getWebsiteEvent optimizations

src/queries/sql/events/getWebsiteEvents.test.ts

@@ -104,7 +104,7 @@ describe('getWebsiteEvents', () => {
     });
   });
 
-  test('clickhouse counts from the lightweight filtered event query and resolves hasData from paged ids', async () => {
+  test('clickhouse counts from the lightweight filtered event query and hydrates wide rows from candidate ids', async () => {
     const { getWebsiteEvents, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });
 
     const result = await getWebsiteEvents('website-1', {
@@ -120,8 +120,12 @@ describe('getWebsiteEvents', () => {
 
     expect(countQuery).toContain('select count(*) as num from (select 1 from (');
     expect(countQuery).not.toContain('event_data');
-    expect(dataQuery).toContain('with paged_events as (');
-    expect(dataQuery).toContain('website_id as websiteId');
+    expect(dataQuery).toContain('with candidate_events as (');
+    expect(dataQuery).toContain('and url_path = {path:String}');
+    expect(dataQuery).toContain('positionCaseInsensitive');
+    expect(dataQuery).toContain(
+      'inner join website_event on website_event.event_id = candidate_events.event_id',
+    );
     expect(dataQuery).toContain('paged_event_data as (');
     expect(dataQuery).toContain('where website_id = {websiteId:UUID}');
     expect(dataQuery).toContain('from umami.event_data_pivot');
@@ -129,6 +133,7 @@ describe('getWebsiteEvents', () => {
     expect(dataQuery).toContain(
       'and event_data_pivot.created_at between {startDate:DateTime64} and {endDate:DateTime64}',
     );
+    expect(dataQuery).toContain('from candidate_events');
     expect(dataQuery).toContain('from paged_events');
     expect(dataQuery).toContain('left join paged_event_data on paged_event_data.event_id = paged_events.id');
     expect(dataQuery).toContain('order by paged_events.createdAt desc');
@@ -191,4 +196,59 @@ describe('getWebsiteEvents', () => {
       'and created_at between {startDate:DateTime64} and {endDate:DateTime64}',
     );
   });
+
+  test('clickhouse narrows row fetches to candidate ids when there are no filters, cohorts, or search', async () => {
+    vi.resetModules();
+
+    const clickhouseRawQuery = vi
+      .fn()
+      .mockResolvedValueOnce([{ num: '10000' }])
+      .mockResolvedValueOnce([{ id: 'event-1' }]);
+
+    vi.doMock('@/lib/db', () => ({
+      CLICKHOUSE: 'clickhouse',
+      PRISMA: 'prisma',
+      runQuery: vi.fn((queries: Record<string, () => unknown>) => queries.clickhouse()),
+    }));
+
+    vi.doMock('@/lib/prisma', () => ({
+      default: {},
+    }));
+
+    vi.doMock('@/lib/clickhouse', () => ({
+      default: {
+        rawQuery: clickhouseRawQuery,
+        parseFilters: vi.fn().mockReturnValue({
+          ...clickhouseParseFiltersResult,
+          filterQuery: '',
+          cohortQuery: '',
+        }),
+      },
+    }));
+
+    const { getWebsiteEvents } = await import('./getWebsiteEvents');
+
+    await getWebsiteEvents('website-1', {
+      page: 1,
+      pageSize: 20,
+      maxResults: 10000,
+    } as any);
+
+    const [countQuery] = clickhouseRawQuery.mock.calls[0];
+    const [dataQuery] = clickhouseRawQuery.mock.calls[1];
+
+    expect(countQuery).toContain('select count(*) as num from (select 1 from (');
+    expect(countQuery).toContain('from website_event');
+    expect(countQuery).not.toContain('from website_event_stats_hourly');
+    expect(dataQuery).toContain('with candidate_events as (');
+    expect(dataQuery).toContain('select');
+    expect(dataQuery).toContain('event_id,');
+    expect(dataQuery).toContain('created_at');
+    expect(dataQuery).toContain('from website_event');
+    expect(dataQuery).toContain('limit 20');
+    expect(dataQuery).toContain(
+      'inner join website_event on website_event.event_id = candidate_events.event_id',
+    );
+    expect(dataQuery).not.toContain('positionCaseInsensitive');
+  });
 });

src/queries/sql/events/getWebsiteEvents.ts

@@ -120,6 +120,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
     ? `and ((positionCaseInsensitive(event_name, {search:String}) > 0 and event_type = ${EVENT_TYPE.customEvent})
            or (positionCaseInsensitive(url_path, {search:String}) > 0 and event_type = ${EVENT_TYPE.pageView}))`
     : '';
+  const candidateLimit = maxResults ? Math.min(+maxResults, offset + size) : offset + size;
 
   const eventQuery = `
     select
@@ -140,36 +141,35 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
 
   const count = await rawQuery(countQuery, queryParams).then(res => res[0].num);
 
-  const data = await rawQuery(
-    `
-    with paged_events as (
-      select
-        event_id as id,
-        website_id as websiteId,
-        session_id as sessionId,
-        created_at as createdAt,
-        hostname,
-        url_path as urlPath,
-        url_query as urlQuery,
-        referrer_path as referrerPath,
-        referrer_query as referrerQuery,
-        referrer_domain as referrerDomain,
-        country,
-        city,
-        device,
-        os,
-        browser,
-        page_title as pageTitle,
-        event_type as eventType,
-        event_name as eventName
-      from website_event
-      ${cohortQuery}
-      where website_id = {websiteId:UUID}
-        and event_type != ${EVENT_TYPE.performance}
-      ${dateQuery}
-      ${filterQuery}
-      ${searchQuery}
+  const dataQuery = `
+    with candidate_events as (
+      ${eventQuery}
       order by created_at desc
+      limit ${candidateLimit}
+    ),
+    paged_events as (
+      select
+        website_event.event_id as id,
+        website_event.website_id as websiteId,
+        website_event.session_id as sessionId,
+        website_event.created_at as createdAt,
+        website_event.hostname,
+        website_event.url_path as urlPath,
+        website_event.url_query as urlQuery,
+        website_event.referrer_path as referrerPath,
+        website_event.referrer_query as referrerQuery,
+        website_event.referrer_domain as referrerDomain,
+        website_event.country,
+        website_event.city,
+        website_event.device,
+        website_event.os,
+        website_event.browser,
+        website_event.page_title as pageTitle,
+        website_event.event_type as eventType,
+        website_event.event_name as eventName
+      from candidate_events
+      inner join website_event on website_event.event_id = candidate_events.event_id
+      order by candidate_events.created_at desc
       limit ${size} offset ${offset}
     ),
     paged_event_data as (
@@ -188,10 +188,9 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
     from paged_events
     left join paged_event_data on paged_event_data.event_id = paged_events.id
     order by paged_events.createdAt desc
-    `,
-    queryParams,
-    FUNCTION_NAME,
-  );
+    `;
+
+  const data = await rawQuery(dataQuery, queryParams, FUNCTION_NAME);
 
   return {
     data,