diff --git a/src/queries/sql/events/getWebsiteEvents.test.ts b/src/queries/sql/events/getWebsiteEvents.test.ts index d48dfb362..3ecf8bea2 100644 --- a/src/queries/sql/events/getWebsiteEvents.test.ts +++ b/src/queries/sql/events/getWebsiteEvents.test.ts @@ -104,7 +104,7 @@ describe('getWebsiteEvents', () => { }); }); - test('clickhouse counts from the lightweight filtered event query and resolves hasData from paged ids', async () => { + test('clickhouse counts from the lightweight filtered event query and hydrates wide rows from candidate ids', async () => { const { getWebsiteEvents, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' }); const result = await getWebsiteEvents('website-1', { @@ -120,8 +120,12 @@ describe('getWebsiteEvents', () => { expect(countQuery).toContain('select count(*) as num from (select 1 from ('); expect(countQuery).not.toContain('event_data'); - expect(dataQuery).toContain('with paged_events as ('); - expect(dataQuery).toContain('website_id as websiteId'); + expect(dataQuery).toContain('with candidate_events as ('); + expect(dataQuery).toContain('and url_path = {path:String}'); + expect(dataQuery).toContain('positionCaseInsensitive'); + expect(dataQuery).toContain( + 'inner join website_event on website_event.event_id = candidate_events.event_id', + ); expect(dataQuery).toContain('paged_event_data as ('); expect(dataQuery).toContain('where website_id = {websiteId:UUID}'); expect(dataQuery).toContain('from umami.event_data_pivot'); @@ -129,6 +133,7 @@ describe('getWebsiteEvents', () => { expect(dataQuery).toContain( 'and event_data_pivot.created_at between {startDate:DateTime64} and {endDate:DateTime64}', ); + expect(dataQuery).toContain('from candidate_events'); expect(dataQuery).toContain('from paged_events'); expect(dataQuery).toContain('left join paged_event_data on paged_event_data.event_id = paged_events.id'); expect(dataQuery).toContain('order by paged_events.createdAt desc'); @@ -191,4 +196,59 @@ describe('getWebsiteEvents', () => { 'and created_at between {startDate:DateTime64} and {endDate:DateTime64}', ); }); + + test('clickhouse narrows row fetches to candidate ids when there are no filters, cohorts, or search', async () => { + vi.resetModules(); + + const clickhouseRawQuery = vi + .fn() + .mockResolvedValueOnce([{ num: '10000' }]) + .mockResolvedValueOnce([{ id: 'event-1' }]); + + vi.doMock('@/lib/db', () => ({ + CLICKHOUSE: 'clickhouse', + PRISMA: 'prisma', + runQuery: vi.fn((queries: Record unknown>) => queries.clickhouse()), + })); + + vi.doMock('@/lib/prisma', () => ({ + default: {}, + })); + + vi.doMock('@/lib/clickhouse', () => ({ + default: { + rawQuery: clickhouseRawQuery, + parseFilters: vi.fn().mockReturnValue({ + ...clickhouseParseFiltersResult, + filterQuery: '', + cohortQuery: '', + }), + }, + })); + + const { getWebsiteEvents } = await import('./getWebsiteEvents'); + + await getWebsiteEvents('website-1', { + page: 1, + pageSize: 20, + maxResults: 10000, + } as any); + + const [countQuery] = clickhouseRawQuery.mock.calls[0]; + const [dataQuery] = clickhouseRawQuery.mock.calls[1]; + + expect(countQuery).toContain('select count(*) as num from (select 1 from ('); + expect(countQuery).toContain('from website_event'); + expect(countQuery).not.toContain('from website_event_stats_hourly'); + expect(dataQuery).toContain('with candidate_events as ('); + expect(dataQuery).toContain('select'); + expect(dataQuery).toContain('event_id,'); + expect(dataQuery).toContain('created_at'); + expect(dataQuery).toContain('from website_event'); + expect(dataQuery).toContain('limit 20'); + expect(dataQuery).toContain( + 'inner join website_event on website_event.event_id = candidate_events.event_id', + ); + expect(dataQuery).not.toContain('positionCaseInsensitive'); + }); }); diff --git a/src/queries/sql/events/getWebsiteEvents.ts b/src/queries/sql/events/getWebsiteEvents.ts index bf1450e74..05435ff8a 100644 --- a/src/queries/sql/events/getWebsiteEvents.ts +++ b/src/queries/sql/events/getWebsiteEvents.ts @@ -120,6 +120,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { ? `and ((positionCaseInsensitive(event_name, {search:String}) > 0 and event_type = ${EVENT_TYPE.customEvent}) or (positionCaseInsensitive(url_path, {search:String}) > 0 and event_type = ${EVENT_TYPE.pageView}))` : ''; + const candidateLimit = maxResults ? Math.min(+maxResults, offset + size) : offset + size; const eventQuery = ` select @@ -140,36 +141,35 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { const count = await rawQuery(countQuery, queryParams).then(res => res[0].num); - const data = await rawQuery( - ` - with paged_events as ( - select - event_id as id, - website_id as websiteId, - session_id as sessionId, - created_at as createdAt, - hostname, - url_path as urlPath, - url_query as urlQuery, - referrer_path as referrerPath, - referrer_query as referrerQuery, - referrer_domain as referrerDomain, - country, - city, - device, - os, - browser, - page_title as pageTitle, - event_type as eventType, - event_name as eventName - from website_event - ${cohortQuery} - where website_id = {websiteId:UUID} - and event_type != ${EVENT_TYPE.performance} - ${dateQuery} - ${filterQuery} - ${searchQuery} + const dataQuery = ` + with candidate_events as ( + ${eventQuery} order by created_at desc + limit ${candidateLimit} + ), + paged_events as ( + select + website_event.event_id as id, + website_event.website_id as websiteId, + website_event.session_id as sessionId, + website_event.created_at as createdAt, + website_event.hostname, + website_event.url_path as urlPath, + website_event.url_query as urlQuery, + website_event.referrer_path as referrerPath, + website_event.referrer_query as referrerQuery, + website_event.referrer_domain as referrerDomain, + website_event.country, + website_event.city, + website_event.device, + website_event.os, + website_event.browser, + website_event.page_title as pageTitle, + website_event.event_type as eventType, + website_event.event_name as eventName + from candidate_events + inner join website_event on website_event.event_id = candidate_events.event_id + order by candidate_events.created_at desc limit ${size} offset ${offset} ), paged_event_data as ( @@ -188,10 +188,9 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { from paged_events left join paged_event_data on paged_event_data.event_id = paged_events.id order by paged_events.createdAt desc - `, - queryParams, - FUNCTION_NAME, - ); + `; + + const data = await rawQuery(dataQuery, queryParams, FUNCTION_NAME); return { data,