diff --git a/src/queries/sql/events/getWebsiteEvents.test.ts b/src/queries/sql/events/getWebsiteEvents.test.ts index 940805545..76eeee558 100644 --- a/src/queries/sql/events/getWebsiteEvents.test.ts +++ b/src/queries/sql/events/getWebsiteEvents.test.ts @@ -5,7 +5,8 @@ const prismaParseFiltersResult = { filterQuery: 'and website_event.url_path = {{path}}', dateQuery: 'and website_event.created_at between {{startDate}} and {{endDate}}', cohortQuery: 'join cohort on cohort.session_id = website_event.session_id', - joinSessionQuery: 'inner join session on website_event.session_id = session.session_id and website_event.website_id = session.website_id', + joinSessionQuery: + 'inner join session on website_event.session_id = session.session_id and website_event.website_id = session.website_id', }; const clickhouseParseFiltersResult = { @@ -16,11 +17,7 @@ const clickhouseParseFiltersResult = { joinSessionQuery: '', }; -async function loadModule({ - mode, -}: { - mode: 'prisma' | 'clickhouse'; -}) { +async function loadModule({ mode }: { mode: 'prisma' | 'clickhouse' }) { vi.resetModules(); const state = { mode }; @@ -32,7 +29,6 @@ async function loadModule({ const clickhouseRawQuery = vi .fn() .mockResolvedValueOnce([{ num: '25' }]) - .mockResolvedValueOnce([{ event_id: 'event-1', created_at: '2026-08-05 10:00:00' }]) .mockResolvedValueOnce([{ id: 'event-1' }]); const clickhouseParseFilters = vi.fn().mockReturnValue(clickhouseParseFiltersResult); @@ -89,10 +85,14 @@ describe('getWebsiteEvents', () => { expect(countQuery).not.toContain('exists('); expect(dataQuery).toContain('with paged_events as ('); expect(dataQuery).toContain('paged_event_data as ('); - expect(dataQuery).toContain('join paged_events on paged_events.event_id = event_data.website_event_id'); + expect(dataQuery).toContain( + 'join paged_events on paged_events.event_id = event_data.website_event_id', + ); expect(dataQuery).toContain('where event_data.website_id = {{websiteId::uuid}}'); expect(dataQuery).toContain('and event_data.created_at between {{startDate}} and {{endDate}}'); - expect(dataQuery).toContain('left join paged_event_data on paged_event_data.event_id = website_event.event_id'); + expect(dataQuery).toContain( + 'left join paged_event_data on paged_event_data.event_id = website_event.event_id', + ); expect(dataQuery).toContain('order by paged_events.created_at desc'); expect(dataQuery).toContain('(paged_event_data.event_id is not null) as "hasData"'); expect(result).toEqual({ @@ -105,7 +105,7 @@ describe('getWebsiteEvents', () => { }); }); - test('clickhouse counts from the lightweight filtered event query and hydrates wide rows from candidate ids', async () => { + test('clickhouse counts from the lightweight filtered event query and resolves hasData from paged ids', async () => { const { getWebsiteEvents, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' }); const result = await getWebsiteEvents('website-1', { @@ -117,34 +117,28 @@ describe('getWebsiteEvents', () => { } as any); const [countQuery] = clickhouseRawQuery.mock.calls[0]; - const [pagedEventsQuery] = clickhouseRawQuery.mock.calls[1]; - const [dataQuery, dataParams] = clickhouseRawQuery.mock.calls[2]; + const [dataQuery] = clickhouseRawQuery.mock.calls[1]; expect(countQuery).toContain('select count(*) as num from (select 1 from ('); expect(countQuery).not.toContain('event_data'); - expect(pagedEventsQuery).toContain('select'); - expect(pagedEventsQuery).toContain('event_id,'); - expect(pagedEventsQuery).toContain('created_at'); - expect(pagedEventsQuery).toContain('from website_event'); - expect(pagedEventsQuery).toContain('and url_path = {path:String}'); - expect(pagedEventsQuery).toContain('positionCaseInsensitive'); - expect(pagedEventsQuery).toContain('order by created_at desc'); - expect(pagedEventsQuery).toContain('limit 20 offset 0'); + expect(dataQuery).toContain('with paged_events as ('); + expect(dataQuery).toContain('website_id as websiteId'); expect(dataQuery).toContain('paged_event_data as ('); expect(dataQuery).toContain('where website_id = {websiteId:UUID}'); expect(dataQuery).toContain('from umami.event_data_pivot'); - expect(dataQuery).toContain('and event_id in {eventIds:Array(UUID)}'); + expect(dataQuery).toContain( + 'inner join paged_events on paged_events.id = event_data_pivot.event_id', + ); expect(dataQuery).toContain( 'and event_data_pivot.created_at between {startDate:DateTime64} and {endDate:DateTime64}', ); - expect(dataQuery).toContain('from website_event'); + expect(dataQuery).toContain('from paged_events'); expect(dataQuery).toContain( - 'left join paged_event_data on paged_event_data.event_id = website_event.event_id', + 'left join paged_event_data on paged_event_data.event_id = paged_events.id', ); - expect(dataQuery).toContain('order by indexOf({eventIds:Array(UUID)}, website_event.event_id)'); + expect(dataQuery).toContain('order by paged_events.createdAt desc'); expect(dataQuery).toContain('toUInt8(1) as has_data'); expect(dataQuery).toContain('ifNull(paged_event_data.has_data, 0) as hasData'); - expect(dataParams).toMatchObject({ eventIds: ['event-1'] }); expect(result).toEqual({ data: [{ id: 'event-1' }], count: '25', @@ -162,7 +156,6 @@ describe('getWebsiteEvents', () => { const clickhouseRawQuery = vi .fn() .mockResolvedValueOnce([{ num: '25' }]) - .mockResolvedValueOnce([{ event_id: 'event-1', created_at: '2026-08-05 10:00:00' }]) .mockResolvedValueOnce([{ id: 'event-1' }]); vi.doMock('@/lib/db', () => ({ @@ -194,7 +187,7 @@ describe('getWebsiteEvents', () => { timezone: 'America/Los_Angeles', } as any); - const [dataQuery] = clickhouseRawQuery.mock.calls[2]; + const [dataQuery] = clickhouseRawQuery.mock.calls[1]; expect(dataQuery).toContain( 'and event_data_pivot.created_at between toTimezone({startDate:DateTime64},{timezone:String}) and toTimezone({endDate:DateTime64},{timezone:String})', @@ -203,105 +196,4 @@ describe('getWebsiteEvents', () => { 'and created_at between {startDate:DateTime64} and {endDate:DateTime64}', ); }); - - test('clickhouse narrows row fetches to candidate ids when there are no filters, cohorts, or search', async () => { - vi.resetModules(); - - const clickhouseRawQuery = vi - .fn() - .mockResolvedValueOnce([{ num: '10000' }]) - .mockResolvedValueOnce([{ event_id: 'event-1', created_at: '2026-08-05 10:00:00' }]) - .mockResolvedValueOnce([{ id: 'event-1' }]); - - vi.doMock('@/lib/db', () => ({ - CLICKHOUSE: 'clickhouse', - PRISMA: 'prisma', - runQuery: vi.fn((queries: Record unknown>) => queries.clickhouse()), - })); - - vi.doMock('@/lib/prisma', () => ({ - default: {}, - })); - - vi.doMock('@/lib/clickhouse', () => ({ - default: { - rawQuery: clickhouseRawQuery, - parseFilters: vi.fn().mockReturnValue({ - ...clickhouseParseFiltersResult, - filterQuery: '', - cohortQuery: '', - }), - }, - })); - - const { getWebsiteEvents } = await import('./getWebsiteEvents'); - - await getWebsiteEvents('website-1', { - page: 1, - pageSize: 20, - maxResults: 10000, - } as any); - - const [countQuery] = clickhouseRawQuery.mock.calls[0]; - const [pagedEventsQuery] = clickhouseRawQuery.mock.calls[1]; - const [dataQuery, dataParams] = clickhouseRawQuery.mock.calls[2]; - - expect(countQuery).toContain('select count(*) as num from (select 1 from ('); - expect(countQuery).toContain('from website_event'); - expect(countQuery).not.toContain('from website_event_stats_hourly'); - expect(pagedEventsQuery).toContain('select'); - expect(pagedEventsQuery).toContain('event_id,'); - expect(pagedEventsQuery).toContain('created_at'); - expect(pagedEventsQuery).toContain('from website_event'); - expect(pagedEventsQuery).toContain('limit 20 offset 0'); - expect(pagedEventsQuery).not.toContain('positionCaseInsensitive'); - expect(dataQuery).toContain('and event_id in {eventIds:Array(UUID)}'); - expect(dataQuery).toContain('from website_event'); - expect(dataQuery).toContain('order by indexOf({eventIds:Array(UUID)}, website_event.event_id)'); - expect(dataParams).toMatchObject({ eventIds: ['event-1'] }); - }); - - test('clickhouse returns an empty page without running the wide hydration query when no ids match', async () => { - vi.resetModules(); - - const clickhouseRawQuery = vi - .fn() - .mockResolvedValueOnce([{ num: '0' }]) - .mockResolvedValueOnce([]); - - vi.doMock('@/lib/db', () => ({ - CLICKHOUSE: 'clickhouse', - PRISMA: 'prisma', - runQuery: vi.fn((queries: Record unknown>) => queries.clickhouse()), - })); - - vi.doMock('@/lib/prisma', () => ({ - default: {}, - })); - - vi.doMock('@/lib/clickhouse', () => ({ - default: { - rawQuery: clickhouseRawQuery, - parseFilters: vi.fn().mockReturnValue(clickhouseParseFiltersResult), - }, - })); - - const { getWebsiteEvents } = await import('./getWebsiteEvents'); - - const result = await getWebsiteEvents('website-1', { - page: 1, - pageSize: 20, - } as any); - - expect(clickhouseRawQuery).toHaveBeenCalledTimes(2); - expect(result).toEqual({ - data: [], - count: '0', - page: 1, - pageSize: 20, - orderBy: undefined, - search: undefined, - isCapped: false, - }); - }); }); diff --git a/src/queries/sql/events/getWebsiteEvents.ts b/src/queries/sql/events/getWebsiteEvents.ts index 24d6de7ba..bf1450e74 100644 --- a/src/queries/sql/events/getWebsiteEvents.ts +++ b/src/queries/sql/events/getWebsiteEvents.ts @@ -139,69 +139,59 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { : `select count(*) as num from (${eventQuery}) t`; const count = await rawQuery(countQuery, queryParams).then(res => res[0].num); - const pagedEventRows = await rawQuery<{ event_id: string; created_at: string }[]>( + + const data = await rawQuery( ` - ${eventQuery} - order by created_at desc - limit ${size} offset ${offset} - `, - queryParams, - FUNCTION_NAME, - ); - - const eventIds = pagedEventRows.map(({ event_id }) => event_id); - - if (!eventIds.length) { - return { - data: [], - count, - page: +page, - pageSize: size, - orderBy, - search, - isCapped: !!maxResults && +count >= +maxResults, - }; - } - - const dataQuery = ` + with paged_events as ( + select + event_id as id, + website_id as websiteId, + session_id as sessionId, + created_at as createdAt, + hostname, + url_path as urlPath, + url_query as urlQuery, + referrer_path as referrerPath, + referrer_query as referrerQuery, + referrer_domain as referrerDomain, + country, + city, + device, + os, + browser, + page_title as pageTitle, + event_type as eventType, + event_name as eventName + from website_event + ${cohortQuery} + where website_id = {websiteId:UUID} + and event_type != ${EVENT_TYPE.performance} + ${dateQuery} + ${filterQuery} + ${searchQuery} + order by created_at desc + limit ${size} offset ${offset} + ), paged_event_data as ( select event_id, toUInt8(1) as has_data from umami.event_data_pivot + inner join paged_events on paged_events.id = event_data_pivot.event_id where website_id = {websiteId:UUID} - and event_id in {eventIds:Array(UUID)} ${hasDataDateQuery} group by event_id ) select - website_event.event_id as id, - website_event.website_id as websiteId, - website_event.session_id as sessionId, - website_event.created_at as createdAt, - website_event.hostname, - website_event.url_path as urlPath, - website_event.url_query as urlQuery, - website_event.referrer_path as referrerPath, - website_event.referrer_query as referrerQuery, - website_event.referrer_domain as referrerDomain, - website_event.country, - website_event.city, - website_event.device, - website_event.os, - website_event.browser, - website_event.page_title as pageTitle, - website_event.event_type as eventType, - website_event.event_name as eventName, + paged_events.*, ifNull(paged_event_data.has_data, 0) as hasData - from website_event - left join paged_event_data on paged_event_data.event_id = website_event.event_id - where website_event.website_id = {websiteId:UUID} - and website_event.event_id in {eventIds:Array(UUID)} - order by indexOf({eventIds:Array(UUID)}, website_event.event_id) - `; - - const data = await rawQuery(dataQuery, { ...queryParams, eventIds }, FUNCTION_NAME); + from paged_events + left join paged_event_data on paged_event_data.event_id = paged_events.id + order by paged_events.createdAt desc + `, + queryParams, + FUNCTION_NAME, + ); return { data,