From b3aec54bb3d5932bd6c5ce7c0d6ea5226e96860b Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 30 Jul 2026 09:34:35 -0700 Subject: [PATCH] Fix excludeBounce query, simplify bounce definition for SPA websites, query routing for performance, add tests --- docker-compose.yml | 1 - src/lib/clickhouse.ts | 8 +- src/lib/constants.ts | 1 - src/lib/prisma.ts | 18 ++- src/permissions/website.test.ts | 22 +-- .../sql/getChannelExpandedMetrics.test.ts | 7 +- src/queries/sql/getChannelExpandedMetrics.ts | 67 +++++---- src/queries/sql/getWebsiteStats.test.ts | 135 ++++++++++++++++++ src/queries/sql/getWebsiteStats.ts | 111 +++++++++----- .../pageviews/getPageviewExpandedMetrics.ts | 62 +++++--- src/queries/sql/reports/getBreakdown.ts | 67 ++++++--- .../sql/sessions/getSessionExpandedMetrics.ts | 61 +++++--- 12 files changed, 410 insertions(+), 150 deletions(-) create mode 100644 src/queries/sql/getWebsiteStats.test.ts diff --git a/docker-compose.yml b/docker-compose.yml index bc8d8da08..348c294ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,6 @@ services: environment: DATABASE_URL: postgresql://umami:umami@db:5432/umami APP_SECRET: replace-me-with-a-random-string - # BOUNCE_THRESHOLD: 2 # OPTIONAL: set the minimum amount custom events to trigger a bounce depends_on: db: condition: service_healthy diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index bfce9ba62..7d3331dd7 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -177,13 +177,17 @@ function getExcludeBounceQuery(filters: Record) { } return `join - (select distinct session_id as exclude_session_id, visit_id as exclude_visit_id + (select session_id as exclude_session_id, visit_id as exclude_visit_id from website_event_stats_hourly where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = 1 + and event_type != 5 group by session_id, visit_id having sum(views) > 1 + or ( + sum(views) = 1 + and sumIf(length(event_name), event_type = 2) > 0 + ) ) excludeBounce on excludeBounce.exclude_session_id = website_event.session_id and excludeBounce.exclude_visit_id = website_event.visit_id diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 5420bc6cc..89e250e73 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -1,5 +1,4 @@ export const CURRENT_VERSION = process.env.currentVersion; -export const BOUNCE_THRESHOLD = Math.max(1, Number.parseInt(process.env.BOUNCE_THRESHOLD, 10) || 1); export const AUTH_TOKEN = 'umami.auth'; export const LOCALE_CONFIG = 'umami.locale'; export const TIMEZONE_CONFIG = 'umami.timezone'; diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts index 18b53a1cb..5c76aef4a 100644 --- a/src/lib/prisma.ts +++ b/src/lib/prisma.ts @@ -2,13 +2,7 @@ import { PrismaPg } from '@prisma/adapter-pg'; import { readReplicas } from '@prisma/extension-read-replicas'; import debug from 'debug'; import { PrismaClient } from '@/generated/prisma/client'; -import { - DATA_TYPE, - DEFAULT_PAGE_SIZE, - FILTER_COLUMNS, - OPERATORS, - SESSION_COLUMNS, -} from './constants'; +import { DATA_TYPE, DEFAULT_PAGE_SIZE, FILTER_COLUMNS, OPERATORS, SESSION_COLUMNS } from './constants'; import { filtersObjectToArray } from './params'; import type { Operator, PropertyFilter, QueryFilters, QueryOptions } from './types'; @@ -249,13 +243,17 @@ function getExcludeBounceQuery(filters: Record) { } return `join - (select distinct session_id, visit_id + (select session_id, visit_id from website_event where website_id = {{websiteId}} and created_at between {{startDate}} and {{endDate}} - and event_type = 1 + and event_type != 5 group by session_id, visit_id - having count(*) > 1 + having sum(case when event_type NOT IN (2, 5) then 1 else 0 end) > 1 + or ( + sum(case when event_type NOT IN (2, 5) then 1 else 0 end) = 1 + and sum(case when event_type = 2 then 1 else 0 end) > 0 + ) ) excludeBounce on excludeBounce.session_id = website_event.session_id and excludeBounce.visit_id = website_event.visit_id diff --git a/src/permissions/website.test.ts b/src/permissions/website.test.ts index b9c00a103..13679f3b6 100644 --- a/src/permissions/website.test.ts +++ b/src/permissions/website.test.ts @@ -1,6 +1,5 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; import { getEntity } from '@/lib/entity'; -import prisma from '@/lib/prisma'; import { getTeamUser, getWebsite } from '@/queries/prisma'; import { canCreateWebsite, @@ -13,6 +12,11 @@ import { canViewWebsite, } from './website'; +const { websiteFindManyMock, teamUserFindManyMock } = vi.hoisted(() => ({ + websiteFindManyMock: vi.fn(), + teamUserFindManyMock: vi.fn(), +})); + vi.mock('@/lib/entity', () => ({ getEntity: vi.fn(), })); @@ -26,10 +30,10 @@ vi.mock('@/lib/prisma', () => ({ default: { client: { website: { - findMany: vi.fn(), + findMany: websiteFindManyMock, }, teamUser: { - findMany: vi.fn(), + findMany: teamUserFindManyMock, }, }, }, @@ -54,8 +58,8 @@ beforeEach(() => { vi.mocked(getEntity).mockReset(); vi.mocked(getWebsite).mockReset(); vi.mocked(getTeamUser).mockReset(); - vi.mocked(prisma.client.website.findMany).mockReset(); - vi.mocked(prisma.client.teamUser.findMany).mockReset(); + websiteFindManyMock.mockReset(); + teamUserFindManyMock.mockReset(); }); describe('canViewWebsite', () => { @@ -139,12 +143,12 @@ describe('canViewBatchWebsites', () => { }); test('returns owned, team, and share allowed ids for a user', async () => { - vi.mocked(prisma.client.website.findMany).mockResolvedValue([ + websiteFindManyMock.mockResolvedValue([ { id: 'owned', userId: 'user-1', teamId: null }, { id: 'team', userId: null, teamId: 'team-1' }, { id: 'foreign', userId: 'other', teamId: null }, ] as any); - vi.mocked(prisma.client.teamUser.findMany).mockResolvedValue([{ teamId: 'team-1' }] as any); + teamUserFindManyMock.mockResolvedValue([{ teamId: 'team-1' }] as any); await expect( canViewBatchWebsites({ user: normalUser, shareToken: { websiteId: 'shared' } as any }, [ @@ -157,10 +161,10 @@ describe('canViewBatchWebsites', () => { }); test('excludes team websites when the user is not a team member', async () => { - vi.mocked(prisma.client.website.findMany).mockResolvedValue([ + websiteFindManyMock.mockResolvedValue([ { id: 'team', userId: null, teamId: 'team-1' }, ] as any); - vi.mocked(prisma.client.teamUser.findMany).mockResolvedValue([] as any); + teamUserFindManyMock.mockResolvedValue([] as any); await expect(canViewBatchWebsites({ user: normalUser }, ['team'])).resolves.toEqual([]); }); diff --git a/src/queries/sql/getChannelExpandedMetrics.test.ts b/src/queries/sql/getChannelExpandedMetrics.test.ts index 452071e86..8edf8cc17 100644 --- a/src/queries/sql/getChannelExpandedMetrics.test.ts +++ b/src/queries/sql/getChannelExpandedMetrics.test.ts @@ -83,7 +83,9 @@ describe('getChannelExpandedMetrics postgres branch', () => { expect(query).toContain('sum(visit_stats.c) as "pageviews"'); expect(query).toContain('count(distinct visit_stats.session_id) as "visitors"'); expect(query).toContain('count(distinct visit_stats.visit_id) as "visits"'); - expect(query).toContain('sum(case when visit_stats.c = 1 then 1 else 0 end) as "bounces"'); + expect(query).toContain('sum(case when visit_stats.c = 1 and coalesce(visit_events.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces"'); + expect(query).toContain('visit_events as'); + expect(query).toContain('left join visit_events'); // getTimestampDiffSQL output is injected into the totaltime column expect(query).toContain('sum(ts_diff(min_time, max_time)) as "totaltime"'); expect(getTimestampDiffSQL).toHaveBeenCalledWith( @@ -151,7 +153,8 @@ describe('getChannelExpandedMetrics clickhouse branch', () => { expect(query).toContain('uniq(t.session_id) as "visitors"'); expect(query).toContain('uniq(t.visit_id) as "visits"'); - expect(query).toContain('sum(if(t.c = 1, 1, 0)) as "bounces"'); + expect(query).toContain('sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces"'); + expect(query).toContain('left join ('); expect(query).toContain('sum(max_time-min_time) as "totaltime"'); expect(query).toContain('where website_id = {websiteId:UUID}'); expect(query).toContain( diff --git a/src/queries/sql/getChannelExpandedMetrics.ts b/src/queries/sql/getChannelExpandedMetrics.ts index 6d9eee301..fa3c1dc23 100644 --- a/src/queries/sql/getChannelExpandedMetrics.ts +++ b/src/queries/sql/getChannelExpandedMetrics.ts @@ -1,6 +1,5 @@ import clickhouse from '@/lib/clickhouse'; import { - BOUNCE_THRESHOLD, EMAIL_DOMAINS, EVENT_TYPE, LLM_DOMAINS, @@ -49,6 +48,29 @@ async function relationalQuery( ...filters, websiteId, }); + const needsBounceEvents = filters.excludeBounce !== true; + const bounceQuery = needsBounceEvents + ? `sum(case when visit_stats.c = 1 and coalesce(visit_events.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",` + : '0 as "bounces",'; + const visitEventsQuery = needsBounceEvents + ? `, + + visit_events as ( + select + session_id, + visit_id, + 1 as has_custom_event + from website_event + where website_id = {{websiteId::uuid}} + and created_at between {{startDate}} and {{endDate}} + and event_type = ${EVENT_TYPE.customEvent} + group by session_id, visit_id)` + : ''; + const visitEventsJoin = needsBounceEvents + ? `left join visit_events + on visit_events.session_id = visit_stats.session_id + and visit_events.visit_id = visit_stats.visit_id` + : ''; return rawQuery( ` @@ -122,33 +144,21 @@ async function relationalQuery( min(created_at) as min_time, max(created_at) as max_time from prefix - group by session_id, visit_id), - - visit_events as ( - select - session_id, - visit_id, - count(*) as events_count - from website_event - where website_id = {{websiteId::uuid}} - and created_at between {{startDate}} and {{endDate}} - and event_type = ${EVENT_TYPE.customEvent} group by session_id, visit_id) + ${visitEventsQuery} select visit_channels.name, sum(visit_stats.c) as "pageviews", count(distinct visit_stats.session_id) as "visitors", count(distinct visit_stats.visit_id) as "visits", - sum(case when visit_stats.c = 1 and coalesce(visit_events.events_count, 0) < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces", + ${bounceQuery} sum(${getTimestampDiffSQL('visit_stats.min_time', 'visit_stats.max_time')}) as "totaltime" from visit_stats join visit_channels on visit_channels.session_id = visit_stats.session_id and visit_channels.visit_id = visit_stats.visit_id - left join visit_events - on visit_events.session_id = visit_stats.session_id - and visit_events.visit_id = visit_stats.visit_id + ${visitEventsJoin} group by visit_channels.name order by visitors desc, visits desc `, @@ -166,6 +176,20 @@ async function clickhouseQuery( ...filters, websiteId, }); + const needsBounceEvents = filters.excludeBounce !== true; + const bounceQuery = needsBounceEvents + ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",` + : '0 as "bounces",'; + const visitEventsJoin = needsBounceEvents + ? `left join ( + select session_id, visit_id, toUInt8(1) as has_custom_event + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = ${EVENT_TYPE.customEvent} + group by session_id, visit_id + ) as e using (session_id, visit_id)` + : ''; return rawQuery( ` @@ -174,7 +198,7 @@ async function clickhouseQuery( sum(t.c) as "pageviews", uniq(t.session_id) as "visitors", uniq(t.visit_id) as "visits", - sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces", + ${bounceQuery} sum(max_time-min_time) as "totaltime" from ( select @@ -229,14 +253,7 @@ async function clickhouseQuery( ) group by session_id, visit_id ) as t - left join ( - select session_id, visit_id, toUInt32(count()) as events_count - from website_event - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = ${EVENT_TYPE.customEvent} - group by session_id, visit_id - ) as e using (session_id, visit_id) + ${visitEventsJoin} group by name order by visitors desc, visits desc; `, diff --git a/src/queries/sql/getWebsiteStats.test.ts b/src/queries/sql/getWebsiteStats.test.ts new file mode 100644 index 000000000..c604683db --- /dev/null +++ b/src/queries/sql/getWebsiteStats.test.ts @@ -0,0 +1,135 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; + +const parseFiltersResult = { + queryParams: { websiteId: 'website-1' }, + filterQuery: 'and website_event.url_path = {{path}}', + joinSessionQuery: 'join session on session.session_id = website_event.session_id', + cohortQuery: 'join cohort on cohort.session_id = website_event.session_id', + excludeBounceQuery: 'join excludeBounce on excludeBounce.visit_id = website_event.visit_id', +}; + +async function loadModule({ + mode, +}: { + mode: 'prisma' | 'clickhouse'; +}) { + vi.resetModules(); + + const state = { mode }; + const prismaRawQuery = vi.fn().mockResolvedValue([{}]); + const prismaParseFilters = vi.fn().mockReturnValue(parseFiltersResult); + const getTimestampDiffSQL = vi.fn().mockReturnValue('ts_diff(t.min_time, t.max_time)'); + const clickhouseRawQuery = vi.fn().mockResolvedValue([{}]); + const clickhouseParseFilters = vi.fn().mockReturnValue(parseFiltersResult); + + vi.doMock('@/lib/db', () => ({ + CLICKHOUSE: 'clickhouse', + PRISMA: 'prisma', + runQuery: vi.fn((queries: Record unknown>) => queries[state.mode]()), + })); + + vi.doMock('@/lib/prisma', () => ({ + default: { + rawQuery: prismaRawQuery, + parseFilters: prismaParseFilters, + getTimestampDiffSQL, + }, + })); + + vi.doMock('@/lib/clickhouse', () => ({ + default: { + rawQuery: clickhouseRawQuery, + parseFilters: clickhouseParseFilters, + }, + })); + + const mod = await import('./getWebsiteStats'); + + return { + getWebsiteStats: mod.getWebsiteStats, + prismaRawQuery, + clickhouseRawQuery, + }; +} + +afterEach(() => { + vi.resetModules(); + vi.clearAllMocks(); +}); + +describe('getWebsiteStats default bounce path', () => { + test('postgres uses a single grouped pass for the default path', async () => { + const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' }); + + await getWebsiteStats('website-1', {}); + + const [query] = prismaRawQuery.mock.calls[0]; + + expect(query).toContain('t.c = 1 and t.has_custom_event = 0'); + expect(query).toContain('max(case when website_event.event_type = 2 then 1 else 0 end) as "has_custom_event"'); + expect(query).not.toContain('left join ('); + }); + + test('clickhouse uses the hourly aggregate view without the extra join', async () => { + const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' }); + + await getWebsiteStats('website-1', {}); + + const [query] = clickhouseRawQuery.mock.calls[0]; + + expect(query).toContain('from website_event_stats_hourly "website_event"'); + expect(query).toContain('sumIf(1, t.c = 1 and t.has_custom_event = 0) as "bounces"'); + expect(query).toContain('has_custom_event'); + expect(query).not.toContain('left join ('); + }); +}); + +describe('getWebsiteStats filtered bounce path', () => { + test('postgres switches to the visit-events join when event filters are present', async () => { + const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' }); + + await getWebsiteStats('website-1', { path: '/pricing' } as any); + + const [query] = prismaRawQuery.mock.calls[0]; + + expect(query).toContain('left join ('); + expect(query).toContain('coalesce(e.has_custom_event, 0) = 0'); + expect(query).toContain('event_type = 2'); + }); + + test('clickhouse switches to the raw-event join when event filters are present', async () => { + const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' }); + + await getWebsiteStats('website-1', { path: '/pricing' } as any); + + const [query] = clickhouseRawQuery.mock.calls[0]; + + expect(query).toContain('from website_event'); + expect(query).toContain('left join ('); + expect(query).toContain('ifNull(e.has_custom_event, 0) = 0'); + }); +}); + +describe('getWebsiteStats excludeBounce path', () => { + test('clickhouse skips the extra join when excludeBounce is enabled', async () => { + const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' }); + + await getWebsiteStats('website-1', { excludeBounce: true }); + + const [query] = clickhouseRawQuery.mock.calls[0]; + + expect(query).toContain('0 as "bounces"'); + expect(query).not.toContain('left join ('); + }); + + test('postgres skips the extra join when excludeBounce is enabled on the filtered path', async () => { + const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' }); + + await getWebsiteStats('website-1', { excludeBounce: true, path: '/pricing' } as any); + + const [query] = prismaRawQuery.mock.calls[0]; + + expect(query).toContain('0 as "bounces"'); + expect(query).not.toContain('left join ('); + }); +}); diff --git a/src/queries/sql/getWebsiteStats.ts b/src/queries/sql/getWebsiteStats.ts index 6ca085db4..681c41abe 100644 --- a/src/queries/sql/getWebsiteStats.ts +++ b/src/queries/sql/getWebsiteStats.ts @@ -1,5 +1,5 @@ import clickhouse from '@/lib/clickhouse'; -import { BOUNCE_THRESHOLD, EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants'; +import { EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import type { QueryFilters } from '@/lib/types'; @@ -35,9 +35,60 @@ async function relationalQuery( }); const { excludeBounce } = filters; + const hasEventFilters = + EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || + !!filters.eventPropertyFilters?.length; + + if (!hasEventFilters) { + return rawQuery( + ` + select + cast(coalesce(sum(t.c), 0) as bigint) as "pageviews", + count(distinct t.session_id) as "visitors", + count(distinct t.visit_id) as "visits", + ${excludeBounce ? '0' : 'coalesce(sum(case when t.c = 1 and t.has_custom_event = 0 then 1 else 0 end), 0)'} as "bounces", + cast(coalesce(sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}), 0) as bigint) as "totaltime" + from ( + select + website_event.session_id, + website_event.visit_id, + sum(case when website_event.event_type NOT IN (2, 5) then 1 else 0 end) as "c", + min(case when website_event.event_type NOT IN (2, 5) then website_event.created_at end) as "min_time", + max(case when website_event.event_type NOT IN (2, 5) then website_event.created_at end) as "max_time", + max(case when website_event.event_type = ${EVENT_TYPE.customEvent} then 1 else 0 end) as "has_custom_event" + from website_event + ${cohortQuery} + ${excludeBounceQuery} + ${joinSessionQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_type != ${EVENT_TYPE.performance} + ${filterQuery} + group by 1, 2 + having sum(case when website_event.event_type NOT IN (2, 5) then 1 else 0 end) > 0 + ) as t + `, + queryParams, + FUNCTION_NAME, + ).then(result => result?.[0]); + } + const bounceQuery = excludeBounce ? '0' - : `coalesce(sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end), 0)`; + : 'coalesce(sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end), 0)'; + const visitEventsJoin = excludeBounce + ? '' + : ` + left join ( + select session_id, visit_id, 1 as "has_custom_event" + from website_event + where website_id = {{websiteId::uuid}} + and created_at between {{startDate}} and {{endDate}} + and event_type = ${EVENT_TYPE.customEvent} + group by 1, 2 + ) as e + on e.session_id = t.session_id + and e.visit_id = t.visit_id`; return rawQuery( ` @@ -53,16 +104,7 @@ async function relationalQuery( website_event.visit_id, count(*) as "c", min(website_event.created_at) as "min_time", - max(website_event.created_at) as "max_time", - max(( - select count(*) - from website_event we2 - where we2.website_id = website_event.website_id - and we2.session_id = website_event.session_id - and we2.visit_id = website_event.visit_id - and we2.created_at between {{startDate}} and {{endDate}} - and we2.event_type = ${EVENT_TYPE.customEvent} - )) as "events_count" + max(website_event.created_at) as "max_time" from website_event ${cohortQuery} ${excludeBounceQuery} @@ -72,8 +114,8 @@ async function relationalQuery( and website_event.event_type NOT IN (2, 5) ${filterQuery} group by 1, 2 - ) as t + ${visitEventsJoin} `, queryParams, FUNCTION_NAME, @@ -92,17 +134,20 @@ async function clickhouseQuery( let sql = ''; const { excludeBounce } = filters; + const hasEventFilters = + EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || + !!filters.eventPropertyFilters?.length; const bounceQuery = excludeBounce ? '0' - : `sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD})`; + : 'sumIf(1, t.c = 1 and t.has_custom_event = 0)'; - if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) { + if (hasEventFilters) { sql = ` select sum(t.c) as "pageviews", uniq(t.session_id) as "visitors", uniq(t.visit_id) as "visits", - ${bounceQuery} as "bounces", + ${excludeBounce ? '0' : 'sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0)'} as "bounces", sum(max_time-min_time) as "totaltime" from ( select @@ -120,14 +165,14 @@ async function clickhouseQuery( ${filterQuery} group by session_id, visit_id ) as t - left join ( - select session_id, visit_id, toUInt32(count()) as events_count + ${excludeBounce ? '' : `left join ( + select session_id, visit_id, toUInt8(1) as has_custom_event from website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = ${EVENT_TYPE.customEvent} group by session_id, visit_id - ) as e using (session_id, visit_id); + ) as e using (session_id, visit_id)`}; `; } else { sql = ` @@ -137,28 +182,24 @@ async function clickhouseQuery( uniq(visit_id) as "visits", ${bounceQuery} as "bounces", sum(max_time-min_time) as "totaltime" - from (select - session_id, - visit_id, - sum(views) c, - min(min_time) min_time, - max(max_time) max_time + from ( + select + session_id, + visit_id, + sum(views) c, + minIf(min_time, event_type NOT IN (2, 5)) min_time, + maxIf(max_time, event_type NOT IN (2, 5)) max_time, + max(if(event_type = ${EVENT_TYPE.customEvent} and length(event_name) > 0, 1, 0)) has_custom_event from website_event_stats_hourly "website_event" ${cohortQuery} ${excludeBounceQuery} - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type NOT IN (2, 5) - ${filterQuery} - group by session_id, visit_id - ) as t - left join ( - select session_id, visit_id, toUInt32(sum(length(event_name))) as events_count - from website_event_stats_hourly where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type != ${EVENT_TYPE.performance} + ${filterQuery} group by session_id, visit_id - ) as e using (session_id, visit_id); + having c > 0 + ) as t `; } diff --git a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts index 153086584..d66b912f0 100644 --- a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts +++ b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts @@ -1,5 +1,10 @@ import clickhouse from '@/lib/clickhouse'; -import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, GROUPED_DOMAINS, SESSION_COLUMNS } from '@/lib/constants'; +import { + EVENT_TYPE, + FILTER_COLUMNS, + GROUPED_DOMAINS, + SESSION_COLUMNS, +} from '@/lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import type { QueryFilters } from '@/lib/types'; @@ -46,6 +51,7 @@ async function relationalQuery( }, { joinSession: SESSION_COLUMNS.includes(type) }, ); + const needsBounceEvents = filters.excludeBounce !== true; const fullPathSearchQuery = type === 'fullPath' && filters.search ? `and (case when website_event.url_query != '' then website_event.url_path || '?' || website_event.url_query else website_event.url_path end) ilike {{fullPathSearch}}` @@ -88,6 +94,21 @@ async function relationalQuery( type === 'fullPath' ? `case when website_event.url_query != '' then website_event.url_path || '?' || website_event.url_query else website_event.url_path end` : column; + const bounceQuery = needsBounceEvents + ? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",` + : '0 as "bounces",'; + const visitEventsJoin = needsBounceEvents + ? `left join ( + select session_id, visit_id, 1 as "has_custom_event" + from website_event + where website_id = {{websiteId::uuid}} + and created_at between {{startDate}} and {{endDate}} + and event_type = ${EVENT_TYPE.customEvent} + group by 1, 2 + ) as e + on e.session_id = t.session_id + and e.visit_id = t.visit_id` + : ''; return rawQuery( ` @@ -96,7 +117,7 @@ async function relationalQuery( sum(t.c) as "pageviews", count(distinct t.session_id) as "visitors", count(distinct t.visit_id) as "visits", - sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces", + ${bounceQuery} sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime" from ( select @@ -105,16 +126,7 @@ async function relationalQuery( website_event.visit_id, count(*) as "c", min(website_event.created_at) as "min_time", - max(website_event.created_at) as "max_time", - max(( - select count(*) - from website_event we2 - where we2.website_id = website_event.website_id - and we2.session_id = website_event.session_id - and we2.visit_id = website_event.visit_id - and we2.created_at between {{startDate}} and {{endDate}} - and we2.event_type = ${EVENT_TYPE.customEvent} - )) as "events_count" + max(website_event.created_at) as "max_time" from website_event ${cohortQuery} ${excludeBounceQuery} @@ -128,6 +140,7 @@ async function relationalQuery( ${filterQuery} group by ${groupByColumn}, website_event.session_id, website_event.visit_id ) as t + ${visitEventsJoin} where name != '' group by name order by visitors desc, visits desc @@ -154,6 +167,7 @@ async function clickhouseQuery( ...filters, websiteId, }); + const needsBounceEvents = filters.excludeBounce !== true; const fullPathSearchQuery = type === 'fullPath' && filters.search ? `and positionCaseInsensitive(if(url_query != '', concat(url_path, '?', url_query), url_path), {fullPathSearch:String}) > 0` @@ -189,6 +203,19 @@ async function clickhouseQuery( } else if (type === 'fullPath') { selectColumn = `if(url_query != '', concat(url_path, '?', url_query), url_path)`; } + const bounceQuery = needsBounceEvents + ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",` + : '0 as "bounces",'; + const visitEventsJoin = needsBounceEvents + ? `left join ( + select session_id, visit_id, toUInt8(1) as has_custom_event + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = ${EVENT_TYPE.customEvent} + group by session_id, visit_id + ) as e using (session_id, visit_id)` + : ''; return rawQuery( ` @@ -197,7 +224,7 @@ async function clickhouseQuery( sum(t.c) as "pageviews", uniq(t.session_id) as "visitors", uniq(t.visit_id) as "visits", - sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces", + ${bounceQuery} sum(max_time-min_time) as "totaltime" from ( select @@ -220,14 +247,7 @@ async function clickhouseQuery( ${filterQuery} group by name, session_id, visit_id ) as t - left join ( - select session_id, visit_id, toUInt32(count()) as events_count - from website_event - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = ${EVENT_TYPE.customEvent} - group by session_id, visit_id - ) as e using (session_id, visit_id) + ${visitEventsJoin} group by name order by visitors desc, visits desc limit ${limit} diff --git a/src/queries/sql/reports/getBreakdown.ts b/src/queries/sql/reports/getBreakdown.ts index 13dcccf17..57258013b 100644 --- a/src/queries/sql/reports/getBreakdown.ts +++ b/src/queries/sql/reports/getBreakdown.ts @@ -1,5 +1,9 @@ import clickhouse from '@/lib/clickhouse'; -import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { + EVENT_TYPE, + FILTER_COLUMNS, + SESSION_COLUMNS, +} from '@/lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import type { QueryFilters } from '@/lib/types'; @@ -31,7 +35,7 @@ async function relationalQuery( ): Promise { const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma; const { startDate, endDate, fields } = parameters; - const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( + const { filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters( { ...filters, websiteId, @@ -43,6 +47,22 @@ async function relationalQuery( joinSession: !!fields.find((name: string) => SESSION_COLUMNS.includes(name)), }, ); + const needsBounceEvents = filters.excludeBounce !== true; + const bounceQuery = needsBounceEvents + ? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",` + : '0 as "bounces",'; + const visitEventsJoin = needsBounceEvents + ? `left join ( + select session_id, visit_id, 1 as "has_custom_event" + from website_event + where website_id = {{websiteId::uuid}} + and created_at between {{startDate}} and {{endDate}} + and event_type = ${EVENT_TYPE.customEvent} + group by 1, 2 + ) as e + on e.session_id = t.session_id + and e.visit_id = t.visit_id` + : ''; return rawQuery( ` @@ -50,7 +70,7 @@ async function relationalQuery( sum(t.c) as "views", count(distinct t.session_id) as "visitors", count(distinct t.visit_id) as "visits", - sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces", + ${bounceQuery} sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime", ${parseFieldsByName(fields)} from ( @@ -60,18 +80,10 @@ async function relationalQuery( website_event.visit_id, count(*) as "c", min(website_event.created_at) as "min_time", - max(website_event.created_at) as "max_time", - max(( - select count(*) - from website_event we2 - where we2.website_id = website_event.website_id - and we2.session_id = website_event.session_id - and we2.visit_id = website_event.visit_id - and we2.created_at between {{startDate}} and {{endDate}} - and we2.event_type = ${EVENT_TYPE.customEvent} - )) as "events_count" + max(website_event.created_at) as "max_time" from website_event ${cohortQuery} + ${excludeBounceQuery} ${joinSessionQuery} where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} @@ -79,6 +91,7 @@ async function relationalQuery( group by ${parseFieldsByName(fields)}, website_event.session_id, website_event.visit_id ) as t + ${visitEventsJoin} group by ${parseFieldsByName(fields)} order by 2 desc, 1 desc limit 500 @@ -94,13 +107,27 @@ async function clickhouseQuery( ): Promise { const { parseFilters, rawQuery } = clickhouse; const { startDate, endDate, fields } = parameters; - const { filterQuery, cohortQuery, queryParams } = parseFilters({ + const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({ ...filters, websiteId, startDate, endDate, eventType: EVENT_TYPE.pageView, }); + const needsBounceEvents = filters.excludeBounce !== true; + const bounceQuery = needsBounceEvents + ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",` + : '0 as "bounces",'; + const visitEventsJoin = needsBounceEvents + ? `left join ( + select session_id, visit_id, toUInt8(1) as has_custom_event + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = ${EVENT_TYPE.customEvent} + group by session_id, visit_id + ) as e using (session_id, visit_id)` + : ''; return rawQuery( ` @@ -108,7 +135,7 @@ async function clickhouseQuery( sum(t.c) as "views", count(distinct t.session_id) as "visitors", count(distinct t.visit_id) as "visits", - sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces", + ${bounceQuery} sum(max_time-min_time) as "totaltime", ${parseFieldsByName(fields)} from ( @@ -121,20 +148,14 @@ async function clickhouseQuery( max(created_at) max_time from website_event ${cohortQuery} + ${excludeBounceQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} group by ${parseFieldsByName(fields)}, session_id, visit_id ) as t - left join ( - select session_id, visit_id, toUInt32(count()) as events_count - from website_event - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = ${EVENT_TYPE.customEvent} - group by session_id, visit_id - ) as e using (session_id, visit_id) + ${visitEventsJoin} group by ${parseFieldsByName(fields)} order by 2 desc, 1 desc limit 500 diff --git a/src/queries/sql/sessions/getSessionExpandedMetrics.ts b/src/queries/sql/sessions/getSessionExpandedMetrics.ts index c917a40f9..1127469f8 100644 --- a/src/queries/sql/sessions/getSessionExpandedMetrics.ts +++ b/src/queries/sql/sessions/getSessionExpandedMetrics.ts @@ -1,5 +1,9 @@ import clickhouse from '@/lib/clickhouse'; -import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { + EVENT_TYPE, + FILTER_COLUMNS, + SESSION_COLUMNS, +} from '@/lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import type { QueryFilters } from '@/lib/types'; @@ -48,11 +52,27 @@ async function relationalQuery( joinSession: SESSION_COLUMNS.includes(type), }, ); + const needsBounceEvents = filters.excludeBounce !== true; const includeCountry = column === 'city' || column === 'region'; if (type === 'language') { column = `lower(left(${type}, 2))`; } + const bounceQuery = needsBounceEvents + ? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",` + : '0 as "bounces",'; + const visitEventsJoin = needsBounceEvents + ? `left join ( + select session_id, visit_id, 1 as "has_custom_event" + from website_event + where website_id = {{websiteId::uuid}} + and created_at between {{startDate}} and {{endDate}} + and event_type = ${EVENT_TYPE.customEvent} + group by 1, 2 + ) as e + on e.session_id = t.session_id + and e.visit_id = t.visit_id` + : ''; return rawQuery( ` @@ -62,7 +82,7 @@ async function relationalQuery( sum(t.c) as "pageviews", count(distinct t.session_id) as "visitors", count(distinct t.visit_id) as "visits", - sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces", + ${bounceQuery} sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime" from ( select @@ -72,16 +92,7 @@ async function relationalQuery( website_event.visit_id, count(*) as "c", min(website_event.created_at) as "min_time", - max(website_event.created_at) as "max_time", - max(( - select count(*) - from website_event we2 - where we2.website_id = website_event.website_id - and we2.session_id = website_event.session_id - and we2.visit_id = website_event.visit_id - and we2.created_at between {{startDate}} and {{endDate}} - and we2.event_type = ${EVENT_TYPE.customEvent} - )) as "events_count" + max(website_event.created_at) as "max_time" from website_event ${cohortQuery} ${excludeBounceQuery} @@ -93,6 +104,7 @@ async function relationalQuery( group by name, website_event.session_id, website_event.visit_id ${includeCountry ? ', country' : ''} ) as t + ${visitEventsJoin} where name != '' group by name ${includeCountry ? ', country' : ''} @@ -117,11 +129,25 @@ async function clickhouseQuery( ...filters, websiteId, }); + const needsBounceEvents = filters.excludeBounce !== true; const includeCountry = column === 'city' || column === 'region'; if (type === 'language') { column = `lower(left(${type}, 2))`; } + const bounceQuery = needsBounceEvents + ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",` + : '0 as "bounces",'; + const visitEventsJoin = needsBounceEvents + ? `left join ( + select session_id, visit_id, toUInt8(1) as has_custom_event + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = ${EVENT_TYPE.customEvent} + group by session_id, visit_id + ) as e using (session_id, visit_id)` + : ''; return rawQuery( ` @@ -131,7 +157,7 @@ async function clickhouseQuery( sum(t.c) as "pageviews", uniq(t.session_id) as "visitors", uniq(t.visit_id) as "visits", - sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces", + ${bounceQuery} sum(max_time-min_time) as "totaltime" from ( select @@ -153,14 +179,7 @@ async function clickhouseQuery( group by name, session_id, visit_id ${includeCountry ? ', country' : ''} ) as t - left join ( - select session_id, visit_id, toUInt32(count()) as events_count - from website_event - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = ${EVENT_TYPE.customEvent} - group by session_id, visit_id - ) as e using (session_id, visit_id) + ${visitEventsJoin} group by name ${includeCountry ? ', country' : ''} order by visitors desc, visits desc