Optimize bounce detection and enforce unique session data entries

This commit is contained in:
Mike Cao committed 2026-08-01 23:14:43 -07:00
commit be1531edf7
16 files changed
+561 -182

No files matched your search

-1
View File
@@ -7,7 +7,6 @@ services:
environment: environment:
DATABASE_URL: postgresql://umami:umami@db:5432/umami DATABASE_URL: postgresql://umami:umami@db:5432/umami
APP_SECRET: replace-me-with-a-random-string APP_SECRET: replace-me-with-a-random-string
# BOUNCE_THRESHOLD: 2 # OPTIONAL: set the minimum amount custom events to trigger a bounce
depends_on: depends_on:
db: db:
condition: service_healthy condition: service_healthy
@@ -0,0 +1,16 @@
WITH ranked_session_data AS (
SELECT
"session_data_id",
ROW_NUMBER() OVER (
PARTITION BY "session_id", "data_key"
ORDER BY "created_at" DESC NULLS LAST, "session_data_id" DESC
) AS row_num
FROM "session_data"
)
DELETE FROM "session_data"
USING ranked_session_data
WHERE "session_data"."session_data_id" = ranked_session_data."session_data_id"
AND ranked_session_data.row_num > 1;
CREATE UNIQUE INDEX "session_data_session_id_data_key_key"
ON "session_data"("session_id", "data_key");
+1
View File
@@ -208,6 +208,7 @@ model SessionData {
@@index([sessionId]) @@index([sessionId])
@@index([sessionId, createdAt]) @@index([sessionId, createdAt])
@@index([websiteId, createdAt, dataKey]) @@index([websiteId, createdAt, dataKey])
@@unique([sessionId, dataKey])
@@map("session_data") @@map("session_data")
} }
+6 -2
View File
@@ -177,13 +177,17 @@ function getExcludeBounceQuery(filters: Record<string, any>) {
} }
return `join return `join
(select distinct session_id as exclude_session_id, visit_id as exclude_visit_id (select session_id as exclude_session_id, visit_id as exclude_visit_id
from website_event_stats_hourly from website_event_stats_hourly
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 1 and event_type != 5
group by session_id, visit_id group by session_id, visit_id
having sum(views) > 1 having sum(views) > 1
or (
sum(views) = 1
and sumIf(length(event_name), event_type = 2) > 0
)
) excludeBounce ) excludeBounce
on excludeBounce.exclude_session_id = website_event.session_id on excludeBounce.exclude_session_id = website_event.session_id
and excludeBounce.exclude_visit_id = website_event.visit_id and excludeBounce.exclude_visit_id = website_event.visit_id
-1
View File
@@ -1,5 +1,4 @@
export const CURRENT_VERSION = process.env.currentVersion; export const CURRENT_VERSION = process.env.currentVersion;
export const BOUNCE_THRESHOLD = Math.max(1, Number.parseInt(process.env.BOUNCE_THRESHOLD, 10) || 1);
export const AUTH_TOKEN = 'umami.auth'; export const AUTH_TOKEN = 'umami.auth';
export const LOCALE_CONFIG = 'umami.locale'; export const LOCALE_CONFIG = 'umami.locale';
export const TIMEZONE_CONFIG = 'umami.timezone'; export const TIMEZONE_CONFIG = 'umami.timezone';
+8 -10
View File
@@ -2,13 +2,7 @@ import { PrismaPg } from '@prisma/adapter-pg';
import { readReplicas } from '@prisma/extension-read-replicas'; import { readReplicas } from '@prisma/extension-read-replicas';
import debug from 'debug'; import debug from 'debug';
import { PrismaClient } from '@/generated/prisma/client'; import { PrismaClient } from '@/generated/prisma/client';
import { import { DATA_TYPE, DEFAULT_PAGE_SIZE, FILTER_COLUMNS, OPERATORS, SESSION_COLUMNS } from './constants';
DATA_TYPE,
DEFAULT_PAGE_SIZE,
FILTER_COLUMNS,
OPERATORS,
SESSION_COLUMNS,
} from './constants';
import { filtersObjectToArray } from './params'; import { filtersObjectToArray } from './params';
import type { Operator, PropertyFilter, QueryFilters, QueryOptions } from './types'; import type { Operator, PropertyFilter, QueryFilters, QueryOptions } from './types';
@@ -249,13 +243,17 @@ function getExcludeBounceQuery(filters: Record<string, any>) {
} }
return `join return `join
(select distinct session_id, visit_id (select session_id, visit_id
from website_event from website_event
where website_id = {{websiteId}} where website_id = {{websiteId}}
and created_at between {{startDate}} and {{endDate}} and created_at between {{startDate}} and {{endDate}}
and event_type = 1 and event_type != 5
group by session_id, visit_id group by session_id, visit_id
having count(*) > 1 having sum(case when event_type NOT IN (2, 5) then 1 else 0 end) > 1
or (
sum(case when event_type NOT IN (2, 5) then 1 else 0 end) = 1
and sum(case when event_type = 2 then 1 else 0 end) > 0
)
) excludeBounce ) excludeBounce
on excludeBounce.session_id = website_event.session_id on excludeBounce.session_id = website_event.session_id
and excludeBounce.visit_id = website_event.visit_id and excludeBounce.visit_id = website_event.visit_id
+15 -17
View File
@@ -1,6 +1,5 @@
import { beforeEach, describe, expect, test, vi } from 'vitest'; import { beforeEach, describe, expect, test, vi } from 'vitest';
import { getEntity } from '@/lib/entity'; import { getEntity } from '@/lib/entity';
import prisma from '@/lib/prisma';
import { getTeamUser, getWebsite } from '@/queries/prisma'; import { getTeamUser, getWebsite } from '@/queries/prisma';
import { import {
canCreateWebsite, canCreateWebsite,
@@ -13,6 +12,11 @@ import {
canViewWebsite, canViewWebsite,
} from './website'; } from './website';
const { websiteFindManyMock, teamUserFindManyMock } = vi.hoisted(() => ({
websiteFindManyMock: vi.fn(),
teamUserFindManyMock: vi.fn(),
}));
vi.mock('@/lib/entity', () => ({ vi.mock('@/lib/entity', () => ({
getEntity: vi.fn(), getEntity: vi.fn(),
})); }));
@@ -26,10 +30,10 @@ vi.mock('@/lib/prisma', () => ({
default: { default: {
client: { client: {
website: { website: {
findMany: vi.fn(), findMany: websiteFindManyMock,
}, },
teamUser: { teamUser: {
findMany: vi.fn(), findMany: teamUserFindManyMock,
}, },
}, },
}, },
@@ -54,8 +58,8 @@ beforeEach(() => {
vi.mocked(getEntity).mockReset(); vi.mocked(getEntity).mockReset();
vi.mocked(getWebsite).mockReset(); vi.mocked(getWebsite).mockReset();
vi.mocked(getTeamUser).mockReset(); vi.mocked(getTeamUser).mockReset();
vi.mocked(prisma.client.website.findMany as (...args: unknown[]) => unknown).mockReset(); websiteFindManyMock.mockReset();
vi.mocked(prisma.client.teamUser.findMany as (...args: unknown[]) => unknown).mockReset(); teamUserFindManyMock.mockReset();
}); });
describe('canViewWebsite', () => { describe('canViewWebsite', () => {
@@ -139,16 +143,12 @@ describe('canViewBatchWebsites', () => {
}); });
test('returns owned, team, and share allowed ids for a user', async () => { test('returns owned, team, and share allowed ids for a user', async () => {
vi.mocked( websiteFindManyMock.mockResolvedValue([
prisma.client.website.findMany as (...args: unknown[]) => Promise<unknown>,
).mockResolvedValue([
{ id: 'owned', userId: 'user-1', teamId: null }, { id: 'owned', userId: 'user-1', teamId: null },
{ id: 'team', userId: null, teamId: 'team-1' }, { id: 'team', userId: null, teamId: 'team-1' },
{ id: 'foreign', userId: 'other', teamId: null }, { id: 'foreign', userId: 'other', teamId: null },
] as any); ] as any);
vi.mocked( teamUserFindManyMock.mockResolvedValue([{ teamId: 'team-1' }] as any);
prisma.client.teamUser.findMany as (...args: unknown[]) => Promise<unknown>,
).mockResolvedValue([{ teamId: 'team-1' }] as any);
await expect( await expect(
canViewBatchWebsites({ user: normalUser, shareToken: { websiteId: 'shared' } as any }, [ canViewBatchWebsites({ user: normalUser, shareToken: { websiteId: 'shared' } as any }, [
@@ -161,12 +161,10 @@ describe('canViewBatchWebsites', () => {
}); });
test('excludes team websites when the user is not a team member', async () => { test('excludes team websites when the user is not a team member', async () => {
vi.mocked( websiteFindManyMock.mockResolvedValue([
prisma.client.website.findMany as (...args: unknown[]) => Promise<unknown>, { id: 'team', userId: null, teamId: 'team-1' },
).mockResolvedValue([{ id: 'team', userId: null, teamId: 'team-1' }] as any); ] as any);
vi.mocked( teamUserFindManyMock.mockResolvedValue([] as any);
prisma.client.teamUser.findMany as (...args: unknown[]) => Promise<unknown>,
).mockResolvedValue([] as any);
await expect(canViewBatchWebsites({ user: normalUser }, ['team'])).resolves.toEqual([]); await expect(canViewBatchWebsites({ user: normalUser }, ['team'])).resolves.toEqual([]);
}); });
@@ -1,8 +1,6 @@
import { beforeEach, describe, expect, test, vi } from 'vitest'; import { beforeEach, describe, expect, test, vi } from 'vitest';
import { import {
BOUNCE_THRESHOLD,
EMAIL_DOMAINS, EMAIL_DOMAINS,
EVENT_TYPE,
LLM_DOMAINS, LLM_DOMAINS,
PAID_AD_PARAMS, PAID_AD_PARAMS,
SEARCH_DOMAINS, SEARCH_DOMAINS,
@@ -85,12 +83,9 @@ describe('getChannelExpandedMetrics postgres branch', () => {
expect(query).toContain('sum(visit_stats.c) as "pageviews"'); expect(query).toContain('sum(visit_stats.c) as "pageviews"');
expect(query).toContain('count(distinct visit_stats.session_id) as "visitors"'); expect(query).toContain('count(distinct visit_stats.session_id) as "visitors"');
expect(query).toContain('count(distinct visit_stats.visit_id) as "visits"'); expect(query).toContain('count(distinct visit_stats.visit_id) as "visits"');
expect(query).toContain('sum(case when visit_stats.c = 1 and coalesce(visit_events.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces"');
expect(query).toContain('visit_events as'); expect(query).toContain('visit_events as');
expect(query).toContain(`and event_type = ${EVENT_TYPE.customEvent}`);
expect(query).toContain('left join visit_events'); expect(query).toContain('left join visit_events');
expect(query).toContain(
`sum(case when visit_stats.c = 1 and coalesce(visit_events.events_count, 0) < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces"`,
);
// getTimestampDiffSQL output is injected into the totaltime column // getTimestampDiffSQL output is injected into the totaltime column
expect(query).toContain('sum(ts_diff(min_time, max_time)) as "totaltime"'); expect(query).toContain('sum(ts_diff(min_time, max_time)) as "totaltime"');
expect(getTimestampDiffSQL).toHaveBeenCalledWith( expect(getTimestampDiffSQL).toHaveBeenCalledWith(
@@ -158,11 +153,8 @@ describe('getChannelExpandedMetrics clickhouse branch', () => {
expect(query).toContain('uniq(t.session_id) as "visitors"'); expect(query).toContain('uniq(t.session_id) as "visitors"');
expect(query).toContain('uniq(t.visit_id) as "visits"'); expect(query).toContain('uniq(t.visit_id) as "visits"');
expect(query).toContain( expect(query).toContain('sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces"');
`sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces"`, expect(query).toContain('left join (');
);
expect(query).toContain(`and event_type = ${EVENT_TYPE.customEvent}`);
expect(query).toContain('as e using (session_id, visit_id)');
expect(query).toContain('sum(max_time-min_time) as "totaltime"'); expect(query).toContain('sum(max_time-min_time) as "totaltime"');
expect(query).toContain('where website_id = {websiteId:UUID}'); expect(query).toContain('where website_id = {websiteId:UUID}');
expect(query).toContain( expect(query).toContain(
+42 -25
View File
@@ -1,6 +1,5 @@
import clickhouse from '@/lib/clickhouse'; import clickhouse from '@/lib/clickhouse';
import { import {
BOUNCE_THRESHOLD,
EMAIL_DOMAINS, EMAIL_DOMAINS,
EVENT_TYPE, EVENT_TYPE,
LLM_DOMAINS, LLM_DOMAINS,
@@ -49,6 +48,29 @@ async function relationalQuery(
...filters, ...filters,
websiteId, websiteId,
}); });
const needsBounceEvents = filters.excludeBounce !== true;
const bounceQuery = needsBounceEvents
? `sum(case when visit_stats.c = 1 and coalesce(visit_events.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`
: '0 as "bounces",';
const visitEventsQuery = needsBounceEvents
? `,
visit_events as (
select
session_id,
visit_id,
1 as has_custom_event
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id)`
: '';
const visitEventsJoin = needsBounceEvents
? `left join visit_events
on visit_events.session_id = visit_stats.session_id
and visit_events.visit_id = visit_stats.visit_id`
: '';
return rawQuery( return rawQuery(
` `
@@ -122,33 +144,21 @@ async function relationalQuery(
min(created_at) as min_time, min(created_at) as min_time,
max(created_at) as max_time max(created_at) as max_time
from prefix from prefix
group by session_id, visit_id),
visit_events as (
select
session_id,
visit_id,
count(*) as events_count
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id) group by session_id, visit_id)
${visitEventsQuery}
select select
visit_channels.name, visit_channels.name,
sum(visit_stats.c) as "pageviews", sum(visit_stats.c) as "pageviews",
count(distinct visit_stats.session_id) as "visitors", count(distinct visit_stats.session_id) as "visitors",
count(distinct visit_stats.visit_id) as "visits", count(distinct visit_stats.visit_id) as "visits",
sum(case when visit_stats.c = 1 and coalesce(visit_events.events_count, 0) < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces", ${bounceQuery}
sum(${getTimestampDiffSQL('visit_stats.min_time', 'visit_stats.max_time')}) as "totaltime" sum(${getTimestampDiffSQL('visit_stats.min_time', 'visit_stats.max_time')}) as "totaltime"
from visit_stats from visit_stats
join visit_channels join visit_channels
on visit_channels.session_id = visit_stats.session_id on visit_channels.session_id = visit_stats.session_id
and visit_channels.visit_id = visit_stats.visit_id and visit_channels.visit_id = visit_stats.visit_id
left join visit_events ${visitEventsJoin}
on visit_events.session_id = visit_stats.session_id
and visit_events.visit_id = visit_stats.visit_id
group by visit_channels.name group by visit_channels.name
order by visitors desc, visits desc order by visitors desc, visits desc
`, `,
@@ -166,6 +176,20 @@ async function clickhouseQuery(
...filters, ...filters,
websiteId, websiteId,
}); });
const needsBounceEvents = filters.excludeBounce !== true;
const bounceQuery = needsBounceEvents
? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`
: '0 as "bounces",';
const visitEventsJoin = needsBounceEvents
? `left join (
select session_id, visit_id, toUInt8(1) as has_custom_event
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id
) as e using (session_id, visit_id)`
: '';
return rawQuery( return rawQuery(
` `
@@ -174,7 +198,7 @@ async function clickhouseQuery(
sum(t.c) as "pageviews", sum(t.c) as "pageviews",
uniq(t.session_id) as "visitors", uniq(t.session_id) as "visitors",
uniq(t.visit_id) as "visits", uniq(t.visit_id) as "visits",
sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces", ${bounceQuery}
sum(max_time-min_time) as "totaltime" sum(max_time-min_time) as "totaltime"
from ( from (
select select
@@ -229,14 +253,7 @@ async function clickhouseQuery(
) )
group by session_id, visit_id group by session_id, visit_id
) as t ) as t
left join ( ${visitEventsJoin}
select session_id, visit_id, toUInt32(count()) as events_count
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id
) as e using (session_id, visit_id)
group by name group by name
order by visitors desc, visits desc; order by visitors desc, visits desc;
`, `,
+135
View File
@@ -0,0 +1,135 @@
import { afterEach, describe, expect, test, vi } from 'vitest';
const parseFiltersResult = {
queryParams: { websiteId: 'website-1' },
filterQuery: 'and website_event.url_path = {{path}}',
joinSessionQuery: 'join session on session.session_id = website_event.session_id',
cohortQuery: 'join cohort on cohort.session_id = website_event.session_id',
excludeBounceQuery: 'join excludeBounce on excludeBounce.visit_id = website_event.visit_id',
};
async function loadModule({
mode,
}: {
mode: 'prisma' | 'clickhouse';
}) {
vi.resetModules();
const state = { mode };
const prismaRawQuery = vi.fn().mockResolvedValue([{}]);
const prismaParseFilters = vi.fn().mockReturnValue(parseFiltersResult);
const getTimestampDiffSQL = vi.fn().mockReturnValue('ts_diff(t.min_time, t.max_time)');
const clickhouseRawQuery = vi.fn().mockResolvedValue([{}]);
const clickhouseParseFilters = vi.fn().mockReturnValue(parseFiltersResult);
vi.doMock('@/lib/db', () => ({
CLICKHOUSE: 'clickhouse',
PRISMA: 'prisma',
runQuery: vi.fn((queries: Record<string, () => unknown>) => queries[state.mode]()),
}));
vi.doMock('@/lib/prisma', () => ({
default: {
rawQuery: prismaRawQuery,
parseFilters: prismaParseFilters,
getTimestampDiffSQL,
},
}));
vi.doMock('@/lib/clickhouse', () => ({
default: {
rawQuery: clickhouseRawQuery,
parseFilters: clickhouseParseFilters,
},
}));
const mod = await import('./getWebsiteStats');
return {
getWebsiteStats: mod.getWebsiteStats,
prismaRawQuery,
clickhouseRawQuery,
};
}
afterEach(() => {
vi.resetModules();
vi.clearAllMocks();
});
describe('getWebsiteStats default bounce path', () => {
test('postgres uses a single grouped pass for the default path', async () => {
const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' });
await getWebsiteStats('website-1', {});
const [query] = prismaRawQuery.mock.calls[0];
expect(query).toContain('t.c = 1 and t.has_custom_event = 0');
expect(query).toContain('max(case when website_event.event_type = 2 then 1 else 0 end) as "has_custom_event"');
expect(query).not.toContain('left join (');
});
test('clickhouse uses the hourly aggregate view without the extra join', async () => {
const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });
await getWebsiteStats('website-1', {});
const [query] = clickhouseRawQuery.mock.calls[0];
expect(query).toContain('from website_event_stats_hourly "website_event"');
expect(query).toContain('sumIf(1, t.c = 1 and t.has_custom_event = 0) as "bounces"');
expect(query).toContain('has_custom_event');
expect(query).not.toContain('left join (');
});
});
describe('getWebsiteStats filtered bounce path', () => {
test('postgres switches to the visit-events join when event filters are present', async () => {
const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' });
await getWebsiteStats('website-1', { path: '/pricing' } as any);
const [query] = prismaRawQuery.mock.calls[0];
expect(query).toContain('left join (');
expect(query).toContain('coalesce(e.has_custom_event, 0) = 0');
expect(query).toContain('event_type = 2');
});
test('clickhouse switches to the raw-event join when event filters are present', async () => {
const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });
await getWebsiteStats('website-1', { path: '/pricing' } as any);
const [query] = clickhouseRawQuery.mock.calls[0];
expect(query).toContain('from website_event');
expect(query).toContain('left join (');
expect(query).toContain('ifNull(e.has_custom_event, 0) = 0');
});
});
describe('getWebsiteStats excludeBounce path', () => {
test('clickhouse skips the extra join when excludeBounce is enabled', async () => {
const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });
await getWebsiteStats('website-1', { excludeBounce: true });
const [query] = clickhouseRawQuery.mock.calls[0];
expect(query).toContain('0 as "bounces"');
expect(query).not.toContain('left join (');
});
test('postgres skips the extra join when excludeBounce is enabled on the filtered path', async () => {
const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' });
await getWebsiteStats('website-1', { excludeBounce: true, path: '/pricing' } as any);
const [query] = prismaRawQuery.mock.calls[0];
expect(query).toContain('0 as "bounces"');
expect(query).not.toContain('left join (');
});
});
+76 -35
View File
@@ -1,5 +1,5 @@
import clickhouse from '@/lib/clickhouse'; import clickhouse from '@/lib/clickhouse';
import { BOUNCE_THRESHOLD, EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants'; import { EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma'; import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types'; import type { QueryFilters } from '@/lib/types';
@@ -35,9 +35,60 @@ async function relationalQuery(
}); });
const { excludeBounce } = filters; const { excludeBounce } = filters;
const hasEventFilters =
EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) ||
!!filters.eventPropertyFilters?.length;
if (!hasEventFilters) {
return rawQuery(
`
select
cast(coalesce(sum(t.c), 0) as bigint) as "pageviews",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
${excludeBounce ? '0' : 'coalesce(sum(case when t.c = 1 and t.has_custom_event = 0 then 1 else 0 end), 0)'} as "bounces",
cast(coalesce(sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}), 0) as bigint) as "totaltime"
from (
select
website_event.session_id,
website_event.visit_id,
sum(case when website_event.event_type NOT IN (2, 5) then 1 else 0 end) as "c",
min(case when website_event.event_type NOT IN (2, 5) then website_event.created_at end) as "min_time",
max(case when website_event.event_type NOT IN (2, 5) then website_event.created_at end) as "max_time",
max(case when website_event.event_type = ${EVENT_TYPE.customEvent} then 1 else 0 end) as "has_custom_event"
from website_event
${cohortQuery}
${excludeBounceQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != ${EVENT_TYPE.performance}
${filterQuery}
group by 1, 2
having sum(case when website_event.event_type NOT IN (2, 5) then 1 else 0 end) > 0
) as t
`,
queryParams,
FUNCTION_NAME,
).then(result => result?.[0]);
}
const bounceQuery = excludeBounce const bounceQuery = excludeBounce
? '0' ? '0'
: `coalesce(sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end), 0)`; : 'coalesce(sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end), 0)';
const visitEventsJoin = excludeBounce
? ''
: `
left join (
select session_id, visit_id, 1 as "has_custom_event"
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and event_type = ${EVENT_TYPE.customEvent}
group by 1, 2
) as e
on e.session_id = t.session_id
and e.visit_id = t.visit_id`;
return rawQuery( return rawQuery(
` `
@@ -53,16 +104,7 @@ async function relationalQuery(
website_event.visit_id, website_event.visit_id,
count(*) as "c", count(*) as "c",
min(website_event.created_at) as "min_time", min(website_event.created_at) as "min_time",
max(website_event.created_at) as "max_time", max(website_event.created_at) as "max_time"
max((
select count(*)
from website_event we2
where we2.website_id = website_event.website_id
and we2.session_id = website_event.session_id
and we2.visit_id = website_event.visit_id
and we2.created_at between {{startDate}} and {{endDate}}
and we2.event_type = ${EVENT_TYPE.customEvent}
)) as "events_count"
from website_event from website_event
${cohortQuery} ${cohortQuery}
${excludeBounceQuery} ${excludeBounceQuery}
@@ -72,8 +114,8 @@ async function relationalQuery(
and website_event.event_type NOT IN (2, 5) and website_event.event_type NOT IN (2, 5)
${filterQuery} ${filterQuery}
group by 1, 2 group by 1, 2
) as t ) as t
${visitEventsJoin}
`, `,
queryParams, queryParams,
FUNCTION_NAME, FUNCTION_NAME,
@@ -92,17 +134,20 @@ async function clickhouseQuery(
let sql = ''; let sql = '';
const { excludeBounce } = filters; const { excludeBounce } = filters;
const hasEventFilters =
EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) ||
!!filters.eventPropertyFilters?.length;
const bounceQuery = excludeBounce const bounceQuery = excludeBounce
? '0' ? '0'
: `sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD})`; : 'sumIf(1, t.c = 1 and t.has_custom_event = 0)';
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) { if (hasEventFilters) {
sql = ` sql = `
select select
sum(t.c) as "pageviews", sum(t.c) as "pageviews",
uniq(t.session_id) as "visitors", uniq(t.session_id) as "visitors",
uniq(t.visit_id) as "visits", uniq(t.visit_id) as "visits",
${bounceQuery} as "bounces", ${excludeBounce ? '0' : 'sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0)'} as "bounces",
sum(max_time-min_time) as "totaltime" sum(max_time-min_time) as "totaltime"
from ( from (
select select
@@ -120,14 +165,14 @@ async function clickhouseQuery(
${filterQuery} ${filterQuery}
group by session_id, visit_id group by session_id, visit_id
) as t ) as t
left join ( ${excludeBounce ? '' : `left join (
select session_id, visit_id, toUInt32(count()) as events_count select session_id, visit_id, toUInt8(1) as has_custom_event
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = ${EVENT_TYPE.customEvent} and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id group by session_id, visit_id
) as e using (session_id, visit_id); ) as e using (session_id, visit_id)`};
`; `;
} else { } else {
sql = ` sql = `
@@ -137,28 +182,24 @@ async function clickhouseQuery(
uniq(visit_id) as "visits", uniq(visit_id) as "visits",
${bounceQuery} as "bounces", ${bounceQuery} as "bounces",
sum(max_time-min_time) as "totaltime" sum(max_time-min_time) as "totaltime"
from (select from (
session_id, select
visit_id, session_id,
sum(views) c, visit_id,
min(min_time) min_time, sum(views) c,
max(max_time) max_time minIf(min_time, event_type NOT IN (2, 5)) min_time,
maxIf(max_time, event_type NOT IN (2, 5)) max_time,
max(if(event_type = ${EVENT_TYPE.customEvent} and length(event_name) > 0, 1, 0)) has_custom_event
from website_event_stats_hourly "website_event" from website_event_stats_hourly "website_event"
${cohortQuery} ${cohortQuery}
${excludeBounceQuery} ${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type NOT IN (2, 5)
${filterQuery}
group by session_id, visit_id
) as t
left join (
select session_id, visit_id, toUInt32(sum(length(event_name))) as events_count
from website_event_stats_hourly
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != ${EVENT_TYPE.performance}
${filterQuery}
group by session_id, visit_id group by session_id, visit_id
) as e using (session_id, visit_id); having c > 0
) as t
`; `;
} }
@@ -1,5 +1,10 @@
import clickhouse from '@/lib/clickhouse'; import clickhouse from '@/lib/clickhouse';
import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, GROUPED_DOMAINS, SESSION_COLUMNS } from '@/lib/constants'; import {
EVENT_TYPE,
FILTER_COLUMNS,
GROUPED_DOMAINS,
SESSION_COLUMNS,
} from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma'; import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types'; import type { QueryFilters } from '@/lib/types';
@@ -46,6 +51,7 @@ async function relationalQuery(
}, },
{ joinSession: SESSION_COLUMNS.includes(type) }, { joinSession: SESSION_COLUMNS.includes(type) },
); );
const needsBounceEvents = filters.excludeBounce !== true;
const fullPathSearchQuery = const fullPathSearchQuery =
type === 'fullPath' && filters.search type === 'fullPath' && filters.search
? `and (case when website_event.url_query != '' then website_event.url_path || '?' || website_event.url_query else website_event.url_path end) ilike {{fullPathSearch}}` ? `and (case when website_event.url_query != '' then website_event.url_path || '?' || website_event.url_query else website_event.url_path end) ilike {{fullPathSearch}}`
@@ -88,6 +94,21 @@ async function relationalQuery(
type === 'fullPath' type === 'fullPath'
? `case when website_event.url_query != '' then website_event.url_path || '?' || website_event.url_query else website_event.url_path end` ? `case when website_event.url_query != '' then website_event.url_path || '?' || website_event.url_query else website_event.url_path end`
: column; : column;
const bounceQuery = needsBounceEvents
? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`
: '0 as "bounces",';
const visitEventsJoin = needsBounceEvents
? `left join (
select session_id, visit_id, 1 as "has_custom_event"
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and event_type = ${EVENT_TYPE.customEvent}
group by 1, 2
) as e
on e.session_id = t.session_id
and e.visit_id = t.visit_id`
: '';
return rawQuery( return rawQuery(
` `
@@ -96,7 +117,7 @@ async function relationalQuery(
sum(t.c) as "pageviews", sum(t.c) as "pageviews",
count(distinct t.session_id) as "visitors", count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits", count(distinct t.visit_id) as "visits",
sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces", ${bounceQuery}
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime" sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime"
from ( from (
select select
@@ -105,16 +126,7 @@ async function relationalQuery(
website_event.visit_id, website_event.visit_id,
count(*) as "c", count(*) as "c",
min(website_event.created_at) as "min_time", min(website_event.created_at) as "min_time",
max(website_event.created_at) as "max_time", max(website_event.created_at) as "max_time"
max((
select count(*)
from website_event we2
where we2.website_id = website_event.website_id
and we2.session_id = website_event.session_id
and we2.visit_id = website_event.visit_id
and we2.created_at between {{startDate}} and {{endDate}}
and we2.event_type = ${EVENT_TYPE.customEvent}
)) as "events_count"
from website_event from website_event
${cohortQuery} ${cohortQuery}
${excludeBounceQuery} ${excludeBounceQuery}
@@ -128,6 +140,7 @@ async function relationalQuery(
${filterQuery} ${filterQuery}
group by ${groupByColumn}, website_event.session_id, website_event.visit_id group by ${groupByColumn}, website_event.session_id, website_event.visit_id
) as t ) as t
${visitEventsJoin}
where name != '' where name != ''
group by name group by name
order by visitors desc, visits desc order by visitors desc, visits desc
@@ -154,6 +167,7 @@ async function clickhouseQuery(
...filters, ...filters,
websiteId, websiteId,
}); });
const needsBounceEvents = filters.excludeBounce !== true;
const fullPathSearchQuery = const fullPathSearchQuery =
type === 'fullPath' && filters.search type === 'fullPath' && filters.search
? `and positionCaseInsensitive(if(url_query != '', concat(url_path, '?', url_query), url_path), {fullPathSearch:String}) > 0` ? `and positionCaseInsensitive(if(url_query != '', concat(url_path, '?', url_query), url_path), {fullPathSearch:String}) > 0`
@@ -189,6 +203,19 @@ async function clickhouseQuery(
} else if (type === 'fullPath') { } else if (type === 'fullPath') {
selectColumn = `if(url_query != '', concat(url_path, '?', url_query), url_path)`; selectColumn = `if(url_query != '', concat(url_path, '?', url_query), url_path)`;
} }
const bounceQuery = needsBounceEvents
? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`
: '0 as "bounces",';
const visitEventsJoin = needsBounceEvents
? `left join (
select session_id, visit_id, toUInt8(1) as has_custom_event
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id
) as e using (session_id, visit_id)`
: '';
return rawQuery( return rawQuery(
` `
@@ -197,7 +224,7 @@ async function clickhouseQuery(
sum(t.c) as "pageviews", sum(t.c) as "pageviews",
uniq(t.session_id) as "visitors", uniq(t.session_id) as "visitors",
uniq(t.visit_id) as "visits", uniq(t.visit_id) as "visits",
sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces", ${bounceQuery}
sum(max_time-min_time) as "totaltime" sum(max_time-min_time) as "totaltime"
from ( from (
select select
@@ -220,14 +247,7 @@ async function clickhouseQuery(
${filterQuery} ${filterQuery}
group by name, session_id, visit_id group by name, session_id, visit_id
) as t ) as t
left join ( ${visitEventsJoin}
select session_id, visit_id, toUInt32(count()) as events_count
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id
) as e using (session_id, visit_id)
group by name group by name
order by visitors desc, visits desc order by visitors desc, visits desc
limit ${limit} limit ${limit}
+44 -23
View File
@@ -1,5 +1,9 @@
import clickhouse from '@/lib/clickhouse'; import clickhouse from '@/lib/clickhouse';
import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; import {
EVENT_TYPE,
FILTER_COLUMNS,
SESSION_COLUMNS,
} from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma'; import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types'; import type { QueryFilters } from '@/lib/types';
@@ -31,7 +35,7 @@ async function relationalQuery(
): Promise<BreakdownData[]> { ): Promise<BreakdownData[]> {
const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma; const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma;
const { startDate, endDate, fields } = parameters; const { startDate, endDate, fields } = parameters;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( const { filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters(
{ {
...filters, ...filters,
websiteId, websiteId,
@@ -43,6 +47,22 @@ async function relationalQuery(
joinSession: !!fields.find((name: string) => SESSION_COLUMNS.includes(name)), joinSession: !!fields.find((name: string) => SESSION_COLUMNS.includes(name)),
}, },
); );
const needsBounceEvents = filters.excludeBounce !== true;
const bounceQuery = needsBounceEvents
? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`
: '0 as "bounces",';
const visitEventsJoin = needsBounceEvents
? `left join (
select session_id, visit_id, 1 as "has_custom_event"
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and event_type = ${EVENT_TYPE.customEvent}
group by 1, 2
) as e
on e.session_id = t.session_id
and e.visit_id = t.visit_id`
: '';
return rawQuery( return rawQuery(
` `
@@ -50,7 +70,7 @@ async function relationalQuery(
sum(t.c) as "views", sum(t.c) as "views",
count(distinct t.session_id) as "visitors", count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits", count(distinct t.visit_id) as "visits",
sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces", ${bounceQuery}
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime", sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime",
${parseFieldsByName(fields)} ${parseFieldsByName(fields)}
from ( from (
@@ -60,18 +80,10 @@ async function relationalQuery(
website_event.visit_id, website_event.visit_id,
count(*) as "c", count(*) as "c",
min(website_event.created_at) as "min_time", min(website_event.created_at) as "min_time",
max(website_event.created_at) as "max_time", max(website_event.created_at) as "max_time"
max((
select count(*)
from website_event we2
where we2.website_id = website_event.website_id
and we2.session_id = website_event.session_id
and we2.visit_id = website_event.visit_id
and we2.created_at between {{startDate}} and {{endDate}}
and we2.event_type = ${EVENT_TYPE.customEvent}
)) as "events_count"
from website_event from website_event
${cohortQuery} ${cohortQuery}
${excludeBounceQuery}
${joinSessionQuery} ${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}} where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}} and website_event.created_at between {{startDate}} and {{endDate}}
@@ -79,6 +91,7 @@ async function relationalQuery(
group by ${parseFieldsByName(fields)}, group by ${parseFieldsByName(fields)},
website_event.session_id, website_event.visit_id website_event.session_id, website_event.visit_id
) as t ) as t
${visitEventsJoin}
group by ${parseFieldsByName(fields)} group by ${parseFieldsByName(fields)}
order by 2 desc, 1 desc order by 2 desc, 1 desc
limit 500 limit 500
@@ -94,13 +107,27 @@ async function clickhouseQuery(
): Promise<BreakdownData[]> { ): Promise<BreakdownData[]> {
const { parseFilters, rawQuery } = clickhouse; const { parseFilters, rawQuery } = clickhouse;
const { startDate, endDate, fields } = parameters; const { startDate, endDate, fields } = parameters;
const { filterQuery, cohortQuery, queryParams } = parseFilters({ const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({
...filters, ...filters,
websiteId, websiteId,
startDate, startDate,
endDate, endDate,
eventType: EVENT_TYPE.pageView, eventType: EVENT_TYPE.pageView,
}); });
const needsBounceEvents = filters.excludeBounce !== true;
const bounceQuery = needsBounceEvents
? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`
: '0 as "bounces",';
const visitEventsJoin = needsBounceEvents
? `left join (
select session_id, visit_id, toUInt8(1) as has_custom_event
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id
) as e using (session_id, visit_id)`
: '';
return rawQuery( return rawQuery(
` `
@@ -108,7 +135,7 @@ async function clickhouseQuery(
sum(t.c) as "views", sum(t.c) as "views",
count(distinct t.session_id) as "visitors", count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits", count(distinct t.visit_id) as "visits",
sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces", ${bounceQuery}
sum(max_time-min_time) as "totaltime", sum(max_time-min_time) as "totaltime",
${parseFieldsByName(fields)} ${parseFieldsByName(fields)}
from ( from (
@@ -121,20 +148,14 @@ async function clickhouseQuery(
max(created_at) max_time max(created_at) max_time
from website_event from website_event
${cohortQuery} ${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery} ${filterQuery}
group by ${parseFieldsByName(fields)}, group by ${parseFieldsByName(fields)},
session_id, visit_id session_id, visit_id
) as t ) as t
left join ( ${visitEventsJoin}
select session_id, visit_id, toUInt32(count()) as events_count
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id
) as e using (session_id, visit_id)
group by ${parseFieldsByName(fields)} group by ${parseFieldsByName(fields)}
order by 2 desc, 1 desc order by 2 desc, 1 desc
limit 500 limit 500
@@ -1,5 +1,9 @@
import clickhouse from '@/lib/clickhouse'; import clickhouse from '@/lib/clickhouse';
import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; import {
EVENT_TYPE,
FILTER_COLUMNS,
SESSION_COLUMNS,
} from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma'; import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types'; import type { QueryFilters } from '@/lib/types';
@@ -48,11 +52,27 @@ async function relationalQuery(
joinSession: SESSION_COLUMNS.includes(type), joinSession: SESSION_COLUMNS.includes(type),
}, },
); );
const needsBounceEvents = filters.excludeBounce !== true;
const includeCountry = column === 'city' || column === 'region'; const includeCountry = column === 'city' || column === 'region';
if (type === 'language') { if (type === 'language') {
column = `lower(left(${type}, 2))`; column = `lower(left(${type}, 2))`;
} }
const bounceQuery = needsBounceEvents
? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`
: '0 as "bounces",';
const visitEventsJoin = needsBounceEvents
? `left join (
select session_id, visit_id, 1 as "has_custom_event"
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and event_type = ${EVENT_TYPE.customEvent}
group by 1, 2
) as e
on e.session_id = t.session_id
and e.visit_id = t.visit_id`
: '';
return rawQuery( return rawQuery(
` `
@@ -62,7 +82,7 @@ async function relationalQuery(
sum(t.c) as "pageviews", sum(t.c) as "pageviews",
count(distinct t.session_id) as "visitors", count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits", count(distinct t.visit_id) as "visits",
sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces", ${bounceQuery}
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime" sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime"
from ( from (
select select
@@ -72,16 +92,7 @@ async function relationalQuery(
website_event.visit_id, website_event.visit_id,
count(*) as "c", count(*) as "c",
min(website_event.created_at) as "min_time", min(website_event.created_at) as "min_time",
max(website_event.created_at) as "max_time", max(website_event.created_at) as "max_time"
max((
select count(*)
from website_event we2
where we2.website_id = website_event.website_id
and we2.session_id = website_event.session_id
and we2.visit_id = website_event.visit_id
and we2.created_at between {{startDate}} and {{endDate}}
and we2.event_type = ${EVENT_TYPE.customEvent}
)) as "events_count"
from website_event from website_event
${cohortQuery} ${cohortQuery}
${excludeBounceQuery} ${excludeBounceQuery}
@@ -93,6 +104,7 @@ async function relationalQuery(
group by name, website_event.session_id, website_event.visit_id group by name, website_event.session_id, website_event.visit_id
${includeCountry ? ', country' : ''} ${includeCountry ? ', country' : ''}
) as t ) as t
${visitEventsJoin}
where name != '' where name != ''
group by name group by name
${includeCountry ? ', country' : ''} ${includeCountry ? ', country' : ''}
@@ -117,11 +129,25 @@ async function clickhouseQuery(
...filters, ...filters,
websiteId, websiteId,
}); });
const needsBounceEvents = filters.excludeBounce !== true;
const includeCountry = column === 'city' || column === 'region'; const includeCountry = column === 'city' || column === 'region';
if (type === 'language') { if (type === 'language') {
column = `lower(left(${type}, 2))`; column = `lower(left(${type}, 2))`;
} }
const bounceQuery = needsBounceEvents
? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`
: '0 as "bounces",';
const visitEventsJoin = needsBounceEvents
? `left join (
select session_id, visit_id, toUInt8(1) as has_custom_event
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id
) as e using (session_id, visit_id)`
: '';
return rawQuery( return rawQuery(
` `
@@ -131,7 +157,7 @@ async function clickhouseQuery(
sum(t.c) as "pageviews", sum(t.c) as "pageviews",
uniq(t.session_id) as "visitors", uniq(t.session_id) as "visitors",
uniq(t.visit_id) as "visits", uniq(t.visit_id) as "visits",
sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces", ${bounceQuery}
sum(max_time-min_time) as "totaltime" sum(max_time-min_time) as "totaltime"
from ( from (
select select
@@ -153,14 +179,7 @@ async function clickhouseQuery(
group by name, session_id, visit_id group by name, session_id, visit_id
${includeCountry ? ', country' : ''} ${includeCountry ? ', country' : ''}
) as t ) as t
left join ( ${visitEventsJoin}
select session_id, visit_id, toUInt32(count()) as events_count
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id
) as e using (session_id, visit_id)
group by name group by name
${includeCountry ? ', country' : ''} ${includeCountry ? ', country' : ''}
order by visitors desc, visits desc order by visitors desc, visits desc
@@ -0,0 +1,74 @@
import { beforeEach, describe, expect, test, vi } from 'vitest';
import { DATA_TYPE } from '@/lib/constants';
import { relationalQuery } from './saveSessionData';
const { writeRawQueryMock } = vi.hoisted(() => ({
writeRawQueryMock: vi.fn(),
}));
vi.mock('@/lib/prisma', () => ({
default: {
writeRawQuery: writeRawQueryMock,
},
}));
describe('relationalQuery', () => {
beforeEach(() => {
writeRawQueryMock.mockReset();
writeRawQueryMock.mockResolvedValue(undefined);
});
test('writes session data with a Postgres upsert keyed by sessionId and dataKey', async () => {
const createdAt = new Date('2026-07-30T10:00:00.000Z');
await relationalQuery({
websiteId: 'website-1',
sessionId: 'session-1',
sessionData: { plan: 'pro' },
distinctId: 'distinct-1',
createdAt,
});
expect(writeRawQueryMock).toHaveBeenCalledTimes(1);
const [query, params, tag] = writeRawQueryMock.mock.calls[0];
expect(query).toContain('insert into session_data');
expect(query).toContain('on conflict (session_id, data_key)');
expect(query).toContain('do update set');
expect(query).toContain('coalesce({{createdAt}}, now())');
expect(query).toContain('created_at = coalesce({{createdAt}}, session_data.created_at)');
expect(query).toContain('{{id}}');
expect(query).toContain('{{websiteId}}');
expect(query).toContain('{{sessionId}}');
expect(query).toContain('{{dataKey}}');
expect(params).toEqual({
id: expect.any(String),
websiteId: 'website-1',
sessionId: 'session-1',
dataKey: 'plan',
stringValue: 'pro',
numberValue: null,
dateValue: null,
dataType: DATA_TYPE.string,
distinctId: 'distinct-1',
createdAt,
});
expect(tag).toBe('saveSessionData');
});
test('preserves default and existing createdAt behavior when createdAt is omitted', async () => {
await relationalQuery({
websiteId: 'website-1',
sessionId: 'session-1',
sessionData: { plan: 'pro' },
distinctId: 'distinct-1',
});
const [query, params] = writeRawQueryMock.mock.calls[0];
expect(query).toContain('coalesce({{createdAt}}, now())');
expect(query).toContain('created_at = coalesce({{createdAt}}, session_data.created_at)');
expect(params.createdAt).toBeUndefined();
});
});
+60 -15
View File
@@ -30,7 +30,7 @@ export async function relationalQuery({
distinctId, distinctId,
createdAt, createdAt,
}: SaveSessionDataArgs) { }: SaveSessionDataArgs) {
const { client } = prisma; const { writeRawQuery } = prisma;
const jsonKeys = flattenJSON(sessionData); const jsonKeys = flattenJSON(sessionData);
const normalizedDistinctId = truncateString(distinctId, FIELD_LENGTH.distinctId); const normalizedDistinctId = truncateString(distinctId, FIELD_LENGTH.distinctId);
@@ -49,24 +49,69 @@ export async function relationalQuery({
})); }));
for (const data of flattenedData) { for (const data of flattenedData) {
const { sessionId, dataKey, ...props } = data; const {
id,
websiteId,
sessionId,
dataKey,
stringValue,
numberValue,
dateValue,
dataType,
distinctId,
createdAt,
} = data;
const updateResult = await client.sessionData.updateMany({ await writeRawQuery(
where: { `
insert into session_data (
session_data_id,
website_id,
session_id,
data_key,
string_value,
number_value,
date_value,
data_type,
distinct_id,
created_at
)
values (
{{id}},
{{websiteId}},
{{sessionId}},
{{dataKey}},
{{stringValue}},
{{numberValue}},
{{dateValue}},
{{dataType}},
{{distinctId}},
coalesce({{createdAt}}, now())
)
on conflict (session_id, data_key)
do update set
website_id = excluded.website_id,
string_value = excluded.string_value,
number_value = excluded.number_value,
date_value = excluded.date_value,
data_type = excluded.data_type,
distinct_id = excluded.distinct_id,
created_at = coalesce({{createdAt}}, session_data.created_at)
`,
{
id,
websiteId,
sessionId, sessionId,
dataKey, dataKey,
stringValue,
numberValue,
dateValue,
dataType,
distinctId,
createdAt,
}, },
data: { 'saveSessionData',
...props, );
},
});
// If no record was updated, create a new one
if (updateResult.count === 0) {
await client.sessionData.create({
data,
});
}
} }
} }