Fix excludeBounce query, simplify bounce definition for SPA websites, query routing for performance, add tests
This commit is contained in:
1 parent
0a80d09d0b
commit
b3aec54bb3
12 files changed
+410
-150
No files matched your search
@@ -83,7 +83,9 @@ describe('getChannelExpandedMetrics postgres branch', () => {
|
||||
expect(query).toContain('sum(visit_stats.c) as "pageviews"');
|
||||
expect(query).toContain('count(distinct visit_stats.session_id) as "visitors"');
|
||||
expect(query).toContain('count(distinct visit_stats.visit_id) as "visits"');
|
||||
expect(query).toContain('sum(case when visit_stats.c = 1 then 1 else 0 end) as "bounces"');
|
||||
expect(query).toContain('sum(case when visit_stats.c = 1 and coalesce(visit_events.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces"');
|
||||
expect(query).toContain('visit_events as');
|
||||
expect(query).toContain('left join visit_events');
|
||||
// getTimestampDiffSQL output is injected into the totaltime column
|
||||
expect(query).toContain('sum(ts_diff(min_time, max_time)) as "totaltime"');
|
||||
expect(getTimestampDiffSQL).toHaveBeenCalledWith(
|
||||
@@ -151,7 +153,8 @@ describe('getChannelExpandedMetrics clickhouse branch', () => {
|
||||
|
||||
expect(query).toContain('uniq(t.session_id) as "visitors"');
|
||||
expect(query).toContain('uniq(t.visit_id) as "visits"');
|
||||
expect(query).toContain('sum(if(t.c = 1, 1, 0)) as "bounces"');
|
||||
expect(query).toContain('sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces"');
|
||||
expect(query).toContain('left join (');
|
||||
expect(query).toContain('sum(max_time-min_time) as "totaltime"');
|
||||
expect(query).toContain('where website_id = {websiteId:UUID}');
|
||||
expect(query).toContain(
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import clickhouse from '@/lib/clickhouse';
|
||||
import {
|
||||
BOUNCE_THRESHOLD,
|
||||
EMAIL_DOMAINS,
|
||||
EVENT_TYPE,
|
||||
LLM_DOMAINS,
|
||||
@@ -49,6 +48,29 @@ async function relationalQuery(
|
||||
...filters,
|
||||
websiteId,
|
||||
});
|
||||
const needsBounceEvents = filters.excludeBounce !== true;
|
||||
const bounceQuery = needsBounceEvents
|
||||
? `sum(case when visit_stats.c = 1 and coalesce(visit_events.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`
|
||||
: '0 as "bounces",';
|
||||
const visitEventsQuery = needsBounceEvents
|
||||
? `,
|
||||
|
||||
visit_events as (
|
||||
select
|
||||
session_id,
|
||||
visit_id,
|
||||
1 as has_custom_event
|
||||
from website_event
|
||||
where website_id = {{websiteId::uuid}}
|
||||
and created_at between {{startDate}} and {{endDate}}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by session_id, visit_id)`
|
||||
: '';
|
||||
const visitEventsJoin = needsBounceEvents
|
||||
? `left join visit_events
|
||||
on visit_events.session_id = visit_stats.session_id
|
||||
and visit_events.visit_id = visit_stats.visit_id`
|
||||
: '';
|
||||
|
||||
return rawQuery(
|
||||
`
|
||||
@@ -122,33 +144,21 @@ async function relationalQuery(
|
||||
min(created_at) as min_time,
|
||||
max(created_at) as max_time
|
||||
from prefix
|
||||
group by session_id, visit_id),
|
||||
|
||||
visit_events as (
|
||||
select
|
||||
session_id,
|
||||
visit_id,
|
||||
count(*) as events_count
|
||||
from website_event
|
||||
where website_id = {{websiteId::uuid}}
|
||||
and created_at between {{startDate}} and {{endDate}}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by session_id, visit_id)
|
||||
${visitEventsQuery}
|
||||
|
||||
select
|
||||
visit_channels.name,
|
||||
sum(visit_stats.c) as "pageviews",
|
||||
count(distinct visit_stats.session_id) as "visitors",
|
||||
count(distinct visit_stats.visit_id) as "visits",
|
||||
sum(case when visit_stats.c = 1 and coalesce(visit_events.events_count, 0) < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces",
|
||||
${bounceQuery}
|
||||
sum(${getTimestampDiffSQL('visit_stats.min_time', 'visit_stats.max_time')}) as "totaltime"
|
||||
from visit_stats
|
||||
join visit_channels
|
||||
on visit_channels.session_id = visit_stats.session_id
|
||||
and visit_channels.visit_id = visit_stats.visit_id
|
||||
left join visit_events
|
||||
on visit_events.session_id = visit_stats.session_id
|
||||
and visit_events.visit_id = visit_stats.visit_id
|
||||
${visitEventsJoin}
|
||||
group by visit_channels.name
|
||||
order by visitors desc, visits desc
|
||||
`,
|
||||
@@ -166,6 +176,20 @@ async function clickhouseQuery(
|
||||
...filters,
|
||||
websiteId,
|
||||
});
|
||||
const needsBounceEvents = filters.excludeBounce !== true;
|
||||
const bounceQuery = needsBounceEvents
|
||||
? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`
|
||||
: '0 as "bounces",';
|
||||
const visitEventsJoin = needsBounceEvents
|
||||
? `left join (
|
||||
select session_id, visit_id, toUInt8(1) as has_custom_event
|
||||
from website_event
|
||||
where website_id = {websiteId:UUID}
|
||||
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by session_id, visit_id
|
||||
) as e using (session_id, visit_id)`
|
||||
: '';
|
||||
|
||||
return rawQuery(
|
||||
`
|
||||
@@ -174,7 +198,7 @@ async function clickhouseQuery(
|
||||
sum(t.c) as "pageviews",
|
||||
uniq(t.session_id) as "visitors",
|
||||
uniq(t.visit_id) as "visits",
|
||||
sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces",
|
||||
${bounceQuery}
|
||||
sum(max_time-min_time) as "totaltime"
|
||||
from (
|
||||
select
|
||||
@@ -229,14 +253,7 @@ async function clickhouseQuery(
|
||||
)
|
||||
group by session_id, visit_id
|
||||
) as t
|
||||
left join (
|
||||
select session_id, visit_id, toUInt32(count()) as events_count
|
||||
from website_event
|
||||
where website_id = {websiteId:UUID}
|
||||
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by session_id, visit_id
|
||||
) as e using (session_id, visit_id)
|
||||
${visitEventsJoin}
|
||||
group by name
|
||||
order by visitors desc, visits desc;
|
||||
`,
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import { afterEach, describe, expect, test, vi } from 'vitest';
|
||||
|
||||
const parseFiltersResult = {
|
||||
queryParams: { websiteId: 'website-1' },
|
||||
filterQuery: 'and website_event.url_path = {{path}}',
|
||||
joinSessionQuery: 'join session on session.session_id = website_event.session_id',
|
||||
cohortQuery: 'join cohort on cohort.session_id = website_event.session_id',
|
||||
excludeBounceQuery: 'join excludeBounce on excludeBounce.visit_id = website_event.visit_id',
|
||||
};
|
||||
|
||||
async function loadModule({
|
||||
mode,
|
||||
}: {
|
||||
mode: 'prisma' | 'clickhouse';
|
||||
}) {
|
||||
vi.resetModules();
|
||||
|
||||
const state = { mode };
|
||||
const prismaRawQuery = vi.fn().mockResolvedValue([{}]);
|
||||
const prismaParseFilters = vi.fn().mockReturnValue(parseFiltersResult);
|
||||
const getTimestampDiffSQL = vi.fn().mockReturnValue('ts_diff(t.min_time, t.max_time)');
|
||||
const clickhouseRawQuery = vi.fn().mockResolvedValue([{}]);
|
||||
const clickhouseParseFilters = vi.fn().mockReturnValue(parseFiltersResult);
|
||||
|
||||
vi.doMock('@/lib/db', () => ({
|
||||
CLICKHOUSE: 'clickhouse',
|
||||
PRISMA: 'prisma',
|
||||
runQuery: vi.fn((queries: Record<string, () => unknown>) => queries[state.mode]()),
|
||||
}));
|
||||
|
||||
vi.doMock('@/lib/prisma', () => ({
|
||||
default: {
|
||||
rawQuery: prismaRawQuery,
|
||||
parseFilters: prismaParseFilters,
|
||||
getTimestampDiffSQL,
|
||||
},
|
||||
}));
|
||||
|
||||
vi.doMock('@/lib/clickhouse', () => ({
|
||||
default: {
|
||||
rawQuery: clickhouseRawQuery,
|
||||
parseFilters: clickhouseParseFilters,
|
||||
},
|
||||
}));
|
||||
|
||||
const mod = await import('./getWebsiteStats');
|
||||
|
||||
return {
|
||||
getWebsiteStats: mod.getWebsiteStats,
|
||||
prismaRawQuery,
|
||||
clickhouseRawQuery,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('getWebsiteStats default bounce path', () => {
|
||||
test('postgres uses a single grouped pass for the default path', async () => {
|
||||
const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' });
|
||||
|
||||
await getWebsiteStats('website-1', {});
|
||||
|
||||
const [query] = prismaRawQuery.mock.calls[0];
|
||||
|
||||
expect(query).toContain('t.c = 1 and t.has_custom_event = 0');
|
||||
expect(query).toContain('max(case when website_event.event_type = 2 then 1 else 0 end) as "has_custom_event"');
|
||||
expect(query).not.toContain('left join (');
|
||||
});
|
||||
|
||||
test('clickhouse uses the hourly aggregate view without the extra join', async () => {
|
||||
const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });
|
||||
|
||||
await getWebsiteStats('website-1', {});
|
||||
|
||||
const [query] = clickhouseRawQuery.mock.calls[0];
|
||||
|
||||
expect(query).toContain('from website_event_stats_hourly "website_event"');
|
||||
expect(query).toContain('sumIf(1, t.c = 1 and t.has_custom_event = 0) as "bounces"');
|
||||
expect(query).toContain('has_custom_event');
|
||||
expect(query).not.toContain('left join (');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getWebsiteStats filtered bounce path', () => {
|
||||
test('postgres switches to the visit-events join when event filters are present', async () => {
|
||||
const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' });
|
||||
|
||||
await getWebsiteStats('website-1', { path: '/pricing' } as any);
|
||||
|
||||
const [query] = prismaRawQuery.mock.calls[0];
|
||||
|
||||
expect(query).toContain('left join (');
|
||||
expect(query).toContain('coalesce(e.has_custom_event, 0) = 0');
|
||||
expect(query).toContain('event_type = 2');
|
||||
});
|
||||
|
||||
test('clickhouse switches to the raw-event join when event filters are present', async () => {
|
||||
const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });
|
||||
|
||||
await getWebsiteStats('website-1', { path: '/pricing' } as any);
|
||||
|
||||
const [query] = clickhouseRawQuery.mock.calls[0];
|
||||
|
||||
expect(query).toContain('from website_event');
|
||||
expect(query).toContain('left join (');
|
||||
expect(query).toContain('ifNull(e.has_custom_event, 0) = 0');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getWebsiteStats excludeBounce path', () => {
|
||||
test('clickhouse skips the extra join when excludeBounce is enabled', async () => {
|
||||
const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });
|
||||
|
||||
await getWebsiteStats('website-1', { excludeBounce: true });
|
||||
|
||||
const [query] = clickhouseRawQuery.mock.calls[0];
|
||||
|
||||
expect(query).toContain('0 as "bounces"');
|
||||
expect(query).not.toContain('left join (');
|
||||
});
|
||||
|
||||
test('postgres skips the extra join when excludeBounce is enabled on the filtered path', async () => {
|
||||
const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' });
|
||||
|
||||
await getWebsiteStats('website-1', { excludeBounce: true, path: '/pricing' } as any);
|
||||
|
||||
const [query] = prismaRawQuery.mock.calls[0];
|
||||
|
||||
expect(query).toContain('0 as "bounces"');
|
||||
expect(query).not.toContain('left join (');
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import clickhouse from '@/lib/clickhouse';
|
||||
import { BOUNCE_THRESHOLD, EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants';
|
||||
import { EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants';
|
||||
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
|
||||
import prisma from '@/lib/prisma';
|
||||
import type { QueryFilters } from '@/lib/types';
|
||||
@@ -35,9 +35,60 @@ async function relationalQuery(
|
||||
});
|
||||
|
||||
const { excludeBounce } = filters;
|
||||
const hasEventFilters =
|
||||
EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) ||
|
||||
!!filters.eventPropertyFilters?.length;
|
||||
|
||||
if (!hasEventFilters) {
|
||||
return rawQuery(
|
||||
`
|
||||
select
|
||||
cast(coalesce(sum(t.c), 0) as bigint) as "pageviews",
|
||||
count(distinct t.session_id) as "visitors",
|
||||
count(distinct t.visit_id) as "visits",
|
||||
${excludeBounce ? '0' : 'coalesce(sum(case when t.c = 1 and t.has_custom_event = 0 then 1 else 0 end), 0)'} as "bounces",
|
||||
cast(coalesce(sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}), 0) as bigint) as "totaltime"
|
||||
from (
|
||||
select
|
||||
website_event.session_id,
|
||||
website_event.visit_id,
|
||||
sum(case when website_event.event_type NOT IN (2, 5) then 1 else 0 end) as "c",
|
||||
min(case when website_event.event_type NOT IN (2, 5) then website_event.created_at end) as "min_time",
|
||||
max(case when website_event.event_type NOT IN (2, 5) then website_event.created_at end) as "max_time",
|
||||
max(case when website_event.event_type = ${EVENT_TYPE.customEvent} then 1 else 0 end) as "has_custom_event"
|
||||
from website_event
|
||||
${cohortQuery}
|
||||
${excludeBounceQuery}
|
||||
${joinSessionQuery}
|
||||
where website_event.website_id = {{websiteId::uuid}}
|
||||
and website_event.created_at between {{startDate}} and {{endDate}}
|
||||
and website_event.event_type != ${EVENT_TYPE.performance}
|
||||
${filterQuery}
|
||||
group by 1, 2
|
||||
having sum(case when website_event.event_type NOT IN (2, 5) then 1 else 0 end) > 0
|
||||
) as t
|
||||
`,
|
||||
queryParams,
|
||||
FUNCTION_NAME,
|
||||
).then(result => result?.[0]);
|
||||
}
|
||||
|
||||
const bounceQuery = excludeBounce
|
||||
? '0'
|
||||
: `coalesce(sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end), 0)`;
|
||||
: 'coalesce(sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end), 0)';
|
||||
const visitEventsJoin = excludeBounce
|
||||
? ''
|
||||
: `
|
||||
left join (
|
||||
select session_id, visit_id, 1 as "has_custom_event"
|
||||
from website_event
|
||||
where website_id = {{websiteId::uuid}}
|
||||
and created_at between {{startDate}} and {{endDate}}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by 1, 2
|
||||
) as e
|
||||
on e.session_id = t.session_id
|
||||
and e.visit_id = t.visit_id`;
|
||||
|
||||
return rawQuery(
|
||||
`
|
||||
@@ -53,16 +104,7 @@ async function relationalQuery(
|
||||
website_event.visit_id,
|
||||
count(*) as "c",
|
||||
min(website_event.created_at) as "min_time",
|
||||
max(website_event.created_at) as "max_time",
|
||||
max((
|
||||
select count(*)
|
||||
from website_event we2
|
||||
where we2.website_id = website_event.website_id
|
||||
and we2.session_id = website_event.session_id
|
||||
and we2.visit_id = website_event.visit_id
|
||||
and we2.created_at between {{startDate}} and {{endDate}}
|
||||
and we2.event_type = ${EVENT_TYPE.customEvent}
|
||||
)) as "events_count"
|
||||
max(website_event.created_at) as "max_time"
|
||||
from website_event
|
||||
${cohortQuery}
|
||||
${excludeBounceQuery}
|
||||
@@ -72,8 +114,8 @@ async function relationalQuery(
|
||||
and website_event.event_type NOT IN (2, 5)
|
||||
${filterQuery}
|
||||
group by 1, 2
|
||||
|
||||
) as t
|
||||
${visitEventsJoin}
|
||||
`,
|
||||
queryParams,
|
||||
FUNCTION_NAME,
|
||||
@@ -92,17 +134,20 @@ async function clickhouseQuery(
|
||||
|
||||
let sql = '';
|
||||
const { excludeBounce } = filters;
|
||||
const hasEventFilters =
|
||||
EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) ||
|
||||
!!filters.eventPropertyFilters?.length;
|
||||
const bounceQuery = excludeBounce
|
||||
? '0'
|
||||
: `sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD})`;
|
||||
: 'sumIf(1, t.c = 1 and t.has_custom_event = 0)';
|
||||
|
||||
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
|
||||
if (hasEventFilters) {
|
||||
sql = `
|
||||
select
|
||||
sum(t.c) as "pageviews",
|
||||
uniq(t.session_id) as "visitors",
|
||||
uniq(t.visit_id) as "visits",
|
||||
${bounceQuery} as "bounces",
|
||||
${excludeBounce ? '0' : 'sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0)'} as "bounces",
|
||||
sum(max_time-min_time) as "totaltime"
|
||||
from (
|
||||
select
|
||||
@@ -120,14 +165,14 @@ async function clickhouseQuery(
|
||||
${filterQuery}
|
||||
group by session_id, visit_id
|
||||
) as t
|
||||
left join (
|
||||
select session_id, visit_id, toUInt32(count()) as events_count
|
||||
${excludeBounce ? '' : `left join (
|
||||
select session_id, visit_id, toUInt8(1) as has_custom_event
|
||||
from website_event
|
||||
where website_id = {websiteId:UUID}
|
||||
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by session_id, visit_id
|
||||
) as e using (session_id, visit_id);
|
||||
) as e using (session_id, visit_id)`};
|
||||
`;
|
||||
} else {
|
||||
sql = `
|
||||
@@ -137,28 +182,24 @@ async function clickhouseQuery(
|
||||
uniq(visit_id) as "visits",
|
||||
${bounceQuery} as "bounces",
|
||||
sum(max_time-min_time) as "totaltime"
|
||||
from (select
|
||||
session_id,
|
||||
visit_id,
|
||||
sum(views) c,
|
||||
min(min_time) min_time,
|
||||
max(max_time) max_time
|
||||
from (
|
||||
select
|
||||
session_id,
|
||||
visit_id,
|
||||
sum(views) c,
|
||||
minIf(min_time, event_type NOT IN (2, 5)) min_time,
|
||||
maxIf(max_time, event_type NOT IN (2, 5)) max_time,
|
||||
max(if(event_type = ${EVENT_TYPE.customEvent} and length(event_name) > 0, 1, 0)) has_custom_event
|
||||
from website_event_stats_hourly "website_event"
|
||||
${cohortQuery}
|
||||
${excludeBounceQuery}
|
||||
where website_id = {websiteId:UUID}
|
||||
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
and event_type NOT IN (2, 5)
|
||||
${filterQuery}
|
||||
group by session_id, visit_id
|
||||
) as t
|
||||
left join (
|
||||
select session_id, visit_id, toUInt32(sum(length(event_name))) as events_count
|
||||
from website_event_stats_hourly
|
||||
where website_id = {websiteId:UUID}
|
||||
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
and event_type != ${EVENT_TYPE.performance}
|
||||
${filterQuery}
|
||||
group by session_id, visit_id
|
||||
) as e using (session_id, visit_id);
|
||||
having c > 0
|
||||
) as t
|
||||
`;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import clickhouse from '@/lib/clickhouse';
|
||||
import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, GROUPED_DOMAINS, SESSION_COLUMNS } from '@/lib/constants';
|
||||
import {
|
||||
EVENT_TYPE,
|
||||
FILTER_COLUMNS,
|
||||
GROUPED_DOMAINS,
|
||||
SESSION_COLUMNS,
|
||||
} from '@/lib/constants';
|
||||
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
|
||||
import prisma from '@/lib/prisma';
|
||||
import type { QueryFilters } from '@/lib/types';
|
||||
@@ -46,6 +51,7 @@ async function relationalQuery(
|
||||
},
|
||||
{ joinSession: SESSION_COLUMNS.includes(type) },
|
||||
);
|
||||
const needsBounceEvents = filters.excludeBounce !== true;
|
||||
const fullPathSearchQuery =
|
||||
type === 'fullPath' && filters.search
|
||||
? `and (case when website_event.url_query != '' then website_event.url_path || '?' || website_event.url_query else website_event.url_path end) ilike {{fullPathSearch}}`
|
||||
@@ -88,6 +94,21 @@ async function relationalQuery(
|
||||
type === 'fullPath'
|
||||
? `case when website_event.url_query != '' then website_event.url_path || '?' || website_event.url_query else website_event.url_path end`
|
||||
: column;
|
||||
const bounceQuery = needsBounceEvents
|
||||
? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`
|
||||
: '0 as "bounces",';
|
||||
const visitEventsJoin = needsBounceEvents
|
||||
? `left join (
|
||||
select session_id, visit_id, 1 as "has_custom_event"
|
||||
from website_event
|
||||
where website_id = {{websiteId::uuid}}
|
||||
and created_at between {{startDate}} and {{endDate}}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by 1, 2
|
||||
) as e
|
||||
on e.session_id = t.session_id
|
||||
and e.visit_id = t.visit_id`
|
||||
: '';
|
||||
|
||||
return rawQuery(
|
||||
`
|
||||
@@ -96,7 +117,7 @@ async function relationalQuery(
|
||||
sum(t.c) as "pageviews",
|
||||
count(distinct t.session_id) as "visitors",
|
||||
count(distinct t.visit_id) as "visits",
|
||||
sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces",
|
||||
${bounceQuery}
|
||||
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime"
|
||||
from (
|
||||
select
|
||||
@@ -105,16 +126,7 @@ async function relationalQuery(
|
||||
website_event.visit_id,
|
||||
count(*) as "c",
|
||||
min(website_event.created_at) as "min_time",
|
||||
max(website_event.created_at) as "max_time",
|
||||
max((
|
||||
select count(*)
|
||||
from website_event we2
|
||||
where we2.website_id = website_event.website_id
|
||||
and we2.session_id = website_event.session_id
|
||||
and we2.visit_id = website_event.visit_id
|
||||
and we2.created_at between {{startDate}} and {{endDate}}
|
||||
and we2.event_type = ${EVENT_TYPE.customEvent}
|
||||
)) as "events_count"
|
||||
max(website_event.created_at) as "max_time"
|
||||
from website_event
|
||||
${cohortQuery}
|
||||
${excludeBounceQuery}
|
||||
@@ -128,6 +140,7 @@ async function relationalQuery(
|
||||
${filterQuery}
|
||||
group by ${groupByColumn}, website_event.session_id, website_event.visit_id
|
||||
) as t
|
||||
${visitEventsJoin}
|
||||
where name != ''
|
||||
group by name
|
||||
order by visitors desc, visits desc
|
||||
@@ -154,6 +167,7 @@ async function clickhouseQuery(
|
||||
...filters,
|
||||
websiteId,
|
||||
});
|
||||
const needsBounceEvents = filters.excludeBounce !== true;
|
||||
const fullPathSearchQuery =
|
||||
type === 'fullPath' && filters.search
|
||||
? `and positionCaseInsensitive(if(url_query != '', concat(url_path, '?', url_query), url_path), {fullPathSearch:String}) > 0`
|
||||
@@ -189,6 +203,19 @@ async function clickhouseQuery(
|
||||
} else if (type === 'fullPath') {
|
||||
selectColumn = `if(url_query != '', concat(url_path, '?', url_query), url_path)`;
|
||||
}
|
||||
const bounceQuery = needsBounceEvents
|
||||
? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`
|
||||
: '0 as "bounces",';
|
||||
const visitEventsJoin = needsBounceEvents
|
||||
? `left join (
|
||||
select session_id, visit_id, toUInt8(1) as has_custom_event
|
||||
from website_event
|
||||
where website_id = {websiteId:UUID}
|
||||
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by session_id, visit_id
|
||||
) as e using (session_id, visit_id)`
|
||||
: '';
|
||||
|
||||
return rawQuery(
|
||||
`
|
||||
@@ -197,7 +224,7 @@ async function clickhouseQuery(
|
||||
sum(t.c) as "pageviews",
|
||||
uniq(t.session_id) as "visitors",
|
||||
uniq(t.visit_id) as "visits",
|
||||
sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces",
|
||||
${bounceQuery}
|
||||
sum(max_time-min_time) as "totaltime"
|
||||
from (
|
||||
select
|
||||
@@ -220,14 +247,7 @@ async function clickhouseQuery(
|
||||
${filterQuery}
|
||||
group by name, session_id, visit_id
|
||||
) as t
|
||||
left join (
|
||||
select session_id, visit_id, toUInt32(count()) as events_count
|
||||
from website_event
|
||||
where website_id = {websiteId:UUID}
|
||||
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by session_id, visit_id
|
||||
) as e using (session_id, visit_id)
|
||||
${visitEventsJoin}
|
||||
group by name
|
||||
order by visitors desc, visits desc
|
||||
limit ${limit}
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import clickhouse from '@/lib/clickhouse';
|
||||
import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
|
||||
import {
|
||||
EVENT_TYPE,
|
||||
FILTER_COLUMNS,
|
||||
SESSION_COLUMNS,
|
||||
} from '@/lib/constants';
|
||||
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
|
||||
import prisma from '@/lib/prisma';
|
||||
import type { QueryFilters } from '@/lib/types';
|
||||
@@ -31,7 +35,7 @@ async function relationalQuery(
|
||||
): Promise<BreakdownData[]> {
|
||||
const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma;
|
||||
const { startDate, endDate, fields } = parameters;
|
||||
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(
|
||||
const { filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters(
|
||||
{
|
||||
...filters,
|
||||
websiteId,
|
||||
@@ -43,6 +47,22 @@ async function relationalQuery(
|
||||
joinSession: !!fields.find((name: string) => SESSION_COLUMNS.includes(name)),
|
||||
},
|
||||
);
|
||||
const needsBounceEvents = filters.excludeBounce !== true;
|
||||
const bounceQuery = needsBounceEvents
|
||||
? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`
|
||||
: '0 as "bounces",';
|
||||
const visitEventsJoin = needsBounceEvents
|
||||
? `left join (
|
||||
select session_id, visit_id, 1 as "has_custom_event"
|
||||
from website_event
|
||||
where website_id = {{websiteId::uuid}}
|
||||
and created_at between {{startDate}} and {{endDate}}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by 1, 2
|
||||
) as e
|
||||
on e.session_id = t.session_id
|
||||
and e.visit_id = t.visit_id`
|
||||
: '';
|
||||
|
||||
return rawQuery(
|
||||
`
|
||||
@@ -50,7 +70,7 @@ async function relationalQuery(
|
||||
sum(t.c) as "views",
|
||||
count(distinct t.session_id) as "visitors",
|
||||
count(distinct t.visit_id) as "visits",
|
||||
sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces",
|
||||
${bounceQuery}
|
||||
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime",
|
||||
${parseFieldsByName(fields)}
|
||||
from (
|
||||
@@ -60,18 +80,10 @@ async function relationalQuery(
|
||||
website_event.visit_id,
|
||||
count(*) as "c",
|
||||
min(website_event.created_at) as "min_time",
|
||||
max(website_event.created_at) as "max_time",
|
||||
max((
|
||||
select count(*)
|
||||
from website_event we2
|
||||
where we2.website_id = website_event.website_id
|
||||
and we2.session_id = website_event.session_id
|
||||
and we2.visit_id = website_event.visit_id
|
||||
and we2.created_at between {{startDate}} and {{endDate}}
|
||||
and we2.event_type = ${EVENT_TYPE.customEvent}
|
||||
)) as "events_count"
|
||||
max(website_event.created_at) as "max_time"
|
||||
from website_event
|
||||
${cohortQuery}
|
||||
${excludeBounceQuery}
|
||||
${joinSessionQuery}
|
||||
where website_event.website_id = {{websiteId::uuid}}
|
||||
and website_event.created_at between {{startDate}} and {{endDate}}
|
||||
@@ -79,6 +91,7 @@ async function relationalQuery(
|
||||
group by ${parseFieldsByName(fields)},
|
||||
website_event.session_id, website_event.visit_id
|
||||
) as t
|
||||
${visitEventsJoin}
|
||||
group by ${parseFieldsByName(fields)}
|
||||
order by 2 desc, 1 desc
|
||||
limit 500
|
||||
@@ -94,13 +107,27 @@ async function clickhouseQuery(
|
||||
): Promise<BreakdownData[]> {
|
||||
const { parseFilters, rawQuery } = clickhouse;
|
||||
const { startDate, endDate, fields } = parameters;
|
||||
const { filterQuery, cohortQuery, queryParams } = parseFilters({
|
||||
const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({
|
||||
...filters,
|
||||
websiteId,
|
||||
startDate,
|
||||
endDate,
|
||||
eventType: EVENT_TYPE.pageView,
|
||||
});
|
||||
const needsBounceEvents = filters.excludeBounce !== true;
|
||||
const bounceQuery = needsBounceEvents
|
||||
? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`
|
||||
: '0 as "bounces",';
|
||||
const visitEventsJoin = needsBounceEvents
|
||||
? `left join (
|
||||
select session_id, visit_id, toUInt8(1) as has_custom_event
|
||||
from website_event
|
||||
where website_id = {websiteId:UUID}
|
||||
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by session_id, visit_id
|
||||
) as e using (session_id, visit_id)`
|
||||
: '';
|
||||
|
||||
return rawQuery(
|
||||
`
|
||||
@@ -108,7 +135,7 @@ async function clickhouseQuery(
|
||||
sum(t.c) as "views",
|
||||
count(distinct t.session_id) as "visitors",
|
||||
count(distinct t.visit_id) as "visits",
|
||||
sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces",
|
||||
${bounceQuery}
|
||||
sum(max_time-min_time) as "totaltime",
|
||||
${parseFieldsByName(fields)}
|
||||
from (
|
||||
@@ -121,20 +148,14 @@ async function clickhouseQuery(
|
||||
max(created_at) max_time
|
||||
from website_event
|
||||
${cohortQuery}
|
||||
${excludeBounceQuery}
|
||||
where website_id = {websiteId:UUID}
|
||||
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
${filterQuery}
|
||||
group by ${parseFieldsByName(fields)},
|
||||
session_id, visit_id
|
||||
) as t
|
||||
left join (
|
||||
select session_id, visit_id, toUInt32(count()) as events_count
|
||||
from website_event
|
||||
where website_id = {websiteId:UUID}
|
||||
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by session_id, visit_id
|
||||
) as e using (session_id, visit_id)
|
||||
${visitEventsJoin}
|
||||
group by ${parseFieldsByName(fields)}
|
||||
order by 2 desc, 1 desc
|
||||
limit 500
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import clickhouse from '@/lib/clickhouse';
|
||||
import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
|
||||
import {
|
||||
EVENT_TYPE,
|
||||
FILTER_COLUMNS,
|
||||
SESSION_COLUMNS,
|
||||
} from '@/lib/constants';
|
||||
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
|
||||
import prisma from '@/lib/prisma';
|
||||
import type { QueryFilters } from '@/lib/types';
|
||||
@@ -48,11 +52,27 @@ async function relationalQuery(
|
||||
joinSession: SESSION_COLUMNS.includes(type),
|
||||
},
|
||||
);
|
||||
const needsBounceEvents = filters.excludeBounce !== true;
|
||||
const includeCountry = column === 'city' || column === 'region';
|
||||
|
||||
if (type === 'language') {
|
||||
column = `lower(left(${type}, 2))`;
|
||||
}
|
||||
const bounceQuery = needsBounceEvents
|
||||
? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`
|
||||
: '0 as "bounces",';
|
||||
const visitEventsJoin = needsBounceEvents
|
||||
? `left join (
|
||||
select session_id, visit_id, 1 as "has_custom_event"
|
||||
from website_event
|
||||
where website_id = {{websiteId::uuid}}
|
||||
and created_at between {{startDate}} and {{endDate}}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by 1, 2
|
||||
) as e
|
||||
on e.session_id = t.session_id
|
||||
and e.visit_id = t.visit_id`
|
||||
: '';
|
||||
|
||||
return rawQuery(
|
||||
`
|
||||
@@ -62,7 +82,7 @@ async function relationalQuery(
|
||||
sum(t.c) as "pageviews",
|
||||
count(distinct t.session_id) as "visitors",
|
||||
count(distinct t.visit_id) as "visits",
|
||||
sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces",
|
||||
${bounceQuery}
|
||||
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime"
|
||||
from (
|
||||
select
|
||||
@@ -72,16 +92,7 @@ async function relationalQuery(
|
||||
website_event.visit_id,
|
||||
count(*) as "c",
|
||||
min(website_event.created_at) as "min_time",
|
||||
max(website_event.created_at) as "max_time",
|
||||
max((
|
||||
select count(*)
|
||||
from website_event we2
|
||||
where we2.website_id = website_event.website_id
|
||||
and we2.session_id = website_event.session_id
|
||||
and we2.visit_id = website_event.visit_id
|
||||
and we2.created_at between {{startDate}} and {{endDate}}
|
||||
and we2.event_type = ${EVENT_TYPE.customEvent}
|
||||
)) as "events_count"
|
||||
max(website_event.created_at) as "max_time"
|
||||
from website_event
|
||||
${cohortQuery}
|
||||
${excludeBounceQuery}
|
||||
@@ -93,6 +104,7 @@ async function relationalQuery(
|
||||
group by name, website_event.session_id, website_event.visit_id
|
||||
${includeCountry ? ', country' : ''}
|
||||
) as t
|
||||
${visitEventsJoin}
|
||||
where name != ''
|
||||
group by name
|
||||
${includeCountry ? ', country' : ''}
|
||||
@@ -117,11 +129,25 @@ async function clickhouseQuery(
|
||||
...filters,
|
||||
websiteId,
|
||||
});
|
||||
const needsBounceEvents = filters.excludeBounce !== true;
|
||||
const includeCountry = column === 'city' || column === 'region';
|
||||
|
||||
if (type === 'language') {
|
||||
column = `lower(left(${type}, 2))`;
|
||||
}
|
||||
const bounceQuery = needsBounceEvents
|
||||
? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`
|
||||
: '0 as "bounces",';
|
||||
const visitEventsJoin = needsBounceEvents
|
||||
? `left join (
|
||||
select session_id, visit_id, toUInt8(1) as has_custom_event
|
||||
from website_event
|
||||
where website_id = {websiteId:UUID}
|
||||
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by session_id, visit_id
|
||||
) as e using (session_id, visit_id)`
|
||||
: '';
|
||||
|
||||
return rawQuery(
|
||||
`
|
||||
@@ -131,7 +157,7 @@ async function clickhouseQuery(
|
||||
sum(t.c) as "pageviews",
|
||||
uniq(t.session_id) as "visitors",
|
||||
uniq(t.visit_id) as "visits",
|
||||
sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces",
|
||||
${bounceQuery}
|
||||
sum(max_time-min_time) as "totaltime"
|
||||
from (
|
||||
select
|
||||
@@ -153,14 +179,7 @@ async function clickhouseQuery(
|
||||
group by name, session_id, visit_id
|
||||
${includeCountry ? ', country' : ''}
|
||||
) as t
|
||||
left join (
|
||||
select session_id, visit_id, toUInt32(count()) as events_count
|
||||
from website_event
|
||||
where website_id = {websiteId:UUID}
|
||||
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
and event_type = ${EVENT_TYPE.customEvent}
|
||||
group by session_id, visit_id
|
||||
) as e using (session_id, visit_id)
|
||||
${visitEventsJoin}
|
||||
group by name
|
||||
${includeCountry ? ', country' : ''}
|
||||
order by visitors desc, visits desc
|
||||
|
||||
Reference in new issue
Block a user