import clickhouse from '@/lib/clickhouse'; import { EMAIL_DOMAINS, EVENT_TYPE, LLM_DOMAINS, PAID_AD_PARAMS, SEARCH_DOMAINS, SHOPPING_DOMAINS, SOCIAL_DOMAINS, VIDEO_DOMAINS, } from '@/lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import type { QueryFilters } from '@/lib/types'; const FUNCTION_NAME = 'getChannelExpandedMetrics'; export interface ChannelExpandedMetricsParameters { limit?: number | string; offset?: number | string; } export interface ChannelExpandedMetricsData { name: string; pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number; } export async function getChannelExpandedMetrics( ...args: [websiteId: string, filters?: QueryFilters] ): Promise { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } async function relationalQuery( websiteId: string, filters: QueryFilters, ): Promise { const { rawQuery, parseFilters, getTimestampDiffSQL } = prisma; const { queryParams, filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, dateQuery } = parseFilters({ ...filters, websiteId, }); const needsBounceEvents = filters.excludeBounce !== true; const bounceQuery = needsBounceEvents ? `sum(case when visit_stats.c = 1 and coalesce(visit_events.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",` : '0 as "bounces",'; const visitEventsQuery = needsBounceEvents ? `, visit_events as ( select session_id, visit_id, 1 as has_custom_event from website_event where website_id = {{websiteId::uuid}} and created_at between {{startDate}} and {{endDate}} and event_type = ${EVENT_TYPE.customEvent} group by session_id, visit_id)` : ''; const visitEventsJoin = needsBounceEvents ? `left join visit_events on visit_events.session_id = visit_stats.session_id and visit_events.visit_id = visit_stats.visit_id` : ''; return rawQuery( ` WITH prefix AS ( select case when website_event.utm_medium LIKE 'p%' OR website_event.utm_medium LIKE '%ppc%' OR website_event.utm_medium LIKE '%retargeting%' OR website_event.utm_medium LIKE '%paid%' then 'paid' else 'organic' end prefix, website_event.referrer_domain, website_event.url_query, website_event.utm_medium, website_event.utm_source, website_event.session_id, website_event.visit_id, website_event.hostname, website_event.event_id, website_event.created_at from website_event ${cohortQuery} ${excludeBounceQuery} ${joinSessionQuery} where website_event.website_id = {{websiteId::uuid}} and website_event.event_type NOT IN (2, 5) ${dateQuery} ${filterQuery}), channels as ( select case when referrer_domain = '' and url_query = '' then 'direct' when ${toPostgresPositionClause('url_query', PAID_AD_PARAMS)} then 'paidAds' when ${toPostgresPositionClause('utm_medium', ['referral', 'app', 'link'])} then 'referral' when utm_medium ilike '%affiliate%' then 'affiliate' when utm_medium ilike '%sms%' or utm_source ilike '%sms%' then 'sms' when ${toPostgresPositionClause('referrer_domain', LLM_DOMAINS)} then 'llm' when ${toPostgresPositionClause('referrer_domain', SEARCH_DOMAINS)} or utm_medium ilike '%organic%' then concat(prefix, 'Search') when ${toPostgresPositionClause('referrer_domain', SOCIAL_DOMAINS)} then concat(prefix, 'Social') when ${toPostgresPositionClause('referrer_domain', EMAIL_DOMAINS)} or utm_medium ilike '%mail%' then 'email' when ${toPostgresPositionClause('referrer_domain', SHOPPING_DOMAINS)} or utm_medium ilike '%shop%' then concat(prefix, 'Shopping') when ${toPostgresPositionClause('referrer_domain', VIDEO_DOMAINS)} or utm_medium ilike '%video%' then concat(prefix, 'Video') when referrer_domain != regexp_replace(hostname, '^www.', '') and referrer_domain != '' then 'referral' else '' end as "name", session_id, visit_id, event_id, created_at from prefix), visit_channels as ( select session_id, visit_id, coalesce(nullif(name, ''), 'direct') as "name" from ( select name, session_id, visit_id, row_number() over ( partition by session_id, visit_id order by case when name != '' then 0 else 1 end, created_at, event_id ) as row_num from channels ) as ranked_channels where row_num = 1), visit_stats as ( select session_id, visit_id, count(*) as c, min(created_at) as min_time, max(created_at) as max_time from prefix group by session_id, visit_id) ${visitEventsQuery} select visit_channels.name, sum(visit_stats.c) as "pageviews", count(distinct visit_stats.session_id) as "visitors", count(distinct visit_stats.visit_id) as "visits", ${bounceQuery} sum(${getTimestampDiffSQL('visit_stats.min_time', 'visit_stats.max_time')}) as "totaltime" from visit_stats join visit_channels on visit_channels.session_id = visit_stats.session_id and visit_channels.visit_id = visit_stats.visit_id ${visitEventsJoin} group by visit_channels.name order by visitors desc, visits desc `, queryParams, FUNCTION_NAME, ); } async function clickhouseQuery( websiteId: string, filters: QueryFilters, ): Promise { const { rawQuery, parseFilters } = clickhouse; const { queryParams, filterQuery, cohortQuery, excludeBounceQuery } = parseFilters({ ...filters, websiteId, }); const needsBounceEvents = filters.excludeBounce !== true; const bounceQuery = needsBounceEvents ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",` : '0 as "bounces",'; const visitEventsJoin = needsBounceEvents ? `left join ( select session_id, visit_id, toUInt8(1) as has_custom_event from website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = ${EVENT_TYPE.customEvent} group by session_id, visit_id ) as e using (session_id, visit_id)` : ''; return rawQuery( ` select name, sum(t.c) as "pageviews", uniq(t.session_id) as "visitors", uniq(t.visit_id) as "visits", ${bounceQuery} sum(max_time-min_time) as "totaltime" from ( select session_id, visit_id, coalesce(nullIf(argMin(name, tuple(if(name != '', 0, 1), created_at, event_id)), ''), 'direct') as name, count(*) c, min(created_at) min_time, max(created_at) max_time from ( select case when multiSearchAny(lower(utm_medium), ['cp', 'ppc', 'retargeting', 'paid']) != 0 then 'paid' else 'organic' end prefix, case when referrer_domain = '' and url_query = '' then 'direct' when multiSearchAny(lower(url_query), [${toClickHouseStringArray( PAID_AD_PARAMS, )}]) != 0 then 'paidAds' when multiSearchAny(lower(utm_medium), ['referral', 'app','link']) != 0 then 'referral' when position(lower(utm_medium), 'affiliate') > 0 then 'affiliate' when position(lower(utm_medium), 'sms') > 0 or position(lower(utm_source), 'sms') > 0 then 'sms' when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray( LLM_DOMAINS, )}]) != 0 then 'llm' when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray( SEARCH_DOMAINS, )}]) != 0 or position(lower(utm_medium), 'organic') > 0 then concat(prefix, 'Search') when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray( SOCIAL_DOMAINS, )}]) != 0 then concat(prefix, 'Social') when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray( EMAIL_DOMAINS, )}]) != 0 or position(lower(utm_medium), 'mail') > 0 then 'email' when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray( SHOPPING_DOMAINS, )}]) != 0 or position(lower(utm_medium), 'shop') > 0 then concat(prefix, 'Shopping') when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray( VIDEO_DOMAINS, )}]) != 0 or position(lower(utm_medium), 'video') > 0 then concat(prefix, 'Video') when referrer_domain != hostname and referrer_domain != '' then 'referral' else '' end AS "name", session_id, visit_id, event_id, created_at from website_event ${cohortQuery} ${excludeBounceQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type NOT IN (2, 5) ${filterQuery} ) group by session_id, visit_id ) as t ${visitEventsJoin} group by name order by visitors desc, visits desc; `, queryParams, FUNCTION_NAME, ); } function toClickHouseStringArray(arr: string[]): string { // Escape backslashes first, then single quotes, for ClickHouse string literals. return arr.map(p => `'${p.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`).join(', '); } function escapePostgresLikeValue(val: string) { // Escape LIKE wildcards/backslashes so the value is matched literally, then // escape single quotes for the surrounding SQL string literal. return val.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_').replace(/'/g, "''"); } function toPostgresPositionClause(column: string, arr: string[]) { return arr.map(val => `${column} ilike '%${escapePostgresLikeValue(val)}%'`).join(' OR\n '); }