From 1f2bd7cad9cace4dd32d28ba0d6b699b02f5ab04 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Fri, 1 May 2026 10:00:30 -0700 Subject: [PATCH] fix property filter css. format date columns in pivot table --- .../[websiteId]/event-data-pivot/route.ts | 4 +- .../[websiteId]/session-data-pivot/route.ts | 4 +- .../property-data/PropertyFilterEditForm.tsx | 2 +- src/queries/sql/events/getEventDataPivot.ts | 16 +- .../sql/sessions/getSessionDataPivot.ts | 146 ++++++++++++------ 5 files changed, 121 insertions(+), 51 deletions(-) diff --git a/src/app/api/websites/[websiteId]/event-data-pivot/route.ts b/src/app/api/websites/[websiteId]/event-data-pivot/route.ts index c43aa9f3f..048f95a6f 100644 --- a/src/app/api/websites/[websiteId]/event-data-pivot/route.ts +++ b/src/app/api/websites/[websiteId]/event-data-pivot/route.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { parseEventPropertyFilters } from '@/lib/params'; import { getQueryFilters, parseRequest } from '@/lib/request'; import { json, unauthorized } from '@/lib/response'; -import { filterParams, pagingParams } from '@/lib/schema'; +import { filterParams, pagingParams, timezoneParam, unitParam } from '@/lib/schema'; import { canViewWebsite } from '@/permissions'; import { getEventDataPivot } from '@/queries/sql/events/getEventDataPivot'; @@ -14,6 +14,8 @@ export async function GET( startAt: z.coerce.number().int(), endAt: z.coerce.number().int(), eventName: z.string(), + timezone: timezoneParam.optional(), + unit: unitParam.optional(), ...filterParams, ...pagingParams, }); diff --git a/src/app/api/websites/[websiteId]/session-data-pivot/route.ts b/src/app/api/websites/[websiteId]/session-data-pivot/route.ts index f27b17781..6bf0e2b12 100644 --- a/src/app/api/websites/[websiteId]/session-data-pivot/route.ts +++ b/src/app/api/websites/[websiteId]/session-data-pivot/route.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { parsePropertyFilters } from '@/lib/params'; import { getQueryFilters, parseRequest } from '@/lib/request'; import { json, unauthorized } from '@/lib/response'; -import { filterParams, pagingParams } from '@/lib/schema'; +import { filterParams, pagingParams, timezoneParam, unitParam } from '@/lib/schema'; import { canViewWebsite } from '@/permissions'; import { getSessionDataPivot } from '@/queries/sql/sessions/getSessionDataPivot'; @@ -14,6 +14,8 @@ export async function GET( startAt: z.coerce.number().int(), endAt: z.coerce.number().int(), propertyName: z.string(), + timezone: timezoneParam.optional(), + unit: unitParam.optional(), ...filterParams, ...pagingParams, }); diff --git a/src/components/property-data/PropertyFilterEditForm.tsx b/src/components/property-data/PropertyFilterEditForm.tsx index 6d2db0b5f..6f5e9bce6 100644 --- a/src/components/property-data/PropertyFilterEditForm.tsx +++ b/src/components/property-data/PropertyFilterEditForm.tsx @@ -60,7 +60,7 @@ export function PropertyFilterEditForm({ }; return ( - + diff --git a/src/queries/sql/events/getEventDataPivot.ts b/src/queries/sql/events/getEventDataPivot.ts index 7a494a1da..6f7551aed 100644 --- a/src/queries/sql/events/getEventDataPivot.ts +++ b/src/queries/sql/events/getEventDataPivot.ts @@ -22,7 +22,7 @@ export async function getEventDataPivot( async function relationalQuery(websiteId: string, eventName: string, filters: QueryFilters, eventFilters: EventPropertyFilter[] = []) { const { timezone = 'utc' } = filters; - const { rawQuery, parseFilters, getPropertyFilterQuery } = prisma; + const { rawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = prisma; const { page = 1, pageSize } = filters; const size = +pageSize || DEFAULT_PAGE_SIZE; const offset = +size * (+page - 1); @@ -84,7 +84,7 @@ async function relationalQuery(websiteId: string, eventName: string, filters: Qu case when event_data.data_type = 1 then event_data.string_value end, case when event_data.data_type = 2 then cast(event_data.number_value as varchar) end, case when event_data.data_type = 3 then event_data.string_value end, - case when event_data.data_type = 4 then cast(event_data.date_value as varchar) end, + case when event_data.data_type = 4 then ${getDateStringSQL('event_data.date_value', 'second', timezone)} end, case when event_data.data_type = 5 then event_data.string_value end, '' ) as "value", @@ -120,7 +120,7 @@ async function relationalQuery(websiteId: string, eventName: string, filters: Qu async function clickhouseQuery(websiteId: string, eventName: string, filters: QueryFilters, eventFilters: EventPropertyFilter[] = []) { const { timezone = 'UTC' } = filters; - const { rawQuery, parseFilters, getPropertyFilterQuery } = clickhouse; + const { rawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = clickhouse; const { page = 1, pageSize } = filters; const size = +pageSize || DEFAULT_PAGE_SIZE; const offset = +size * (+page - 1); @@ -160,7 +160,15 @@ async function clickhouseQuery(websiteId: string, eventName: string, filters: Qu event_data_pivot.url_path as urlPath, event_data_pivot.created_at as createdAt, groupArrayMerge(property_keys) as propertyKeys, - groupArrayMerge(property_values) as propertyValues + arrayMap( + (value, dataType) -> if( + dataType = 4, + ${getDateStringSQL('parseDateTimeBestEffortOrNull(value)', 'second', timezone)}, + value + ), + groupArrayMerge(property_values), + groupArrayMerge(property_types) + ) as propertyValues from umami.event_data_pivot any left join ( select * diff --git a/src/queries/sql/sessions/getSessionDataPivot.ts b/src/queries/sql/sessions/getSessionDataPivot.ts index 6ea7b6b5f..5ce922008 100644 --- a/src/queries/sql/sessions/getSessionDataPivot.ts +++ b/src/queries/sql/sessions/getSessionDataPivot.ts @@ -27,7 +27,7 @@ async function relationalQuery( propertyFilters: PropertyFilter[] = [], ): Promise> { const { timezone = 'utc' } = filters; - const { rawQuery, parseFilters, getPropertyFilterQuery } = prisma; + const { rawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = prisma; const { page = 1, pageSize } = filters; const size = +pageSize || DEFAULT_PAGE_SIZE; const offset = +size * (+page - 1); @@ -43,7 +43,7 @@ async function relationalQuery( timezone, ); - const countResult = await rawQuery( + const countResult = (await rawQuery( ` with filtered_sessions as ( select distinct website_event.session_id @@ -67,11 +67,11 @@ async function relationalQuery( from selected_property_sessions `, { ...queryParams, websiteId, propertyName, ...pfParams }, - ); + )) as { num: number }[]; const count = countResult[0].num; - const rows = await rawQuery( + const rows = (await rawQuery( ` with filtered_sessions as ( select distinct website_event.session_id @@ -91,41 +91,68 @@ async function relationalQuery( where session_data.website_id = {{websiteId::uuid}} and session_data.data_key = {{propertyName}} ), + latest_session_properties as ( + select + ranked.session_id, + ranked.distinct_id, + ranked.created_at, + ranked.data_key, + ranked.data_type, + ranked.string_value, + ranked.number_value, + ranked.date_value + from ( + select + session_data.session_id, + session_data.distinct_id, + session_data.created_at, + session_data.data_key, + session_data.data_type, + session_data.string_value, + session_data.number_value, + session_data.date_value, + row_number() over ( + partition by session_data.session_id, session_data.data_key + order by session_data.created_at desc, session_data.session_data_id desc + ) as row_num + from session_data + join selected_property_sessions + on selected_property_sessions.session_id = session_data.session_id + where session_data.website_id = {{websiteId::uuid}} + ) ranked + where ranked.row_num = 1 + ), paged_sessions as ( - select session_data.session_id - from session_data - join selected_property_sessions - on selected_property_sessions.session_id = session_data.session_id - where session_data.website_id = {{websiteId::uuid}} - group by session_data.session_id - order by max(session_data.created_at) desc + select latest_session_properties.session_id + from latest_session_properties + group by latest_session_properties.session_id + order by max(latest_session_properties.created_at) desc limit ${size} offset ${offset} ) select - session_data.session_id as "sessionId", - coalesce(max(session_data.distinct_id), '') as "distinctId", - max(session_data.created_at) as "createdAt", - array_agg(session_data.data_key order by session_data.data_key asc) as "propertyKeys", + latest_session_properties.session_id as "sessionId", + coalesce(max(latest_session_properties.distinct_id), '') as "distinctId", + max(latest_session_properties.created_at) as "createdAt", + array_agg(latest_session_properties.data_key order by latest_session_properties.data_key asc) as "propertyKeys", array_agg( coalesce( - case when session_data.data_type = 1 then session_data.string_value end, - case when session_data.data_type = 2 then cast(session_data.number_value as varchar) end, - case when session_data.data_type = 3 then session_data.string_value end, - case when session_data.data_type = 4 then cast(session_data.date_value as varchar) end, - case when session_data.data_type = 5 then session_data.string_value end, + case when latest_session_properties.data_type = 1 then latest_session_properties.string_value end, + case when latest_session_properties.data_type = 2 then cast(latest_session_properties.number_value as varchar) end, + case when latest_session_properties.data_type = 3 then latest_session_properties.string_value end, + case when latest_session_properties.data_type = 4 then ${getDateStringSQL('latest_session_properties.date_value', 'second', timezone)} end, + case when latest_session_properties.data_type = 5 then latest_session_properties.string_value end, '' ) - order by session_data.data_key asc + order by latest_session_properties.data_key asc ) as "propertyValues" - from session_data - join paged_sessions on paged_sessions.session_id = session_data.session_id - where session_data.website_id = {{websiteId::uuid}} - group by session_data.session_id - order by max(session_data.created_at) desc + from latest_session_properties + join paged_sessions on paged_sessions.session_id = latest_session_properties.session_id + group by latest_session_properties.session_id + order by max(latest_session_properties.created_at) desc `, { ...queryParams, websiteId, propertyName, ...pfParams }, FUNCTION_NAME, - ); + )) as SessionDataPivotRow[]; return { data: rows, count, page: +page, pageSize: size }; } @@ -137,7 +164,7 @@ async function clickhouseQuery( propertyFilters: PropertyFilter[] = [], ): Promise> { const { timezone = 'UTC' } = filters; - const { rawQuery, parseFilters, getPropertyFilterQuery } = clickhouse; + const { rawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = clickhouse; const { page = 1, pageSize } = filters; const size = +pageSize || DEFAULT_PAGE_SIZE; const offset = +size * (+page - 1); @@ -149,7 +176,7 @@ async function clickhouseQuery( timezone, ); - const count = await rawQuery( + const countResult = (await rawQuery( ` with filtered_sessions as ( select distinct website_event.session_id @@ -171,9 +198,10 @@ async function clickhouseQuery( from selected_property_sessions `, { ...queryParams, websiteId, propertyName, ...pfParams }, - ).then((res: any) => res[0].num); + )) as { num: number }[]; + const count = countResult[0].num; - const data = await rawQuery( + const data = (await rawQuery( ` with filtered_sessions as ( select distinct website_event.session_id @@ -190,26 +218,56 @@ async function clickhouseQuery( join filtered_sessions on filtered_sessions.session_id = session_data.session_id where session_data.website_id = {websiteId:UUID} and session_data.data_key = {propertyName:String} + ), + latest_session_properties as ( + select + session_data.session_id as session_id, + argMax(ifNull(session_data.distinct_id, ''), session_data.created_at) as distinct_id, + session_data.data_key as data_key, + argMax(session_data.data_type, session_data.created_at) as data_type, + argMax(session_data.string_value, session_data.created_at) as string_value, + argMax(session_data.number_value, session_data.created_at) as number_value, + argMax(session_data.date_value, session_data.created_at) as date_value, + max(session_data.created_at) as created_at + from session_data final + join selected_property_sessions + on selected_property_sessions.session_id = session_data.session_id + where session_data.website_id = {websiteId:UUID} + group by + session_data.session_id, + session_data.data_key + ), + paged_sessions as ( + select + latest_session_properties.session_id + from latest_session_properties + group by latest_session_properties.session_id + order by max(latest_session_properties.created_at) desc + limit ${size} offset ${offset} ) select - session_data_pivot.session_id as sessionId, - session_data_pivot.distinct_id as distinctId, - maxMerge(session_data_pivot.created_at) as createdAt, - groupArrayMerge(session_data_pivot.property_keys) as propertyKeys, - groupArrayMerge(session_data_pivot.property_values) as propertyValues - from umami.session_data_pivot session_data_pivot - join selected_property_sessions - on selected_property_sessions.session_id = session_data_pivot.session_id - where session_data_pivot.website_id = {websiteId:UUID} + latest_session_properties.session_id as sessionId, + max(latest_session_properties.distinct_id) as distinctId, + max(latest_session_properties.created_at) as createdAt, + groupArray(latest_session_properties.data_key) as propertyKeys, + groupArray( + multiIf( + latest_session_properties.data_type IN (1, 3, 5), ifNull(latest_session_properties.string_value, ''), + latest_session_properties.data_type = 2, toString(ifNull(latest_session_properties.number_value, 0)), + latest_session_properties.data_type = 4, ${getDateStringSQL('latest_session_properties.date_value', 'second', timezone)}, + '' + ) + ) as propertyValues + from latest_session_properties + join paged_sessions + on paged_sessions.session_id = latest_session_properties.session_id group by - session_data_pivot.session_id, - session_data_pivot.distinct_id + latest_session_properties.session_id order by createdAt desc - limit ${size} offset ${offset} `, { ...queryParams, websiteId, propertyName, ...pfParams }, FUNCTION_NAME, - ); + )) as SessionDataPivotRow[]; return { data, count, page: +page, pageSize: size }; }