Event data properties refactor and design

src/queries/sql/events/getEventDataFields.ts

@@ -13,7 +13,7 @@ export async function getEventDataFields(...args: [websiteId: string, filters: Q
 }
 
 async function relationalQuery(websiteId: string, filters: QueryFilters) {
-  const { rawQuery, parseFilters, getDateSQL } = prisma;
+  const { rawQuery, parseFilters } = prisma;
   const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({
     ...filters,
     websiteId,
@@ -24,11 +24,6 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
     select
       data_key as "propertyName",
       data_type as "dataType",
-      case 
-        when data_type = 2 then replace(string_value, '.0000', '') 
-        when data_type = 4 then ${getDateSQL('date_value', 'hour')} 
-        else string_value
-      end as "value",
       count(*) as "total"
     from event_data
     join website_event on website_event.event_id = event_data.website_event_id
@@ -39,9 +34,8 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
     where event_data.website_id = {{websiteId::uuid}}
       and event_data.created_at between {{startDate}} and {{endDate}}
     ${filterQuery}
-    group by data_key, data_type, value
-    order by 2 desc
-    limit 100
+    group by data_key, data_type
+    order by "total" desc, "propertyName" asc
     `,
     queryParams,
     FUNCTION_NAME,
@@ -51,7 +45,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
 async function clickhouseQuery(
   websiteId: string,
   filters: QueryFilters,
-): Promise<{ propertyName: string; dataType: number; propertyValue: string; total: number }[]> {
+): Promise<{ propertyName: string; dataType: number; total: number }[]> {
   const { rawQuery, parseFilters } = clickhouse;
   const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId });
 
@@ -60,13 +54,10 @@ async function clickhouseQuery(
     select
       data_key as propertyName,
       data_type as dataType,
-      multiIf(data_type = 2, replaceAll(string_value, '.0000', ''),
-              data_type = 4, toString(date_trunc('hour', date_value)),
-              string_value) as "value",
       count(*) as "total"
     from event_data
     any left join (
-          select * 
+          select *
           from website_event
           where website_id = {websiteId:UUID}
             and created_at between {startDate:DateTime64} and {endDate:DateTime64}
@@ -78,9 +69,8 @@ async function clickhouseQuery(
     where event_data.website_id = {websiteId:UUID}
       and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}
     ${filterQuery}
-    group by data_key, data_type, value
-    order by 2 desc
-    limit 100
+    group by data_key, data_type
+    order by total desc, propertyName asc
     `,
     queryParams,
     FUNCTION_NAME,

src/queries/sql/events/getEventDataNumericSeries.ts

@@ -0,0 +1,104 @@
+import clickhouse from '@/lib/clickhouse';
+import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
+import prisma from '@/lib/prisma';
+import type { QueryFilters } from '@/lib/types';
+
+const FUNCTION_NAME = 'getEventDataNumericSeries';
+
+export async function getEventDataNumericSeries(
+  ...args: [
+    websiteId: string,
+    eventName: string,
+    propertyName: string,
+    metric: 'sum' | 'avg',
+    filters: QueryFilters,
+  ]
+) {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(...args),
+    [CLICKHOUSE]: () => clickhouseQuery(...args),
+  });
+}
+
+async function relationalQuery(
+  websiteId: string,
+  eventName: string,
+  propertyName: string,
+  metric: 'sum' | 'avg',
+  filters: QueryFilters,
+) {
+  const { timezone = 'utc', unit = 'day' } = filters;
+  const { rawQuery, getDateSQL, parseFilters } = prisma;
+  const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({
+    ...filters,
+    websiteId,
+  });
+  const aggFn = metric === 'avg' ? 'avg' : 'sum';
+
+  return rawQuery(
+    `
+    select
+      ${getDateSQL('event_data.created_at', unit, timezone)} t,
+      ${aggFn}(cast(event_data.number_value as decimal)) y
+    from event_data
+    join website_event on website_event.event_id = event_data.website_event_id
+      and website_event.website_id = {{websiteId::uuid}}
+      and website_event.created_at between {{startDate}} and {{endDate}}
+      and website_event.event_type = 2
+      and website_event.event_name = {{eventName}}
+    ${cohortQuery}
+    ${joinSessionQuery}
+    where event_data.website_id = {{websiteId::uuid}}
+      and event_data.created_at between {{startDate}} and {{endDate}}
+      and event_data.data_key = {{propertyName}}
+      and event_data.data_type = 2
+      ${filterQuery}
+    group by 1
+    order by 1
+    `,
+    { ...queryParams, eventName, propertyName },
+    FUNCTION_NAME,
+  );
+}
+
+async function clickhouseQuery(
+  websiteId: string,
+  eventName: string,
+  propertyName: string,
+  metric: 'sum' | 'avg',
+  filters: QueryFilters,
+): Promise<{ t: string; y: number }[]> {
+  const { timezone = 'UTC', unit = 'day' } = filters;
+  const { rawQuery, getDateSQL, parseFilters } = clickhouse;
+  const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId });
+  const aggFn = metric === 'avg' ? 'avg' : 'sum';
+
+  return rawQuery(
+    `
+    select
+      ${getDateSQL('event_data.created_at', unit, timezone)} as t,
+      ${aggFn}(event_data.number_value) as y
+    from event_data
+    any left join (
+          select *
+          from website_event
+          where website_id = {websiteId:UUID}
+            and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+            and event_type = 2
+            and event_name = {eventName:String}) website_event
+    on website_event.event_id = event_data.event_id
+      and website_event.session_id = event_data.session_id
+      and website_event.website_id = event_data.website_id
+    ${cohortQuery}
+    where event_data.website_id = {websiteId:UUID}
+      and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}
+      and event_data.data_key = {propertyName:String}
+      and event_data.data_type = 2
+    ${filterQuery}
+    group by t
+    order by t
+    `,
+    { ...queryParams, eventName, propertyName },
+    FUNCTION_NAME,
+  );
+}

src/queries/sql/events/getEventDataPivot.ts

@@ -0,0 +1,173 @@
+import clickhouse from '@/lib/clickhouse';
+import { DEFAULT_PAGE_SIZE } from '@/lib/constants';
+import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
+import prisma from '@/lib/prisma';
+import type { QueryFilters } from '@/lib/types';
+
+const FUNCTION_NAME = 'getEventDataPivot';
+
+export async function getEventDataPivot(
+  ...args: [websiteId: string, eventName: string, filters: QueryFilters]
+) {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(...args),
+    [CLICKHOUSE]: () => clickhouseQuery(...args),
+  });
+}
+
+async function relationalQuery(websiteId: string, eventName: string, filters: QueryFilters) {
+  const { rawQuery, parseFilters } = prisma;
+  const { page = 1, pageSize } = filters;
+  const size = +pageSize || DEFAULT_PAGE_SIZE;
+  const offset = +size * (+page - 1);
+
+  const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({
+    ...filters,
+    websiteId,
+  });
+
+  const countResult = await rawQuery(
+    `
+    select count(distinct website_event.event_id) as num
+    from website_event
+    join event_data on event_data.website_event_id = website_event.event_id
+      and event_data.website_id = {{websiteId::uuid}}
+      and event_data.created_at between {{startDate}} and {{endDate}}
+    ${cohortQuery}
+    ${joinSessionQuery}
+    where website_event.website_id = {{websiteId::uuid}}
+      and website_event.created_at between {{startDate}} and {{endDate}}
+      and website_event.event_name = {{eventName}}
+      ${filterQuery}
+    `,
+    { ...queryParams, eventName },
+  );
+
+  const count = countResult[0].num;
+
+  const rows = await rawQuery(
+    `
+    with paged_events as (
+      select website_event.event_id
+      from website_event
+      join event_data on event_data.website_event_id = website_event.event_id
+        and event_data.website_id = {{websiteId::uuid}}
+        and event_data.created_at between {{startDate}} and {{endDate}}
+      ${cohortQuery}
+      ${joinSessionQuery}
+      where website_event.website_id = {{websiteId::uuid}}
+        and website_event.created_at between {{startDate}} and {{endDate}}
+        and website_event.event_name = {{eventName}}
+        ${filterQuery}
+      group by website_event.event_id
+      order by max(website_event.created_at) desc
+      limit ${size} offset ${offset}
+    )
+    select
+      website_event.event_id as "eventId",
+      website_event.session_id as "sessionId",
+      website_event.event_name as "eventName",
+      website_event.url_path as "urlPath",
+      website_event.created_at as "createdAt",
+      event_data.data_key as "dataKey",
+      coalesce(
+        case when event_data.data_type = 1 then event_data.string_value end,
+        case when event_data.data_type = 2 then cast(event_data.number_value as varchar) end,
+        case when event_data.data_type = 4 then cast(event_data.date_value as varchar) end,
+        ''
+      ) as "value",
+      event_data.data_type as "dataType"
+    from event_data
+    join website_event on website_event.event_id = event_data.website_event_id
+      and website_event.website_id = {{websiteId::uuid}}
+    join paged_events on paged_events.event_id = event_data.website_event_id
+    where event_data.website_id = {{websiteId::uuid}}
+      and event_data.created_at between {{startDate}} and {{endDate}}
+    order by website_event.created_at desc
+    `,
+    { ...queryParams, eventName },
+    FUNCTION_NAME,
+  );
+
+  // Pivot flat rows into one record per event
+  const eventMap = new Map<
+    string,
+    { eventId: string; sessionId: string; eventName: string; urlPath: string; createdAt: Date; propertyKeys: string[]; propertyValues: string[] }
+  >();
+  for (const { eventId, sessionId, eventName: name, urlPath, createdAt, dataKey, value } of rows) {
+    if (!eventMap.has(eventId)) {
+      eventMap.set(eventId, { eventId, sessionId, eventName: name, urlPath, createdAt, propertyKeys: [], propertyValues: [] });
+    }
+    const entry = eventMap.get(eventId);
+    entry.propertyKeys.push(dataKey);
+    entry.propertyValues.push(value ?? '');
+  }
+
+  return { data: [...eventMap.values()], count, page: +page, pageSize: size };
+}
+
+async function clickhouseQuery(websiteId: string, eventName: string, filters: QueryFilters) {
+  const { rawQuery, parseFilters } = clickhouse;
+  const { page = 1, pageSize } = filters;
+  const size = +pageSize || DEFAULT_PAGE_SIZE;
+  const offset = +size * (+page - 1);
+
+  const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId });
+
+  const count = await rawQuery(
+    `
+    select count() as num
+    from umami.event_data_pivot
+    any left join (
+          select *
+          from website_event
+          where website_id = {websiteId:UUID}
+            and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+            and event_type = 2
+            and event_name = {eventName:String}) website_event
+    on website_event.event_id = event_data_pivot.event_id
+      and website_event.session_id = event_data_pivot.session_id
+      and website_event.website_id = event_data_pivot.website_id
+    ${cohortQuery}
+    where event_data_pivot.website_id = {websiteId:UUID}
+      and event_data_pivot.created_at between {startDate:DateTime64} and {endDate:DateTime64}
+    ${filterQuery}
+    `,
+    { ...queryParams, eventName },
+  ).then((res: any) => res[0].num);
+
+  const data = await rawQuery(
+    `
+    select
+      event_id as eventId,
+      session_id as sessionId,
+      event_name as eventName,
+      url_path as urlPath,
+      created_at as createdAt,
+      groupArrayMerge(property_keys) as propertyKeys,
+      groupArrayMerge(property_values) as propertyValues
+    from umami.event_data_pivot
+    any left join (
+          select *
+          from website_event
+          where website_id = {websiteId:UUID}
+            and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+            and event_type = 2
+            and event_name = {eventName:String}) website_event
+    on website_event.event_id = event_data_pivot.event_id
+      and website_event.session_id = event_data_pivot.session_id
+      and website_event.website_id = event_data_pivot.website_id
+    ${cohortQuery}
+    where event_data_pivot.website_id = {websiteId:UUID}
+      and event_data_pivot.created_at between {startDate:DateTime64} and {endDate:DateTime64}
+    ${filterQuery}
+    group by event_id, session_id, event_name, url_path, created_at
+    order by created_at desc
+    limit ${size} offset ${offset}
+    `,
+    { ...queryParams, eventName },
+    FUNCTION_NAME,
+  );
+
+  return { data, count, page: +page, pageSize: size };
+}

src/queries/sql/events/getEventDataProperties.ts

@@ -31,6 +31,7 @@ async function relationalQuery(
     select
       website_event.event_name as "eventName",
       event_data.data_key as "propertyName",
+      event_data.data_type as "dataType",
       count(*) as "total"
     from event_data 
     join website_event on website_event.event_id = event_data.website_event_id
@@ -41,8 +42,8 @@ async function relationalQuery(
     where event_data.website_id = {{websiteId::uuid}}
       and event_data.created_at between {{startDate}} and {{endDate}}
     ${filterQuery}
-    group by website_event.event_name, event_data.data_key
-    order by 3 desc
+    group by website_event.event_name, event_data.data_key, event_data.data_type
+    order by 4 desc
     limit 500
     `,
     queryParams,
@@ -53,7 +54,7 @@ async function relationalQuery(
 async function clickhouseQuery(
   websiteId: string,
   filters: QueryFilters & { propertyName?: string },
-): Promise<{ eventName: string; propertyName: string; total: number }[]> {
+): Promise<{ eventName: string; propertyName: string; dataType: number; total: number }[]> {
   const { rawQuery, parseFilters } = clickhouse;
   const { filterQuery, cohortQuery, queryParams } = parseFilters(
     { ...filters, websiteId },
@@ -67,6 +68,7 @@ async function clickhouseQuery(
     select
       event_name as eventName,
       data_key as propertyName,
+      data_type as dataType,
       count(*) as total
     from event_data
     any left join (
@@ -82,8 +84,8 @@ async function clickhouseQuery(
     where event_data.website_id = {websiteId:UUID}
       and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}
     ${filterQuery}
-    group by event_name, data_key
-    order by 1, 3 desc
+    group by event_name, data_key, data_type
+    order by 1, 4 desc
     limit 500
     `,
     queryParams,

src/queries/sql/events/getEventDataPropertySeries.ts

@@ -0,0 +1,96 @@
+import clickhouse from '@/lib/clickhouse';
+import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
+import prisma from '@/lib/prisma';
+import type { QueryFilters } from '@/lib/types';
+
+const FUNCTION_NAME = 'getEventDataPropertySeries';
+
+export async function getEventDataPropertySeries(
+  ...args: [websiteId: string, eventName: string, propertyName: string, filters: QueryFilters]
+) {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(...args),
+    [CLICKHOUSE]: () => clickhouseQuery(...args),
+  });
+}
+
+async function relationalQuery(
+  websiteId: string,
+  eventName: string,
+  propertyName: string,
+  filters: QueryFilters,
+) {
+  const { timezone = 'utc', unit = 'day' } = filters;
+  const { rawQuery, getDateSQL, parseFilters } = prisma;
+  const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({
+    ...filters,
+    websiteId,
+  });
+
+  return rawQuery(
+    `
+    select
+      event_data.string_value as x,
+      ${getDateSQL('event_data.created_at', unit, timezone)} t,
+      count(*) y
+    from event_data
+    join website_event on website_event.event_id = event_data.website_event_id
+      and website_event.website_id = {{websiteId::uuid}}
+      and website_event.created_at between {{startDate}} and {{endDate}}
+      and website_event.event_type = 2
+      and website_event.event_name = {{eventName}}
+    ${cohortQuery}
+    ${joinSessionQuery}
+    where event_data.website_id = {{websiteId::uuid}}
+      and event_data.created_at between {{startDate}} and {{endDate}}
+      and event_data.data_key = {{propertyName}}
+      and event_data.data_type = 1
+      ${filterQuery}
+    group by 1, 2
+    order by 2
+    `,
+    { ...queryParams, eventName, propertyName },
+    FUNCTION_NAME,
+  );
+}
+
+async function clickhouseQuery(
+  websiteId: string,
+  eventName: string,
+  propertyName: string,
+  filters: QueryFilters,
+): Promise<{ x: string; t: string; y: number }[]> {
+  const { timezone = 'UTC', unit = 'day' } = filters;
+  const { rawQuery, getDateSQL, parseFilters } = clickhouse;
+  const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId });
+
+  return rawQuery(
+    `
+    select
+      event_data.string_value as x,
+      ${getDateSQL('event_data.created_at', unit, timezone)} as t,
+      count() as y
+    from event_data
+    any left join (
+          select *
+          from website_event
+          where website_id = {websiteId:UUID}
+            and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+            and event_type = 2
+            and event_name = {eventName:String}) website_event
+    on website_event.event_id = event_data.event_id
+      and website_event.session_id = event_data.session_id
+      and website_event.website_id = event_data.website_id
+    ${cohortQuery}
+    where event_data.website_id = {websiteId:UUID}
+      and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}
+      and event_data.data_key = {propertyName:String}
+      and event_data.data_type = 1
+    ${filterQuery}
+    group by x, t
+    order by t
+    `,
+    { ...queryParams, eventName, propertyName },
+    FUNCTION_NAME,
+  );
+}

src/queries/sql/index.ts

@@ -1,7 +1,10 @@
 export * from './events/getEventDataById';
 export * from './events/getEventDataEvents';
 export * from './events/getEventDataFields';
+export * from './events/getEventDataNumericSeries';
+export * from './events/getEventDataPivot';
 export * from './events/getEventDataProperties';
+export * from './events/getEventDataPropertySeries';
 export * from './events/getEventDataStats';
 export * from './events/getEventDataUsage';
 export * from './events/getEventDataValues';