Add scroll heatmaps.

src/queries/sql/heatmap/extractHeatmapEvents.ts

@@ -2,6 +2,7 @@ import { HEATMAP_EVENT_TYPE } from '@/lib/constants';
 
 const RRWEB_TYPE_INCREMENTAL = 3;
 const RRWEB_TYPE_META = 4;
+const RRWEB_TYPE_CUSTOM = 5;
 const RRWEB_SOURCE_MOUSE_INTERACTION = 2;
 const RRWEB_SOURCE_VIEWPORT_RESIZE = 4;
 const RRWEB_MOUSE_CLICK = 2;
@@ -13,6 +14,7 @@ export interface ExtractedHeatmapEvent {
   y: number | null;
   viewportW: number | null;
   viewportH: number | null;
+  pageH: number | null;
   scrollPct: number | null;
   urlPath: string;
   createdAt: Date;
@@ -46,6 +48,36 @@ export function extractHeatmapEvents(events: any[]): ExtractedHeatmapEvent[] {
       continue;
     }
 
+    if (ev.type === RRWEB_TYPE_CUSTOM && ev.data) {
+      if (ev.data.tag === 'url-change') {
+        const path = safePathname(ev.data.payload?.url);
+        if (path) urlPath = path;
+        continue;
+      }
+
+      if (ev.data.tag === 'scroll-progress' && ev.data.payload) {
+        const p = ev.data.payload;
+        const path = safePathname(p.url) ?? urlPath;
+        if (path === null) continue;
+        out.push({
+          eventType: HEATMAP_EVENT_TYPE.scroll,
+          nodeId: null,
+          x: null,
+          y: null,
+          viewportW: typeof p.viewportW === 'number' ? p.viewportW : viewportW,
+          viewportH: typeof p.viewportH === 'number' ? p.viewportH : viewportH,
+          pageH: typeof p.pageH === 'number' ? p.pageH : null,
+          scrollPct:
+            typeof p.scrollPct === 'number'
+              ? Math.max(0, Math.min(100, Math.round(p.scrollPct)))
+              : null,
+          urlPath: path,
+          createdAt: new Date(typeof ev.timestamp === 'number' ? ev.timestamp : Date.now()),
+        });
+      }
+      continue;
+    }
+
     if (ev.type !== RRWEB_TYPE_INCREMENTAL || !ev.data) continue;
 
     const d = ev.data;
@@ -68,6 +100,7 @@ export function extractHeatmapEvents(events: any[]): ExtractedHeatmapEvent[] {
         y: typeof d.y === 'number' ? Math.round(d.y) : null,
         viewportW,
         viewportH,
+        pageH: null,
         scrollPct: null,
         urlPath,
         createdAt: new Date(typeof ev.timestamp === 'number' ? ev.timestamp : Date.now()),

src/queries/sql/heatmap/getHeatmap.ts

@@ -7,11 +7,15 @@ const FUNCTION_NAME = 'getHeatmap';
 
 const POINT_LIMIT = 5000;
 const PAGE_LIMIT = 100;
+const SCROLL_BUCKET_SIZE = 5;
+
+export type HeatmapMode = 'click' | 'scroll';
 
 export interface HeatmapParameters {
   startDate: Date;
   endDate: Date;
   urlPath?: string;
+  mode?: HeatmapMode;
 }
 
 export interface HeatmapPage {
@@ -29,9 +33,22 @@ export interface HeatmapPoint {
   count: number;
 }
 
+export interface HeatmapScrollBucket {
+  depth: number;
+  sessions: number;
+}
+
 export interface HeatmapResult {
+  mode: HeatmapMode;
   pages: HeatmapPage[];
   points: HeatmapPoint[];
+  scroll: {
+    buckets: HeatmapScrollBucket[];
+    totalSessions: number;
+    pageH: number | null;
+    viewportW: number | null;
+    viewportH: number | null;
+  };
 }
 
 export async function getHeatmap(
@@ -44,11 +61,20 @@ export async function getHeatmap(
   });
 }
 
+const emptyScroll = (): HeatmapResult['scroll'] => ({
+  buckets: [],
+  totalSessions: 0,
+  pageH: null,
+  viewportW: null,
+  viewportH: null,
+});
+
 async function relationalQuery(
   websiteId: string,
-  { startDate, endDate, urlPath }: HeatmapParameters,
+  { startDate, endDate, urlPath, mode = 'click' }: HeatmapParameters,
 ): Promise<HeatmapResult> {
   const { rawQuery } = prisma;
+  const eventType = mode === 'scroll' ? HEATMAP_EVENT_TYPE.scroll : HEATMAP_EVENT_TYPE.click;
 
   const pages: HeatmapPage[] = await rawQuery(
     `
@@ -64,17 +90,73 @@ async function relationalQuery(
     order by count desc
     limit ${PAGE_LIMIT}
     `,
-    {
-      websiteId,
-      eventType: HEATMAP_EVENT_TYPE.click,
-      startDate,
-      endDate,
-    },
+    { websiteId, eventType, startDate, endDate },
     FUNCTION_NAME,
   );
 
   if (!urlPath) {
-    return { pages, points: [] };
+    return { mode, pages, points: [], scroll: emptyScroll() };
+  }
+
+  if (mode === 'scroll') {
+    const bucketRows: { depth: number | string; sessions: number | string }[] = await rawQuery(
+      `
+      select
+        (floor(max_pct / ${SCROLL_BUCKET_SIZE}) * ${SCROLL_BUCKET_SIZE})::int as depth,
+        count(*)::int as sessions
+      from (
+        select visit_id, max(scroll_pct) as max_pct
+        from heatmap_event
+        where website_id = {{websiteId::uuid}}
+          and event_type = {{eventType}}
+          and url_path = {{urlPath}}
+          and created_at between {{startDate}} and {{endDate}}
+          and scroll_pct is not null
+        group by visit_id
+      ) per_session
+      group by depth
+      order by depth
+      `,
+      { websiteId, eventType, urlPath, startDate, endDate },
+      FUNCTION_NAME,
+    );
+
+    const dimRows: {
+      totalSessions: number | string;
+      pageH: number | null;
+      viewportW: number | null;
+      viewportH: number | null;
+    }[] = await rawQuery(
+      `
+      select
+        count(distinct visit_id)::int as "totalSessions",
+        (mode() within group (order by page_h))::int as "pageH",
+        (mode() within group (order by viewport_w))::int as "viewportW",
+        (mode() within group (order by viewport_h))::int as "viewportH"
+      from heatmap_event
+      where website_id = {{websiteId::uuid}}
+        and event_type = {{eventType}}
+        and url_path = {{urlPath}}
+        and created_at between {{startDate}} and {{endDate}}
+        and scroll_pct is not null
+      `,
+      { websiteId, eventType, urlPath, startDate, endDate },
+      FUNCTION_NAME,
+    );
+
+    const dim = dimRows[0];
+    return {
+      mode,
+      pages,
+      points: [],
+      scroll: {
+        buckets: bucketRows.map(r => ({ depth: Number(r.depth), sessions: Number(r.sessions) })),
+        totalSessions: Number(dim?.totalSessions ?? 0),
+        pageH: dim?.pageH ?? null,
+        viewportW: dim?.viewportW ?? null,
+        viewportH: dim?.viewportH ?? null,
+      },
+    };
   }
 
   const rawPoints: HeatmapPoint[] = await rawQuery(
@@ -99,24 +181,19 @@ async function relationalQuery(
     order by count desc
     limit ${POINT_LIMIT}
     `,
-    {
-      websiteId,
-      eventType: HEATMAP_EVENT_TYPE.click,
-      urlPath,
-      startDate,
-      endDate,
-    },
+    { websiteId, eventType, urlPath, startDate, endDate },
     FUNCTION_NAME,
   );
 
-  return { pages, points: rawPoints };
+  return { mode, pages, points: rawPoints, scroll: emptyScroll() };
 }
 
 async function clickhouseQuery(
   websiteId: string,
-  { startDate, endDate, urlPath }: HeatmapParameters,
+  { startDate, endDate, urlPath, mode = 'click' }: HeatmapParameters,
 ): Promise<HeatmapResult> {
   const { rawQuery } = clickhouse;
+  const eventType = mode === 'scroll' ? HEATMAP_EVENT_TYPE.scroll : HEATMAP_EVENT_TYPE.click;
 
   const pageRows = await rawQuery<
     { urlPath: string; count: string | number; sessions: string | number }[]
@@ -134,12 +211,7 @@ async function clickhouseQuery(
     order by count desc
     limit ${PAGE_LIMIT}
     `,
-    {
-      websiteId,
-      eventType: HEATMAP_EVENT_TYPE.click,
-      startDate,
-      endDate,
-    },
+    { websiteId, eventType, startDate, endDate },
     FUNCTION_NAME,
   );
 
@@ -150,7 +222,72 @@ async function clickhouseQuery(
   }));
 
   if (!urlPath) {
-    return { pages, points: [] };
+    return { mode, pages, points: [], scroll: emptyScroll() };
+  }
+
+  if (mode === 'scroll') {
+    const bucketRows = await rawQuery<{ depth: number | string; sessions: number | string }[]>(
+      `
+      select
+        intDiv(max_pct, ${SCROLL_BUCKET_SIZE}) * ${SCROLL_BUCKET_SIZE} as depth,
+        count() as sessions
+      from (
+        select visit_id, max(scroll_pct) as max_pct
+        from heatmap_event
+        where website_id = {websiteId:UUID}
+          and event_type = {eventType:UInt8}
+          and url_path = {urlPath:String}
+          and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+          and scroll_pct is not null
+        group by visit_id
+      )
+      group by depth
+      order by depth
+      `,
+      { websiteId, eventType, urlPath, startDate, endDate },
+      FUNCTION_NAME,
+    );
+
+    const dimRows = await rawQuery<
+      {
+        totalSessions: number | string;
+        pageH: number | null;
+        viewportW: number | null;
+        viewportH: number | null;
+      }[]
+    >(
+      `
+      select
+        uniq(visit_id) as totalSessions,
+        toInt32OrNull(toString(arrayElement(topK(1)(page_h), 1))) as pageH,
+        toInt32OrNull(toString(arrayElement(topK(1)(viewport_w), 1))) as viewportW,
+        toInt32OrNull(toString(arrayElement(topK(1)(viewport_h), 1))) as viewportH
+      from heatmap_event
+      where website_id = {websiteId:UUID}
+        and event_type = {eventType:UInt8}
+        and url_path = {urlPath:String}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+        and scroll_pct is not null
+      `,
+      { websiteId, eventType, urlPath, startDate, endDate },
+      FUNCTION_NAME,
+    );
+
+    const dim = dimRows[0];
+    return {
+      mode,
+      pages,
+      points: [],
+      scroll: {
+        buckets: bucketRows.map(r => ({ depth: Number(r.depth), sessions: Number(r.sessions) })),
+        totalSessions: Number(dim?.totalSessions ?? 0),
+        pageH: dim?.pageH === null || dim?.pageH === undefined ? null : Number(dim.pageH),
+        viewportW:
+          dim?.viewportW === null || dim?.viewportW === undefined ? null : Number(dim.viewportW),
+        viewportH:
+          dim?.viewportH === null || dim?.viewportH === undefined ? null : Number(dim.viewportH),
+      },
+    };
   }
 
   const pointRows = await rawQuery<
@@ -184,13 +321,7 @@ async function clickhouseQuery(
     order by count desc
     limit ${POINT_LIMIT}
     `,
-    {
-      websiteId,
-      eventType: HEATMAP_EVENT_TYPE.click,
-      urlPath,
-      startDate,
-      endDate,
-    },
+    { websiteId, eventType, urlPath, startDate, endDate },
     FUNCTION_NAME,
   );
 
@@ -203,5 +334,5 @@ async function clickhouseQuery(
     count: Number(p.count),
   }));
 
-  return { pages, points };
+  return { mode, pages, points, scroll: emptyScroll() };
 }

src/queries/sql/heatmap/saveHeatmapEvents.ts

@@ -15,6 +15,7 @@ export interface HeatmapEventRow {
   y: number | null;
   viewportW: number | null;
   viewportH: number | null;
+  pageH: number | null;
   scrollPct: number | null;
   createdAt: Date;
 }
@@ -42,6 +43,7 @@ async function relationalQuery(rows: HeatmapEventRow[]) {
       y: r.y,
       viewportW: r.viewportW,
       viewportH: r.viewportH,
+      pageH: r.pageH,
       scrollPct: r.scrollPct,
       createdAt: r.createdAt,
     })),
@@ -64,6 +66,7 @@ async function clickhouseQuery(rows: HeatmapEventRow[]) {
     y: r.y,
     viewport_w: r.viewportW,
     viewport_h: r.viewportH,
+    page_h: r.pageH,
     scroll_pct: r.scrollPct,
     created_at: getUTCString(r.createdAt),
   }));