POC for playwright based screenshots for heatmaps
This commit is contained in:
1 parent
1f3f805a30
commit
6de5adaefc
12 files changed
+938
-1139
No files matched your search
@@ -0,0 +1,524 @@
|
||||
import prisma from '@/lib/prisma';
|
||||
import { uuid } from '@/lib/crypto';
|
||||
import { getWebsite } from '@/queries/prisma';
|
||||
|
||||
const SNAPSHOT_STATUS = {
|
||||
pending: 'pending',
|
||||
ready: 'ready',
|
||||
failed: 'failed',
|
||||
} as const;
|
||||
|
||||
const SNAPSHOT_RETRY_DELAY_MS = 15 * 60 * 1000;
|
||||
const SNAPSHOT_PENDING_WINDOW_MS = 30 * 1000;
|
||||
|
||||
export type HeatmapSnapshotStatus = (typeof SNAPSHOT_STATUS)[keyof typeof SNAPSHOT_STATUS];
|
||||
|
||||
export interface HeatmapSnapshotImage {
|
||||
id: string;
|
||||
imageUrl: string | null;
|
||||
status: HeatmapSnapshotStatus;
|
||||
mimeType: string | null;
|
||||
pageW: number;
|
||||
pageH: number;
|
||||
viewportW: number;
|
||||
viewportH: number;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
interface SnapshotRecord {
|
||||
id: string;
|
||||
websiteId: string;
|
||||
urlPath: string;
|
||||
viewportW: number;
|
||||
viewportH: number;
|
||||
pageW: number;
|
||||
pageH: number;
|
||||
status: HeatmapSnapshotStatus;
|
||||
mimeType: string | null;
|
||||
imageSize: number | null;
|
||||
error: string | null;
|
||||
hasImage: boolean;
|
||||
updatedAt: Date | string | null;
|
||||
}
|
||||
|
||||
interface EnsureHeatmapSnapshotOptions {
|
||||
websiteId: string;
|
||||
urlPath: string;
|
||||
viewportW: number | null;
|
||||
viewportH: number | null;
|
||||
pageW: number | null;
|
||||
pageH: number | null;
|
||||
}
|
||||
|
||||
interface CaptureResult {
|
||||
imageData: Buffer;
|
||||
mimeType: string;
|
||||
pageW: number;
|
||||
pageH: number;
|
||||
}
|
||||
|
||||
async function measurePage(page: any) {
|
||||
return page.evaluate(() => {
|
||||
const doc = document.documentElement;
|
||||
const body = document.body;
|
||||
const root = document.scrollingElement || doc || body;
|
||||
let maxRight = 0;
|
||||
let maxBottom = 0;
|
||||
|
||||
if (body) {
|
||||
const walker = document.createTreeWalker(body, NodeFilter.SHOW_ELEMENT);
|
||||
let node = walker.currentNode as Element | null;
|
||||
|
||||
while (node) {
|
||||
const rect = node.getBoundingClientRect?.();
|
||||
|
||||
if (rect && (rect.width > 0 || rect.height > 0)) {
|
||||
maxRight = Math.max(maxRight, rect.right);
|
||||
maxBottom = Math.max(maxBottom, rect.bottom);
|
||||
}
|
||||
|
||||
node = walker.nextNode() as Element | null;
|
||||
}
|
||||
}
|
||||
|
||||
const pageW = Math.max(
|
||||
window.innerWidth,
|
||||
root?.scrollWidth || 0,
|
||||
doc?.scrollWidth || 0,
|
||||
body?.scrollWidth || 0,
|
||||
Math.ceil(maxRight),
|
||||
);
|
||||
const pageH = Math.max(
|
||||
window.innerHeight,
|
||||
root?.scrollHeight || 0,
|
||||
doc?.scrollHeight || 0,
|
||||
body?.scrollHeight || 0,
|
||||
Math.ceil(maxBottom),
|
||||
);
|
||||
|
||||
return {
|
||||
pageW: Math.ceil(pageW),
|
||||
pageH: Math.ceil(pageH),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function getSchema() {
|
||||
const databaseUrl = process.env.DATABASE_URL;
|
||||
|
||||
if (!databaseUrl) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const connectionUrl = new URL(databaseUrl);
|
||||
|
||||
return connectionUrl.searchParams.get('schema');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function rawExecute(sql: string, data: Record<string, any> = {}) {
|
||||
const params: any[] = [];
|
||||
const schema = getSchema();
|
||||
|
||||
if (schema) {
|
||||
await prisma.client.$executeRawUnsafe(`SET search_path TO "${schema}";`);
|
||||
}
|
||||
|
||||
const query = sql.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*}}/g, (...args) => {
|
||||
const [, name, type] = args;
|
||||
|
||||
params.push(data[name]);
|
||||
|
||||
return `$${params.length}${type ?? ''}`;
|
||||
});
|
||||
|
||||
return prisma.client.$executeRawUnsafe(query, ...params);
|
||||
}
|
||||
|
||||
async function findSnapshot(
|
||||
websiteId: string,
|
||||
urlPath: string,
|
||||
viewportW: number,
|
||||
viewportH: number,
|
||||
): Promise<SnapshotRecord | null> {
|
||||
const rows = await prisma.rawQuery(
|
||||
`
|
||||
select
|
||||
snapshot_id as id,
|
||||
website_id as "websiteId",
|
||||
url_path as "urlPath",
|
||||
viewport_w as "viewportW",
|
||||
viewport_h as "viewportH",
|
||||
page_w as "pageW",
|
||||
page_h as "pageH",
|
||||
status,
|
||||
mime_type as "mimeType",
|
||||
image_size as "imageSize",
|
||||
error,
|
||||
image_data is not null as "hasImage",
|
||||
updated_at as "updatedAt"
|
||||
from heatmap_snapshot
|
||||
where website_id = {{websiteId::uuid}}
|
||||
and url_path = {{urlPath}}
|
||||
and viewport_w = {{viewportW}}
|
||||
and viewport_h = {{viewportH}}
|
||||
limit 1
|
||||
`,
|
||||
{ websiteId, urlPath, viewportW, viewportH },
|
||||
'findHeatmapSnapshot',
|
||||
);
|
||||
|
||||
return rows?.[0] ?? null;
|
||||
}
|
||||
|
||||
function getSnapshotImageUrl(websiteId: string, snapshotId: string) {
|
||||
return `/api/websites/${websiteId}/heatmaps/snapshots/${snapshotId}`;
|
||||
}
|
||||
|
||||
function mapSnapshot(websiteId: string, row: SnapshotRecord): HeatmapSnapshotImage {
|
||||
return {
|
||||
id: row.id,
|
||||
imageUrl: row.status === SNAPSHOT_STATUS.ready && row.hasImage ? getSnapshotImageUrl(websiteId, row.id) : null,
|
||||
status: row.status,
|
||||
mimeType: row.mimeType,
|
||||
pageW: Number(row.pageW),
|
||||
pageH: Number(row.pageH),
|
||||
viewportW: Number(row.viewportW),
|
||||
viewportH: Number(row.viewportH),
|
||||
error: row.error,
|
||||
};
|
||||
}
|
||||
|
||||
function getFirstDomain(domain?: string | null) {
|
||||
return domain?.split(',')[0]?.trim() || null;
|
||||
}
|
||||
|
||||
function getWebsiteOrigin(domain?: string | null) {
|
||||
const host = getFirstDomain(domain);
|
||||
|
||||
if (!host) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (host.startsWith('http://') || host.startsWith('https://')) {
|
||||
return new URL(host);
|
||||
}
|
||||
|
||||
const protocol =
|
||||
host.startsWith('localhost') || host.startsWith('127.0.0.1') || host.startsWith('[::1]')
|
||||
? 'http'
|
||||
: 'https';
|
||||
|
||||
return new URL(`${protocol}://${host}`);
|
||||
}
|
||||
|
||||
function buildCaptureUrl(domain: string | null | undefined, urlPath: string) {
|
||||
const origin = getWebsiteOrigin(domain);
|
||||
|
||||
if (!origin) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new URL(urlPath || '/', origin).toString();
|
||||
}
|
||||
|
||||
export function shouldSkipSnapshot(urlPath: string) {
|
||||
// Internal Umami app routes cannot be rendered from the tracked website domain.
|
||||
return urlPath.startsWith('/teams/');
|
||||
}
|
||||
|
||||
async function upsertSnapshotRecord({
|
||||
id,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status,
|
||||
mimeType,
|
||||
imageData,
|
||||
error,
|
||||
}: {
|
||||
id: string;
|
||||
websiteId: string;
|
||||
urlPath: string;
|
||||
viewportW: number;
|
||||
viewportH: number;
|
||||
pageW: number;
|
||||
pageH: number;
|
||||
status: HeatmapSnapshotStatus;
|
||||
mimeType: string | null;
|
||||
imageData: Buffer | null;
|
||||
error: string | null;
|
||||
}) {
|
||||
return rawExecute(
|
||||
`
|
||||
insert into heatmap_snapshot (
|
||||
snapshot_id,
|
||||
website_id,
|
||||
url_path,
|
||||
viewport_w,
|
||||
viewport_h,
|
||||
page_w,
|
||||
page_h,
|
||||
status,
|
||||
mime_type,
|
||||
image_data,
|
||||
image_size,
|
||||
error,
|
||||
created_at,
|
||||
updated_at
|
||||
)
|
||||
values (
|
||||
{{id::uuid}},
|
||||
{{websiteId::uuid}},
|
||||
{{urlPath}},
|
||||
{{viewportW}},
|
||||
{{viewportH}},
|
||||
{{pageW}},
|
||||
{{pageH}},
|
||||
{{status}},
|
||||
{{mimeType}},
|
||||
{{imageData}},
|
||||
{{imageSize}},
|
||||
{{error}},
|
||||
now(),
|
||||
now()
|
||||
)
|
||||
on conflict (website_id, url_path, viewport_w, viewport_h) do update
|
||||
set
|
||||
page_w = excluded.page_w,
|
||||
page_h = excluded.page_h,
|
||||
status = excluded.status,
|
||||
mime_type = excluded.mime_type,
|
||||
image_data = excluded.image_data,
|
||||
image_size = excluded.image_size,
|
||||
error = excluded.error,
|
||||
updated_at = now()
|
||||
`,
|
||||
{
|
||||
id,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status,
|
||||
mimeType,
|
||||
imageData,
|
||||
imageSize: imageData?.byteLength ?? null,
|
||||
error,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function captureSnapshot(
|
||||
url: string,
|
||||
viewportW: number,
|
||||
viewportH: number,
|
||||
pageW?: number,
|
||||
): Promise<CaptureResult> {
|
||||
const { chromium } = await import('@playwright/test');
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const initialViewportW = Math.max(viewportW, pageW || 0);
|
||||
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: initialViewportW, height: viewportH },
|
||||
screen: { width: initialViewportW, height: viewportH },
|
||||
deviceScaleFactor: 1,
|
||||
ignoreHTTPSErrors: true,
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
|
||||
await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => undefined);
|
||||
await page.waitForTimeout(500);
|
||||
|
||||
let dimensions = await measurePage(page);
|
||||
let currentWidth = initialViewportW;
|
||||
let captureWidth = Math.max(initialViewportW, dimensions.pageW);
|
||||
|
||||
for (let i = 0; i < 3; i++) {
|
||||
if (captureWidth <= currentWidth) {
|
||||
break;
|
||||
}
|
||||
|
||||
currentWidth = captureWidth;
|
||||
|
||||
await page.setViewportSize({
|
||||
width: currentWidth,
|
||||
height: viewportH,
|
||||
});
|
||||
await page.waitForLoadState('networkidle', { timeout: 3000 }).catch(() => undefined);
|
||||
await page.waitForTimeout(300);
|
||||
|
||||
dimensions = await measurePage(page);
|
||||
captureWidth = Math.max(currentWidth, dimensions.pageW);
|
||||
}
|
||||
|
||||
const imageData = Buffer.from(await page.screenshot({ fullPage: true, type: 'png' }));
|
||||
|
||||
await context.close();
|
||||
|
||||
return {
|
||||
imageData,
|
||||
mimeType: 'image/png',
|
||||
pageW: dimensions.pageW,
|
||||
pageH: dimensions.pageH,
|
||||
};
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureHeatmapSnapshot({
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
}: EnsureHeatmapSnapshotOptions): Promise<HeatmapSnapshotImage | null> {
|
||||
if (!urlPath || !viewportW || !viewportH || !pageW || !pageH) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (shouldSkipSnapshot(urlPath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const existing = await findSnapshot(websiteId, urlPath, viewportW, viewportH);
|
||||
|
||||
if (existing?.status === SNAPSHOT_STATUS.ready && existing.hasImage) {
|
||||
return mapSnapshot(websiteId, existing);
|
||||
}
|
||||
|
||||
const updatedAt = existing?.updatedAt ? new Date(existing.updatedAt) : null;
|
||||
const ageMs = updatedAt ? Date.now() - updatedAt.getTime() : Number.POSITIVE_INFINITY;
|
||||
|
||||
if (
|
||||
existing?.status === SNAPSHOT_STATUS.pending &&
|
||||
ageMs < SNAPSHOT_PENDING_WINDOW_MS
|
||||
) {
|
||||
return mapSnapshot(websiteId, existing);
|
||||
}
|
||||
|
||||
if (
|
||||
existing?.status === SNAPSHOT_STATUS.failed &&
|
||||
ageMs < SNAPSHOT_RETRY_DELAY_MS
|
||||
) {
|
||||
return mapSnapshot(websiteId, existing);
|
||||
}
|
||||
|
||||
const snapshotId = existing?.id ?? uuid();
|
||||
const website = await getWebsite(websiteId);
|
||||
const captureUrl = buildCaptureUrl(website?.domain, urlPath);
|
||||
|
||||
if (!captureUrl) {
|
||||
await upsertSnapshotRecord({
|
||||
id: snapshotId,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status: SNAPSHOT_STATUS.failed,
|
||||
mimeType: null,
|
||||
imageData: null,
|
||||
error: 'Website domain is not configured for screenshot capture.',
|
||||
});
|
||||
|
||||
const failed = await findSnapshot(websiteId, urlPath, viewportW, viewportH);
|
||||
|
||||
return failed ? mapSnapshot(websiteId, failed) : null;
|
||||
}
|
||||
|
||||
await upsertSnapshotRecord({
|
||||
id: snapshotId,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status: SNAPSHOT_STATUS.pending,
|
||||
mimeType: null,
|
||||
imageData: null,
|
||||
error: null,
|
||||
});
|
||||
|
||||
try {
|
||||
const capture = await captureSnapshot(captureUrl, viewportW, viewportH, pageW);
|
||||
|
||||
await upsertSnapshotRecord({
|
||||
id: snapshotId,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW: capture.pageW,
|
||||
pageH: capture.pageH,
|
||||
status: SNAPSHOT_STATUS.ready,
|
||||
mimeType: capture.mimeType,
|
||||
imageData: capture.imageData,
|
||||
error: null,
|
||||
});
|
||||
} catch (error) {
|
||||
await upsertSnapshotRecord({
|
||||
id: snapshotId,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status: SNAPSHOT_STATUS.failed,
|
||||
mimeType: null,
|
||||
imageData: null,
|
||||
error: error instanceof Error ? error.message.slice(0, 500) : 'Screenshot capture failed.',
|
||||
});
|
||||
}
|
||||
|
||||
const snapshot = await findSnapshot(websiteId, urlPath, viewportW, viewportH);
|
||||
|
||||
return snapshot ? mapSnapshot(websiteId, snapshot) : null;
|
||||
}
|
||||
|
||||
export async function getHeatmapSnapshotImage(
|
||||
websiteId: string,
|
||||
snapshotId: string,
|
||||
): Promise<{ mimeType: string; imageData: Buffer } | null> {
|
||||
const rows = await prisma.rawQuery(
|
||||
`
|
||||
select
|
||||
mime_type as "mimeType",
|
||||
image_data as "imageData"
|
||||
from heatmap_snapshot
|
||||
where snapshot_id = {{snapshotId::uuid}}
|
||||
and website_id = {{websiteId::uuid}}
|
||||
and status = 'ready'
|
||||
and image_data is not null
|
||||
limit 1
|
||||
`,
|
||||
{ websiteId, snapshotId },
|
||||
'getHeatmapSnapshotImage',
|
||||
);
|
||||
|
||||
const row = rows?.[0];
|
||||
|
||||
if (!row?.imageData || !row?.mimeType) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
mimeType: row.mimeType,
|
||||
imageData: Buffer.from(row.imageData),
|
||||
};
|
||||
}
|
||||
@@ -3,12 +3,17 @@ import { HEATMAP_EVENT_TYPE } from '@/lib/constants';
|
||||
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
|
||||
import prisma from '@/lib/prisma';
|
||||
import type { QueryFilters } from '@/lib/types';
|
||||
import {
|
||||
ensureHeatmapSnapshot,
|
||||
shouldSkipSnapshot,
|
||||
type HeatmapSnapshotImage,
|
||||
} from './ensureHeatmapSnapshot';
|
||||
|
||||
const FUNCTION_NAME = 'getHeatmap';
|
||||
|
||||
const POINT_LIMIT = 5000;
|
||||
const PAGE_LIMIT = 100;
|
||||
const SCROLL_BUCKET_SIZE = 5;
|
||||
const SCROLL_BUCKET_SIZE = 10;
|
||||
|
||||
export type HeatmapMode = 'click' | 'scroll';
|
||||
|
||||
@@ -41,12 +46,7 @@ export interface HeatmapScrollBucket {
|
||||
sessions: number;
|
||||
}
|
||||
|
||||
export interface HeatmapSnapshot {
|
||||
replayId: string;
|
||||
timestamp: number;
|
||||
chunkIndex: number | null;
|
||||
eventIndex: number | null;
|
||||
}
|
||||
export type HeatmapSnapshot = HeatmapSnapshotImage;
|
||||
|
||||
export interface HeatmapResult {
|
||||
mode: HeatmapMode;
|
||||
@@ -73,13 +73,6 @@ export async function getHeatmap(
|
||||
});
|
||||
}
|
||||
|
||||
interface SnapshotRow {
|
||||
replayId: string;
|
||||
timestamp: number | string;
|
||||
chunkIndex: number | string | null;
|
||||
eventIndex: number | string | null;
|
||||
}
|
||||
|
||||
interface HeatmapFilterContext {
|
||||
joinQuery: string;
|
||||
queryParams: Record<string, any>;
|
||||
@@ -107,7 +100,7 @@ async function relationalQuery(
|
||||
`
|
||||
: '';
|
||||
|
||||
const pages: HeatmapPage[] = await rawQuery(
|
||||
const rawPages: HeatmapPage[] = await rawQuery(
|
||||
`
|
||||
select
|
||||
h.url_path as "urlPath",
|
||||
@@ -126,6 +119,7 @@ async function relationalQuery(
|
||||
{ ...filterContext.queryParams, websiteId, eventType, startDate, endDate },
|
||||
FUNCTION_NAME,
|
||||
);
|
||||
const pages = rawPages.filter(page => !shouldSkipSnapshot(page.urlPath));
|
||||
|
||||
if (!urlPath) {
|
||||
return { mode, pages, points: [], snapshot: null, scroll: emptyScroll() };
|
||||
@@ -190,17 +184,13 @@ async function relationalQuery(
|
||||
viewportW: dim?.viewportW ?? null,
|
||||
viewportH: dim?.viewportH ?? null,
|
||||
};
|
||||
const snapshot = await getRelationalSnapshot(rawQuery, {
|
||||
const snapshot = await ensureHeatmapSnapshot({
|
||||
websiteId,
|
||||
eventType,
|
||||
urlPath,
|
||||
startDate,
|
||||
endDate,
|
||||
pageW: scroll.pageW,
|
||||
pageH: scroll.pageH,
|
||||
viewportW: scroll.viewportW,
|
||||
viewportH: scroll.viewportH,
|
||||
filterContext,
|
||||
pageW: scroll.pageW,
|
||||
pageH: scroll.pageH,
|
||||
});
|
||||
|
||||
return {
|
||||
@@ -257,156 +247,18 @@ async function relationalQuery(
|
||||
);
|
||||
|
||||
const viewport = pickSnapshotViewport(rawPoints);
|
||||
const snapshot = await getRelationalSnapshot(rawQuery, {
|
||||
const snapshot = await ensureHeatmapSnapshot({
|
||||
websiteId,
|
||||
eventType,
|
||||
urlPath,
|
||||
startDate,
|
||||
endDate,
|
||||
pageW: null,
|
||||
pageH: null,
|
||||
viewportW: viewport?.width ?? null,
|
||||
viewportH: viewport?.height ?? null,
|
||||
filterContext,
|
||||
pageW: viewport?.pageW ?? null,
|
||||
pageH: viewport?.pageH ?? null,
|
||||
});
|
||||
|
||||
return { mode, pages, points: rawPoints, snapshot, scroll: emptyScroll() };
|
||||
}
|
||||
|
||||
async function getRelationalSnapshot(
|
||||
rawQuery: typeof prisma.rawQuery,
|
||||
{
|
||||
websiteId,
|
||||
eventType,
|
||||
urlPath,
|
||||
startDate,
|
||||
endDate,
|
||||
pageW,
|
||||
pageH,
|
||||
viewportW,
|
||||
viewportH,
|
||||
filterContext,
|
||||
}: {
|
||||
websiteId: string;
|
||||
eventType: number;
|
||||
urlPath: string;
|
||||
startDate: Date;
|
||||
endDate: Date;
|
||||
pageW: number | null;
|
||||
pageH: number | null;
|
||||
viewportW: number | null;
|
||||
viewportH: number | null;
|
||||
filterContext: HeatmapFilterContext;
|
||||
},
|
||||
): Promise<HeatmapSnapshot | null> {
|
||||
const pageFilter =
|
||||
pageW && pageH
|
||||
? `
|
||||
and h.page_w = {{pageW}}
|
||||
and h.page_h = {{pageH}}
|
||||
`
|
||||
: '';
|
||||
const viewportFilter =
|
||||
viewportW && viewportH
|
||||
? `
|
||||
and h.viewport_w = {{viewportW}}
|
||||
and h.viewport_h = {{viewportH}}
|
||||
`
|
||||
: '';
|
||||
|
||||
const rows: SnapshotRow[] = await rawQuery(
|
||||
`
|
||||
select
|
||||
h.visit_id as "replayId",
|
||||
coalesce(h.replay_time_ms, (extract(epoch from h.created_at) * 1000)::bigint) as "timestamp",
|
||||
h.replay_chunk_index as "chunkIndex",
|
||||
h.replay_event_index as "eventIndex"
|
||||
from heatmap_event h
|
||||
inner join (
|
||||
select distinct visit_id
|
||||
from session_replay
|
||||
where website_id = {{websiteId::uuid}}
|
||||
) sr on sr.visit_id = h.visit_id
|
||||
${filterContext.joinQuery}
|
||||
where h.website_id = {{websiteId::uuid}}
|
||||
and h.event_type = {{eventType}}
|
||||
and h.url_path = {{urlPath}}
|
||||
and h.created_at between {{startDate}} and {{endDate}}
|
||||
and h.replay_chunk_index is not null
|
||||
and h.replay_event_index is not null
|
||||
and h.replay_time_ms is not null
|
||||
${pageFilter}
|
||||
${viewportFilter}
|
||||
order by
|
||||
h.replay_chunk_index asc nulls last,
|
||||
h.replay_event_index asc nulls last,
|
||||
h.replay_time_ms asc,
|
||||
h.created_at asc
|
||||
limit 1
|
||||
`,
|
||||
{
|
||||
...filterContext.queryParams,
|
||||
websiteId,
|
||||
eventType,
|
||||
urlPath,
|
||||
startDate,
|
||||
endDate,
|
||||
pageW,
|
||||
pageH,
|
||||
viewportW,
|
||||
viewportH,
|
||||
},
|
||||
FUNCTION_NAME,
|
||||
);
|
||||
|
||||
if (rows.length > 0) {
|
||||
return mapSnapshot(rows[0]);
|
||||
}
|
||||
|
||||
const fallbackRows: SnapshotRow[] = await rawQuery(
|
||||
`
|
||||
select
|
||||
h.visit_id as "replayId",
|
||||
coalesce(h.replay_time_ms, (extract(epoch from h.created_at) * 1000)::bigint) as "timestamp",
|
||||
h.replay_chunk_index as "chunkIndex",
|
||||
h.replay_event_index as "eventIndex"
|
||||
from heatmap_event h
|
||||
inner join (
|
||||
select distinct visit_id
|
||||
from session_replay
|
||||
where website_id = {{websiteId::uuid}}
|
||||
) sr on sr.visit_id = h.visit_id
|
||||
${filterContext.joinQuery}
|
||||
where h.website_id = {{websiteId::uuid}}
|
||||
and h.event_type = {{eventType}}
|
||||
and h.url_path = {{urlPath}}
|
||||
and h.created_at between {{startDate}} and {{endDate}}
|
||||
${pageFilter}
|
||||
${viewportFilter}
|
||||
order by
|
||||
h.replay_chunk_index asc nulls last,
|
||||
h.replay_event_index asc nulls last,
|
||||
h.created_at asc
|
||||
limit 1
|
||||
`,
|
||||
{
|
||||
...filterContext.queryParams,
|
||||
websiteId,
|
||||
eventType,
|
||||
urlPath,
|
||||
startDate,
|
||||
endDate,
|
||||
pageW,
|
||||
pageH,
|
||||
viewportW,
|
||||
viewportH,
|
||||
},
|
||||
FUNCTION_NAME,
|
||||
);
|
||||
|
||||
return mapSnapshot(fallbackRows[0]);
|
||||
}
|
||||
|
||||
async function clickhouseQuery(
|
||||
websiteId: string,
|
||||
parameters: HeatmapParameters,
|
||||
@@ -451,11 +303,13 @@ async function clickhouseQuery(
|
||||
FUNCTION_NAME,
|
||||
);
|
||||
|
||||
const pages: HeatmapPage[] = pageRows.map(p => ({
|
||||
urlPath: p.urlPath,
|
||||
count: Number(p.count),
|
||||
sessions: Number(p.sessions),
|
||||
}));
|
||||
const pages: HeatmapPage[] = pageRows
|
||||
.map(p => ({
|
||||
urlPath: p.urlPath,
|
||||
count: Number(p.count),
|
||||
sessions: Number(p.sessions),
|
||||
}))
|
||||
.filter(page => !shouldSkipSnapshot(page.urlPath));
|
||||
|
||||
if (!urlPath) {
|
||||
return { mode, pages, points: [], snapshot: null, scroll: emptyScroll() };
|
||||
@@ -524,17 +378,13 @@ async function clickhouseQuery(
|
||||
viewportH:
|
||||
dim?.viewportH === null || dim?.viewportH === undefined ? null : Number(dim.viewportH),
|
||||
};
|
||||
const snapshot = await getClickhouseSnapshot(rawQuery, {
|
||||
const snapshot = await ensureHeatmapSnapshot({
|
||||
websiteId,
|
||||
eventType,
|
||||
urlPath,
|
||||
startDate,
|
||||
endDate,
|
||||
pageW: scroll.pageW,
|
||||
pageH: scroll.pageH,
|
||||
viewportW: scroll.viewportW,
|
||||
viewportH: scroll.viewportH,
|
||||
filterContext,
|
||||
pageW: scroll.pageW,
|
||||
pageH: scroll.pageH,
|
||||
});
|
||||
|
||||
return {
|
||||
@@ -617,156 +467,18 @@ async function clickhouseQuery(
|
||||
}));
|
||||
|
||||
const viewport = pickSnapshotViewport(points);
|
||||
const snapshot = await getClickhouseSnapshot(rawQuery, {
|
||||
const snapshot = await ensureHeatmapSnapshot({
|
||||
websiteId,
|
||||
eventType,
|
||||
urlPath,
|
||||
startDate,
|
||||
endDate,
|
||||
pageW: null,
|
||||
pageH: null,
|
||||
viewportW: viewport?.width ?? null,
|
||||
viewportH: viewport?.height ?? null,
|
||||
filterContext,
|
||||
pageW: viewport?.pageW ?? null,
|
||||
pageH: viewport?.pageH ?? null,
|
||||
});
|
||||
|
||||
return { mode, pages, points, snapshot, scroll: emptyScroll() };
|
||||
}
|
||||
|
||||
async function getClickhouseSnapshot(
|
||||
rawQuery: typeof clickhouse.rawQuery,
|
||||
{
|
||||
websiteId,
|
||||
eventType,
|
||||
urlPath,
|
||||
startDate,
|
||||
endDate,
|
||||
pageW,
|
||||
pageH,
|
||||
viewportW,
|
||||
viewportH,
|
||||
filterContext,
|
||||
}: {
|
||||
websiteId: string;
|
||||
eventType: number;
|
||||
urlPath: string;
|
||||
startDate: Date;
|
||||
endDate: Date;
|
||||
pageW: number | null;
|
||||
pageH: number | null;
|
||||
viewportW: number | null;
|
||||
viewportH: number | null;
|
||||
filterContext: HeatmapFilterContext;
|
||||
},
|
||||
): Promise<HeatmapSnapshot | null> {
|
||||
const pageFilter =
|
||||
pageW && pageH
|
||||
? `
|
||||
and h.page_w = {pageW:UInt32}
|
||||
and h.page_h = {pageH:UInt32}
|
||||
`
|
||||
: '';
|
||||
const viewportFilter =
|
||||
viewportW && viewportH
|
||||
? `
|
||||
and h.viewport_w = {viewportW:UInt32}
|
||||
and h.viewport_h = {viewportH:UInt32}
|
||||
`
|
||||
: '';
|
||||
|
||||
const rows = await rawQuery<SnapshotRow[]>(
|
||||
`
|
||||
select
|
||||
toString(h.visit_id) as replayId,
|
||||
ifNull(h.replay_time_ms, toInt64(toUnixTimestamp(h.created_at)) * 1000) as timestamp,
|
||||
h.replay_chunk_index as chunkIndex,
|
||||
h.replay_event_index as eventIndex
|
||||
from heatmap_event h
|
||||
inner join (
|
||||
select distinct visit_id
|
||||
from session_replay
|
||||
where website_id = {websiteId:UUID}
|
||||
) sr on sr.visit_id = h.visit_id
|
||||
${filterContext.joinQuery}
|
||||
where h.website_id = {websiteId:UUID}
|
||||
and h.event_type = {eventType:UInt8}
|
||||
and h.url_path = {urlPath:String}
|
||||
and h.created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
and h.replay_chunk_index is not null
|
||||
and h.replay_event_index is not null
|
||||
and h.replay_time_ms is not null
|
||||
${pageFilter}
|
||||
${viewportFilter}
|
||||
order by
|
||||
h.replay_chunk_index asc,
|
||||
h.replay_event_index asc,
|
||||
h.replay_time_ms asc,
|
||||
h.created_at asc
|
||||
limit 1
|
||||
`,
|
||||
{
|
||||
...filterContext.queryParams,
|
||||
websiteId,
|
||||
eventType,
|
||||
urlPath,
|
||||
startDate,
|
||||
endDate,
|
||||
pageW,
|
||||
pageH,
|
||||
viewportW,
|
||||
viewportH,
|
||||
},
|
||||
FUNCTION_NAME,
|
||||
);
|
||||
|
||||
if (rows.length > 0) {
|
||||
return mapSnapshot(rows[0]);
|
||||
}
|
||||
|
||||
const fallbackRows = await rawQuery<SnapshotRow[]>(
|
||||
`
|
||||
select
|
||||
toString(h.visit_id) as replayId,
|
||||
ifNull(h.replay_time_ms, toInt64(toUnixTimestamp(h.created_at)) * 1000) as timestamp,
|
||||
h.replay_chunk_index as chunkIndex,
|
||||
h.replay_event_index as eventIndex
|
||||
from heatmap_event h
|
||||
inner join (
|
||||
select distinct visit_id
|
||||
from session_replay
|
||||
where website_id = {websiteId:UUID}
|
||||
) sr on sr.visit_id = h.visit_id
|
||||
${filterContext.joinQuery}
|
||||
where h.website_id = {websiteId:UUID}
|
||||
and h.event_type = {eventType:UInt8}
|
||||
and h.url_path = {urlPath:String}
|
||||
and h.created_at between {startDate:DateTime64} and {endDate:DateTime64}
|
||||
${pageFilter}
|
||||
${viewportFilter}
|
||||
order by
|
||||
h.replay_chunk_index asc,
|
||||
h.replay_event_index asc,
|
||||
h.created_at asc
|
||||
limit 1
|
||||
`,
|
||||
{
|
||||
...filterContext.queryParams,
|
||||
websiteId,
|
||||
eventType,
|
||||
urlPath,
|
||||
startDate,
|
||||
endDate,
|
||||
pageW,
|
||||
pageH,
|
||||
viewportW,
|
||||
viewportH,
|
||||
},
|
||||
FUNCTION_NAME,
|
||||
);
|
||||
|
||||
return mapSnapshot(fallbackRows[0]);
|
||||
}
|
||||
|
||||
function emptyScroll(): HeatmapResult['scroll'] {
|
||||
return {
|
||||
buckets: [],
|
||||
@@ -839,21 +551,6 @@ function pickSnapshotViewport(
|
||||
};
|
||||
}
|
||||
|
||||
function mapSnapshot(row?: SnapshotRow | null): HeatmapSnapshot | null {
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
replayId: row.replayId,
|
||||
timestamp: Number(row.timestamp),
|
||||
chunkIndex:
|
||||
row.chunkIndex === null || row.chunkIndex === undefined ? null : Number(row.chunkIndex),
|
||||
eventIndex:
|
||||
row.eventIndex === null || row.eventIndex === undefined ? null : Number(row.eventIndex),
|
||||
};
|
||||
}
|
||||
|
||||
function getRelationalHeatmapFilterContext(
|
||||
websiteId: string,
|
||||
filters: QueryFilters,
|
||||
|
||||
@@ -29,12 +29,38 @@ export interface HeatmapEventRow {
|
||||
export async function saveHeatmapEvents(rows: HeatmapEventRow[]) {
|
||||
if (!rows?.length) return;
|
||||
|
||||
const normalizedRows = rows.map(r => ({
|
||||
...r,
|
||||
nodeId: toInt(r.nodeId),
|
||||
x: toInt(r.x),
|
||||
y: toInt(r.y),
|
||||
pageX: toInt(r.pageX),
|
||||
pageY: toInt(r.pageY),
|
||||
pageW: toInt(r.pageW),
|
||||
viewportW: toInt(r.viewportW),
|
||||
viewportH: toInt(r.viewportH),
|
||||
pageH: toInt(r.pageH),
|
||||
scrollPct: toScrollPct(r.scrollPct),
|
||||
}));
|
||||
|
||||
return runQuery({
|
||||
[PRISMA]: () => relationalQuery(rows),
|
||||
[CLICKHOUSE]: () => clickhouseQuery(rows),
|
||||
[PRISMA]: () => relationalQuery(normalizedRows),
|
||||
[CLICKHOUSE]: () => clickhouseQuery(normalizedRows),
|
||||
});
|
||||
}
|
||||
|
||||
function toInt(value: number | null) {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? Math.round(value) : null;
|
||||
}
|
||||
|
||||
function toScrollPct(value: number | null) {
|
||||
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Math.max(0, Math.min(100, Math.round(value)));
|
||||
}
|
||||
|
||||
async function relationalQuery(rows: HeatmapEventRow[]) {
|
||||
return prisma.client.heatmapEvent.createMany({
|
||||
data: rows.map(r => ({
|
||||
|
||||
Reference in new issue
Block a user