fix: harden analytics writes and hide internal server errors from client

This commit is contained in:
Francis Cao committed 2026-06-22 22:07:07 -07:00
1 parent 770b4d59d2
commit 129681ebd7
15 files changed
+217 -128

No files matched your search

+59 -66
View File
@@ -1,14 +1,8 @@
import clickhouse from '@/lib/clickhouse';
import {
EVENT_NAME_LENGTH,
FIELD_VALUE_LENGTH,
HOSTNAME_LENGTH,
PAGE_TITLE_LENGTH,
TAG_LENGTH,
URL_LENGTH,
} from '@/lib/constants';
import { FIELD_LENGTH } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { truncateString } from '@/lib/format';
import kafka from '@/lib/kafka';
import prisma from '@/lib/prisma';
import { saveEventData } from './saveEventData';
@@ -76,10 +70,6 @@ export async function saveEvent(args: SaveEventArgs) {
});
}
function truncate(value: string | null | undefined, maxLength: number) {
return value ? value.substring(0, maxLength) : value;
}
async function relationalQuery({
websiteId,
sessionId,
@@ -121,27 +111,27 @@ async function relationalQuery({
websiteId,
sessionId,
visitId,
urlPath: truncate(urlPath, URL_LENGTH),
urlQuery: truncate(urlQuery, URL_LENGTH),
utmSource: truncate(utmSource, FIELD_VALUE_LENGTH),
utmMedium: truncate(utmMedium, FIELD_VALUE_LENGTH),
utmCampaign: truncate(utmCampaign, FIELD_VALUE_LENGTH),
utmContent: truncate(utmContent, FIELD_VALUE_LENGTH),
utmTerm: truncate(utmTerm, FIELD_VALUE_LENGTH),
referrerPath: truncate(referrerPath, URL_LENGTH),
referrerQuery: truncate(referrerQuery, URL_LENGTH),
referrerDomain: truncate(referrerDomain, URL_LENGTH),
pageTitle: truncate(pageTitle, PAGE_TITLE_LENGTH),
gclid: truncate(gclid, FIELD_VALUE_LENGTH),
fbclid: truncate(fbclid, FIELD_VALUE_LENGTH),
msclkid: truncate(msclkid, FIELD_VALUE_LENGTH),
ttclid: truncate(ttclid, FIELD_VALUE_LENGTH),
lifatid: truncate(lifatid, FIELD_VALUE_LENGTH),
twclid: truncate(twclid, FIELD_VALUE_LENGTH),
urlPath: truncateString(urlPath, FIELD_LENGTH.url),
urlQuery: truncateString(urlQuery, FIELD_LENGTH.url),
utmSource: truncateString(utmSource, FIELD_LENGTH.fieldValue),
utmMedium: truncateString(utmMedium, FIELD_LENGTH.fieldValue),
utmCampaign: truncateString(utmCampaign, FIELD_LENGTH.fieldValue),
utmContent: truncateString(utmContent, FIELD_LENGTH.fieldValue),
utmTerm: truncateString(utmTerm, FIELD_LENGTH.fieldValue),
referrerPath: truncateString(referrerPath, FIELD_LENGTH.url),
referrerQuery: truncateString(referrerQuery, FIELD_LENGTH.url),
referrerDomain: truncateString(referrerDomain, FIELD_LENGTH.url),
pageTitle: truncateString(pageTitle, FIELD_LENGTH.pageTitle),
gclid: truncateString(gclid, FIELD_LENGTH.fieldValue),
fbclid: truncateString(fbclid, FIELD_LENGTH.fieldValue),
msclkid: truncateString(msclkid, FIELD_LENGTH.fieldValue),
ttclid: truncateString(ttclid, FIELD_LENGTH.fieldValue),
lifatid: truncateString(lifatid, FIELD_LENGTH.fieldValue),
twclid: truncateString(twclid, FIELD_LENGTH.fieldValue),
eventType,
eventName: truncate(eventName, EVENT_NAME_LENGTH) ?? null,
tag: truncate(tag, TAG_LENGTH),
hostname: truncate(hostname, HOSTNAME_LENGTH),
eventName: truncateString(eventName, FIELD_LENGTH.eventName) ?? null,
tag: truncateString(tag, FIELD_LENGTH.tag),
hostname: truncateString(hostname, FIELD_LENGTH.hostname),
lcp,
inp,
cls,
@@ -156,8 +146,8 @@ async function relationalQuery({
websiteId,
sessionId,
eventId: websiteEventId,
urlPath: truncate(urlPath, URL_LENGTH),
eventName: truncate(eventName, EVENT_NAME_LENGTH),
urlPath: truncateString(urlPath, FIELD_LENGTH.url),
eventName: truncateString(eventName, FIELD_LENGTH.eventName),
eventData,
createdAt,
});
@@ -169,7 +159,7 @@ async function relationalQuery({
websiteId,
sessionId,
eventId: websiteEventId,
eventName: truncate(eventName, EVENT_NAME_LENGTH),
eventName: truncateString(eventName, FIELD_LENGTH.eventName),
currency,
revenue,
createdAt,
@@ -229,37 +219,40 @@ async function clickhouseQuery({
session_id: sessionId,
visit_id: visitId,
event_id: eventId,
country: country,
region: country && region ? (region.includes('-') ? region : `${country}-${region}`) : null,
city: city,
url_path: truncate(urlPath, URL_LENGTH),
url_query: truncate(urlQuery, URL_LENGTH),
utm_source: truncate(utmSource, FIELD_VALUE_LENGTH),
utm_medium: truncate(utmMedium, FIELD_VALUE_LENGTH),
utm_campaign: truncate(utmCampaign, FIELD_VALUE_LENGTH),
utm_content: truncate(utmContent, FIELD_VALUE_LENGTH),
utm_term: truncate(utmTerm, FIELD_VALUE_LENGTH),
referrer_path: truncate(referrerPath, URL_LENGTH),
referrer_query: truncate(referrerQuery, URL_LENGTH),
referrer_domain: truncate(referrerDomain, URL_LENGTH),
page_title: truncate(pageTitle, PAGE_TITLE_LENGTH),
gclid: truncate(gclid, FIELD_VALUE_LENGTH),
fbclid: truncate(fbclid, FIELD_VALUE_LENGTH),
msclkid: truncate(msclkid, FIELD_VALUE_LENGTH),
ttclid: truncate(ttclid, FIELD_VALUE_LENGTH),
li_fat_id: truncate(lifatid, FIELD_VALUE_LENGTH),
twclid: truncate(twclid, FIELD_VALUE_LENGTH),
region: truncateString(
country && region ? (region.includes('-') ? region : `${country}-${region}`) : null,
FIELD_LENGTH.region,
),
city: truncateString(city, FIELD_LENGTH.city),
url_path: truncateString(urlPath, FIELD_LENGTH.url),
url_query: truncateString(urlQuery, FIELD_LENGTH.url),
utm_source: truncateString(utmSource, FIELD_LENGTH.fieldValue),
utm_medium: truncateString(utmMedium, FIELD_LENGTH.fieldValue),
utm_campaign: truncateString(utmCampaign, FIELD_LENGTH.fieldValue),
utm_content: truncateString(utmContent, FIELD_LENGTH.fieldValue),
utm_term: truncateString(utmTerm, FIELD_LENGTH.fieldValue),
referrer_path: truncateString(referrerPath, FIELD_LENGTH.url),
referrer_query: truncateString(referrerQuery, FIELD_LENGTH.url),
referrer_domain: truncateString(referrerDomain, FIELD_LENGTH.url),
page_title: truncateString(pageTitle, FIELD_LENGTH.pageTitle),
gclid: truncateString(gclid, FIELD_LENGTH.fieldValue),
fbclid: truncateString(fbclid, FIELD_LENGTH.fieldValue),
msclkid: truncateString(msclkid, FIELD_LENGTH.fieldValue),
ttclid: truncateString(ttclid, FIELD_LENGTH.fieldValue),
li_fat_id: truncateString(lifatid, FIELD_LENGTH.fieldValue),
twclid: truncateString(twclid, FIELD_LENGTH.fieldValue),
event_type: eventType,
event_name: truncate(eventName, EVENT_NAME_LENGTH) ?? null,
tag: truncate(tag, TAG_LENGTH),
distinct_id: distinctId,
event_name: truncateString(eventName, FIELD_LENGTH.eventName) ?? null,
tag: truncateString(tag, FIELD_LENGTH.tag),
distinct_id: truncateString(distinctId, FIELD_LENGTH.distinctId),
created_at: getUTCString(createdAt),
browser: browser,
os: os,
device: device,
screen: screen,
language: language,
hostname: truncate(hostname, HOSTNAME_LENGTH),
browser: truncateString(browser, FIELD_LENGTH.browser),
os: truncateString(os, FIELD_LENGTH.os),
device: truncateString(device, FIELD_LENGTH.device),
screen: truncateString(screen, FIELD_LENGTH.screen),
language: truncateString(language, FIELD_LENGTH.language),
hostname: truncateString(hostname, FIELD_LENGTH.hostname),
country: truncateString(country, FIELD_LENGTH.country),
lcp: lcp,
inp: inp,
cls: cls,
@@ -278,8 +271,8 @@ async function clickhouseQuery({
websiteId,
sessionId,
eventId,
urlPath: truncate(urlPath, URL_LENGTH),
eventName: truncate(eventName, EVENT_NAME_LENGTH),
urlPath: truncateString(urlPath, FIELD_LENGTH.url),
eventName: truncateString(eventName, FIELD_LENGTH.eventName),
eventData,
createdAt,
});
+9 -8
View File
@@ -1,8 +1,9 @@
import clickhouse from '@/lib/clickhouse';
import { DATA_TYPE } from '@/lib/constants';
import { DATA_TYPE, FIELD_LENGTH } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { flattenJSON, getStringValue } from '@/lib/data';
import { flattenJSON, getStoredStringValue } from '@/lib/data';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { truncateString } from '@/lib/format';
import kafka from '@/lib/kafka';
import prisma from '@/lib/prisma';
import type { DynamicData } from '@/lib/types';
@@ -34,8 +35,8 @@ async function relationalQuery(data: SaveEventDataArgs) {
id: uuid(),
websiteEventId: eventId,
websiteId,
dataKey: a.key,
stringValue: getStringValue(a.value, a.dataType),
dataKey: truncateString(a.key, FIELD_LENGTH.dataKey),
stringValue: getStoredStringValue(a.value, a.dataType),
numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
dataType: a.dataType,
@@ -60,11 +61,11 @@ async function clickhouseQuery(data: SaveEventDataArgs) {
website_id: websiteId,
session_id: sessionId,
event_id: eventId,
url_path: urlPath,
event_name: eventName,
data_key: key,
url_path: truncateString(urlPath, FIELD_LENGTH.url),
event_name: truncateString(eventName, FIELD_LENGTH.eventName),
data_key: truncateString(key, FIELD_LENGTH.dataKey),
data_type: dataType,
string_value: getStringValue(value, dataType),
string_value: getStoredStringValue(value, dataType),
number_value: dataType === DATA_TYPE.number ? value : null,
date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
created_at: getUTCString(createdAt),
+4 -2
View File
@@ -1,5 +1,7 @@
import { FIELD_LENGTH } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { PRISMA, runQuery } from '@/lib/db';
import { truncateString } from '@/lib/format';
import prisma from '@/lib/prisma';
export interface SaveRevenueArgs {
@@ -27,8 +29,8 @@ async function relationalQuery(data: SaveRevenueArgs) {
websiteId,
sessionId,
eventId,
eventName,
currency,
eventName: truncateString(eventName, FIELD_LENGTH.eventName),
currency: truncateString(currency, FIELD_LENGTH.currency),
revenue,
createdAt,
},
+15 -1
View File
@@ -1,10 +1,24 @@
import type { Prisma } from '@/generated/prisma/client';
import { FIELD_LENGTH } from '@/lib/constants';
import { truncateString } from '@/lib/format';
import prisma from '@/lib/prisma';
const FUNCTION_NAME = 'createSession';
export async function createSession(data: Prisma.SessionCreateInput) {
const { rawQuery } = prisma;
const normalizedData: Prisma.SessionCreateInput = {
...data,
browser: truncateString(data.browser, FIELD_LENGTH.browser),
os: truncateString(data.os, FIELD_LENGTH.os),
device: truncateString(data.device, FIELD_LENGTH.device),
screen: truncateString(data.screen, FIELD_LENGTH.screen),
language: truncateString(data.language, FIELD_LENGTH.language),
country: truncateString(data.country, FIELD_LENGTH.country),
region: truncateString(data.region, FIELD_LENGTH.region),
city: truncateString(data.city, FIELD_LENGTH.city),
distinctId: truncateString(data.distinctId, FIELD_LENGTH.distinctId),
};
await rawQuery(
`
@@ -38,7 +52,7 @@ export async function createSession(data: Prisma.SessionCreateInput) {
)
on conflict (session_id) do nothing
`,
data,
normalizedData,
FUNCTION_NAME,
);
}
+11 -8
View File
@@ -1,8 +1,9 @@
import clickhouse from '@/lib/clickhouse';
import { DATA_TYPE } from '@/lib/constants';
import { DATA_TYPE, FIELD_LENGTH } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { flattenJSON, getStringValue } from '@/lib/data';
import { flattenJSON, getStoredStringValue } from '@/lib/data';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { truncateString } from '@/lib/format';
import kafka from '@/lib/kafka';
import prisma from '@/lib/prisma';
import type { DynamicData } from '@/lib/types';
@@ -32,17 +33,18 @@ export async function relationalQuery({
const { client } = prisma;
const jsonKeys = flattenJSON(sessionData);
const normalizedDistinctId = truncateString(distinctId, FIELD_LENGTH.distinctId);
const flattenedData = jsonKeys.map(a => ({
id: uuid(),
websiteId,
sessionId,
dataKey: a.key,
stringValue: getStringValue(a.value, a.dataType),
dataKey: truncateString(a.key, FIELD_LENGTH.dataKey),
stringValue: getStoredStringValue(a.value, a.dataType),
numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
dataType: a.dataType,
distinctId,
distinctId: normalizedDistinctId,
createdAt,
}));
@@ -79,17 +81,18 @@ async function clickhouseQuery({
const { sendMessage } = kafka;
const jsonKeys = flattenJSON(sessionData);
const normalizedDistinctId = truncateString(distinctId, FIELD_LENGTH.distinctId);
const messages = jsonKeys.map(({ key, value, dataType }) => {
return {
website_id: websiteId,
session_id: sessionId,
data_key: key,
data_key: truncateString(key, FIELD_LENGTH.dataKey),
data_type: dataType,
string_value: getStringValue(value, dataType),
string_value: getStoredStringValue(value, dataType),
number_value: dataType === DATA_TYPE.number ? value : null,
date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
distinct_id: distinctId,
distinct_id: normalizedDistinctId,
created_at: getUTCString(createdAt),
};
});