fix: harden analytics writes and hide internal server errors from client
This commit is contained in:
1 parent
770b4d59d2
commit
129681ebd7
15 files changed
+217
-128
No files matched your search
@@ -1,14 +1,8 @@
|
||||
import clickhouse from '@/lib/clickhouse';
|
||||
import {
|
||||
EVENT_NAME_LENGTH,
|
||||
FIELD_VALUE_LENGTH,
|
||||
HOSTNAME_LENGTH,
|
||||
PAGE_TITLE_LENGTH,
|
||||
TAG_LENGTH,
|
||||
URL_LENGTH,
|
||||
} from '@/lib/constants';
|
||||
import { FIELD_LENGTH } from '@/lib/constants';
|
||||
import { uuid } from '@/lib/crypto';
|
||||
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
|
||||
import { truncateString } from '@/lib/format';
|
||||
import kafka from '@/lib/kafka';
|
||||
import prisma from '@/lib/prisma';
|
||||
import { saveEventData } from './saveEventData';
|
||||
@@ -76,10 +70,6 @@ export async function saveEvent(args: SaveEventArgs) {
|
||||
});
|
||||
}
|
||||
|
||||
function truncate(value: string | null | undefined, maxLength: number) {
|
||||
return value ? value.substring(0, maxLength) : value;
|
||||
}
|
||||
|
||||
async function relationalQuery({
|
||||
websiteId,
|
||||
sessionId,
|
||||
@@ -121,27 +111,27 @@ async function relationalQuery({
|
||||
websiteId,
|
||||
sessionId,
|
||||
visitId,
|
||||
urlPath: truncate(urlPath, URL_LENGTH),
|
||||
urlQuery: truncate(urlQuery, URL_LENGTH),
|
||||
utmSource: truncate(utmSource, FIELD_VALUE_LENGTH),
|
||||
utmMedium: truncate(utmMedium, FIELD_VALUE_LENGTH),
|
||||
utmCampaign: truncate(utmCampaign, FIELD_VALUE_LENGTH),
|
||||
utmContent: truncate(utmContent, FIELD_VALUE_LENGTH),
|
||||
utmTerm: truncate(utmTerm, FIELD_VALUE_LENGTH),
|
||||
referrerPath: truncate(referrerPath, URL_LENGTH),
|
||||
referrerQuery: truncate(referrerQuery, URL_LENGTH),
|
||||
referrerDomain: truncate(referrerDomain, URL_LENGTH),
|
||||
pageTitle: truncate(pageTitle, PAGE_TITLE_LENGTH),
|
||||
gclid: truncate(gclid, FIELD_VALUE_LENGTH),
|
||||
fbclid: truncate(fbclid, FIELD_VALUE_LENGTH),
|
||||
msclkid: truncate(msclkid, FIELD_VALUE_LENGTH),
|
||||
ttclid: truncate(ttclid, FIELD_VALUE_LENGTH),
|
||||
lifatid: truncate(lifatid, FIELD_VALUE_LENGTH),
|
||||
twclid: truncate(twclid, FIELD_VALUE_LENGTH),
|
||||
urlPath: truncateString(urlPath, FIELD_LENGTH.url),
|
||||
urlQuery: truncateString(urlQuery, FIELD_LENGTH.url),
|
||||
utmSource: truncateString(utmSource, FIELD_LENGTH.fieldValue),
|
||||
utmMedium: truncateString(utmMedium, FIELD_LENGTH.fieldValue),
|
||||
utmCampaign: truncateString(utmCampaign, FIELD_LENGTH.fieldValue),
|
||||
utmContent: truncateString(utmContent, FIELD_LENGTH.fieldValue),
|
||||
utmTerm: truncateString(utmTerm, FIELD_LENGTH.fieldValue),
|
||||
referrerPath: truncateString(referrerPath, FIELD_LENGTH.url),
|
||||
referrerQuery: truncateString(referrerQuery, FIELD_LENGTH.url),
|
||||
referrerDomain: truncateString(referrerDomain, FIELD_LENGTH.url),
|
||||
pageTitle: truncateString(pageTitle, FIELD_LENGTH.pageTitle),
|
||||
gclid: truncateString(gclid, FIELD_LENGTH.fieldValue),
|
||||
fbclid: truncateString(fbclid, FIELD_LENGTH.fieldValue),
|
||||
msclkid: truncateString(msclkid, FIELD_LENGTH.fieldValue),
|
||||
ttclid: truncateString(ttclid, FIELD_LENGTH.fieldValue),
|
||||
lifatid: truncateString(lifatid, FIELD_LENGTH.fieldValue),
|
||||
twclid: truncateString(twclid, FIELD_LENGTH.fieldValue),
|
||||
eventType,
|
||||
eventName: truncate(eventName, EVENT_NAME_LENGTH) ?? null,
|
||||
tag: truncate(tag, TAG_LENGTH),
|
||||
hostname: truncate(hostname, HOSTNAME_LENGTH),
|
||||
eventName: truncateString(eventName, FIELD_LENGTH.eventName) ?? null,
|
||||
tag: truncateString(tag, FIELD_LENGTH.tag),
|
||||
hostname: truncateString(hostname, FIELD_LENGTH.hostname),
|
||||
lcp,
|
||||
inp,
|
||||
cls,
|
||||
@@ -156,8 +146,8 @@ async function relationalQuery({
|
||||
websiteId,
|
||||
sessionId,
|
||||
eventId: websiteEventId,
|
||||
urlPath: truncate(urlPath, URL_LENGTH),
|
||||
eventName: truncate(eventName, EVENT_NAME_LENGTH),
|
||||
urlPath: truncateString(urlPath, FIELD_LENGTH.url),
|
||||
eventName: truncateString(eventName, FIELD_LENGTH.eventName),
|
||||
eventData,
|
||||
createdAt,
|
||||
});
|
||||
@@ -169,7 +159,7 @@ async function relationalQuery({
|
||||
websiteId,
|
||||
sessionId,
|
||||
eventId: websiteEventId,
|
||||
eventName: truncate(eventName, EVENT_NAME_LENGTH),
|
||||
eventName: truncateString(eventName, FIELD_LENGTH.eventName),
|
||||
currency,
|
||||
revenue,
|
||||
createdAt,
|
||||
@@ -229,37 +219,40 @@ async function clickhouseQuery({
|
||||
session_id: sessionId,
|
||||
visit_id: visitId,
|
||||
event_id: eventId,
|
||||
country: country,
|
||||
region: country && region ? (region.includes('-') ? region : `${country}-${region}`) : null,
|
||||
city: city,
|
||||
url_path: truncate(urlPath, URL_LENGTH),
|
||||
url_query: truncate(urlQuery, URL_LENGTH),
|
||||
utm_source: truncate(utmSource, FIELD_VALUE_LENGTH),
|
||||
utm_medium: truncate(utmMedium, FIELD_VALUE_LENGTH),
|
||||
utm_campaign: truncate(utmCampaign, FIELD_VALUE_LENGTH),
|
||||
utm_content: truncate(utmContent, FIELD_VALUE_LENGTH),
|
||||
utm_term: truncate(utmTerm, FIELD_VALUE_LENGTH),
|
||||
referrer_path: truncate(referrerPath, URL_LENGTH),
|
||||
referrer_query: truncate(referrerQuery, URL_LENGTH),
|
||||
referrer_domain: truncate(referrerDomain, URL_LENGTH),
|
||||
page_title: truncate(pageTitle, PAGE_TITLE_LENGTH),
|
||||
gclid: truncate(gclid, FIELD_VALUE_LENGTH),
|
||||
fbclid: truncate(fbclid, FIELD_VALUE_LENGTH),
|
||||
msclkid: truncate(msclkid, FIELD_VALUE_LENGTH),
|
||||
ttclid: truncate(ttclid, FIELD_VALUE_LENGTH),
|
||||
li_fat_id: truncate(lifatid, FIELD_VALUE_LENGTH),
|
||||
twclid: truncate(twclid, FIELD_VALUE_LENGTH),
|
||||
region: truncateString(
|
||||
country && region ? (region.includes('-') ? region : `${country}-${region}`) : null,
|
||||
FIELD_LENGTH.region,
|
||||
),
|
||||
city: truncateString(city, FIELD_LENGTH.city),
|
||||
url_path: truncateString(urlPath, FIELD_LENGTH.url),
|
||||
url_query: truncateString(urlQuery, FIELD_LENGTH.url),
|
||||
utm_source: truncateString(utmSource, FIELD_LENGTH.fieldValue),
|
||||
utm_medium: truncateString(utmMedium, FIELD_LENGTH.fieldValue),
|
||||
utm_campaign: truncateString(utmCampaign, FIELD_LENGTH.fieldValue),
|
||||
utm_content: truncateString(utmContent, FIELD_LENGTH.fieldValue),
|
||||
utm_term: truncateString(utmTerm, FIELD_LENGTH.fieldValue),
|
||||
referrer_path: truncateString(referrerPath, FIELD_LENGTH.url),
|
||||
referrer_query: truncateString(referrerQuery, FIELD_LENGTH.url),
|
||||
referrer_domain: truncateString(referrerDomain, FIELD_LENGTH.url),
|
||||
page_title: truncateString(pageTitle, FIELD_LENGTH.pageTitle),
|
||||
gclid: truncateString(gclid, FIELD_LENGTH.fieldValue),
|
||||
fbclid: truncateString(fbclid, FIELD_LENGTH.fieldValue),
|
||||
msclkid: truncateString(msclkid, FIELD_LENGTH.fieldValue),
|
||||
ttclid: truncateString(ttclid, FIELD_LENGTH.fieldValue),
|
||||
li_fat_id: truncateString(lifatid, FIELD_LENGTH.fieldValue),
|
||||
twclid: truncateString(twclid, FIELD_LENGTH.fieldValue),
|
||||
event_type: eventType,
|
||||
event_name: truncate(eventName, EVENT_NAME_LENGTH) ?? null,
|
||||
tag: truncate(tag, TAG_LENGTH),
|
||||
distinct_id: distinctId,
|
||||
event_name: truncateString(eventName, FIELD_LENGTH.eventName) ?? null,
|
||||
tag: truncateString(tag, FIELD_LENGTH.tag),
|
||||
distinct_id: truncateString(distinctId, FIELD_LENGTH.distinctId),
|
||||
created_at: getUTCString(createdAt),
|
||||
browser: browser,
|
||||
os: os,
|
||||
device: device,
|
||||
screen: screen,
|
||||
language: language,
|
||||
hostname: truncate(hostname, HOSTNAME_LENGTH),
|
||||
browser: truncateString(browser, FIELD_LENGTH.browser),
|
||||
os: truncateString(os, FIELD_LENGTH.os),
|
||||
device: truncateString(device, FIELD_LENGTH.device),
|
||||
screen: truncateString(screen, FIELD_LENGTH.screen),
|
||||
language: truncateString(language, FIELD_LENGTH.language),
|
||||
hostname: truncateString(hostname, FIELD_LENGTH.hostname),
|
||||
country: truncateString(country, FIELD_LENGTH.country),
|
||||
lcp: lcp,
|
||||
inp: inp,
|
||||
cls: cls,
|
||||
@@ -278,8 +271,8 @@ async function clickhouseQuery({
|
||||
websiteId,
|
||||
sessionId,
|
||||
eventId,
|
||||
urlPath: truncate(urlPath, URL_LENGTH),
|
||||
eventName: truncate(eventName, EVENT_NAME_LENGTH),
|
||||
urlPath: truncateString(urlPath, FIELD_LENGTH.url),
|
||||
eventName: truncateString(eventName, FIELD_LENGTH.eventName),
|
||||
eventData,
|
||||
createdAt,
|
||||
});
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import clickhouse from '@/lib/clickhouse';
|
||||
import { DATA_TYPE } from '@/lib/constants';
|
||||
import { DATA_TYPE, FIELD_LENGTH } from '@/lib/constants';
|
||||
import { uuid } from '@/lib/crypto';
|
||||
import { flattenJSON, getStringValue } from '@/lib/data';
|
||||
import { flattenJSON, getStoredStringValue } from '@/lib/data';
|
||||
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
|
||||
import { truncateString } from '@/lib/format';
|
||||
import kafka from '@/lib/kafka';
|
||||
import prisma from '@/lib/prisma';
|
||||
import type { DynamicData } from '@/lib/types';
|
||||
@@ -34,8 +35,8 @@ async function relationalQuery(data: SaveEventDataArgs) {
|
||||
id: uuid(),
|
||||
websiteEventId: eventId,
|
||||
websiteId,
|
||||
dataKey: a.key,
|
||||
stringValue: getStringValue(a.value, a.dataType),
|
||||
dataKey: truncateString(a.key, FIELD_LENGTH.dataKey),
|
||||
stringValue: getStoredStringValue(a.value, a.dataType),
|
||||
numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
|
||||
dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
|
||||
dataType: a.dataType,
|
||||
@@ -60,11 +61,11 @@ async function clickhouseQuery(data: SaveEventDataArgs) {
|
||||
website_id: websiteId,
|
||||
session_id: sessionId,
|
||||
event_id: eventId,
|
||||
url_path: urlPath,
|
||||
event_name: eventName,
|
||||
data_key: key,
|
||||
url_path: truncateString(urlPath, FIELD_LENGTH.url),
|
||||
event_name: truncateString(eventName, FIELD_LENGTH.eventName),
|
||||
data_key: truncateString(key, FIELD_LENGTH.dataKey),
|
||||
data_type: dataType,
|
||||
string_value: getStringValue(value, dataType),
|
||||
string_value: getStoredStringValue(value, dataType),
|
||||
number_value: dataType === DATA_TYPE.number ? value : null,
|
||||
date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
|
||||
created_at: getUTCString(createdAt),
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { FIELD_LENGTH } from '@/lib/constants';
|
||||
import { uuid } from '@/lib/crypto';
|
||||
import { PRISMA, runQuery } from '@/lib/db';
|
||||
import { truncateString } from '@/lib/format';
|
||||
import prisma from '@/lib/prisma';
|
||||
|
||||
export interface SaveRevenueArgs {
|
||||
@@ -27,8 +29,8 @@ async function relationalQuery(data: SaveRevenueArgs) {
|
||||
websiteId,
|
||||
sessionId,
|
||||
eventId,
|
||||
eventName,
|
||||
currency,
|
||||
eventName: truncateString(eventName, FIELD_LENGTH.eventName),
|
||||
currency: truncateString(currency, FIELD_LENGTH.currency),
|
||||
revenue,
|
||||
createdAt,
|
||||
},
|
||||
|
||||
@@ -1,10 +1,24 @@
|
||||
import type { Prisma } from '@/generated/prisma/client';
|
||||
import { FIELD_LENGTH } from '@/lib/constants';
|
||||
import { truncateString } from '@/lib/format';
|
||||
import prisma from '@/lib/prisma';
|
||||
|
||||
const FUNCTION_NAME = 'createSession';
|
||||
|
||||
export async function createSession(data: Prisma.SessionCreateInput) {
|
||||
const { rawQuery } = prisma;
|
||||
const normalizedData: Prisma.SessionCreateInput = {
|
||||
...data,
|
||||
browser: truncateString(data.browser, FIELD_LENGTH.browser),
|
||||
os: truncateString(data.os, FIELD_LENGTH.os),
|
||||
device: truncateString(data.device, FIELD_LENGTH.device),
|
||||
screen: truncateString(data.screen, FIELD_LENGTH.screen),
|
||||
language: truncateString(data.language, FIELD_LENGTH.language),
|
||||
country: truncateString(data.country, FIELD_LENGTH.country),
|
||||
region: truncateString(data.region, FIELD_LENGTH.region),
|
||||
city: truncateString(data.city, FIELD_LENGTH.city),
|
||||
distinctId: truncateString(data.distinctId, FIELD_LENGTH.distinctId),
|
||||
};
|
||||
|
||||
await rawQuery(
|
||||
`
|
||||
@@ -38,7 +52,7 @@ export async function createSession(data: Prisma.SessionCreateInput) {
|
||||
)
|
||||
on conflict (session_id) do nothing
|
||||
`,
|
||||
data,
|
||||
normalizedData,
|
||||
FUNCTION_NAME,
|
||||
);
|
||||
}
|
||||
@@ -1,8 +1,9 @@
|
||||
import clickhouse from '@/lib/clickhouse';
|
||||
import { DATA_TYPE } from '@/lib/constants';
|
||||
import { DATA_TYPE, FIELD_LENGTH } from '@/lib/constants';
|
||||
import { uuid } from '@/lib/crypto';
|
||||
import { flattenJSON, getStringValue } from '@/lib/data';
|
||||
import { flattenJSON, getStoredStringValue } from '@/lib/data';
|
||||
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
|
||||
import { truncateString } from '@/lib/format';
|
||||
import kafka from '@/lib/kafka';
|
||||
import prisma from '@/lib/prisma';
|
||||
import type { DynamicData } from '@/lib/types';
|
||||
@@ -32,17 +33,18 @@ export async function relationalQuery({
|
||||
const { client } = prisma;
|
||||
|
||||
const jsonKeys = flattenJSON(sessionData);
|
||||
const normalizedDistinctId = truncateString(distinctId, FIELD_LENGTH.distinctId);
|
||||
|
||||
const flattenedData = jsonKeys.map(a => ({
|
||||
id: uuid(),
|
||||
websiteId,
|
||||
sessionId,
|
||||
dataKey: a.key,
|
||||
stringValue: getStringValue(a.value, a.dataType),
|
||||
dataKey: truncateString(a.key, FIELD_LENGTH.dataKey),
|
||||
stringValue: getStoredStringValue(a.value, a.dataType),
|
||||
numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
|
||||
dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
|
||||
dataType: a.dataType,
|
||||
distinctId,
|
||||
distinctId: normalizedDistinctId,
|
||||
createdAt,
|
||||
}));
|
||||
|
||||
@@ -79,17 +81,18 @@ async function clickhouseQuery({
|
||||
const { sendMessage } = kafka;
|
||||
|
||||
const jsonKeys = flattenJSON(sessionData);
|
||||
const normalizedDistinctId = truncateString(distinctId, FIELD_LENGTH.distinctId);
|
||||
|
||||
const messages = jsonKeys.map(({ key, value, dataType }) => {
|
||||
return {
|
||||
website_id: websiteId,
|
||||
session_id: sessionId,
|
||||
data_key: key,
|
||||
data_key: truncateString(key, FIELD_LENGTH.dataKey),
|
||||
data_type: dataType,
|
||||
string_value: getStringValue(value, dataType),
|
||||
string_value: getStoredStringValue(value, dataType),
|
||||
number_value: dataType === DATA_TYPE.number ? value : null,
|
||||
date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
|
||||
distinct_id: distinctId,
|
||||
distinct_id: normalizedDistinctId,
|
||||
created_at: getUTCString(createdAt),
|
||||
};
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user