Add API integration test suite with Postgres and ClickHouse coverage

This commit is contained in:
Mike Cao committed 2026-09-06 16:42:53 -07:00
1 parent 950ae63f87
commit e18769e8cf
59 files changed
+8971 -28

No files matched your search

+30
View File
@@ -0,0 +1,30 @@
export interface AllowlistEntry {
/** Operation key, e.g. "POST /api/auth/sso" */
key: string;
/** Why this operation cannot be exercised by the harness. */
reason: string;
/** Only applies when running against this backend (UMAMI_TEST_DB); omit for both. */
db?: 'postgres' | 'clickhouse';
}
/**
* Operations that are intentionally not covered. Keep this list short and every
* entry justified — the reporter warns when an allowlisted operation is
* actually called, so entries can be removed once they gain a test.
*/
export const allowlist: AllowlistEntry[] = [
{
key: 'POST /api/auth/sso',
reason:
'Cloud single sign-on handoff. Requires REDIS_URL (returns 500 "Redis is disabled" otherwise); Redis is out of scope for the harness.',
},
{
key: 'GET /api/auth/subscription',
reason: 'Cloud billing/subscription lookup; only meaningful under CLOUD_MODE.',
},
{
key: 'DELETE /api/websites/{websiteId}/sessions/{sessionId}',
reason: 'Session deletion needs relational storage; sessions.spec.ts skips it on ClickHouse.',
db: 'clickhouse',
},
];
+82
View File
@@ -0,0 +1,82 @@
import { describe, expect, test } from 'vitest';
import { createMatcher, getPathname } from './matcher';
const templates = [
'/api/websites',
'/api/websites/charts',
'/api/websites/{websiteId}',
'/api/websites/{websiteId}/replays',
'/api/websites/{websiteId}/replays/saved',
'/api/websites/{websiteId}/replays/{replayId}',
'/api/websites/{websiteId}/replays/saved/{replayId}',
'/api/websites/{websiteId}/sessions/stats',
'/api/websites/{websiteId}/sessions/weekly',
'/api/websites/{websiteId}/sessions/{sessionId}',
'/api/reports',
'/api/reports/funnel',
'/api/reports/{reportId}',
'/api/teams/join',
'/api/teams/{teamId}',
'/api/share/{slug}',
'/api/share/id/{shareId}',
];
const match = createMatcher(templates);
describe('getPathname', () => {
test('strips origin, query and hash', () => {
expect(getPathname('http://localhost:3100/api/websites?page=1#x')).toBe('/api/websites');
expect(getPathname('/api/websites?page=1')).toBe('/api/websites');
});
test('strips trailing slashes but keeps root', () => {
expect(getPathname('/api/websites/')).toBe('/api/websites');
expect(getPathname('/')).toBe('/');
});
});
describe('createMatcher', () => {
test('prefers literal segments over parameters', () => {
expect(match('/api/websites/charts')).toBe('/api/websites/charts');
expect(match('/api/websites/abc-123')).toBe('/api/websites/{websiteId}');
expect(match('/api/reports/funnel')).toBe('/api/reports/funnel');
expect(match('/api/reports/abc-123')).toBe('/api/reports/{reportId}');
expect(match('/api/teams/join')).toBe('/api/teams/join');
expect(match('/api/teams/abc-123')).toBe('/api/teams/{teamId}');
});
test('resolves nested literal/parameter ambiguity', () => {
expect(match('/api/websites/w1/replays/saved')).toBe('/api/websites/{websiteId}/replays/saved');
expect(match('/api/websites/w1/replays/r1')).toBe(
'/api/websites/{websiteId}/replays/{replayId}',
);
expect(match('/api/websites/w1/replays/saved/r1')).toBe(
'/api/websites/{websiteId}/replays/saved/{replayId}',
);
expect(match('/api/websites/w1/sessions/stats')).toBe(
'/api/websites/{websiteId}/sessions/stats',
);
expect(match('/api/websites/w1/sessions/weekly')).toBe(
'/api/websites/{websiteId}/sessions/weekly',
);
expect(match('/api/websites/w1/sessions/s1')).toBe(
'/api/websites/{websiteId}/sessions/{sessionId}',
);
});
test('distinguishes templates by segment count', () => {
expect(match('/api/share/abcdef')).toBe('/api/share/{slug}');
expect(match('/api/share/id/abcdef')).toBe('/api/share/id/{shareId}');
});
test('ignores query strings and absolute URLs', () => {
expect(match('http://localhost:3100/api/websites/w1?startAt=1&endAt=2')).toBe(
'/api/websites/{websiteId}',
);
});
test('returns null for unknown paths', () => {
expect(match('/api/nope')).toBeNull();
expect(match('/api/websites/w1/unknown')).toBeNull();
});
});
+57
View File
@@ -0,0 +1,57 @@
export type PathMatcher = (url: string) => string | null;
/** Strips origin, query string, hash and trailing slashes from a URL or path. */
export function getPathname(url: string): string {
const raw = /^https?:\/\//i.test(url) ? new URL(url).pathname : url.split(/[?#]/)[0];
return raw.length > 1 ? raw.replace(/\/+$/, '') : raw;
}
/**
* Maps a concrete request path back to an OpenAPI path template.
*
* Candidates must have the same number of segments. A literal segment match
* scores higher than a `{param}` match, so `/api/websites/charts` resolves to
* `/api/websites/charts` rather than `/api/websites/{websiteId}`, and
* `/api/reports/funnel` beats `/api/reports/{reportId}`.
*/
export function createMatcher(templates: string[]): PathMatcher {
const candidates = [...new Set(templates)].map(template => ({
template,
segments: template.split('/').filter(Boolean),
}));
return url => {
const segments = getPathname(url).split('/').filter(Boolean);
let best: { template: string; score: number } | null = null;
for (const candidate of candidates) {
if (candidate.segments.length !== segments.length) {
continue;
}
let score = 0;
let matches = true;
for (let i = 0; i < segments.length; i++) {
const expected = candidate.segments[i];
const actual = segments[i];
if (expected === actual) {
score += 1;
} else if (expected.startsWith('{') && expected.endsWith('}') && actual) {
// parameter segment: matches anything non-empty
} else {
matches = false;
break;
}
}
if (matches && (!best || score > best.score)) {
best = { template: candidate.template, score };
}
}
return best?.template ?? null;
};
}
+57
View File
@@ -0,0 +1,57 @@
import { readFileSync } from 'node:fs';
import { OPENAPI_FILE } from '../paths';
/**
* The coverage oracle is the generated OpenAPI document. `pnpm build` cannot
* succeed with an undocumented route (see src/openapi/coverage.ts), so every
* operation in public/openapi.json is exactly the set of endpoints that exist.
*/
export interface ApiOperation {
/** Same format as src/openapi/operation.ts getOperationKey: "GET /api/websites" */
key: string;
method: string;
/** OpenAPI path template, e.g. "/api/websites/{websiteId}" */
path: string;
/** Route file that implements the operation (x-umami-source) */
source: string;
tag: string;
}
const METHODS = ['get', 'post', 'put', 'patch', 'delete', 'head', 'options'];
let cached: ApiOperation[] | undefined;
export function loadOperations(): ApiOperation[] {
if (cached) {
return cached;
}
const document = JSON.parse(readFileSync(OPENAPI_FILE, 'utf8'));
const operations: ApiOperation[] = [];
for (const [pathTemplate, pathItem] of Object.entries<Record<string, any>>(
document.paths ?? {},
)) {
for (const method of METHODS) {
const operation = pathItem?.[method];
if (!operation) {
continue;
}
operations.push({
key: `${method.toUpperCase()} ${pathTemplate}`,
method: method.toUpperCase(),
path: pathTemplate,
source: operation['x-umami-source'] ?? '',
tag: operation.tags?.[0] ?? 'Other',
});
}
}
cached = operations.sort(
(a, b) => a.path.localeCompare(b.path) || a.method.localeCompare(b.method),
);
return cached;
}
+61
View File
@@ -0,0 +1,61 @@
import { appendFileSync, mkdirSync } from 'node:fs';
import path from 'node:path';
import { COVERAGE_DIR } from '../paths';
import { createMatcher, getPathname, type PathMatcher } from './matcher';
import { loadOperations } from './oracle';
/**
* Records every API call made through ApiClient as "METHOD /path/{template}".
*
* Playwright workers are separate processes, so each process appends to its
* own log file; the reporter merges them at the end of the run.
*/
export const UNMATCHED_PREFIX = 'UNMATCHED ';
let enabled = true;
let matcher: PathMatcher | undefined;
let logFile: string | undefined;
const seen = new Set<string>();
/**
* Global setup calls this so the seeding traffic does not count as coverage —
* only calls made from specs should satisfy the coverage check.
*/
export function disableCoverageRecording() {
enabled = false;
}
function getMatcher() {
if (!matcher) {
matcher = createMatcher(loadOperations().map(operation => operation.path));
}
return matcher;
}
function getLogFile() {
if (!logFile) {
mkdirSync(COVERAGE_DIR, { recursive: true });
logFile = path.join(COVERAGE_DIR, `worker-${process.pid}.log`);
}
return logFile;
}
export function recordApiCall(method: string, url: string) {
if (!enabled) {
return;
}
const template = getMatcher()(url);
const line = template
? `${method.toUpperCase()} ${template}`
: `${UNMATCHED_PREFIX}${method.toUpperCase()} ${getPathname(url)}`;
if (seen.has(line)) {
return;
}
seen.add(line);
appendFileSync(getLogFile(), `${line}\n`);
}
+138
View File
@@ -0,0 +1,138 @@
import { existsSync, readdirSync, readFileSync } from 'node:fs';
import path from 'node:path';
import { COVERAGE_DIR } from '../paths';
import { allowlist } from './allowlist';
import { type ApiOperation, loadOperations } from './oracle';
import { UNMATCHED_PREFIX } from './recorder';
export interface CoverageSummary {
total: number;
covered: string[];
allowlisted: string[];
uncovered: ApiOperation[];
/** Allowlisted operations that were actually called — the entry can be removed. */
staleAllowlist: string[];
/** Allowlist keys that don't exist in the OpenAPI document. */
unknownAllowlist: string[];
/** Calls whose path could not be mapped to any documented operation. */
unmatched: string[];
}
export function readRecordedCalls() {
const covered = new Set<string>();
const unmatched = new Set<string>();
if (!existsSync(COVERAGE_DIR)) {
return { covered, unmatched };
}
for (const file of readdirSync(COVERAGE_DIR)) {
if (!file.endsWith('.log')) {
continue;
}
for (const line of readFileSync(path.join(COVERAGE_DIR, file), 'utf8').split('\n')) {
const entry = line.trim();
if (!entry) {
continue;
}
if (entry.startsWith(UNMATCHED_PREFIX)) {
unmatched.add(entry.slice(UNMATCHED_PREFIX.length));
} else {
covered.add(entry);
}
}
}
return { covered, unmatched };
}
export function computeCoverage(): CoverageSummary {
const operations = loadOperations();
const { covered, unmatched } = readRecordedCalls();
const known = new Set(operations.map(operation => operation.key));
const db = process.env.UMAMI_TEST_DB === 'clickhouse' ? 'clickhouse' : 'postgres';
const allowed = new Set(
allowlist.filter(entry => !entry.db || entry.db === db).map(entry => entry.key),
);
return {
total: operations.length,
covered: operations.filter(op => covered.has(op.key)).map(op => op.key),
allowlisted: [...allowed].filter(key => known.has(key) && !covered.has(key)),
uncovered: operations.filter(op => !covered.has(op.key) && !allowed.has(op.key)),
staleAllowlist: [...allowed].filter(key => covered.has(key)),
unknownAllowlist: [...allowed].filter(key => !known.has(key)),
unmatched: [...unmatched].sort(),
};
}
function groupByTag(operations: ApiOperation[]) {
const groups = new Map<string, ApiOperation[]>();
for (const operation of operations) {
const group = groups.get(operation.tag) ?? [];
group.push(operation);
groups.set(operation.tag, group);
}
return [...groups.entries()].sort(([a], [b]) => a.localeCompare(b));
}
export function formatCoverage(summary: CoverageSummary): string {
const lines: string[] = [];
const { total, covered, allowlisted, uncovered } = summary;
lines.push('');
lines.push(
`API coverage: ${covered.length + allowlisted.length}/${total} operations ` +
`(${covered.length} covered, ${allowlisted.length} allowlisted, ${uncovered.length} uncovered)`,
);
if (uncovered.length) {
lines.push('');
lines.push('Uncovered operations:');
for (const [tag, operations] of groupByTag(uncovered)) {
lines.push(` ${tag}`);
for (const operation of operations) {
const source = operation.source ? ` <- ${operation.source}` : '';
lines.push(` ${operation.key}${source}`);
}
}
}
if (summary.staleAllowlist.length) {
lines.push('');
lines.push('Allowlisted operations that were called (remove them from coverage/allowlist.ts):');
for (const key of summary.staleAllowlist) {
lines.push(` ${key}`);
}
}
if (summary.unknownAllowlist.length) {
lines.push('');
lines.push('Allowlist entries that do not exist in public/openapi.json:');
for (const key of summary.unknownAllowlist) {
lines.push(` ${key}`);
}
}
if (summary.unmatched.length) {
lines.push('');
lines.push('Calls that did not match any documented operation:');
for (const entry of summary.unmatched) {
lines.push(` ${entry}`);
}
}
lines.push('');
return lines.join('\n');
}
+51
View File
@@ -0,0 +1,51 @@
import { mkdirSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import type { FullResult, Reporter } from '@playwright/test/reporter';
import { COVERAGE_DIR } from '../paths';
import { computeCoverage, formatCoverage } from './report';
/**
* Enforces endpoint coverage at the end of the run.
*
* API_COVERAGE=enforce (default) fail a passing run if any operation is uncovered
* API_COVERAGE=report print the summary only
* API_COVERAGE=off skip entirely
*/
class ApiCoverageReporter implements Reporter {
printsToStdio() {
return true;
}
async onEnd(result: FullResult) {
const mode = process.env.API_COVERAGE ?? 'enforce';
if (mode === 'off') {
return;
}
const summary = computeCoverage();
mkdirSync(COVERAGE_DIR, { recursive: true });
writeFileSync(path.join(COVERAGE_DIR, 'summary.json'), JSON.stringify(summary, null, 2));
console.log(formatCoverage(summary));
if (mode !== 'enforce' || summary.uncovered.length === 0) {
return;
}
if (result.status !== 'passed') {
console.log('API coverage not enforced because the run did not pass.\n');
return;
}
console.log(
`API coverage check failed: ${summary.uncovered.length} operation(s) were never called. ` +
'Add a spec that exercises them or an entry in tests/api/coverage/allowlist.ts.\n',
);
return { status: 'failed' as const };
}
}
export default ApiCoverageReporter;