Add API integration test suite with Postgres and ClickHouse coverage
This commit is contained in:
1 parent
950ae63f87
commit
e18769e8cf
59 files changed
+8971
-28
No files matched your search
@@ -0,0 +1,30 @@
|
||||
export interface AllowlistEntry {
|
||||
/** Operation key, e.g. "POST /api/auth/sso" */
|
||||
key: string;
|
||||
/** Why this operation cannot be exercised by the harness. */
|
||||
reason: string;
|
||||
/** Only applies when running against this backend (UMAMI_TEST_DB); omit for both. */
|
||||
db?: 'postgres' | 'clickhouse';
|
||||
}
|
||||
|
||||
/**
|
||||
* Operations that are intentionally not covered. Keep this list short and every
|
||||
* entry justified — the reporter warns when an allowlisted operation is
|
||||
* actually called, so entries can be removed once they gain a test.
|
||||
*/
|
||||
export const allowlist: AllowlistEntry[] = [
|
||||
{
|
||||
key: 'POST /api/auth/sso',
|
||||
reason:
|
||||
'Cloud single sign-on handoff. Requires REDIS_URL (returns 500 "Redis is disabled" otherwise); Redis is out of scope for the harness.',
|
||||
},
|
||||
{
|
||||
key: 'GET /api/auth/subscription',
|
||||
reason: 'Cloud billing/subscription lookup; only meaningful under CLOUD_MODE.',
|
||||
},
|
||||
{
|
||||
key: 'DELETE /api/websites/{websiteId}/sessions/{sessionId}',
|
||||
reason: 'Session deletion needs relational storage; sessions.spec.ts skips it on ClickHouse.',
|
||||
db: 'clickhouse',
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,82 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import { createMatcher, getPathname } from './matcher';
|
||||
|
||||
const templates = [
|
||||
'/api/websites',
|
||||
'/api/websites/charts',
|
||||
'/api/websites/{websiteId}',
|
||||
'/api/websites/{websiteId}/replays',
|
||||
'/api/websites/{websiteId}/replays/saved',
|
||||
'/api/websites/{websiteId}/replays/{replayId}',
|
||||
'/api/websites/{websiteId}/replays/saved/{replayId}',
|
||||
'/api/websites/{websiteId}/sessions/stats',
|
||||
'/api/websites/{websiteId}/sessions/weekly',
|
||||
'/api/websites/{websiteId}/sessions/{sessionId}',
|
||||
'/api/reports',
|
||||
'/api/reports/funnel',
|
||||
'/api/reports/{reportId}',
|
||||
'/api/teams/join',
|
||||
'/api/teams/{teamId}',
|
||||
'/api/share/{slug}',
|
||||
'/api/share/id/{shareId}',
|
||||
];
|
||||
|
||||
const match = createMatcher(templates);
|
||||
|
||||
describe('getPathname', () => {
|
||||
test('strips origin, query and hash', () => {
|
||||
expect(getPathname('http://localhost:3100/api/websites?page=1#x')).toBe('/api/websites');
|
||||
expect(getPathname('/api/websites?page=1')).toBe('/api/websites');
|
||||
});
|
||||
|
||||
test('strips trailing slashes but keeps root', () => {
|
||||
expect(getPathname('/api/websites/')).toBe('/api/websites');
|
||||
expect(getPathname('/')).toBe('/');
|
||||
});
|
||||
});
|
||||
|
||||
describe('createMatcher', () => {
|
||||
test('prefers literal segments over parameters', () => {
|
||||
expect(match('/api/websites/charts')).toBe('/api/websites/charts');
|
||||
expect(match('/api/websites/abc-123')).toBe('/api/websites/{websiteId}');
|
||||
expect(match('/api/reports/funnel')).toBe('/api/reports/funnel');
|
||||
expect(match('/api/reports/abc-123')).toBe('/api/reports/{reportId}');
|
||||
expect(match('/api/teams/join')).toBe('/api/teams/join');
|
||||
expect(match('/api/teams/abc-123')).toBe('/api/teams/{teamId}');
|
||||
});
|
||||
|
||||
test('resolves nested literal/parameter ambiguity', () => {
|
||||
expect(match('/api/websites/w1/replays/saved')).toBe('/api/websites/{websiteId}/replays/saved');
|
||||
expect(match('/api/websites/w1/replays/r1')).toBe(
|
||||
'/api/websites/{websiteId}/replays/{replayId}',
|
||||
);
|
||||
expect(match('/api/websites/w1/replays/saved/r1')).toBe(
|
||||
'/api/websites/{websiteId}/replays/saved/{replayId}',
|
||||
);
|
||||
expect(match('/api/websites/w1/sessions/stats')).toBe(
|
||||
'/api/websites/{websiteId}/sessions/stats',
|
||||
);
|
||||
expect(match('/api/websites/w1/sessions/weekly')).toBe(
|
||||
'/api/websites/{websiteId}/sessions/weekly',
|
||||
);
|
||||
expect(match('/api/websites/w1/sessions/s1')).toBe(
|
||||
'/api/websites/{websiteId}/sessions/{sessionId}',
|
||||
);
|
||||
});
|
||||
|
||||
test('distinguishes templates by segment count', () => {
|
||||
expect(match('/api/share/abcdef')).toBe('/api/share/{slug}');
|
||||
expect(match('/api/share/id/abcdef')).toBe('/api/share/id/{shareId}');
|
||||
});
|
||||
|
||||
test('ignores query strings and absolute URLs', () => {
|
||||
expect(match('http://localhost:3100/api/websites/w1?startAt=1&endAt=2')).toBe(
|
||||
'/api/websites/{websiteId}',
|
||||
);
|
||||
});
|
||||
|
||||
test('returns null for unknown paths', () => {
|
||||
expect(match('/api/nope')).toBeNull();
|
||||
expect(match('/api/websites/w1/unknown')).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
export type PathMatcher = (url: string) => string | null;
|
||||
|
||||
/** Strips origin, query string, hash and trailing slashes from a URL or path. */
|
||||
export function getPathname(url: string): string {
|
||||
const raw = /^https?:\/\//i.test(url) ? new URL(url).pathname : url.split(/[?#]/)[0];
|
||||
|
||||
return raw.length > 1 ? raw.replace(/\/+$/, '') : raw;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a concrete request path back to an OpenAPI path template.
|
||||
*
|
||||
* Candidates must have the same number of segments. A literal segment match
|
||||
* scores higher than a `{param}` match, so `/api/websites/charts` resolves to
|
||||
* `/api/websites/charts` rather than `/api/websites/{websiteId}`, and
|
||||
* `/api/reports/funnel` beats `/api/reports/{reportId}`.
|
||||
*/
|
||||
export function createMatcher(templates: string[]): PathMatcher {
|
||||
const candidates = [...new Set(templates)].map(template => ({
|
||||
template,
|
||||
segments: template.split('/').filter(Boolean),
|
||||
}));
|
||||
|
||||
return url => {
|
||||
const segments = getPathname(url).split('/').filter(Boolean);
|
||||
let best: { template: string; score: number } | null = null;
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.segments.length !== segments.length) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let score = 0;
|
||||
let matches = true;
|
||||
|
||||
for (let i = 0; i < segments.length; i++) {
|
||||
const expected = candidate.segments[i];
|
||||
const actual = segments[i];
|
||||
|
||||
if (expected === actual) {
|
||||
score += 1;
|
||||
} else if (expected.startsWith('{') && expected.endsWith('}') && actual) {
|
||||
// parameter segment: matches anything non-empty
|
||||
} else {
|
||||
matches = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (matches && (!best || score > best.score)) {
|
||||
best = { template: candidate.template, score };
|
||||
}
|
||||
}
|
||||
|
||||
return best?.template ?? null;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { OPENAPI_FILE } from '../paths';
|
||||
|
||||
/**
|
||||
* The coverage oracle is the generated OpenAPI document. `pnpm build` cannot
|
||||
* succeed with an undocumented route (see src/openapi/coverage.ts), so every
|
||||
* operation in public/openapi.json is exactly the set of endpoints that exist.
|
||||
*/
|
||||
export interface ApiOperation {
|
||||
/** Same format as src/openapi/operation.ts getOperationKey: "GET /api/websites" */
|
||||
key: string;
|
||||
method: string;
|
||||
/** OpenAPI path template, e.g. "/api/websites/{websiteId}" */
|
||||
path: string;
|
||||
/** Route file that implements the operation (x-umami-source) */
|
||||
source: string;
|
||||
tag: string;
|
||||
}
|
||||
|
||||
const METHODS = ['get', 'post', 'put', 'patch', 'delete', 'head', 'options'];
|
||||
|
||||
let cached: ApiOperation[] | undefined;
|
||||
|
||||
export function loadOperations(): ApiOperation[] {
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
const document = JSON.parse(readFileSync(OPENAPI_FILE, 'utf8'));
|
||||
const operations: ApiOperation[] = [];
|
||||
|
||||
for (const [pathTemplate, pathItem] of Object.entries<Record<string, any>>(
|
||||
document.paths ?? {},
|
||||
)) {
|
||||
for (const method of METHODS) {
|
||||
const operation = pathItem?.[method];
|
||||
|
||||
if (!operation) {
|
||||
continue;
|
||||
}
|
||||
|
||||
operations.push({
|
||||
key: `${method.toUpperCase()} ${pathTemplate}`,
|
||||
method: method.toUpperCase(),
|
||||
path: pathTemplate,
|
||||
source: operation['x-umami-source'] ?? '',
|
||||
tag: operation.tags?.[0] ?? 'Other',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
cached = operations.sort(
|
||||
(a, b) => a.path.localeCompare(b.path) || a.method.localeCompare(b.method),
|
||||
);
|
||||
|
||||
return cached;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { appendFileSync, mkdirSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { COVERAGE_DIR } from '../paths';
|
||||
import { createMatcher, getPathname, type PathMatcher } from './matcher';
|
||||
import { loadOperations } from './oracle';
|
||||
|
||||
/**
|
||||
* Records every API call made through ApiClient as "METHOD /path/{template}".
|
||||
*
|
||||
* Playwright workers are separate processes, so each process appends to its
|
||||
* own log file; the reporter merges them at the end of the run.
|
||||
*/
|
||||
export const UNMATCHED_PREFIX = 'UNMATCHED ';
|
||||
|
||||
let enabled = true;
|
||||
let matcher: PathMatcher | undefined;
|
||||
let logFile: string | undefined;
|
||||
const seen = new Set<string>();
|
||||
|
||||
/**
|
||||
* Global setup calls this so the seeding traffic does not count as coverage —
|
||||
* only calls made from specs should satisfy the coverage check.
|
||||
*/
|
||||
export function disableCoverageRecording() {
|
||||
enabled = false;
|
||||
}
|
||||
|
||||
function getMatcher() {
|
||||
if (!matcher) {
|
||||
matcher = createMatcher(loadOperations().map(operation => operation.path));
|
||||
}
|
||||
|
||||
return matcher;
|
||||
}
|
||||
|
||||
function getLogFile() {
|
||||
if (!logFile) {
|
||||
mkdirSync(COVERAGE_DIR, { recursive: true });
|
||||
logFile = path.join(COVERAGE_DIR, `worker-${process.pid}.log`);
|
||||
}
|
||||
|
||||
return logFile;
|
||||
}
|
||||
|
||||
export function recordApiCall(method: string, url: string) {
|
||||
if (!enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const template = getMatcher()(url);
|
||||
const line = template
|
||||
? `${method.toUpperCase()} ${template}`
|
||||
: `${UNMATCHED_PREFIX}${method.toUpperCase()} ${getPathname(url)}`;
|
||||
|
||||
if (seen.has(line)) {
|
||||
return;
|
||||
}
|
||||
|
||||
seen.add(line);
|
||||
appendFileSync(getLogFile(), `${line}\n`);
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { COVERAGE_DIR } from '../paths';
|
||||
import { allowlist } from './allowlist';
|
||||
import { type ApiOperation, loadOperations } from './oracle';
|
||||
import { UNMATCHED_PREFIX } from './recorder';
|
||||
|
||||
export interface CoverageSummary {
|
||||
total: number;
|
||||
covered: string[];
|
||||
allowlisted: string[];
|
||||
uncovered: ApiOperation[];
|
||||
/** Allowlisted operations that were actually called — the entry can be removed. */
|
||||
staleAllowlist: string[];
|
||||
/** Allowlist keys that don't exist in the OpenAPI document. */
|
||||
unknownAllowlist: string[];
|
||||
/** Calls whose path could not be mapped to any documented operation. */
|
||||
unmatched: string[];
|
||||
}
|
||||
|
||||
export function readRecordedCalls() {
|
||||
const covered = new Set<string>();
|
||||
const unmatched = new Set<string>();
|
||||
|
||||
if (!existsSync(COVERAGE_DIR)) {
|
||||
return { covered, unmatched };
|
||||
}
|
||||
|
||||
for (const file of readdirSync(COVERAGE_DIR)) {
|
||||
if (!file.endsWith('.log')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const line of readFileSync(path.join(COVERAGE_DIR, file), 'utf8').split('\n')) {
|
||||
const entry = line.trim();
|
||||
|
||||
if (!entry) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.startsWith(UNMATCHED_PREFIX)) {
|
||||
unmatched.add(entry.slice(UNMATCHED_PREFIX.length));
|
||||
} else {
|
||||
covered.add(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { covered, unmatched };
|
||||
}
|
||||
|
||||
export function computeCoverage(): CoverageSummary {
|
||||
const operations = loadOperations();
|
||||
const { covered, unmatched } = readRecordedCalls();
|
||||
const known = new Set(operations.map(operation => operation.key));
|
||||
const db = process.env.UMAMI_TEST_DB === 'clickhouse' ? 'clickhouse' : 'postgres';
|
||||
const allowed = new Set(
|
||||
allowlist.filter(entry => !entry.db || entry.db === db).map(entry => entry.key),
|
||||
);
|
||||
|
||||
return {
|
||||
total: operations.length,
|
||||
covered: operations.filter(op => covered.has(op.key)).map(op => op.key),
|
||||
allowlisted: [...allowed].filter(key => known.has(key) && !covered.has(key)),
|
||||
uncovered: operations.filter(op => !covered.has(op.key) && !allowed.has(op.key)),
|
||||
staleAllowlist: [...allowed].filter(key => covered.has(key)),
|
||||
unknownAllowlist: [...allowed].filter(key => !known.has(key)),
|
||||
unmatched: [...unmatched].sort(),
|
||||
};
|
||||
}
|
||||
|
||||
function groupByTag(operations: ApiOperation[]) {
|
||||
const groups = new Map<string, ApiOperation[]>();
|
||||
|
||||
for (const operation of operations) {
|
||||
const group = groups.get(operation.tag) ?? [];
|
||||
group.push(operation);
|
||||
groups.set(operation.tag, group);
|
||||
}
|
||||
|
||||
return [...groups.entries()].sort(([a], [b]) => a.localeCompare(b));
|
||||
}
|
||||
|
||||
export function formatCoverage(summary: CoverageSummary): string {
|
||||
const lines: string[] = [];
|
||||
const { total, covered, allowlisted, uncovered } = summary;
|
||||
|
||||
lines.push('');
|
||||
lines.push(
|
||||
`API coverage: ${covered.length + allowlisted.length}/${total} operations ` +
|
||||
`(${covered.length} covered, ${allowlisted.length} allowlisted, ${uncovered.length} uncovered)`,
|
||||
);
|
||||
|
||||
if (uncovered.length) {
|
||||
lines.push('');
|
||||
lines.push('Uncovered operations:');
|
||||
|
||||
for (const [tag, operations] of groupByTag(uncovered)) {
|
||||
lines.push(` ${tag}`);
|
||||
|
||||
for (const operation of operations) {
|
||||
const source = operation.source ? ` <- ${operation.source}` : '';
|
||||
lines.push(` ${operation.key}${source}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (summary.staleAllowlist.length) {
|
||||
lines.push('');
|
||||
lines.push('Allowlisted operations that were called (remove them from coverage/allowlist.ts):');
|
||||
|
||||
for (const key of summary.staleAllowlist) {
|
||||
lines.push(` ${key}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (summary.unknownAllowlist.length) {
|
||||
lines.push('');
|
||||
lines.push('Allowlist entries that do not exist in public/openapi.json:');
|
||||
|
||||
for (const key of summary.unknownAllowlist) {
|
||||
lines.push(` ${key}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (summary.unmatched.length) {
|
||||
lines.push('');
|
||||
lines.push('Calls that did not match any documented operation:');
|
||||
|
||||
for (const entry of summary.unmatched) {
|
||||
lines.push(` ${entry}`);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { mkdirSync, writeFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type { FullResult, Reporter } from '@playwright/test/reporter';
|
||||
import { COVERAGE_DIR } from '../paths';
|
||||
import { computeCoverage, formatCoverage } from './report';
|
||||
|
||||
/**
|
||||
* Enforces endpoint coverage at the end of the run.
|
||||
*
|
||||
* API_COVERAGE=enforce (default) fail a passing run if any operation is uncovered
|
||||
* API_COVERAGE=report print the summary only
|
||||
* API_COVERAGE=off skip entirely
|
||||
*/
|
||||
class ApiCoverageReporter implements Reporter {
|
||||
printsToStdio() {
|
||||
return true;
|
||||
}
|
||||
|
||||
async onEnd(result: FullResult) {
|
||||
const mode = process.env.API_COVERAGE ?? 'enforce';
|
||||
|
||||
if (mode === 'off') {
|
||||
return;
|
||||
}
|
||||
|
||||
const summary = computeCoverage();
|
||||
|
||||
mkdirSync(COVERAGE_DIR, { recursive: true });
|
||||
writeFileSync(path.join(COVERAGE_DIR, 'summary.json'), JSON.stringify(summary, null, 2));
|
||||
|
||||
console.log(formatCoverage(summary));
|
||||
|
||||
if (mode !== 'enforce' || summary.uncovered.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.status !== 'passed') {
|
||||
console.log('API coverage not enforced because the run did not pass.\n');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`API coverage check failed: ${summary.uncovered.length} operation(s) were never called. ` +
|
||||
'Add a spec that exercises them or an entry in tests/api/coverage/allowlist.ts.\n',
|
||||
);
|
||||
|
||||
return { status: 'failed' as const };
|
||||
}
|
||||
}
|
||||
|
||||
export default ApiCoverageReporter;
|
||||
Reference in new issue
Block a user