umami/scripts/test-api.ts

/* eslint-disable no-console */
/**
 * Orchestrates the API integration suite:
 *
 *   1. docker compose up (builds the umami image, starts Postgres [+ ClickHouse])
 *   2. waits for /api/heartbeat
 *   3. runs Playwright against the container (tests/api, playwright.api.config.ts)
 *   4. dumps container logs on failure
 *   5. tears the stack down (unless --keep)
 *
 * Usage: tsx scripts/test-api.ts [options] [-- playwright args]
 */
import { spawnSync } from 'node:child_process';
import { mkdirSync, writeFileSync } from 'node:fs';
import { createRequire } from 'node:module';
import { connect } from 'node:net';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { assertDisposableTarget } from '../tests/api/paths';

const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const COMPOSE_FILE = 'docker-compose.test.yml';
const PROJECT_NAME = process.env.COMPOSE_PROJECT_NAME || 'umami-api-test';
const PORT = process.env.UMAMI_TEST_PORT || '3100';
const BASE_URL = process.env.PLAYWRIGHT_BASE_URL || `http://localhost:${PORT}`;
// PLAYWRIGHT_BASE_URL selects an already-running server: no Compose stack is
// built, started, or torn down for it.
const EXTERNAL_TARGET = !!process.env.PLAYWRIGHT_BASE_URL;
const HEARTBEAT_TIMEOUT_MS = 180_000;
// Mirrors RUNTIME_DIR in tests/api/paths.ts (one state dir per target host).
const LOG_FILE = path.join(
  ROOT_DIR,
  'tests/api/.runtime',
  new URL(BASE_URL).host.replace(/[^a-z0-9.-]+/gi, '-'),
  'docker.log',
);

const HELP = `
Usage: pnpm test:api [options] [-- playwright args]

Options:
  --clickhouse        Run against Postgres + ClickHouse (analytics stored in ClickHouse)
  --keep              Leave the containers running after the tests finish
  --no-build          Skip the image build (use the existing UMAMI_TEST_IMAGE)
  --up-only           Start the stack, wait for readiness, and exit (see --keep)
  --down              Tear down the stack and exit
  --grep <expr>       Only run matching tests (implies --coverage-report)
  --coverage-report   Print endpoint coverage without failing the run
  -h, --help          Show this help

Environment:
  UMAMI_TEST_PORT     Host port for the app (default 3100)
  UMAMI_TEST_IMAGE    Image tag to build/run (default umami-test:local)
  COMPOSE_PROJECT_NAME  Compose project name (default umami-api-test)
  PLAYWRIGHT_BASE_URL  Test an already-running server instead of starting the
                      Compose stack (pass --clickhouse if it stores analytics there)
  API_ALLOW_DESTRUCTIVE=1  Required for non-loopback PLAYWRIGHT_BASE_URL targets: the suite
                      recreates websites, ingests data and toggles global 2FA
  API_SKIP_SEED=1     Reuse the previous seed when re-running against a kept stack
                      (each run seeds additively otherwise)
`;

interface Options {
  clickhouse: boolean;
  keep: boolean;
  build: boolean;
  upOnly: boolean;
  down: boolean;
  grep?: string;
  coverageReport: boolean;
  passthrough: string[];
}

function parseArgs(argv: string[]): Options {
  const options: Options = {
    clickhouse: false,
    keep: false,
    build: true,
    upOnly: false,
    down: false,
    coverageReport: false,
    passthrough: [],
  };

  for (let i = 0; i < argv.length; i++) {
    const arg = argv[i];

    if (arg === '--') {
      options.passthrough.push(...argv.slice(i + 1));
      break;
    }

    if (arg.startsWith('--grep=')) {
      options.grep = arg.slice('--grep='.length);
      continue;
    }

    switch (arg) {
      case '--clickhouse':
        options.clickhouse = true;
        break;
      case '--keep':
        options.keep = true;
        break;
      case '--no-build':
        options.build = false;
        break;
      case '--up-only':
        options.upOnly = true;
        break;
      case '--down':
        options.down = true;
        break;
      case '--coverage-report':
        options.coverageReport = true;
        break;
      case '--grep':
        options.grep = argv[++i];
        break;
      case '-h':
      case '--help':
        console.log(HELP);
        process.exit(0);
        break;
      default:
        console.error(`Unknown option: ${arg}`);
        console.log(HELP);
        process.exit(2);
    }
  }

  return options;
}

function run(command: string, args: string[], env: Record<string, string> = {}) {
  const result = spawnSync(command, args, {
    cwd: ROOT_DIR,
    stdio: 'inherit',
    env: { ...process.env, ...env },
  });

  if (result.error) {
    throw result.error;
  }

  return result.status ?? 1;
}

function capture(command: string, args: string[]) {
  const result = spawnSync(command, args, {
    cwd: ROOT_DIR,
    encoding: 'utf8',
    env: process.env,
    maxBuffer: 64 * 1024 * 1024,
  });

  return `${result.stdout ?? ''}${result.stderr ?? ''}`;
}

function composeArgs(profiles: string[], args: string[]) {
  return [
    'compose',
    '-f',
    COMPOSE_FILE,
    '-p',
    PROJECT_NAME,
    ...profiles.flatMap(profile => ['--profile', profile]),
    ...args,
  ];
}

// Docker reports a taken host port as an opaque "ports are not available" error
// after the image has already been built, so check up front. Probe by connecting
// rather than binding: on Windows a second listen() on a busy port can succeed
// unless the owner set SO_EXCLUSIVEADDRUSE, so a bind test gives false negatives.
function isPortFree(port: number) {
  return new Promise<boolean>(resolve => {
    const socket = connect({ host: '127.0.0.1', port });
    const done = (free: boolean) => {
      socket.destroy();
      resolve(free);
    };
    socket.setTimeout(1000);
    socket.once('connect', () => done(false));
    socket.once('timeout', () => done(true));
    socket.once('error', () => done(true));
  });
}

async function waitForHeartbeat(url: string, timeoutMs: number) {
  const deadline = Date.now() + timeoutMs;
  let lastError = '';

  while (Date.now() < deadline) {
    try {
      const response = await fetch(`${url}/api/heartbeat`);

      if (response.ok) {
        return;
      }

      lastError = `HTTP ${response.status}`;
    } catch (error) {
      lastError = (error as Error).message;
    }

    await new Promise(resolve => setTimeout(resolve, 1000));
  }

  throw new Error(`Timed out waiting for ${url}/api/heartbeat (${lastError})`);
}

async function main() {
  const options = parseArgs(process.argv.slice(2));
  const profile = options.clickhouse ? 'clickhouse' : 'postgres';
  const appService = options.clickhouse ? 'umami-clickhouse' : 'umami';
  const logServices = options.clickhouse ? [appService, 'db', 'clickhouse'] : [appService, 'db'];

  const teardown = () => {
    console.log('\nStopping test stack...');
    run('docker', composeArgs(['postgres', 'clickhouse'], ['down', '-v', '--remove-orphans']));
  };

  if (EXTERNAL_TARGET && (options.upOnly || options.down)) {
    console.error(
      'PLAYWRIGHT_BASE_URL selects an already-running server; --up-only and --down manage ' +
        'the local Compose stack and cannot be combined with it.',
    );
    return 2;
  }

  if (options.down) {
    teardown();
    return 0;
  }

  const dumpLogs = () => {
    const logs = capture('docker', composeArgs([profile], ['logs', '--no-color', ...logServices]));

    mkdirSync(path.dirname(LOG_FILE), { recursive: true });
    writeFileSync(LOG_FILE, logs);

    console.log(`\n--- docker compose logs (${logServices.join(', ')}) ---\n`);
    console.log(logs);
    console.log(`--- end of logs (saved to ${path.relative(ROOT_DIR, LOG_FILE)}) ---\n`);
  };

  let exitCode = 1;
  let started = false;

  const onSignal = () => {
    console.log('\nInterrupted.');

    if (started && !options.keep) {
      teardown();
    }

    process.exit(130);
  };

  process.on('SIGINT', onSignal);
  process.on('SIGTERM', onSignal);

  try {
    if (EXTERNAL_TARGET) {
      try {
        assertDisposableTarget(BASE_URL);
      } catch (error) {
        console.error((error as Error).message);
        return 2;
      }
      console.log(`Using the already-running server at ${BASE_URL} (PLAYWRIGHT_BASE_URL is set).`);
    } else {
      // A kept stack from a previous run legitimately owns the port; only
      // complain when something else is listening on it.
      const ownStackRunning = capture(
        'docker',
        composeArgs([profile], ['ps', '-q', '--status', 'running', appService]),
      ).trim();

      if (!ownStackRunning && !(await isPortFree(Number(PORT)))) {
        console.error(
          `Port ${PORT} is already in use on this machine, so the umami container cannot be published there.
` +
            `Stop whatever is listening on ${PORT} (e.g. another dev server), or pick a different port:
` +
            `  UMAMI_TEST_PORT=3200 pnpm test:api`,
        );
        return 2;
      }

      console.log(`Starting test stack (profile: ${profile}, port: ${PORT})...`);

      const upArgs = ['up', '-d', '--wait', '--wait-timeout', '300'];

      if (options.build) {
        upArgs.push('--build');
      }

      started = true;

      if (run('docker', composeArgs([profile], upArgs)) !== 0) {
        console.error('docker compose up failed.');
        dumpLogs();
        return 1;
      }
    }

    await waitForHeartbeat(BASE_URL, HEARTBEAT_TIMEOUT_MS);

    console.log(`umami is ready at ${BASE_URL}`);

    if (options.upOnly) {
      options.keep = true;
      console.log(
        `\nRun tests with: PLAYWRIGHT_BASE_URL=${BASE_URL} UMAMI_TEST_DB=${profile} pnpm test:api:run` +
          '\nRe-running against the same stack? Add API_SKIP_SEED=1 (seeding is additive).' +
          '\nStop the stack with: pnpm test:api:down',
      );
      return 0;
    }

    // Invoke the Playwright CLI through the current node binary instead of
    // `pnpm exec`: on Windows that resolves to pnpm.cmd, which Node refuses to
    // spawn without a shell (EINVAL, CVE-2024-27980 hardening).
    const playwrightCli = path.join(
      path.dirname(createRequire(import.meta.url).resolve('@playwright/test/package.json')),
      'cli.js',
    );
    const playwrightArgs = [playwrightCli, 'test', '-c', 'playwright.api.config.ts'];

    if (options.grep) {
      playwrightArgs.push('--grep', options.grep);
    }

    playwrightArgs.push(...options.passthrough);

    exitCode = run(process.execPath, playwrightArgs, {
      PLAYWRIGHT_BASE_URL: BASE_URL,
      PLAYWRIGHT_SKIP_WEB_SERVER: '1',
      UMAMI_TEST_DB: profile,
      API_COVERAGE:
        process.env.API_COVERAGE ?? (options.grep || options.coverageReport ? 'report' : 'enforce'),
    });

    if (exitCode !== 0 && started) {
      dumpLogs();
    }

    return exitCode;
  } finally {
    process.off('SIGINT', onSignal);
    process.off('SIGTERM', onSignal);

    if (started && !options.keep) {
      teardown();
    } else if (started) {
      console.log(`\nStack left running (${BASE_URL}). Stop it with: pnpm test:api:down`);
    }
  }
}

main()
  .then(code => {
    process.exit(code);
  })
  .catch(error => {
    console.error(error);
    process.exit(1);
  });