Files
umami/src/lib/kafka.ts
T
Leon Acosta 2771abc6a7 verify Kafka broker TLS certificate by default
When SASL credentials are present, getClient() connected with
`rejectUnauthorized: false`, so the SASL password was sent over a TLS
connection whose broker certificate was never verified — a MITM could
present any certificate and capture the credentials.

Verify by default; set KAFKA_SSL_ALLOW_UNAUTHORIZED=true to opt out for
self-signed brokers.
2026-07-04 18:45:45 +07:00

166 lines
3.9 KiB
TypeScript

import type * as tls from 'node:tls';
import debug from 'debug';
import { Kafka, logLevel, type Producer, type RecordMetadata, type SASLOptions } from 'kafkajs';
import { serializeError } from 'serialize-error';
import { KAFKA, KAFKA_PRODUCER } from '@/lib/db';
const log = debug('umami:kafka');
const CONNECT_TIMEOUT = 5000;
const SEND_TIMEOUT = 3000;
const ACKS = 1;
const DEFAULT_MAX_MESSAGE_BYTES = 900_000;
let kafka: Kafka;
let producer: Producer;
const enabled = Boolean(process.env.KAFKA_URL && process.env.KAFKA_BROKER);
type KafkaMessage = Record<string, unknown>;
type KafkaProducerMessage = { value: string };
function getMaxMessageBytes() {
const size = Number(process.env.KAFKA_MAX_MESSAGE_BYTES);
return Number.isFinite(size) && size > 0 ? size : DEFAULT_MAX_MESSAGE_BYTES;
}
function getMessageSize(value: string) {
return Buffer.byteLength(value, 'utf8');
}
function getMessages(message: KafkaMessage | KafkaMessage[]) {
const items = Array.isArray(message) ? message : [message];
return items.map(item => {
const value = JSON.stringify(item);
return { value, size: getMessageSize(value) };
});
}
function getClient() {
const { username, password } = new URL(process.env.KAFKA_URL);
const brokers = process.env.KAFKA_BROKER.split(',');
const mechanism =
(process.env.KAFKA_SASL_MECHANISM as 'plain' | 'scram-sha-256' | 'scram-sha-512') || 'plain';
const ssl: { ssl?: tls.ConnectionOptions | boolean; sasl?: SASLOptions } =
username && password
? {
ssl: {
rejectUnauthorized: process.env.KAFKA_SSL_ALLOW_UNAUTHORIZED !== 'true',
},
sasl: {
mechanism,
username,
password,
},
}
: {};
const client: Kafka = new Kafka({
clientId: 'umami',
brokers: brokers,
connectionTimeout: CONNECT_TIMEOUT,
logLevel: logLevel.ERROR,
...ssl,
});
if (process.env.NODE_ENV !== 'production') {
globalThis[KAFKA] = client;
}
log('Kafka initialized');
return client;
}
async function getProducer(): Promise<Producer> {
const producer = kafka.producer();
await producer.connect();
if (process.env.NODE_ENV !== 'production') {
globalThis[KAFKA_PRODUCER] = producer;
}
log('Kafka producer initialized');
return producer;
}
async function sendMessage(
topic: string,
message: KafkaMessage | KafkaMessage[],
): Promise<RecordMetadata[]> {
try {
await connect();
const maxMessageBytes = getMaxMessageBytes();
const messages = getMessages(message);
const result: RecordMetadata[] = [];
let batch: KafkaProducerMessage[] = [];
let batchSize = 0;
for (const { value, size } of messages) {
if (size > maxMessageBytes) {
log('Kafka message dropped: topic=%s size=%d max=%d', topic, size, maxMessageBytes);
continue;
}
if (batch.length && batchSize + size > maxMessageBytes) {
result.push(
...(await producer.send({
topic,
messages: batch,
timeout: SEND_TIMEOUT,
acks: ACKS,
})),
);
batch = [];
batchSize = 0;
}
batch.push({ value });
batchSize += size;
}
if (batch.length) {
result.push(
...(await producer.send({
topic,
messages: batch,
timeout: SEND_TIMEOUT,
acks: ACKS,
})),
);
}
return result;
} catch (e) {
// eslint-disable-next-line no-console
console.log('KAFKA ERROR:', serializeError(e));
return [];
}
}
async function connect(): Promise<Kafka> {
if (!kafka) {
kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (globalThis[KAFKA] || getClient());
if (kafka) {
producer = globalThis[KAFKA_PRODUCER] || (await getProducer());
}
}
return kafka;
}
export default {
enabled,
client: kafka,
producer,
log,
connect,
sendMessage,
};