Implement first-pass load reduction controls

packages/bus/src/jetstream.ts

@@ -84,6 +84,50 @@ export const ensureStream = async (
   }
 };
 
+const parseBoundedNumber = (value: string | undefined, fallback: number): number => {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed < 0) {
+    return fallback;
+  }
+  return Math.floor(parsed);
+};
+
+export type StreamRetentionClass = "raw" | "derived";
+
+export const resolveStreamRetention = (
+  streamClass: StreamRetentionClass,
+  env: Record<string, string | undefined> = process.env
+): Pick<StreamConfig, "max_bytes" | "max_age"> => {
+  if (streamClass === "raw") {
+    return {
+      max_age: parseBoundedNumber(env.STREAM_RAW_MAX_AGE_MS, 7_200_000),
+      max_bytes: parseBoundedNumber(env.STREAM_RAW_MAX_BYTES, 1_073_741_824)
+    };
+  }
+
+  return {
+    max_age: parseBoundedNumber(env.STREAM_DERIVED_MAX_AGE_MS, 86_400_000),
+    max_bytes: parseBoundedNumber(env.STREAM_DERIVED_MAX_BYTES, 536_870_912)
+  };
+};
+
+export const buildStreamConfig = (
+  name: string,
+  subject: string,
+  streamClass: StreamRetentionClass,
+  env: Record<string, string | undefined> = process.env
+): StreamConfig => ({
+  name,
+  subjects: [subject],
+  retention: "limits",
+  storage: "file",
+  discard: "old",
+  max_msgs_per_subject: -1,
+  max_msgs: -1,
+  ...resolveStreamRetention(streamClass, env),
+  num_replicas: 1
+});
+
 export const buildDurableConsumer = (
   durableName: string,
   deliverSubject: string = createInbox()

packages/observability/src/logger.ts

@@ -22,20 +22,46 @@ export type LoggerOptions = {
   service: string;
   now?: () => string;
   sink?: (record: LogRecord) => void;
+  level?: LogLevel;
 };
 
 const defaultSink = (record: LogRecord) => {
   console.log(JSON.stringify(record));
 };
 
+const LOG_LEVEL_ORDER: Record<LogLevel, number> = {
+  debug: 10,
+  info: 20,
+  warn: 30,
+  error: 40
+};
+
+const resolveLogLevel = (value: string | undefined): LogLevel => {
+  switch ((value ?? "").trim().toLowerCase()) {
+    case "debug":
+    case "info":
+    case "warn":
+    case "error":
+      return value!.trim().toLowerCase() as LogLevel;
+    default:
+      return "info";
+  }
+};
+
 export const createLogger = ({
   service,
   now = () => new Date().toISOString(),
-  sink = defaultSink
+  sink = defaultSink,
+  level = resolveLogLevel(process.env.LOG_LEVEL)
 }: LoggerOptions): Logger => {
-  const write = (level: LogLevel, msg: string, context?: LogContext) => {
+  const levelThreshold = resolveLogLevel(level);
+
+  const write = (recordLevel: LogLevel, msg: string, context?: LogContext) => {
+    if (LOG_LEVEL_ORDER[recordLevel] < LOG_LEVEL_ORDER[levelThreshold]) {
+      return;
+    }
     const record: LogRecord = {
-      level,
+      level: recordLevel,
       service,
       msg,
       ts: now(),

packages/storage/src/clickhouse.ts

@@ -449,6 +449,157 @@ export const insertAlert = async (client: ClickHouseClient, alert: AlertEvent):
   });
 };
 
+export type ClickHouseBatchWriterOptions = {
+  flushIntervalMs?: number;
+  maxRows?: number;
+  onError?: (table: string, error: unknown, rowCount: number) => void;
+};
+
+type BatchState = {
+  rows: unknown[];
+  timer: ReturnType<typeof setTimeout> | null;
+  flushing: Promise<void> | null;
+};
+
+const createBatchState = (): BatchState => ({
+  rows: [],
+  timer: null,
+  flushing: null
+});
+
+export class ClickHouseBatchWriter {
+  private readonly flushIntervalMs: number;
+  private readonly maxRows: number;
+  private readonly states = new Map<string, BatchState>();
+
+  constructor(
+    private readonly client: ClickHouseClient,
+    options: ClickHouseBatchWriterOptions = {}
+  ) {
+    this.flushIntervalMs = Math.max(1, Math.floor(options.flushIntervalMs ?? 100));
+    this.maxRows = Math.max(1, Math.floor(options.maxRows ?? 250));
+    this.onError = options.onError;
+  }
+
+  private readonly onError?: (table: string, error: unknown, rowCount: number) => void;
+
+  enqueue(table: string, row: unknown): void {
+    const state = this.states.get(table) ?? createBatchState();
+    if (!this.states.has(table)) {
+      this.states.set(table, state);
+    }
+
+    state.rows.push(row);
+
+    if (state.rows.length >= this.maxRows) {
+      void this.flush(table);
+      return;
+    }
+
+    if (!state.timer) {
+      state.timer = setTimeout(() => {
+        state.timer = null;
+        void this.flush(table);
+      }, this.flushIntervalMs);
+    }
+  }
+
+  async flush(table: string): Promise<void> {
+    const state = this.states.get(table);
+    if (!state) {
+      return;
+    }
+
+    if (state.flushing) {
+      await state.flushing;
+      return;
+    }
+
+    if (state.timer) {
+      clearTimeout(state.timer);
+      state.timer = null;
+    }
+
+    if (state.rows.length === 0) {
+      return;
+    }
+
+    const rows = state.rows.splice(0, state.rows.length);
+    state.flushing = this.client
+      .insert({
+        table,
+        values: rows,
+        format: "JSONEachRow"
+      })
+      .catch((error) => {
+        this.onError?.(table, error, rows.length);
+      })
+      .finally(() => {
+        state.flushing = null;
+      });
+
+    await state.flushing;
+  }
+
+  async flushAll(): Promise<void> {
+    for (const table of this.states.keys()) {
+      await this.flush(table);
+    }
+  }
+
+  async close(): Promise<void> {
+    for (const state of this.states.values()) {
+      if (state.timer) {
+        clearTimeout(state.timer);
+        state.timer = null;
+      }
+    }
+    await this.flushAll();
+  }
+}
+
+export const enqueueEquityPrintJoinInsert = (
+  writer: ClickHouseBatchWriter,
+  join: EquityPrintJoin
+): void => {
+  writer.enqueue(EQUITY_PRINT_JOINS_TABLE, toEquityPrintJoinRecord(join));
+};
+
+export const enqueueInferredDarkInsert = (
+  writer: ClickHouseBatchWriter,
+  event: InferredDarkEvent
+): void => {
+  writer.enqueue(INFERRED_DARK_TABLE, toInferredDarkRecord(event));
+};
+
+export const enqueueFlowPacketInsert = (
+  writer: ClickHouseBatchWriter,
+  packet: FlowPacket
+): void => {
+  writer.enqueue(FLOW_PACKETS_TABLE, toFlowPacketRecord(packet));
+};
+
+export const enqueueSmartMoneyEventInsert = (
+  writer: ClickHouseBatchWriter,
+  event: SmartMoneyEvent
+): void => {
+  writer.enqueue(SMART_MONEY_EVENTS_TABLE, toSmartMoneyEventRecord(event));
+};
+
+export const enqueueClassifierHitInsert = (
+  writer: ClickHouseBatchWriter,
+  hit: ClassifierHitEvent
+): void => {
+  writer.enqueue(CLASSIFIER_HITS_TABLE, toClassifierHitRecord(hit));
+};
+
+export const enqueueAlertInsert = (
+  writer: ClickHouseBatchWriter,
+  alert: AlertEvent
+): void => {
+  writer.enqueue(ALERTS_TABLE, toAlertRecord(alert));
+};
+
 const clampLimit = (limit: number): number => {
   if (!Number.isFinite(limit)) {
     return 100;