Add classifier hits and alerts pipeline

Add NATS subjects + ClickHouse tables for classifier hits/alerts, evaluate sweep/spike rules in compute, expose API/WS endpoints, and cover storage helpers with tests.

packages/bus/src/subjects.ts

@@ -4,3 +4,7 @@ export const STREAM_EQUITY_PRINTS = "EQUITY_PRINTS";
 export const SUBJECT_EQUITY_PRINTS = "equities.prints";
 export const STREAM_FLOW_PACKETS = "FLOW_PACKETS";
 export const SUBJECT_FLOW_PACKETS = "flow.packets";
+export const STREAM_CLASSIFIER_HITS = "CLASSIFIER_HITS";
+export const SUBJECT_CLASSIFIER_HITS = "flow.classifier_hits";
+export const STREAM_ALERTS = "ALERTS";
+export const SUBJECT_ALERTS = "flow.alerts";

packages/storage/src/alerts.ts

@@ -0,0 +1,93 @@
+import type { AlertEvent, ClassifierHit } from "@islandflow/types";
+
+export const ALERTS_TABLE = "alerts";
+
+export type AlertRecord = {
+  source_ts: number;
+  ingest_ts: number;
+  seq: number;
+  trace_id: string;
+  score: number;
+  severity: string;
+  hits_json: string;
+  evidence_refs_json: string;
+};
+
+export const alertsTableDDL = (): string => {
+  return `
+CREATE TABLE IF NOT EXISTS ${ALERTS_TABLE} (
+  source_ts UInt64,
+  ingest_ts UInt64,
+  seq UInt64,
+  trace_id String,
+  score Float64,
+  severity String,
+  hits_json String,
+  evidence_refs_json String
+)
+ENGINE = MergeTree
+ORDER BY (source_ts, seq)
+`;
+};
+
+export const toAlertRecord = (alert: AlertEvent): AlertRecord => {
+  return {
+    source_ts: alert.source_ts,
+    ingest_ts: alert.ingest_ts,
+    seq: alert.seq,
+    trace_id: alert.trace_id,
+    score: alert.score,
+    severity: alert.severity,
+    hits_json: JSON.stringify(alert.hits),
+    evidence_refs_json: JSON.stringify(alert.evidence_refs)
+  };
+};
+
+const safeHitArray = (value: string): ClassifierHit[] => {
+  try {
+    const parsed = JSON.parse(value);
+    if (Array.isArray(parsed)) {
+      return parsed.map((entry) => {
+        const record = entry as Partial<ClassifierHit>;
+        return {
+          classifier_id: String(record.classifier_id ?? ""),
+          confidence: Number(record.confidence ?? 0),
+          direction: String(record.direction ?? ""),
+          explanations: Array.isArray(record.explanations)
+            ? record.explanations.map((item) => String(item))
+            : []
+        };
+      });
+    }
+  } catch {
+    // ignore
+  }
+
+  return [];
+};
+
+const safeStringArray = (value: string): string[] => {
+  try {
+    const parsed = JSON.parse(value);
+    if (Array.isArray(parsed)) {
+      return parsed.map((entry) => String(entry));
+    }
+  } catch {
+    // ignore
+  }
+
+  return [];
+};
+
+export const fromAlertRecord = (record: AlertRecord): AlertEvent => {
+  return {
+    source_ts: record.source_ts,
+    ingest_ts: record.ingest_ts,
+    seq: record.seq,
+    trace_id: record.trace_id,
+    score: record.score,
+    severity: record.severity,
+    hits: safeHitArray(record.hits_json),
+    evidence_refs: safeStringArray(record.evidence_refs_json)
+  };
+};

packages/storage/src/classifier-hits.ts

@@ -0,0 +1,70 @@
+import type { ClassifierHitEvent } from "@islandflow/types";
+
+export const CLASSIFIER_HITS_TABLE = "classifier_hits";
+
+export type ClassifierHitRecord = {
+  source_ts: number;
+  ingest_ts: number;
+  seq: number;
+  trace_id: string;
+  classifier_id: string;
+  confidence: number;
+  direction: string;
+  explanations_json: string;
+};
+
+export const classifierHitsTableDDL = (): string => {
+  return `
+CREATE TABLE IF NOT EXISTS ${CLASSIFIER_HITS_TABLE} (
+  source_ts UInt64,
+  ingest_ts UInt64,
+  seq UInt64,
+  trace_id String,
+  classifier_id String,
+  confidence Float64,
+  direction String,
+  explanations_json String
+)
+ENGINE = MergeTree
+ORDER BY (source_ts, seq)
+`;
+};
+
+export const toClassifierHitRecord = (hit: ClassifierHitEvent): ClassifierHitRecord => {
+  return {
+    source_ts: hit.source_ts,
+    ingest_ts: hit.ingest_ts,
+    seq: hit.seq,
+    trace_id: hit.trace_id,
+    classifier_id: hit.classifier_id,
+    confidence: hit.confidence,
+    direction: hit.direction,
+    explanations_json: JSON.stringify(hit.explanations)
+  };
+};
+
+const safeJsonArray = (value: string): string[] => {
+  try {
+    const parsed = JSON.parse(value);
+    if (Array.isArray(parsed)) {
+      return parsed.map((entry) => String(entry));
+    }
+  } catch {
+    // ignore
+  }
+
+  return [];
+};
+
+export const fromClassifierHitRecord = (record: ClassifierHitRecord): ClassifierHitEvent => {
+  return {
+    source_ts: record.source_ts,
+    ingest_ts: record.ingest_ts,
+    seq: record.seq,
+    trace_id: record.trace_id,
+    classifier_id: record.classifier_id,
+    confidence: record.confidence,
+    direction: record.direction,
+    explanations: safeJsonArray(record.explanations_json)
+  };
+};

packages/storage/src/clickhouse.ts

@@ -1,6 +1,18 @@
 import { createClient, type ClickHouseClient } from "@clickhouse/client";
-import { EquityPrintSchema, FlowPacketSchema, OptionPrintSchema } from "@islandflow/types";
-import type { EquityPrint, FlowPacket, OptionPrint } from "@islandflow/types";
+import {
+  AlertEventSchema,
+  ClassifierHitEventSchema,
+  EquityPrintSchema,
+  FlowPacketSchema,
+  OptionPrintSchema
+} from "@islandflow/types";
+import type {
+  AlertEvent,
+  ClassifierHitEvent,
+  EquityPrint,
+  FlowPacket,
+  OptionPrint
+} from "@islandflow/types";
 import {
   normalizeOptionPrint,
   optionPrintsTableDDL,
@@ -18,6 +30,20 @@ import {
   toFlowPacketRecord,
   type FlowPacketRecord
 } from "./flow-packets";
+import {
+  CLASSIFIER_HITS_TABLE,
+  classifierHitsTableDDL,
+  fromClassifierHitRecord,
+  toClassifierHitRecord,
+  type ClassifierHitRecord
+} from "./classifier-hits";
+import {
+  ALERTS_TABLE,
+  alertsTableDDL,
+  fromAlertRecord,
+  toAlertRecord,
+  type AlertRecord
+} from "./alerts";
 
 export type ClickHouseOptions = {
   url: string;
@@ -59,6 +85,20 @@ export const ensureFlowPacketsTable = async (
   });
 };
 
+export const ensureClassifierHitsTable = async (
+  client: ClickHouseClient
+): Promise<void> => {
+  await client.exec({
+    query: classifierHitsTableDDL()
+  });
+};
+
+export const ensureAlertsTable = async (client: ClickHouseClient): Promise<void> => {
+  await client.exec({
+    query: alertsTableDDL()
+  });
+};
+
 export const insertOptionPrint = async (
   client: ClickHouseClient,
   print: OptionPrint
@@ -95,6 +135,27 @@ export const insertFlowPacket = async (
   });
 };
 
+export const insertClassifierHit = async (
+  client: ClickHouseClient,
+  hit: ClassifierHitEvent
+): Promise<void> => {
+  const record = toClassifierHitRecord(hit);
+  await client.insert({
+    table: CLASSIFIER_HITS_TABLE,
+    values: [record],
+    format: "JSONEachRow"
+  });
+};
+
+export const insertAlert = async (client: ClickHouseClient, alert: AlertEvent): Promise<void> => {
+  const record = toAlertRecord(alert);
+  await client.insert({
+    table: ALERTS_TABLE,
+    values: [record],
+    format: "JSONEachRow"
+  });
+};
+
 const clampLimit = (limit: number): number => {
   if (!Number.isFinite(limit)) {
     return 100;
@@ -196,6 +257,42 @@ const normalizeFlowPacketRow = (row: unknown): FlowPacketRecord | null => {
   };
 };
 
+const normalizeClassifierHitRow = (row: unknown): ClassifierHitRecord | null => {
+  if (!row || typeof row !== "object") {
+    return null;
+  }
+
+  const record = row as Record<string, unknown>;
+  return {
+    source_ts: coerceNumber(record.source_ts) as number,
+    ingest_ts: coerceNumber(record.ingest_ts) as number,
+    seq: coerceNumber(record.seq) as number,
+    trace_id: String(record.trace_id ?? ""),
+    classifier_id: String(record.classifier_id ?? ""),
+    confidence: Number(coerceNumber(record.confidence) ?? 0),
+    direction: String(record.direction ?? ""),
+    explanations_json: String(record.explanations_json ?? "[]")
+  };
+};
+
+const normalizeAlertRow = (row: unknown): AlertRecord | null => {
+  if (!row || typeof row !== "object") {
+    return null;
+  }
+
+  const record = row as Record<string, unknown>;
+  return {
+    source_ts: coerceNumber(record.source_ts) as number,
+    ingest_ts: coerceNumber(record.ingest_ts) as number,
+    seq: coerceNumber(record.seq) as number,
+    trace_id: String(record.trace_id ?? ""),
+    score: Number(coerceNumber(record.score) ?? 0),
+    severity: String(record.severity ?? ""),
+    hits_json: String(record.hits_json ?? "[]"),
+    evidence_refs_json: String(record.evidence_refs_json ?? "[]")
+  };
+};
+
 export const fetchRecentOptionPrints = async (
   client: ClickHouseClient,
   limit: number
@@ -242,6 +339,42 @@ export const fetchRecentFlowPackets = async (
   return FlowPacketSchema.array().parse(packets);
 };
 
+export const fetchRecentClassifierHits = async (
+  client: ClickHouseClient,
+  limit: number
+): Promise<ClassifierHitEvent[]> => {
+  const safeLimit = clampLimit(limit);
+  const result = await client.query({
+    query: `SELECT * FROM ${CLASSIFIER_HITS_TABLE} ORDER BY source_ts DESC, seq DESC LIMIT ${safeLimit}`,
+    format: "JSONEachRow"
+  });
+
+  const rows = await result.json<unknown[]>();
+  const records = rows
+    .map(normalizeClassifierHitRow)
+    .filter((record): record is ClassifierHitRecord => record !== null);
+  const hits = records.map(fromClassifierHitRecord);
+  return ClassifierHitEventSchema.array().parse(hits);
+};
+
+export const fetchRecentAlerts = async (
+  client: ClickHouseClient,
+  limit: number
+): Promise<AlertEvent[]> => {
+  const safeLimit = clampLimit(limit);
+  const result = await client.query({
+    query: `SELECT * FROM ${ALERTS_TABLE} ORDER BY source_ts DESC, seq DESC LIMIT ${safeLimit}`,
+    format: "JSONEachRow"
+  });
+
+  const rows = await result.json<unknown[]>();
+  const records = rows
+    .map(normalizeAlertRow)
+    .filter((record): record is AlertRecord => record !== null);
+  const alerts = records.map(fromAlertRecord);
+  return AlertEventSchema.array().parse(alerts);
+};
+
 export const fetchOptionPrintsAfter = async (
   client: ClickHouseClient,
   afterTs: number,

packages/storage/src/index.ts

@@ -1,4 +1,6 @@
 export * from "./clickhouse";
+export * from "./classifier-hits";
+export * from "./alerts";
 export * from "./flow-packets";
 export * from "./equity-prints";
 export * from "./option-prints";

packages/storage/tests/alerts.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, it } from "bun:test";
+import { alertsTableDDL, ALERTS_TABLE, fromAlertRecord, toAlertRecord } from "../src/alerts";
+
+const alert = {
+  source_ts: 10,
+  ingest_ts: 20,
+  seq: 1,
+  trace_id: "alert:fp-1",
+  score: 78,
+  severity: "medium",
+  hits: [
+    {
+      classifier_id: "large_bullish_call_sweep",
+      confidence: 0.72,
+      direction: "bullish",
+      explanations: ["Likely call sweep.", "Premium $50000."]
+    }
+  ],
+  evidence_refs: ["flowpacket:1", "print:1"]
+};
+
+describe("alerts storage helpers", () => {
+  it("includes the correct table name in the DDL", () => {
+    const ddl = alertsTableDDL();
+    expect(ddl).toContain(ALERTS_TABLE);
+    expect(ddl).toContain("CREATE TABLE IF NOT EXISTS");
+  });
+
+  it("round-trips alert records", () => {
+    const record = toAlertRecord(alert);
+    const restored = fromAlertRecord(record);
+    expect(restored.hits).toEqual(alert.hits);
+    expect(restored.evidence_refs).toEqual(alert.evidence_refs);
+    expect(restored.severity).toBe(alert.severity);
+  });
+});

packages/storage/tests/classifier-hits.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from "bun:test";
+import {
+  classifierHitsTableDDL,
+  CLASSIFIER_HITS_TABLE,
+  fromClassifierHitRecord,
+  toClassifierHitRecord
+} from "../src/classifier-hits";
+
+const hit = {
+  source_ts: 10,
+  ingest_ts: 20,
+  seq: 1,
+  trace_id: "classifier:large_bullish_call_sweep:fp-1",
+  classifier_id: "large_bullish_call_sweep",
+  confidence: 0.72,
+  direction: "bullish",
+  explanations: ["Likely call sweep.", "Premium $50000."]
+};
+
+describe("classifier hits storage helpers", () => {
+  it("includes the correct table name in the DDL", () => {
+    const ddl = classifierHitsTableDDL();
+    expect(ddl).toContain(CLASSIFIER_HITS_TABLE);
+    expect(ddl).toContain("CREATE TABLE IF NOT EXISTS");
+  });
+
+  it("round-trips classifier hit records", () => {
+    const record = toClassifierHitRecord(hit);
+    const restored = fromClassifierHitRecord(record);
+    expect(restored.explanations).toEqual(hit.explanations);
+    expect(restored.classifier_id).toBe(hit.classifier_id);
+    expect(restored.direction).toBe(hit.direction);
+  });
+});