Add dark inference pipeline

packages/storage/src/clickhouse.ts

@@ -5,6 +5,7 @@ import {
   EquityPrintSchema,
   EquityQuoteSchema,
   EquityPrintJoinSchema,
+  InferredDarkEventSchema,
   FlowPacketSchema,
   OptionNBBOSchema,
   OptionPrintSchema
@@ -15,6 +16,7 @@ import type {
   EquityPrint,
   EquityQuote,
   EquityPrintJoin,
+  InferredDarkEvent,
   FlowPacket,
   OptionNBBO,
   OptionPrint
@@ -42,6 +44,13 @@ import {
   toEquityPrintJoinRecord,
   type EquityPrintJoinRecord
 } from "./equity-print-joins";
+import {
+  inferredDarkTableDDL,
+  INFERRED_DARK_TABLE,
+  fromInferredDarkRecord,
+  toInferredDarkRecord,
+  type InferredDarkRecord
+} from "./inferred-dark";
 import {
   FLOW_PACKETS_TABLE,
   flowPacketsTableDDL,
@@ -120,6 +129,14 @@ export const ensureEquityPrintJoinsTable = async (
   });
 };
 
+export const ensureInferredDarkTable = async (
+  client: ClickHouseClient
+): Promise<void> => {
+  await client.exec({
+    query: inferredDarkTableDDL()
+  });
+};
+
 export const ensureFlowPacketsTable = async (
   client: ClickHouseClient
 ): Promise<void> => {
@@ -202,6 +219,18 @@ export const insertEquityPrintJoin = async (
   });
 };
 
+export const insertInferredDark = async (
+  client: ClickHouseClient,
+  event: InferredDarkEvent
+): Promise<void> => {
+  const record = toInferredDarkRecord(event);
+  await client.insert({
+    table: INFERRED_DARK_TABLE,
+    values: [record],
+    format: "JSONEachRow"
+  });
+};
+
 export const insertFlowPacket = async (
   client: ClickHouseClient,
   packet: FlowPacket
@@ -367,6 +396,23 @@ const normalizeEquityPrintJoinRow = (row: unknown): EquityPrintJoinRecord | null
   };
 };
 
+const normalizeInferredDarkRow = (row: unknown): InferredDarkRecord | null => {
+  if (!row || typeof row !== "object") {
+    return null;
+  }
+
+  const record = row as Record<string, unknown>;
+  return {
+    source_ts: coerceNumber(record.source_ts) as number,
+    ingest_ts: coerceNumber(record.ingest_ts) as number,
+    seq: coerceNumber(record.seq) as number,
+    trace_id: String(record.trace_id ?? ""),
+    type: String(record.type ?? ""),
+    confidence: Number(coerceNumber(record.confidence) ?? 0),
+    evidence_refs_json: String(record.evidence_refs_json ?? "[]")
+  };
+};
+
 const normalizeFlowPacketRow = (row: unknown): FlowPacketRecord | null => {
   if (!row || typeof row !== "object") {
     return null;
@@ -497,6 +543,24 @@ export const fetchRecentEquityPrintJoins = async (
   return EquityPrintJoinSchema.array().parse(joins);
 };
 
+export const fetchRecentInferredDark = async (
+  client: ClickHouseClient,
+  limit: number
+): Promise<InferredDarkEvent[]> => {
+  const safeLimit = clampLimit(limit);
+  const result = await client.query({
+    query: `SELECT * FROM ${INFERRED_DARK_TABLE} ORDER BY source_ts DESC, seq DESC LIMIT ${safeLimit}`,
+    format: "JSONEachRow"
+  });
+
+  const rows = await result.json<unknown[]>();
+  const records = rows
+    .map(normalizeInferredDarkRow)
+    .filter((record): record is InferredDarkRecord => record !== null);
+  const events = records.map(fromInferredDarkRecord);
+  return InferredDarkEventSchema.array().parse(events);
+};
+
 export const fetchRecentFlowPackets = async (
   client: ClickHouseClient,
   limit: number
@@ -649,3 +713,26 @@ export const fetchEquityPrintJoinsAfter = async (
   const joins = records.map(fromEquityPrintJoinRecord);
   return EquityPrintJoinSchema.array().parse(joins);
 };
+
+export const fetchInferredDarkAfter = async (
+  client: ClickHouseClient,
+  afterTs: number,
+  afterSeq: number,
+  limit: number
+): Promise<InferredDarkEvent[]> => {
+  const safeLimit = clampLimit(limit);
+  const safeAfterTs = clampCursor(afterTs);
+  const safeAfterSeq = clampCursor(afterSeq);
+
+  const result = await client.query({
+    query: `SELECT * FROM ${INFERRED_DARK_TABLE} WHERE (source_ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY source_ts ASC, seq ASC LIMIT ${safeLimit}`,
+    format: "JSONEachRow"
+  });
+
+  const rows = await result.json<unknown[]>();
+  const records = rows
+    .map(normalizeInferredDarkRow)
+    .filter((record): record is InferredDarkRecord => record !== null);
+  const events = records.map(fromInferredDarkRecord);
+  return InferredDarkEventSchema.array().parse(events);
+};

packages/storage/src/index.ts

@@ -5,5 +5,6 @@ export * from "./flow-packets";
 export * from "./equity-prints";
 export * from "./equity-quotes";
 export * from "./equity-print-joins";
+export * from "./inferred-dark";
 export * from "./option-prints";
 export * from "./option-nbbo";

packages/storage/src/inferred-dark.ts

@@ -0,0 +1,66 @@
+import type { InferredDarkEvent } from "@islandflow/types";
+
+export const INFERRED_DARK_TABLE = "inferred_dark";
+
+export type InferredDarkRecord = {
+  source_ts: number;
+  ingest_ts: number;
+  seq: number;
+  trace_id: string;
+  type: string;
+  confidence: number;
+  evidence_refs_json: string;
+};
+
+export const inferredDarkTableDDL = (): string => {
+  return `
+CREATE TABLE IF NOT EXISTS ${INFERRED_DARK_TABLE} (
+  source_ts UInt64,
+  ingest_ts UInt64,
+  seq UInt64,
+  trace_id String,
+  type String,
+  confidence Float64,
+  evidence_refs_json String
+)
+ENGINE = MergeTree
+ORDER BY (source_ts, seq)
+`;
+};
+
+export const toInferredDarkRecord = (event: InferredDarkEvent): InferredDarkRecord => {
+  return {
+    source_ts: event.source_ts,
+    ingest_ts: event.ingest_ts,
+    seq: event.seq,
+    trace_id: event.trace_id,
+    type: event.type,
+    confidence: event.confidence,
+    evidence_refs_json: JSON.stringify(event.evidence_refs)
+  };
+};
+
+const safeStringArray = (value: string): string[] => {
+  try {
+    const parsed = JSON.parse(value);
+    if (Array.isArray(parsed)) {
+      return parsed.map((entry) => String(entry));
+    }
+  } catch {
+    // ignore
+  }
+
+  return [];
+};
+
+export const fromInferredDarkRecord = (record: InferredDarkRecord): InferredDarkEvent => {
+  return {
+    source_ts: record.source_ts,
+    ingest_ts: record.ingest_ts,
+    seq: record.seq,
+    trace_id: record.trace_id,
+    type: record.type,
+    confidence: record.confidence,
+    evidence_refs: safeStringArray(record.evidence_refs_json)
+  };
+};