Add smart-money option signal path and tape filters

packages/storage/src/clickhouse.ts

@@ -20,11 +20,14 @@ import type {
   InferredDarkEvent,
   FlowPacket,
   OptionNBBO,
-  OptionPrint
+  OptionPrint,
+  OptionFlowFilters,
+  OptionFlowView
 } from "@islandflow/types";
 import {
   normalizeOptionPrint,
   optionPrintsTableDDL,
+  optionPrintsTableMigrations,
   OPTION_PRINTS_TABLE
 } from "./option-prints";
 import { normalizeOptionNBBO, optionNBBOTableDDL, OPTION_NBBO_TABLE } from "./option-nbbo";
@@ -221,6 +224,9 @@ export const ensureOptionPrintsTable = async (
   await client.exec({
     query: optionPrintsTableDDL()
   });
+  for (const query of optionPrintsTableMigrations()) {
+    await client.exec({ query });
+  }
 };
 
 export const ensureOptionNBBOTable = async (
@@ -499,19 +505,78 @@ const normalizeNumericFields = (
 
 const normalizeOptionRow = (row: unknown): unknown => {
   if (row && typeof row === "object") {
-    return normalizeNumericFields(row as Record<string, unknown>, [
+    const record = normalizeNumericFields(row as Record<string, unknown>, [
       "source_ts",
       "ingest_ts",
       "seq",
       "ts",
       "price",
-      "size"
+      "size",
+      "notional"
     ]);
+
+    if ("is_etf" in record) {
+      record.is_etf = Boolean(record.is_etf);
+    }
+    if ("signal_pass" in record) {
+      record.signal_pass = Boolean(record.signal_pass);
+    }
+    if (record.signal_reasons == null) {
+      record.signal_reasons = [];
+    }
+    return record;
   }
 
   return row;
 };
 
+export type OptionPrintQueryFilters = {
+  view?: OptionFlowView;
+  minNotional?: number;
+  security?: "stock" | "etf" | "all";
+  optionTypes?: string[];
+  nbboSides?: string[];
+};
+
+const buildOptionPrintFilterConditions = (
+  filters: OptionPrintQueryFilters | undefined,
+  tracePrefix: string | undefined
+): string[] => {
+  const conditions: string[] = [];
+  const traceCondition = buildTracePrefixCondition(tracePrefix);
+  if (traceCondition) {
+    conditions.push(traceCondition);
+  }
+
+  if (!filters) {
+    return conditions;
+  }
+
+  if ((filters.view ?? "signal") === "signal") {
+    conditions.push("signal_pass = 1");
+  }
+
+  if (typeof filters.minNotional === "number" && Number.isFinite(filters.minNotional)) {
+    conditions.push(`notional >= ${filters.minNotional}`);
+  }
+
+  if (filters.security === "stock") {
+    conditions.push("(is_etf = 0 OR is_etf IS NULL)");
+  } else if (filters.security === "etf") {
+    conditions.push("is_etf = 1");
+  }
+
+  if (filters.optionTypes && filters.optionTypes.length > 0) {
+    conditions.push(`option_type IN (${buildStringList(filters.optionTypes)})`);
+  }
+
+  if (filters.nbboSides && filters.nbboSides.length > 0) {
+    conditions.push(`nbbo_side IN (${buildStringList(filters.nbboSides)})`);
+  }
+
+  return conditions;
+};
+
 const normalizeOptionNbboRow = (row: unknown): unknown => {
   if (row && typeof row === "object") {
     return normalizeNumericFields(row as Record<string, unknown>, [
@@ -683,11 +748,12 @@ const normalizeAlertRow = (row: unknown): AlertRecord | null => {
 export const fetchRecentOptionPrints = async (
   client: ClickHouseClient,
   limit: number,
-  tracePrefix?: string
+  tracePrefix?: string,
+  filters?: OptionPrintQueryFilters
 ): Promise<OptionPrint[]> => {
   const safeLimit = clampLimit(limit);
-  const condition = buildTracePrefixCondition(tracePrefix);
-  const whereClause = condition ? ` WHERE ${condition}` : "";
+  const conditions = buildOptionPrintFilterConditions(filters, tracePrefix);
+  const whereClause = conditions.length > 0 ? ` WHERE ${conditions.join(" AND ")}` : "";
   const result = await client.query({
     query: `SELECT * FROM ${OPTION_PRINTS_TABLE}${whereClause} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
     format: "JSONEachRow"
@@ -855,16 +921,19 @@ export const fetchOptionPrintsAfter = async (
   afterTs: number,
   afterSeq: number,
   limit: number,
-  tracePrefix?: string
+  tracePrefix?: string,
+  filters?: OptionPrintQueryFilters
 ): Promise<OptionPrint[]> => {
   const safeLimit = clampLimit(limit);
   const safeAfterTs = clampCursor(afterTs);
   const safeAfterSeq = clampCursor(afterSeq);
-  const traceCondition = buildTracePrefixCondition(tracePrefix);
-  const traceClause = traceCondition ? ` AND ${traceCondition}` : "";
+  const conditions = [
+    `((ts, seq) > (${safeAfterTs}, ${safeAfterSeq}))`,
+    ...buildOptionPrintFilterConditions(filters, tracePrefix)
+  ];
 
   const result = await client.query({
-    query: `SELECT * FROM ${OPTION_PRINTS_TABLE} WHERE (ts, seq) > (${safeAfterTs}, ${safeAfterSeq})${traceClause} ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
+    query: `SELECT * FROM ${OPTION_PRINTS_TABLE} WHERE ${conditions.join(" AND ")} ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
     format: "JSONEachRow"
   });
 
@@ -1122,14 +1191,14 @@ export const fetchOptionPrintsBefore = async (
   beforeTs: number,
   beforeSeq: number,
   limit: number,
-  tracePrefix?: string
+  tracePrefix?: string,
+  filters?: OptionPrintQueryFilters
 ): Promise<OptionPrint[]> => {
   const safeLimit = clampLimit(limit);
-  const conditions = [buildBeforeTupleCondition("ts", "seq", beforeTs, beforeSeq)];
-  const traceCondition = buildTracePrefixCondition(tracePrefix);
-  if (traceCondition) {
-    conditions.push(traceCondition);
-  }
+  const conditions = [
+    buildBeforeTupleCondition("ts", "seq", beforeTs, beforeSeq),
+    ...buildOptionPrintFilterConditions(filters, tracePrefix)
+  ];
 
   const result = await client.query({
     query: `SELECT * FROM ${OPTION_PRINTS_TABLE} WHERE ${conditions.join(" AND ")} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,

packages/storage/src/option-prints.ts

@@ -14,16 +14,38 @@ CREATE TABLE IF NOT EXISTS ${OPTION_PRINTS_TABLE} (
   price Float64,
   size UInt32,
   exchange String,
-  conditions Array(String)
+  conditions Array(String),
+  underlying_id Nullable(String),
+  option_type Nullable(String),
+  notional Nullable(Float64),
+  nbbo_side Nullable(String),
+  is_etf Nullable(Bool),
+  signal_pass Nullable(Bool),
+  signal_reasons Array(String) DEFAULT [],
+  signal_profile Nullable(String)
 )
 ENGINE = MergeTree
 ORDER BY (ts, option_contract_id)
 `;
 };
 
+export const optionPrintsTableMigrations = (): string[] => {
+  return [
+    `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS underlying_id Nullable(String)`,
+    `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS option_type Nullable(String)`,
+    `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS notional Nullable(Float64)`,
+    `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS nbbo_side Nullable(String)`,
+    `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS is_etf Nullable(Bool)`,
+    `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS signal_pass Nullable(Bool)`,
+    `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS signal_reasons Array(String) DEFAULT []`,
+    `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS signal_profile Nullable(String)`
+  ];
+};
+
 export const normalizeOptionPrint = (print: OptionPrint): OptionPrint => {
   return {
     ...print,
-    conditions: print.conditions ?? []
+    conditions: print.conditions ?? [],
+    signal_reasons: print.signal_reasons ?? []
   };
 };

packages/storage/tests/option-prints.test.ts

@@ -1,5 +1,10 @@
 import { describe, expect, it } from "bun:test";
-import { createClickHouseClient, fetchOptionPrintsBefore, fetchOptionPrintsByTraceIds } from "../src/clickhouse";
+import {
+  createClickHouseClient,
+  fetchOptionPrintsBefore,
+  fetchOptionPrintsByTraceIds,
+  fetchRecentOptionPrints
+} from "../src/clickhouse";
 import { normalizeOptionPrint, optionPrintsTableDDL, OPTION_PRINTS_TABLE } from "../src/option-prints";
 
 const basePrint = {
@@ -38,12 +43,24 @@ describe("option-prints storage helpers", () => {
       };
     };
 
+    await fetchRecentOptionPrints(client, 25, undefined, {
+      view: "signal",
+      security: "stock",
+      nbboSides: ["AA", "A"],
+      optionTypes: ["call"],
+      minNotional: 25_000
+    });
     await fetchOptionPrintsBefore(client, 100, 5, 20, "alpaca");
     await fetchOptionPrintsByTraceIds(client, ["trace-1", "trace-2"]);
 
-    expect(queries[0]).toContain("(ts, seq) < (100, 5)");
-    expect(queries[0]).toContain("startsWith(trace_id, 'alpaca')");
-    expect(queries[0]).toContain("ORDER BY ts DESC, seq DESC LIMIT 20");
-    expect(queries[1]).toContain("trace_id IN ('trace-1', 'trace-2')");
+    expect(queries[0]).toContain("signal_pass = 1");
+    expect(queries[0]).toContain("(is_etf = 0 OR is_etf IS NULL)");
+    expect(queries[0]).toContain("nbbo_side IN ('AA', 'A')");
+    expect(queries[0]).toContain("option_type IN ('call')");
+    expect(queries[0]).toContain("notional >= 25000");
+    expect(queries[1]).toContain("(ts, seq) < (100, 5)");
+    expect(queries[1]).toContain("startsWith(trace_id, 'alpaca')");
+    expect(queries[1]).toContain("ORDER BY ts DESC, seq DESC LIMIT 20");
+    expect(queries[2]).toContain("trace_id IN ('trace-1', 'trace-2')");
   });
 });