+
+
+
+
+ );
+};
+
type AlertSeverityStripProps = {
alerts: AlertEvent[];
};
@@ -1503,6 +1832,7 @@ export default function HomePage() {
const [selectedAlert, setSelectedAlert] = useState
+
+
+ {error ? (
+
+
+ {statusText}
+
+
+ {lastUpdate ? `Updated ${formatTime(lastUpdate)}` : "Waiting for data"}
+
+ Chart error: {error}
+ ) : !hasData ? (
+
+ {mode === "live"
+ ? "No candles yet. Start candles service."
+ : "No candles for this replay window."}
+
+ ) : null}
+
+
+
+
diff --git a/packages/storage/src/clickhouse.ts b/packages/storage/src/clickhouse.ts
index 0aea7c5..594dec9 100644
--- a/packages/storage/src/clickhouse.ts
+++ b/packages/storage/src/clickhouse.ts
@@ -327,7 +327,8 @@ const coerceNumber = (value: unknown): unknown => {
};
const quoteString = (value: string): string => {
- return JSON.stringify(value);
+ const escaped = value.replace(/'/g, "''");
+ return `'${escaped}'`;
};
const normalizeNumericFields = (
diff --git a/scripts/dev.ts b/scripts/dev.ts
index 84fbc51..97651cf 100644
--- a/scripts/dev.ts
+++ b/scripts/dev.ts
@@ -1,3 +1,5 @@
+import net from "node:net";
+
type ChildSpec = {
name: string;
cmd: string[];
@@ -12,6 +14,54 @@ type Child = {
const children: Child[] = [];
let shuttingDown = false;
+const sleep = (delayMs: number): Promise => {
+ return new Promise((resolve) => setTimeout(resolve, delayMs));
+};
+
+const parseUrlHostPort = (
+ value: string,
+ fallbackHost: string,
+ fallbackPort: number
+): { host: string; port: number } => {
+ const candidate = value.split(",")[0]?.trim() ?? "";
+ if (!candidate) {
+ return { host: fallbackHost, port: fallbackPort };
+ }
+
+ try {
+ const url = new URL(candidate.includes("://") ? candidate : `tcp://${candidate}`);
+ const port = url.port ? Number(url.port) : fallbackPort;
+ return { host: url.hostname || fallbackHost, port };
+ } catch {
+ return { host: fallbackHost, port: fallbackPort };
+ }
+};
+
+const checkTcp = (host: string, port: number, timeoutMs = 1000): Promise => {
+ return new Promise((resolve) => {
+ const socket = net.connect({ host, port });
+ const finalize = (ok: boolean) => {
+ socket.removeAllListeners();
+ socket.destroy();
+ resolve(ok);
+ };
+
+ socket.setTimeout(timeoutMs);
+ socket.once("connect", () => finalize(true));
+ socket.once("error", () => finalize(false));
+ socket.once("timeout", () => finalize(false));
+ });
+};
+
+const checkHttp = async (url: string): Promise => {
+ try {
+ const response = await fetch(url);
+ return response.ok;
+ } catch {
+ return false;
+ }
+};
+
const spawnChild = ({ name, cmd, cwd }: ChildSpec): void => {
const proc = Bun.spawn(cmd, {
cwd,
@@ -56,8 +106,44 @@ const shutdown = (code: number): void => {
process.on("SIGINT", () => shutdown(0));
process.on("SIGTERM", () => shutdown(0));
-const tasks: ChildSpec[] = [
- { name: "infra", cmd: ["docker", "compose", "up"] },
+const waitForInfra = async (): Promise => {
+ const natsTarget = parseUrlHostPort(process.env.NATS_URL ?? "", "127.0.0.1", 4222);
+ const redisTarget = parseUrlHostPort(process.env.REDIS_URL ?? "", "127.0.0.1", 6379);
+ const clickhouseUrl = process.env.CLICKHOUSE_URL ?? "http://127.0.0.1:8123";
+ const deadline = Date.now() + 90_000;
+ let lastLog = 0;
+
+ while (Date.now() < deadline) {
+ const [natsOk, redisOk, clickhouseOk] = await Promise.all([
+ checkTcp(natsTarget.host, natsTarget.port),
+ checkTcp(redisTarget.host, redisTarget.port),
+ checkHttp(`${clickhouseUrl.replace(/\/$/, "")}/ping`)
+ ]);
+
+ if (natsOk && redisOk && clickhouseOk) {
+ console.log("[dev] Infra ready");
+ return;
+ }
+
+ const now = Date.now();
+ if (now - lastLog > 5000) {
+ console.log(
+ `[dev] Waiting for infra... nats=${natsOk ? "up" : "down"} redis=${
+ redisOk ? "up" : "down"
+ } clickhouse=${clickhouseOk ? "up" : "down"}`
+ );
+ lastLog = now;
+ }
+
+ await sleep(1000);
+ }
+
+ console.error("[dev] Infra not ready after 90s. Check Docker/ports and retry.");
+ shutdown(1);
+};
+
+const infraTask: ChildSpec = { name: "infra", cmd: ["docker", "compose", "up"] };
+const serviceTasks: ChildSpec[] = [
{ name: "web", cmd: ["bun", "run", "dev"], cwd: "apps/web" },
{ name: "ingest-options", cmd: ["bun", "run", "dev"], cwd: "services/ingest-options" },
{ name: "ingest-equities", cmd: ["bun", "run", "dev"], cwd: "services/ingest-equities" },
@@ -68,7 +154,10 @@ const tasks: ChildSpec[] = [
{ name: "api", cmd: ["bun", "run", "dev"], cwd: "services/api" }
];
-for (const task of tasks) {
+spawnChild(infraTask);
+await waitForInfra();
+
+for (const task of serviceTasks) {
spawnChild(task);
}
diff --git a/services/api/src/index.ts b/services/api/src/index.ts
index d789a42..df57d59 100644
--- a/services/api/src/index.ts
+++ b/services/api/src/index.ts
@@ -76,10 +76,10 @@ const logger = createLogger({ service });
const envSchema = z.object({
API_PORT: z.coerce.number().int().positive().default(4000),
- NATS_URL: z.string().default("nats://localhost:4222"),
- CLICKHOUSE_URL: z.string().default("http://localhost:8123"),
+ NATS_URL: z.string().default("nats://127.0.0.1:4222"),
+ CLICKHOUSE_URL: z.string().default("http://127.0.0.1:8123"),
CLICKHOUSE_DATABASE: z.string().default("default"),
- REDIS_URL: z.string().default("redis://localhost:6379"),
+ REDIS_URL: z.string().default("redis://127.0.0.1:6379"),
REST_DEFAULT_LIMIT: z.coerce.number().int().positive().default(200)
});
@@ -311,7 +311,7 @@ const run = async () => {
servers: env.NATS_URL,
name: service
},
- { attempts: 20, delayMs: 500 }
+ { attempts: 120, delayMs: 500 }
);
await ensureStream(jsm, {
diff --git a/services/candles/src/index.ts b/services/candles/src/index.ts
index dfb773d..a625509 100644
--- a/services/candles/src/index.ts
+++ b/services/candles/src/index.ts
@@ -26,11 +26,11 @@ const logger = createLogger({ service });
const metrics = createMetrics({ service });
const envSchema = z.object({
- NATS_URL: z.string().default("nats://localhost:4222"),
- CLICKHOUSE_URL: z.string().default("http://localhost:8123"),
+ NATS_URL: z.string().default("nats://127.0.0.1:4222"),
+ CLICKHOUSE_URL: z.string().default("http://127.0.0.1:8123"),
CLICKHOUSE_DATABASE: z.string().default("default"),
- REDIS_URL: z.string().default("redis://localhost:6379"),
- CANDLE_INTERVALS_MS: z.string().default("1000,5000,60000"),
+ REDIS_URL: z.string().default("redis://127.0.0.1:6379"),
+ CANDLE_INTERVALS_MS: z.string().default("60000,300000"),
CANDLE_MAX_LATE_MS: z.coerce.number().int().nonnegative().default(0),
CANDLE_CACHE_LIMIT: z.coerce.number().int().nonnegative().default(2000),
CANDLE_DELIVER_POLICY: z
@@ -185,7 +185,7 @@ const emitCandle = async (
const run = async () => {
logger.info("service starting");
- const intervalsMs = parseIntervals(env.CANDLE_INTERVALS_MS, [1000, 5000, 60000]);
+ const intervalsMs = parseIntervals(env.CANDLE_INTERVALS_MS, [60000, 300000]);
if (intervalsMs.length === 0) {
throw new Error("CANDLE_INTERVALS_MS produced no valid intervals");
}
@@ -200,7 +200,7 @@ const run = async () => {
servers: env.NATS_URL,
name: service
},
- { attempts: 20, delayMs: 500 }
+ { attempts: 120, delayMs: 500 }
);
await ensureStream(jsm, {
diff --git a/services/compute/src/index.ts b/services/compute/src/index.ts
index 0aa2d67..042e6df 100644
--- a/services/compute/src/index.ts
+++ b/services/compute/src/index.ts
@@ -74,10 +74,10 @@ const service = "compute";
const logger = createLogger({ service });
const envSchema = z.object({
- NATS_URL: z.string().default("nats://localhost:4222"),
- CLICKHOUSE_URL: z.string().default("http://localhost:8123"),
+ NATS_URL: z.string().default("nats://127.0.0.1:4222"),
+ CLICKHOUSE_URL: z.string().default("http://127.0.0.1:8123"),
CLICKHOUSE_DATABASE: z.string().default("default"),
- REDIS_URL: z.string().default("redis://localhost:6379"),
+ REDIS_URL: z.string().default("redis://127.0.0.1:6379"),
CLUSTER_WINDOW_MS: z.coerce.number().int().positive().default(500),
ROLLING_WINDOW_SIZE: z.coerce.number().int().positive().default(50),
ROLLING_TTL_SEC: z.coerce.number().int().nonnegative().default(86400),
@@ -758,7 +758,7 @@ const run = async () => {
servers: env.NATS_URL,
name: service
},
- { attempts: 20, delayMs: 500 }
+ { attempts: 120, delayMs: 500 }
);
await ensureStream(jsm, {
diff --git a/services/ingest-equities/src/adapters/synthetic.ts b/services/ingest-equities/src/adapters/synthetic.ts
index 861ffa7..6aa9f16 100644
--- a/services/ingest-equities/src/adapters/synthetic.ts
+++ b/services/ingest-equities/src/adapters/synthetic.ts
@@ -22,6 +22,10 @@ const DARK_SEQUENCE: DarkScenario[] = [
"sell",
"sell"
];
+const SYNTHETIC_SYMBOLS = [
+ "SPY",
+ ...SP500_SYMBOLS.filter((symbol) => symbol !== "SPY")
+];
const hashSymbol = (value: string): number => {
let hash = 0;
@@ -138,7 +142,7 @@ export const createSyntheticEquitiesAdapter = (
const now = Date.now();
const batchSize = 3;
- const darkSymbol = SP500_SYMBOLS[darkSymbolIndex % SP500_SYMBOLS.length];
+ const darkSymbol = SYNTHETIC_SYMBOLS[darkSymbolIndex % SYNTHETIC_SYMBOLS.length];
const darkHash = hashSymbol(darkSymbol);
const darkBase = 25 + (darkHash % 475);
const darkDrift = ((darkStep % 24) - 12) * 0.08;
@@ -189,7 +193,7 @@ export const createSyntheticEquitiesAdapter = (
for (let i = 0; i < batchSize; i += 1) {
seq += 1;
- const symbol = SP500_SYMBOLS[(seq + i) % SP500_SYMBOLS.length];
+ const symbol = SYNTHETIC_SYMBOLS[(seq + i) % SYNTHETIC_SYMBOLS.length];
const symbolHash = hashSymbol(symbol);
const basePrice = 25 + (symbolHash % 475);
const mid = formatPrice(basePrice + ((seq % 40) - 20) * 0.05);
diff --git a/services/ingest-equities/src/index.ts b/services/ingest-equities/src/index.ts
index de0c324..1572aa2 100644
--- a/services/ingest-equities/src/index.ts
+++ b/services/ingest-equities/src/index.ts
@@ -30,8 +30,8 @@ const service = "ingest-equities";
const logger = createLogger({ service });
const envSchema = z.object({
- NATS_URL: z.string().default("nats://localhost:4222"),
- CLICKHOUSE_URL: z.string().default("http://localhost:8123"),
+ NATS_URL: z.string().default("nats://127.0.0.1:4222"),
+ CLICKHOUSE_URL: z.string().default("http://127.0.0.1:8123"),
CLICKHOUSE_DATABASE: z.string().default("default"),
EQUITIES_INGEST_ADAPTER: z.string().min(1).default("synthetic"),
EMIT_INTERVAL_MS: z.coerce.number().int().positive().default(1000),
@@ -129,7 +129,7 @@ const run = async () => {
servers: env.NATS_URL,
name: service
},
- { attempts: 20, delayMs: 500 }
+ { attempts: 120, delayMs: 500 }
);
await ensureStream(jsm, {
diff --git a/services/ingest-options/src/adapters/synthetic.ts b/services/ingest-options/src/adapters/synthetic.ts
index 23bf302..a5cdf41 100644
--- a/services/ingest-options/src/adapters/synthetic.ts
+++ b/services/ingest-options/src/adapters/synthetic.ts
@@ -17,6 +17,10 @@ type Burst = {
seed: number;
};
+const SYNTHETIC_SYMBOLS = [
+ "SPY",
+ ...SP500_SYMBOLS.filter((symbol) => symbol !== "SPY")
+];
const MS_PER_DAY = 24 * 60 * 60 * 1000;
const EXPIRY_OFFSETS = [0, 1, 7, 14, 28, 45, 60, 90];
const EXCHANGES = ["CBOE", "PHLX", "ISE", "ARCA", "BOX", "MIAX"];
@@ -177,7 +181,7 @@ const formatExpiry = (now: number, offsetDays: number): string => {
};
const buildBurst = (burstIndex: number, now: number): Burst => {
- const symbol = SP500_SYMBOLS[burstIndex % SP500_SYMBOLS.length];
+ const symbol = SYNTHETIC_SYMBOLS[burstIndex % SYNTHETIC_SYMBOLS.length];
const symbolHash = hashSymbol(symbol);
const seed = symbolHash + burstIndex * 7;
const scenario = pickWeighted(SCENARIOS, seed);
diff --git a/services/ingest-options/src/index.ts b/services/ingest-options/src/index.ts
index 01de7b9..3fc61ba 100644
--- a/services/ingest-options/src/index.ts
+++ b/services/ingest-options/src/index.ts
@@ -28,8 +28,8 @@ const service = "ingest-options";
const logger = createLogger({ service });
const envSchema = z.object({
- NATS_URL: z.string().default("nats://localhost:4222"),
- CLICKHOUSE_URL: z.string().default("http://localhost:8123"),
+ NATS_URL: z.string().default("nats://127.0.0.1:4222"),
+ CLICKHOUSE_URL: z.string().default("http://127.0.0.1:8123"),
CLICKHOUSE_DATABASE: z.string().default("default"),
OPTIONS_INGEST_ADAPTER: z.string().min(1).default("synthetic"),
ALPACA_KEY_ID: z.string().default(""),
@@ -225,7 +225,7 @@ const run = async () => {
servers: env.NATS_URL,
name: service
},
- { attempts: 20, delayMs: 500 }
+ { attempts: 120, delayMs: 500 }
);
await ensureStream(jsm, {
From f96f5699ef3c06f8978881a006cc77e2a7f6bf9a Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Fri, 9 Jan 2026 16:01:08 -0500
Subject: [PATCH 007/237] Add additional flow classifiers
---
services/compute/src/classifiers.ts | 569 +++++++++++++++++++--
services/compute/tests/classifiers.test.ts | 92 +++-
2 files changed, 627 insertions(+), 34 deletions(-)
diff --git a/services/compute/src/classifiers.ts b/services/compute/src/classifiers.ts
index 5c9bbff..824886e 100644
--- a/services/compute/src/classifiers.ts
+++ b/services/compute/src/classifiers.ts
@@ -14,6 +14,8 @@ export type ClassifierConfig = {
minAggressorRatio: number;
};
+const MS_PER_DAY = 86_400_000;
+
const clamp = (value: number, min = 0, max = 1): number => {
if (!Number.isFinite(value)) {
return min;
@@ -33,8 +35,29 @@ const getNumberFeature = (packet: FlowPacket, key: string): number => {
return typeof value === "number" && Number.isFinite(value) ? value : 0;
};
+const getStringFeature = (packet: FlowPacket, key: string): string => {
+ const value = packet.features[key];
+ return typeof value === "string" ? value : "";
+};
+
const formatPct = (value: number): string => `${Math.round(value * 100)}%`;
+const getAggressorContext = (
+ packet: FlowPacket
+): {
+ coverage: number;
+ aggressiveBuyRatio: number;
+ aggressiveSellRatio: number;
+ aggressiveRatio: number;
+} => {
+ return {
+ coverage: getNumberFeature(packet, "nbbo_coverage_ratio"),
+ aggressiveBuyRatio: getNumberFeature(packet, "nbbo_aggressive_buy_ratio"),
+ aggressiveSellRatio: getNumberFeature(packet, "nbbo_aggressive_sell_ratio"),
+ aggressiveRatio: getNumberFeature(packet, "nbbo_aggressive_ratio")
+ };
+};
+
const applyAggressorAdjustment = (
confidence: number,
coverage: number,
@@ -61,6 +84,124 @@ const applyAggressorAdjustment = (
return { confidence: adjusted, note };
};
+type LargeActivity = {
+ count: number;
+ totalPremium: number;
+ totalSize: number;
+ windowMs: number;
+ premiumZ: number;
+ sizeZ: number;
+ premiumBaselineReady: boolean;
+ sizeBaselineReady: boolean;
+ passesAbsolute: boolean;
+ passesZ: boolean;
+ baselineNote: string;
+};
+
+const getLargeActivity = (packet: FlowPacket, config: ClassifierConfig): LargeActivity => {
+ const count = getNumberFeature(packet, "count");
+ const totalPremium = getNumberFeature(packet, "total_premium");
+ const totalSize = getNumberFeature(packet, "total_size");
+ const windowMs = getNumberFeature(packet, "window_ms");
+ const premiumZ = getNumberFeature(packet, "total_premium_z");
+ const sizeZ = getNumberFeature(packet, "total_size_z");
+ const premiumBaseline = getNumberFeature(packet, "total_premium_baseline_n");
+ const sizeBaseline = getNumberFeature(packet, "total_size_baseline_n");
+
+ const premiumBaselineReady = premiumBaseline >= config.zMinSamples;
+ const sizeBaselineReady = sizeBaseline >= config.zMinSamples;
+ const passesAbsolute = totalSize >= config.spikeMinSize && totalPremium >= config.spikeMinPremium;
+ const passesZ =
+ (premiumBaselineReady && premiumZ >= config.spikeMinPremiumZ) ||
+ (sizeBaselineReady && sizeZ >= config.spikeMinSizeZ);
+
+ const baselineNote =
+ premiumBaselineReady || sizeBaselineReady
+ ? `Baseline z-scores: premium ${premiumZ.toFixed(2)}, size ${sizeZ.toFixed(2)}.`
+ : "Baseline z-scores unavailable.";
+
+ return {
+ count,
+ totalPremium,
+ totalSize,
+ windowMs,
+ premiumZ,
+ sizeZ,
+ premiumBaselineReady,
+ sizeBaselineReady,
+ passesAbsolute,
+ passesZ,
+ baselineNote
+ };
+};
+
+const applySideAggressorAdjustment = (
+ confidence: number,
+ coverage: number,
+ ratio: number,
+ config: ClassifierConfig,
+ label: string
+): { confidence: number; note: string } => {
+ const normalizedCoverage = clamp(coverage, 0, 1);
+ const normalizedRatio = clamp(ratio, 0, 1);
+ let adjusted = confidence;
+
+ if (normalizedCoverage <= 0) {
+ return {
+ confidence: adjusted - 0.15,
+ note: "Aggressor mix unavailable (no NBBO coverage)."
+ };
+ }
+
+ if (normalizedCoverage < config.minNbboCoverage) {
+ adjusted -= 0.1;
+ }
+
+ if (normalizedRatio >= config.minAggressorRatio) {
+ adjusted += 0.05;
+ } else {
+ adjusted -= 0.1;
+ }
+
+ const note = `Aggressor mix ${formatPct(normalizedRatio)} ${label}, NBBO coverage ${formatPct(
+ normalizedCoverage
+ )}.`;
+
+ return { confidence: adjusted, note };
+};
+
+const getReferenceTs = (packet: FlowPacket): number | null => {
+ const endTs = getNumberFeature(packet, "end_ts");
+ if (endTs > 0) {
+ return endTs;
+ }
+
+ if (Number.isFinite(packet.source_ts) && packet.source_ts > 0) {
+ return packet.source_ts;
+ }
+
+ return null;
+};
+
+const getDteDays = (packet: FlowPacket, contract: ParsedContract): number | null => {
+ const expiryTs = Date.parse(`${contract.expiry}T00:00:00Z`);
+ if (!Number.isFinite(expiryTs)) {
+ return null;
+ }
+
+ const referenceTs = getReferenceTs(packet);
+ if (!referenceTs) {
+ return null;
+ }
+
+ const diffMs = expiryTs - referenceTs;
+ if (diffMs < 0) {
+ return null;
+ }
+
+ return Math.ceil(diffMs / MS_PER_DAY);
+};
+
const buildSweepHit = (
packet: FlowPacket,
contract: ParsedContract,
@@ -130,43 +271,30 @@ const buildSweepHit = (
};
const buildSpikeHit = (packet: FlowPacket, config: ClassifierConfig): ClassifierHit | null => {
- const count = getNumberFeature(packet, "count");
- const totalPremium = getNumberFeature(packet, "total_premium");
- const totalSize = getNumberFeature(packet, "total_size");
- const windowMs = getNumberFeature(packet, "window_ms");
- const premiumZ = getNumberFeature(packet, "total_premium_z");
- const sizeZ = getNumberFeature(packet, "total_size_z");
- const premiumBaseline = getNumberFeature(packet, "total_premium_baseline_n");
- const sizeBaseline = getNumberFeature(packet, "total_size_baseline_n");
- const coverage = getNumberFeature(packet, "nbbo_coverage_ratio");
- const aggressiveBuyRatio = getNumberFeature(packet, "nbbo_aggressive_buy_ratio");
- const aggressiveSellRatio = getNumberFeature(packet, "nbbo_aggressive_sell_ratio");
+ const activity = getLargeActivity(packet, config);
+ const { coverage, aggressiveBuyRatio, aggressiveSellRatio } = getAggressorContext(packet);
const aggressiveRatio = Math.max(aggressiveBuyRatio, aggressiveSellRatio);
- const premiumBaselineReady = premiumBaseline >= config.zMinSamples;
- const sizeBaselineReady = sizeBaseline >= config.zMinSamples;
- const passesAbsolute = totalSize >= config.spikeMinSize && totalPremium >= config.spikeMinPremium;
- const passesZ =
- (premiumBaselineReady && premiumZ >= config.spikeMinPremiumZ) ||
- (sizeBaselineReady && sizeZ >= config.spikeMinSizeZ);
-
- if (!passesAbsolute && !passesZ) {
+ if (!activity.passesAbsolute && !activity.passesZ) {
return null;
}
let confidence = 0.5;
- if (totalSize >= config.spikeMinSize * 2) {
+ if (activity.totalSize >= config.spikeMinSize * 2) {
confidence += 0.15;
}
- if (totalPremium >= config.spikeMinPremium * 2) {
+ if (activity.totalPremium >= config.spikeMinPremium * 2) {
confidence += 0.15;
}
- if (count >= 3) {
+ if (activity.count >= 3) {
confidence += 0.1;
}
- if (passesZ) {
+ if (activity.passesZ) {
confidence += 0.1;
- if (premiumZ >= config.spikeMinPremiumZ + 1 || sizeZ >= config.spikeMinSizeZ + 1) {
+ if (
+ activity.premiumZ >= config.spikeMinPremiumZ + 1 ||
+ activity.sizeZ >= config.spikeMinSizeZ + 1
+ ) {
confidence += 0.05;
}
}
@@ -174,20 +302,365 @@ const buildSpikeHit = (packet: FlowPacket, config: ClassifierConfig): Classifier
const aggressor = applyAggressorAdjustment(confidence, coverage, aggressiveRatio, config);
confidence = clamp(aggressor.confidence, 0, 0.9);
- const baselineNote =
- premiumBaselineReady || sizeBaselineReady
- ? `Baseline z-scores: premium ${premiumZ.toFixed(2)}, size ${sizeZ.toFixed(2)}.`
- : "Baseline z-scores unavailable.";
-
return {
classifier_id: "unusual_contract_spike",
confidence,
direction: "neutral",
explanations: [
- `Unusual contract spike: ${count} prints in ${Math.round(windowMs)}ms for ${packet.features.option_contract_id ?? packet.id}.`,
- `Premium ${formatUsd(totalPremium)} across ${Math.round(totalSize)} contracts.`,
+ `Unusual contract spike: ${activity.count} prints in ${Math.round(activity.windowMs)}ms for ${packet.features.option_contract_id ?? packet.id}.`,
+ `Premium ${formatUsd(activity.totalPremium)} across ${Math.round(activity.totalSize)} contracts.`,
`Thresholds: >=${config.spikeMinSize} contracts and >=${formatUsd(config.spikeMinPremium)} premium or z>=${config.spikeMinPremiumZ.toFixed(1)}.`,
- baselineNote,
+ activity.baselineNote,
+ aggressor.note
+ ]
+ };
+};
+
+const buildOverwriteHit = (
+ packet: FlowPacket,
+ contract: ParsedContract,
+ config: ClassifierConfig
+): ClassifierHit | null => {
+ if (contract.right !== "C") {
+ return null;
+ }
+
+ const activity = getLargeActivity(packet, config);
+ if (!activity.passesAbsolute && !activity.passesZ) {
+ return null;
+ }
+
+ const { coverage, aggressiveSellRatio } = getAggressorContext(packet);
+ let confidence = 0.45;
+ if (activity.totalPremium >= config.spikeMinPremium * 2) {
+ confidence += 0.15;
+ }
+ if (activity.totalSize >= config.spikeMinSize * 2) {
+ confidence += 0.1;
+ }
+ if (activity.count >= 3) {
+ confidence += 0.05;
+ }
+ if (activity.passesZ) {
+ confidence += 0.1;
+ }
+
+ const aggressor = applySideAggressorAdjustment(
+ confidence,
+ coverage,
+ aggressiveSellRatio,
+ config,
+ "sell-side"
+ );
+ confidence = clamp(aggressor.confidence, 0, 0.9);
+
+ return {
+ classifier_id: "large_call_sell_overwrite",
+ confidence,
+ direction: "bearish",
+ explanations: [
+ `Likely call overwrite: ${activity.count} prints in ${Math.round(activity.windowMs)}ms for ${packet.features.option_contract_id ?? packet.id}.`,
+ `Premium ${formatUsd(activity.totalPremium)} across ${Math.round(activity.totalSize)} contracts.`,
+ `Thresholds: >=${config.spikeMinSize} contracts and >=${formatUsd(config.spikeMinPremium)} premium or z>=${config.spikeMinPremiumZ.toFixed(1)}.`,
+ "Direction inferred from sell-side aggressor mix.",
+ activity.baselineNote,
+ aggressor.note
+ ]
+ };
+};
+
+const buildPutWriteHit = (
+ packet: FlowPacket,
+ contract: ParsedContract,
+ config: ClassifierConfig
+): ClassifierHit | null => {
+ if (contract.right !== "P") {
+ return null;
+ }
+
+ const activity = getLargeActivity(packet, config);
+ if (!activity.passesAbsolute && !activity.passesZ) {
+ return null;
+ }
+
+ const { coverage, aggressiveSellRatio } = getAggressorContext(packet);
+ let confidence = 0.45;
+ if (activity.totalPremium >= config.spikeMinPremium * 2) {
+ confidence += 0.15;
+ }
+ if (activity.totalSize >= config.spikeMinSize * 2) {
+ confidence += 0.1;
+ }
+ if (activity.count >= 3) {
+ confidence += 0.05;
+ }
+ if (activity.passesZ) {
+ confidence += 0.1;
+ }
+
+ const aggressor = applySideAggressorAdjustment(
+ confidence,
+ coverage,
+ aggressiveSellRatio,
+ config,
+ "sell-side"
+ );
+ confidence = clamp(aggressor.confidence, 0, 0.9);
+
+ return {
+ classifier_id: "large_put_sell_write",
+ confidence,
+ direction: "bullish",
+ explanations: [
+ `Likely put write: ${activity.count} prints in ${Math.round(activity.windowMs)}ms for ${packet.features.option_contract_id ?? packet.id}.`,
+ `Premium ${formatUsd(activity.totalPremium)} across ${Math.round(activity.totalSize)} contracts.`,
+ `Thresholds: >=${config.spikeMinSize} contracts and >=${formatUsd(config.spikeMinPremium)} premium or z>=${config.spikeMinPremiumZ.toFixed(1)}.`,
+ "Direction inferred from sell-side aggressor mix.",
+ activity.baselineNote,
+ aggressor.note
+ ]
+ };
+};
+
+const buildStraddleStrangleHit = (
+ packet: FlowPacket,
+ config: ClassifierConfig
+): ClassifierHit | null => {
+ const structureType = getStringFeature(packet, "structure_type");
+ if (structureType !== "straddle" && structureType !== "strangle") {
+ return null;
+ }
+
+ const activity = getLargeActivity(packet, config);
+ const { coverage, aggressiveBuyRatio, aggressiveSellRatio, aggressiveRatio } =
+ getAggressorContext(packet);
+ const structureLegs = getNumberFeature(packet, "structure_legs");
+ const structureStrikes = getNumberFeature(packet, "structure_strikes");
+ const strikeSpan = getNumberFeature(packet, "structure_strike_span");
+
+ let confidence = 0.45;
+ if (activity.totalPremium >= config.spikeMinPremium) {
+ confidence += 0.1;
+ }
+ if (activity.totalSize >= config.spikeMinSize) {
+ confidence += 0.05;
+ }
+ if (structureLegs >= 4) {
+ confidence += 0.05;
+ }
+
+ const aggressor = applyAggressorAdjustment(confidence, coverage, aggressiveRatio, config);
+ confidence = clamp(aggressor.confidence, 0, 0.85);
+
+ let volBias = "mixed aggressor skew";
+ if (aggressiveBuyRatio >= aggressiveSellRatio + 0.1) {
+ volBias = "buy-side skew suggests long volatility";
+ } else if (aggressiveSellRatio >= aggressiveBuyRatio + 0.1) {
+ volBias = "sell-side skew suggests short volatility";
+ }
+
+ const skewNote = `Aggressor skew: buy ${formatPct(aggressiveBuyRatio)}, sell ${formatPct(
+ aggressiveSellRatio
+ )}; ${volBias}.`;
+
+ return {
+ classifier_id: structureType === "straddle" ? "straddle" : "strangle",
+ confidence,
+ direction: "neutral",
+ explanations: [
+ `Likely ${structureType}: ${structureLegs} legs across ${structureStrikes} strikes (span ${strikeSpan}).`,
+ `Premium ${formatUsd(activity.totalPremium)} across ${Math.round(activity.totalSize)} contracts.`,
+ skewNote,
+ aggressor.note
+ ]
+ };
+};
+
+const buildVerticalSpreadHit = (
+ packet: FlowPacket,
+ config: ClassifierConfig
+): ClassifierHit | null => {
+ const structureType = getStringFeature(packet, "structure_type");
+ if (structureType !== "vertical") {
+ return null;
+ }
+
+ const structureRights = getStringFeature(packet, "structure_rights");
+ if (structureRights !== "C" && structureRights !== "P") {
+ return null;
+ }
+
+ const activity = getLargeActivity(packet, config);
+ const { coverage, aggressiveBuyRatio, aggressiveSellRatio } = getAggressorContext(packet);
+ const structureLegs = getNumberFeature(packet, "structure_legs");
+ const structureStrikes = getNumberFeature(packet, "structure_strikes");
+ const strikeSpan = getNumberFeature(packet, "structure_strike_span");
+
+ let confidence = 0.5;
+ if (activity.totalPremium >= config.spikeMinPremium) {
+ confidence += 0.1;
+ }
+ if (activity.totalSize >= config.spikeMinSize) {
+ confidence += 0.05;
+ }
+ if (structureLegs >= 3) {
+ confidence += 0.05;
+ }
+
+ let direction: "bullish" | "bearish" | "neutral" = "neutral";
+ let biasNote = "Debit/credit bias unclear (insufficient aggressor data).";
+ let aggressorNote = "Aggressor mix unavailable (no NBBO coverage).";
+ const hasAggressor = coverage > 0 && aggressiveBuyRatio + aggressiveSellRatio > 0;
+ if (hasAggressor) {
+ const buyDominant = aggressiveBuyRatio >= aggressiveSellRatio;
+ const dominantRatio = buyDominant ? aggressiveBuyRatio : aggressiveSellRatio;
+ const label = buyDominant ? "buy-side" : "sell-side";
+ const aggressor = applySideAggressorAdjustment(
+ confidence,
+ coverage,
+ dominantRatio,
+ config,
+ label
+ );
+ confidence = aggressor.confidence;
+ aggressorNote = aggressor.note;
+
+ const spreadBias = buyDominant ? "debit" : "credit";
+ biasNote = `Aggressor skew: buy ${formatPct(aggressiveBuyRatio)}, sell ${formatPct(
+ aggressiveSellRatio
+ )}; suggests ${spreadBias} ${structureRights === "C" ? "call" : "put"} vertical.`;
+
+ if (structureRights === "C") {
+ direction = buyDominant ? "bullish" : "bearish";
+ } else {
+ direction = buyDominant ? "bearish" : "bullish";
+ }
+ } else {
+ confidence -= 0.1;
+ }
+
+ confidence = clamp(confidence, 0, 0.85);
+
+ return {
+ classifier_id: "vertical_spread",
+ confidence,
+ direction,
+ explanations: [
+ `Likely vertical spread: ${structureLegs} legs across ${structureStrikes} strikes (span ${strikeSpan}).`,
+ `Premium ${formatUsd(activity.totalPremium)} across ${Math.round(activity.totalSize)} contracts.`,
+ biasNote,
+ aggressorNote,
+ "Direction inferred from debit/credit bias."
+ ]
+ };
+};
+
+const buildLadderHit = (
+ packet: FlowPacket,
+ config: ClassifierConfig
+): ClassifierHit | null => {
+ const structureType = getStringFeature(packet, "structure_type");
+ if (structureType !== "ladder") {
+ return null;
+ }
+
+ const activity = getLargeActivity(packet, config);
+ const { coverage, aggressiveRatio } = getAggressorContext(packet);
+ const structureRights = getStringFeature(packet, "structure_rights");
+ const structureLegs = getNumberFeature(packet, "structure_legs");
+ const structureStrikes = getNumberFeature(packet, "structure_strikes");
+ const strikeSpan = getNumberFeature(packet, "structure_strike_span");
+
+ const qualifies =
+ activity.totalPremium >= config.spikeMinPremium ||
+ activity.totalSize >= config.spikeMinSize ||
+ activity.passesZ;
+ if (!qualifies) {
+ return null;
+ }
+
+ let confidence = 0.45;
+ if (activity.totalPremium >= config.spikeMinPremium * 2) {
+ confidence += 0.1;
+ }
+ if (activity.totalSize >= config.spikeMinSize * 2) {
+ confidence += 0.1;
+ }
+ if (structureStrikes >= 4) {
+ confidence += 0.05;
+ }
+ if (activity.passesZ) {
+ confidence += 0.05;
+ }
+
+ const aggressor = applyAggressorAdjustment(confidence, coverage, aggressiveRatio, config);
+ confidence = clamp(aggressor.confidence, 0, 0.85);
+
+ let direction: "bullish" | "bearish" | "neutral" = "neutral";
+ if (structureRights === "C") {
+ direction = "bullish";
+ } else if (structureRights === "P") {
+ direction = "bearish";
+ }
+
+ return {
+ classifier_id: "ladder_accumulation",
+ confidence,
+ direction,
+ explanations: [
+ `Likely multi-strike ladder accumulation: ${structureLegs} legs across ${structureStrikes} strikes (span ${strikeSpan}).`,
+ `Premium ${formatUsd(activity.totalPremium)} across ${Math.round(activity.totalSize)} contracts.`,
+ `Thresholds: ladder structure plus >=${config.spikeMinSize} contracts or >=${formatUsd(config.spikeMinPremium)} premium.`,
+ "Direction inferred from call/put ladder.",
+ activity.baselineNote,
+ aggressor.note
+ ]
+ };
+};
+
+const buildFarDatedHit = (
+ packet: FlowPacket,
+ contract: ParsedContract,
+ config: ClassifierConfig
+): ClassifierHit | null => {
+ const dteDays = getDteDays(packet, contract);
+ if (!dteDays || dteDays < 60) {
+ return null;
+ }
+
+ const activity = getLargeActivity(packet, config);
+ if (!activity.passesAbsolute && !activity.passesZ) {
+ return null;
+ }
+
+ const { coverage, aggressiveRatio } = getAggressorContext(packet);
+ let confidence = 0.5;
+ if (dteDays >= 90) {
+ confidence += 0.05;
+ }
+ if (activity.totalPremium >= config.spikeMinPremium * 2) {
+ confidence += 0.1;
+ }
+ if (activity.totalSize >= config.spikeMinSize * 2) {
+ confidence += 0.05;
+ }
+ if (activity.passesZ) {
+ confidence += 0.1;
+ }
+
+ const aggressor = applyAggressorAdjustment(confidence, coverage, aggressiveRatio, config);
+ confidence = clamp(aggressor.confidence, 0, 0.85);
+
+ return {
+ classifier_id: "far_dated_conviction",
+ confidence,
+ direction: contract.right === "C" ? "bullish" : "bearish",
+ explanations: [
+ `Likely far-dated ${contract.right === "C" ? "call" : "put"} positioning: ${dteDays} DTE for ${packet.features.option_contract_id ?? packet.id}.`,
+ `Premium ${formatUsd(activity.totalPremium)} across ${Math.round(activity.totalSize)} contracts.`,
+ `Thresholds: DTE >=60 and >=${config.spikeMinSize} contracts or >=${formatUsd(config.spikeMinPremium)} premium (or z-scores).`,
+ "Direction inferred from call/put right.",
+ activity.baselineNote,
aggressor.note
]
};
@@ -222,5 +695,37 @@ export const evaluateClassifiers = (
hits.push(spikeHit);
}
+ if (contract) {
+ const overwriteHit = buildOverwriteHit(packet, contract, config);
+ if (overwriteHit) {
+ hits.push(overwriteHit);
+ }
+
+ const putWriteHit = buildPutWriteHit(packet, contract, config);
+ if (putWriteHit) {
+ hits.push(putWriteHit);
+ }
+
+ const farDatedHit = buildFarDatedHit(packet, contract, config);
+ if (farDatedHit) {
+ hits.push(farDatedHit);
+ }
+ }
+
+ const structureHit = buildStraddleStrangleHit(packet, config);
+ if (structureHit) {
+ hits.push(structureHit);
+ }
+
+ const verticalHit = buildVerticalSpreadHit(packet, config);
+ if (verticalHit) {
+ hits.push(verticalHit);
+ }
+
+ const ladderHit = buildLadderHit(packet, config);
+ if (ladderHit) {
+ hits.push(ladderHit);
+ }
+
return hits;
};
diff --git a/services/compute/tests/classifiers.test.ts b/services/compute/tests/classifiers.test.ts
index 8903963..81aece7 100644
--- a/services/compute/tests/classifiers.test.ts
+++ b/services/compute/tests/classifiers.test.ts
@@ -15,12 +15,14 @@ const baseConfig: ClassifierConfig = {
minAggressorRatio: 0.55
};
+const DEFAULT_TS = Date.UTC(2024, 0, 2);
+
const buildPacket = (
overrides: Record
): FlowPacket => {
return {
- source_ts: 1,
- ingest_ts: 1,
+ source_ts: DEFAULT_TS,
+ ingest_ts: DEFAULT_TS,
seq: 1,
trace_id: "trace",
id: "packet",
@@ -32,6 +34,8 @@ const buildPacket = (
total_size: 20,
first_price: 1,
last_price: 1.01,
+ start_ts: DEFAULT_TS - 500,
+ end_ts: DEFAULT_TS,
window_ms: 500,
...overrides
},
@@ -100,3 +104,87 @@ describe("classifier z-score behavior", () => {
expect((highHit?.confidence ?? 0)).toBeGreaterThan(lowHit?.confidence ?? 0);
});
});
+
+describe("classifier structure and positioning signals", () => {
+ test("call overwrite triggers on sell-side aggressor mix", () => {
+ const packet = buildPacket({
+ option_contract_id: "SPY-2024-03-15-450-C",
+ total_premium: 80_000,
+ total_size: 800,
+ nbbo_coverage_ratio: 0.9,
+ nbbo_aggressive_sell_ratio: 0.7,
+ nbbo_aggressive_buy_ratio: 0.3
+ });
+ const hits = evaluateClassifiers(packet, baseConfig);
+ expect(hits.some((hit) => hit.classifier_id === "large_call_sell_overwrite")).toBe(true);
+ });
+
+ test("put write triggers on sell-side aggressor mix", () => {
+ const packet = buildPacket({
+ option_contract_id: "SPY-2024-03-15-450-P",
+ total_premium: 75_000,
+ total_size: 700,
+ nbbo_coverage_ratio: 0.85,
+ nbbo_aggressive_sell_ratio: 0.68,
+ nbbo_aggressive_buy_ratio: 0.32
+ });
+ const hits = evaluateClassifiers(packet, baseConfig);
+ expect(hits.some((hit) => hit.classifier_id === "large_put_sell_write")).toBe(true);
+ });
+
+ test("straddle classifier triggers on structure tag", () => {
+ const packet = buildPacket({
+ structure_type: "straddle",
+ structure_legs: 2,
+ structure_strikes: 1,
+ structure_rights: "C/P",
+ structure_strike_span: 0
+ });
+ const hits = evaluateClassifiers(packet, baseConfig);
+ expect(hits.some((hit) => hit.classifier_id === "straddle")).toBe(true);
+ });
+
+ test("vertical spread infers direction from aggressor skew", () => {
+ const packet = buildPacket({
+ structure_type: "vertical",
+ structure_legs: 2,
+ structure_strikes: 2,
+ structure_rights: "C",
+ structure_strike_span: 5,
+ total_premium: 55_000,
+ total_size: 600,
+ nbbo_coverage_ratio: 0.85,
+ nbbo_aggressive_buy_ratio: 0.7,
+ nbbo_aggressive_sell_ratio: 0.3
+ });
+ const hits = evaluateClassifiers(packet, baseConfig);
+ const hit = hits.find((candidate) => candidate.classifier_id === "vertical_spread");
+ expect(hit?.direction).toBe("bullish");
+ });
+
+ test("ladder accumulation triggers on multi-strike structures", () => {
+ const packet = buildPacket({
+ structure_type: "ladder",
+ structure_legs: 3,
+ structure_strikes: 3,
+ structure_rights: "C",
+ structure_strike_span: 10,
+ total_premium: 60_000,
+ total_size: 650
+ });
+ const hits = evaluateClassifiers(packet, baseConfig);
+ expect(hits.some((hit) => hit.classifier_id === "ladder_accumulation")).toBe(true);
+ });
+
+ test("far-dated conviction triggers on 60DTE threshold", () => {
+ const packet = buildPacket({
+ option_contract_id: "SPY-2024-04-19-450-C",
+ end_ts: DEFAULT_TS,
+ total_premium: 70_000,
+ total_size: 800
+ });
+ const hits = evaluateClassifiers(packet, baseConfig);
+ const hit = hits.find((candidate) => candidate.classifier_id === "far_dated_conviction");
+ expect(hit?.direction).toBe("bullish");
+ });
+});
From 2752025fbc3cbd23fe76520e6053c16772d81e95 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Fri, 9 Jan 2026 16:11:38 -0500
Subject: [PATCH 008/237] Join underlying quotes for 0DTE classifier
---
.env.example | 3 +
README.md | 3 +
services/compute/src/classifiers.ts | 84 ++++++++++++++++++++++
services/compute/src/index.ts | 38 +++++++++-
services/compute/tests/classifiers.test.ts | 20 +++++-
5 files changed, 145 insertions(+), 3 deletions(-)
diff --git a/.env.example b/.env.example
index 36bc452..a589881 100644
--- a/.env.example
+++ b/.env.example
@@ -67,3 +67,6 @@ CLASSIFIER_SPIKE_MIN_SIZE_Z=2
CLASSIFIER_Z_MIN_SAMPLES=12
CLASSIFIER_MIN_NBBO_COVERAGE=0.5
CLASSIFIER_MIN_AGGRESSOR_RATIO=0.55
+CLASSIFIER_0DTE_MAX_ATM_PCT=0.01
+CLASSIFIER_0DTE_MIN_PREMIUM=20000
+CLASSIFIER_0DTE_MIN_SIZE=400
diff --git a/README.md b/README.md
index d163311..56597bc 100644
--- a/README.md
+++ b/README.md
@@ -196,6 +196,9 @@ Provider links: [Interactive Brokers](https://www.interactivebrokers.com/), [IBK
- `CLASSIFIER_Z_MIN_SAMPLES` (default `12`) — minimum samples before z-scores apply.
- `CLASSIFIER_MIN_NBBO_COVERAGE` (default `0.5`) — NBBO coverage ratio gate.
- `CLASSIFIER_MIN_AGGRESSOR_RATIO` (default `0.55`) — aggressor ratio gate.
+- `CLASSIFIER_0DTE_MAX_ATM_PCT` (default `0.01`) — max ATM distance as pct of underlying for 0DTE gamma punch.
+- `CLASSIFIER_0DTE_MIN_PREMIUM` (default `20000`) — 0DTE gamma punch premium floor.
+- `CLASSIFIER_0DTE_MIN_SIZE` (default `400`) — 0DTE gamma punch size floor.
### Testing + throttling
diff --git a/services/compute/src/classifiers.ts b/services/compute/src/classifiers.ts
index 824886e..3eef846 100644
--- a/services/compute/src/classifiers.ts
+++ b/services/compute/src/classifiers.ts
@@ -12,6 +12,9 @@ export type ClassifierConfig = {
zMinSamples: number;
minNbboCoverage: number;
minAggressorRatio: number;
+ zeroDteMaxAtmPct: number;
+ zeroDteMinPremium: number;
+ zeroDteMinSize: number;
};
const MS_PER_DAY = 86_400_000;
@@ -42,6 +45,13 @@ const getStringFeature = (packet: FlowPacket, key: string): string => {
const formatPct = (value: number): string => `${Math.round(value * 100)}%`;
+const formatPctPrecise = (value: number, digits = 2): string => {
+ if (!Number.isFinite(value)) {
+ return "0%";
+ }
+ return `${(value * 100).toFixed(digits)}%`;
+};
+
const getAggressorContext = (
packet: FlowPacket
): {
@@ -183,6 +193,14 @@ const getReferenceTs = (packet: FlowPacket): number | null => {
return null;
};
+const getReferenceDay = (packet: FlowPacket): string | null => {
+ const referenceTs = getReferenceTs(packet);
+ if (!referenceTs) {
+ return null;
+ }
+ return new Date(referenceTs).toISOString().slice(0, 10);
+};
+
const getDteDays = (packet: FlowPacket, contract: ParsedContract): number | null => {
const expiryTs = Date.parse(`${contract.expiry}T00:00:00Z`);
if (!Number.isFinite(expiryTs)) {
@@ -666,6 +684,67 @@ const buildFarDatedHit = (
};
};
+const buildZeroDteGammaPunchHit = (
+ packet: FlowPacket,
+ contract: ParsedContract,
+ config: ClassifierConfig
+): ClassifierHit | null => {
+ const referenceDay = getReferenceDay(packet);
+ if (!referenceDay || contract.expiry !== referenceDay) {
+ return null;
+ }
+
+ const activity = getLargeActivity(packet, config);
+ if (
+ activity.totalPremium < config.zeroDteMinPremium ||
+ activity.totalSize < config.zeroDteMinSize
+ ) {
+ return null;
+ }
+
+ const underlyingMid = getNumberFeature(packet, "underlying_mid");
+ if (!Number.isFinite(underlyingMid) || underlyingMid <= 0) {
+ return null;
+ }
+
+ const strike = contract.strike;
+ const atmPct = Math.abs(strike - underlyingMid) / underlyingMid;
+ if (atmPct > config.zeroDteMaxAtmPct) {
+ return null;
+ }
+
+ const { coverage, aggressiveRatio } = getAggressorContext(packet);
+ let confidence = 0.55;
+ if (atmPct <= config.zeroDteMaxAtmPct * 0.5) {
+ confidence += 0.05;
+ }
+ if (activity.totalPremium >= config.zeroDteMinPremium * 2) {
+ confidence += 0.1;
+ }
+ if (activity.totalSize >= config.zeroDteMinSize * 2) {
+ confidence += 0.05;
+ }
+
+ const aggressor = applyAggressorAdjustment(confidence, coverage, aggressiveRatio, config);
+ confidence = clamp(aggressor.confidence, 0, 0.9);
+
+ return {
+ classifier_id: "zero_dte_gamma_punch",
+ confidence,
+ direction: contract.right === "C" ? "bullish" : "bearish",
+ explanations: [
+ `Likely 0DTE gamma punch: ${packet.features.option_contract_id ?? packet.id} near ATM.`,
+ `Underlying mid ${formatUsd(underlyingMid)}, strike ${formatUsd(strike)} (${formatPctPrecise(atmPct)} from ATM).`,
+ `Premium ${formatUsd(activity.totalPremium)} across ${Math.round(activity.totalSize)} contracts.`,
+ `Thresholds: DTE=0, ATM <=${formatPctPrecise(config.zeroDteMaxAtmPct)}, >=${formatUsd(
+ config.zeroDteMinPremium
+ )} premium, >=${config.zeroDteMinSize} contracts.`,
+ activity.baselineNote,
+ aggressor.note
+ ]
+ };
+};
+
export const evaluateClassifiers = (
packet: FlowPacket,
config: ClassifierConfig
@@ -710,6 +789,11 @@ export const evaluateClassifiers = (
if (farDatedHit) {
hits.push(farDatedHit);
}
+
+ const zeroDteHit = buildZeroDteGammaPunchHit(packet, contract, config);
+ if (zeroDteHit) {
+ hits.push(zeroDteHit);
+ }
}
const structureHit = buildStraddleStrangleHit(packet, config);
diff --git a/services/compute/src/index.ts b/services/compute/src/index.ts
index 042e6df..315c65f 100644
--- a/services/compute/src/index.ts
+++ b/services/compute/src/index.ts
@@ -115,7 +115,10 @@ const envSchema = z.object({
CLASSIFIER_SPIKE_MIN_SIZE_Z: z.coerce.number().nonnegative().default(2),
CLASSIFIER_Z_MIN_SAMPLES: z.coerce.number().int().nonnegative().default(12),
CLASSIFIER_MIN_NBBO_COVERAGE: z.coerce.number().min(0).max(1).default(0.5),
- CLASSIFIER_MIN_AGGRESSOR_RATIO: z.coerce.number().min(0).max(1).default(0.55)
+ CLASSIFIER_MIN_AGGRESSOR_RATIO: z.coerce.number().min(0).max(1).default(0.55),
+ CLASSIFIER_0DTE_MAX_ATM_PCT: z.coerce.number().min(0).max(1).default(0.01),
+ CLASSIFIER_0DTE_MIN_PREMIUM: z.coerce.number().positive().default(20_000),
+ CLASSIFIER_0DTE_MIN_SIZE: z.coerce.number().int().positive().default(400)
});
const env = readEnv(envSchema);
@@ -130,7 +133,10 @@ const classifierConfig: ClassifierConfig = {
spikeMinSizeZ: env.CLASSIFIER_SPIKE_MIN_SIZE_Z,
zMinSamples: env.CLASSIFIER_Z_MIN_SAMPLES,
minNbboCoverage: env.CLASSIFIER_MIN_NBBO_COVERAGE,
- minAggressorRatio: env.CLASSIFIER_MIN_AGGRESSOR_RATIO
+ minAggressorRatio: env.CLASSIFIER_MIN_AGGRESSOR_RATIO,
+ zeroDteMaxAtmPct: env.CLASSIFIER_0DTE_MAX_ATM_PCT,
+ zeroDteMinPremium: env.CLASSIFIER_0DTE_MIN_PREMIUM,
+ zeroDteMinSize: env.CLASSIFIER_0DTE_MIN_SIZE
};
const darkInferenceConfig: DarkInferenceConfig = {
@@ -485,6 +491,34 @@ const flushCluster = async (
window_ms: env.CLUSTER_WINDOW_MS
};
+ const parsedContract = parseContractId(cluster.contractId);
+ if (parsedContract?.root) {
+ features.underlying_id = parsedContract.root;
+ const quoteJoin = selectEquityQuote(parsedContract.root, cluster.endTs);
+ if (!quoteJoin.quote) {
+ joinQuality.underlying_quote_missing = 1;
+ } else {
+ joinQuality.underlying_quote_age_ms = quoteJoin.ageMs;
+ if (quoteJoin.stale) {
+ joinQuality.underlying_quote_stale = 1;
+ } else {
+ const bid = quoteJoin.quote.bid;
+ const ask = quoteJoin.quote.ask;
+ if (Number.isFinite(bid) && Number.isFinite(ask) && ask > 0) {
+ const mid = (bid + ask) / 2;
+ const spread = ask - bid;
+ features.underlying_quote_ts = quoteJoin.quote.ts;
+ features.underlying_bid = bid;
+ features.underlying_ask = ask;
+ features.underlying_mid = roundTo(mid);
+ features.underlying_spread = roundTo(spread);
+ } else {
+ joinQuality.underlying_quote_missing = 1;
+ }
+ }
+ }
+ }
+
const placementTotal =
cluster.placements.aa +
cluster.placements.a +
diff --git a/services/compute/tests/classifiers.test.ts b/services/compute/tests/classifiers.test.ts
index 81aece7..ab3b110 100644
--- a/services/compute/tests/classifiers.test.ts
+++ b/services/compute/tests/classifiers.test.ts
@@ -12,7 +12,10 @@ const baseConfig: ClassifierConfig = {
spikeMinSizeZ: 2,
zMinSamples: 12,
minNbboCoverage: 0.5,
- minAggressorRatio: 0.55
+ minAggressorRatio: 0.55,
+ zeroDteMaxAtmPct: 0.01,
+ zeroDteMinPremium: 20_000,
+ zeroDteMinSize: 400
};
const DEFAULT_TS = Date.UTC(2024, 0, 2);
@@ -187,4 +190,19 @@ describe("classifier structure and positioning signals", () => {
const hit = hits.find((candidate) => candidate.classifier_id === "far_dated_conviction");
expect(hit?.direction).toBe("bullish");
});
+
+ test("zero dte gamma punch triggers when ATM and large", () => {
+ const packet = buildPacket({
+ option_contract_id: "SPY-2024-01-02-450-C",
+ total_premium: 35_000,
+ total_size: 600,
+ underlying_mid: 450,
+ nbbo_coverage_ratio: 0.8,
+ nbbo_aggressive_buy_ratio: 0.7,
+ nbbo_aggressive_sell_ratio: 0.3
+ });
+ const hits = evaluateClassifiers(packet, baseConfig);
+ const hit = hits.find((candidate) => candidate.classifier_id === "zero_dte_gamma_punch");
+ expect(hit?.direction).toBe("bullish");
+ });
});
From 6951dddfdff3c73d68d9c3a07208d626e2663648 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Fri, 9 Jan 2026 17:09:05 -0500
Subject: [PATCH 009/237] Add replay for flow alerts and hits
---
apps/web/app/page.tsx | 78 ++++++++++++++++++------------
packages/storage/src/clickhouse.ts | 69 ++++++++++++++++++++++++++
services/api/src/index.ts | 27 +++++++++++
3 files changed, 142 insertions(+), 32 deletions(-)
diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx
index 01dd36c..8e182bb 100644
--- a/apps/web/app/page.tsx
+++ b/apps/web/app/page.tsx
@@ -1957,27 +1957,41 @@ export default function HomePage() {
getReplayKey: disableReplayGrouping
});
- const flowHold = useCallback(() => !flowScroll.isAtTopRef.current, [flowScroll.isAtTopRef]);
- const flow = useFlowStream(
- mode === "live",
- flowScroll.onNewItems,
- flowAnchor.capture,
- flowHold,
- flowScroll.resumeTick
- );
- const alerts = useLiveStream({
- enabled: mode === "live",
- wsPath: "/ws/alerts",
- expectedType: "alert",
- onNewItems: alertsScroll.onNewItems,
- captureScroll: alertsAnchor.capture
+ const flow = useTape({
+ mode,
+ wsPath: "/ws/flow",
+ replayPath: "/replay/flow",
+ latestPath: "/flow/packets",
+ expectedType: "flow-packet",
+ batchSize: mode === "replay" ? 120 : undefined,
+ pollMs: mode === "replay" ? 200 : undefined,
+ captureScroll: flowAnchor.capture,
+ onNewItems: flowScroll.onNewItems,
+ getReplayKey: disableReplayGrouping
});
- const classifierHits = useLiveStream({
- enabled: mode === "live",
+ const alerts = useTape({
+ mode,
+ wsPath: "/ws/alerts",
+ replayPath: "/replay/alerts",
+ latestPath: "/flow/alerts",
+ expectedType: "alert",
+ batchSize: mode === "replay" ? 120 : undefined,
+ pollMs: mode === "replay" ? 200 : undefined,
+ captureScroll: alertsAnchor.capture,
+ onNewItems: alertsScroll.onNewItems,
+ getReplayKey: disableReplayGrouping
+ });
+ const classifierHits = useTape({
+ mode,
wsPath: "/ws/classifier-hits",
+ replayPath: "/replay/classifier-hits",
+ latestPath: "/flow/classifier-hits",
expectedType: "classifier-hit",
+ batchSize: mode === "replay" ? 120 : undefined,
+ pollMs: mode === "replay" ? 200 : undefined,
+ captureScroll: classifierAnchor.capture,
onNewItems: classifierScroll.onNewItems,
- captureScroll: classifierAnchor.capture
+ getReplayKey: disableReplayGrouping
});
useLayoutEffect(() => {
@@ -2498,7 +2512,7 @@ export default function HomePage() {
+
+
+
+ Equity Chart
++ Server-built {formatIntervalLabel(chartIntervalMs)} candles for {chartTicker}. +
+
+
+
+ {CANDLE_INTERVALS.map((interval) => (
+
+ ))}
+
+ {activeTickers.length > 1 ? (
+ Charting first of {activeTickers.length} tickers
+ ) : (
+ Charting {chartTicker}
+ )}
+
diff --git a/apps/web/package.json b/apps/web/package.json
index edab5bd..b61eb2e 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -9,6 +9,7 @@
},
"dependencies": {
"@islandflow/types": "workspace:*",
+ "lightweight-charts": "^4.2.0",
"next": "^14.2.4",
"react": "^18.3.1",
"react-dom": "^18.3.1"
diff --git a/bun.lock b/bun.lock
index 557deeb..0408e06 100644
--- a/bun.lock
+++ b/bun.lock
@@ -9,6 +9,7 @@
"name": "@islandflow/web",
"dependencies": {
"@islandflow/types": "workspace:*",
+ "lightweight-charts": "^4.2.0",
"next": "^14.2.4",
"react": "^18.3.1",
"react-dom": "^18.3.1",
@@ -205,10 +206,14 @@
"csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="],
+ "fancy-canvas": ["fancy-canvas@2.1.0", "", {}, "sha512-nifxXJ95JNLFR2NgRV4/MxVP45G9909wJTEKz5fg/TZS20JJZA6hfgRVh/bC9bwl2zBtBNcYPjiBE4njQHVBwQ=="],
+
"graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="],
"js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],
+ "lightweight-charts": ["lightweight-charts@4.2.3", "", { "dependencies": { "fancy-canvas": "2.1.0" } }, "sha512-5kS/2hY3wNYNzhnS8Gb+GAS07DX8GPF2YVDnd2NMC85gJVQ6RLU6YrXNgNJ6eg0AnWPwCnvaGtYmGky3HiLQEw=="],
+
"loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="],
"nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
From a95bb1375dfb3fd0ae970685b9ac98aad87aa0f8 Mon Sep 17 00:00:00 2001
From: dirtydishes <35477874+dirtydishes@users.noreply.github.com>
Date: Wed, 7 Jan 2026 16:57:54 -0500
Subject: [PATCH 005/237] Document environment configuration
---
README.md | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 91 insertions(+), 7 deletions(-)
diff --git a/README.md b/README.md
index 9860a69..d163311 100644
--- a/README.md
+++ b/README.md
@@ -110,13 +110,97 @@ Run just the web app (fixed to port 3000):
Run just the API:
- `bun --cwd services/api run dev`
-Adapter selection (env):
-- Options: `OPTIONS_INGEST_ADAPTER` (defaults to `synthetic`; supported: `synthetic`, `alpaca`, `ibkr`, `databento`)
-- Equities: `EQUITIES_INGEST_ADAPTER` (defaults to `synthetic`)
-- Compute: `COMPUTE_DELIVER_POLICY` (`new` default), `COMPUTE_CONSUMER_RESET` (force skip backlog)
-- Rolling stats: `REDIS_URL`, `ROLLING_WINDOW_SIZE`, `ROLLING_TTL_SEC`
-- Classifier tuning: `CLASSIFIER_SWEEP_MIN_PREMIUM_Z`, `CLASSIFIER_SPIKE_MIN_PREMIUM_Z`, `CLASSIFIER_SPIKE_MIN_SIZE_Z`, `CLASSIFIER_Z_MIN_SAMPLES`
-- Aggressor gating: `CLASSIFIER_MIN_NBBO_COVERAGE`, `CLASSIFIER_MIN_AGGRESSOR_RATIO`
+## Environment Configuration
+
+All runtime configuration is driven by `.env`. Start by copying `.env.example` and edit the values you need. Defaults below match `.env.example` unless otherwise noted.
+
+### Core infrastructure
+
+These define how services connect to the event bus and storage backends. Documentation links are provided for convenience.
+
+- `NATS_URL` (default `nats://localhost:4222`) — NATS JetStream endpoint. See [NATS](https://nats.io/) and [JetStream](https://docs.nats.io/nats-concepts/jetstream).
+- `CLICKHOUSE_URL` (default `http://localhost:8123`) — ClickHouse HTTP endpoint. See [ClickHouse](https://clickhouse.com/).
+- `CLICKHOUSE_DATABASE` (default `default`) — ClickHouse database name.
+- `REDIS_URL` (default `redis://localhost:6379`) — Redis endpoint for rolling stats. See [Redis](https://redis.io/).
+
+### Adapter selection
+
+- `OPTIONS_INGEST_ADAPTER` (default `synthetic`) — options ingest adapter: `synthetic`, `alpaca`, `ibkr`, `databento`.
+- `EQUITIES_INGEST_ADAPTER` (default `synthetic`) — equities ingest adapter.
+- `EMIT_INTERVAL_MS` (default `1000`) — synthetic equities emit cadence.
+
+### Alpaca options adapter (dev-only)
+
+Provider links: [Alpaca](https://alpaca.markets/), [Alpaca Market Data API](https://alpaca.markets/docs/api-references/market-data-api/).
+
+- `ALPACA_KEY_ID`, `ALPACA_SECRET_KEY` — credentials.
+- `ALPACA_REST_URL` (default `https://data.alpaca.markets`) — REST endpoint.
+- `ALPACA_WS_BASE_URL` (default `wss://stream.data.alpaca.markets/v1beta1`) — streaming endpoint.
+- `ALPACA_FEED` (default `indicative`) — use `opra` when you have a subscription.
+- `ALPACA_UNDERLYINGS` (default `SPY,NVDA,AAPL`) — comma-separated list of symbols.
+- `ALPACA_STRIKES_PER_SIDE` (default `8`) — strikes per side around ATM.
+- `ALPACA_MAX_DTE_DAYS` (default `30`) — expiry horizon.
+- `ALPACA_MONEYNESS_PCT` (default `0.06`) — ATM band for strike selection.
+- `ALPACA_MONEYNESS_FALLBACK_PCT` (default `0.1`) — fallback band if strikes are sparse.
+- `ALPACA_MAX_QUOTES` (default `200`) — subscription size guardrail.
+
+### Databento historical replay adapter
+
+Provider links: [Databento](https://databento.com/), [Databento API](https://databento.com/docs/api-reference).
+
+- `DATABENTO_API_KEY` — API key.
+- `DATABENTO_DATASET` (default `OPRA.PILLAR`) — dataset.
+- `DATABENTO_SCHEMA` (default `trades`) — schema.
+- `DATABENTO_START` — ISO date/time start for replay.
+- `DATABENTO_END` — ISO date/time end (optional).
+- `DATABENTO_SYMBOLS` (default `SPY.OPT`) — comma list or dataset symbols.
+- `DATABENTO_STYPE_IN` (default `parent`) — input symbology type.
+- `DATABENTO_STYPE_OUT` (default `instrument_id`) — output symbology type.
+- `DATABENTO_LIMIT` (default `0`) — record cap (0 means no cap).
+- `DATABENTO_PRICE_SCALE` (default `1`) — divide raw price by this value.
+- `DATABENTO_PYTHON_BIN` (default `py/.venv/bin/python`) — Python executable for replay sidecar.
+
+### IBKR options adapter (Python sidecar)
+
+Provider links: [Interactive Brokers](https://www.interactivebrokers.com/), [IBKR API docs](https://interactivebrokers.github.io/).
+
+- `IBKR_HOST` (default `127.0.0.1`) — TWS/Gateway host.
+- `IBKR_PORT` (default `7497`) — TWS/Gateway port.
+- `IBKR_CLIENT_ID` (default `0`) — API client ID.
+- `IBKR_SYMBOL` (default `SPY`) — underlying symbol.
+- `IBKR_EXPIRY` (default `20250117`) — expiry in `YYYYMMDD`.
+- `IBKR_STRIKE` (default `450`) — strike price.
+- `IBKR_RIGHT` (default `C`) — option right (`C` or `P`).
+- `IBKR_EXCHANGE` (default `SMART`) — exchange route.
+- `IBKR_CURRENCY` (default `USD`) — currency.
+- `IBKR_PYTHON_BIN` (default `python3`) — Python executable for sidecar.
+
+### Compute + market-structure tuning
+
+- `COMPUTE_DELIVER_POLICY` (default `new`) — consumer start behavior (`new` or `all`).
+- `COMPUTE_CONSUMER_RESET` (default `false`) — force consumer reset (skip backlog).
+- `NBBO_MAX_AGE_MS` (default `1000`) — max allowed NBBO age for joins.
+- `NEXT_PUBLIC_NBBO_MAX_AGE_MS` (default `1000`) — UI-visible NBBO age for display gating.
+- `ROLLING_WINDOW_SIZE` (default `50`) — rolling stats window length.
+- `ROLLING_TTL_SEC` (default `86400`) — rolling stats TTL in seconds.
+
+### Classifier thresholds
+
+- `CLASSIFIER_SWEEP_MIN_PREMIUM` (default `40000`) — absolute sweep premium floor.
+- `CLASSIFIER_SWEEP_MIN_COUNT` (default `3`) — minimum leg count for sweeps.
+- `CLASSIFIER_SWEEP_MIN_PREMIUM_Z` (default `2`) — sweep premium z-score threshold.
+- `CLASSIFIER_SPIKE_MIN_PREMIUM` (default `20000`) — absolute spike premium floor.
+- `CLASSIFIER_SPIKE_MIN_SIZE` (default `400`) — absolute spike size floor.
+- `CLASSIFIER_SPIKE_MIN_PREMIUM_Z` (default `2.5`) — spike premium z-score threshold.
+- `CLASSIFIER_SPIKE_MIN_SIZE_Z` (default `2`) — spike size z-score threshold.
+- `CLASSIFIER_Z_MIN_SAMPLES` (default `12`) — minimum samples before z-scores apply.
+- `CLASSIFIER_MIN_NBBO_COVERAGE` (default `0.5`) — NBBO coverage ratio gate.
+- `CLASSIFIER_MIN_AGGRESSOR_RATIO` (default `0.55`) — aggressor ratio gate.
+
+### Testing + throttling
+
+- `TESTING_MODE` (default `false`) — enable ingest throttling for local dev.
+- `TESTING_THROTTLE_MS` (default `200`) — minimum spacing between emitted prints.
Testing mode (throttles ingest to reduce CPU):
- `TESTING_MODE=true` enables throttling
From 1583a5041210970ac9c10ee9d269da1a4864fcc9 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Fri, 9 Jan 2026 15:29:41 -0500
Subject: [PATCH 006/237] Improve local defaults and replay candle fetch
---
.env.example | 6 +-
apps/web/app/page.tsx | 77 ++++++++++++---
packages/storage/src/clickhouse.ts | 3 +-
scripts/dev.ts | 95 ++++++++++++++++++-
services/api/src/index.ts | 8 +-
services/candles/src/index.ts | 12 +--
services/compute/src/index.ts | 8 +-
.../ingest-equities/src/adapters/synthetic.ts | 8 +-
services/ingest-equities/src/index.ts | 6 +-
.../ingest-options/src/adapters/synthetic.ts | 6 +-
services/ingest-options/src/index.ts | 6 +-
11 files changed, 193 insertions(+), 42 deletions(-)
diff --git a/.env.example b/.env.example
index 508bd81..36bc452 100644
--- a/.env.example
+++ b/.env.example
@@ -1,7 +1,7 @@
-NATS_URL=nats://localhost:4222
-CLICKHOUSE_URL=http://localhost:8123
+NATS_URL=nats://127.0.0.1:4222
+CLICKHOUSE_URL=http://127.0.0.1:8123
CLICKHOUSE_DATABASE=default
-REDIS_URL=redis://localhost:6379
+REDIS_URL=redis://127.0.0.1:6379
# Options ingest
OPTIONS_INGEST_ADAPTER=synthetic
diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx
index 3828c78..01dd36c 100644
--- a/apps/web/app/page.tsx
+++ b/apps/web/app/page.tsx
@@ -20,9 +20,8 @@ const NBBO_MAX_AGE_MS_SAFE =
Number.isFinite(NBBO_MAX_AGE_MS) && NBBO_MAX_AGE_MS > 0 ? NBBO_MAX_AGE_MS : 1000;
const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1"]);
const CANDLE_INTERVALS = [
- { label: "1s", ms: 1000 },
- { label: "5s", ms: 5000 },
- { label: "1m", ms: 60000 }
+ { label: "1m", ms: 60000 },
+ { label: "5m", ms: 300000 }
];
type CandlestickSeries = ReturnType;
@@ -63,6 +62,23 @@ const toChartCandle = (candle: EquityCandle): ChartCandle => {
};
};
+const readErrorDetail = async (response: Response): Promise => {
+ const text = await response.text();
+ if (!text) {
+ return "";
+ }
+ try {
+ const payload = JSON.parse(text) as {
+ detail?: string;
+ error?: string;
+ message?: string;
+ };
+ return payload.detail ?? payload.error ?? payload.message ?? text;
+ } catch {
+ return text;
+ }
+};
+
type WsStatus = "connecting" | "connected" | "disconnected";
type TapeMode = "live" | "replay";
@@ -1218,15 +1234,28 @@ type CandleChartProps = {
ticker: string;
intervalMs: number;
mode: TapeMode;
+ replayTime?: number | null;
};
-const CandleChart = ({ ticker, intervalMs, mode }: CandleChartProps) => {
+const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChartProps) => {
const containerRef = useRef(null);
const chartRef = useRef(null);
const seriesRef = useRef(null);
const socketRef = useRef(null);
const reconnectRef = useRef(null);
const lastCandleRef = useRef<{ time: UTCTimestamp; seq: number } | null>(null);
+ const replayBucket = useMemo(() => {
+ if (mode !== "replay" || replayTime === null) {
+ return null;
+ }
+ return Math.floor(replayTime / intervalMs);
+ }, [mode, replayTime, intervalMs]);
+ const replayEndTs = useMemo(() => {
+ if (replayBucket === null) {
+ return null;
+ }
+ return (replayBucket + 1) * intervalMs - 1;
+ }, [replayBucket, intervalMs]);
const [ready, setReady] = useState(false);
const [status, setStatus] = useState(mode === "live" ? "connecting" : "connected");
const [lastUpdate, setLastUpdate] = useState(null);
@@ -1307,6 +1336,16 @@ const CandleChart = ({ ticker, intervalMs, mode }: CandleChartProps) => {
return;
}
+ if (mode === "replay" && replayBucket === null) {
+ setError(null);
+ setHasData(false);
+ setLastUpdate(null);
+ lastCandleRef.current = null;
+ seriesRef.current.setData([]);
+ setStatus("connected");
+ return;
+ }
+
let active = true;
setError(null);
setHasData(false);
@@ -1322,9 +1361,15 @@ const CandleChart = ({ ticker, intervalMs, mode }: CandleChartProps) => {
url.searchParams.set("interval_ms", intervalMs.toString());
url.searchParams.set("limit", "300");
url.searchParams.set("cache", "1");
+ if (mode === "replay" && replayEndTs !== null) {
+ url.searchParams.set("end_ts", replayEndTs.toString());
+ }
const response = await fetch(url.toString());
if (!response.ok) {
- throw new Error(`Candle fetch failed (${response.status})`);
+ const detail = await readErrorDetail(response);
+ throw new Error(
+ `Candle fetch failed (${response.status})${detail ? `: ${detail}` : ""}`
+ );
}
const payload = (await response.json()) as { data?: EquityCandle[] };
if (!active || !seriesRef.current) {
@@ -1361,7 +1406,7 @@ const CandleChart = ({ ticker, intervalMs, mode }: CandleChartProps) => {
return () => {
active = false;
};
- }, [ready, ticker, intervalMs, mode]);
+ }, [ready, ticker, intervalMs, mode, replayBucket, replayEndTs]);
useEffect(() => {
if (!ready || mode !== "live" || !seriesRef.current) {
@@ -1471,6 +1516,13 @@ const CandleChart = ({ ticker, intervalMs, mode }: CandleChartProps) => {
}, [intervalMs]);
const statusText = statusLabel(status, false, mode);
+ const intervalLabel = formatIntervalLabel(intervalMs);
+ const emptyLabel =
+ mode === "live"
+ ? status === "connected"
+ ? `No candles yet. First ${intervalLabel} candle appears after the window closes.`
+ : "Chart offline. Start candles service."
+ : "No candles for this replay window.";
return (
-
@@ -1487,11 +1539,7 @@ const CandleChart = ({ ticker, intervalMs, mode }: CandleChartProps) => {
{error ? (
);
@@ -2280,7 +2328,12 @@ export default function HomePage() {
Charting {chartTicker}
)}
Chart error: {error}
) : !hasData ? (
-
- {mode === "live"
- ? "No candles yet. Start candles service."
- : "No candles for this replay window."}
-
+ {emptyLabel}
) : null}
Flow Packets
-Deterministic clusters (live only).
+Deterministic clusters.
@@ -2521,13 +2535,13 @@ export default function HomePage() {
- {mode !== "live" ? (
-
Flow packets are live-only in this build.
- ) : filteredFlow.length === 0 ? (
+ {filteredFlow.length === 0 ? (
{tickerSet.size > 0
? "No flow packets match the current filter."
- : "No flow packets yet. Start compute."}
+ : mode === "live"
+ ? "No flow packets yet. Start compute."
+ : "Replay queue empty. Ensure ClickHouse has data."}
) : (
filteredFlow.map((packet) => {
@@ -2640,7 +2654,7 @@ export default function HomePage() {
replayComplete={alerts.replayComplete}
paused={alerts.paused}
dropped={alerts.dropped}
- mode="live"
+ mode={mode}
onTogglePause={alerts.togglePause}
/>
- {mode !== "live" ? (
-
Alerts are live-only in this build.
- ) : filteredAlerts.length === 0 ? (
+ {filteredAlerts.length === 0 ? (
{tickerSet.size > 0
? "No alerts match the current filter."
- : "No alerts yet. Start compute."}
+ : mode === "live"
+ ? "No alerts yet. Start compute."
+ : "Replay queue empty. Ensure ClickHouse has data."}
) : (
filteredAlerts.map((alert) => {
@@ -2716,7 +2730,7 @@ export default function HomePage() {
replayComplete={classifierHits.replayComplete}
paused={classifierHits.paused}
dropped={classifierHits.dropped}
- mode="live"
+ mode={mode}
onTogglePause={classifierHits.togglePause}
/>
- {mode !== "live" ? (
- => {
+ const safeLimit = clampLimit(limit);
+ const safeAfterTs = clampCursor(afterTs);
+ const safeAfterSeq = clampCursor(afterSeq);
+
+ const result = await client.query({
+ query: `SELECT * FROM ${FLOW_PACKETS_TABLE} WHERE (source_ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY source_ts ASC, seq ASC LIMIT ${safeLimit}`,
+ format: "JSONEachRow"
+ });
+
+ const rows = await result.json();
+ const records = rows
+ .map(normalizeFlowPacketRow)
+ .filter((record): record is FlowPacketRecord => record !== null);
+ const packets = records.map(fromFlowPacketRecord);
+ return FlowPacketSchema.array().parse(packets);
+};
+
+export const fetchClassifierHitsAfter = async (
+ client: ClickHouseClient,
+ afterTs: number,
+ afterSeq: number,
+ limit: number
+): Promise => {
+ const safeLimit = clampLimit(limit);
+ const safeAfterTs = clampCursor(afterTs);
+ const safeAfterSeq = clampCursor(afterSeq);
+
+ const result = await client.query({
+ query: `SELECT * FROM ${CLASSIFIER_HITS_TABLE} WHERE (source_ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY source_ts ASC, seq ASC LIMIT ${safeLimit}`,
+ format: "JSONEachRow"
+ });
+
+ const rows = await result.json();
+ const records = rows
+ .map(normalizeClassifierHitRow)
+ .filter((record): record is ClassifierHitRecord => record !== null);
+ const hits = records.map(fromClassifierHitRecord);
+ return ClassifierHitEventSchema.array().parse(hits);
+};
+
+export const fetchAlertsAfter = async (
+ client: ClickHouseClient,
+ afterTs: number,
+ afterSeq: number,
+ limit: number
+): Promise => {
+ const safeLimit = clampLimit(limit);
+ const safeAfterTs = clampCursor(afterTs);
+ const safeAfterSeq = clampCursor(afterSeq);
+
+ const result = await client.query({
+ query: `SELECT * FROM ${ALERTS_TABLE} WHERE (source_ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY source_ts ASC, seq ASC LIMIT ${safeLimit}`,
+ format: "JSONEachRow"
+ });
+
+ const rows = await result.json();
+ const records = rows
+ .map(normalizeAlertRow)
+ .filter((record): record is AlertRecord => record !== null);
+ const alerts = records.map(fromAlertRecord);
+ return AlertEventSchema.array().parse(alerts);
+};
diff --git a/services/api/src/index.ts b/services/api/src/index.ts
index df57d59..a345aba 100644
--- a/services/api/src/index.ts
+++ b/services/api/src/index.ts
@@ -38,6 +38,9 @@ import {
ensureFlowPacketsTable,
ensureOptionNBBOTable,
ensureOptionPrintsTable,
+ fetchAlertsAfter,
+ fetchClassifierHitsAfter,
+ fetchFlowPacketsAfter,
fetchRecentAlerts,
fetchRecentClassifierHits,
fetchRecentEquityPrintJoins,
@@ -916,6 +919,30 @@ const run = async () => {
return jsonResponse({ data, next });
}
+ if (req.method === "GET" && url.pathname === "/replay/flow") {
+ const { afterTs, afterSeq, limit } = parseReplayParams(url);
+ const data = await fetchFlowPacketsAfter(clickhouse, afterTs, afterSeq, limit);
+ const last = data.at(-1);
+ const next = last ? { ts: last.source_ts, seq: last.seq } : null;
+ return jsonResponse({ data, next });
+ }
+
+ if (req.method === "GET" && url.pathname === "/replay/classifier-hits") {
+ const { afterTs, afterSeq, limit } = parseReplayParams(url);
+ const data = await fetchClassifierHitsAfter(clickhouse, afterTs, afterSeq, limit);
+ const last = data.at(-1);
+ const next = last ? { ts: last.source_ts, seq: last.seq } : null;
+ return jsonResponse({ data, next });
+ }
+
+ if (req.method === "GET" && url.pathname === "/replay/alerts") {
+ const { afterTs, afterSeq, limit } = parseReplayParams(url);
+ const data = await fetchAlertsAfter(clickhouse, afterTs, afterSeq, limit);
+ const last = data.at(-1);
+ const next = last ? { ts: last.source_ts, seq: last.seq } : null;
+ return jsonResponse({ data, next });
+ }
+
if (req.method === "GET" && url.pathname === "/ws/options") {
if (serverRef.upgrade(req, { data: { channel: "options" } })) {
return new Response(null, { status: 101 });
From 980bb4f1b17a3667b5fdd59b69ab2a31c0f4d4d4 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Fri, 9 Jan 2026 17:22:09 -0500
Subject: [PATCH 010/237] Add replay streaming service
---
.env.example | 9 +
README.md | 10 +
scripts/dev.ts | 12 +
services/replay/package.json | 16 ++
services/replay/src/index.ts | 458 ++++++++++++++++++++++++++++++++++
services/replay/tsconfig.json | 7 +
6 files changed, 512 insertions(+)
create mode 100644 services/replay/package.json
create mode 100644 services/replay/src/index.ts
create mode 100644 services/replay/tsconfig.json
diff --git a/.env.example b/.env.example
index a589881..ae5499b 100644
--- a/.env.example
+++ b/.env.example
@@ -70,3 +70,12 @@ CLASSIFIER_MIN_AGGRESSOR_RATIO=0.55
CLASSIFIER_0DTE_MAX_ATM_PCT=0.01
CLASSIFIER_0DTE_MIN_PREMIUM=20000
CLASSIFIER_0DTE_MIN_SIZE=400
+
+# Replay service
+REPLAY_ENABLED=false
+REPLAY_STREAMS=options,nbbo,equities,equity-quotes
+REPLAY_START_TS=0
+REPLAY_END_TS=0
+REPLAY_SPEED=1
+REPLAY_BATCH_SIZE=200
+REPLAY_LOG_EVERY=1000
diff --git a/README.md b/README.md
index 56597bc..3468c9c 100644
--- a/README.md
+++ b/README.md
@@ -200,6 +200,16 @@ Provider links: [Interactive Brokers](https://www.interactivebrokers.com/), [IBK
- `CLASSIFIER_0DTE_MIN_PREMIUM` (default `20000`) — 0DTE gamma punch premium floor.
- `CLASSIFIER_0DTE_MIN_SIZE` (default `400`) — 0DTE gamma punch size floor.
+### Replay service
+
+- `REPLAY_ENABLED` (default `false`) — start the replay streamer when running `bun run dev`.
+- `REPLAY_STREAMS` (default `options,nbbo,equities,equity-quotes`) — comma list of streams to re-publish.
+- `REPLAY_START_TS` (default `0`) — start timestamp in ms since epoch (0 means beginning).
+- `REPLAY_END_TS` (default `0`) — end timestamp in ms since epoch (0 means no end).
+- `REPLAY_SPEED` (default `1`) — playback speed (1 = real-time, 2 = 2x, 0 = as fast as possible).
+- `REPLAY_BATCH_SIZE` (default `200`) — batch size per ClickHouse fetch.
+- `REPLAY_LOG_EVERY` (default `1000`) — log progress every N events.
+
### Testing + throttling
- `TESTING_MODE` (default `false`) — enable ingest throttling for local dev.
diff --git a/scripts/dev.ts b/scripts/dev.ts
index 97651cf..b8bd265 100644
--- a/scripts/dev.ts
+++ b/scripts/dev.ts
@@ -18,6 +18,14 @@ const sleep = (delayMs: number): Promise => {
return new Promise((resolve) => setTimeout(resolve, delayMs));
};
+const parseBool = (value: string | undefined): boolean => {
+ if (!value) {
+ return false;
+ }
+ const normalized = value.trim().toLowerCase();
+ return ["1", "true", "yes", "on"].includes(normalized);
+};
+
const parseUrlHostPort = (
value: string,
fallbackHost: string,
@@ -154,6 +162,10 @@ const serviceTasks: ChildSpec[] = [
{ name: "api", cmd: ["bun", "run", "dev"], cwd: "services/api" }
];
+if (parseBool(process.env.REPLAY_ENABLED)) {
+ serviceTasks.push({ name: "replay", cmd: ["bun", "run", "dev"], cwd: "services/replay" });
+}
+
spawnChild(infraTask);
await waitForInfra();
diff --git a/services/replay/package.json b/services/replay/package.json
new file mode 100644
index 0000000..23bcfe3
--- /dev/null
+++ b/services/replay/package.json
@@ -0,0 +1,16 @@
+{
+ "name": "@islandflow/replay",
+ "private": true,
+ "type": "module",
+ "scripts": {
+ "dev": "bun run src/index.ts"
+ },
+ "dependencies": {
+ "@islandflow/bus": "workspace:*",
+ "@islandflow/config": "workspace:*",
+ "@islandflow/observability": "workspace:*",
+ "@islandflow/storage": "workspace:*",
+ "@islandflow/types": "workspace:*",
+ "zod": "^3.23.8"
+ }
+}
diff --git a/services/replay/src/index.ts b/services/replay/src/index.ts
new file mode 100644
index 0000000..9de942b
--- /dev/null
+++ b/services/replay/src/index.ts
@@ -0,0 +1,458 @@
+import { readEnv } from "@islandflow/config";
+import { createLogger, createMetrics } from "@islandflow/observability";
+import {
+ SUBJECT_EQUITY_PRINTS,
+ SUBJECT_EQUITY_QUOTES,
+ SUBJECT_OPTION_NBBO,
+ SUBJECT_OPTION_PRINTS,
+ STREAM_EQUITY_PRINTS,
+ STREAM_EQUITY_QUOTES,
+ STREAM_OPTION_NBBO,
+ STREAM_OPTION_PRINTS,
+ connectJetStreamWithRetry,
+ ensureStream,
+ publishJson
+} from "@islandflow/bus";
+import {
+ createClickHouseClient,
+ fetchEquityPrintsAfter,
+ fetchEquityQuotesAfter,
+ fetchOptionNBBOAfter,
+ fetchOptionPrintsAfter
+} from "@islandflow/storage";
+import type { EquityPrint, EquityQuote, OptionNBBO, OptionPrint } from "@islandflow/types";
+import { z } from "zod";
+
+const service = "replay";
+const logger = createLogger({ service });
+const metrics = createMetrics({ service });
+
+const envSchema = z.object({
+ NATS_URL: z.string().default("nats://127.0.0.1:4222"),
+ CLICKHOUSE_URL: z.string().default("http://127.0.0.1:8123"),
+ CLICKHOUSE_DATABASE: z.string().default("default"),
+ REPLAY_STREAMS: z.string().default("options,nbbo,equities,equity-quotes"),
+ REPLAY_START_TS: z.coerce.number().int().nonnegative().default(0),
+ REPLAY_END_TS: z.coerce.number().int().nonnegative().default(0),
+ REPLAY_SPEED: z.coerce.number().nonnegative().default(1),
+ REPLAY_BATCH_SIZE: z.coerce.number().int().positive().default(200),
+ REPLAY_LOG_EVERY: z.coerce.number().int().positive().default(1000)
+});
+
+const env = readEnv(envSchema);
+
+type ReplayCursor = {
+ ts: number;
+ seq: number;
+};
+
+type ReplayStreamKind = "options" | "nbbo" | "equities" | "equity-quotes";
+
+type ReplayEvent = OptionPrint | OptionNBBO | EquityPrint | EquityQuote;
+
+type FetchAfter = (
+ afterTs: number,
+ afterSeq: number,
+ limit: number
+) => Promise;
+
+type ReplayStream = {
+ kind: ReplayStreamKind;
+ subject: string;
+ streamName: string;
+ fetchAfter: FetchAfter;
+ buffer: ReplayEvent[];
+ cursor: ReplayCursor;
+ done: boolean;
+ emitted: number;
+ rank: number;
+};
+
+// Tie-breaker order favors quotes before prints when timestamps match.
+const STREAM_ORDER: ReplayStreamKind[] = ["nbbo", "options", "equity-quotes", "equities"];
+
+const STREAM_DEFS: Record<
+ ReplayStreamKind,
+ {
+ subject: string;
+ streamName: string;
+ rank: number;
+ fetchAfter: (client: ReturnType, afterTs: number, afterSeq: number, limit: number) => Promise;
+ }
+> = {
+ options: {
+ subject: SUBJECT_OPTION_PRINTS,
+ streamName: STREAM_OPTION_PRINTS,
+ rank: STREAM_ORDER.indexOf("options"),
+ fetchAfter: (client, afterTs, afterSeq, limit) =>
+ fetchOptionPrintsAfter(client, afterTs, afterSeq, limit)
+ },
+ nbbo: {
+ subject: SUBJECT_OPTION_NBBO,
+ streamName: STREAM_OPTION_NBBO,
+ rank: STREAM_ORDER.indexOf("nbbo"),
+ fetchAfter: (client, afterTs, afterSeq, limit) =>
+ fetchOptionNBBOAfter(client, afterTs, afterSeq, limit)
+ },
+ equities: {
+ subject: SUBJECT_EQUITY_PRINTS,
+ streamName: STREAM_EQUITY_PRINTS,
+ rank: STREAM_ORDER.indexOf("equities"),
+ fetchAfter: (client, afterTs, afterSeq, limit) =>
+ fetchEquityPrintsAfter(client, afterTs, afterSeq, limit)
+ },
+ "equity-quotes": {
+ subject: SUBJECT_EQUITY_QUOTES,
+ streamName: STREAM_EQUITY_QUOTES,
+ rank: STREAM_ORDER.indexOf("equity-quotes"),
+ fetchAfter: (client, afterTs, afterSeq, limit) =>
+ fetchEquityQuotesAfter(client, afterTs, afterSeq, limit)
+ }
+};
+
+const sleep = (delayMs: number): Promise =>
+ new Promise((resolve) => setTimeout(resolve, delayMs));
+
+const normalizeStreamName = (value: string): ReplayStreamKind | null => {
+ switch (value.trim().toLowerCase()) {
+ case "options":
+ case "option-prints":
+ case "option_prints":
+ case "options-prints":
+ return "options";
+ case "nbbo":
+ case "option-nbbo":
+ case "option_nbbo":
+ case "options-nbbo":
+ return "nbbo";
+ case "equities":
+ case "equity":
+ case "equity-prints":
+ case "equity_prints":
+ return "equities";
+ case "equity-quotes":
+ case "equity_quotes":
+ case "quotes":
+ return "equity-quotes";
+ case "all":
+ return null;
+ default:
+ return null;
+ }
+};
+
+const parseStreamList = (value: string): ReplayStreamKind[] => {
+ const tokens = value
+ .split(",")
+ .map((entry) => entry.trim())
+ .filter(Boolean);
+
+ if (tokens.some((entry) => entry.toLowerCase() === "all")) {
+ return [...STREAM_ORDER];
+ }
+
+ const seen = new Set();
+ const result: ReplayStreamKind[] = [];
+ const invalid: string[] = [];
+
+ for (const token of tokens) {
+ const kind = normalizeStreamName(token);
+ if (!kind) {
+ invalid.push(token);
+ continue;
+ }
+ if (!seen.has(kind)) {
+ seen.add(kind);
+ result.push(kind);
+ }
+ }
+
+ if (invalid.length > 0) {
+ throw new Error(`Unknown replay stream(s): ${invalid.join(", ")}`);
+ }
+
+ if (result.length === 0) {
+ throw new Error("No replay streams selected.");
+ }
+
+ return result;
+};
+
+const buildStreamConfig = (name: string, subject: string) => ({
+ name,
+ subjects: [subject],
+ retention: "limits",
+ storage: "file",
+ discard: "old",
+ max_msgs_per_subject: -1,
+ max_msgs: -1,
+ max_bytes: -1,
+ max_age: 0,
+ num_replicas: 1
+});
+
+const buildStartCursor = (startTs: number): ReplayCursor => {
+ if (startTs <= 0) {
+ return { ts: 0, seq: 0 };
+ }
+
+ const adjusted = Math.max(0, startTs - 1);
+ return { ts: adjusted, seq: 0 };
+};
+
+const getEventTs = (event: ReplayEvent): number => (Number.isFinite(event.ts) ? event.ts : 0);
+
+const getEventIngestTs = (event: ReplayEvent): number =>
+ Number.isFinite(event.ingest_ts) ? event.ingest_ts : 0;
+
+const getEventSeq = (event: ReplayEvent): number => (Number.isFinite(event.seq) ? event.seq : 0);
+
+const pickNextEvent = (streams: ReplayStream[]): { stream: ReplayStream; event: ReplayEvent } | null => {
+ let choice: { stream: ReplayStream; event: ReplayEvent } | null = null;
+
+ for (const stream of streams) {
+ const event = stream.buffer[0];
+ if (!event) {
+ continue;
+ }
+
+ if (!choice) {
+ choice = { stream, event };
+ continue;
+ }
+
+ const candidateTs = getEventTs(event);
+ const currentTs = getEventTs(choice.event);
+ if (candidateTs !== currentTs) {
+ if (candidateTs < currentTs) {
+ choice = { stream, event };
+ }
+ continue;
+ }
+
+ const candidateIngest = getEventIngestTs(event);
+ const currentIngest = getEventIngestTs(choice.event);
+ if (candidateIngest !== currentIngest) {
+ if (candidateIngest < currentIngest) {
+ choice = { stream, event };
+ }
+ continue;
+ }
+
+ const candidateSeq = getEventSeq(event);
+ const currentSeq = getEventSeq(choice.event);
+ if (candidateSeq !== currentSeq) {
+ if (candidateSeq < currentSeq) {
+ choice = { stream, event };
+ }
+ continue;
+ }
+
+ if (stream.rank < choice.stream.rank) {
+ choice = { stream, event };
+ }
+ }
+
+ return choice;
+};
+
+const retry = async (
+ label: string,
+ attempts: number,
+ delayMs: number,
+ task: () => Promise
+): Promise => {
+ let lastError: unknown;
+
+ for (let attempt = 1; attempt <= attempts; attempt += 1) {
+ try {
+ return await task();
+ } catch (error) {
+ lastError = error;
+ logger.warn(`${label} attempt failed`, {
+ attempt,
+ error: error instanceof Error ? error.message : String(error)
+ });
+
+ if (attempt < attempts) {
+ await sleep(delayMs);
+ }
+ }
+ }
+
+ throw lastError ?? new Error(`${label} failed after retries`);
+};
+
+const run = async () => {
+ logger.info("service starting");
+
+ if (env.REPLAY_END_TS > 0 && env.REPLAY_END_TS < env.REPLAY_START_TS) {
+ throw new Error("REPLAY_END_TS must be >= REPLAY_START_TS when set.");
+ }
+
+ const streamKinds = parseStreamList(env.REPLAY_STREAMS);
+
+ const { nc, js, jsm } = await connectJetStreamWithRetry(
+ {
+ servers: env.NATS_URL,
+ name: service
+ },
+ { attempts: 120, delayMs: 500 }
+ );
+
+ for (const kind of streamKinds) {
+ const def = STREAM_DEFS[kind];
+ await ensureStream(jsm, buildStreamConfig(def.streamName, def.subject));
+ }
+
+ const clickhouse = createClickHouseClient({
+ url: env.CLICKHOUSE_URL,
+ database: env.CLICKHOUSE_DATABASE
+ });
+
+ await retry("clickhouse ready", 20, 500, async () => {
+ await clickhouse.query({ query: "SELECT 1", format: "JSONEachRow" });
+ });
+
+ const startCursor = buildStartCursor(env.REPLAY_START_TS);
+ const streams: ReplayStream[] = streamKinds.map((kind) => {
+ const def = STREAM_DEFS[kind];
+ return {
+ kind,
+ subject: def.subject,
+ streamName: def.streamName,
+ fetchAfter: (afterTs, afterSeq, limit) => def.fetchAfter(clickhouse, afterTs, afterSeq, limit),
+ buffer: [],
+ cursor: { ...startCursor },
+ done: false,
+ emitted: 0,
+ rank: def.rank
+ };
+ });
+
+ logger.info("replay configured", {
+ streams: streams.map((stream) => stream.kind),
+ start_ts: env.REPLAY_START_TS,
+ end_ts: env.REPLAY_END_TS > 0 ? env.REPLAY_END_TS : null,
+ speed: env.REPLAY_SPEED,
+ batch_size: env.REPLAY_BATCH_SIZE
+ });
+
+ let stopping = false;
+ let baseEventTs: number | null = null;
+ let startWallMs = 0;
+ let totalEmitted = 0;
+
+ const shutdown = async (signal: string) => {
+ if (stopping) {
+ return;
+ }
+ stopping = true;
+ logger.info("service stopping", { signal });
+ await nc.drain();
+ await clickhouse.close();
+ process.exit(0);
+ };
+
+ process.on("SIGINT", () => void shutdown("SIGINT"));
+ process.on("SIGTERM", () => void shutdown("SIGTERM"));
+
+ const speed = env.REPLAY_SPEED;
+ const endTs = env.REPLAY_END_TS > 0 ? env.REPLAY_END_TS : null;
+
+ while (!stopping) {
+ for (const stream of streams) {
+ if (stream.done || stream.buffer.length > 0) {
+ continue;
+ }
+
+ const data = await stream.fetchAfter(
+ stream.cursor.ts,
+ stream.cursor.seq,
+ env.REPLAY_BATCH_SIZE
+ );
+
+ if (data.length === 0) {
+ stream.done = true;
+ logger.info("replay stream exhausted", { stream: stream.kind });
+ continue;
+ }
+
+ stream.buffer = data;
+ metrics.gauge("replay.buffer_depth", data.length, { stream: stream.kind });
+ }
+
+ const next = pickNextEvent(streams);
+ if (!next) {
+ break;
+ }
+
+ const { stream, event } = next;
+ const eventTs = getEventTs(event);
+ const eventSeq = getEventSeq(event);
+
+ if (endTs !== null && eventTs > endTs) {
+ logger.info("replay reached end timestamp", { end_ts: endTs, last_ts: eventTs });
+ break;
+ }
+
+ if (baseEventTs === null) {
+ baseEventTs = eventTs;
+ startWallMs = Date.now();
+ }
+
+ if (speed > 0 && baseEventTs !== null) {
+ const targetMs = startWallMs + (eventTs - baseEventTs) / speed;
+ const delayMs = Math.max(0, targetMs - Date.now());
+ if (delayMs > 0) {
+ await sleep(delayMs);
+ }
+ }
+
+ try {
+ await publishJson(js, stream.subject, event);
+ } catch (error) {
+ logger.error("failed to publish replay event", {
+ error: error instanceof Error ? error.message : String(error),
+ stream: stream.kind,
+ ts: eventTs,
+ seq: eventSeq
+ });
+ throw error;
+ }
+
+ stream.buffer.shift();
+ stream.cursor = { ts: eventTs, seq: eventSeq };
+ stream.emitted += 1;
+ totalEmitted += 1;
+ metrics.count("replay.emitted", 1, { stream: stream.kind });
+
+ if (totalEmitted % env.REPLAY_LOG_EVERY === 0) {
+ logger.info("replay progress", {
+ emitted: totalEmitted,
+ last_ts: eventTs
+ });
+ }
+ }
+
+ logger.info("replay complete", {
+ emitted: totalEmitted,
+ streams: streams.map((stream) => ({
+ stream: stream.kind,
+ emitted: stream.emitted
+ }))
+ });
+
+ await nc.drain();
+ await clickhouse.close();
+ process.exit(0);
+};
+
+try {
+ await run();
+} catch (error) {
+ logger.error("replay service failed", {
+ error: error instanceof Error ? error.message : String(error)
+ });
+ process.exit(1);
+}
diff --git a/services/replay/tsconfig.json b/services/replay/tsconfig.json
new file mode 100644
index 0000000..d8c6443
--- /dev/null
+++ b/services/replay/tsconfig.json
@@ -0,0 +1,7 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": {
+ "types": []
+ },
+ "include": ["src/**/*.ts"]
+}
From 9edf8fcbc5dfafe182c1bb413c52980590a19cd0 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Sat, 10 Jan 2026 16:08:23 -0500
Subject: [PATCH 011/237] Emit equity candles on schedule to avoid stalled
charts
---
services/candles/src/aggregator.ts | 12 ++++++++++++
services/candles/src/index.ts | 14 ++++++++++++++
services/candles/tests/aggregator.test.ts | 15 +++++++++++++++
3 files changed, 41 insertions(+)
diff --git a/services/candles/src/aggregator.ts b/services/candles/src/aggregator.ts
index e00d9b9..37a8921 100644
--- a/services/candles/src/aggregator.ts
+++ b/services/candles/src/aggregator.ts
@@ -233,6 +233,18 @@ export class CandleAggregator {
return { emitted, droppedLate };
}
+ flushExpired(now: number): EquityCandle[] {
+ const watermark = Math.max(0, Math.floor(now) - this.maxLateMs);
+ const emitted: EquityCandle[] = [];
+
+ for (const state of this.stateByKey.values()) {
+ state.lastTsSeen = Math.max(state.lastTsSeen, watermark);
+ emitted.push(...flushState(state, watermark));
+ }
+
+ return emitted;
+ }
+
drain(): EquityCandle[] {
const emitted: EquityCandle[] = [];
diff --git a/services/candles/src/index.ts b/services/candles/src/index.ts
index a625509..a02ab70 100644
--- a/services/candles/src/index.ts
+++ b/services/candles/src/index.ts
@@ -329,6 +329,18 @@ const run = async () => {
let droppedLate = 0;
let lastLateLog = Date.now();
+ const flushExpired = async () => {
+ const expired = aggregator.flushExpired(Date.now());
+ for (const candle of expired) {
+ const validated = EquityCandleSchema.parse(candle);
+ await emitCandle(clickhouse, js, redis, validated, env.CANDLE_CACHE_LIMIT);
+ }
+ };
+
+ const flushTimer = setInterval(() => {
+ void flushExpired();
+ }, 1000);
+
const loop = async () => {
for await (const msg of subscription.messages) {
try {
@@ -365,6 +377,8 @@ const run = async () => {
const shutdown = async (signal: string) => {
logger.info("service stopping", { signal });
+ clearInterval(flushTimer);
+ await flushExpired();
const remaining = aggregator.drain();
for (const candle of remaining) {
const validated = EquityCandleSchema.parse(candle);
diff --git a/services/candles/tests/aggregator.test.ts b/services/candles/tests/aggregator.test.ts
index 79a39b2..bbfe57b 100644
--- a/services/candles/tests/aggregator.test.ts
+++ b/services/candles/tests/aggregator.test.ts
@@ -78,4 +78,19 @@ describe("CandleAggregator", () => {
expect(lateResult.emitted).toHaveLength(0);
expect(lateResult.droppedLate).toBe(1);
});
+
+ test("flushes expired windows without new prints", () => {
+ const aggregator = new CandleAggregator({ intervalsMs: [1000], maxLateMs: 0 });
+
+ const first = buildPrint({ ts: 1000, price: 10, size: 100, seq: 1 });
+ const second = buildPrint({ ts: 1500, price: 12, size: 50, seq: 2 });
+
+ aggregator.ingest(first);
+ aggregator.ingest(second);
+
+ const emitted = aggregator.flushExpired(2500);
+ expect(emitted).toHaveLength(1);
+ expect(emitted[0].ts).toBe(1000);
+ expect(emitted[0].close).toBe(12);
+ });
});
From af328f1b32d660f30408f96a239db4a312d7a16b Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Sat, 10 Jan 2026 16:09:52 -0500
Subject: [PATCH 012/237] NOW THIS IS A PROPER FUCKING CLI! OpenCode <3
From 4a22fcc635c571972a18e16487dd23919f0bd0f3 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Sun, 11 Jan 2026 10:47:11 -0500
Subject: [PATCH 013/237] Fix options tape replay formatting
---
apps/web/app/page.tsx | 35 +++++++++++++++++++++++++++++------
1 file changed, 29 insertions(+), 6 deletions(-)
diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx
index 8e182bb..ce8fe40 100644
--- a/apps/web/app/page.tsx
+++ b/apps/web/app/page.tsx
@@ -232,7 +232,13 @@ const buildApiUrl = (path: string): string => {
};
const formatPrice = (price: number): string => {
- return price.toFixed(2);
+ if (!Number.isFinite(price)) {
+ return "0.00";
+ }
+ return price.toLocaleString(undefined, {
+ minimumFractionDigits: 2,
+ maximumFractionDigits: 2
+ });
};
const formatSize = (size: number): string => {
@@ -257,6 +263,19 @@ const formatUsd = (value: number): string => {
});
};
+const normalizeContractId = (value: string): string => value.trim();
+
+const formatContractLabel = (value: string): string => {
+ const normalized = normalizeContractId(value);
+ if (!normalized) {
+ return "Unknown contract";
+ }
+ if (/^\d+$/.test(normalized)) {
+ return `Instrument ${normalized}`;
+ }
+ return normalized;
+};
+
const formatDateTime = (ts: number): string => {
const date = new Date(ts);
return `${date.toLocaleDateString()} ${date.toLocaleTimeString()}`;
@@ -2032,13 +2051,14 @@ export default function HomePage() {
const nbboMap = useMemo(() => {
const map = new Map();
for (const quote of nbbo.items) {
- const existing = map.get(quote.option_contract_id);
+ const contractId = normalizeContractId(quote.option_contract_id);
+ const existing = map.get(contractId);
if (
!existing ||
quote.ts > existing.ts ||
(quote.ts === existing.ts && quote.seq >= existing.seq)
) {
- map.set(quote.option_contract_id, quote);
+ map.set(contractId, quote);
}
}
return map;
@@ -2196,7 +2216,7 @@ export default function HomePage() {
return options.items;
}
return options.items.filter((print) =>
- matchesTicker(extractUnderlying(print.option_contract_id))
+ matchesTicker(extractUnderlying(normalizeContractId(print.option_contract_id)))
);
}, [options.items, matchesTicker, tickerSet]);
@@ -2386,20 +2406,23 @@ export default function HomePage() {
) : (
filteredOptions.map((print) => {
- const quote = nbboMap.get(print.option_contract_id);
+ const contractId = normalizeContractId(print.option_contract_id);
+ const quote = nbboMap.get(contractId);
const nbboAge = quote ? Math.abs(print.ts - quote.ts) : null;
const nbboStale = nbboAge !== null && nbboAge > NBBO_MAX_AGE_MS_SAFE;
const nbboMid = quote ? (quote.bid + quote.ask) / 2 : null;
const nbboSide = classifyNbboSide(print.price, quote);
+ const notional = print.price * print.size * 100;
return (
Classifier hits are live-only in this build.
- ) : filteredClassifierHits.length === 0 ? (
+ {filteredClassifierHits.length === 0 ? (
{tickerSet.size > 0
? "No classifier hits match the current filter."
- : "No classifier hits yet. Start compute."}
+ : mode === "live"
+ ? "No classifier hits yet. Start compute."
+ : "Replay queue empty. Ensure ClickHouse has data."}
) : (
filteredClassifierHits.map((hit) => {
diff --git a/packages/storage/src/clickhouse.ts b/packages/storage/src/clickhouse.ts
index 594dec9..8ed0aff 100644
--- a/packages/storage/src/clickhouse.ts
+++ b/packages/storage/src/clickhouse.ts
@@ -860,3 +860,72 @@ export const fetchInferredDarkAfter = async (
const events = records.map(fromInferredDarkRecord);
return InferredDarkEventSchema.array().parse(events);
};
+
+export const fetchFlowPacketsAfter = async (
+ client: ClickHouseClient,
+ afterTs: number,
+ afterSeq: number,
+ limit: number
+): Promise
-
Date: Tue, 20 Jan 2026 12:40:31 -0500
Subject: [PATCH 024/237] Add classifier and dark markers to chart
---
apps/web/app/page.tsx | 259 +++++++++++++++++++++++++++++++++++++++++-
1 file changed, 257 insertions(+), 2 deletions(-)
diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx
index d32e8bc..e6b1430 100644
--- a/apps/web/app/page.tsx
+++ b/apps/web/app/page.tsx
@@ -12,7 +12,7 @@ import type {
OptionNBBO,
OptionPrint
} from "@islandflow/types";
-import { createChart, type IChartApi, type UTCTimestamp } from "lightweight-charts";
+import { createChart, type IChartApi, type SeriesMarker, type UTCTimestamp } from "lightweight-charts";
const MAX_ITEMS = 500;
const NBBO_MAX_AGE_MS = Number(process.env.NEXT_PUBLIC_NBBO_MAX_AGE_MS);
@@ -1371,9 +1371,26 @@ type CandleChartProps = {
intervalMs: number;
mode: TapeMode;
replayTime?: number | null;
+ classifierHits: ClassifierHitEvent[];
+ inferredDark: InferredDarkEvent[];
+ onClassifierHitClick: (hit: ClassifierHitEvent) => void;
+ onInferredDarkClick: (event: InferredDarkEvent) => void;
};
-const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChartProps) => {
+type MarkerAction =
+ | { kind: "hit"; hit: ClassifierHitEvent }
+ | { kind: "dark"; event: InferredDarkEvent };
+
+const CandleChart = ({
+ ticker,
+ intervalMs,
+ mode,
+ replayTime = null,
+ classifierHits,
+ inferredDark,
+ onClassifierHitClick,
+ onInferredDarkClick
+}: CandleChartProps) => {
const containerRef = useRef(null);
const chartRef = useRef(null);
const seriesRef = useRef(null);
@@ -1383,6 +1400,11 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
const overlayReconnectRef = useRef(null);
const lastCandleRef = useRef<{ time: UTCTimestamp; seq: number } | null>(null);
+ const markerLookupRef = useRef>(new Map());
+ const [visibleRangeMs, setVisibleRangeMs] = useState<{ from: number; to: number } | null>(null);
+ const onHitClickRef = useRef(onClassifierHitClick);
+ const onDarkClickRef = useRef(onInferredDarkClick);
+
const overlayCanvasRef = useRef(null);
const overlayCtxRef = useRef(null);
const overlayDataRef = useRef([]);
@@ -1457,6 +1479,132 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
drawOverlay([...overlayDataRef.current, ...overlayLiveRef.current]);
}, [drawOverlay, ticker, intervalMs, mode]);
+ useEffect(() => {
+ onHitClickRef.current = onClassifierHitClick;
+ }, [onClassifierHitClick]);
+
+ useEffect(() => {
+ onDarkClickRef.current = onInferredDarkClick;
+ }, [onInferredDarkClick]);
+
+ const markerBundle = useMemo(() => {
+ const lookup = new Map();
+ const markers: SeriesMarker[] = [];
+
+ if (!visibleRangeMs) {
+ return { markers, lookup };
+ }
+
+ const { from, to } = visibleRangeMs;
+ const inRangeHits = classifierHits
+ .filter((hit) => hit.source_ts >= from && hit.source_ts <= to)
+ .sort((a, b) => {
+ const delta = a.source_ts - b.source_ts;
+ if (delta !== 0) {
+ return delta;
+ }
+ return a.seq - b.seq;
+ });
+ const inRangeDark = inferredDark
+ .filter((event) => event.source_ts >= from && event.source_ts <= to)
+ .sort((a, b) => {
+ const delta = a.source_ts - b.source_ts;
+ if (delta !== 0) {
+ return delta;
+ }
+ return a.seq - b.seq;
+ });
+
+ const MAX_HIT_MARKERS = 220;
+ const MAX_DARK_MARKERS = 120;
+ const MAX_TOTAL_MARKERS = 320;
+
+ const cappedHits =
+ inRangeHits.length > MAX_HIT_MARKERS
+ ? inRangeHits.slice(inRangeHits.length - MAX_HIT_MARKERS)
+ : inRangeHits;
+ const cappedDark =
+ inRangeDark.length > MAX_DARK_MARKERS
+ ? inRangeDark.slice(inRangeDark.length - MAX_DARK_MARKERS)
+ : inRangeDark;
+
+ for (const hit of cappedHits) {
+ const direction = normalizeDirection(hit.direction);
+ const markerId = `hit:${hit.trace_id}:${hit.seq}`;
+ lookup.set(markerId, { kind: "hit", hit });
+
+ markers.push({
+ id: markerId,
+ time: toChartTime(hit.source_ts),
+ position: direction === "bullish" ? "belowBar" : "aboveBar",
+ color:
+ direction === "bullish"
+ ? "#2f6d4f"
+ : direction === "bearish"
+ ? "#c46f2a"
+ : "rgba(111, 91, 57, 0.9)",
+ shape:
+ direction === "bullish"
+ ? "arrowUp"
+ : direction === "bearish"
+ ? "arrowDown"
+ : "circle",
+ text: hit.classifier_id ? hit.classifier_id.slice(0, 3).toUpperCase() : "H"
+ });
+ }
+
+ for (const event of cappedDark) {
+ const markerId = `dark:${event.trace_id}:${event.seq}`;
+ lookup.set(markerId, { kind: "dark", event });
+ markers.push({
+ id: markerId,
+ time: toChartTime(event.source_ts),
+ position: "aboveBar",
+ color: "rgba(31, 74, 123, 0.9)",
+ shape: "square",
+ text: "D"
+ });
+ }
+
+ markers.sort((a, b) => {
+ const delta = Number(a.time) - Number(b.time);
+ if (delta !== 0) {
+ return delta;
+ }
+ return String(a.id ?? "").localeCompare(String(b.id ?? ""));
+ });
+
+ const cappedMarkers =
+ markers.length > MAX_TOTAL_MARKERS
+ ? markers.slice(markers.length - MAX_TOTAL_MARKERS)
+ : markers;
+
+ if (cappedMarkers !== markers) {
+ const nextLookup = new Map();
+ for (const marker of cappedMarkers) {
+ const id = marker.id;
+ if (typeof id !== "string") {
+ continue;
+ }
+ const action = lookup.get(id);
+ if (action) {
+ nextLookup.set(id, action);
+ }
+ }
+ return { markers: cappedMarkers, lookup: nextLookup };
+ }
+
+ return { markers: cappedMarkers, lookup };
+ }, [classifierHits, inferredDark, visibleRangeMs]);
+
+ useEffect(() => {
+ if (!seriesRef.current) {
+ return;
+ }
+ markerLookupRef.current = markerBundle.lookup;
+ seriesRef.current.setMarkers(markerBundle.markers);
+ }, [markerBundle]);
+
const replayBucket = useMemo(() => {
if (mode !== "replay" || replayTime === null) {
return null;
@@ -1533,6 +1681,47 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
seriesRef.current = series;
setReady(true);
+ const timeScale = chart.timeScale();
+ const updateVisibleRange = () => {
+ const range = timeScale.getVisibleRange();
+ if (!range) {
+ setVisibleRangeMs(null);
+ return;
+ }
+ const from = chartTimeToMs(range.from);
+ const to = chartTimeToMs(range.to);
+ if (from === null || to === null) {
+ setVisibleRangeMs(null);
+ return;
+ }
+
+ setVisibleRangeMs({
+ from: Math.min(from, to),
+ to: Math.max(from, to)
+ });
+ };
+
+ const clickHandler = (param: { hoveredObjectId?: unknown }) => {
+ const hovered = param.hoveredObjectId;
+ if (hovered === null || hovered === undefined) {
+ return;
+ }
+ const key = typeof hovered === "string" ? hovered : String(hovered);
+ const action = markerLookupRef.current.get(key);
+ if (!action) {
+ return;
+ }
+ if (action.kind === "hit") {
+ onHitClickRef.current(action.hit);
+ } else {
+ onDarkClickRef.current(action.event);
+ }
+ };
+
+ updateVisibleRange();
+ timeScale.subscribeVisibleTimeRangeChange(updateVisibleRange);
+ chart.subscribeClick(clickHandler);
+
const resizeObserver = new ResizeObserver((entries) => {
const entry = entries[0];
if (!entry) {
@@ -1559,6 +1748,8 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
return () => {
resizeObserver.disconnect();
+ timeScale.unsubscribeVisibleTimeRangeChange(updateVisibleRange);
+ chart.unsubscribeClick(clickHandler);
chart.remove();
chartRef.current = null;
seriesRef.current = null;
@@ -2607,6 +2798,14 @@ export default function HomePage() {
return extractUnderlying(match[1]);
}, []);
+ const extractPacketIdFromClassifierHitTrace = useCallback((traceId: string): string | null => {
+ const idx = traceId.indexOf("flowpacket:");
+ if (idx < 0) {
+ return null;
+ }
+ return traceId.slice(idx);
+ }, []);
+
const inferAlertUnderlying = useCallback(
(alert: AlertEvent): string | null => {
const fromTrace = extractUnderlyingFromTrace(alert.trace_id);
@@ -2699,6 +2898,58 @@ export default function HomePage() {
});
}, [classifierHits.items, extractUnderlyingFromTrace, matchesTicker, tickerSet]);
+ const chartClassifierHits = useMemo(() => {
+ const desired = chartTicker.toUpperCase();
+ return classifierHits.items
+ .filter((hit) => extractUnderlyingFromTrace(hit.trace_id) === desired)
+ .sort((a, b) => {
+ const delta = a.source_ts - b.source_ts;
+ if (delta !== 0) {
+ return delta;
+ }
+ return a.seq - b.seq;
+ });
+ }, [chartTicker, classifierHits.items, extractUnderlyingFromTrace]);
+
+ const chartInferredDark = useMemo(() => {
+ const desired = chartTicker.toUpperCase();
+ return inferredDark.items
+ .filter((event) => inferDarkUnderlying(event, equityPrintMap, equityJoinMap) === desired)
+ .sort((a, b) => {
+ const delta = a.source_ts - b.source_ts;
+ if (delta !== 0) {
+ return delta;
+ }
+ return a.seq - b.seq;
+ });
+ }, [chartTicker, inferredDark.items, equityJoinMap, equityPrintMap]);
+
+ const handleClassifierMarkerClick = useCallback(
+ (hit: ClassifierHitEvent) => {
+ const packetId = extractPacketIdFromClassifierHitTrace(hit.trace_id);
+ if (!packetId) {
+ return;
+ }
+
+ const desiredTrace = `alert:${packetId}`;
+ const alert = alerts.items.find(
+ (item) => item.trace_id === desiredTrace || item.evidence_refs[0] === packetId
+ );
+ if (!alert) {
+ return;
+ }
+
+ setSelectedDarkEvent(null);
+ setSelectedAlert(alert);
+ },
+ [alerts.items, extractPacketIdFromClassifierHitTrace]
+ );
+
+ const handleDarkMarkerClick = useCallback((event: InferredDarkEvent) => {
+ setSelectedAlert(null);
+ setSelectedDarkEvent(event);
+ }, []);
+
const lastSeen = useMemo(() => {
return [
options.lastUpdate,
@@ -2803,6 +3054,10 @@ export default function HomePage() {
intervalMs={chartIntervalMs}
mode={mode}
replayTime={equities.replayTime}
+ classifierHits={chartClassifierHits}
+ inferredDark={chartInferredDark}
+ onClassifierHitClick={handleClassifierMarkerClick}
+ onInferredDarkClick={handleDarkMarkerClick}
/>
From dcda4006e9cd41854a1086a25c0dc80d43f93188 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Tue, 20 Jan 2026 12:40:35 -0500
Subject: [PATCH 025/237] Add typescript-language-server dev dependency
---
bun.lock | 19 +++++++++++++++++++
package.json | 3 +++
2 files changed, 22 insertions(+)
diff --git a/bun.lock b/bun.lock
index 0408e06..d6e99c6 100644
--- a/bun.lock
+++ b/bun.lock
@@ -4,6 +4,9 @@
"workspaces": {
"": {
"name": "islandflow",
+ "devDependencies": {
+ "typescript-language-server": "^5.1.3",
+ },
},
"apps/web": {
"name": "@islandflow/web",
@@ -99,6 +102,7 @@
"@islandflow/observability": "workspace:*",
"@islandflow/storage": "workspace:*",
"@islandflow/types": "workspace:*",
+ "ws": "^8.18.3",
"zod": "^3.23.8",
},
},
@@ -122,6 +126,17 @@
"@islandflow/observability": "workspace:*",
},
},
+ "services/replay": {
+ "name": "@islandflow/replay",
+ "dependencies": {
+ "@islandflow/bus": "workspace:*",
+ "@islandflow/config": "workspace:*",
+ "@islandflow/observability": "workspace:*",
+ "@islandflow/storage": "workspace:*",
+ "@islandflow/types": "workspace:*",
+ "zod": "^3.23.8",
+ },
+ },
},
"packages": {
"@clickhouse/client": ["@clickhouse/client@0.2.10", "", { "dependencies": { "@clickhouse/client-common": "0.2.10" } }, "sha512-ZwBgzjEAFN/ogS0ym5KHVbR7Hx/oYCX01qGp2baEyfN2HM73kf/7Vp3GvMHWRy+zUXISONEtFv7UTViOXnmFrg=="],
@@ -148,6 +163,8 @@
"@islandflow/refdata": ["@islandflow/refdata@workspace:services/refdata"],
+ "@islandflow/replay": ["@islandflow/replay@workspace:services/replay"],
+
"@islandflow/storage": ["@islandflow/storage@workspace:packages/storage"],
"@islandflow/types": ["@islandflow/types@workspace:packages/types"],
@@ -248,6 +265,8 @@
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
+ "typescript-language-server": ["typescript-language-server@5.1.3", "", { "bin": { "typescript-language-server": "lib/cli.mjs" } }, "sha512-r+pAcYtWdN8tKlYZPwiiHNA2QPjXnI02NrW5Sf2cVM3TRtuQ3V9EKKwOxqwaQ0krsaEXk/CbN90I5erBuf84Vg=="],
+
"undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
"ws": ["ws@8.18.3", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg=="],
diff --git a/package.json b/package.json
index 6a8122b..0d570a9 100644
--- a/package.json
+++ b/package.json
@@ -13,5 +13,8 @@
"dev:infra:down": "docker compose down",
"dev:web": "bun --cwd=apps/web run dev",
"dev:services": "bun run scripts/dev-services.ts"
+ },
+ "devDependencies": {
+ "typescript-language-server": "^5.1.3"
}
}
From f08abec68a4c0730b0bf8929b20c44727fb7e5e4 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Tue, 20 Jan 2026 12:57:38 -0500
Subject: [PATCH 026/237] Open classifier hit drawer when alert missing
---
apps/web/app/page.tsx | 228 ++++++++++++++++++++++++++++++++++++++----
1 file changed, 206 insertions(+), 22 deletions(-)
diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx
index e6b1430..579ad2c 100644
--- a/apps/web/app/page.tsx
+++ b/apps/web/app/page.tsx
@@ -2388,6 +2388,114 @@ const AlertDrawer = ({ alert, flowPacket, evidence, onClose }: AlertDrawerProps)
);
};
+type ClassifierHitDrawerProps = {
+ hit: ClassifierHitEvent;
+ flowPacket: FlowPacket | null;
+ evidence: EvidenceItem[];
+ onClose: () => void;
+};
+
+const ClassifierHitDrawer = ({ hit, flowPacket, evidence, onClose }: ClassifierHitDrawerProps) => {
+ const direction = normalizeDirection(hit.direction);
+ const evidencePrints = evidence.filter((item) => item.kind === "print");
+ const unknownCount = evidence.filter((item) => item.kind === "unknown").length;
+
+ return (
+
+ );
+};
+
type DarkDrawerProps = {
event: InferredDarkEvent;
evidence: DarkEvidenceItem[];
@@ -2513,6 +2621,7 @@ export default function HomePage() {
const [replaySource, setReplaySource] = useState(null);
const [selectedAlert, setSelectedAlert] = useState(null);
const [selectedDarkEvent, setSelectedDarkEvent] = useState(null);
+ const [selectedClassifierHit, setSelectedClassifierHit] = useState(null);
const [filterInput, setFilterInput] = useState("");
const [chartIntervalMs, setChartIntervalMs] = useState(CANDLE_INTERVALS[0].ms);
@@ -2779,6 +2888,7 @@ export default function HomePage() {
setSelectedAlert(null);
}
setSelectedDarkEvent(null);
+ setSelectedClassifierHit(null);
}, [mode]);
const extractPacketContract = useCallback((packet: FlowPacket): string => {
@@ -2806,6 +2916,43 @@ export default function HomePage() {
return traceId.slice(idx);
}, []);
+ const selectedClassifierPacketId = useMemo(() => {
+ if (!selectedClassifierHit) {
+ return null;
+ }
+ return extractPacketIdFromClassifierHitTrace(selectedClassifierHit.trace_id);
+ }, [extractPacketIdFromClassifierHitTrace, selectedClassifierHit]);
+
+ const selectedClassifierFlowPacket = useMemo(() => {
+ if (!selectedClassifierPacketId) {
+ return null;
+ }
+ return flowPacketMap.get(selectedClassifierPacketId) ?? null;
+ }, [flowPacketMap, selectedClassifierPacketId]);
+
+ const selectedClassifierEvidence = useMemo((): EvidenceItem[] => {
+ if (!selectedClassifierHit) {
+ return [];
+ }
+
+ if (!selectedClassifierPacketId) {
+ return [];
+ }
+
+ const packet = flowPacketMap.get(selectedClassifierPacketId);
+ if (!packet) {
+ return [];
+ }
+
+ return packet.members.map((id) => {
+ const print = optionPrintMap.get(id);
+ if (print) {
+ return { kind: "print", id, print };
+ }
+ return { kind: "unknown", id };
+ });
+ }, [flowPacketMap, optionPrintMap, selectedClassifierHit, selectedClassifierPacketId]);
+
const inferAlertUnderlying = useCallback(
(alert: AlertEvent): string | null => {
const fromTrace = extractUnderlyingFromTrace(alert.trace_id);
@@ -2924,29 +3071,50 @@ export default function HomePage() {
});
}, [chartTicker, inferredDark.items, equityJoinMap, equityPrintMap]);
- const handleClassifierMarkerClick = useCallback(
- (hit: ClassifierHitEvent) => {
+ const findAlertForClassifierHit = useCallback(
+ (hit: ClassifierHitEvent): AlertEvent | null => {
const packetId = extractPacketIdFromClassifierHitTrace(hit.trace_id);
if (!packetId) {
- return;
+ return null;
}
const desiredTrace = `alert:${packetId}`;
- const alert = alerts.items.find(
- (item) => item.trace_id === desiredTrace || item.evidence_refs[0] === packetId
+ return (
+ alerts.items.find(
+ (item) => item.trace_id === desiredTrace || item.evidence_refs[0] === packetId
+ ) ?? null
);
- if (!alert) {
- return;
- }
-
- setSelectedDarkEvent(null);
- setSelectedAlert(alert);
},
[alerts.items, extractPacketIdFromClassifierHitTrace]
);
+ const openFromClassifierHit = useCallback(
+ (hit: ClassifierHitEvent) => {
+ const alert = findAlertForClassifierHit(hit);
+ if (alert) {
+ setSelectedClassifierHit(null);
+ setSelectedDarkEvent(null);
+ setSelectedAlert(alert);
+ return;
+ }
+
+ setSelectedAlert(null);
+ setSelectedDarkEvent(null);
+ setSelectedClassifierHit(hit);
+ },
+ [findAlertForClassifierHit]
+ );
+
+ const handleClassifierMarkerClick = useCallback(
+ (hit: ClassifierHitEvent) => {
+ openFromClassifierHit(hit);
+ },
+ [openFromClassifierHit]
+ );
+
const handleDarkMarkerClick = useCallback((event: InferredDarkEvent) => {
setSelectedAlert(null);
+ setSelectedClassifierHit(null);
setSelectedDarkEvent(event);
}, []);
@@ -3401,6 +3569,7 @@ export default function HomePage() {
type="button"
onClick={() => {
setSelectedDarkEvent(null);
+ setSelectedClassifierHit(null);
setSelectedAlert(alert);
}}
>
@@ -3468,7 +3637,12 @@ export default function HomePage() {
filteredClassifierHits.map((hit) => {
const direction = normalizeDirection(hit.direction);
return (
-
{print.option_contract_id}
+ {formatContractLabel(contractId)}
${formatPrice(print.price)}
{formatSize(print.size)}x
{print.exchange}
+ Notional ${formatUsd(notional)}
{print.conditions?.length ? (
{print.conditions.join(", ")}
) : null}
From c8c8094594c69aff003ede73ec194ca71c39c80a Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Sun, 11 Jan 2026 10:47:17 -0500
Subject: [PATCH 014/237] Fix dev shutdown to stop orphaned services
---
scripts/dev-services.ts | 63 ++++++++++++++++++++++++++++++++++-----
scripts/dev.ts | 66 ++++++++++++++++++++++++++++++++++++-----
2 files changed, 115 insertions(+), 14 deletions(-)
diff --git a/scripts/dev-services.ts b/scripts/dev-services.ts
index f435237..bd3cf7b 100644
--- a/scripts/dev-services.ts
+++ b/scripts/dev-services.ts
@@ -12,12 +12,61 @@ type Child = {
const children: Child[] = [];
let shuttingDown = false;
+const sleep = (delayMs: number): Promise => {
+ return new Promise((resolve) => setTimeout(resolve, delayMs));
+};
+
+const waitForExit = async (proc: Bun.Subprocess, timeoutMs: number): Promise => {
+ const result = await Promise.race([
+ proc.exited.then(() => true),
+ sleep(timeoutMs).then(() => false)
+ ]);
+ return result;
+};
+
+const signalProcess = (pid: number, signal: NodeJS.Signals): boolean => {
+ try {
+ process.kill(-pid, signal);
+ return true;
+ } catch {
+ try {
+ process.kill(pid, signal);
+ return true;
+ } catch {
+ return false;
+ }
+ }
+};
+
+const stopChild = async (child: Child, timeoutMs = 5000): Promise => {
+ const pid = child.process.pid;
+ if (!pid) {
+ return;
+ }
+
+ if (!signalProcess(pid, "SIGINT")) {
+ return;
+ }
+
+ const exited = await waitForExit(child.process, timeoutMs);
+ if (exited) {
+ return;
+ }
+
+ if (!signalProcess(pid, "SIGKILL")) {
+ return;
+ }
+
+ await waitForExit(child.process, 2000);
+};
+
const spawnChild = ({ name, cmd, cwd }: ChildSpec): void => {
const proc = Bun.spawn(cmd, {
cwd,
stdin: "inherit",
stdout: "inherit",
- stderr: "inherit"
+ stderr: "inherit",
+ detached: true
});
children.push({ name, process: proc });
@@ -30,26 +79,26 @@ const spawnChild = ({ name, cmd, cwd }: ChildSpec): void => {
const exitCode = code ?? 0;
const statusLabel = exitCode === 0 ? "exited" : "failed";
console.error(`[dev-services] ${name} ${statusLabel} (${exitCode})`);
- shutdown(exitCode);
+ void shutdown(exitCode);
});
};
-const shutdown = (code: number): void => {
+const shutdown = async (code: number): Promise => {
if (shuttingDown) {
return;
}
shuttingDown = true;
- for (const child of children) {
- child.process.kill();
+ if (children.length > 0) {
+ await Promise.all(children.map((child) => stopChild(child)));
}
process.exit(code);
};
-process.on("SIGINT", () => shutdown(0));
-process.on("SIGTERM", () => shutdown(0));
+process.on("SIGINT", () => void shutdown(0));
+process.on("SIGTERM", () => void shutdown(0));
const tasks: ChildSpec[] = [
{ name: "ingest-options", cmd: ["bun", "run", "dev"], cwd: "services/ingest-options" },
diff --git a/scripts/dev.ts b/scripts/dev.ts
index b8bd265..c13a338 100644
--- a/scripts/dev.ts
+++ b/scripts/dev.ts
@@ -18,6 +18,50 @@ const sleep = (delayMs: number): Promise => {
return new Promise((resolve) => setTimeout(resolve, delayMs));
};
+const waitForExit = async (proc: Bun.Subprocess, timeoutMs: number): Promise => {
+ const result = await Promise.race([
+ proc.exited.then(() => true),
+ sleep(timeoutMs).then(() => false)
+ ]);
+ return result;
+};
+
+const signalProcess = (pid: number, signal: NodeJS.Signals): boolean => {
+ try {
+ process.kill(-pid, signal);
+ return true;
+ } catch {
+ try {
+ process.kill(pid, signal);
+ return true;
+ } catch {
+ return false;
+ }
+ }
+};
+
+const stopChild = async (child: Child, timeoutMs = 5000): Promise => {
+ const pid = child.process.pid;
+ if (!pid) {
+ return;
+ }
+
+ if (!signalProcess(pid, "SIGINT")) {
+ return;
+ }
+
+ const exited = await waitForExit(child.process, timeoutMs);
+ if (exited) {
+ return;
+ }
+
+ if (!signalProcess(pid, "SIGKILL")) {
+ return;
+ }
+
+ await waitForExit(child.process, 2000);
+};
+
const parseBool = (value: string | undefined): boolean => {
if (!value) {
return false;
@@ -75,7 +119,8 @@ const spawnChild = ({ name, cmd, cwd }: ChildSpec): void => {
cwd,
stdin: "inherit",
stdout: "inherit",
- stderr: "inherit"
+ stderr: "inherit",
+ detached: true
});
children.push({ name, process: proc });
@@ -93,26 +138,33 @@ const spawnChild = ({ name, cmd, cwd }: ChildSpec): void => {
"[dev] Infra failed. Ensure Docker is installed and the daemon is running (OrbStack or Docker Desktop), then retry."
);
}
- shutdown(exitCode);
+ void shutdown(exitCode);
});
};
-const shutdown = (code: number): void => {
+const shutdown = async (code: number): Promise => {
if (shuttingDown) {
return;
}
shuttingDown = true;
- for (const child of children) {
- child.process.kill();
+ const infra = children.find((child) => child.name === "infra") ?? null;
+ const services = children.filter((child) => child.name !== "infra");
+
+ if (services.length > 0) {
+ await Promise.all(services.map((child) => stopChild(child)));
+ }
+
+ if (infra) {
+ await stopChild(infra, 8000);
}
process.exit(code);
};
-process.on("SIGINT", () => shutdown(0));
-process.on("SIGTERM", () => shutdown(0));
+process.on("SIGINT", () => void shutdown(0));
+process.on("SIGTERM", () => void shutdown(0));
const waitForInfra = async (): Promise => {
const natsTarget = parseUrlHostPort(process.env.NATS_URL ?? "", "127.0.0.1", 4222);
From 4f743437d170d6b2e7b6d817f99cdd86495ad18e Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Sun, 11 Jan 2026 12:11:13 -0500
Subject: [PATCH 015/237] Add Databento NBBO replay ingestion
---
.../ingest-options/py/databento_replay.py | 68 +++++-
.../ingest-options/src/adapters/databento.ts | 206 ++++++++++++------
services/ingest-options/src/index.ts | 2 +
3 files changed, 200 insertions(+), 76 deletions(-)
diff --git a/services/ingest-options/py/databento_replay.py b/services/ingest-options/py/databento_replay.py
index cd15c4a..ea98b29 100644
--- a/services/ingest-options/py/databento_replay.py
+++ b/services/ingest-options/py/databento_replay.py
@@ -122,17 +122,32 @@ class SymbolResolver:
return len(self._pending)
+def _first_attr(record, names: list[str]):
+ for name in names:
+ if not name:
+ continue
+ value = getattr(record, name, None)
+ if value is not None:
+ return value
+ return None
+
+
+def _to_int(value, default: int = 0) -> int:
+ try:
+ return int(value)
+ except (TypeError, ValueError):
+ return default
+
+
def build_payload(record, symbol_override: str | None = None) -> dict | None:
ts_event = getattr(record, "ts_event", None)
- price = getattr(record, "price", None)
- size = getattr(record, "size", None)
symbol = symbol_override or (
getattr(record, "symbol", None)
or getattr(record, "raw_symbol", None)
or getattr(record, "instrument_id", None)
)
- if ts_event is None or price is None or size is None or symbol is None:
+ if ts_event is None or symbol is None:
return None
ts_ms = normalize_ts(ts_event)
@@ -144,21 +159,49 @@ def build_payload(record, symbol_override: str | None = None) -> dict | None:
or getattr(record, "publisher_id", None)
or getattr(record, "exchange_id", None)
)
- conditions = getattr(record, "conditions", None) or getattr(record, "condition", None)
- if isinstance(conditions, str):
- conditions = [conditions]
+
+ price = getattr(record, "price", None)
+ size = getattr(record, "size", None)
+ if price is not None and size is not None:
+ conditions = getattr(record, "conditions", None) or getattr(record, "condition", None)
+ if isinstance(conditions, str):
+ conditions = [conditions]
+
+ payload = {
+ "type": "trade",
+ "ts": ts_ms,
+ "price": float(price),
+ "size": int(size),
+ "symbol": stringify(symbol),
+ }
+
+ if exchange is not None:
+ payload["exchange"] = stringify(exchange)
+ if conditions:
+ payload["conditions"] = conditions
+
+ return payload
+
+ bid = _first_attr(record, ["bid_px", "bid_price", "bid"])
+ ask = _first_attr(record, ["ask_px", "ask_price", "ask"])
+ if bid is None or ask is None:
+ return None
+
+ bid_size = _first_attr(record, ["bid_sz", "bid_size", "bid_qty", "bid_q"])
+ ask_size = _first_attr(record, ["ask_sz", "ask_size", "ask_qty", "ask_q"])
payload = {
+ "type": "nbbo",
"ts": ts_ms,
- "price": float(price),
- "size": int(size),
+ "bid": float(bid),
+ "ask": float(ask),
+ "bidSize": int(bid_size) if bid_size is not None else 0,
+ "askSize": int(ask_size) if ask_size is not None else 0,
"symbol": stringify(symbol),
}
if exchange is not None:
payload["exchange"] = stringify(exchange)
- if conditions:
- payload["conditions"] = conditions
return payload
@@ -239,7 +282,10 @@ def main() -> int:
date = dt.datetime.utcfromtimestamp(ts_ms / 1000).date()
if is_numeric_symbol(symbol):
- instrument_id = int(symbol)
+ instrument_id = _to_int(symbol, default=-1)
+ if instrument_id < 0:
+ return
+
mapped = resolver.lookup(instrument_id, date)
if mapped:
emit_payload(build_payload(record, mapped))
diff --git a/services/ingest-options/src/adapters/databento.ts b/services/ingest-options/src/adapters/databento.ts
index d1f5925..6f174d8 100644
--- a/services/ingest-options/src/adapters/databento.ts
+++ b/services/ingest-options/src/adapters/databento.ts
@@ -5,6 +5,7 @@ type DatabentoOptionsAdapterConfig = {
apiKey: string;
dataset: string;
schema: string;
+ nbboSchema: string;
start: string;
end?: string;
symbols: string;
@@ -16,6 +17,7 @@ type DatabentoOptionsAdapterConfig = {
};
type DatabentoTradeMessage = {
+ type: "trade";
ts: number;
price: number;
size: number;
@@ -24,6 +26,19 @@ type DatabentoTradeMessage = {
conditions?: string[] | string;
};
+type DatabentoNbboMessage = {
+ type: "nbbo";
+ ts: number;
+ bid: number;
+ ask: number;
+ bidSize?: number;
+ askSize?: number;
+ symbol: string;
+ exchange?: string;
+};
+
+type DatabentoReplayMessage = DatabentoTradeMessage | DatabentoNbboMessage;
+
type OptionContract = {
root: string;
expiry: string;
@@ -139,45 +154,39 @@ export const createDatabentoOptionsAdapter = (
}
const scriptPath = new URL("../../py/databento_replay.py", import.meta.url).pathname;
- const args = [
- config.pythonBin,
- scriptPath,
- "--dataset",
- config.dataset,
- "--schema",
- config.schema,
- "--start",
- config.start,
- "--symbols",
- config.symbols,
- "--stype-in",
- config.stypeIn,
- "--stype-out",
- config.stypeOut
- ];
- if (config.end) {
- args.push("--end", config.end);
- }
+ const buildArgs = (schema: string): string[] => {
+ const args = [
+ config.pythonBin,
+ scriptPath,
+ "--dataset",
+ config.dataset,
+ "--schema",
+ schema,
+ "--start",
+ config.start,
+ "--symbols",
+ config.symbols,
+ "--stype-in",
+ config.stypeIn,
+ "--stype-out",
+ config.stypeOut
+ ];
- if (config.limit > 0) {
- args.push("--limit", String(config.limit));
- }
-
- const child = Bun.spawn(args, {
- stdout: "pipe",
- stderr: "inherit",
- env: {
- ...Bun.env,
- DATABENTO_API_KEY: config.apiKey
+ if (config.end) {
+ args.push("--end", config.end);
}
- });
- if (!child.stdout) {
- throw new Error("Databento adapter failed to attach stdout.");
- }
+ if (config.limit > 0) {
+ args.push("--limit", String(config.limit));
+ }
- let seq = 0;
+ return args;
+ };
+
+ const children: Bun.Subprocess[] = [];
+ let tradeSeq = 0;
+ let nbboSeq = 0;
const contractIdCache = new Map();
const warnedSymbols = new Set();
@@ -201,55 +210,122 @@ export const createDatabentoOptionsAdapter = (
const handleLine = (line: string) => {
try {
- const payload = JSON.parse(line) as DatabentoTradeMessage;
- if (!payload) {
+ const payload = JSON.parse(line) as DatabentoReplayMessage;
+ if (!payload || typeof payload !== "object") {
return;
}
- const price = Number(payload.price);
- const size = Number(payload.size);
- if (!Number.isFinite(price) || !Number.isFinite(size)) {
- return;
- }
-
- const symbol = String(payload.symbol ?? "").trim();
+ const symbol = String((payload as { symbol?: unknown }).symbol ?? "").trim();
if (!symbol) {
return;
}
- const sourceTs = normalizeTimestamp(Number(payload.ts));
+ const sourceTs = normalizeTimestamp(Number((payload as { ts?: unknown }).ts));
+ if (!Number.isFinite(sourceTs)) {
+ return;
+ }
+
const ingestTs = Date.now();
- seq += 1;
+ const contractId = resolveContractId(symbol);
- const scaledPrice = config.priceScale === 1 ? price : price / config.priceScale;
+ if (payload.type === "trade") {
+ const price = Number(payload.price);
+ const size = Number(payload.size);
+ if (!Number.isFinite(price) || !Number.isFinite(size)) {
+ return;
+ }
- const conditions = Array.isArray(payload.conditions)
- ? payload.conditions.map((entry) => String(entry))
- : typeof payload.conditions === "string"
- ? [payload.conditions]
- : undefined;
+ const scaledPrice =
+ config.priceScale === 1 ? price : price / config.priceScale;
- void handlers.onTrade({
- source_ts: sourceTs,
- ingest_ts: ingestTs,
- seq,
- trace_id: `databento-${seq}`,
- ts: sourceTs,
- option_contract_id: resolveContractId(symbol),
- price: scaledPrice,
- size,
- exchange: payload.exchange ? String(payload.exchange) : "OPRA",
- conditions
- });
+ const conditions = Array.isArray(payload.conditions)
+ ? payload.conditions.map((entry) => String(entry))
+ : typeof payload.conditions === "string"
+ ? [payload.conditions]
+ : undefined;
+
+ tradeSeq += 1;
+ void handlers.onTrade({
+ source_ts: sourceTs,
+ ingest_ts: ingestTs,
+ seq: tradeSeq,
+ trace_id: `databento-${tradeSeq}`,
+ ts: sourceTs,
+ option_contract_id: contractId,
+ price: scaledPrice,
+ size,
+ exchange: payload.exchange ? String(payload.exchange) : "OPRA",
+ conditions
+ });
+ return;
+ }
+
+ if (payload.type === "nbbo") {
+ if (!handlers.onNBBO) {
+ return;
+ }
+
+ const bid = Number(payload.bid);
+ const ask = Number(payload.ask);
+ if (!Number.isFinite(bid) || !Number.isFinite(ask)) {
+ return;
+ }
+
+ const scaledBid = config.priceScale === 1 ? bid : bid / config.priceScale;
+ const scaledAsk = config.priceScale === 1 ? ask : ask / config.priceScale;
+
+ const bidSize = Math.max(0, Math.floor(Number(payload.bidSize ?? 0)));
+ const askSize = Math.max(0, Math.floor(Number(payload.askSize ?? 0)));
+
+ nbboSeq += 1;
+ void handlers.onNBBO({
+ source_ts: sourceTs,
+ ingest_ts: ingestTs,
+ seq: nbboSeq,
+ trace_id: `databento-${nbboSeq}`,
+ ts: sourceTs,
+ option_contract_id: contractId,
+ bid: scaledBid,
+ ask: scaledAsk,
+ bidSize,
+ askSize
+ });
+ }
} catch {
// Ignore malformed lines to keep replay streaming.
}
};
- void readLines(child.stdout, handleLine);
+ const spawnStream = (schema: string): void => {
+ const trimmed = schema.trim();
+ if (!trimmed) {
+ return;
+ }
+
+ const child = Bun.spawn(buildArgs(trimmed), {
+ stdout: "pipe",
+ stderr: "inherit",
+ env: {
+ ...Bun.env,
+ DATABENTO_API_KEY: config.apiKey
+ }
+ });
+
+ if (!child.stdout) {
+ throw new Error("Databento adapter failed to attach stdout.");
+ }
+
+ children.push(child);
+ void readLines(child.stdout, handleLine);
+ };
+
+ spawnStream(config.schema);
+ spawnStream(config.nbboSchema);
return () => {
- child.kill();
+ for (const child of children) {
+ child.kill();
+ }
};
}
};
diff --git a/services/ingest-options/src/index.ts b/services/ingest-options/src/index.ts
index 3fc61ba..5d51678 100644
--- a/services/ingest-options/src/index.ts
+++ b/services/ingest-options/src/index.ts
@@ -46,6 +46,7 @@ const envSchema = z.object({
DATABENTO_API_KEY: z.string().default(""),
DATABENTO_DATASET: z.string().default("OPRA.PILLAR"),
DATABENTO_SCHEMA: z.string().default("trades"),
+ DATABENTO_NBBO_SCHEMA: z.string().default("tbbo"),
DATABENTO_START: z.string().default(""),
DATABENTO_END: z.string().default(""),
DATABENTO_SYMBOLS: z.string().default("ALL"),
@@ -188,6 +189,7 @@ const selectAdapter = (name: string): OptionIngestAdapter => {
apiKey: env.DATABENTO_API_KEY,
dataset: env.DATABENTO_DATASET,
schema: env.DATABENTO_SCHEMA,
+ nbboSchema: env.DATABENTO_NBBO_SCHEMA,
start: env.DATABENTO_START,
end: env.DATABENTO_END || undefined,
symbols: env.DATABENTO_SYMBOLS,
From 04188851b33689b8beb00b0390c412e05fe9f4aa Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Sun, 11 Jan 2026 14:26:47 -0500
Subject: [PATCH 016/237] Sync replay source for options and NBBO
---
apps/web/app/page.tsx | 75 +++++++++++++++++++++++++++++++++++++++----
1 file changed, 69 insertions(+), 6 deletions(-)
diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx
index ce8fe40..d496531 100644
--- a/apps/web/app/page.tsx
+++ b/apps/web/app/page.tsx
@@ -122,6 +122,29 @@ const extractTracePrefix = (item: T): string | null => {
return inferTracePrefix(traceId);
};
+const extractReplaySource = (item: T): string | null => {
+ const prefix = extractTracePrefix(item);
+ if (!prefix) {
+ return null;
+ }
+
+ const normalized = prefix.toLowerCase();
+ if (normalized.startsWith("synthetic")) {
+ return "synthetic";
+ }
+ if (normalized.startsWith("databento")) {
+ return "databento";
+ }
+ if (normalized.startsWith("alpaca")) {
+ return "alpaca";
+ }
+ if (normalized.startsWith("ibkr")) {
+ return "ibkr";
+ }
+
+ return prefix;
+};
+
type SortableItem = {
ts?: number;
source_ts?: number;
@@ -604,6 +627,8 @@ type TapeConfig = {
onNewItems?: (count: number) => void;
getItemTs?: (item: T) => number;
getReplayKey?: (item: T) => string | null;
+ replaySourceKey?: string | null;
+ onReplaySourceKey?: (key: string | null) => void;
};
const useTape = (
@@ -614,6 +639,8 @@ const useTape = (
const pollMs = config.pollMs ?? 1000;
const getItemTs = config.getItemTs ?? extractSortTs;
const getReplayKey = config.getReplayKey ?? extractTracePrefix;
+ const replaySourceKey = config.replaySourceKey ?? null;
+ const onReplaySourceKey = config.onReplaySourceKey;
const [status, setStatus] = useState("connecting");
const [items, setItems] = useState([]);
const [lastUpdate, setLastUpdate] = useState(null);
@@ -627,6 +654,7 @@ const useTape = (
const replayEndRef = useRef(null);
const replayCompleteRef = useRef(false);
const replaySourceRef = useRef(null);
+ const replaySourceNotifiedRef = useRef(null);
const emptyPollsRef = useRef(0);
const pausedRef = useRef(paused);
const pendingRef = useRef([]);
@@ -690,6 +718,7 @@ const useTape = (
setReplayComplete(false);
replayCompleteRef.current = false;
replaySourceRef.current = null;
+ replaySourceNotifiedRef.current = null;
emptyPollsRef.current = 0;
setDropped(0);
setStatus("connecting");
@@ -697,7 +726,7 @@ const useTape = (
pendingRef.current = [];
pendingCountRef.current = 0;
cancelFlush();
- }, [mode, cancelFlush]);
+ }, [mode, replaySourceKey, cancelFlush]);
useEffect(() => {
if (mode !== "replay" || !latestPath) {
@@ -863,7 +892,12 @@ const useTape = (
const payload = (await response.json()) as ReplayResponse;
let sourcePrefix = replaySourceRef.current;
- if (!sourcePrefix) {
+ if (replaySourceKey) {
+ if (sourcePrefix !== replaySourceKey) {
+ sourcePrefix = replaySourceKey;
+ replaySourceRef.current = replaySourceKey;
+ }
+ } else if (!sourcePrefix) {
const firstWithTrace = payload.data.find((item) => getReplayKey(item));
if (firstWithTrace) {
sourcePrefix = getReplayKey(firstWithTrace);
@@ -871,6 +905,11 @@ const useTape = (
}
}
+ if (onReplaySourceKey && sourcePrefix && replaySourceNotifiedRef.current !== sourcePrefix) {
+ replaySourceNotifiedRef.current = sourcePrefix;
+ onReplaySourceKey(sourcePrefix);
+ }
+
const filtered = sourcePrefix
? payload.data.filter((item) => getReplayKey(item) === sourcePrefix)
: payload.data;
@@ -915,7 +954,7 @@ const useTape = (
await new Promise((resolve) => setTimeout(resolve, 0));
}
- if (hasForeign) {
+ if (!replaySourceKey && hasForeign) {
replayCompleteRef.current = true;
setReplayComplete(true);
setStatus("disconnected");
@@ -943,7 +982,18 @@ const useTape = (
window.clearInterval(interval);
cancelFlush();
};
- }, [mode, replayPath, batchSize, pollMs, scheduleFlush, cancelFlush, getItemTs, getReplayKey]);
+ }, [
+ mode,
+ replayPath,
+ batchSize,
+ pollMs,
+ scheduleFlush,
+ cancelFlush,
+ getItemTs,
+ getReplayKey,
+ replaySourceKey,
+ onReplaySourceKey
+ ]);
return {
status,
@@ -1896,10 +1946,19 @@ const formatFlowMetric = (value: number, suffix?: string): string => {
export default function HomePage() {
const [mode, setMode] = useState("live");
+ const [replaySource, setReplaySource] = useState(null);
const [selectedAlert, setSelectedAlert] = useState(null);
const [selectedDarkEvent, setSelectedDarkEvent] = useState(null);
const [filterInput, setFilterInput] = useState("");
const [chartIntervalMs, setChartIntervalMs] = useState(CANDLE_INTERVALS[0].ms);
+
+ const handleReplaySource = useCallback((value: string | null) => {
+ setReplaySource(value);
+ }, []);
+
+ useEffect(() => {
+ setReplaySource(null);
+ }, [mode]);
const optionsScroll = useListScroll();
const equitiesScroll = useListScroll();
const flowScroll = useListScroll();
@@ -1927,7 +1986,9 @@ export default function HomePage() {
batchSize: mode === "replay" ? 120 : undefined,
pollMs: mode === "replay" ? 200 : undefined,
captureScroll: optionsAnchor.capture,
- onNewItems: optionsScroll.onNewItems
+ onNewItems: optionsScroll.onNewItems,
+ getReplayKey: extractReplaySource,
+ onReplaySourceKey: handleReplaySource
});
const equities = useTape({
@@ -1960,7 +2021,9 @@ export default function HomePage() {
latestPath: "/nbbo/options",
expectedType: "option-nbbo",
batchSize: mode === "replay" ? 120 : undefined,
- pollMs: mode === "replay" ? 200 : undefined
+ pollMs: mode === "replay" ? 200 : undefined,
+ getReplayKey: extractReplaySource,
+ replaySourceKey: replaySource
});
const inferredDark = useTape({
From debbc1046ba4e5d67686d537741eea2ba4033f7f Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Sun, 11 Jan 2026 16:44:34 -0500
Subject: [PATCH 017/237] Add replay source filter for option replay
---
apps/web/app/page.tsx | 9 ++++++-
packages/storage/src/clickhouse.ts | 39 ++++++++++++++++++++++++------
services/api/src/index.ts | 34 +++++++++++++++++++++++---
3 files changed, 69 insertions(+), 13 deletions(-)
diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx
index d496531..b6beebc 100644
--- a/apps/web/app/page.tsx
+++ b/apps/web/app/page.tsx
@@ -743,6 +743,9 @@ const useTape = (
try {
const url = new URL(buildApiUrl(latestPath));
url.searchParams.set("limit", "1");
+ if (replaySourceKey) {
+ url.searchParams.set("source", replaySourceKey);
+ }
const response = await fetch(url.toString());
if (!response.ok) {
throw new Error(`Replay baseline failed with ${response.status}`);
@@ -763,7 +766,7 @@ const useTape = (
return () => {
active = false;
};
- }, [mode, latestPath, getItemTs]);
+ }, [mode, latestPath, getItemTs, replaySourceKey]);
useEffect(() => {
if (mode !== "live") {
@@ -883,6 +886,10 @@ const useTape = (
url.searchParams.set("after_ts", cursor.ts.toString());
url.searchParams.set("after_seq", cursor.seq.toString());
url.searchParams.set("limit", batchSize.toString());
+ const desiredSource = replaySourceKey ?? replaySourceRef.current;
+ if (desiredSource) {
+ url.searchParams.set("source", desiredSource);
+ }
const response = await fetch(url.toString());
if (!response.ok) {
diff --git a/packages/storage/src/clickhouse.ts b/packages/storage/src/clickhouse.ts
index 8ed0aff..9a42b9f 100644
--- a/packages/storage/src/clickhouse.ts
+++ b/packages/storage/src/clickhouse.ts
@@ -331,6 +331,17 @@ const quoteString = (value: string): string => {
return `'${escaped}'`;
};
+const buildTracePrefixCondition = (tracePrefix: string | undefined): string | null => {
+ if (!tracePrefix) {
+ return null;
+ }
+ const normalized = tracePrefix.trim();
+ if (!normalized) {
+ return null;
+ }
+ return `startsWith(trace_id, ${quoteString(normalized)})`;
+};
+
const normalizeNumericFields = (
row: Record,
fields: string[]
@@ -531,11 +542,14 @@ const normalizeAlertRow = (row: unknown): AlertRecord | null => {
export const fetchRecentOptionPrints = async (
client: ClickHouseClient,
- limit: number
+ limit: number,
+ tracePrefix?: string
): Promise => {
const safeLimit = clampLimit(limit);
+ const condition = buildTracePrefixCondition(tracePrefix);
+ const whereClause = condition ? ` WHERE ${condition}` : "";
const result = await client.query({
- query: `SELECT * FROM ${OPTION_PRINTS_TABLE} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
+ query: `SELECT * FROM ${OPTION_PRINTS_TABLE}${whereClause} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
format: "JSONEachRow"
});
@@ -545,11 +559,14 @@ export const fetchRecentOptionPrints = async (
export const fetchRecentOptionNBBO = async (
client: ClickHouseClient,
- limit: number
+ limit: number,
+ tracePrefix?: string
): Promise => {
const safeLimit = clampLimit(limit);
+ const condition = buildTracePrefixCondition(tracePrefix);
+ const whereClause = condition ? ` WHERE ${condition}` : "";
const result = await client.query({
- query: `SELECT * FROM ${OPTION_NBBO_TABLE} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
+ query: `SELECT * FROM ${OPTION_NBBO_TABLE}${whereClause} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
format: "JSONEachRow"
});
@@ -697,14 +714,17 @@ export const fetchOptionPrintsAfter = async (
client: ClickHouseClient,
afterTs: number,
afterSeq: number,
- limit: number
+ limit: number,
+ tracePrefix?: string
): Promise => {
const safeLimit = clampLimit(limit);
const safeAfterTs = clampCursor(afterTs);
const safeAfterSeq = clampCursor(afterSeq);
+ const traceCondition = buildTracePrefixCondition(tracePrefix);
+ const traceClause = traceCondition ? ` AND ${traceCondition}` : "";
const result = await client.query({
- query: `SELECT * FROM ${OPTION_PRINTS_TABLE} WHERE (ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
+ query: `SELECT * FROM ${OPTION_PRINTS_TABLE} WHERE (ts, seq) > (${safeAfterTs}, ${safeAfterSeq})${traceClause} ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
format: "JSONEachRow"
});
@@ -716,14 +736,17 @@ export const fetchOptionNBBOAfter = async (
client: ClickHouseClient,
afterTs: number,
afterSeq: number,
- limit: number
+ limit: number,
+ tracePrefix?: string
): Promise => {
const safeLimit = clampLimit(limit);
const safeAfterTs = clampCursor(afterTs);
const safeAfterSeq = clampCursor(afterSeq);
+ const traceCondition = buildTracePrefixCondition(tracePrefix);
+ const traceClause = traceCondition ? ` AND ${traceCondition}` : "";
const result = await client.query({
- query: `SELECT * FROM ${OPTION_NBBO_TABLE} WHERE (ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
+ query: `SELECT * FROM ${OPTION_NBBO_TABLE} WHERE (ts, seq) > (${safeAfterTs}, ${safeAfterSeq})${traceClause} ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
format: "JSONEachRow"
});
diff --git a/services/api/src/index.ts b/services/api/src/index.ts
index a345aba..d282b86 100644
--- a/services/api/src/index.ts
+++ b/services/api/src/index.ts
@@ -122,6 +122,14 @@ const replayParamsSchema = z.object({
after_seq: z.coerce.number().int().nonnegative().default(0),
limit: z.coerce.number().int().positive().max(1000).default(200)
});
+
+const replaySourceSchema = z
+ .string()
+ .trim()
+ .min(1)
+ .max(64)
+ .regex(/^[A-Za-z0-9][A-Za-z0-9_-]*$/)
+ .transform((value) => value.toLowerCase());
const candleQuerySchema = z.object({
underlying_id: z.string().min(1),
interval_ms: z.coerce.number().int().positive(),
@@ -193,6 +201,20 @@ const parseReplayParams = (url: URL): { afterTs: number; afterSeq: number; limit
};
};
+const parseReplaySource = (url: URL): string | null => {
+ const raw = url.searchParams.get("source");
+ if (!raw) {
+ return null;
+ }
+
+ const trimmed = raw.trim();
+ if (!trimmed) {
+ return null;
+ }
+
+ return replaySourceSchema.parse(trimmed);
+};
+
const parseBooleanParam = (value: string | null | undefined): boolean => {
if (!value) {
return false;
@@ -756,13 +778,15 @@ const run = async () => {
if (req.method === "GET" && url.pathname === "/prints/options") {
const limit = parseLimit(url.searchParams.get("limit"));
- const data = await fetchRecentOptionPrints(clickhouse, limit);
+ const source = parseReplaySource(url) ?? undefined;
+ const data = await fetchRecentOptionPrints(clickhouse, limit, source);
return jsonResponse({ data });
}
if (req.method === "GET" && url.pathname === "/nbbo/options") {
const limit = parseLimit(url.searchParams.get("limit"));
- const data = await fetchRecentOptionNBBO(clickhouse, limit);
+ const source = parseReplaySource(url) ?? undefined;
+ const data = await fetchRecentOptionNBBO(clickhouse, limit, source);
return jsonResponse({ data });
}
@@ -847,7 +871,8 @@ const run = async () => {
if (req.method === "GET" && url.pathname === "/replay/options") {
const { afterTs, afterSeq, limit } = parseReplayParams(url);
- const data = await fetchOptionPrintsAfter(clickhouse, afterTs, afterSeq, limit);
+ const source = parseReplaySource(url) ?? undefined;
+ const data = await fetchOptionPrintsAfter(clickhouse, afterTs, afterSeq, limit, source);
const last = data.at(-1);
const next = last ? { ts: last.ts, seq: last.seq } : null;
return jsonResponse({ data, next });
@@ -855,7 +880,8 @@ const run = async () => {
if (req.method === "GET" && url.pathname === "/replay/nbbo") {
const { afterTs, afterSeq, limit } = parseReplayParams(url);
- const data = await fetchOptionNBBOAfter(clickhouse, afterTs, afterSeq, limit);
+ const source = parseReplaySource(url) ?? undefined;
+ const data = await fetchOptionNBBOAfter(clickhouse, afterTs, afterSeq, limit, source);
const last = data.at(-1);
const next = last ? { ts: last.ts, seq: last.seq } : null;
return jsonResponse({ data, next });
From 1175dd00cc6f4808a127e95343b8ab3220aa92d6 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Tue, 20 Jan 2026 10:43:36 -0500
Subject: [PATCH 018/237] Add Alpaca equities ingest adapter
Adds an Alpaca-backed equities adapter with exchange metadata mapping for conservative off-exchange tagging and a small helper test suite.
---
services/ingest-equities/package.json | 1 +
.../ingest-equities/src/adapters/alpaca.ts | 335 ++++++++++++++++++
services/ingest-equities/src/index.ts | 28 ++
services/ingest-equities/tests/alpaca.test.ts | 29 ++
4 files changed, 393 insertions(+)
create mode 100644 services/ingest-equities/src/adapters/alpaca.ts
create mode 100644 services/ingest-equities/tests/alpaca.test.ts
diff --git a/services/ingest-equities/package.json b/services/ingest-equities/package.json
index 5452f2f..bf85916 100644
--- a/services/ingest-equities/package.json
+++ b/services/ingest-equities/package.json
@@ -11,6 +11,7 @@
"@islandflow/observability": "workspace:*",
"@islandflow/storage": "workspace:*",
"@islandflow/types": "workspace:*",
+ "ws": "^8.18.3",
"zod": "^3.23.8"
}
}
diff --git a/services/ingest-equities/src/adapters/alpaca.ts b/services/ingest-equities/src/adapters/alpaca.ts
new file mode 100644
index 0000000..97b7205
--- /dev/null
+++ b/services/ingest-equities/src/adapters/alpaca.ts
@@ -0,0 +1,335 @@
+import { createLogger } from "@islandflow/observability";
+import type { EquityPrint, EquityQuote } from "@islandflow/types";
+import type { EquityIngestAdapter, EquityIngestHandlers } from "./types";
+import WebSocket from "ws";
+
+export type AlpacaEquitiesFeed = "iex" | "sip";
+
+export type AlpacaEquitiesAdapterConfig = {
+ keyId: string;
+ secretKey: string;
+ restUrl: string;
+ wsBaseUrl: string;
+ feed: AlpacaEquitiesFeed;
+ symbols: string[];
+};
+
+type AlpacaExchangeMetaEntry = {
+ code: string;
+ name: string;
+};
+
+type AlpacaTradeMessage = {
+ T: "t";
+ S: string;
+ t: string;
+ p: number;
+ s: number;
+ x?: string;
+ c?: string[];
+ z?: string;
+};
+
+type AlpacaQuoteMessage = {
+ T: "q";
+ S: string;
+ t: string;
+ bp: number;
+ ap: number;
+ bs?: number;
+ as?: number;
+ bx?: string;
+ ax?: string;
+ c?: string[];
+ z?: string;
+};
+
+const logger = createLogger({ service: "ingest-equities" });
+
+const normalizeSymbols = (symbols: string[]): string[] => {
+ const seen = new Set();
+ const result: string[] = [];
+
+ for (const entry of symbols) {
+ const symbol = entry.trim().toUpperCase();
+ if (!symbol || seen.has(symbol)) {
+ continue;
+ }
+
+ seen.add(symbol);
+ result.push(symbol);
+ }
+
+ return result;
+};
+
+const buildHeaders = (config: AlpacaEquitiesAdapterConfig): Record => ({
+ "APCA-API-KEY-ID": config.keyId,
+ "APCA-API-SECRET-KEY": config.secretKey
+});
+
+const parseTimestamp = (value: string): number => {
+ const parsed = Date.parse(value);
+ if (Number.isFinite(parsed)) {
+ return parsed;
+ }
+ return Date.now();
+};
+
+const decodePayload = (data: WebSocket.RawData): unknown => {
+ if (typeof data === "string") {
+ return JSON.parse(data) as unknown;
+ }
+
+ if (data instanceof ArrayBuffer) {
+ return JSON.parse(new TextDecoder().decode(new Uint8Array(data))) as unknown;
+ }
+
+ if (ArrayBuffer.isView(data)) {
+ return JSON.parse(new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))) as unknown;
+ }
+
+ return JSON.parse(new TextDecoder().decode(new Uint8Array(data as ArrayBuffer))) as unknown;
+};
+
+const extractExchangeMeta = (payload: unknown): AlpacaExchangeMetaEntry[] => {
+ if (!Array.isArray(payload)) {
+ return [];
+ }
+
+ const result: AlpacaExchangeMetaEntry[] = [];
+
+ for (const entry of payload) {
+ if (!entry || typeof entry !== "object") {
+ continue;
+ }
+ const candidate = entry as Record;
+ const code = typeof candidate.code === "string" ? candidate.code : typeof candidate.exchange === "string" ? candidate.exchange : null;
+ const name = typeof candidate.name === "string" ? candidate.name : typeof candidate.description === "string" ? candidate.description : null;
+ if (!code || !name) {
+ continue;
+ }
+
+ result.push({ code, name });
+ }
+
+ return result;
+};
+
+const buildExchangeNameMap = (entries: AlpacaExchangeMetaEntry[]): Map => {
+ const map = new Map();
+ for (const entry of entries) {
+ const code = entry.code.trim();
+ const name = entry.name.trim();
+ if (!code || !name) {
+ continue;
+ }
+ map.set(code, name);
+ }
+ return map;
+};
+
+const OFF_EXCHANGE_HINTS = ["FINRA", "TRF", "ADF", "OTC", "Trade Reporting Facility", "Alternative Display Facility"];
+
+export const inferOffExchangeFlag = (exchangeCode: string | undefined, exchangeNameMap: Map): boolean => {
+ if (!exchangeCode) {
+ return false;
+ }
+
+ const name = exchangeNameMap.get(exchangeCode) ?? "";
+ const normalized = name.toUpperCase();
+
+ if (normalized) {
+ return OFF_EXCHANGE_HINTS.some((hint) => normalized.includes(hint.toUpperCase()));
+ }
+
+ // Conservative fallback: only tag the most common FINRA code when no mapping is available.
+ return exchangeCode.toUpperCase() === "D";
+};
+
+const buildWsUrl = (wsBaseUrl: string, feed: AlpacaEquitiesFeed): string => {
+ const parsed = new URL(wsBaseUrl);
+ return `${parsed.origin}/v2/${feed}`;
+};
+
+const fetchExchangeMeta = async (config: AlpacaEquitiesAdapterConfig): Promise> => {
+ const url = new URL("/v2/stocks/meta/exchanges", config.restUrl);
+
+ try {
+ const response = await fetch(url.toString(), {
+ headers: buildHeaders(config)
+ });
+
+ if (!response.ok) {
+ logger.warn("alpaca exchange meta request failed", {
+ status: response.status
+ });
+ return new Map();
+ }
+
+ const payload = (await response.json()) as unknown;
+ const entries = extractExchangeMeta(payload);
+ return buildExchangeNameMap(entries);
+ } catch (error) {
+ logger.warn("alpaca exchange meta request error", {
+ error: error instanceof Error ? error.message : String(error)
+ });
+ return new Map();
+ }
+};
+
+export const createAlpacaEquitiesAdapter = (
+ config: AlpacaEquitiesAdapterConfig
+): EquityIngestAdapter => {
+ return {
+ name: "alpaca",
+ start: async (handlers: EquityIngestHandlers) => {
+ if (!config.keyId || !config.secretKey) {
+ throw new Error("Alpaca equities adapter requires ALPACA_KEY_ID and ALPACA_SECRET_KEY.");
+ }
+
+ const symbols = normalizeSymbols(config.symbols);
+ if (symbols.length === 0) {
+ throw new Error("Alpaca equities adapter requires at least one symbol.");
+ }
+
+ const exchangeNameMap = await fetchExchangeMeta(config);
+ const wsUrl = buildWsUrl(config.wsBaseUrl, config.feed);
+ const ws = new WebSocket(wsUrl);
+
+ let seq = 0;
+ let stopped = false;
+ let authenticated = false;
+
+ ws.on("open", () => {
+ ws.send(
+ JSON.stringify({
+ action: "auth",
+ key: config.keyId,
+ secret: config.secretKey
+ })
+ );
+ });
+
+ const subscribe = () => {
+ const message: Record = {
+ action: "subscribe",
+ trades: symbols
+ };
+
+ if (handlers.onQuote) {
+ message.quotes = symbols;
+ }
+
+ ws.send(JSON.stringify(message));
+ };
+
+ ws.on("message", (data) => {
+ if (stopped) {
+ return;
+ }
+
+ let payload: unknown;
+ try {
+ payload = decodePayload(data);
+ } catch (error) {
+ logger.warn("alpaca equities message decode failed", {
+ error: error instanceof Error ? error.message : String(error)
+ });
+ return;
+ }
+
+ if (!Array.isArray(payload)) {
+ return;
+ }
+
+ for (const entry of payload) {
+ if (!entry || typeof entry !== "object") {
+ continue;
+ }
+
+ const message = entry as (AlpacaTradeMessage | AlpacaQuoteMessage | { T?: string; msg?: string });
+ const type = message.T;
+
+ if (type === "success") {
+ const msg = (message as { msg?: string }).msg ?? "";
+ if (msg === "authenticated") {
+ authenticated = true;
+ subscribe();
+ }
+ continue;
+ }
+
+ if (type === "subscription") {
+ continue;
+ }
+
+ if (type === "error") {
+ logger.error("alpaca equities stream error", { message });
+ continue;
+ }
+
+ if (type === "t") {
+ const trade = message as AlpacaTradeMessage;
+ const sourceTs = parseTimestamp(trade.t);
+ seq += 1;
+ const exchangeCode = trade.x ?? "";
+
+ void handlers.onTrade({
+ source_ts: sourceTs,
+ ingest_ts: Date.now(),
+ seq,
+ trace_id: `alpaca-equities-${seq}`,
+ ts: sourceTs,
+ underlying_id: trade.S,
+ price: trade.p,
+ size: trade.s,
+ exchange: exchangeCode || "ALPACA",
+ offExchangeFlag: inferOffExchangeFlag(exchangeCode, exchangeNameMap)
+ } satisfies EquityPrint);
+
+ continue;
+ }
+
+ if (type === "q" && handlers.onQuote) {
+ const quote = message as AlpacaQuoteMessage;
+ const sourceTs = parseTimestamp(quote.t);
+ seq += 1;
+
+ void handlers.onQuote({
+ source_ts: sourceTs,
+ ingest_ts: Date.now(),
+ seq,
+ trace_id: `alpaca-equity-quote-${seq}`,
+ ts: sourceTs,
+ underlying_id: quote.S,
+ bid: quote.bp,
+ ask: quote.ap
+ } satisfies EquityQuote);
+
+ continue;
+ }
+ }
+ });
+
+ ws.on("error", (error) => {
+ logger.error("alpaca equities websocket error", {
+ error: error instanceof Error ? error.message : String(error)
+ });
+ });
+
+ ws.on("close", (code, reason) => {
+ logger.warn("alpaca equities websocket closed", {
+ code,
+ reason: reason.toString(),
+ authenticated
+ });
+ });
+
+ return () => {
+ stopped = true;
+ ws.close();
+ };
+ }
+ };
+};
diff --git a/services/ingest-equities/src/index.ts b/services/ingest-equities/src/index.ts
index 1572aa2..3644e7a 100644
--- a/services/ingest-equities/src/index.ts
+++ b/services/ingest-equities/src/index.ts
@@ -22,6 +22,7 @@ import {
type EquityPrint,
type EquityQuote
} from "@islandflow/types";
+import { createAlpacaEquitiesAdapter } from "./adapters/alpaca";
import { createSyntheticEquitiesAdapter } from "./adapters/synthetic";
import type { EquityIngestAdapter, StopHandler } from "./adapters/types";
import { z } from "zod";
@@ -35,6 +36,15 @@ const envSchema = z.object({
CLICKHOUSE_DATABASE: z.string().default("default"),
EQUITIES_INGEST_ADAPTER: z.string().min(1).default("synthetic"),
EMIT_INTERVAL_MS: z.coerce.number().int().positive().default(1000),
+
+ // Alpaca (equities)
+ ALPACA_KEY_ID: z.string().default(""),
+ ALPACA_SECRET_KEY: z.string().default(""),
+ ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"),
+ ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"),
+ ALPACA_UNDERLYINGS: z.string().default("SPY,NVDA,AAPL"),
+ ALPACA_EQUITIES_FEED: z.enum(["iex", "sip"]).default("iex"),
+
TESTING_MODE: z
.preprocess((value) => {
if (typeof value === "string") {
@@ -113,11 +123,29 @@ const retry = async (
throw lastError ?? new Error(`${label} failed after retries`);
};
+const parseSymbolList = (value: string): string[] => {
+ return value
+ .split(",")
+ .map((entry) => entry.trim())
+ .filter(Boolean);
+};
+
const selectAdapter = (name: string): EquityIngestAdapter => {
if (name === "synthetic") {
return createSyntheticEquitiesAdapter({ emitIntervalMs: env.EMIT_INTERVAL_MS });
}
+ if (name === "alpaca") {
+ return createAlpacaEquitiesAdapter({
+ keyId: env.ALPACA_KEY_ID,
+ secretKey: env.ALPACA_SECRET_KEY,
+ restUrl: env.ALPACA_REST_URL,
+ wsBaseUrl: env.ALPACA_WS_BASE_URL,
+ feed: env.ALPACA_EQUITIES_FEED,
+ symbols: parseSymbolList(env.ALPACA_UNDERLYINGS)
+ });
+ }
+
throw new Error(`Unknown ingest adapter: ${name}`);
};
diff --git a/services/ingest-equities/tests/alpaca.test.ts b/services/ingest-equities/tests/alpaca.test.ts
new file mode 100644
index 0000000..c7422b0
--- /dev/null
+++ b/services/ingest-equities/tests/alpaca.test.ts
@@ -0,0 +1,29 @@
+import { describe, expect, test } from "bun:test";
+import { inferOffExchangeFlag } from "../src/adapters/alpaca";
+
+describe("alpaca equities adapter helpers", () => {
+ test("inferOffExchangeFlag tags FINRA/TRF venues as off-exchange", () => {
+ const map = new Map([
+ ["D", "FINRA / Nasdaq TRF"],
+ ["N", "FINRA / NYSE TRF"],
+ ["Q", "NASDAQ"],
+ ["P", "NYSE ARCA"],
+ ["O", "OTC Markets"]
+ ]);
+
+ expect(inferOffExchangeFlag("D", map)).toBe(true);
+ expect(inferOffExchangeFlag("N", map)).toBe(true);
+ expect(inferOffExchangeFlag("O", map)).toBe(true);
+ expect(inferOffExchangeFlag("Q", map)).toBe(false);
+ expect(inferOffExchangeFlag("P", map)).toBe(false);
+ });
+
+ test("inferOffExchangeFlag falls back conservatively when no mapping", () => {
+ const empty = new Map();
+
+ expect(inferOffExchangeFlag(undefined, empty)).toBe(false);
+ expect(inferOffExchangeFlag("", empty)).toBe(false);
+ expect(inferOffExchangeFlag("D", empty)).toBe(true);
+ expect(inferOffExchangeFlag("N", empty)).toBe(false);
+ });
+});
From 0e1b9666b48cef92d145a87cb5446552ed52bb5f Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Tue, 20 Jan 2026 10:43:38 -0500
Subject: [PATCH 019/237] Document Alpaca equities adapter env
Updates the env example and README to cover the Alpaca equities feed selection and adapter availability.
---
.env.example | 2 ++
README.md | 9 ++++++---
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/.env.example b/.env.example
index ae5499b..0c7385a 100644
--- a/.env.example
+++ b/.env.example
@@ -45,6 +45,8 @@ IBKR_PYTHON_BIN=python3
# Equities ingest
EQUITIES_INGEST_ADAPTER=synthetic
EMIT_INTERVAL_MS=1000
+# When using EQUITIES_INGEST_ADAPTER=alpaca, choose "iex" (free) or "sip" (paid).
+ALPACA_EQUITIES_FEED=iex
# Testing mode
TESTING_MODE=false
diff --git a/README.md b/README.md
index 3468c9c..724436d 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,7 @@ Done now (in repo):
- UI: alerts + classifier hits panels, ticker filter, evidence drawer, severity strip
- Databento historical replay adapter (options) with symbol mapping
- Alpaca options adapter (dev-only, bounded contract list)
+- Alpaca equities adapter (stocks trades/quotes via WS)
- IBKR options adapter (single-underlying bridge via `ib_insync`)
- Dark-pool-style inference (absorbed blocks, stealth accumulation, distribution) with WS/REST surfaces and UI list
- Testing-mode throttling for ingest to reduce CPU during local dev
@@ -29,7 +30,7 @@ Done now (in repo):
In progress / blocked:
- Production-grade licensed live data feeds (beyond current dev/test bridges)
- Advanced clustering (spreads/rolls beyond basic structure tags)
-- Candles/overlays service (scaffolded, not yet emitting data)
+- Chart overlays beyond basic candles (candles service emits data; UI overlays still limited)
Not started:
- Reference data/corporate action enrichment
@@ -45,7 +46,7 @@ Not started:
## Current Capabilities
- Synthetic options/equity prints with deterministic sequencing across the S&P 500
-- Ingest adapter seam (env-selected; options default `synthetic`, equities default `synthetic`)
+- Ingest adapter seam (env-selected; options default `synthetic`, equities: `synthetic` or `alpaca`)
- Raw event persistence in ClickHouse + streaming via NATS JetStream
- Deterministic option FlowPacket clustering (time-window)
- Rolling stats baselines in Redis with z-score features on FlowPackets
@@ -53,6 +54,7 @@ Not started:
- Aggressor mix features from NBBO placement on FlowPackets
- Classifiers + alert scoring (rule-first) with WS/REST endpoints
- API gateway with REST, WS, and replay endpoints
+- Server-built equity candles (service + REST/WS surfaces)
- UI tapes for options/equities/flow packets + alerts/hits with live/replay toggle and pause controls
- Alpaca options adapter (dev-only) with bounded contract selection
- IBKR options adapter (single-underlying bridge via Python sidecar)
@@ -126,7 +128,7 @@ These define how services connect to the event bus and storage backends. Documen
### Adapter selection
- `OPTIONS_INGEST_ADAPTER` (default `synthetic`) — options ingest adapter: `synthetic`, `alpaca`, `ibkr`, `databento`.
-- `EQUITIES_INGEST_ADAPTER` (default `synthetic`) — equities ingest adapter.
+- `EQUITIES_INGEST_ADAPTER` (default `synthetic`) — equities ingest adapter: `synthetic`, `alpaca`.
- `EMIT_INTERVAL_MS` (default `1000`) — synthetic equities emit cadence.
### Alpaca options adapter (dev-only)
@@ -137,6 +139,7 @@ Provider links: [Alpaca](https://alpaca.markets/), [Alpaca Market Data API](http
- `ALPACA_REST_URL` (default `https://data.alpaca.markets`) — REST endpoint.
- `ALPACA_WS_BASE_URL` (default `wss://stream.data.alpaca.markets/v1beta1`) — streaming endpoint.
- `ALPACA_FEED` (default `indicative`) — use `opra` when you have a subscription.
+- `ALPACA_EQUITIES_FEED` (default `iex`) — equities feed: `iex` (free) or `sip` (paid).
- `ALPACA_UNDERLYINGS` (default `SPY,NVDA,AAPL`) — comma-separated list of symbols.
- `ALPACA_STRIKES_PER_SIDE` (default `8`) — strikes per side around ATM.
- `ALPACA_MAX_DTE_DAYS` (default `30`) — expiry horizon.
From 6503f9380aa3afa97c1aec59e2f0fe9b16501cd8 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Tue, 20 Jan 2026 10:43:41 -0500
Subject: [PATCH 020/237] Ignore local assistant artifacts
Adds patterns for local session/token usage artifacts so they don't show up as untracked changes.
---
.gitignore | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/.gitignore b/.gitignore
index 0ac6b0d..e603126 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,7 @@ coverage/
logs/
.tmp/
apps/web/.next/
+
+# Local assistant artifacts
+session-ses_*.md
+token-usage-output.txt
From 7fc9f5cf667e2f923fe69c5b7d3e9fae8c64d7b7 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Tue, 20 Jan 2026 11:05:28 -0500
Subject: [PATCH 021/237] Add equity prints range query
Adds a ClickHouse helper to fetch equity prints for a single underlying over a bounded time range with stable ts/seq ordering.
---
packages/storage/src/clickhouse.ts | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/packages/storage/src/clickhouse.ts b/packages/storage/src/clickhouse.ts
index 9a42b9f..730bdaf 100644
--- a/packages/storage/src/clickhouse.ts
+++ b/packages/storage/src/clickhouse.ts
@@ -773,6 +773,30 @@ export const fetchEquityPrintsAfter = async (
return EquityPrintSchema.array().parse(rows.map(normalizeEquityRow));
};
+export const fetchEquityPrintsRange = async (
+ client: ClickHouseClient,
+ underlyingId: string,
+ startTs: number,
+ endTs: number,
+ limit: number
+): Promise => {
+ const safeLimit = clampLimit(limit);
+ const safeStart = clampCursor(startTs);
+ const safeEnd = clampCursor(endTs);
+ const rangeStart = Math.min(safeStart, safeEnd);
+ const rangeEnd = Math.max(safeStart, safeEnd);
+ const safeUnderlying = quoteString(underlyingId);
+
+ const result = await client.query({
+ query: `SELECT * FROM ${EQUITY_PRINTS_TABLE} WHERE underlying_id = ${safeUnderlying} AND ts >= ${rangeStart} AND ts <= ${rangeEnd} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
+ format: "JSONEachRow"
+ });
+
+ const rows = await result.json();
+ const parsed = EquityPrintSchema.array().parse(rows.map(normalizeEquityRow));
+ return parsed.reverse();
+};
+
export const fetchEquityQuotesAfter = async (
client: ClickHouseClient,
afterTs: number,
From 52f7ad82c68bda2d7f9f531ae013d92a958d6ff4 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Tue, 20 Jan 2026 11:05:40 -0500
Subject: [PATCH 022/237] Add equities prints range endpoint
Adds GET /prints/equities/range for fetching underlying-scoped equity prints over a requested time window, used by chart overlays.
---
services/api/src/index.ts | 42 +++++++++++++++++++++++++++++++++++++++
1 file changed, 42 insertions(+)
diff --git a/services/api/src/index.ts b/services/api/src/index.ts
index d282b86..54c7ae3 100644
--- a/services/api/src/index.ts
+++ b/services/api/src/index.ts
@@ -51,6 +51,7 @@ import {
fetchEquityCandlesRange,
fetchRecentOptionNBBO,
fetchEquityPrintsAfter,
+ fetchEquityPrintsRange,
fetchEquityPrintJoinsAfter,
fetchEquityQuotesAfter,
fetchInferredDarkAfter,
@@ -143,6 +144,13 @@ const candleReplaySchema = replayParamsSchema.extend({
interval_ms: z.coerce.number().int().positive()
});
+const equityPrintRangeSchema = z.object({
+ underlying_id: z.string().min(1),
+ start_ts: z.coerce.number().int().nonnegative(),
+ end_ts: z.coerce.number().int().nonnegative(),
+ limit: limitSchema.optional()
+});
+
type Channel =
| "options"
| "options-nbbo"
@@ -223,6 +231,24 @@ const parseBooleanParam = (value: string | null | undefined): boolean => {
return ["1", "true", "yes", "on"].includes(normalized);
};
+const parseEquityPrintRangeParams = (
+ url: URL
+): { underlyingId: string; startTs: number; endTs: number; limit: number } => {
+ const params = equityPrintRangeSchema.parse({
+ underlying_id: url.searchParams.get("underlying_id") ?? undefined,
+ start_ts: url.searchParams.get("start_ts") ?? undefined,
+ end_ts: url.searchParams.get("end_ts") ?? undefined,
+ limit: url.searchParams.get("limit") ?? undefined
+ });
+
+ return {
+ underlyingId: params.underlying_id,
+ startTs: params.start_ts,
+ endTs: params.end_ts,
+ limit: params.limit ?? env.REST_DEFAULT_LIMIT
+ };
+};
+
const parseCandleParams = (
url: URL
): {
@@ -796,6 +822,22 @@ const run = async () => {
return jsonResponse({ data });
}
+ if (req.method === "GET" && url.pathname === "/prints/equities/range") {
+ try {
+ const { underlyingId, startTs, endTs, limit } = parseEquityPrintRangeParams(url);
+ const data = await fetchEquityPrintsRange(clickhouse, underlyingId, startTs, endTs, limit);
+ return jsonResponse({ data });
+ } catch (error) {
+ return jsonResponse(
+ {
+ error: "invalid equity range query",
+ detail: error instanceof Error ? error.message : String(error)
+ },
+ 400
+ );
+ }
+ }
+
if (req.method === "GET" && url.pathname === "/quotes/equities") {
const limit = parseLimit(url.searchParams.get("limit"));
const data = await fetchRecentEquityQuotes(clickhouse, limit);
From 4dd7b038100763e6e34bd230998214d252501ed8 Mon Sep 17 00:00:00 2001
From: dirtydishes
Date: Tue, 20 Jan 2026 11:05:49 -0500
Subject: [PATCH 023/237] Overlay off-exchange equity prints on chart
Adds a canvas overlay to the equity candle chart to render off-exchange prints with a toggle/legend, fetching viewport-bounded data and updating live via the equities websocket.
---
apps/web/app/globals.css | 33 ++++
apps/web/app/page.tsx | 372 ++++++++++++++++++++++++++++++++++++++-
2 files changed, 402 insertions(+), 3 deletions(-)
diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css
index 5d46329..2da9583 100644
--- a/apps/web/app/globals.css
+++ b/apps/web/app/globals.css
@@ -336,6 +336,39 @@ h1 {
color: #5b4c34;
}
+.overlay-toggle {
+ border: 1px solid rgba(31, 74, 123, 0.35);
+ border-radius: 999px;
+ padding: 6px 12px;
+ background: rgba(31, 74, 123, 0.12);
+ color: #1f4a7b;
+ font-size: 0.7rem;
+ letter-spacing: 0.12em;
+ text-transform: uppercase;
+ cursor: pointer;
+}
+
+.overlay-toggle.overlay-toggle-on {
+ border-color: rgba(31, 74, 123, 0.6);
+ background: rgba(31, 74, 123, 0.2);
+}
+
+.overlay-toggle:focus-visible {
+ outline: 2px solid rgba(31, 74, 123, 0.4);
+ outline-offset: 2px;
+}
+
+.overlay-legend {
+ color: #6f5b39;
+ font-size: 0.75rem;
+}
+
+@media (max-width: 700px) {
+ .overlay-legend {
+ flex: 1 1 100%;
+ }
+}
+
.chart-status {
display: inline-flex;
align-items: center;
diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx
index b6beebc..d32e8bc 100644
--- a/apps/web/app/page.tsx
+++ b/apps/web/app/page.tsx
@@ -26,6 +26,13 @@ const CANDLE_INTERVALS = [
type CandlestickSeries = ReturnType;
+type EquityOverlayPoint = {
+ ts: number;
+ price: number;
+ size: number;
+ offExchangeFlag: boolean;
+};
+
type ChartCandle = {
time: UTCTimestamp;
open: number;
@@ -52,6 +59,37 @@ const toChartTime = (ts: number): UTCTimestamp => {
return Math.floor(ts / 1000) as UTCTimestamp;
};
+type ChartTimeLike = number | string | { year: number; month: number; day: number };
+
+const chartTimeToMs = (value: ChartTimeLike): number | null => {
+ if (typeof value === "number") {
+ return Math.floor(value * 1000);
+ }
+
+ if (typeof value === "string") {
+ const parsed = Date.parse(value);
+ return Number.isFinite(parsed) ? parsed : null;
+ }
+
+ if (value && typeof value === "object") {
+ const { year, month, day } = value;
+ if (
+ Number.isFinite(year) &&
+ Number.isFinite(month) &&
+ Number.isFinite(day) &&
+ year >= 1970 &&
+ month >= 1 &&
+ month <= 12 &&
+ day >= 1 &&
+ day <= 31
+ ) {
+ return Date.UTC(year, month - 1, day);
+ }
+ }
+
+ return null;
+};
+
const toChartCandle = (candle: EquityCandle): ChartCandle => {
return {
time: toChartTime(candle.ts),
@@ -62,6 +100,28 @@ const toChartCandle = (candle: EquityCandle): ChartCandle => {
};
};
+const clamp = (value: number, min: number, max: number): number => {
+ if (!Number.isFinite(value)) {
+ return min;
+ }
+ return Math.max(min, Math.min(max, value));
+};
+
+const sampleToLimit = (items: T[], limit: number): T[] => {
+ if (items.length <= limit) {
+ return items;
+ }
+
+ const safeLimit = Math.max(1, Math.floor(limit));
+ const step = Math.ceil(items.length / safeLimit);
+ const sampled: T[] = [];
+ for (let idx = 0; idx < items.length; idx += step) {
+ sampled.push(items[idx]);
+ }
+
+ return sampled;
+};
+
const readErrorDetail = async (response: Response): Promise => {
const text = await response.text();
if (!text) {
@@ -1319,7 +1379,84 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
const seriesRef = useRef(null);
const socketRef = useRef(null);
const reconnectRef = useRef(null);
+ const overlaySocketRef = useRef(null);
+ const overlayReconnectRef = useRef(null);
const lastCandleRef = useRef<{ time: UTCTimestamp; seq: number } | null>(null);
+
+ const overlayCanvasRef = useRef(null);
+ const overlayCtxRef = useRef(null);
+ const overlayDataRef = useRef([]);
+ const overlayLiveRef = useRef([]);
+ const overlayLastFetchRef = useRef<{ startTs: number; endTs: number; ticker: string } | null>(
+ null
+ );
+ const overlayFetchAbortRef = useRef(null);
+ const overlayTimerRef = useRef(null);
+
+ const [overlayEnabled, setOverlayEnabled] = useState(true);
+
+ const drawOverlay = useCallback(
+ (points: EquityOverlayPoint[]) => {
+ const canvas = overlayCanvasRef.current;
+ const ctx = overlayCtxRef.current;
+ const chart = chartRef.current;
+ if (!canvas || !ctx || !chart) {
+ return;
+ }
+
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+
+ if (!overlayEnabled || points.length === 0) {
+ canvas.style.opacity = "0";
+ return;
+ }
+
+ const timeScale = chart.timeScale();
+ if (!seriesRef.current) {
+ canvas.style.opacity = "0";
+ return;
+ }
+
+ const filtered = points.filter((point) => point.offExchangeFlag);
+ const sampled = sampleToLimit(filtered, 1400);
+
+ const maxRadius = 10;
+ const minRadius = 2;
+ const maxSize = Math.max(1, ...sampled.map((point) => point.size));
+
+ ctx.globalAlpha = 0.9;
+ ctx.fillStyle = "rgba(31, 74, 123, 0.55)";
+ ctx.strokeStyle = "rgba(31, 74, 123, 0.95)";
+
+ for (const point of sampled) {
+ const x = timeScale.timeToCoordinate(toChartTime(point.ts));
+ const y = seriesRef.current.priceToCoordinate(point.price);
+ if (x === null || y === null) {
+ continue;
+ }
+
+ const radius = clamp(
+ minRadius + (Math.sqrt(point.size) / Math.sqrt(maxSize)) * (maxRadius - minRadius),
+ minRadius,
+ maxRadius
+ );
+
+ ctx.beginPath();
+ ctx.arc(x, y, radius, 0, Math.PI * 2);
+ ctx.fill();
+ ctx.stroke();
+ }
+
+ ctx.globalAlpha = 1;
+ canvas.style.opacity = "1";
+ },
+ [overlayEnabled]
+ );
+
+ useEffect(() => {
+ drawOverlay([...overlayDataRef.current, ...overlayLiveRef.current]);
+ }, [drawOverlay, ticker, intervalMs, mode]);
+
const replayBucket = useMemo(() => {
if (mode !== "replay" || replayTime === null) {
return null;
@@ -1371,6 +1508,19 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
}
});
+ const overlayCanvas = document.createElement("canvas");
+ overlayCanvas.width = Math.max(1, Math.floor(width));
+ overlayCanvas.height = Math.max(1, Math.floor(height));
+ overlayCanvas.style.position = "absolute";
+ overlayCanvas.style.inset = "0";
+ overlayCanvas.style.pointerEvents = "none";
+ overlayCanvas.style.zIndex = "2";
+ overlayCanvas.style.opacity = "0";
+ container.style.position = "relative";
+ container.appendChild(overlayCanvas);
+ overlayCanvasRef.current = overlayCanvas;
+ overlayCtxRef.current = overlayCanvas.getContext("2d");
+
const series = chart.addCandlestickSeries({
upColor: "#2f6d4f",
downColor: "#c46f2a",
@@ -1390,10 +1540,18 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
}
const { width: nextWidth, height: nextHeight } = entry.contentRect;
if (Number.isFinite(nextWidth) && Number.isFinite(nextHeight)) {
+ const nextW = Math.max(1, Math.floor(nextWidth));
+ const nextH = Math.max(1, Math.floor(nextHeight));
chart.applyOptions({
- width: Math.max(1, Math.floor(nextWidth)),
- height: Math.max(1, Math.floor(nextHeight))
+ width: nextW,
+ height: nextH
});
+
+ const canvas = overlayCanvasRef.current;
+ if (canvas) {
+ canvas.width = nextW;
+ canvas.height = nextH;
+ }
}
});
@@ -1404,6 +1562,9 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
chart.remove();
chartRef.current = null;
seriesRef.current = null;
+ overlayCtxRef.current = null;
+ overlayCanvasRef.current?.remove();
+ overlayCanvasRef.current = null;
};
}, []);
@@ -1418,6 +1579,9 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
setLastUpdate(null);
lastCandleRef.current = null;
seriesRef.current.setData([]);
+ overlayDataRef.current = [];
+ overlayLiveRef.current = [];
+ overlayLastFetchRef.current = null;
setStatus("connected");
return;
}
@@ -1428,6 +1592,9 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
setLastUpdate(null);
lastCandleRef.current = null;
seriesRef.current.setData([]);
+ overlayDataRef.current = [];
+ overlayLiveRef.current = [];
+ overlayLastFetchRef.current = null;
setStatus(mode === "live" ? "connecting" : "connected");
const fetchCandles = async () => {
@@ -1460,6 +1627,7 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
const chartData = sorted.map(toChartCandle);
seriesRef.current.setData(chartData);
chartRef.current?.timeScale().fitContent();
+ drawOverlay([...overlayDataRef.current, ...overlayLiveRef.current]);
if (sorted.length > 0) {
const last = sorted[sorted.length - 1];
@@ -1477,10 +1645,125 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
}
};
+
+ const ensureOverlayListener = () => {
+ if (!chartRef.current) {
+ return;
+ }
+
+ const handler = () => {
+ const combined = [...overlayDataRef.current, ...overlayLiveRef.current];
+ drawOverlay(combined);
+ scheduleOverlayFetch();
+ };
+
+ chartRef.current.timeScale().subscribeVisibleTimeRangeChange(handler);
+ return () => {
+ chartRef.current?.timeScale().unsubscribeVisibleTimeRangeChange(handler);
+ };
+ };
+
+ const cancelOverlayFetch = () => {
+ if (overlayFetchAbortRef.current) {
+ overlayFetchAbortRef.current.abort();
+ overlayFetchAbortRef.current = null;
+ }
+ };
+
+ const fetchOverlayRange = async (startTs: number, endTs: number) => {
+ cancelOverlayFetch();
+ const abort = new AbortController();
+ overlayFetchAbortRef.current = abort;
+
+ const url = new URL(buildApiUrl("/prints/equities/range"));
+ url.searchParams.set("underlying_id", ticker);
+ url.searchParams.set("start_ts", Math.floor(startTs).toString());
+ url.searchParams.set("end_ts", Math.floor(endTs).toString());
+ url.searchParams.set("limit", "2500");
+
+ const response = await fetch(url.toString(), { signal: abort.signal });
+ if (!response.ok) {
+ const detail = await readErrorDetail(response);
+ throw new Error(
+ `Equity range fetch failed (${response.status})${detail ? `: ${detail}` : ""}`
+ );
+ }
+
+ const payload = (await response.json()) as { data?: EquityPrint[] };
+ const prints = payload.data ?? [];
+ overlayDataRef.current = prints.map((print) => ({
+ ts: print.ts,
+ price: print.price,
+ size: print.size,
+ offExchangeFlag: print.offExchangeFlag
+ }));
+ overlayLiveRef.current = [];
+ overlayLastFetchRef.current = { startTs, endTs, ticker };
+ };
+
+ function scheduleOverlayFetch() {
+ if (overlayTimerRef.current !== null) {
+ window.clearTimeout(overlayTimerRef.current);
+ }
+
+ overlayTimerRef.current = window.setTimeout(() => {
+ if (!active || !chartRef.current || !seriesRef.current) {
+ return;
+ }
+
+ const timeScale = chartRef.current.timeScale();
+ const range = timeScale.getVisibleRange();
+ if (!range) {
+ return;
+ }
+
+ const startTs = chartTimeToMs(range.from);
+ const endTs = chartTimeToMs(range.to);
+ if (startTs === null || endTs === null) {
+ return;
+ }
+ const last = overlayLastFetchRef.current;
+
+ const needsFetch =
+ !last ||
+ last.ticker !== ticker ||
+ startTs < last.startTs ||
+ endTs > last.endTs ||
+ Math.abs(endTs - last.endTs) > intervalMs * 6;
+
+ if (!needsFetch) {
+ return;
+ }
+
+ void fetchOverlayRange(startTs, endTs)
+ .then(() => {
+ drawOverlay([...overlayDataRef.current, ...overlayLiveRef.current]);
+ })
+ .catch((error) => {
+ if (!active) {
+ return;
+ }
+ if (error instanceof DOMException && error.name === "AbortError") {
+ return;
+ }
+ console.warn("Overlay fetch failed", error);
+ });
+ }, 180);
+ }
+
+ const overlayUnsubscribe = ensureOverlayListener();
+ scheduleOverlayFetch();
+
void fetchCandles();
return () => {
active = false;
+ cancelOverlayFetch();
+ if (overlayTimerRef.current !== null) {
+ window.clearTimeout(overlayTimerRef.current);
+ overlayTimerRef.current = null;
+ }
+ overlayUnsubscribe?.();
};
}, [ready, ticker, intervalMs, mode, replayBucket, replayEndTs]);
@@ -1493,6 +1776,15 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
window.clearTimeout(reconnectRef.current);
reconnectRef.current = null;
}
+
+ if (overlaySocketRef.current) {
+ overlaySocketRef.current.close();
+ }
+ if (overlayReconnectRef.current !== null) {
+ window.clearTimeout(overlayReconnectRef.current);
+ overlayReconnectRef.current = null;
+ }
+
return;
}
@@ -1545,6 +1837,7 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
lastCandleRef.current = { time: chartCandle.time, seq: candle.seq };
setHasData(true);
setLastUpdate(candle.ingest_ts ?? candle.ts);
+ drawOverlay([...overlayDataRef.current, ...overlayLiveRef.current]);
} catch (error) {
console.warn("Failed to parse candle payload", error);
}
@@ -1567,7 +1860,64 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
};
};
+ const connectOverlay = () => {
+ if (!active) {
+ return;
+ }
+
+ const socket = new WebSocket(buildWsUrl("/ws/equities"));
+ overlaySocketRef.current = socket;
+
+ socket.onmessage = (event) => {
+ if (!active) {
+ return;
+ }
+
+ try {
+ const message = JSON.parse(event.data) as StreamMessage;
+ if (!message || message.type !== "equity-print") {
+ return;
+ }
+
+ const print = message.payload;
+ if (print.underlying_id !== ticker) {
+ return;
+ }
+
+ overlayLiveRef.current.push({
+ ts: print.ts,
+ price: print.price,
+ size: print.size,
+ offExchangeFlag: print.offExchangeFlag
+ });
+
+ if (overlayLiveRef.current.length > 1500) {
+ overlayLiveRef.current = overlayLiveRef.current.slice(-1500);
+ }
+
+ drawOverlay([...overlayDataRef.current, ...overlayLiveRef.current]);
+ } catch (error) {
+ console.warn("Failed to parse equity print payload", error);
+ }
+ };
+
+ socket.onclose = () => {
+ if (!active) {
+ return;
+ }
+ overlayReconnectRef.current = window.setTimeout(connectOverlay, 1500);
+ };
+
+ socket.onerror = () => {
+ if (!active) {
+ return;
+ }
+ socket.close();
+ };
+ };
+
connect();
+ connectOverlay();
return () => {
active = false;
@@ -1578,8 +1928,16 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
if (socketRef.current) {
socketRef.current.close();
}
+
+ if (overlayReconnectRef.current !== null) {
+ window.clearTimeout(overlayReconnectRef.current);
+ overlayReconnectRef.current = null;
+ }
+ if (overlaySocketRef.current) {
+ overlaySocketRef.current.close();
+ }
};
- }, [ready, mode, ticker, intervalMs]);
+ }, [ready, mode, ticker, intervalMs, drawOverlay]);
useEffect(() => {
if (!chartRef.current) {
@@ -1610,6 +1968,14 @@ const CandleChart = ({ ticker, intervalMs, mode, replayTime = null }: CandleChar
{lastUpdate ? `Updated ${formatTime(lastUpdate)}` : "Waiting for data"}
+
+ Blue circles = off-exchange trades
{error ? (
From 0f3e05085cfa277ae1283b39ba45e1afbb0793ae Mon Sep 17 00:00:00 2001
From: dirtydishes
+
);
})
)}
@@ -3528,15 +3702,16 @@ export default function HomePage() {
const underlying = inferDarkUnderlying(event, equityPrintMap, equityJoinMap);
const evidenceCount = event.evidence_refs.length;
return (
-