From aece60c84f779a9dba2003b8e5dcfa80b6eeb339 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Wed, 6 May 2026 23:18:38 -0400 Subject: [PATCH 01/15] Fix tape rail navigation --- .beads/issues.jsonl | 2 ++ apps/web/app/terminal.tsx | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 91007f6..741bac4 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,5 @@ +{"_type":"issue","id":"islandflow-uj7","title":"Fix home to tape navigation","description":"Home rail Tape navigation was not reliably switching to the tape route. Use browser-native top-level navigation for Home/Tape rail links so /tape remains reachable even if client router handling stalls.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T03:18:14Z","created_by":"dirtydishes","updated_at":"2026-05-07T03:18:21Z","started_at":"2026-05-07T03:18:20Z","closed_at":"2026-05-07T03:18:21Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-84s","title":"Implement seamless /tape live-to-history scroll gate","description":"Implement seamless live-to-ClickHouse scroll-gated history for /tape panes, including split live/history buffers in the web client, snapshot_limit support on live subscriptions, a bundled options support lookup endpoint, ClickHouse helpers for parity hydration, and test coverage for live head retention, background history loading, scoped options deep-hydration, and historical options decor restoration.\n","status":"in_progress","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T02:10:43Z","created_by":"dirtydishes","updated_at":"2026-05-07T02:10:47Z","started_at":"2026-05-07T02:10:47Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-sh1","title":"Fix live websocket stale lag and reconnect loop","description":"Investigate and fix API live consumer lag causing stale timestamps, feed-behind status, and reconnect loops. Optimize live cache persistence path, add lag telemetry/alerts, and validate in runtime.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T17:04:34Z","created_by":"dirtydishes","updated_at":"2026-05-04T17:09:44Z","started_at":"2026-05-04T17:04:38Z","closed_at":"2026-05-04T17:09:44Z","close_reason":"Completed: optimized live cache persistence path, added lag telemetry, deployed api via docker compose on di, verified ws freshness and low hotFeedLagMs","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-b3o","title":"Implement options tape table with execution spot","description":"Redesign OptionsPane into a dense classifier-colored table and preserve execution-time underlying spot on option prints from equity quote mid.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:41:59Z","created_by":"dirtydishes","updated_at":"2026-05-04T05:14:26Z","started_at":"2026-05-04T04:42:08Z","closed_at":"2026-05-04T05:14:26Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-ug1","title":"Fix false NBBO-missing badges in live Options tape","description":"Investigate and fix client-side cases where Options rows show NBBO missing/stale even when a fresh NBBO quote exists in the live nbbo map. Update rendering logic to prefer fresh quote-derived status and add regression tests.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-29T15:58:31Z","created_by":"dirtydishes","updated_at":"2026-04-29T16:01:28Z","started_at":"2026-04-29T15:58:35Z","closed_at":"2026-04-29T16:01:28Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index e4c67a6..ef96942 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -1,6 +1,5 @@ "use client"; -import Link from "next/link"; import { usePathname } from "next/navigation"; import { createContext, @@ -6998,13 +6997,13 @@ export function TerminalAppShell({ children }: { children: ReactNode }) { {NAV_ITEMS.map((item) => { const active = pathname === item.href; return ( - {item.label} - + ); })} From 9ca0e5241110c0d640c9ade1fb041f5d2cfc42c9 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Wed, 6 May 2026 23:28:50 -0400 Subject: [PATCH 02/15] Fix tape history scroll gate attachment --- apps/web/app/terminal.tsx | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index ef96942..8419290 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -1218,6 +1218,7 @@ export const getOptionTableSnapshot = ( type ListScrollState = { listRef: React.RefObject; + listNode: HTMLDivElement | null; setListRef: (node: HTMLDivElement | null) => void; isAtTop: boolean; isAtTopRef: React.MutableRefObject; @@ -1309,6 +1310,7 @@ const useListScroll = (): ListScrollState => { return { listRef, + listNode, setListRef, isAtTop, isAtTopRef, @@ -1369,6 +1371,7 @@ const useScrollAnchor = ( const useBottomHistoryGate = ( listRef: React.RefObject, + listNode: HTMLDivElement | null, enabled: boolean, onLoadOlder: () => void ): void => { @@ -1381,7 +1384,7 @@ const useBottomHistoryGate = ( if (!enabled) { return; } - const element = listRef.current; + const element = listNode ?? listRef.current; if (!element) { return; } @@ -1398,7 +1401,7 @@ const useBottomHistoryGate = ( return () => { element.removeEventListener("scroll", maybeLoad); }; - }, [enabled, listRef]); + }, [enabled, listNode, listRef]); }; type VirtualListResult = { @@ -6099,7 +6102,7 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredOptions.slice(0, limit) : state.filteredOptions; const virtual = useVirtualList(items, state.optionsScroll.listRef, !limit, 36); - useBottomHistoryGate(state.optionsScroll.listRef, state.mode === "live" && !limit, () => + useBottomHistoryGate(state.optionsScroll.listRef, state.optionsScroll.listNode, state.mode === "live" && !limit, () => void state.liveSession.loadOlder("options") ); @@ -6276,7 +6279,7 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredEquities.slice(0, limit) : state.filteredEquities; const virtual = useVirtualList(items, state.equitiesScroll.listRef, !limit, 36); - useBottomHistoryGate(state.equitiesScroll.listRef, state.mode === "live" && !limit, () => + useBottomHistoryGate(state.equitiesScroll.listRef, state.equitiesScroll.listNode, state.mode === "live" && !limit, () => void state.liveSession.loadOlder("equities") ); @@ -6374,7 +6377,7 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredFlow.slice(0, limit) : state.filteredFlow; const virtual = useVirtualList(items, state.flowScroll.listRef, !limit, 44); - useBottomHistoryGate(state.flowScroll.listRef, state.mode === "live" && !limit, () => + useBottomHistoryGate(state.flowScroll.listRef, state.flowScroll.listNode, state.mode === "live" && !limit, () => void state.liveSession.loadOlder("flow") ); @@ -6516,7 +6519,7 @@ const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => const state = useTerminal(); const items = limit ? state.filteredAlerts.slice(0, limit) : state.filteredAlerts; const virtual = useVirtualList(items, state.alertsScroll.listRef, !limit, 46); - useBottomHistoryGate(state.alertsScroll.listRef, state.mode === "live" && !limit, () => + useBottomHistoryGate(state.alertsScroll.listRef, state.alertsScroll.listNode, state.mode === "live" && !limit, () => void state.liveSession.loadOlder("alerts") ); @@ -6615,7 +6618,7 @@ type ClassifierPaneProps = { const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { const state = useTerminal(); - useBottomHistoryGate(state.classifierScroll.listRef, state.mode === "live" && !limit, () => { + useBottomHistoryGate(state.classifierScroll.listRef, state.classifierScroll.listNode, state.mode === "live" && !limit, () => { void state.liveSession.loadOlder("smart-money"); void state.liveSession.loadOlder("classifier-hits"); }); @@ -6745,7 +6748,7 @@ const DarkPane = ({ limit, className }: DarkPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredInferredDark.slice(0, limit) : state.filteredInferredDark; const virtual = useVirtualList(items, state.darkScroll.listRef, !limit, 44); - useBottomHistoryGate(state.darkScroll.listRef, state.mode === "live" && !limit, () => + useBottomHistoryGate(state.darkScroll.listRef, state.darkScroll.listNode, state.mode === "live" && !limit, () => void state.liveSession.loadOlder("inferred-dark") ); From 1d3865c8fcbc02318cad30d9024571e51eb047b4 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Wed, 6 May 2026 23:42:35 -0400 Subject: [PATCH 03/15] Fix Tape navigation from home --- apps/web/app/terminal.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 8419290..7a66d5b 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -1,5 +1,6 @@ "use client"; +import Link from "next/link"; import { usePathname } from "next/navigation"; import { createContext, @@ -7000,13 +7001,13 @@ export function TerminalAppShell({ children }: { children: ReactNode }) { {NAV_ITEMS.map((item) => { const active = pathname === item.href; return ( - {item.label} - + ); })} From d81b4c0cfb97671d79d7c932d48eb6c443535c1a Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Wed, 6 May 2026 23:49:04 -0400 Subject: [PATCH 04/15] Restore scoped live history retention --- apps/web/app/terminal.test.ts | 49 +++++++++++++++++++++++++++++++++++ apps/web/app/terminal.tsx | 24 +++++++++-------- 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index 2071762..c96d86e 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "bun:test"; import { NAV_ITEMS, + appendHistoryTail, buildDefaultFlowFilters, classifierToneForFamily, deriveAlertDirection, @@ -9,6 +10,7 @@ import { formatOptionContractLabel, flushPausableTapeData, getAlertWindowAnchorTs, + getLiveHistoryRetentionCap, getOptionTableSnapshot, getLiveFeedStatus, getLiveManifest, @@ -240,6 +242,53 @@ describe("live tape pausable helpers", () => { }); }); +describe("live tape history helpers", () => { + it("appends older scoped rows behind the hot live head", () => { + const liveHead = Array.from({ length: 100 }, (_, idx) => + makeItem(`hot-${idx}`, 200 - idx, 2_000 - idx) + ); + const older = [makeItem("older-1", 99, 999), makeItem("older-2", 98, 998)]; + + const next = appendHistoryTail([], older, liveHead, 5000); + + expect(next.map((item) => item.trace_id)).toEqual(["older-1", "older-2"]); + }); + + it("skips duplicates already present in the live head", () => { + const liveHead = [makeItem("latest", 3, 300), makeItem("duplicate", 2, 200)]; + const older = [makeItem("duplicate", 2, 200), makeItem("older", 1, 100)]; + + const next = appendHistoryTail([], older, liveHead, 5000); + + expect(next.map((item) => item.trace_id)).toEqual(["older"]); + }); + + it("trims the history tail to the soft cap", () => { + const current = [makeItem("existing", 4, 400)]; + const older = [makeItem("older-1", 3, 300), makeItem("older-2", 2, 200)]; + + const next = appendHistoryTail(current, older, [], 2); + + expect(next.map((item) => item.trace_id)).toEqual(["existing", "older-1"]); + }); + + it("keeps scoped option and equity history on the normal retention cap", () => { + expect( + getLiveHistoryRetentionCap({ + channel: "options", + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + } as any) + ).toBeGreaterThan(0); + expect( + getLiveHistoryRetentionCap({ + channel: "equities", + underlying_ids: ["AAPL"] + } as any) + ).toBeGreaterThan(0); + }); +}); + describe("options display formatters", () => { it("formats dashed option contracts as ticker strike expiry", () => { expect(formatOptionContractLabel("SPY-2025-01-17-450-C")).toEqual({ diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 7a66d5b..d20be39 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -510,7 +510,7 @@ const EMPTY_PAUSABLE_TAPE = { dropped: 0 }; -const appendHistoryTail = ( +export const appendHistoryTail = ( current: T[], incoming: T[], liveHead: T[], @@ -541,6 +541,16 @@ const appendHistoryTail = ( return cap > 0 ? appended.slice(0, cap) : appended; }; +export const getLiveHistoryRetentionCap = (subscription: LiveSubscription): number => { + switch (subscription.channel) { + case "options": + case "equities": + return LIVE_HISTORY_SOFT_CAP; + default: + return LIVE_HISTORY_SOFT_CAP; + } +}; + export const getLiveFeedStatus = ( sourceStatus: WsStatus, freshestTs: number | null, @@ -2924,21 +2934,13 @@ const useLiveSession = ( switch (subscription.channel) { case "options": - mergeOlder( - setOptionsHistory, - options, - subscription.underlying_ids?.length || subscription.option_contract_id ? 0 : LIVE_HISTORY_SOFT_CAP - ); + mergeOlder(setOptionsHistory, options, getLiveHistoryRetentionCap(subscription)); break; case "nbbo": mergeOlder(setNbboHistory, nbbo); break; case "equities": - mergeOlder( - setEquitiesHistory, - equities, - subscription.underlying_ids?.length ? 0 : LIVE_HISTORY_SOFT_CAP - ); + mergeOlder(setEquitiesHistory, equities, getLiveHistoryRetentionCap(subscription)); break; case "equity-quotes": break; From 034d24f8acaa6650604bd49eec02bdf5f66615ff Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Thu, 7 May 2026 00:39:26 -0400 Subject: [PATCH 05/15] Restore continuous live tape history --- apps/web/app/terminal.test.ts | 75 ++++++++ apps/web/app/terminal.tsx | 327 +++++++++++++++++++++++--------- services/api/src/index.ts | 8 +- services/api/src/live.ts | 28 +-- services/api/tests/live.test.ts | 162 +++++++++++++++- 5 files changed, 483 insertions(+), 117 deletions(-) diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index c96d86e..78c7c70 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "bun:test"; +import { getSubscriptionKey as getLiveSubscriptionKey } from "@islandflow/types"; import { NAV_ITEMS, appendHistoryTail, @@ -10,10 +11,12 @@ import { formatOptionContractLabel, flushPausableTapeData, getAlertWindowAnchorTs, + getScopedLiveAutoHydrationChannels, getLiveHistoryRetentionCap, getOptionTableSnapshot, getLiveFeedStatus, getLiveManifest, + mergeNewestWithOverflow, normalizeAlertSeverity, nextFlowFilterPopoverState, projectPausableTapeState, @@ -243,6 +246,36 @@ describe("live tape pausable helpers", () => { }); describe("live tape history helpers", () => { + it("promotes hot-window overflow into the history tail", () => { + const currentHot = [makeItem("hot-3", 3, 300), makeItem("hot-2", 2, 200), makeItem("hot-1", 1, 100)]; + const incoming = [makeItem("hot-4", 4, 400)]; + + const { kept, evicted } = mergeNewestWithOverflow(incoming, currentHot, 3); + const nextHistory = appendHistoryTail([], evicted, kept, 5000); + + expect(kept.map((item) => item.trace_id)).toEqual(["hot-4", "hot-3", "hot-2"]); + expect(nextHistory.map((item) => item.trace_id)).toEqual(["hot-1"]); + }); + + it("keeps the combined tape continuous beyond the hot live window", () => { + let hot: Array> = []; + let history: Array> = []; + + for (let seq = 1; seq <= 5; seq += 1) { + const { kept, evicted } = mergeNewestWithOverflow([makeItem(`row-${seq}`, seq, seq * 100)], hot, 2); + hot = kept; + history = appendHistoryTail(history, evicted, hot, 5000); + } + + expect([...hot, ...history].map((item) => item.trace_id)).toEqual([ + "row-5", + "row-4", + "row-3", + "row-2", + "row-1" + ]); + }); + it("appends older scoped rows behind the hot live head", () => { const liveHead = Array.from({ length: 100 }, (_, idx) => makeItem(`hot-${idx}`, 200 - idx, 2_000 - idx) @@ -263,6 +296,16 @@ describe("live tape history helpers", () => { expect(next.map((item) => item.trace_id)).toEqual(["older"]); }); + it("dedupes the seam between promoted overflow and fetched history", () => { + const currentHot = [makeItem("hot-3", 3, 300), makeItem("hot-2", 2, 200), makeItem("hot-1", 1, 100)]; + const { kept, evicted } = mergeNewestWithOverflow([makeItem("hot-4", 4, 400)], currentHot, 3); + const promoted = appendHistoryTail([], evicted, kept, 5000); + const merged = appendHistoryTail(promoted, [makeItem("hot-1", 1, 100), makeItem("older", 0, 50)], kept, 5000); + + expect(merged.map((item) => item.trace_id)).toEqual(["hot-1", "older"]); + expect(new Set([...kept, ...merged].map((item) => item.trace_id)).size).toBe(kept.length + merged.length); + }); + it("trims the history tail to the soft cap", () => { const current = [makeItem("existing", 4, 400)]; const older = [makeItem("older-1", 3, 300), makeItem("older-2", 2, 200)]; @@ -287,6 +330,38 @@ describe("live tape history helpers", () => { } as any) ).toBeGreaterThan(0); }); + + it("keeps auto-hydrating scoped live history while next_before exists", () => { + const manifest = getLiveManifest( + "/tape", + "AAPL", + 60000, + buildDefaultFlowFilters(), + { + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }, + { underlying_ids: ["AAPL"] } + ); + const historyCursors = Object.fromEntries( + manifest.map((subscription) => [getLiveSubscriptionKey(subscription), { ts: 1, seq: 1 }]) + ); + + expect( + getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {}) + ).toEqual(["options", "equities"]); + expect( + getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, { + [getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "options")!)]: true + }) + ).toEqual(["equities"]); + expect( + getScopedLiveAutoHydrationChannels(true, "/tape", manifest, { + ...historyCursors, + [getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "equities")!)]: null + }, {}) + ).toEqual(["options"]); + }); }); describe("options display formatters", () => { diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index d20be39..72edbd5 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -368,15 +368,15 @@ const buildItemKey = (item: SortableItem): string | null => { return null; }; -const mergeNewest = ( +export const mergeNewestWithOverflow = ( incoming: T[], existing: T[], limit = LIVE_HOT_WINDOW, onTrim?: (evicted: number) => void -): T[] => { +): { kept: T[]; evicted: T[] } => { const combined = [...incoming, ...existing]; if (combined.length === 0) { - return combined; + return { kept: combined, evicted: [] }; } const seen = new Set(); @@ -402,12 +402,24 @@ const mergeNewest = ( }); const safeLimit = Math.max(1, Math.floor(limit)); - const evicted = Math.max(0, deduped.length - safeLimit); - if (evicted > 0) { - onTrim?.(evicted); + const evicted = deduped.slice(safeLimit); + if (evicted.length > 0) { + onTrim?.(evicted.length); } - return deduped.slice(0, safeLimit); + return { + kept: deduped.slice(0, safeLimit), + evicted + }; +}; + +const mergeNewest = ( + incoming: T[], + existing: T[], + limit = LIVE_HOT_WINDOW, + onTrim?: (evicted: number) => void +): T[] => { + return mergeNewestWithOverflow(incoming, existing, limit, onTrim).kept; }; const getTapeItemKey = (item: SortableItem): string => { @@ -520,25 +532,27 @@ export const appendHistoryTail = ( return current; } - const seen = new Set(); - for (const item of liveHead) { - seen.add(getTapeItemKey(item)); - } - for (const item of current) { - seen.add(getTapeItemKey(item)); - } + const seen = new Set(liveHead.map((item) => getTapeItemKey(item))); + const combined: T[] = []; - const appended = [...current]; - for (const item of incoming) { + for (const item of [...current, ...incoming]) { const key = getTapeItemKey(item); if (seen.has(key)) { continue; } seen.add(key); - appended.push(item); + combined.push(item); } - return cap > 0 ? appended.slice(0, cap) : appended; + combined.sort((a, b) => { + const delta = extractSortTs(b) - extractSortTs(a); + if (delta !== 0) { + return delta; + } + return extractSortSeq(b) - extractSortSeq(a); + }); + + return cap > 0 ? combined.slice(0, cap) : combined; }; export const getLiveHistoryRetentionCap = (subscription: LiveSubscription): number => { @@ -551,6 +565,36 @@ export const getLiveHistoryRetentionCap = (subscription: LiveSubscription): numb } }; +export const getScopedLiveAutoHydrationChannels = ( + enabled: boolean, + pathname: string, + manifest: LiveSubscription[], + historyCursors: Partial>, + historyLoading: Partial> +): Array> => { + if (!enabled || pathname !== "/tape") { + return []; + } + + const channels: Array> = []; + for (const subscription of manifest) { + const scoped = + (subscription.channel === "options" && + (subscription.underlying_ids?.length || subscription.option_contract_id)) || + (subscription.channel === "equities" && subscription.underlying_ids?.length); + if (!scoped) { + continue; + } + + const key = getLiveSubscriptionKey(subscription); + if (historyCursors[key] && !historyLoading[key]) { + channels.push(subscription.channel); + } + } + + return channels; +}; + export const getLiveFeedStatus = ( sourceStatus: WsStatus, freshestTs: number | null, @@ -2544,6 +2588,27 @@ const useLiveSession = ( const [inferredDarkHistory, setInferredDarkHistory] = useState([]); const [chartCandles, setChartCandles] = useState([]); const [chartOverlay, setChartOverlay] = useState([]); + const optionsRef = useRef([]); + const nbboRef = useRef([]); + const equitiesRef = useRef([]); + const equityQuotesRef = useRef([]); + const equityJoinsRef = useRef([]); + const flowRef = useRef([]); + const smartMoneyRef = useRef([]); + const classifierHitsRef = useRef([]); + const alertsRef = useRef([]); + const inferredDarkRef = useRef([]); + const chartCandlesRef = useRef([]); + const chartOverlayRef = useRef([]); + const optionsHistoryRef = useRef([]); + const nbboHistoryRef = useRef([]); + const equitiesHistoryRef = useRef([]); + const equityJoinsHistoryRef = useRef([]); + const flowHistoryRef = useRef([]); + const smartMoneyHistoryRef = useRef([]); + const classifierHitsHistoryRef = useRef([]); + const alertsHistoryRef = useRef([]); + const inferredDarkHistoryRef = useRef([]); const socketRef = useRef(null); const reconnectRef = useRef(null); const idleWatchdogRef = useRef(null); @@ -2556,6 +2621,27 @@ const useLiveSession = ( [pathname, chartTicker, chartIntervalMs, flowFilters, optionScope, equityScope] ); + const replaceArrayState = ( + setter: Dispatch>, + ref: { current: T[] }, + next: T[] + ): void => { + ref.current = next; + setter(next); + }; + + const mergeHistoryState = ( + setter: Dispatch>, + ref: { current: T[] }, + incoming: T[], + liveHead: T[], + cap = LIVE_HISTORY_SOFT_CAP + ): void => { + const next = appendHistoryTail(ref.current, incoming, liveHead, cap); + ref.current = next; + setter(next); + }; + useEffect(() => { if (!enabled) { setStatus("disconnected"); @@ -2586,6 +2672,27 @@ const useLiveSession = ( setInferredDarkHistory([]); setChartCandles([]); setChartOverlay([]); + optionsRef.current = []; + nbboRef.current = []; + equitiesRef.current = []; + equityQuotesRef.current = []; + equityJoinsRef.current = []; + flowRef.current = []; + smartMoneyRef.current = []; + classifierHitsRef.current = []; + alertsRef.current = []; + inferredDarkRef.current = []; + chartCandlesRef.current = []; + chartOverlayRef.current = []; + optionsHistoryRef.current = []; + nbboHistoryRef.current = []; + equitiesHistoryRef.current = []; + equityJoinsHistoryRef.current = []; + flowHistoryRef.current = []; + smartMoneyHistoryRef.current = []; + classifierHitsHistoryRef.current = []; + alertsHistoryRef.current = []; + inferredDarkHistoryRef.current = []; subscribedKeysRef.current = new Set(); subscribedMapRef.current = new Map(); if (socketRef.current) { @@ -2642,62 +2749,112 @@ const useLiveSession = ( const updateAt = Date.now(); const mergeItems = ( - setter: React.Dispatch>, + setter: Dispatch>, + ref: { current: T[] }, nextItems: T[], - retentionLimit = LIVE_HOT_WINDOW + retentionLimit = LIVE_HOT_WINDOW, + history?: { + setter: Dispatch>; + ref: { current: T[] }; + cap?: number; + } ) => { - setter((prev) => - message.op === "snapshot" - ? shouldRetainLiveSnapshotHistory( - subscription.channel, - true, - nextItems.length, - prev.length - ) - ? prev - : (nextItems as T[]) - : mergeNewest(nextItems as T[], prev, retentionLimit, (evicted) => - incrementRetentionMetric("hotWindowEvictions", evicted) - ) + if (message.op === "snapshot") { + const next = shouldRetainLiveSnapshotHistory( + subscription.channel, + true, + nextItems.length, + ref.current.length + ) + ? ref.current + : nextItems; + replaceArrayState(setter, ref, next); + return; + } + + const { kept, evicted } = mergeNewestWithOverflow( + nextItems, + ref.current, + retentionLimit, + (evictedCount) => incrementRetentionMetric("hotWindowEvictions", evictedCount) ); + replaceArrayState(setter, ref, kept); + if (history && evicted.length > 0) { + mergeHistoryState(history.setter, history.ref, evicted, kept, history.cap); + } }; switch (subscription.channel) { case "options": - mergeItems(setOptions, items as OptionPrint[], LIVE_HOT_WINDOW_OPTIONS); + mergeItems(setOptions, optionsRef, items as OptionPrint[], LIVE_HOT_WINDOW_OPTIONS, { + setter: setOptionsHistory, + ref: optionsHistoryRef, + cap: getLiveHistoryRetentionCap(subscription) + }); break; case "nbbo": - mergeItems(setNbbo, items as OptionNBBO[]); + mergeItems(setNbbo, nbboRef, items as OptionNBBO[], LIVE_HOT_WINDOW, { + setter: setNbboHistory, + ref: nbboHistoryRef + }); break; case "equities": - mergeItems(setEquities, items as EquityPrint[]); + mergeItems(setEquities, equitiesRef, items as EquityPrint[], LIVE_HOT_WINDOW, { + setter: setEquitiesHistory, + ref: equitiesHistoryRef, + cap: getLiveHistoryRetentionCap(subscription) + }); break; case "equity-quotes": - mergeItems(setEquityQuotes, items as EquityQuote[]); + mergeItems(setEquityQuotes, equityQuotesRef, items as EquityQuote[]); break; case "equity-joins": - mergeItems(setEquityJoins, items as EquityPrintJoin[]); + mergeItems(setEquityJoins, equityJoinsRef, items as EquityPrintJoin[], LIVE_HOT_WINDOW, { + setter: setEquityJoinsHistory, + ref: equityJoinsHistoryRef + }); break; case "flow": - mergeItems(setFlow, items as FlowPacket[]); + mergeItems(setFlow, flowRef, items as FlowPacket[], LIVE_HOT_WINDOW, { + setter: setFlowHistory, + ref: flowHistoryRef + }); break; case "smart-money": - mergeItems(setSmartMoney, items as SmartMoneyEvent[]); + mergeItems(setSmartMoney, smartMoneyRef, items as SmartMoneyEvent[], LIVE_HOT_WINDOW, { + setter: setSmartMoneyHistory, + ref: smartMoneyHistoryRef + }); break; case "classifier-hits": - mergeItems(setClassifierHits, items as ClassifierHitEvent[]); + mergeItems( + setClassifierHits, + classifierHitsRef, + items as ClassifierHitEvent[], + LIVE_HOT_WINDOW, + { + setter: setClassifierHitsHistory, + ref: classifierHitsHistoryRef + } + ); break; case "alerts": - mergeItems(setAlerts, items as AlertEvent[]); + mergeItems(setAlerts, alertsRef, items as AlertEvent[], LIVE_HOT_WINDOW, { + setter: setAlertsHistory, + ref: alertsHistoryRef + }); break; case "inferred-dark": - mergeItems(setInferredDark, items as InferredDarkEvent[]); + mergeItems(setInferredDark, inferredDarkRef, items as InferredDarkEvent[], LIVE_HOT_WINDOW, { + setter: setInferredDarkHistory, + ref: inferredDarkHistoryRef + }); break; case "equity-candles": - mergeItems(setChartCandles, items as EquityCandle[]); + mergeItems(setChartCandles, chartCandlesRef, items as EquityCandle[]); break; case "equity-overlay": - mergeItems(setChartOverlay, items as EquityPrint[]); + mergeItems(setChartOverlay, chartOverlayRef, items as EquityPrint[]); break; } @@ -2839,10 +2996,14 @@ const useLiveSession = ( .filter((channel) => channel === "options" || channel === "equities") ); if (resetScopedChannels.has("options")) { + optionsRef.current = []; + optionsHistoryRef.current = []; setOptions([]); setOptionsHistory([]); } if (resetScopedChannels.has("equities")) { + equitiesRef.current = []; + equitiesHistoryRef.current = []; setEquities([]); setEquitiesHistory([]); } @@ -2926,41 +3087,56 @@ const useLiveSession = ( const mergeOlder = ( setter: Dispatch>, + ref: { current: T[] }, liveHead: T[], cap = LIVE_HISTORY_SOFT_CAP ) => { - setter((prev) => appendHistoryTail(prev, older as T[], liveHead, cap)); + mergeHistoryState(setter, ref, older as T[], liveHead, cap); }; switch (subscription.channel) { case "options": - mergeOlder(setOptionsHistory, options, getLiveHistoryRetentionCap(subscription)); + mergeOlder( + setOptionsHistory, + optionsHistoryRef, + optionsRef.current, + getLiveHistoryRetentionCap(subscription) + ); break; case "nbbo": - mergeOlder(setNbboHistory, nbbo); + mergeOlder(setNbboHistory, nbboHistoryRef, nbboRef.current); break; case "equities": - mergeOlder(setEquitiesHistory, equities, getLiveHistoryRetentionCap(subscription)); + mergeOlder( + setEquitiesHistory, + equitiesHistoryRef, + equitiesRef.current, + getLiveHistoryRetentionCap(subscription) + ); break; case "equity-quotes": break; case "equity-joins": - mergeOlder(setEquityJoinsHistory, equityJoins); + mergeOlder(setEquityJoinsHistory, equityJoinsHistoryRef, equityJoinsRef.current); break; case "flow": - mergeOlder(setFlowHistory, flow); + mergeOlder(setFlowHistory, flowHistoryRef, flowRef.current); break; case "smart-money": - mergeOlder(setSmartMoneyHistory, smartMoney); + mergeOlder(setSmartMoneyHistory, smartMoneyHistoryRef, smartMoneyRef.current); break; case "classifier-hits": - mergeOlder(setClassifierHitsHistory, classifierHits); + mergeOlder( + setClassifierHitsHistory, + classifierHitsHistoryRef, + classifierHitsRef.current + ); break; case "alerts": - mergeOlder(setAlertsHistory, alerts); + mergeOlder(setAlertsHistory, alertsHistoryRef, alertsRef.current); break; case "inferred-dark": - mergeOlder(setInferredDarkHistory, inferredDark); + mergeOlder(setInferredDarkHistory, inferredDarkHistoryRef, inferredDarkRef.current); break; } @@ -2978,41 +3154,18 @@ const useLiveSession = ( setHistoryLoading((current) => ({ ...current, [key]: false })); } }, - [ - enabled, - manifest, - historyCursors, - historyLoading, - options, - nbbo, - equities, - equityJoins, - flow, - smartMoney, - classifierHits, - alerts, - inferredDark - ] + [enabled, manifest, historyCursors, historyLoading] ); useEffect(() => { - if (!enabled || pathname !== "/tape") { - return; - } - const scoped = manifest.filter( - (subscription) => - (subscription.channel === "options" && - (subscription.underlying_ids?.length || subscription.option_contract_id)) || - (subscription.channel === "equities" && subscription.underlying_ids?.length) - ); - if (scoped.length === 0) { - return; - } - for (const subscription of scoped) { - const key = getLiveSubscriptionKey(subscription); - if (historyCursors[key] && !historyLoading[key]) { - void loadOlder(subscription.channel); - } + for (const channel of getScopedLiveAutoHydrationChannels( + enabled, + pathname, + manifest, + historyCursors, + historyLoading + )) { + void loadOlder(channel); } }, [enabled, pathname, manifest, historyCursors, historyLoading, loadOlder]); diff --git a/services/api/src/index.ts b/services/api/src/index.ts index c450ea7..ff72307 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -112,7 +112,7 @@ import { } from "@islandflow/types"; import { createClient } from "redis"; import { z } from "zod"; -import { LIVE_FEED_LOOKBACK_MS, LiveStateManager, shouldFanoutLiveEvent } from "./live"; +import { LiveStateManager, shouldFanoutLiveEvent } from "./live"; const service = "api"; const logger = createLogger({ service }); @@ -617,14 +617,12 @@ const parseLiveOptionPrintFilters = (url: URL): OptionPrintQueryFilters => { return { ...storageFilters, underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids"), - optionContractId: url.searchParams.get("option_contract_id") ?? undefined, - sinceTs: Date.now() - LIVE_FEED_LOOKBACK_MS + optionContractId: url.searchParams.get("option_contract_id") ?? undefined }; }; const parseLiveEquityPrintFilters = (url: URL): EquityPrintQueryFilters => ({ - underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids"), - sinceTs: Date.now() - LIVE_FEED_LOOKBACK_MS + underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids") }); const matchesScopedOptionSubscription = ( diff --git a/services/api/src/live.ts b/services/api/src/live.ts index aa4281c..2907214 100644 --- a/services/api/src/live.ts +++ b/services/api/src/live.ts @@ -408,13 +408,7 @@ export class LiveStateManager { const config = this.generic[channel]; if (this.redis?.isOpen) { const payloads = await this.redis.lRange(config.redisKey, 0, config.limit - 1); - const cached = normalizeGenericItems( - channel, - parseJsonList(payloads, config.parse).filter((item) => - isWithinLiveFeedLookback(channel, item) - ), - config - ); + const cached = normalizeGenericItems(channel, parseJsonList(payloads, config.parse), config); if (cached.length > 0) { this.genericItems.set(channel, cached); this.stats.genericHydrateFromRedis += 1; @@ -434,9 +428,7 @@ export class LiveStateManager { const fresh = normalizeGenericItems( channel, - (await config.fetchRecent(this.clickhouse, config.limit)).filter((item) => - isWithinLiveFeedLookback(channel, item) - ), + await config.fetchRecent(this.clickhouse, config.limit), config ); this.stats.genericHydrateFromClickHouse += 1; @@ -467,8 +459,7 @@ export class LiveStateManager { optionTypes: subscription.filters?.optionTypes, minNotional: subscription.filters?.minNotional, underlyingIds: subscription.underlying_ids, - optionContractId: subscription.option_contract_id, - sinceTs: Date.now() - LIVE_FEED_LOOKBACK_MS + optionContractId: subscription.option_contract_id }; const items = await fetchRecentOptionPrints( this.clickhouse, @@ -487,7 +478,6 @@ export class LiveStateManager { const config = this.generic.options; const limit = snapshotLimitFor(subscription, config.limit); const items = (this.genericItems.get("options") ?? []).filter((item) => - isWithinLiveFeedLookback("options", item) && matchesOptionPrintFilters(item, subscription.filters) ).slice(0, limit); return { @@ -501,7 +491,6 @@ export class LiveStateManager { const config = this.generic.flow; const limit = snapshotLimitFor(subscription, config.limit); const items = (this.genericItems.get("flow") ?? []).filter((item) => - isWithinLiveFeedLookback("flow", item) && matchesFlowPacketFilters(item, subscription.filters) ).slice(0, limit); return { @@ -516,8 +505,7 @@ export class LiveStateManager { const limit = snapshotLimitFor(subscription, config.limit); if (subscription.underlying_ids?.length) { const filters: EquityPrintQueryFilters = { - underlyingIds: subscription.underlying_ids, - sinceTs: Date.now() - LIVE_FEED_LOOKBACK_MS + underlyingIds: subscription.underlying_ids }; const items = await fetchRecentEquityPrints(this.clickhouse, limit, filters); return { @@ -527,9 +515,7 @@ export class LiveStateManager { next_before: nextBeforeForItems(items, config.cursor) }; } - const items = (this.genericItems.get("equities") ?? []).filter((item) => - isWithinLiveFeedLookback("equities", item) - ).slice(0, limit); + const items = (this.genericItems.get("equities") ?? []).slice(0, limit); return { subscription, items, @@ -568,9 +554,7 @@ export class LiveStateManager { default: { const config = this.generic[subscription.channel]; const limit = snapshotLimitFor(subscription, config.limit); - const items = (this.genericItems.get(subscription.channel) ?? []).filter((item) => - isWithinLiveFeedLookback(subscription.channel, item) - ).slice(0, limit); + const items = (this.genericItems.get(subscription.channel) ?? []).slice(0, limit); return { subscription, items, diff --git a/services/api/tests/live.test.ts b/services/api/tests/live.test.ts index 3cb789e..898d2fa 100644 --- a/services/api/tests/live.test.ts +++ b/services/api/tests/live.test.ts @@ -7,15 +7,17 @@ import { shouldFanoutLiveEvent } from "../src/live"; -const makeClickHouse = (): ClickHouseClient => +const makeClickHouse = ( + queryResolver?: (query: string) => unknown[] +): ClickHouseClient => ({ exec: async () => {}, insert: async () => {}, ping: async () => ({ success: true }), close: async () => {}, - query: async () => ({ + query: async ({ query }: { query: string }) => ({ async json() { - return [] as T; + return (queryResolver?.(query) ?? []) as T; } }) }) as ClickHouseClient; @@ -408,6 +410,160 @@ describe("LiveStateManager", () => { ]); }); + it("seeds scoped option snapshots from clickhouse rows older than 24h", async () => { + const now = Date.now(); + const staleTs = now - 25 * 60 * 60 * 1000; + const manager = new LiveStateManager( + makeClickHouse((query) => + query.includes("FROM option_prints") + ? [ + { + source_ts: staleTs, + ingest_ts: staleTs + 1, + seq: 1, + trace_id: "opt-ancient", + ts: staleTs, + option_contract_id: "AAPL-2025-01-17-200-C", + underlying_id: "AAPL", + price: 1, + size: 10, + exchange: "X", + signal_pass: true + } + ] + : [] + ), + null + ); + + const snapshot = await manager.getSnapshot({ + channel: "options", + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }); + + expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ + "opt-ancient" + ]); + expect(snapshot.next_before).toEqual({ ts: staleTs, seq: 1 }); + expect(isLiveItemFresh("options", snapshot.items[0], now)).toBe(false); + }); + + it("seeds scoped equity snapshots from clickhouse rows older than 24h", async () => { + const now = Date.now(); + const staleTs = now - 25 * 60 * 60 * 1000; + const manager = new LiveStateManager( + makeClickHouse((query) => + query.includes("FROM equity_prints") + ? [ + { + source_ts: staleTs, + ingest_ts: staleTs + 1, + seq: 1, + trace_id: "eq-ancient", + ts: staleTs, + underlying_id: "AAPL", + price: 100, + size: 10, + exchange: "X", + offExchangeFlag: false + } + ] + : [] + ), + null + ); + + const snapshot = await manager.getSnapshot({ + channel: "equities", + underlying_ids: ["AAPL"] + }); + + expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ + "eq-ancient" + ]); + expect(snapshot.next_before).toEqual({ ts: staleTs, seq: 1 }); + expect(isLiveItemFresh("equities", snapshot.items[0], now)).toBe(false); + }); + + it("hydrates retained rows older than 24h into generic live snapshots and keeps them stale", async () => { + const redis = makeRedis(); + const now = Date.now(); + const staleTs = now - 25 * 60 * 60 * 1000; + + await redis.lPush( + "live:options", + JSON.stringify({ + source_ts: staleTs, + ingest_ts: staleTs + 1, + seq: 1, + trace_id: "opt-retained", + ts: staleTs, + option_contract_id: "AAPL-2025-01-17-200-C", + underlying_id: "AAPL", + price: 1, + size: 10, + exchange: "X", + signal_pass: true + }) + ); + await redis.hSet("live:cursors", "options", JSON.stringify({ ts: staleTs, seq: 1 })); + + await redis.lPush( + "live:equities", + JSON.stringify({ + source_ts: staleTs, + ingest_ts: staleTs + 1, + seq: 2, + trace_id: "eq-retained", + ts: staleTs, + underlying_id: "AAPL", + price: 100, + size: 10, + exchange: "X", + offExchangeFlag: false + }) + ); + await redis.hSet("live:cursors", "equities", JSON.stringify({ ts: staleTs, seq: 2 })); + + await redis.lPush( + "live:flow", + JSON.stringify({ + source_ts: staleTs, + ingest_ts: staleTs + 1, + seq: 3, + trace_id: "flow-retained", + id: "flow-retained", + members: ["opt-retained"], + features: {}, + join_quality: {} + }) + ); + await redis.hSet("live:cursors", "flow", JSON.stringify({ ts: staleTs, seq: 3 })); + + const manager = new LiveStateManager(makeClickHouse(), redis as never); + await manager.hydrate(); + + const [optionsSnapshot, equitiesSnapshot, flowSnapshot] = await Promise.all([ + manager.getSnapshot({ channel: "options" }), + manager.getSnapshot({ channel: "equities" }), + manager.getSnapshot({ channel: "flow" }) + ]); + + expect((optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ + "opt-retained" + ]); + expect((equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ + "eq-retained" + ]); + expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([ + "flow-retained" + ]); + expect(isLiveItemFresh("options", optionsSnapshot.items[0], now)).toBe(false); + expect(isLiveItemFresh("equities", equitiesSnapshot.items[0], now)).toBe(false); + expect(isLiveItemFresh("flow", flowSnapshot.items[0], now)).toBe(false); + }); + it("keeps only the newest NBBO quote per contract across hydrate and ingest", async () => { const redis = makeRedis(); const now = Date.now(); From e69bf295c88e6b8899a9a3679a159e14f9a8966e Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Thu, 7 May 2026 01:52:20 -0400 Subject: [PATCH 06/15] Stabilize tape virtualization and scoped live health --- .beads/issues.jsonl | 1 + .env.example | 10 +- apps/web/app/globals.css | 13 +- apps/web/app/terminal.test.ts | 58 +++ apps/web/app/terminal.tsx | 834 ++++++++++++++++++++++---------- apps/web/package.json | 1 + bun.lock | 5 + packages/types/src/live.ts | 24 +- services/api/src/index.ts | 31 +- services/api/src/live.ts | 45 ++ services/api/tests/live.test.ts | 117 +++++ 11 files changed, 866 insertions(+), 273 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 741bac4..b7f0a79 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-2ij","title":"Harden tape virtualization, scoped focus, and live feed health","description":"Implement the coordinated tape stability plan across web and API.\n\nScope:\n- replace fixed-height tape virtualization with measured virtualization and virtual-end history loading\n- replace scrollHeight anchoring with key-based anchor restore\n- compose canonical tape lists across seed/live/history sources\n- preserve clicked contract/ticker context during scoped focus transitions\n- separate backend hot-channel health from scoped quiet empty states\n- shrink browser hot windows and modestly reduce server cache limits\n- add regression tests and development instrumentation\n\nAcceptance:\n- no giant blank spacer gaps during tape scrolling\n- scroll remains stable while live data and history mutate the list\n- clicked deep-history option/equity rows remain visible immediately after focus\n- narrow scopes do not surface Feed behind unless backend channel health is stale\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T05:35:18Z","created_by":"dirtydishes","updated_at":"2026-05-07T05:52:14Z","started_at":"2026-05-07T05:35:21Z","closed_at":"2026-05-07T05:52:14Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-uj7","title":"Fix home to tape navigation","description":"Home rail Tape navigation was not reliably switching to the tape route. Use browser-native top-level navigation for Home/Tape rail links so /tape remains reachable even if client router handling stalls.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T03:18:14Z","created_by":"dirtydishes","updated_at":"2026-05-07T03:18:21Z","started_at":"2026-05-07T03:18:20Z","closed_at":"2026-05-07T03:18:21Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-84s","title":"Implement seamless /tape live-to-history scroll gate","description":"Implement seamless live-to-ClickHouse scroll-gated history for /tape panes, including split live/history buffers in the web client, snapshot_limit support on live subscriptions, a bundled options support lookup endpoint, ClickHouse helpers for parity hydration, and test coverage for live head retention, background history loading, scoped options deep-hydration, and historical options decor restoration.\n","status":"in_progress","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T02:10:43Z","created_by":"dirtydishes","updated_at":"2026-05-07T02:10:47Z","started_at":"2026-05-07T02:10:47Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-sh1","title":"Fix live websocket stale lag and reconnect loop","description":"Investigate and fix API live consumer lag causing stale timestamps, feed-behind status, and reconnect loops. Optimize live cache persistence path, add lag telemetry/alerts, and validate in runtime.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T17:04:34Z","created_by":"dirtydishes","updated_at":"2026-05-04T17:09:44Z","started_at":"2026-05-04T17:04:38Z","closed_at":"2026-05-04T17:09:44Z","close_reason":"Completed: optimized live cache persistence path, added lag telemetry, deployed api via docker compose on di, verified ws freshness and low hotFeedLagMs","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/.env.example b/.env.example index 50f9c5a..4d5ac1b 100644 --- a/.env.example +++ b/.env.example @@ -58,8 +58,8 @@ API_DELIVER_POLICY=new API_CONSUMER_RESET=false NBBO_MAX_AGE_MS=1000 NEXT_PUBLIC_NBBO_MAX_AGE_MS=1000 -NEXT_PUBLIC_LIVE_HOT_WINDOW=2000 -NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS=25000 +NEXT_PUBLIC_LIVE_HOT_WINDOW=600 +NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS=1200 NEXT_PUBLIC_PINNED_EVIDENCE_TTL_MS=1200000 NEXT_PUBLIC_PINNED_EVIDENCE_MAX_ITEMS=4000 ROLLING_WINDOW_SIZE=50 @@ -100,12 +100,12 @@ REPLAY_BATCH_SIZE=200 REPLAY_LOG_EVERY=1000 # API live retention (generic channels) -LIVE_LIMIT_OPTIONS=10000 +LIVE_LIMIT_OPTIONS=2000 LIVE_LIMIT_NBBO=10000 -LIVE_LIMIT_EQUITIES=10000 +LIVE_LIMIT_EQUITIES=2000 LIVE_LIMIT_EQUITY_QUOTES=10000 LIVE_LIMIT_EQUITY_JOINS=10000 -LIVE_LIMIT_FLOW=10000 +LIVE_LIMIT_FLOW=2000 LIVE_LIMIT_CLASSIFIER_HITS=10000 LIVE_LIMIT_ALERTS=10000 LIVE_LIMIT_INFERRED_DARK=10000 diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index 5af91c1..ab3f6ed 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -967,6 +967,11 @@ h3 { min-width: 980px; } +.data-table-body { + position: relative; + min-width: 100%; +} + .data-table-options { min-width: 1280px; } @@ -1024,10 +1029,16 @@ h3 { text-align: left; } -.data-table-row:nth-child(even) { +.data-table-row.is-even { background: rgba(255, 255, 255, 0.022); } +.data-table-virtual-row { + position: absolute; + left: 0; + width: 100%; +} + .data-table-row:hover, .data-table-row:focus-visible { outline: none; diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index 78c7c70..16ce0ad 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -5,12 +5,15 @@ import { appendHistoryTail, buildDefaultFlowFilters, classifierToneForFamily, + composeTapeItems, deriveAlertDirection, countActiveFlowFilterGroups, + findAnchorRestoreIndex, formatCompactUsd, formatOptionContractLabel, flushPausableTapeData, getAlertWindowAnchorTs, + getHotChannelFeedStatus, getScopedLiveAutoHydrationChannels, getLiveHistoryRetentionCap, getOptionTableSnapshot, @@ -246,6 +249,37 @@ describe("live tape pausable helpers", () => { }); describe("live tape history helpers", () => { + it("composes tape items across seed, live, and history without seam duplicates", () => { + const seed = [makeItem("seed", 1, 100), makeItem("dup", 2, 200)]; + const live = [makeItem("live", 5, 500), makeItem("dup", 2, 200)]; + const history = [makeItem("old", 0, 50), makeItem("mid", 3, 300)]; + + expect(composeTapeItems(seed, live, history).map((item) => item.trace_id)).toEqual([ + "live", + "mid", + "dup", + "seed", + "old" + ]); + }); + + it("keeps a clicked seed row visible before scoped live and history arrive", () => { + const clicked = makeItem("clicked", 3, 300); + + expect(composeTapeItems([clicked], [], []).map((item) => item.trace_id)).toEqual(["clicked"]); + }); + + it("drops focus seed duplicates once equivalent live or history rows arrive", () => { + const clicked = makeItem("clicked", 3, 300); + const live = [makeItem("new", 4, 400)]; + const history = [makeItem("clicked", 3, 300)]; + + expect(composeTapeItems([clicked], live, history).map((item) => item.trace_id)).toEqual([ + "new", + "clicked" + ]); + }); + it("promotes hot-window overflow into the history tail", () => { const currentHot = [makeItem("hot-3", 3, 300), makeItem("hot-2", 2, 200), makeItem("hot-1", 1, 100)]; const incoming = [makeItem("hot-4", 4, 400)]; @@ -362,6 +396,21 @@ describe("live tape history helpers", () => { }, {}) ).toEqual(["options"]); }); + + it("restores the same anchor key after live insertions at the top", () => { + const nextKeys = ["new-1", "new-2", "anchor", "after-1", "after-2"]; + expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(2); + }); + + it("falls forward to the nearest surviving key when the anchor is evicted", () => { + const nextKeys = ["new-1", "after-1", "after-2"]; + expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(1); + }); + + it("keeps the same anchor when history is appended at the bottom", () => { + const nextKeys = ["anchor", "after-1", "after-2", "older-1", "older-2"]; + expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(0); + }); }); describe("options display formatters", () => { @@ -533,4 +582,13 @@ describe("signals helpers", () => { expect(statusLabel("connected", false, "live")).toBe("Connected"); expect(statusLabel("stale", false, "live")).toBe("Feed behind"); }); + + it("treats healthy scoped channels as connected even when no matching rows are visible", () => { + expect(getHotChannelFeedStatus("connected", { healthy: true })).toBe("connected"); + }); + + it("surfaces feed behind only when the backend channel health is stale", () => { + expect(getHotChannelFeedStatus("connected", { healthy: false })).toBe("stale"); + expect(getHotChannelFeedStatus("disconnected", { healthy: true })).toBe("disconnected"); + }); }); diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 72edbd5..2718ed7 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -17,6 +17,7 @@ import { type ReactNode, type SetStateAction } from "react"; +import { useVirtualizer, type Virtualizer } from "@tanstack/react-virtual"; import type { AlertEvent, ClassifierHitEvent, @@ -28,6 +29,7 @@ import type { FlowPacket, InferredDarkEvent, LiveServerMessage, + LiveHotChannelHealthMap, LiveSubscription, OptionFlowFilters, OptionNbboSide, @@ -62,10 +64,10 @@ const parseBoundedInt = ( return Math.max(min, Math.min(max, Math.floor(parsed))); }; -const LIVE_HOT_WINDOW = parseBoundedInt(process.env.NEXT_PUBLIC_LIVE_HOT_WINDOW, 100, 1, 100000); +const LIVE_HOT_WINDOW = parseBoundedInt(process.env.NEXT_PUBLIC_LIVE_HOT_WINDOW, 600, 1, 100000); const LIVE_HOT_WINDOW_OPTIONS = parseBoundedInt( process.env.NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS, - 100, + 1200, 1, 100000 ); @@ -145,6 +147,11 @@ type SelectedInstrument = | { kind: "equity"; underlyingId: string } | { kind: "option-contract"; contractId: string; underlyingId: string }; +type TapeFocusSeed = { + scopeKey: string; + items: T[]; +}; + const formatIntervalLabel = (intervalMs: number): string => { const match = CANDLE_INTERVALS.find((interval) => interval.ms === intervalMs); if (match) { @@ -343,6 +350,42 @@ const frontendRetentionMetrics: Record = { pinnedStoreSize: 0 }; +const DEV_TAPE_DEBUG = process.env.NODE_ENV !== "production"; + +type TapeDebugMetricKey = + | "anchorRestoreCount" + | "anchorRestoreFallbackCount" + | "virtualRowMeasurementCount" + | "focusSeedRowCount" + | "scopedQuietTransitions"; + +const frontendTapeDebugMetrics: Record = { + anchorRestoreCount: 0, + anchorRestoreFallbackCount: 0, + virtualRowMeasurementCount: 0, + focusSeedRowCount: 0, + scopedQuietTransitions: 0 +}; + +const bumpTapeDebugMetric = (key: TapeDebugMetricKey, count = 1): void => { + frontendTapeDebugMetrics[key] += count; + if (DEV_TAPE_DEBUG && typeof window !== "undefined") { + (window as typeof window & { __IF_TAPE_DEBUG__?: Record }).__IF_TAPE_DEBUG__ = + frontendTapeDebugMetrics; + } +}; + +const logTapeDebug = (message: string, payload?: Record): void => { + if (!DEV_TAPE_DEBUG) { + return; + } + if (payload) { + console.debug(`[tape] ${message}`, payload); + return; + } + console.debug(`[tape] ${message}`); +}; + const incrementRetentionMetric = (key: RetentionMetricKey, count = 1): void => { frontendRetentionMetrics[key] += count; }; @@ -426,6 +469,24 @@ const getTapeItemKey = (item: SortableItem): string => { return buildItemKey(item) ?? `${extractSortTs(item)}:${extractSortSeq(item)}`; }; +export const composeTapeItems = ( + seedItems: T[], + liveItems: T[], + historyItems: T[] +): T[] => { + const deduped = new Map(); + for (const item of [...seedItems, ...liveItems, ...historyItems]) { + deduped.set(getTapeItemKey(item), item); + } + return Array.from(deduped.values()).sort((a, b) => { + const delta = extractSortTs(b) - extractSortTs(a); + if (delta !== 0) { + return delta; + } + return extractSortSeq(b) - extractSortSeq(a); + }); +}; + type PausableTapeData = { visible: T[]; queued: T[]; @@ -618,9 +679,45 @@ export const getLiveFeedStatus = ( return behindMs > behindDelayMs ? "stale" : "connected"; }; +export const getHotChannelFeedStatus = ( + sourceStatus: WsStatus, + health: { healthy: boolean } | null | undefined +): WsStatus => { + if (sourceStatus !== "connected") { + return sourceStatus; + } + if (!health) { + return "connected"; + } + return health.healthy ? "connected" : "stale"; +}; + +export const findAnchorRestoreIndex = ( + keys: string[], + anchorKey: string, + fallbackKeys: string[] +): number => { + const directIndex = keys.indexOf(anchorKey); + if (directIndex >= 0) { + return directIndex; + } + + const indexByKey = new Map(keys.map((key, index) => [key, index])); + for (const key of fallbackKeys) { + const index = indexByKey.get(key); + if (typeof index === "number") { + return index; + } + } + + return -1; +}; + type TapeState = { status: WsStatus; items: T[]; + liveItems?: T[]; + historyItems?: T[]; lastUpdate: number | null; replayTime: number | null; replayComplete: boolean; @@ -1380,7 +1477,26 @@ const useScrollAnchor = ( listRef: React.RefObject, isAtTopRef: React.MutableRefObject ) => { - const pendingRef = useRef<{ height: number } | null>(null); + const pendingRef = useRef<{ + key: string; + offset: number; + fallbackKeys: string[]; + } | null>(null); + + const readRenderedRows = useCallback((element: HTMLDivElement) => { + return Array.from(element.querySelectorAll("[data-tape-key][data-row-start][data-row-size]")) + .map((node) => { + const key = node.dataset.tapeKey; + const start = Number(node.dataset.rowStart); + const size = Number(node.dataset.rowSize); + if (!key || !Number.isFinite(start) || !Number.isFinite(size)) { + return null; + } + return { key, start, size }; + }) + .filter((row): row is { key: string; start: number; size: number } => row !== null) + .sort((a, b) => a.start - b.start); + }, []); const capture = useCallback(() => { if (isAtTopRef.current) { @@ -1393,10 +1509,27 @@ const useScrollAnchor = ( return; } + const rows = readRenderedRows(el); + if (rows.length === 0) { + pendingRef.current = null; + return; + } + + const scrollTop = el.scrollTop; + const anchorIndex = rows.findIndex((row) => row.start + row.size > scrollTop); + const resolvedIndex = anchorIndex >= 0 ? anchorIndex : 0; + const anchorRow = rows[resolvedIndex]; + if (!anchorRow) { + pendingRef.current = null; + return; + } + pendingRef.current = { - height: el.scrollHeight + key: anchorRow.key, + offset: Math.max(0, scrollTop - anchorRow.start), + fallbackKeys: rows.slice(resolvedIndex).map((row) => row.key) }; - }, [isAtTopRef, listRef]); + }, [isAtTopRef, listRef, readRenderedRows]); const apply = useCallback(() => { const pending = pendingRef.current; @@ -1414,20 +1547,41 @@ const useScrollAnchor = ( return; } - const delta = el.scrollHeight - pending.height; - if (delta !== 0) { - el.scrollTop = Math.max(0, el.scrollTop + delta); + const rows = readRenderedRows(el); + if (rows.length === 0) { + return; + } + + const keys = rows.map((row) => row.key); + const restoreIndex = findAnchorRestoreIndex(keys, pending.key, pending.fallbackKeys); + if (restoreIndex < 0) { + return; + } + + const row = rows[restoreIndex]; + if (!row) { + return; + } + + el.scrollTop = Math.max(0, row.start + pending.offset); + bumpTapeDebugMetric("anchorRestoreCount", 1); + if (row.key !== pending.key) { + bumpTapeDebugMetric("anchorRestoreFallbackCount", 1); + logTapeDebug("anchor restore fallback", { + requested_key: pending.key, + restored_key: row.key + }); } pendingRef.current = null; - }, [isAtTopRef, listRef]); + }, [isAtTopRef, listRef, readRenderedRows]); return { capture, apply }; }; -const useBottomHistoryGate = ( - listRef: React.RefObject, - listNode: HTMLDivElement | null, +const useVirtualHistoryGate = ( enabled: boolean, + itemCount: number, + lastVirtualIndex: number, onLoadOlder: () => void ): void => { const loadRef = useRef(onLoadOlder); @@ -1436,107 +1590,97 @@ const useBottomHistoryGate = ( }, [onLoadOlder]); useEffect(() => { - if (!enabled) { + if (!enabled || itemCount === 0) { return; } - const element = listNode ?? listRef.current; - if (!element) { + if (lastVirtualIndex < itemCount - 1) { return; } - - const maybeLoad = () => { - const threshold = Math.max(240, element.clientHeight * 0.5); - if (element.scrollTop + element.clientHeight >= element.scrollHeight - threshold) { - loadRef.current(); - } - }; - - maybeLoad(); - element.addEventListener("scroll", maybeLoad); - return () => { - element.removeEventListener("scroll", maybeLoad); - }; - }, [enabled, listNode, listRef]); + loadRef.current(); + }, [enabled, itemCount, lastVirtualIndex]); }; -type VirtualListResult = { - visibleItems: T[]; - topSpacerHeight: number; - bottomSpacerHeight: number; +type MeasuredVirtualListResult = { + totalSize: number; + virtualItems: MeasuredVirtualRow[]; + measureElement: (node: HTMLElement | null) => void; + virtualizer: Virtualizer; }; -const useVirtualList = ( +type MeasuredVirtualRow = { + item: T; + key: string; + index: number; + start: number; + size: number; + end: number; +}; + +const useMeasuredVirtualList = ( items: T[], listRef: React.RefObject, - enabled: boolean, - rowHeight: number, - overscan = 8 -): VirtualListResult => { - const [range, setRange] = useState<{ start: number; end: number }>({ - start: 0, - end: items.length + estimateSize: number, + overscan: number, + debugLabel: string +): MeasuredVirtualListResult => { + const virtualizer = useVirtualizer({ + count: items.length, + getScrollElement: () => listRef.current, + estimateSize: () => estimateSize, + overscan, + getItemKey: (index) => getTapeItemKey(items[index] as SortableItem), + measureElement: (node) => { + bumpTapeDebugMetric("virtualRowMeasurementCount", 1); + return node.getBoundingClientRect().height; + } }); - const recompute = useCallback(() => { - if (!enabled) { - setRange({ start: 0, end: items.length }); - return; + const virtualItems: MeasuredVirtualRow[] = virtualizer.getVirtualItems().map((virtualItem) => { + const item = items[virtualItem.index] as T | undefined; + if (!item) { + return null; } - - const element = listRef.current; - if (!element) { - setRange({ start: 0, end: Math.min(items.length, 80) }); - return; - } - - const viewportHeight = Math.max(rowHeight, element.clientHeight); - const visibleCount = Math.ceil(viewportHeight / rowHeight); - const start = Math.max(0, Math.floor(element.scrollTop / rowHeight) - overscan); - const end = Math.min(items.length, start + visibleCount + overscan * 2); - setRange({ start, end }); - }, [enabled, items.length, listRef, overscan, rowHeight]); - - useEffect(() => { - recompute(); - }, [items.length, recompute]); - - useEffect(() => { - if (!enabled) { - return; - } - - const element = listRef.current; - if (!element) { - return; - } - - const onScroll = () => recompute(); - const onResize = () => recompute(); - - element.addEventListener("scroll", onScroll); - window.addEventListener("resize", onResize); - - return () => { - element.removeEventListener("scroll", onScroll); - window.removeEventListener("resize", onResize); - }; - }, [enabled, listRef, recompute]); - - if (!enabled) { return { - visibleItems: items, - topSpacerHeight: 0, - bottomSpacerHeight: 0 + item, + key: getTapeItemKey(item), + index: virtualItem.index, + start: virtualItem.start, + size: virtualItem.size, + end: virtualItem.end }; - } + }).filter((virtualItem): virtualItem is MeasuredVirtualRow => virtualItem !== null); - const start = Math.min(range.start, items.length); - const end = Math.min(Math.max(range.end, start), items.length); + useEffect(() => { + if (!DEV_TAPE_DEBUG || items.length === 0) { + return; + } + const element = listRef.current; + if (!element) { + return; + } + const first = virtualItems[0]; + const last = virtualItems.at(-1); + if (!first || !last) { + return; + } + const visibleTopGap = Math.max(0, first.start - element.scrollTop); + const visibleBottomGap = Math.max(0, element.scrollTop + element.clientHeight - last.end); + if (visibleTopGap > element.clientHeight || visibleBottomGap > element.clientHeight) { + console.warn("[tape] false-gap watchdog", { + pane: debugLabel, + item_count: items.length, + visible_top_gap: visibleTopGap, + visible_bottom_gap: visibleBottomGap, + viewport_height: element.clientHeight + }); + } + }, [debugLabel, items.length, listRef, virtualItems]); return { - visibleItems: items.slice(start, end), - topSpacerHeight: start * rowHeight, - bottomSpacerHeight: Math.max(0, (items.length - end) * rowHeight) + totalSize: virtualizer.getTotalSize(), + virtualItems, + measureElement: virtualizer.measureElement, + virtualizer }; }; @@ -2018,6 +2162,8 @@ const toStaticTapeState = ( ): TapeState => ({ status, items, + liveItems: items, + historyItems: [], lastUpdate, replayTime: null, replayComplete: false, @@ -2032,10 +2178,8 @@ type PausableTapeViewConfig = { sourceItems: T[]; historyTail?: T[]; lastUpdate: number | null; - freshnessMs: number; onNewItems?: (count: number) => void; captureScroll?: () => void; - getItemTs?: (item: T) => number; retentionLimit?: number; shouldHold?: () => boolean; resumeSignal?: number; @@ -2046,17 +2190,6 @@ const usePausableTapeView = ( ): TapeState => { const [paused, setPaused] = useState(false); const [data, setData] = useState>(EMPTY_PAUSABLE_TAPE); - const [clock, setClock] = useState(() => Date.now()); - - useEffect(() => { - const handle = window.setInterval(() => { - setClock(Date.now()); - }, 1000); - - return () => { - window.clearInterval(handle); - }; - }, []); useEffect(() => { if (!config.enabled) { @@ -2132,38 +2265,16 @@ const usePausableTapeView = ( setPaused((current) => !current); }, []); - const getItemTs = config.getItemTs ?? extractSortTs; - const freshestTs = useMemo(() => { - if (config.sourceItems.length === 0) { - return null; - } - - let newest = Number.NEGATIVE_INFINITY; - for (const item of config.sourceItems) { - newest = Math.max(newest, getItemTs(item)); - } - - return Number.isFinite(newest) ? newest : null; - }, [config.sourceItems, getItemTs]); - - const status = config.enabled - ? getLiveFeedStatus( - config.sourceStatus, - freshestTs, - config.freshnessMs, - clock, - LIVE_FEED_BEHIND_DELAY_MS - ) - : "disconnected"; + const status = config.enabled ? config.sourceStatus : "disconnected"; const projected = projectPausableTapeState(data.visible, status, config.lastUpdate); - const items = useMemo( - () => [...projected.items, ...(config.historyTail ?? [])], - [projected.items, config.historyTail] - ); + const historyItems = config.historyTail ?? []; + const items = useMemo(() => composeTapeItems([], projected.items, historyItems), [projected.items, historyItems]); return { status, items, + liveItems: projected.items, + historyItems, lastUpdate: projected.lastUpdate, replayTime: null, replayComplete: false, @@ -2412,6 +2523,7 @@ type LiveSessionState = { status: WsStatus; connectedAt: number | null; lastUpdate: number | null; + channelHealth: LiveHotChannelHealthMap; lastEventByChannel: Partial>; manifest: LiveSubscription[]; historyCursors: Partial>; @@ -2561,6 +2673,12 @@ const useLiveSession = ( const [status, setStatus] = useState(enabled ? "connecting" : "disconnected"); const [connectedAt, setConnectedAt] = useState(null); const [lastUpdate, setLastUpdate] = useState(null); + const [channelHealth, setChannelHealth] = useState({ + options: { freshness_age_ms: null, healthy: false }, + nbbo: { freshness_age_ms: null, healthy: false }, + equities: { freshness_age_ms: null, healthy: false }, + flow: { freshness_age_ms: null, healthy: false } + }); const [lastEventByChannel, setLastEventByChannel] = useState< Partial> >({}); @@ -2647,6 +2765,12 @@ const useLiveSession = ( setStatus("disconnected"); setConnectedAt(null); setLastUpdate(null); + setChannelHealth({ + options: { freshness_age_ms: null, healthy: false }, + nbbo: { freshness_age_ms: null, healthy: false }, + equities: { freshness_age_ms: null, healthy: false }, + flow: { freshness_age_ms: null, healthy: false } + }); setLastEventByChannel({}); setHistoryCursors({}); setHistoryLoading({}); @@ -2736,6 +2860,7 @@ const useLiveSession = ( const handleMessage = (message: LiveServerMessage) => { if (message.op === "ready" || message.op === "heartbeat") { + setChannelHealth(message.channel_health); return; } if (message.op === "error") { @@ -3173,6 +3298,7 @@ const useLiveSession = ( status, connectedAt, lastUpdate, + channelHealth, lastEventByChannel, manifest, historyCursors, @@ -4512,6 +4638,8 @@ const useTerminalState = () => { const [selectedClassifierHit, setSelectedClassifierHit] = useState(null); const [selectedSmartMoneyEvent, setSelectedSmartMoneyEvent] = useState(null); const [selectedInstrument, setSelectedInstrument] = useState(null); + const [optionFocusSeed, setOptionFocusSeed] = useState | null>(null); + const [equityFocusSeed, setEquityFocusSeed] = useState | null>(null); const [filterInput, setFilterInput] = useState(""); const [flowFilters, setFlowFilters] = useState(() => buildDefaultFlowFilters()); const [chartIntervalMs, setChartIntervalMs] = useState(CANDLE_INTERVALS[0].ms); @@ -4524,6 +4652,14 @@ const useTerminalState = () => { }, [filterInput]); const tickerSet = useMemo(() => new Set(activeTickers), [activeTickers]); const instrumentUnderlying = selectedInstrument?.underlyingId.toUpperCase() ?? null; + const optionFocusScopeKey = + selectedInstrument?.kind === "option-contract" + ? `option-contract:${selectedInstrument.contractId}` + : null; + const equityFocusScopeKey = + selectedInstrument?.kind === "equity" + ? `equity:${selectedInstrument.underlyingId.toUpperCase()}` + : null; const optionScope = useMemo( () => ({ underlying_ids: activeTickers.length > 0 ? activeTickers : instrumentUnderlying ? [instrumentUnderlying] : undefined, @@ -4767,13 +4903,19 @@ const useTerminalState = () => { getReplayKey: disableReplayGrouping }); + const optionsChannelStatus = getHotChannelFeedStatus(liveSession.status, liveSession.channelHealth.options); + const equitiesChannelStatus = getHotChannelFeedStatus( + liveSession.status, + liveSession.channelHealth.equities + ); + const flowChannelStatus = getHotChannelFeedStatus(liveSession.status, liveSession.channelHealth.flow); + const liveOptions = usePausableTapeView({ enabled: mode === "live", - sourceStatus: liveSession.status, + sourceStatus: optionsChannelStatus, sourceItems: liveSession.options, historyTail: liveSession.optionsHistory, lastUpdate: liveSession.lastUpdate, - freshnessMs: LIVE_OPTIONS_STALE_MS, retentionLimit: LIVE_HOT_WINDOW_OPTIONS, captureScroll: optionsAnchor.capture, onNewItems: optionsScroll.onNewItems, @@ -4782,11 +4924,10 @@ const useTerminalState = () => { }); const liveEquities = usePausableTapeView({ enabled: mode === "live", - sourceStatus: liveSession.status, + sourceStatus: equitiesChannelStatus, sourceItems: liveSession.equities, historyTail: liveSession.equitiesHistory, lastUpdate: liveSession.lastUpdate, - freshnessMs: LIVE_EQUITIES_STALE_MS, captureScroll: equitiesAnchor.capture, onNewItems: equitiesScroll.onNewItems, shouldHold: () => !equitiesScroll.isAtTopRef.current, @@ -4794,40 +4935,87 @@ const useTerminalState = () => { }); const liveFlow = usePausableTapeView({ enabled: mode === "live", - sourceStatus: liveSession.status, + sourceStatus: flowChannelStatus, sourceItems: liveSession.flow, historyTail: liveSession.flowHistory, lastUpdate: liveSession.lastUpdate, - freshnessMs: LIVE_FLOW_STALE_MS, captureScroll: flowAnchor.capture, onNewItems: flowScroll.onNewItems, shouldHold: () => !flowScroll.isAtTopRef.current, - resumeSignal: flowScroll.resumeTick, - getItemTs: (item) => item.source_ts + resumeSignal: flowScroll.resumeTick }); - const optionsFeed = mode === "live" ? liveOptions : options; + const seededLiveOptionsItems = useMemo( + () => + composeTapeItems( + optionFocusSeed?.scopeKey === optionFocusScopeKey ? optionFocusSeed.items : [], + liveOptions.liveItems ?? [], + liveOptions.historyItems ?? [] + ), + [liveOptions.historyItems, liveOptions.liveItems, optionFocusScopeKey, optionFocusSeed] + ); + const seededLiveEquitiesItems = useMemo( + () => + composeTapeItems( + equityFocusSeed?.scopeKey === equityFocusScopeKey ? equityFocusSeed.items : [], + liveEquities.liveItems ?? [], + liveEquities.historyItems ?? [] + ), + [equityFocusScopeKey, equityFocusSeed, liveEquities.historyItems, liveEquities.liveItems] + ); + + const optionsFeed = + mode === "live" ? { ...liveOptions, items: seededLiveOptionsItems } : options; const nbboFeed = - mode === "live" ? toStaticTapeState(liveSession.status, [...liveSession.nbbo, ...liveSession.nbboHistory], liveSession.lastUpdate) : nbbo; - const equitiesFeed = mode === "live" ? liveEquities : equities; + mode === "live" + ? toStaticTapeState( + getHotChannelFeedStatus(liveSession.status, liveSession.channelHealth.nbbo), + composeTapeItems([], liveSession.nbbo, liveSession.nbboHistory), + liveSession.lastUpdate + ) + : nbbo; + const equitiesFeed = + mode === "live" ? { ...liveEquities, items: seededLiveEquitiesItems } : equities; const equityJoinsFeed = mode === "live" - ? toStaticTapeState(liveSession.status, [...liveSession.equityJoins, ...liveSession.equityJoinsHistory], liveSession.lastUpdate) + ? toStaticTapeState( + liveSession.status, + composeTapeItems([], liveSession.equityJoins, liveSession.equityJoinsHistory), + liveSession.lastUpdate + ) : equityJoins; const flowFeed = mode === "live" ? liveFlow : flow; const alertsFeed = - mode === "live" ? toStaticTapeState(liveSession.status, [...liveSession.alerts, ...liveSession.alertsHistory], liveSession.lastUpdate) : alerts; + mode === "live" + ? toStaticTapeState( + liveSession.status, + composeTapeItems([], liveSession.alerts, liveSession.alertsHistory), + liveSession.lastUpdate + ) + : alerts; const classifierHitsFeed = mode === "live" - ? toStaticTapeState(liveSession.status, [...liveSession.classifierHits, ...liveSession.classifierHitsHistory], liveSession.lastUpdate) + ? toStaticTapeState( + liveSession.status, + composeTapeItems([], liveSession.classifierHits, liveSession.classifierHitsHistory), + liveSession.lastUpdate + ) : classifierHits; const smartMoneyFeed = mode === "live" - ? toStaticTapeState(liveSession.status, [...liveSession.smartMoney, ...liveSession.smartMoneyHistory], liveSession.lastUpdate) + ? toStaticTapeState( + liveSession.status, + composeTapeItems([], liveSession.smartMoney, liveSession.smartMoneyHistory), + liveSession.lastUpdate + ) : smartMoney; const inferredDarkFeed = mode === "live" - ? toStaticTapeState(liveSession.status, [...liveSession.inferredDark, ...liveSession.inferredDarkHistory], liveSession.lastUpdate) + ? toStaticTapeState( + liveSession.status, + composeTapeItems([], liveSession.inferredDark, liveSession.inferredDarkHistory), + liveSession.lastUpdate + ) : inferredDark; useLayoutEffect(() => { @@ -5528,12 +5716,126 @@ const useTerminalState = () => { return equitiesFeed.items.filter((print) => matchesTicker(print.underlying_id)); }, [equitiesFeed.items, matchesTicker, tickerSet, instrumentUnderlying]); + useEffect(() => { + if (!optionFocusSeed) { + return; + } + if (optionFocusSeed.scopeKey !== optionFocusScopeKey) { + setOptionFocusSeed(null); + return; + } + const composedBaseItems = composeTapeItems([], liveOptions.liveItems ?? [], liveOptions.historyItems ?? []); + const liveKeys = new Set(composedBaseItems.map((item) => getTapeItemKey(item))); + if (optionFocusSeed.items.every((item) => liveKeys.has(getTapeItemKey(item)))) { + setOptionFocusSeed(null); + } + }, [liveOptions.historyItems, liveOptions.liveItems, optionFocusScopeKey, optionFocusSeed]); + + useEffect(() => { + if (!equityFocusSeed) { + return; + } + if (equityFocusSeed.scopeKey !== equityFocusScopeKey) { + setEquityFocusSeed(null); + return; + } + const composedBaseItems = composeTapeItems([], liveEquities.liveItems ?? [], liveEquities.historyItems ?? []); + const liveKeys = new Set(composedBaseItems.map((item) => getTapeItemKey(item))); + if (equityFocusSeed.items.every((item) => liveKeys.has(getTapeItemKey(item)))) { + setEquityFocusSeed(null); + } + }, [equityFocusScopeKey, equityFocusSeed, liveEquities.historyItems, liveEquities.liveItems]); + + const focusOptionContract = useCallback( + (print: OptionPrint) => { + const contractId = normalizeContractId(print.option_contract_id); + const parsed = parseOptionContractId(contractId); + const underlyingId = (print.underlying_id ?? parsed?.root ?? extractUnderlying(contractId)).toUpperCase(); + const scopeKey = `option-contract:${contractId}`; + const seedItems = composeTapeItems( + [print], + filteredOptions.filter((candidate) => normalizeContractId(candidate.option_contract_id) === contractId), + [] + ); + setOptionFocusSeed({ scopeKey, items: seedItems }); + bumpTapeDebugMetric("focusSeedRowCount", seedItems.length); + logTapeDebug("option focus seed captured", { + contract_id: contractId, + row_count: seedItems.length + }); + setSelectedInstrument({ + kind: "option-contract", + contractId, + underlyingId + }); + }, + [filteredOptions] + ); + + const focusEquityTicker = useCallback( + (print: EquityPrint) => { + const underlyingId = print.underlying_id.toUpperCase(); + const scopeKey = `equity:${underlyingId}`; + const seedItems = composeTapeItems( + [print], + filteredEquities.filter((candidate) => candidate.underlying_id.toUpperCase() === underlyingId), + [] + ); + setEquityFocusSeed({ scopeKey, items: seedItems }); + bumpTapeDebugMetric("focusSeedRowCount", seedItems.length); + logTapeDebug("equity focus seed captured", { + underlying_id: underlyingId, + row_count: seedItems.length + }); + setSelectedInstrument({ + kind: "equity", + underlyingId + }); + }, + [filteredEquities] + ); + const equitiesSilentWarning = shouldShowEquitiesSilentFeedWarning({ wsStatus: liveSession.status, equitiesSubscribed: mode === "live" && equitiesLiveSubscriptionActive, connectedAt: liveSession.connectedAt, lastEquitiesEventAt: liveSession.lastEventByChannel.equities ?? null }); + const optionsScopeActive = Boolean( + optionScope.option_contract_id || optionScope.underlying_ids?.length + ); + const equitiesScopeActive = Boolean(equityScope.underlying_ids?.length); + const optionsScopedQuiet = + mode === "live" && + optionsScopeActive && + optionsChannelStatus === "connected" && + filteredOptions.length === 0; + const equitiesScopedQuiet = + mode === "live" && + equitiesScopeActive && + equitiesChannelStatus === "connected" && + filteredEquities.length === 0; + + const previousScopedQuietRef = useRef({ + options: optionsScopedQuiet, + equities: equitiesScopedQuiet + }); + + useEffect(() => { + const previous = previousScopedQuietRef.current; + if (previous.options !== optionsScopedQuiet) { + bumpTapeDebugMetric("scopedQuietTransitions", 1); + logTapeDebug("options scoped quiet transition", { active: optionsScopedQuiet }); + } + if (previous.equities !== equitiesScopedQuiet) { + bumpTapeDebugMetric("scopedQuietTransitions", 1); + logTapeDebug("equities scoped quiet transition", { active: equitiesScopedQuiet }); + } + previousScopedQuietRef.current = { + options: optionsScopedQuiet, + equities: equitiesScopedQuiet + }; + }, [equitiesScopedQuiet, optionsScopedQuiet]); const filteredInferredDark = useMemo(() => { if (tickerSet.size === 0) { @@ -5924,6 +6226,8 @@ const useTerminalState = () => { selectedSmartMoneyEvidence, filteredOptions, filteredEquities, + optionsScopedQuiet, + equitiesScopedQuiet, equitiesSilentWarning, filteredInferredDark, filteredFlow, @@ -5932,6 +6236,8 @@ const useTerminalState = () => { filteredClassifierHits, chartSmartMoneyEvents, chartInferredDark, + focusOptionContract, + focusEquityTicker, openFromSmartMoneyEvent, openFromClassifierHit, handleSmartMoneyMarkerClick, @@ -6257,8 +6563,8 @@ type OptionsPaneProps = { const OptionsPane = ({ limit }: OptionsPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredOptions.slice(0, limit) : state.filteredOptions; - const virtual = useVirtualList(items, state.optionsScroll.listRef, !limit, 36); - useBottomHistoryGate(state.optionsScroll.listRef, state.optionsScroll.listNode, state.mode === "live" && !limit, () => + const virtual = useMeasuredVirtualList(items, state.optionsScroll.listRef, 36, 12, "options"); + useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("options") ); @@ -6289,12 +6595,16 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => {
{items.length === 0 ? (
- {state.tickerSet.size > 0 - ? "No option prints match the current filter." - : state.mode === "live" - ? state.options.status === "stale" - ? "Feed behind. Waiting for fresh option prints." - : "No option prints yet. Start ingest-options." + {state.mode === "live" + ? state.options.status === "stale" + ? "Feed behind. Waiting for fresh option prints." + : state.optionsScopedQuiet + ? "No recent option prints for this scope yet." + : state.tickerSet.size > 0 + ? "No option prints match the current filter." + : "No option prints yet. Start ingest-options." + : state.tickerSet.size > 0 + ? "No option prints match the current filter." : "Replay queue empty. Ensure ClickHouse has data."}
) : ( @@ -6314,10 +6624,12 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { IV CLASSIFIER
- {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {virtual.visibleItems.map((print) => { +
+ {virtual.virtualItems.map(({ item: print, key, index, start, size }) => { const contractId = normalizeContractId(print.option_contract_id); const parsed = parseOptionContractId(contractId); const contractDisplay = formatOptionContractLabel(contractId); @@ -6334,15 +6646,21 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { const underlyingId = (print.underlying_id ?? parsed?.root ?? extractUnderlying(contractId)).toUpperCase(); const focusContract = (event: ReactMouseEvent) => { event.stopPropagation(); - state.setSelectedInstrument({ - kind: "option-contract", - contractId, - underlyingId - }); + state.focusOptionContract(print); }; + const rowStyle = { + ...(decor + ? ({ "--classifier-intensity": decor.intensity } as CSSProperties) + : undefined), + top: `${start}px` + } as CSSProperties; const commonProps = { - className: `data-table-row data-table-row-button data-table-row-classified data-table-row-options${decor ? ` is-classified classifier-${decor.tone}` : ""}`, - style: decor ? ({ "--classifier-intensity": decor.intensity } as CSSProperties) : undefined + className: `data-table-row data-table-row-button data-table-row-classified data-table-row-options data-table-virtual-row${index % 2 === 1 ? " is-even" : ""}${decor ? ` is-classified classifier-${decor.tone}` : ""}`, + style: rowStyle, + "data-row-start": String(start), + "data-row-size": String(size), + "data-tape-key": key, + ref: virtual.measureElement }; const cells = ( <> @@ -6389,7 +6707,7 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { ) : ( -
+
{cells}
); })} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null} +
)} @@ -6434,8 +6750,8 @@ type EquitiesPaneProps = { const EquitiesPane = ({ limit }: EquitiesPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredEquities.slice(0, limit) : state.filteredEquities; - const virtual = useVirtualList(items, state.equitiesScroll.listRef, !limit, 36); - useBottomHistoryGate(state.equitiesScroll.listRef, state.equitiesScroll.listNode, state.mode === "live" && !limit, () => + const virtual = useMeasuredVirtualList(items, state.equitiesScroll.listRef, 36, 10, "equities"); + useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("equities") ); @@ -6466,14 +6782,18 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => {
{items.length === 0 ? (
- {state.tickerSet.size > 0 - ? "No equity prints match the current filter." - : state.mode === "live" - ? state.equitiesSilentWarning - ? "Connected but no equity prints received. Check ingest-equities." - : state.equities.status === "stale" - ? "Feed behind. Waiting for fresh equity prints." - : "No equity prints yet. Start ingest-equities." + {state.mode === "live" + ? state.equities.status === "stale" + ? "Feed behind. Waiting for fresh equity prints." + : state.equitiesScopedQuiet + ? "No recent equity prints for this scope yet." + : state.tickerSet.size > 0 + ? "No equity prints match the current filter." + : state.equitiesSilentWarning + ? "Connected but no equity prints received. Check ingest-equities." + : "No equity prints yet. Start ingest-equities." + : state.tickerSet.size > 0 + ? "No equity prints match the current filter." : "Replay queue empty. Ensure ClickHouse has data."}
) : ( @@ -6487,22 +6807,23 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => { VENUE TAPE
- {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {virtual.visibleItems.map((print) => ( -
+
+ {virtual.virtualItems.map(({ item: print, key, index, start, size }) => ( +
{formatTime(print.ts)} @@ -6513,9 +6834,7 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => { {print.offExchangeFlag ? "Off-Ex" : "Lit"}
))} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null} +
)} @@ -6532,8 +6851,8 @@ type FlowPaneProps = { const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredFlow.slice(0, limit) : state.filteredFlow; - const virtual = useVirtualList(items, state.flowScroll.listRef, !limit, 44); - useBottomHistoryGate(state.flowScroll.listRef, state.flowScroll.listNode, state.mode === "live" && !limit, () => + const virtual = useMeasuredVirtualList(items, state.flowScroll.listRef, 44, 8, "flow"); + useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("flow") ); @@ -6586,10 +6905,8 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { NBBO QUALITY
- {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {virtual.visibleItems.map((packet) => { +
+ {virtual.virtualItems.map(({ item: packet, key, index, start, size }) => { const features = packet.features ?? {}; const contract = String(features.option_contract_id ?? packet.id ?? "unknown"); const count = parseNumber(features.count, packet.members.length); @@ -6641,7 +6958,15 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { ].filter(Boolean).join(" | "); return ( -
+
{formatTime(startTs)} → {formatTime(endTs)} {contract} {formatFlowMetric(count)} @@ -6654,9 +6979,7 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => {
); })} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null} +
)} @@ -6674,8 +6997,8 @@ type AlertsPaneProps = { const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredAlerts.slice(0, limit) : state.filteredAlerts; - const virtual = useVirtualList(items, state.alertsScroll.listRef, !limit, 46); - useBottomHistoryGate(state.alertsScroll.listRef, state.alertsScroll.listNode, state.mode === "live" && !limit, () => + const virtual = useMeasuredVirtualList(items, state.alertsScroll.listRef, 46, 8, "alerts"); + useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("alerts") ); @@ -6726,19 +7049,22 @@ const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => DIR NOTE
- {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {virtual.visibleItems.map((alert) => { +
+ {virtual.virtualItems.map(({ item: alert, key, index, start, size }) => { const primary = alert.hits[0]; const direction = deriveAlertDirection(alert); const severity = normalizeAlertSeverity(alert); return ( ); })} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null} +
)} @@ -6774,10 +7098,6 @@ type ClassifierPaneProps = { const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { const state = useTerminal(); - useBottomHistoryGate(state.classifierScroll.listRef, state.classifierScroll.listNode, state.mode === "live" && !limit, () => { - void state.liveSession.loadOlder("smart-money"); - void state.liveSession.loadOlder("classifier-hits"); - }); const smartMoneyItems = limit ? state.filteredSmartMoneyEvents.slice(0, limit) : state.filteredSmartMoneyEvents; const legacyItems = smartMoneyItems.length === 0 @@ -6787,12 +7107,11 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { : []; const items: Array = smartMoneyItems.length > 0 ? smartMoneyItems : legacyItems; - const virtual = useVirtualList( - items, - state.classifierScroll.listRef, - !limit, - 44 - ); + const virtual = useMeasuredVirtualList(items, state.classifierScroll.listRef, 44, 8, "classifier"); + useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => { + void state.liveSession.loadOlder("smart-money"); + void state.liveSession.loadOlder("classifier-hits"); + }); const showingSmartMoney = smartMoneyItems.length > 0; return ( @@ -6839,19 +7158,23 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { PROB NOTE
- {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {showingSmartMoney ? (virtual.visibleItems as SmartMoneyEvent[]).map((event) => { +
+ {showingSmartMoney ? virtual.virtualItems.map(({ item, key, index, start, size }) => { + const event = item as SmartMoneyEvent; const primaryScore = event.profile_scores.find((score) => score.profile_id === event.primary_profile_id) ?? event.profile_scores[0]; const direction = normalizeDirection(event.primary_direction); return ( ); - }) : (virtual.visibleItems as ClassifierHitEvent[]).map((hit) => { + }) : virtual.virtualItems.map(({ item, key, index, start, size }) => { + const hit = item as ClassifierHitEvent; const direction = normalizeDirection(hit.direction); return ( ); })} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null} +
)} @@ -6903,8 +7230,8 @@ type DarkPaneProps = { const DarkPane = ({ limit, className }: DarkPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredInferredDark.slice(0, limit) : state.filteredInferredDark; - const virtual = useVirtualList(items, state.darkScroll.listRef, !limit, 44); - useBottomHistoryGate(state.darkScroll.listRef, state.darkScroll.listNode, state.mode === "live" && !limit, () => + const virtual = useMeasuredVirtualList(items, state.darkScroll.listRef, 44, 8, "dark"); + useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("inferred-dark") ); @@ -6953,18 +7280,21 @@ const DarkPane = ({ limit, className }: DarkPaneProps) => { EVIDENCE NOTE - {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {virtual.visibleItems.map((event) => { +
+ {virtual.virtualItems.map(({ item: event, key, index, start, size }) => { const underlying = inferDarkUnderlying(event, state.equityPrintMap, state.equityJoinMap); const evidenceCount = event.evidence_refs.length; return ( ); })} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null} +
)} diff --git a/apps/web/package.json b/apps/web/package.json index b61eb2e..8ab6906 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -9,6 +9,7 @@ }, "dependencies": { "@islandflow/types": "workspace:*", + "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", "next": "^14.2.4", "react": "^18.3.1", diff --git a/bun.lock b/bun.lock index de67cb2..47fc572 100644 --- a/bun.lock +++ b/bun.lock @@ -12,6 +12,7 @@ "name": "@islandflow/web", "dependencies": { "@islandflow/types": "workspace:*", + "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", "next": "^14.2.4", "react": "^18.3.1", @@ -208,6 +209,10 @@ "@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="], + "@tanstack/react-virtual": ["@tanstack/react-virtual@3.13.24", "", { "dependencies": { "@tanstack/virtual-core": "3.14.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aIJvz5OSkhNIhZIpYivrxrPTKYsjW9Uzy+sP/mx0S3sev2HyvPb7xmjbYvokzEpfgYHy/HjzJ2zFAETuUfgCpg=="], + + "@tanstack/virtual-core": ["@tanstack/virtual-core@3.14.0", "", {}, "sha512-JLANqGy/D6k4Ujmh8Tr25lGimuOXNiaVyXaCAZS0W+1390sADdGnyUdSWNIfd49gebtIxGMij4IktRVzrdr12Q=="], + "@types/node": ["@types/node@20.19.27", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-N2clP5pJhB2YnZJ3PIHFk5RkygRX5WO/5f0WC08tp0wd+sv0rsJk3MqWn3CbNmT2J505a5336jaQj4ph1AdMug=="], "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], diff --git a/packages/types/src/live.ts b/packages/types/src/live.ts index 01fe4af..0787c84 100644 --- a/packages/types/src/live.ts +++ b/packages/types/src/live.ts @@ -54,6 +54,24 @@ export const LiveChannelSchema = z.enum([ export type LiveChannel = z.infer; export type LiveGenericChannel = z.infer; +export const LiveHotChannelSchema = z.enum(["options", "nbbo", "equities", "flow"]); +export type LiveHotChannel = z.infer; + +export const LiveChannelHealthSchema = z.object({ + freshness_age_ms: z.number().int().nonnegative().nullable(), + healthy: z.boolean() +}); + +export type LiveChannelHealth = z.infer; + +export const LiveHotChannelHealthSchema = z.object({ + options: LiveChannelHealthSchema, + nbbo: LiveChannelHealthSchema, + equities: LiveChannelHealthSchema, + flow: LiveChannelHealthSchema +}); + +export type LiveHotChannelHealthMap = z.infer; export const LiveSubscriptionSchema = z.discriminatedUnion("channel", [ z.object({ @@ -152,7 +170,8 @@ export const LiveClientMessageSchema = z.discriminatedUnion("op", [ export type LiveClientMessage = z.infer; export const LiveReadyMessageSchema = z.object({ - op: z.literal("ready") + op: z.literal("ready"), + channel_health: LiveHotChannelHealthSchema }); export type LiveReadyMessage = z.infer; @@ -175,7 +194,8 @@ export type LiveEventMessage = z.infer; export const LiveHeartbeatMessageSchema = z.object({ op: z.literal("heartbeat"), - ts: z.number().int().nonnegative() + ts: z.number().int().nonnegative(), + channel_health: LiveHotChannelHealthSchema }); export type LiveHeartbeatMessage = z.infer; diff --git a/services/api/src/index.ts b/services/api/src/index.ts index ff72307..3035897 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -112,7 +112,7 @@ import { } from "@islandflow/types"; import { createClient } from "redis"; import { z } from "zod"; -import { LiveStateManager, shouldFanoutLiveEvent } from "./live"; +import { HOT_LIVE_REDIS_KEYS, LiveStateManager, shouldFanoutLiveEvent } from "./live"; const service = "api"; const logger = createLogger({ service }); @@ -138,13 +138,6 @@ const state = { shutdownPromise: null as Promise | null }; -const HOT_LIVE_REDIS_KEYS = { - options: "live:options", - equities: "live:equities", - flow: "live:flow", - nbbo: "live:nbbo" -} as const; - const getErrorMessage = (error: unknown): string => { return error instanceof Error ? error.message : String(error); }; @@ -908,6 +901,7 @@ const run = async () => { }; const liveStateMetricsTimer = setInterval(() => { const snapshot = liveState.getStatsSnapshot(); + const hotFeedHealth = liveState.getHotChannelHealth(); const hotFeedLagMs = { options: snapshot.freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.options] ?? null, equities: snapshot.freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.equities] ?? null, @@ -916,7 +910,12 @@ const run = async () => { }; logger.info("live cache metrics", { ...snapshot, - hotFeedLagMs + hotFeedLagMs, + hotFeedHealth, + snapshotSourceCounts: { + generic_cache_snapshot: snapshot.genericCacheSnapshots, + scoped_clickhouse_snapshot: snapshot.scopedClickHouseSnapshots + } }); warnLiveLag("options", hotFeedLagMs.options); warnLiveLag("equities", hotFeedLagMs.equities); @@ -1892,9 +1891,13 @@ const run = async () => { websocket: { open: (socket: any) => { if (socket.data.channel === "live") { - sendLiveMessage(socket, { op: "ready" }); + sendLiveMessage(socket, { op: "ready", channel_health: liveState.getHotChannelHealth() }); const heartbeat = setInterval(() => { - sendLiveMessage(socket, { op: "heartbeat", ts: Date.now() }); + sendLiveMessage(socket, { + op: "heartbeat", + ts: Date.now(), + channel_health: liveState.getHotChannelHealth() + }); }, 15000); liveHeartbeats.set(socket, heartbeat); } else if (socket.data.channel === "options") { @@ -1935,7 +1938,11 @@ const run = async () => { : new TextDecoder().decode(message instanceof Uint8Array ? message : new Uint8Array(message)); const parsed = LiveClientMessageSchema.parse(JSON.parse(payload)); if (parsed.op === "ping") { - sendLiveMessage(socket, { op: "heartbeat", ts: Date.now() }); + sendLiveMessage(socket, { + op: "heartbeat", + ts: Date.now(), + channel_health: liveState.getHotChannelHealth() + }); return; } diff --git a/services/api/src/live.ts b/services/api/src/live.ts index 2907214..bd579da 100644 --- a/services/api/src/live.ts +++ b/services/api/src/live.ts @@ -25,7 +25,10 @@ import { FeedSnapshot, FlowPacketSchema, InferredDarkEventSchema, + LiveChannelHealth, LiveGenericChannel, + LiveHotChannel, + LiveHotChannelHealthMap, LiveSubscription, matchesFlowPacketFilters, matchesOptionPrintFilters, @@ -81,6 +84,13 @@ export const LIVE_FRESHNESS_THRESHOLDS: Partial; + export type GenericLiveLimits = Record; const parseGenericLimit = ( @@ -357,6 +367,8 @@ export class LiveStateManager { private readonly stats = { genericHydrateFromRedis: 0, genericHydrateFromClickHouse: 0, + genericCacheSnapshots: 0, + scopedClickHouseSnapshots: 0, trimOperations: 0, cacheDepthByKey: new Map(), freshnessAgeMsByKey: new Map() @@ -373,6 +385,8 @@ export class LiveStateManager { getStatsSnapshot(): { genericHydrateFromRedis: number; genericHydrateFromClickHouse: number; + genericCacheSnapshots: number; + scopedClickHouseSnapshots: number; trimOperations: number; cacheDepthByKey: Record; freshnessAgeMsByKey: Record; @@ -380,12 +394,37 @@ export class LiveStateManager { return { genericHydrateFromRedis: this.stats.genericHydrateFromRedis, genericHydrateFromClickHouse: this.stats.genericHydrateFromClickHouse, + genericCacheSnapshots: this.stats.genericCacheSnapshots, + scopedClickHouseSnapshots: this.stats.scopedClickHouseSnapshots, trimOperations: this.stats.trimOperations, cacheDepthByKey: Object.fromEntries(this.stats.cacheDepthByKey), freshnessAgeMsByKey: Object.fromEntries(this.stats.freshnessAgeMsByKey) }; } + getHotChannelHealth(): LiveHotChannelHealthMap { + return { + options: this.getChannelHealth("options"), + nbbo: this.getChannelHealth("nbbo"), + equities: this.getChannelHealth("equities"), + flow: this.getChannelHealth("flow") + }; + } + + private getChannelHealth(channel: LiveHotChannel): LiveChannelHealth { + const listKey = HOT_LIVE_REDIS_KEYS[channel]; + const thresholdMs = LIVE_FRESHNESS_THRESHOLDS[channel]; + const freshnessAgeMs = this.stats.freshnessAgeMsByKey.get(listKey) ?? null; + return { + freshness_age_ms: freshnessAgeMs, + healthy: + freshnessAgeMs !== null && + typeof thresholdMs === "number" && + Number.isFinite(freshnessAgeMs) && + freshnessAgeMs <= thresholdMs + }; + } + private updateFreshnessMetric(listKey: string, channel: LiveChannel, item: unknown, now = Date.now()): void { const ts = channel === "equity-candles" || channel === "equity-overlay" @@ -448,6 +487,7 @@ export class LiveStateManager { const scoped = Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id); if (subscription.filters?.view === "raw" || scoped) { + this.stats.scopedClickHouseSnapshots += 1; const limit = snapshotLimitFor(subscription, this.generic.options.limit); const storageFilters: OptionPrintQueryFilters = { view: subscription.filters?.view ?? "signal", @@ -476,6 +516,7 @@ export class LiveStateManager { } const config = this.generic.options; + this.stats.genericCacheSnapshots += 1; const limit = snapshotLimitFor(subscription, config.limit); const items = (this.genericItems.get("options") ?? []).filter((item) => matchesOptionPrintFilters(item, subscription.filters) @@ -489,6 +530,7 @@ export class LiveStateManager { } case "flow": { const config = this.generic.flow; + this.stats.genericCacheSnapshots += 1; const limit = snapshotLimitFor(subscription, config.limit); const items = (this.genericItems.get("flow") ?? []).filter((item) => matchesFlowPacketFilters(item, subscription.filters) @@ -504,6 +546,7 @@ export class LiveStateManager { const config = this.generic.equities; const limit = snapshotLimitFor(subscription, config.limit); if (subscription.underlying_ids?.length) { + this.stats.scopedClickHouseSnapshots += 1; const filters: EquityPrintQueryFilters = { underlyingIds: subscription.underlying_ids }; @@ -515,6 +558,7 @@ export class LiveStateManager { next_before: nextBeforeForItems(items, config.cursor) }; } + this.stats.genericCacheSnapshots += 1; const items = (this.genericItems.get("equities") ?? []).slice(0, limit); return { subscription, @@ -553,6 +597,7 @@ export class LiveStateManager { } default: { const config = this.generic[subscription.channel]; + this.stats.genericCacheSnapshots += 1; const limit = snapshotLimitFor(subscription, config.limit); const items = (this.genericItems.get(subscription.channel) ?? []).slice(0, limit); return { diff --git a/services/api/tests/live.test.ts b/services/api/tests/live.test.ts index 898d2fa..55232cc 100644 --- a/services/api/tests/live.test.ts +++ b/services/api/tests/live.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "bun:test"; import type { ClickHouseClient } from "@islandflow/storage"; import { + HOT_LIVE_REDIS_KEYS, LiveStateManager, isLiveItemFresh, resolveGenericLiveLimits, @@ -729,6 +730,122 @@ describe("LiveStateManager", () => { expect(persisted).toHaveLength(1); }); + it("includes hot-channel health for options, nbbo, equities, and flow", async () => { + const manager = new LiveStateManager(makeClickHouse(), null); + const now = Date.now(); + + await manager.ingest("options", { + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "opt-health", + ts: now, + option_contract_id: "AAPL-2025-01-17-200-C", + price: 1, + size: 10, + exchange: "X" + }); + await manager.ingest("nbbo", { + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "nbbo-health", + ts: now, + option_contract_id: "AAPL-2025-01-17-200-C", + bid: 1, + ask: 1.1, + bidSize: 10, + askSize: 10 + }); + await manager.ingest("equities", { + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "eq-health", + ts: now, + underlying_id: "AAPL", + price: 100, + size: 10, + exchange: "X", + offExchangeFlag: false + }); + await manager.ingest("flow", { + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "flow-health", + id: "flow-health", + members: [], + features: {}, + join_quality: {} + }); + + const health = manager.getHotChannelHealth(); + expect(health.options.healthy).toBe(true); + expect(health.nbbo.healthy).toBe(true); + expect(health.equities.healthy).toBe(true); + expect(health.flow.healthy).toBe(true); + expect(health.options.freshness_age_ms).not.toBeNull(); + expect(health.nbbo.freshness_age_ms).not.toBeNull(); + expect(health.equities.freshness_age_ms).not.toBeNull(); + expect(health.flow.freshness_age_ms).not.toBeNull(); + }); + + it("tracks generic cache and scoped clickhouse snapshot sources separately", async () => { + const manager = new LiveStateManager(makeClickHouse(() => []), null); + const now = Date.now(); + + await manager.ingest("options", { + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "opt-snapshot", + ts: now, + option_contract_id: "SPY-2025-01-17-500-C", + price: 1, + size: 10, + exchange: "X" + }); + + await manager.getSnapshot({ channel: "options" }); + await manager.getSnapshot({ + channel: "options", + underlying_ids: ["QQQ"], + option_contract_id: "QQQ-2025-01-17-400-C" + }); + + const stats = manager.getStatsSnapshot(); + expect(stats.genericCacheSnapshots).toBe(1); + expect(stats.scopedClickHouseSnapshots).toBe(1); + }); + + it("keeps backend channel health healthy when a scoped query is quiet", async () => { + const manager = new LiveStateManager(makeClickHouse(() => []), null); + const now = Date.now(); + + await manager.ingest("options", { + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "opt-global", + ts: now, + option_contract_id: "SPY-2025-01-17-500-C", + price: 1, + size: 10, + exchange: "X" + }); + + const quietSnapshot = await manager.getSnapshot({ + channel: "options", + underlying_ids: ["QQQ"], + option_contract_id: "QQQ-2025-01-17-400-C" + }); + + expect(quietSnapshot.items).toEqual([]); + expect(manager.getHotChannelHealth().options.healthy).toBe(true); + expect(manager.getStatsSnapshot().freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.options]).toBeLessThanOrEqual(50); + }); + it("exposes freshness helper for feed status", () => { expect(isLiveItemFresh("options", { ts: 1000 }, 1010)).toBe(true); expect(isLiveItemFresh("options", { ts: 1000 }, 20_001)).toBe(false); From dc0aeaa7d2309ce3be63b70d3405da54170c3e3f Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Thu, 7 May 2026 02:08:02 -0400 Subject: [PATCH 07/15] Fix Docker workspace lockfile drift and add sync guard --- .beads/issues.jsonl | 1 + deployment/docker/README.md | 17 ++ deployment/docker/workspace-root/bun.lock | 6 + deployment/docker/workspace-root/package.json | 4 +- .../docker/workspace-root/tsconfig.base.json | 4 +- package.json | 4 +- scripts/check-docker-workspace.ts | 244 ++++++++++++++++++ scripts/sync-docker-workspace.ts | 19 ++ 8 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 scripts/check-docker-workspace.ts create mode 100644 scripts/sync-docker-workspace.ts diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index b7f0a79..9229f49 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-35g","title":"Fix Docker deployment workspace lockfile drift","description":"Refresh deployment/docker workspace lockfile for Docker builds, add a drift guard for Docker-built workspaces, and document the separate deployment snapshot so frozen Bun installs cannot fail when repo dependencies change.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T06:02:06Z","created_by":"dirtydishes","updated_at":"2026-05-07T06:07:50Z","started_at":"2026-05-07T06:02:15Z","closed_at":"2026-05-07T06:07:50Z","close_reason":"Completed: synced deployment Docker workspace snapshot from repo root, refreshed deployment bun.lock, added sync/check scripts, and documented maintenance workflow. Local docker compose build validation is blocked here because Docker daemon is unavailable.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-2ij","title":"Harden tape virtualization, scoped focus, and live feed health","description":"Implement the coordinated tape stability plan across web and API.\n\nScope:\n- replace fixed-height tape virtualization with measured virtualization and virtual-end history loading\n- replace scrollHeight anchoring with key-based anchor restore\n- compose canonical tape lists across seed/live/history sources\n- preserve clicked contract/ticker context during scoped focus transitions\n- separate backend hot-channel health from scoped quiet empty states\n- shrink browser hot windows and modestly reduce server cache limits\n- add regression tests and development instrumentation\n\nAcceptance:\n- no giant blank spacer gaps during tape scrolling\n- scroll remains stable while live data and history mutate the list\n- clicked deep-history option/equity rows remain visible immediately after focus\n- narrow scopes do not surface Feed behind unless backend channel health is stale\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T05:35:18Z","created_by":"dirtydishes","updated_at":"2026-05-07T05:52:14Z","started_at":"2026-05-07T05:35:21Z","closed_at":"2026-05-07T05:52:14Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-uj7","title":"Fix home to tape navigation","description":"Home rail Tape navigation was not reliably switching to the tape route. Use browser-native top-level navigation for Home/Tape rail links so /tape remains reachable even if client router handling stalls.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T03:18:14Z","created_by":"dirtydishes","updated_at":"2026-05-07T03:18:21Z","started_at":"2026-05-07T03:18:20Z","closed_at":"2026-05-07T03:18:21Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-84s","title":"Implement seamless /tape live-to-history scroll gate","description":"Implement seamless live-to-ClickHouse scroll-gated history for /tape panes, including split live/history buffers in the web client, snapshot_limit support on live subscriptions, a bundled options support lookup endpoint, ClickHouse helpers for parity hydration, and test coverage for live head retention, background history loading, scoped options deep-hydration, and historical options decor restoration.\n","status":"in_progress","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T02:10:43Z","created_by":"dirtydishes","updated_at":"2026-05-07T02:10:47Z","started_at":"2026-05-07T02:10:47Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/deployment/docker/README.md b/deployment/docker/README.md index de7c805..dca5fbe 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -21,6 +21,7 @@ It is separate from the repo-root `docker-compose.yml`, which is still the light - `deployment/docker/Dockerfile.service`: shared Bun runtime image for most services - `deployment/docker/Dockerfile.ingest-options`: Bun runtime plus Python dependencies for Databento and IBKR adapters - `deployment/docker/Dockerfile.web`: multi-stage build for the Next.js web app +- `deployment/docker/workspace-root/`: deployment-specific workspace snapshot (`package.json`, `tsconfig.base.json`, `bun.lock`) used by Docker builds - `deployment/docker/clickhouse/listen.xml`: forces ClickHouse to listen on IPv4 for other containers on the Docker network - `deployment/docker/.env.example`: container-oriented environment template @@ -185,6 +186,22 @@ If NPM is on multiple networks and names collide (for example another stack also ## Updating the deployment +This deployment installs dependencies from `deployment/docker/workspace-root/bun.lock` (not the repo-root lockfile). + +When dependencies change in any workspace used by Docker builds, refresh and validate the deployment snapshot first: + +```bash +bun run sync:docker-workspace +bun run check:docker-workspace +``` + +Then validate the VPS build path: + +```bash +cd deployment/docker +docker compose build web +``` + When you pull new code: ```bash diff --git a/deployment/docker/workspace-root/bun.lock b/deployment/docker/workspace-root/bun.lock index d6e99c6..47fc572 100644 --- a/deployment/docker/workspace-root/bun.lock +++ b/deployment/docker/workspace-root/bun.lock @@ -12,6 +12,7 @@ "name": "@islandflow/web", "dependencies": { "@islandflow/types": "workspace:*", + "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", "next": "^14.2.4", "react": "^18.3.1", @@ -81,6 +82,7 @@ "@islandflow/bus": "workspace:*", "@islandflow/config": "workspace:*", "@islandflow/observability": "workspace:*", + "@islandflow/refdata": "workspace:*", "@islandflow/storage": "workspace:*", "@islandflow/types": "workspace:*", "redis": "^5.10.0", @@ -207,6 +209,10 @@ "@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="], + "@tanstack/react-virtual": ["@tanstack/react-virtual@3.13.24", "", { "dependencies": { "@tanstack/virtual-core": "3.14.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aIJvz5OSkhNIhZIpYivrxrPTKYsjW9Uzy+sP/mx0S3sev2HyvPb7xmjbYvokzEpfgYHy/HjzJ2zFAETuUfgCpg=="], + + "@tanstack/virtual-core": ["@tanstack/virtual-core@3.14.0", "", {}, "sha512-JLANqGy/D6k4Ujmh8Tr25lGimuOXNiaVyXaCAZS0W+1390sADdGnyUdSWNIfd49gebtIxGMij4IktRVzrdr12Q=="], + "@types/node": ["@types/node@20.19.27", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-N2clP5pJhB2YnZJ3PIHFk5RkygRX5WO/5f0WC08tp0wd+sv0rsJk3MqWn3CbNmT2J505a5336jaQj4ph1AdMug=="], "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], diff --git a/deployment/docker/workspace-root/package.json b/deployment/docker/workspace-root/package.json index 0d570a9..8240012 100644 --- a/deployment/docker/workspace-root/package.json +++ b/deployment/docker/workspace-root/package.json @@ -12,7 +12,9 @@ "dev:infra": "docker compose up", "dev:infra:down": "docker compose down", "dev:web": "bun --cwd=apps/web run dev", - "dev:services": "bun run scripts/dev-services.ts" + "dev:services": "bun run scripts/dev-services.ts", + "sync:docker-workspace": "bun run scripts/sync-docker-workspace.ts", + "check:docker-workspace": "bun run scripts/check-docker-workspace.ts" }, "devDependencies": { "typescript-language-server": "^5.1.3" diff --git a/deployment/docker/workspace-root/tsconfig.base.json b/deployment/docker/workspace-root/tsconfig.base.json index f98f46a..34b15d2 100644 --- a/deployment/docker/workspace-root/tsconfig.base.json +++ b/deployment/docker/workspace-root/tsconfig.base.json @@ -8,6 +8,6 @@ "isolatedModules": true, "resolveJsonModule": true, "skipLibCheck": true, - "noEmit": true - } + "noEmit": true, + }, } diff --git a/package.json b/package.json index 0d570a9..8240012 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,9 @@ "dev:infra": "docker compose up", "dev:infra:down": "docker compose down", "dev:web": "bun --cwd=apps/web run dev", - "dev:services": "bun run scripts/dev-services.ts" + "dev:services": "bun run scripts/dev-services.ts", + "sync:docker-workspace": "bun run scripts/sync-docker-workspace.ts", + "check:docker-workspace": "bun run scripts/check-docker-workspace.ts" }, "devDependencies": { "typescript-language-server": "^5.1.3" diff --git a/scripts/check-docker-workspace.ts b/scripts/check-docker-workspace.ts new file mode 100644 index 0000000..bc0d33e --- /dev/null +++ b/scripts/check-docker-workspace.ts @@ -0,0 +1,244 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +type DependencyMap = Record; + +type LockWorkspace = { + name?: string; + dependencies?: DependencyMap; + devDependencies?: DependencyMap; + optionalDependencies?: DependencyMap; + peerDependencies?: DependencyMap; +}; + +type BunLock = { + lockfileVersion?: number; + configVersion?: number; + workspaces?: Record; + packages?: Record; +}; + +type RootPackageManifest = { + workspaces?: string[]; +}; + +const repoRoot = path.resolve(import.meta.dir, ".."); +const deploymentRoot = path.join(repoRoot, "deployment/docker/workspace-root"); + +const rootPackagePath = path.join(repoRoot, "package.json"); +const deploymentPackagePath = path.join(deploymentRoot, "package.json"); +const rootTsconfigPath = path.join(repoRoot, "tsconfig.base.json"); +const deploymentTsconfigPath = path.join(deploymentRoot, "tsconfig.base.json"); +const rootLockPath = path.join(repoRoot, "bun.lock"); +const deploymentLockPath = path.join(deploymentRoot, "bun.lock"); + +const readUtf8 = async (filePath: string): Promise => { + return readFile(filePath, "utf8"); +}; + +const parseObjectLiteral = async (filePath: string): Promise => { + const raw = await readUtf8(filePath); + try { + const parsed = Function(`"use strict"; return (${raw});`)() as T; + return parsed; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to parse ${filePath}: ${message}`); + } +}; + +const stableSortObject = (value: unknown): unknown => { + if (Array.isArray(value)) { + return value.map(stableSortObject); + } + if (value && typeof value === "object") { + const entries = Object.entries(value as Record) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, nested]) => [key, stableSortObject(nested)] as const); + return Object.fromEntries(entries); + } + return value; +}; + +const stableStringify = (value: unknown): string => { + return JSON.stringify(stableSortObject(value)); +}; + +const listWorkspacePaths = async (workspacePatterns: string[]): Promise => { + const paths = new Set(); + + for (const pattern of workspacePatterns) { + const globPattern = pattern.endsWith("/") ? `${pattern}package.json` : `${pattern}/package.json`; + const glob = new Bun.Glob(globPattern); + for await (const match of glob.scan({ cwd: repoRoot })) { + const normalized = match.replaceAll("\\", "/"); + paths.add(path.posix.dirname(normalized)); + } + } + + return Array.from(paths).sort((a, b) => a.localeCompare(b)); +}; + +const normalizedDependencyMap = (input: DependencyMap | undefined): DependencyMap => { + if (!input) { + return {}; + } + return Object.fromEntries( + Object.entries(input) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, version]) => [name, version]) + ); +}; + +const formatDependencyDiff = ( + workspacePath: string, + section: string, + expected: DependencyMap, + actual: DependencyMap +): string[] => { + const issues: string[] = []; + const expectedKeys = new Set(Object.keys(expected)); + const actualKeys = new Set(Object.keys(actual)); + + for (const key of expectedKeys) { + if (!actualKeys.has(key)) { + issues.push(`${workspacePath} ${section}: missing ${key}@${expected[key]}`); + continue; + } + if (expected[key] !== actual[key]) { + issues.push( + `${workspacePath} ${section}: ${key} expected ${expected[key]} but found ${actual[key]}` + ); + } + } + + for (const key of actualKeys) { + if (!expectedKeys.has(key)) { + issues.push(`${workspacePath} ${section}: extra ${key}@${actual[key]}`); + } + } + + return issues; +}; + +const check = async (): Promise => { + const issues: string[] = []; + + const [rootPackage, deploymentPackage, rootTsconfig, deploymentTsconfig, rootLock, deploymentLock] = + await Promise.all([ + parseObjectLiteral(rootPackagePath), + parseObjectLiteral(deploymentPackagePath), + parseObjectLiteral(rootTsconfigPath), + parseObjectLiteral(deploymentTsconfigPath), + parseObjectLiteral(rootLockPath), + parseObjectLiteral(deploymentLockPath) + ]); + + const rootPackageSnapshot = stableStringify(rootPackage); + const deploymentPackageSnapshot = stableStringify(deploymentPackage); + if (rootPackageSnapshot !== deploymentPackageSnapshot) { + issues.push( + "deployment/docker/workspace-root/package.json does not match repo-root package.json" + ); + } + + const rootTsconfigSnapshot = stableStringify(rootTsconfig); + const deploymentTsconfigSnapshot = stableStringify(deploymentTsconfig); + if (rootTsconfigSnapshot !== deploymentTsconfigSnapshot) { + issues.push( + "deployment/docker/workspace-root/tsconfig.base.json does not match repo-root tsconfig.base.json" + ); + } + + const rootWorkspaces = rootLock.workspaces ?? {}; + const deploymentWorkspaces = deploymentLock.workspaces ?? {}; + + const workspacePatterns = rootPackage.workspaces ?? []; + const workspacePackagePaths = await listWorkspacePaths(workspacePatterns); + for (const workspacePath of workspacePackagePaths) { + const packageJsonPath = path.join(repoRoot, workspacePath, "package.json"); + const workspacePackage = (await parseObjectLiteral(packageJsonPath)) as LockWorkspace; + const deploymentWorkspace = deploymentWorkspaces[workspacePath]; + + if (!deploymentWorkspace) { + issues.push(`deployment lock is missing workspace entry: ${workspacePath}`); + continue; + } + + const sections: Array = [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies" + ]; + for (const section of sections) { + const expectedMap = normalizedDependencyMap(workspacePackage[section] as DependencyMap | undefined); + const actualMap = normalizedDependencyMap( + deploymentWorkspace[section] as DependencyMap | undefined + ); + issues.push(...formatDependencyDiff(workspacePath, section, expectedMap, actualMap)); + } + } + + const workspacePaths = Array.from( + new Set([...Object.keys(rootWorkspaces), ...Object.keys(deploymentWorkspaces)]) + ).sort((a, b) => a.localeCompare(b)); + + for (const workspacePath of workspacePaths) { + const rootWorkspace = rootWorkspaces[workspacePath]; + const deploymentWorkspace = deploymentWorkspaces[workspacePath]; + + if (!rootWorkspace) { + issues.push(`deployment lock has unexpected workspace entry: ${workspacePath}`); + continue; + } + if (!deploymentWorkspace) { + issues.push(`deployment lock is missing workspace entry: ${workspacePath}`); + continue; + } + + if ((rootWorkspace.name ?? "") !== (deploymentWorkspace.name ?? "")) { + issues.push( + `${workspacePath} name mismatch: expected ${rootWorkspace.name ?? "(none)"} but found ${ + deploymentWorkspace.name ?? "(none)" + }` + ); + } + + const sections: Array = [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies" + ]; + for (const section of sections) { + const expectedMap = normalizedDependencyMap(rootWorkspace[section] as DependencyMap | undefined); + const actualMap = normalizedDependencyMap( + deploymentWorkspace[section] as DependencyMap | undefined + ); + issues.push(...formatDependencyDiff(workspacePath, section, expectedMap, actualMap)); + } + } + + const rootPackagesSnapshot = stableStringify(rootLock.packages ?? {}); + const deploymentPackagesSnapshot = stableStringify(deploymentLock.packages ?? {}); + if (rootPackagesSnapshot !== deploymentPackagesSnapshot) { + issues.push( + "deployment/docker/workspace-root/bun.lock package resolutions differ from repo-root bun.lock" + ); + } + + if (issues.length > 0) { + console.error("Docker workspace snapshot is out of sync:"); + for (const issue of issues) { + console.error(`- ${issue}`); + } + console.error("Run: bun run sync:docker-workspace"); + return 1; + } + + console.log("Docker workspace snapshot is in sync."); + return 0; +}; + +process.exitCode = await check(); diff --git a/scripts/sync-docker-workspace.ts b/scripts/sync-docker-workspace.ts new file mode 100644 index 0000000..e20b293 --- /dev/null +++ b/scripts/sync-docker-workspace.ts @@ -0,0 +1,19 @@ +import { copyFile } from "node:fs/promises"; +import path from "node:path"; + +const repoRoot = path.resolve(import.meta.dir, ".."); +const deploymentRoot = path.join(repoRoot, "deployment/docker/workspace-root"); + +const filesToSync = [ + "package.json", + "bun.lock", + "tsconfig.base.json" +] as const; + +for (const fileName of filesToSync) { + const source = path.join(repoRoot, fileName); + const destination = path.join(deploymentRoot, fileName); + await copyFile(source, destination); + console.log(`synced ${fileName}`); +} + From 088bd37e84e6798d198d9103a8104eaea4f5ec80 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Thu, 7 May 2026 02:17:17 -0400 Subject: [PATCH 08/15] Fix terminal virtualization and hydration crash handling --- .beads/issues.jsonl | 1 + apps/web/app/terminal.tsx | 76 +++++++++++++++++++++++++++++++-------- 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 9229f49..c755228 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-9xs","title":"Fix terminal hydration and virtual row measurement crash","description":"Fix client crash caused by options-support hydration on non-JSON/404 responses and satisfy tanstack virtual measured-row data-index requirement across virtualized tables.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T06:14:33Z","created_by":"dirtydishes","updated_at":"2026-05-07T06:17:09Z","started_at":"2026-05-07T06:14:43Z","closed_at":"2026-05-07T06:17:09Z","close_reason":"Completed: added data-index attributes on measured virtual rows, hardened options-support hydration error handling/content-type validation, and guarded trace-id hydration loops against malformed payload entries.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-35g","title":"Fix Docker deployment workspace lockfile drift","description":"Refresh deployment/docker workspace lockfile for Docker builds, add a drift guard for Docker-built workspaces, and document the separate deployment snapshot so frozen Bun installs cannot fail when repo dependencies change.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T06:02:06Z","created_by":"dirtydishes","updated_at":"2026-05-07T06:07:50Z","started_at":"2026-05-07T06:02:15Z","closed_at":"2026-05-07T06:07:50Z","close_reason":"Completed: synced deployment Docker workspace snapshot from repo root, refreshed deployment bun.lock, added sync/check scripts, and documented maintenance workflow. Local docker compose build validation is blocked here because Docker daemon is unavailable.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-2ij","title":"Harden tape virtualization, scoped focus, and live feed health","description":"Implement the coordinated tape stability plan across web and API.\n\nScope:\n- replace fixed-height tape virtualization with measured virtualization and virtual-end history loading\n- replace scrollHeight anchoring with key-based anchor restore\n- compose canonical tape lists across seed/live/history sources\n- preserve clicked contract/ticker context during scoped focus transitions\n- separate backend hot-channel health from scoped quiet empty states\n- shrink browser hot windows and modestly reduce server cache limits\n- add regression tests and development instrumentation\n\nAcceptance:\n- no giant blank spacer gaps during tape scrolling\n- scroll remains stable while live data and history mutate the list\n- clicked deep-history option/equity rows remain visible immediately after focus\n- narrow scopes do not surface Feed behind unless backend channel health is stale\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T05:35:18Z","created_by":"dirtydishes","updated_at":"2026-05-07T05:52:14Z","started_at":"2026-05-07T05:35:21Z","closed_at":"2026-05-07T05:52:14Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-uj7","title":"Fix home to tape navigation","description":"Home rail Tape navigation was not reliably switching to the tape route. Use browser-native top-level navigation for Home/Tape rail links so /tape remains reachable even if client router handling stalls.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T03:18:14Z","created_by":"dirtydishes","updated_at":"2026-05-07T03:18:21Z","started_at":"2026-05-07T03:18:20Z","closed_at":"2026-05-07T03:18:21Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 2718ed7..444a02d 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -234,19 +234,31 @@ const sampleToLimit = (items: T[], limit: number): T[] => { }; const readErrorDetail = async (response: Response): Promise => { + const statusLabel = `HTTP ${response.status}${response.statusText ? ` ${response.statusText}` : ""}`; const text = await response.text(); if (!text) { - return ""; + return statusLabel; } + const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""; + const trimmed = text.trimStart(); + const truncated = text.length > 600 ? `${text.slice(0, 600)}...` : text; + + if (!contentType.includes("application/json")) { + if (/^ { .then((payload: { data?: OptionPrint[] }) => { const next = new Map(); for (const item of payload.data ?? []) { + if (!item || !item.trace_id) { + continue; + } next.set(item.trace_id, item); } if (next.size > 0) { @@ -5241,6 +5256,9 @@ const useTerminalState = () => { .then((payload: { data?: EquityPrintJoin[] }) => { const next = new Map(); for (const item of payload.data ?? []) { + if (!item || !item.id || !item.trace_id) { + continue; + } next.set(item.id, item); next.set(item.trace_id, item); if (item.print_trace_id) { @@ -5441,6 +5459,12 @@ const useTerminalState = () => { if (!response.ok) { throw new Error(await readErrorDetail(response)); } + const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""; + if (!contentType.includes("application/json")) { + throw new Error( + `Unexpected content type from /lookup/options-support: ${contentType || "unknown"}` + ); + } return response.json() as Promise<{ packets?: FlowPacket[]; smart_money?: SmartMoneyEvent[]; @@ -5455,19 +5479,28 @@ const useTerminalState = () => { const now = Date.now(); const packetMap = new Map(); for (const packet of payload.packets ?? []) { + if (!packet || !packet.id) { + continue; + } packetMap.set(packet.id, packet); } if (packetMap.size > 0) { setPinnedFlowPacketMap((prev) => upsertPinnedEntries(prev, packetMap, now)); } if (payload.smart_money?.length) { + const filtered = payload.smart_money.filter((item): item is SmartMoneyEvent => + Boolean(item && item.trace_id) + ); setOptionSupportSmartMoney((prev) => - mergeNewest(payload.smart_money ?? [], prev, PINNED_EVIDENCE_MAX_ITEMS) + mergeNewest(filtered, prev, PINNED_EVIDENCE_MAX_ITEMS) ); } if (payload.classifier_hits?.length) { + const filtered = payload.classifier_hits.filter((item): item is ClassifierHitEvent => + Boolean(item && item.trace_id) + ); setOptionSupportClassifierHits((prev) => - mergeNewest(payload.classifier_hits ?? [], prev, PINNED_EVIDENCE_MAX_ITEMS) + mergeNewest(filtered, prev, PINNED_EVIDENCE_MAX_ITEMS) ); } if (payload.nbbo_by_trace_id) { @@ -5630,11 +5663,14 @@ const useTerminalState = () => { } return response.json(); }) - .then((payload: { data?: OptionPrint[] }) => { - const next = new Map(); - for (const item of payload.data ?? []) { - next.set(item.trace_id, item); - } + .then((payload: { data?: OptionPrint[] }) => { + const next = new Map(); + for (const item of payload.data ?? []) { + if (!item || !item.trace_id) { + continue; + } + next.set(item.trace_id, item); + } if (next.size > 0) { setPinnedOptionPrintMap((prev) => upsertPinnedEntries(prev, next, Date.now())); } @@ -5939,11 +5975,14 @@ const useTerminalState = () => { } return response.json(); }) - .then((payload: { data?: OptionPrint[] }) => { - const next = new Map(); - for (const item of payload.data ?? []) { - next.set(item.trace_id, item); - } + .then((payload: { data?: OptionPrint[] }) => { + const next = new Map(); + for (const item of payload.data ?? []) { + if (!item || !item.trace_id) { + continue; + } + next.set(item.trace_id, item); + } if (next.size > 0) { const now = Date.now(); setPinnedOptionPrintMap((prev) => upsertPinnedEntries(prev, next, now)); @@ -6657,6 +6696,7 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { const commonProps = { className: `data-table-row data-table-row-button data-table-row-classified data-table-row-options data-table-virtual-row${index % 2 === 1 ? " is-even" : ""}${decor ? ` is-classified classifier-${decor.tone}` : ""}`, style: rowStyle, + "data-index": index, "data-row-start": String(start), "data-row-size": String(size), "data-tape-key": key, @@ -6813,6 +6853,7 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => { className={`data-table-row data-table-row-equities data-table-virtual-row${index % 2 === 1 ? " is-even" : ""}`} key={key} ref={virtual.measureElement} + data-index={index} data-row-start={String(start)} data-row-size={String(size)} data-tape-key={key} @@ -6962,6 +7003,7 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { className={`data-table-row data-table-row-flow data-table-virtual-row${index % 2 === 1 ? " is-even" : ""}${nbboStale || nbboMissing ? " data-table-row-warn" : ""}`} key={key} ref={virtual.measureElement} + data-index={index} data-row-start={String(start)} data-row-size={String(size)} data-tape-key={key} @@ -7061,6 +7103,7 @@ const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => key={key} type="button" ref={virtual.measureElement} + data-index={index} data-row-start={String(start)} data-row-size={String(size)} data-tape-key={key} @@ -7171,6 +7214,7 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { key={key} type="button" ref={virtual.measureElement} + data-index={index} data-row-start={String(start)} data-row-size={String(size)} data-tape-key={key} @@ -7199,6 +7243,7 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { key={key} type="button" ref={virtual.measureElement} + data-index={index} data-row-start={String(start)} data-row-size={String(size)} data-tape-key={key} @@ -7291,6 +7336,7 @@ const DarkPane = ({ limit, className }: DarkPaneProps) => { key={key} type="button" ref={virtual.measureElement} + data-index={index} data-row-start={String(start)} data-row-size={String(size)} data-tape-key={key} From bb1df9b58b58c0105a340f4679b63ffb9bb8b181 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Thu, 7 May 2026 02:18:11 -0400 Subject: [PATCH 09/15] Clean up terminal hydration promise-chain formatting --- apps/web/app/terminal.tsx | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 444a02d..58a0aea 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -5663,14 +5663,14 @@ const useTerminalState = () => { } return response.json(); }) - .then((payload: { data?: OptionPrint[] }) => { - const next = new Map(); - for (const item of payload.data ?? []) { - if (!item || !item.trace_id) { - continue; - } - next.set(item.trace_id, item); + .then((payload: { data?: OptionPrint[] }) => { + const next = new Map(); + for (const item of payload.data ?? []) { + if (!item || !item.trace_id) { + continue; } + next.set(item.trace_id, item); + } if (next.size > 0) { setPinnedOptionPrintMap((prev) => upsertPinnedEntries(prev, next, Date.now())); } @@ -5975,14 +5975,14 @@ const useTerminalState = () => { } return response.json(); }) - .then((payload: { data?: OptionPrint[] }) => { - const next = new Map(); - for (const item of payload.data ?? []) { - if (!item || !item.trace_id) { - continue; - } - next.set(item.trace_id, item); + .then((payload: { data?: OptionPrint[] }) => { + const next = new Map(); + for (const item of payload.data ?? []) { + if (!item || !item.trace_id) { + continue; } + next.set(item.trace_id, item); + } if (next.size > 0) { const now = Date.now(); setPinnedOptionPrintMap((prev) => upsertPinnedEntries(prev, next, now)); From 9c351d12d1b47a6a8f8075f2fa7caa6ed3181659 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Thu, 7 May 2026 22:03:09 -0400 Subject: [PATCH 10/15] Refine route-scoped tape subscriptions and table virtualization - Scope live channels by route and trim unused feed work - Switch tape tables to fixed-height virtual rows with separate scroll containers - Add tests for route feature maps and virtual config --- apps/web/app/globals.css | 29 +- apps/web/app/terminal.test.ts | 113 ++- apps/web/app/terminal.tsx | 1382 ++++++++++++++++++++------------- deploy-branch.sh | 6 + deploy.sh | 7 + 5 files changed, 955 insertions(+), 582 deletions(-) create mode 100755 deploy-branch.sh create mode 100755 deploy.sh diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index ab3f6ed..ed3edc2 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -956,17 +956,27 @@ h3 { .data-table-wrap { flex: 1 1 auto; min-height: 0; - overflow: auto; + overflow-x: auto; + overflow-y: hidden; border-top: 1px solid var(--border); border-bottom: 1px solid var(--border); background: rgba(5, 8, 12, 0.42); } .data-table { - display: block; + display: flex; + flex-direction: column; + min-height: 0; min-width: 980px; } +.data-table-scroll { + flex: 1 1 auto; + min-height: 0; + overflow-y: auto; + overflow-x: hidden; +} + .data-table-body { position: relative; min-width: 100%; @@ -1004,10 +1014,8 @@ h3 { } .data-table-head { - position: sticky; - top: 0; - z-index: 2; - min-height: 30px; + flex: 0 0 auto; + height: 30px; padding: 0 10px; border-bottom: 1px solid rgba(255, 255, 255, 0.095); background: rgba(8, 11, 16, 0.98); @@ -1019,7 +1027,7 @@ h3 { .data-table-row { width: 100%; - min-height: 40px; + height: 40px; padding: 0 10px; border: 0; border-bottom: 1px solid rgba(255, 255, 255, 0.055); @@ -1035,6 +1043,7 @@ h3 { .data-table-virtual-row { position: absolute; + top: 0; left: 0; width: 100%; } @@ -1050,18 +1059,18 @@ h3 { } .data-table-row-options { - min-height: 36px; + height: 36px; } .data-table-row-equities { - min-height: 34px; + height: 36px; } .data-table-row-flow, .data-table-row-alerts, .data-table-row-classifier, .data-table-row-dark { - min-height: 44px; + height: 44px; } .data-table-row-classified { diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index 16ce0ad..e4d9a52 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -19,12 +19,15 @@ import { getOptionTableSnapshot, getLiveFeedStatus, getLiveManifest, + getRouteFeatures, + getTapeVirtualConfig, mergeNewestWithOverflow, normalizeAlertSeverity, nextFlowFilterPopoverState, projectPausableTapeState, reducePausableTapeData, shouldRetainLiveSnapshotHistory, + shouldIncludeEquitiesForDarkUnderlyingFallback, shouldShowEquitiesSilentFeedWarning, selectPrimaryClassifierHit, smartMoneyProfileLabel, @@ -51,15 +54,13 @@ const makeAlert = (overrides: Record = {}) => }) as any; describe("live manifest", () => { - it("includes options on home and tape", () => { + it("includes only tape channels on /tape", () => { const filters = buildDefaultFlowFilters(); - for (const pathname of ["/", "/tape"]) { - expect( - getLiveManifest(pathname, "SPY", 60000, filters).some( - (subscription) => subscription.channel === "options" - ) - ).toBe(true); - } + const channels = getLiveManifest("/tape", "SPY", 60000, filters).map( + (subscription) => subscription.channel + ); + + expect(channels).toEqual(["options", "nbbo", "equities", "flow"]); }); it("dedupes tape options subscription", () => { @@ -72,37 +73,29 @@ describe("live manifest", () => { expect(tapeOptionsSubscriptions).toHaveLength(1); }); - it("keeps option filters on baseline subscription across page changes", () => { + it("keeps option filters on /tape options subscriptions", () => { const filters = { ...buildDefaultFlowFilters(), minNotional: 125_000 }; - const homeOptionsSubscription = getLiveManifest("/", "SPY", 60000, filters).find( - (subscription) => subscription.channel === "options" - ); const tapeOptionsSubscription = getLiveManifest("/tape", "SPY", 60000, filters).find( (subscription) => subscription.channel === "options" ); - expect(homeOptionsSubscription?.filters).toBe(filters); expect(tapeOptionsSubscription?.filters).toBe(filters); }); - it("applies global flow filters to flow subscriptions on home and tape", () => { + it("applies global flow filters to flow subscriptions on /tape", () => { const filters = { ...buildDefaultFlowFilters(), minNotional: 50_000 }; - const homeFlowSubscription = getLiveManifest("/", "SPY", 60000, filters).find( - (subscription) => subscription.channel === "flow" - ); const tapeFlowSubscription = getLiveManifest("/tape", "SPY", 60000, filters).find( (subscription) => subscription.channel === "flow" ); - expect(homeFlowSubscription?.filters).toBe(filters); expect(tapeFlowSubscription?.filters).toBe(filters); }); @@ -131,6 +124,90 @@ describe("live manifest", () => { expect(optionsSubscription?.option_contract_id).toBe("AAPL-2025-01-17-200-C"); expect(equitiesSubscription?.underlying_ids).toEqual(["AAPL"]); }); + + it("scopes /signals subscriptions to signals channels only", () => { + const channels = getLiveManifest("/signals", "SPY", 60000, buildDefaultFlowFilters()).map( + (subscription) => subscription.channel + ); + + expect(channels).toEqual([ + "alerts", + "smart-money", + "classifier-hits", + "inferred-dark", + "equity-joins" + ]); + }); + + it("scopes /charts subscriptions to chart channels only", () => { + const channels = getLiveManifest("/charts", "SPY", 60000, buildDefaultFlowFilters()).map( + (subscription) => subscription.channel + ); + + expect(channels).toEqual([ + "smart-money", + "inferred-dark", + "equity-joins", + "equity-candles", + "equity-overlay" + ]); + }); +}); + +describe("route feature map", () => { + it("maps /tape to tape panes and dependencies", () => { + const features = getRouteFeatures("/tape"); + expect(features.showOptionsPane).toBe(true); + expect(features.showEquitiesPane).toBe(true); + expect(features.showFlowPane).toBe(true); + expect(features.needsClassifierDecor).toBe(true); + expect(features.alerts).toBe(false); + }); + + it("maps /signals to signal panes and dependencies", () => { + const features = getRouteFeatures("/signals"); + expect(features.showAlertsPane).toBe(true); + expect(features.showClassifierPane).toBe(true); + expect(features.showDarkPane).toBe(true); + expect(features.options).toBe(false); + expect(features.equityJoins).toBe(true); + }); + + it("maps /charts to chart panes and dependencies", () => { + const features = getRouteFeatures("/charts"); + expect(features.showChartPane).toBe(true); + expect(features.showFocusPane).toBe(true); + expect(features.equityCandles).toBe(true); + expect(features.equityOverlay).toBe(true); + expect(features.alerts).toBe(false); + }); +}); + +describe("fixed tape virtualization config", () => { + it("uses expected fixed row heights and overscan by table", () => { + expect(getTapeVirtualConfig("options")).toEqual({ rowHeight: 36, overscan: 24, debugLabel: "options" }); + expect(getTapeVirtualConfig("equities")).toEqual({ rowHeight: 36, overscan: 20, debugLabel: "equities" }); + expect(getTapeVirtualConfig("flow")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "flow" }); + expect(getTapeVirtualConfig("alerts")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "alerts" }); + expect(getTapeVirtualConfig("classifier")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "classifier" }); + expect(getTapeVirtualConfig("dark")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "dark" }); + }); +}); + +describe("dark underlying route dependency helper", () => { + it("does not keep extra equities subscriptions when joins+trace fallback are sufficient", () => { + expect(shouldIncludeEquitiesForDarkUnderlyingFallback()).toBe(false); + expect( + getLiveManifest("/signals", "SPY", 60000, buildDefaultFlowFilters()).some( + (subscription) => subscription.channel === "equities" + ) + ).toBe(false); + expect( + getLiveManifest("/charts", "SPY", 60000, buildDefaultFlowFilters()).some( + (subscription) => subscription.channel === "equities" + ) + ).toBe(false); + }); }); describe("terminal navigation", () => { diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 58a0aea..01ee884 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -4,6 +4,7 @@ import Link from "next/link"; import { usePathname } from "next/navigation"; import { createContext, + memo, useCallback, useContext, useEffect, @@ -17,7 +18,7 @@ import { type ReactNode, type SetStateAction } from "react"; -import { useVirtualizer, type Virtualizer } from "@tanstack/react-virtual"; +import { useVirtualizer } from "@tanstack/react-virtual"; import type { AlertEvent, ClassifierHitEvent, @@ -125,6 +126,206 @@ const LIVE_SESSION_HOT_CHANNELS = new Set([ "equity-overlay" ]); +type TapeVirtualPane = "options" | "equities" | "flow" | "alerts" | "classifier" | "dark"; + +type TapeVirtualListConfig = { + rowHeight: number; + overscan: number; + debugLabel: TapeVirtualPane; +}; + +const TAPE_VIRTUAL_CONFIG: Record = { + options: { rowHeight: 36, overscan: 24, debugLabel: "options" }, + equities: { rowHeight: 36, overscan: 20, debugLabel: "equities" }, + flow: { rowHeight: 44, overscan: 16, debugLabel: "flow" }, + alerts: { rowHeight: 44, overscan: 16, debugLabel: "alerts" }, + classifier: { rowHeight: 44, overscan: 16, debugLabel: "classifier" }, + dark: { rowHeight: 44, overscan: 16, debugLabel: "dark" } +}; + +export const getTapeVirtualConfig = (pane: TapeVirtualPane): TapeVirtualListConfig => + TAPE_VIRTUAL_CONFIG[pane]; + +type RouteFeatures = { + options: boolean; + nbbo: boolean; + equities: boolean; + flow: boolean; + alerts: boolean; + smartMoney: boolean; + classifierHits: boolean; + inferredDark: boolean; + equityJoins: boolean; + equityCandles: boolean; + equityOverlay: boolean; + showOptionsPane: boolean; + showEquitiesPane: boolean; + showFlowPane: boolean; + showAlertsPane: boolean; + showClassifierPane: boolean; + showDarkPane: boolean; + showChartPane: boolean; + showFocusPane: boolean; + showReplayConsole: boolean; + needsClassifierDecor: boolean; + needsAlertEvidencePrefetch: boolean; + needsDarkUnderlying: boolean; +}; + +export const shouldIncludeEquitiesForDarkUnderlyingFallback = (): boolean => { + return false; +}; + +export const getRouteFeatures = (pathname: string): RouteFeatures => { + const includeEquitiesFallback = shouldIncludeEquitiesForDarkUnderlyingFallback(); + const normalizedPath = + pathname === "/tape" || + pathname === "/signals" || + pathname === "/charts" || + pathname === "/replay" + ? pathname + : "/"; + + switch (normalizedPath) { + case "/tape": + return { + options: true, + nbbo: true, + equities: true, + flow: true, + alerts: false, + smartMoney: false, + classifierHits: false, + inferredDark: false, + equityJoins: false, + equityCandles: false, + equityOverlay: false, + showOptionsPane: true, + showEquitiesPane: true, + showFlowPane: true, + showAlertsPane: false, + showClassifierPane: false, + showDarkPane: false, + showChartPane: false, + showFocusPane: false, + showReplayConsole: false, + needsClassifierDecor: true, + needsAlertEvidencePrefetch: false, + needsDarkUnderlying: false + }; + case "/signals": + return { + options: false, + nbbo: false, + equities: includeEquitiesFallback, + flow: false, + alerts: true, + smartMoney: true, + classifierHits: true, + inferredDark: true, + equityJoins: true, + equityCandles: false, + equityOverlay: false, + showOptionsPane: false, + showEquitiesPane: false, + showFlowPane: false, + showAlertsPane: true, + showClassifierPane: true, + showDarkPane: true, + showChartPane: false, + showFocusPane: false, + showReplayConsole: false, + needsClassifierDecor: false, + needsAlertEvidencePrefetch: true, + needsDarkUnderlying: true + }; + case "/charts": + return { + options: false, + nbbo: false, + equities: includeEquitiesFallback, + flow: false, + alerts: false, + smartMoney: true, + classifierHits: false, + inferredDark: true, + equityJoins: true, + equityCandles: true, + equityOverlay: true, + showOptionsPane: false, + showEquitiesPane: false, + showFlowPane: false, + showAlertsPane: false, + showClassifierPane: false, + showDarkPane: false, + showChartPane: true, + showFocusPane: true, + showReplayConsole: false, + needsClassifierDecor: false, + needsAlertEvidencePrefetch: false, + needsDarkUnderlying: true + }; + case "/replay": + return { + options: false, + nbbo: false, + equities: false, + flow: false, + alerts: false, + smartMoney: false, + classifierHits: false, + inferredDark: false, + equityJoins: false, + equityCandles: false, + equityOverlay: false, + showOptionsPane: true, + showEquitiesPane: false, + showFlowPane: true, + showAlertsPane: true, + showClassifierPane: false, + showDarkPane: false, + showChartPane: false, + showFocusPane: false, + showReplayConsole: true, + needsClassifierDecor: true, + needsAlertEvidencePrefetch: true, + needsDarkUnderlying: false + }; + case "/": + default: + return { + options: false, + nbbo: false, + equities: true, + flow: false, + alerts: true, + smartMoney: true, + classifierHits: false, + inferredDark: true, + equityJoins: true, + equityCandles: true, + equityOverlay: true, + showOptionsPane: false, + showEquitiesPane: true, + showFlowPane: false, + showAlertsPane: true, + showClassifierPane: false, + showDarkPane: false, + showChartPane: true, + showFocusPane: false, + showReplayConsole: false, + needsClassifierDecor: false, + needsAlertEvidencePrefetch: true, + needsDarkUnderlying: true + }; + } +}; + +const EMPTY_ALERT_EVENTS: AlertEvent[] = []; +const EMPTY_CLASSIFIER_HIT_EVENTS: ClassifierHitEvent[] = []; +const EMPTY_SMART_MONEY_EVENTS: SmartMoneyEvent[] = []; +const EMPTY_INFERRED_DARK_EVENTS: InferredDarkEvent[] = []; + type CandlestickSeries = ReturnType; type EquityOverlayPoint = { @@ -981,14 +1182,6 @@ const extractUnderlying = (contractId: string): string => { return contractId.split("-")[0]?.toUpperCase() ?? contractId.toUpperCase(); }; -const extractEquityTraceFromJoin = (joinId: string): string | null => { - const match = joinId.match(/^equityjoin:(.+)$/); - if (match?.[1]) { - return match[1]; - } - return joinId.trim().length > 0 ? joinId.trim() : null; -}; - const normalizeJoinRefCandidates = (value: string): string[] => { const ref = value.trim(); if (!ref) { @@ -1042,7 +1235,6 @@ const formatDarkTrace = (traceId: string): string => { const inferDarkUnderlying = ( event: InferredDarkEvent, - equityPrints: Map, equityJoins: Map ): string | null => { for (const ref of event.evidence_refs) { @@ -1061,17 +1253,6 @@ const inferDarkUnderlying = ( return match[1].toUpperCase(); } - for (const ref of event.evidence_refs) { - const traceId = extractEquityTraceFromJoin(ref); - if (!traceId) { - continue; - } - const print = equityPrints.get(traceId); - if (print) { - return print.underlying_id.toUpperCase(); - } - } - return null; }; @@ -1286,6 +1467,10 @@ type ClassifierDecor = { intensity: number; }; +const EMPTY_CLASSIFIER_HITS_BY_PACKET_ID = new Map(); +const EMPTY_PACKET_ID_BY_OPTION_TRACE_ID = new Map(); +const EMPTY_CLASSIFIER_DECOR_BY_OPTION_TRACE_ID = new Map(); + const SMART_MONEY_PROFILE_TONES: Record = { institutional_directional: "green", retail_whale: "amber", @@ -1612,14 +1797,12 @@ const useVirtualHistoryGate = ( }, [enabled, itemCount, lastVirtualIndex]); }; -type MeasuredVirtualListResult = { +type TapeVirtualListResult = { totalSize: number; - virtualItems: MeasuredVirtualRow[]; - measureElement: (node: HTMLElement | null) => void; - virtualizer: Virtualizer; + virtualItems: TapeVirtualRow[]; }; -type MeasuredVirtualRow = { +type TapeVirtualRow = { item: T; key: string; index: number; @@ -1628,39 +1811,36 @@ type MeasuredVirtualRow = { end: number; }; -const useMeasuredVirtualList = ( +const useTapeVirtualList = ( items: T[], listRef: React.RefObject, - estimateSize: number, - overscan: number, - debugLabel: string -): MeasuredVirtualListResult => { + config: TapeVirtualListConfig +): TapeVirtualListResult => { const virtualizer = useVirtualizer({ count: items.length, getScrollElement: () => listRef.current, - estimateSize: () => estimateSize, - overscan, - getItemKey: (index) => getTapeItemKey(items[index] as SortableItem), - measureElement: (node) => { - bumpTapeDebugMetric("virtualRowMeasurementCount", 1); - return node.getBoundingClientRect().height; - } + estimateSize: () => config.rowHeight, + overscan: config.overscan, + getItemKey: (index) => getTapeItemKey(items[index] as SortableItem) }); - const virtualItems: MeasuredVirtualRow[] = virtualizer.getVirtualItems().map((virtualItem) => { - const item = items[virtualItem.index] as T | undefined; - if (!item) { - return null; - } - return { - item, - key: getTapeItemKey(item), - index: virtualItem.index, - start: virtualItem.start, - size: virtualItem.size, - end: virtualItem.end - }; - }).filter((virtualItem): virtualItem is MeasuredVirtualRow => virtualItem !== null); + const virtualItems: TapeVirtualRow[] = virtualizer + .getVirtualItems() + .map((virtualItem) => { + const item = items[virtualItem.index] as T | undefined; + if (!item) { + return null; + } + return { + item, + key: getTapeItemKey(item), + index: virtualItem.index, + start: virtualItem.start, + size: virtualItem.size, + end: virtualItem.end + }; + }) + .filter((virtualItem): virtualItem is TapeVirtualRow => virtualItem !== null); useEffect(() => { if (!DEV_TAPE_DEBUG || items.length === 0) { @@ -1679,20 +1859,18 @@ const useMeasuredVirtualList = ( const visibleBottomGap = Math.max(0, element.scrollTop + element.clientHeight - last.end); if (visibleTopGap > element.clientHeight || visibleBottomGap > element.clientHeight) { console.warn("[tape] false-gap watchdog", { - pane: debugLabel, + pane: config.debugLabel, item_count: items.length, visible_top_gap: visibleTopGap, visible_bottom_gap: visibleBottomGap, viewport_height: element.clientHeight }); } - }, [debugLabel, items.length, listRef, virtualItems]); + }, [config.debugLabel, items.length, listRef, virtualItems]); return { totalSize: virtualizer.getTotalSize(), - virtualItems, - measureElement: virtualizer.measureElement, - virtualizer + virtualItems }; }; @@ -2635,42 +2813,56 @@ export const getLiveManifest = ( optionScope?: Pick, "underlying_ids" | "option_contract_id">, equityScope?: Pick, "underlying_ids"> ): LiveSubscription[] => { - const baselineSubs: LiveSubscription[] = [{ channel: "options", filters: flowFilters, ...optionScope }]; - const chartSubs: LiveSubscription[] = [ - { channel: "equity-candles", underlying_id: chartTicker, interval_ms: chartIntervalMs }, - { channel: "equity-overlay", underlying_id: chartTicker } - ]; + const features = getRouteFeatures(pathname); + const subscriptions: LiveSubscription[] = []; - if (pathname === "/tape") { - const optionsSub: Extract = { + if (features.options) { + subscriptions.push({ channel: "options", filters: flowFilters, ...optionScope, snapshot_limit: LIVE_HOT_WINDOW_OPTIONS - }; - const tapeSubs: LiveSubscription[] = [ - optionsSub, - { channel: "nbbo", snapshot_limit: LIVE_HOT_WINDOW }, - { channel: "equities", ...equityScope, snapshot_limit: LIVE_HOT_WINDOW }, - { channel: "flow", filters: flowFilters, snapshot_limit: LIVE_HOT_WINDOW }, - { channel: "smart-money", snapshot_limit: LIVE_HOT_WINDOW }, - { channel: "classifier-hits", snapshot_limit: LIVE_HOT_WINDOW }, - { channel: "alerts", snapshot_limit: LIVE_HOT_WINDOW }, - { channel: "inferred-dark", snapshot_limit: LIVE_HOT_WINDOW } - ]; - return dedupeLiveSubscriptions(tapeSubs); + }); + } + if (features.nbbo) { + subscriptions.push({ channel: "nbbo", snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.equities) { + subscriptions.push({ channel: "equities", ...equityScope, snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.flow) { + subscriptions.push({ channel: "flow", filters: flowFilters, snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.alerts) { + subscriptions.push({ channel: "alerts", snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.smartMoney) { + subscriptions.push({ channel: "smart-money", snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.classifierHits) { + subscriptions.push({ channel: "classifier-hits", snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.inferredDark) { + subscriptions.push({ channel: "inferred-dark", snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.equityJoins) { + subscriptions.push({ channel: "equity-joins", snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.equityCandles) { + subscriptions.push({ + channel: "equity-candles", + underlying_id: chartTicker, + interval_ms: chartIntervalMs + }); + } + if (features.equityOverlay) { + subscriptions.push({ + channel: "equity-overlay", + underlying_id: chartTicker + }); } - return dedupeLiveSubscriptions([ - ...baselineSubs, - { channel: "equities", ...equityScope }, - { channel: "flow", filters: flowFilters }, - { channel: "alerts" }, - { channel: "smart-money" }, - { channel: "classifier-hits" }, - { channel: "inferred-dark" }, - ...chartSubs - ]); + return dedupeLiveSubscriptions(subscriptions); }; const useLiveSession = ( @@ -4643,6 +4835,7 @@ const formatFlowMetric = (value: number, suffix?: string): string => { const useTerminalState = () => { const pathname = usePathname(); + const routeFeatures = useMemo(() => getRouteFeatures(pathname), [pathname]); const [mode, setMode] = useState("live"); const [replaySource, setReplaySource] = useState(null); const [selectedAlert, setSelectedAlert] = useState(null); @@ -4711,13 +4904,7 @@ const useTerminalState = () => { optionScope, equityScope ); - const equitiesLiveSubscriptionActive = useMemo( - () => - getLiveManifest(pathname, chartTicker.toUpperCase(), chartIntervalMs, flowFilters, optionScope, equityScope).some( - (sub) => sub.channel === "equities" - ), - [pathname, chartTicker, chartIntervalMs, flowFilters, optionScope, equityScope] - ); + const equitiesLiveSubscriptionActive = routeFeatures.equities; const handleReplaySource = useCallback((value: string | null) => { setReplaySource(value); @@ -5080,16 +5267,6 @@ const useTerminalState = () => { return map; }, [optionsFeed.items]); - const equityPrintMap = useMemo(() => { - const map = new Map(); - for (const print of equitiesFeed.items) { - if (print.trace_id) { - map.set(print.trace_id, print); - } - } - return map; - }, [equitiesFeed.items]); - const equityJoinMap = useMemo(() => { const map = new Map(); for (const join of equityJoinsFeed.items) { @@ -5317,11 +5494,11 @@ const useTerminalState = () => { }, [selectedDarkEvent, resolvedEquityJoinMap]); const selectedDarkUnderlying = useMemo(() => { - if (!selectedDarkEvent) { + if (!routeFeatures.needsDarkUnderlying || !selectedDarkEvent) { return null; } - return inferDarkUnderlying(selectedDarkEvent, equityPrintMap, resolvedEquityJoinMap); - }, [selectedDarkEvent, resolvedEquityJoinMap, equityPrintMap]); + return inferDarkUnderlying(selectedDarkEvent, resolvedEquityJoinMap); + }, [routeFeatures.needsDarkUnderlying, selectedDarkEvent, resolvedEquityJoinMap]); useEffect(() => { if (mode !== "live") { @@ -5358,6 +5535,9 @@ const useTerminalState = () => { }, []); const classifierHitsByPacketId = useMemo(() => { + if (!routeFeatures.needsClassifierDecor) { + return EMPTY_CLASSIFIER_HITS_BY_PACKET_ID; + } const map = new Map(); for (const hit of [...classifierHitsFeed.items, ...optionSupportClassifierHits]) { const packetId = extractPacketIdFromClassifierHitTrace(hit.trace_id); @@ -5367,9 +5547,17 @@ const useTerminalState = () => { map.set(packetId, [...(map.get(packetId) ?? []), hit]); } return map; - }, [classifierHitsFeed.items, optionSupportClassifierHits, extractPacketIdFromClassifierHitTrace]); + }, [ + classifierHitsFeed.items, + optionSupportClassifierHits, + extractPacketIdFromClassifierHitTrace, + routeFeatures.needsClassifierDecor + ]); const smartMoneyByPacketId = useMemo(() => { + if (!routeFeatures.needsClassifierDecor) { + return new Map(); + } const map = new Map(); for (const event of [...smartMoneyFeed.items, ...optionSupportSmartMoney]) { for (const packetId of event.packet_ids) { @@ -5380,9 +5568,12 @@ const useTerminalState = () => { } } return map; - }, [smartMoneyFeed.items, optionSupportSmartMoney]); + }, [smartMoneyFeed.items, optionSupportSmartMoney, routeFeatures.needsClassifierDecor]); const packetIdByOptionTraceId = useMemo(() => { + if (!routeFeatures.needsClassifierDecor) { + return EMPTY_PACKET_ID_BY_OPTION_TRACE_ID; + } const map = new Map(); for (const packet of resolvedFlowPacketMap.values()) { for (const member of packet.members) { @@ -5390,9 +5581,12 @@ const useTerminalState = () => { } } return map; - }, [resolvedFlowPacketMap]); + }, [resolvedFlowPacketMap, routeFeatures.needsClassifierDecor]); const classifierDecorByOptionTraceId = useMemo(() => { + if (!routeFeatures.needsClassifierDecor) { + return EMPTY_CLASSIFIER_DECOR_BY_OPTION_TRACE_ID; + } const map = new Map(); for (const [traceId, packetId] of packetIdByOptionTraceId) { const smartMoneyEvent = smartMoneyByPacketId.get(packetId); @@ -5406,10 +5600,15 @@ const useTerminalState = () => { } } return map; - }, [classifierHitsByPacketId, packetIdByOptionTraceId, smartMoneyByPacketId]); + }, [ + classifierHitsByPacketId, + packetIdByOptionTraceId, + smartMoneyByPacketId, + routeFeatures.needsClassifierDecor + ]); useEffect(() => { - if (mode !== "live" || optionsFeed.items.length === 0) { + if (!routeFeatures.needsClassifierDecor || mode !== "live" || optionsFeed.items.length === 0) { return; } @@ -5525,7 +5724,8 @@ const useTerminalState = () => { optionsFeed.items, classifierDecorByOptionTraceId, packetIdByOptionTraceId, - historicalNbboByTraceId + historicalNbboByTraceId, + routeFeatures.needsClassifierDecor ]); const selectedClassifierPacketId = useMemo(() => { @@ -5874,14 +6074,23 @@ const useTerminalState = () => { }, [equitiesScopedQuiet, optionsScopedQuiet]); const filteredInferredDark = useMemo(() => { + if (!routeFeatures.inferredDark) { + return EMPTY_INFERRED_DARK_EVENTS; + } if (tickerSet.size === 0) { return inferredDarkFeed.items; } return inferredDarkFeed.items.filter((event) => { - const underlying = inferDarkUnderlying(event, equityPrintMap, resolvedEquityJoinMap); + const underlying = inferDarkUnderlying(event, resolvedEquityJoinMap); return matchesTicker(underlying); }); - }, [resolvedEquityJoinMap, equityPrintMap, inferredDarkFeed.items, matchesTicker, tickerSet]); + }, [ + resolvedEquityJoinMap, + inferredDarkFeed.items, + matchesTicker, + tickerSet, + routeFeatures.inferredDark + ]); const filteredFlow = useMemo(() => { return flowFeed.items.filter((packet) => { @@ -5896,13 +6105,31 @@ const useTerminalState = () => { }, [flowFeed.items, flowFilters, extractPacketContract, matchesTicker, tickerSet]); const filteredAlerts = useMemo(() => { + if (!routeFeatures.showAlertsPane && !routeFeatures.needsAlertEvidencePrefetch) { + return EMPTY_ALERT_EVENTS; + } if (tickerSet.size === 0) { return alertsFeed.items; } return alertsFeed.items.filter((alert) => matchesTicker(inferAlertUnderlying(alert))); - }, [alertsFeed.items, inferAlertUnderlying, matchesTicker, tickerSet]); + }, [ + alertsFeed.items, + inferAlertUnderlying, + matchesTicker, + tickerSet, + routeFeatures.showAlertsPane, + routeFeatures.needsAlertEvidencePrefetch + ]); - const visibleAlerts = useMemo(() => filteredAlerts.slice(0, 12), [filteredAlerts]); + const visibleAlerts = useMemo(() => { + if (routeFeatures.needsAlertEvidencePrefetch) { + return filteredAlerts.slice(0, 12); + } + if (routeFeatures.showAlertsPane) { + return filteredAlerts.slice(0, 12); + } + return EMPTY_ALERT_EVENTS; + }, [filteredAlerts, routeFeatures.needsAlertEvidencePrefetch, routeFeatures.showAlertsPane]); const visibleAlertEvidenceRefs = useMemo(() => { const refs = new Set(); @@ -5915,7 +6142,7 @@ const useTerminalState = () => { }, [visibleAlerts]); useEffect(() => { - if (mode !== "live" || visibleAlerts.length === 0) { + if (!routeFeatures.needsAlertEvidencePrefetch || mode !== "live" || visibleAlerts.length === 0) { return; } @@ -5997,7 +6224,8 @@ const useTerminalState = () => { visibleAlerts, visibleAlertEvidenceRefs, resolvedFlowPacketMap, - resolvedOptionPrintMap + resolvedOptionPrintMap, + routeFeatures.needsAlertEvidencePrefetch ]); const activePinnedFlowKeys = useMemo(() => { @@ -6083,6 +6311,9 @@ const useTerminalState = () => { }, []); const filteredClassifierHits = useMemo(() => { + if (!routeFeatures.classifierHits) { + return EMPTY_CLASSIFIER_HIT_EVENTS; + } if (tickerSet.size === 0) { return classifierHitsFeed.items; } @@ -6090,16 +6321,28 @@ const useTerminalState = () => { const underlying = extractUnderlyingFromTrace(hit.trace_id); return matchesTicker(underlying); }); - }, [classifierHitsFeed.items, extractUnderlyingFromTrace, matchesTicker, tickerSet]); + }, [ + classifierHitsFeed.items, + extractUnderlyingFromTrace, + matchesTicker, + tickerSet, + routeFeatures.classifierHits + ]); const filteredSmartMoneyEvents = useMemo(() => { + if (!routeFeatures.smartMoney) { + return EMPTY_SMART_MONEY_EVENTS; + } if (tickerSet.size === 0) { return smartMoneyFeed.items; } return smartMoneyFeed.items.filter((event) => matchesTicker(event.underlying_id)); - }, [matchesTicker, smartMoneyFeed.items, tickerSet]); + }, [matchesTicker, smartMoneyFeed.items, tickerSet, routeFeatures.smartMoney]); const chartSmartMoneyEvents = useMemo(() => { + if (!routeFeatures.showChartPane && !routeFeatures.showFocusPane) { + return EMPTY_SMART_MONEY_EVENTS; + } const desired = chartTicker.toUpperCase(); return smartMoneyFeed.items .filter((event) => event.underlying_id.toUpperCase() === desired) @@ -6110,12 +6353,15 @@ const useTerminalState = () => { } return a.seq - b.seq; }); - }, [chartTicker, smartMoneyFeed.items]); + }, [chartTicker, smartMoneyFeed.items, routeFeatures.showChartPane, routeFeatures.showFocusPane]); const chartInferredDark = useMemo(() => { + if (!routeFeatures.showChartPane && !routeFeatures.showFocusPane) { + return EMPTY_INFERRED_DARK_EVENTS; + } const desired = chartTicker.toUpperCase(); return inferredDarkFeed.items - .filter((event) => inferDarkUnderlying(event, equityPrintMap, resolvedEquityJoinMap) === desired) + .filter((event) => inferDarkUnderlying(event, resolvedEquityJoinMap) === desired) .sort((a, b) => { const delta = a.source_ts - b.source_ts; if (delta !== 0) { @@ -6123,7 +6369,13 @@ const useTerminalState = () => { } return a.seq - b.seq; }); - }, [chartTicker, inferredDarkFeed.items, resolvedEquityJoinMap, equityPrintMap]); + }, [ + chartTicker, + inferredDarkFeed.items, + resolvedEquityJoinMap, + routeFeatures.showChartPane, + routeFeatures.showFocusPane + ]); const findAlertForClassifierHit = useCallback( (hit: ClassifierHitEvent): AlertEvent | null => { @@ -6183,18 +6435,47 @@ const useTerminalState = () => { }, []); const lastSeen = useMemo(() => { - return [ - optionsFeed.lastUpdate, - equitiesFeed.lastUpdate, - inferredDarkFeed.lastUpdate, - flowFeed.lastUpdate, - alertsFeed.lastUpdate, - smartMoneyFeed.lastUpdate, - classifierHitsFeed.lastUpdate - ] + const updates: Array = []; + if (routeFeatures.options || routeFeatures.showOptionsPane) { + updates.push(optionsFeed.lastUpdate); + } + if (routeFeatures.equities || routeFeatures.showEquitiesPane) { + updates.push(equitiesFeed.lastUpdate); + } + if (routeFeatures.inferredDark || routeFeatures.showDarkPane || routeFeatures.showFocusPane) { + updates.push(inferredDarkFeed.lastUpdate); + } + if (routeFeatures.flow || routeFeatures.showFlowPane) { + updates.push(flowFeed.lastUpdate); + } + if (routeFeatures.alerts || routeFeatures.showAlertsPane) { + updates.push(alertsFeed.lastUpdate); + } + if (routeFeatures.smartMoney || routeFeatures.showClassifierPane || routeFeatures.showChartPane || routeFeatures.showFocusPane) { + updates.push(smartMoneyFeed.lastUpdate); + } + if (routeFeatures.classifierHits || routeFeatures.showClassifierPane) { + updates.push(classifierHitsFeed.lastUpdate); + } + return updates .filter((value): value is number => value !== null) .sort((a, b) => b - a)[0] ?? null; }, [ + routeFeatures.options, + routeFeatures.showOptionsPane, + routeFeatures.equities, + routeFeatures.showEquitiesPane, + routeFeatures.inferredDark, + routeFeatures.showDarkPane, + routeFeatures.showFocusPane, + routeFeatures.flow, + routeFeatures.showFlowPane, + routeFeatures.alerts, + routeFeatures.showAlertsPane, + routeFeatures.smartMoney, + routeFeatures.showClassifierPane, + routeFeatures.showChartPane, + routeFeatures.classifierHits, optionsFeed.lastUpdate, equitiesFeed.lastUpdate, inferredDarkFeed.lastUpdate, @@ -6242,13 +6523,13 @@ const useTerminalState = () => { smartMoney: smartMoneyFeed, classifierHits: classifierHitsFeed, liveSession, + routeFeatures, activeTickers, tickerSet, chartTicker, nbboMap, historicalNbboByTraceId, optionPrintMap: resolvedOptionPrintMap, - equityPrintMap, equityJoinMap: resolvedEquityJoinMap, flowPacketMap: resolvedFlowPacketMap, classifierHitsByPacketId, @@ -6512,36 +6793,6 @@ export const FlowFilterPopover = ({ filters, onChange }: FlowFilterPopoverProps) ); }; -const FlowFilterControls = () => { - const state = useTerminal(); - - return ; -}; - -const ContractFilterControl = () => { - const state = useTerminal(); - const selected = state.selectedInstrument; - const isContractFilterActive = selected?.kind === "option-contract"; - - return ( - - ); -}; - type PaneProps = { title: string; status?: ReactNode; @@ -6596,13 +6847,13 @@ const ShellMetricStrip = () => { }; type OptionsPaneProps = { + state: TerminalState; limit?: number; }; -const OptionsPane = ({ limit }: OptionsPaneProps) => { - const state = useTerminal(); +const OptionsPane = memo(({ state, limit }: OptionsPaneProps) => { const items = limit ? state.filteredOptions.slice(0, limit) : state.filteredOptions; - const virtual = useMeasuredVirtualList(items, state.optionsScroll.listRef, 36, 12, "options"); + const virtual = useTapeVirtualList(items, state.optionsScroll.listRef, getTapeVirtualConfig("options")); useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("options") ); @@ -6647,7 +6898,7 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."} ) : ( -
+
TIME @@ -6663,117 +6914,117 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { IV CLASSIFIER
-
- {virtual.virtualItems.map(({ item: print, key, index, start, size }) => { - const contractId = normalizeContractId(print.option_contract_id); - const parsed = parseOptionContractId(contractId); - const contractDisplay = formatOptionContractLabel(contractId); - const quote = state.historicalNbboByTraceId.get(print.trace_id) ?? state.nbboMap.get(contractId); - const hasPreservedNbbo = typeof print.execution_nbbo_side === "string"; - const nbboSide = - print.execution_nbbo_side ?? - print.nbbo_side ?? - (!hasPreservedNbbo ? classifyNbboSide(print.price, quote) : null); - const notional = print.notional ?? print.price * print.size * 100; - const spot = print.execution_underlying_spot; - const iv = print.execution_iv; - const decor = state.classifierDecorByOptionTraceId.get(print.trace_id); - const underlyingId = (print.underlying_id ?? parsed?.root ?? extractUnderlying(contractId)).toUpperCase(); - const focusContract = (event: ReactMouseEvent) => { - event.stopPropagation(); - state.focusOptionContract(print); - }; - const rowStyle = { - ...(decor - ? ({ "--classifier-intensity": decor.intensity } as CSSProperties) - : undefined), - top: `${start}px` - } as CSSProperties; - const commonProps = { - className: `data-table-row data-table-row-button data-table-row-classified data-table-row-options data-table-virtual-row${index % 2 === 1 ? " is-even" : ""}${decor ? ` is-classified classifier-${decor.tone}` : ""}`, - style: rowStyle, - "data-index": index, - "data-row-start": String(start), - "data-row-size": String(size), - "data-tape-key": key, - ref: virtual.measureElement - }; - const cells = ( - <> - {formatTime(print.ts)} - - - - - - - - - - - - - {typeof spot === "number" ? formatPrice(spot) : "--"} - - {formatSize(print.size)}@{formatPrice(print.price)}_{nbboSide ?? "--"} - - {print.option_type ?? "--"} - ${formatCompactUsd(notional)} - - {nbboSide ? ( - {nbboSide} - ) : ( - "--" - )} - - {typeof iv === "number" ? formatPct(iv) : "--"} - {decor ? humanizeClassifierId(decor.family) : "--"} - - ); - - return decor ? ( - - ) : ( -
- {cells} + {virtual.virtualItems.map(({ item: print, key, index, start, size }) => { + const contractId = normalizeContractId(print.option_contract_id); + const parsed = parseOptionContractId(contractId); + const contractDisplay = formatOptionContractLabel(contractId); + const quote = state.historicalNbboByTraceId.get(print.trace_id) ?? state.nbboMap.get(contractId); + const hasPreservedNbbo = typeof print.execution_nbbo_side === "string"; + const nbboSide = + print.execution_nbbo_side ?? + print.nbbo_side ?? + (!hasPreservedNbbo ? classifyNbboSide(print.price, quote) : null); + const notional = print.notional ?? print.price * print.size * 100; + const spot = print.execution_underlying_spot; + const iv = print.execution_iv; + const decor = state.classifierDecorByOptionTraceId.get(print.trace_id); + const focusContract = (event: ReactMouseEvent) => { + event.stopPropagation(); + state.focusOptionContract(print); + }; + const rowStyle = { + ...(decor + ? ({ "--classifier-intensity": decor.intensity } as CSSProperties) + : undefined), + transform: `translateY(${start}px)` + } as CSSProperties; + const commonProps = { + className: `data-table-row data-table-row-button data-table-row-classified data-table-row-options data-table-virtual-row${index % 2 === 1 ? " is-even" : ""}${decor ? ` is-classified classifier-${decor.tone}` : ""}`, + style: rowStyle, + "data-index": index, + "data-row-start": String(start), + "data-row-size": String(size), + "data-tape-key": key + }; + const cells = ( + <> + {formatTime(print.ts)} + + + + + + + + + + + + + {typeof spot === "number" ? formatPrice(spot) : "--"} + + {formatSize(print.size)}@{formatPrice(print.price)}_{nbboSide ?? "--"} + + {print.option_type ?? "--"} + ${formatCompactUsd(notional)} + + {nbboSide ? ( + {nbboSide} + ) : ( + "--" + )} + + {typeof iv === "number" ? formatPct(iv) : "--"} + {decor ? humanizeClassifierId(decor.family) : "--"} + + ); + + return decor ? ( + + ) : ( +
+ {cells} +
+ ); + })}
- ); - })}
@@ -6781,16 +7032,16 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => {
); -}; +}); type EquitiesPaneProps = { + state: TerminalState; limit?: number; }; -const EquitiesPane = ({ limit }: EquitiesPaneProps) => { - const state = useTerminal(); +const EquitiesPane = memo(({ state, limit }: EquitiesPaneProps) => { const items = limit ? state.filteredEquities.slice(0, limit) : state.filteredEquities; - const virtual = useMeasuredVirtualList(items, state.equitiesScroll.listRef, 36, 10, "equities"); + const virtual = useTapeVirtualList(items, state.equitiesScroll.listRef, getTapeVirtualConfig("equities")); useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("equities") ); @@ -6837,7 +7088,7 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."} ) : ( -
+
TIME @@ -6847,52 +7098,53 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => { VENUE TAPE
-
- {virtual.virtualItems.map(({ item: print, key, index, start, size }) => ( -
- {formatTime(print.ts)} - - - - ${formatPrice(print.price)} - {formatSize(print.size)}x - {print.exchange} - {print.offExchangeFlag ? "Off-Ex" : "Lit"} +
+
+ {virtual.virtualItems.map(({ item: print, key, index, start, size }) => ( +
+ {formatTime(print.ts)} + + + + ${formatPrice(print.price)} + {formatSize(print.size)}x + {print.exchange} + {print.offExchangeFlag ? "Off-Ex" : "Lit"} +
+ ))} +
- ))} -
)}
); -}; +}); type FlowPaneProps = { + state: TerminalState; limit?: number; title?: string; }; -const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { - const state = useTerminal(); +const FlowPane = memo(({ state, limit, title = "Flow" }: FlowPaneProps) => { const items = limit ? state.filteredFlow.slice(0, limit) : state.filteredFlow; - const virtual = useMeasuredVirtualList(items, state.flowScroll.listRef, 44, 8, "flow"); + const virtual = useTapeVirtualList(items, state.flowScroll.listRef, getTapeVirtualConfig("flow")); useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("flow") ); @@ -6933,7 +7185,7 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."}
) : ( -
+
TIME @@ -6946,100 +7198,101 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { NBBO QUALITY
-
- {virtual.virtualItems.map(({ item: packet, key, index, start, size }) => { - const features = packet.features ?? {}; - const contract = String(features.option_contract_id ?? packet.id ?? "unknown"); - const count = parseNumber(features.count, packet.members.length); - const totalSize = parseNumber(features.total_size, 0); - const totalNotional = parseNumber(features.total_notional, Number.NaN); - const notional = Number.isFinite(totalNotional) - ? totalNotional - : parseNumber(features.total_premium, 0) * 100; - const startTs = parseNumber(features.start_ts, packet.source_ts); - const endTs = parseNumber(features.end_ts, startTs); - const windowMs = parseNumber(features.window_ms, 0); - const structureType = - typeof features.structure_type === "string" ? features.structure_type : ""; - const structureLegs = parseNumber(features.structure_legs, 0); - const structureRights = - typeof features.structure_rights === "string" ? features.structure_rights : ""; - const structureStrikes = parseNumber(features.structure_strikes, 0); - const nbboBid = parseNumber(features.nbbo_bid, Number.NaN); - const nbboAsk = parseNumber(features.nbbo_ask, Number.NaN); - const nbboMid = parseNumber(features.nbbo_mid, Number.NaN); - const nbboSpread = parseNumber(features.nbbo_spread, Number.NaN); - const aggressiveBuyRatio = parseNumber(features.nbbo_aggressive_buy_ratio, Number.NaN); - const aggressiveSellRatio = parseNumber( - features.nbbo_aggressive_sell_ratio, - Number.NaN - ); - const aggressiveCoverage = parseNumber(features.nbbo_coverage_ratio, Number.NaN); - const insideRatio = parseNumber(features.nbbo_inside_ratio, Number.NaN); - const nbboAge = parseNumber(packet.join_quality.nbbo_age_ms, Number.NaN); - const nbboStale = parseNumber(packet.join_quality.nbbo_stale, 0) > 0; - const nbboMissing = parseNumber(packet.join_quality.nbbo_missing, 0) > 0; - const structureLabel = structureType - ? `${structureType.replace(/_/g, " ")}${structureRights ? ` ${structureRights}` : ""}${structureLegs > 0 ? ` ${structureLegs}L` : ""}${structureStrikes > 0 ? ` ${structureStrikes}K` : ""}` - : "--"; - const nbboLabel = Number.isFinite(nbboBid) && Number.isFinite(nbboAsk) - ? `${formatPrice(nbboBid)} x ${formatPrice(nbboAsk)}` - : Number.isFinite(nbboMid) - ? `Mid ${formatPrice(nbboMid)}` - : "--"; - const qualityLabel = [ - Number.isFinite(aggressiveCoverage) && aggressiveCoverage > 0 - ? `Agg ${formatPct(aggressiveBuyRatio)}/${formatPct(aggressiveSellRatio)} ${formatPct(aggressiveCoverage)} cov` - : null, - Number.isFinite(insideRatio) && insideRatio > 0 ? `In ${formatPct(insideRatio)}` : null, - Number.isFinite(nbboSpread) ? `Spr ${formatPrice(nbboSpread)}` : null, - Number.isFinite(nbboAge) ? `${Math.round(nbboAge)}ms` : null, - nbboStale ? "Stale" : null, - nbboMissing ? "Missing" : null - ].filter(Boolean).join(" | "); +
+
+ {virtual.virtualItems.map(({ item: packet, key, index, start, size }) => { + const features = packet.features ?? {}; + const contract = String(features.option_contract_id ?? packet.id ?? "unknown"); + const count = parseNumber(features.count, packet.members.length); + const totalSize = parseNumber(features.total_size, 0); + const totalNotional = parseNumber(features.total_notional, Number.NaN); + const notional = Number.isFinite(totalNotional) + ? totalNotional + : parseNumber(features.total_premium, 0) * 100; + const startTs = parseNumber(features.start_ts, packet.source_ts); + const endTs = parseNumber(features.end_ts, startTs); + const windowMs = parseNumber(features.window_ms, 0); + const structureType = + typeof features.structure_type === "string" ? features.structure_type : ""; + const structureLegs = parseNumber(features.structure_legs, 0); + const structureRights = + typeof features.structure_rights === "string" ? features.structure_rights : ""; + const structureStrikes = parseNumber(features.structure_strikes, 0); + const nbboBid = parseNumber(features.nbbo_bid, Number.NaN); + const nbboAsk = parseNumber(features.nbbo_ask, Number.NaN); + const nbboMid = parseNumber(features.nbbo_mid, Number.NaN); + const nbboSpread = parseNumber(features.nbbo_spread, Number.NaN); + const aggressiveBuyRatio = parseNumber(features.nbbo_aggressive_buy_ratio, Number.NaN); + const aggressiveSellRatio = parseNumber( + features.nbbo_aggressive_sell_ratio, + Number.NaN + ); + const aggressiveCoverage = parseNumber(features.nbbo_coverage_ratio, Number.NaN); + const insideRatio = parseNumber(features.nbbo_inside_ratio, Number.NaN); + const nbboAge = parseNumber(packet.join_quality.nbbo_age_ms, Number.NaN); + const nbboStale = parseNumber(packet.join_quality.nbbo_stale, 0) > 0; + const nbboMissing = parseNumber(packet.join_quality.nbbo_missing, 0) > 0; + const structureLabel = structureType + ? `${structureType.replace(/_/g, " ")}${structureRights ? ` ${structureRights}` : ""}${structureLegs > 0 ? ` ${structureLegs}L` : ""}${structureStrikes > 0 ? ` ${structureStrikes}K` : ""}` + : "--"; + const nbboLabel = Number.isFinite(nbboBid) && Number.isFinite(nbboAsk) + ? `${formatPrice(nbboBid)} x ${formatPrice(nbboAsk)}` + : Number.isFinite(nbboMid) + ? `Mid ${formatPrice(nbboMid)}` + : "--"; + const qualityLabel = [ + Number.isFinite(aggressiveCoverage) && aggressiveCoverage > 0 + ? `Agg ${formatPct(aggressiveBuyRatio)}/${formatPct(aggressiveSellRatio)} ${formatPct(aggressiveCoverage)} cov` + : null, + Number.isFinite(insideRatio) && insideRatio > 0 ? `In ${formatPct(insideRatio)}` : null, + Number.isFinite(nbboSpread) ? `Spr ${formatPrice(nbboSpread)}` : null, + Number.isFinite(nbboAge) ? `${Math.round(nbboAge)}ms` : null, + nbboStale ? "Stale" : null, + nbboMissing ? "Missing" : null + ].filter(Boolean).join(" | "); - return ( -
- {formatTime(startTs)} → {formatTime(endTs)} - {contract} - {formatFlowMetric(count)} - {formatFlowMetric(totalSize)} - ${formatUsd(notional)} - {windowMs > 0 ? formatFlowMetric(windowMs, "ms") : "--"} - {structureLabel} - {nbboLabel} - {qualityLabel || "--"} + return ( +
+ {formatTime(startTs)} → {formatTime(endTs)} + {contract} + {formatFlowMetric(count)} + {formatFlowMetric(totalSize)} + ${formatUsd(notional)} + {windowMs > 0 ? formatFlowMetric(windowMs, "ms") : "--"} + {structureLabel} + {nbboLabel} + {qualityLabel || "--"} +
+ ); + })} +
- ); - })} -
)}
); -}; +}); type AlertsPaneProps = { + state: TerminalState; limit?: number; withStrip?: boolean; className?: string; }; -const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => { - const state = useTerminal(); +const AlertsPane = memo(({ state, limit, withStrip = false, className }: AlertsPaneProps) => { const items = limit ? state.filteredAlerts.slice(0, limit) : state.filteredAlerts; - const virtual = useMeasuredVirtualList(items, state.alertsScroll.listRef, 46, 8, "alerts"); + const virtual = useTapeVirtualList(items, state.alertsScroll.listRef, getTapeVirtualConfig("alerts")); useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("alerts") ); @@ -7080,7 +7333,7 @@ const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => : "Replay queue empty. Ensure ClickHouse has data."}
) : ( -
+
TIME @@ -7091,56 +7344,57 @@ const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => DIR NOTE
-
- {virtual.virtualItems.map(({ item: alert, key, index, start, size }) => { - const primary = alert.hits[0]; - const direction = deriveAlertDirection(alert); - const severity = normalizeAlertSeverity(alert); +
+
+ {virtual.virtualItems.map(({ item: alert, key, index, start, size }) => { + const primary = alert.hits[0]; + const direction = deriveAlertDirection(alert); + const severity = normalizeAlertSeverity(alert); - return ( - - ); - })} -
+ return ( + + ); + })} +
+
)}
); -}; +}); type ClassifierPaneProps = { + state: TerminalState; limit?: number; className?: string; }; -const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { - const state = useTerminal(); +const ClassifierPane = memo(({ state, limit, className }: ClassifierPaneProps) => { const smartMoneyItems = limit ? state.filteredSmartMoneyEvents.slice(0, limit) : state.filteredSmartMoneyEvents; const legacyItems = smartMoneyItems.length === 0 @@ -7150,7 +7404,7 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { : []; const items: Array = smartMoneyItems.length > 0 ? smartMoneyItems : legacyItems; - const virtual = useMeasuredVirtualList(items, state.classifierScroll.listRef, 44, 8, "classifier"); + const virtual = useTapeVirtualList(items, state.classifierScroll.listRef, getTapeVirtualConfig("classifier")); useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => { void state.liveSession.loadOlder("smart-money"); void state.liveSession.loadOlder("classifier-hits"); @@ -7192,7 +7446,7 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."} ) : ( -
+
TIME @@ -7201,81 +7455,81 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { PROB NOTE
-
- {showingSmartMoney ? virtual.virtualItems.map(({ item, key, index, start, size }) => { - const event = item as SmartMoneyEvent; - const primaryScore = - event.profile_scores.find((score) => score.profile_id === event.primary_profile_id) ?? - event.profile_scores[0]; - const direction = normalizeDirection(event.primary_direction); - return ( - - ); - }) : virtual.virtualItems.map(({ item, key, index, start, size }) => { - const hit = item as ClassifierHitEvent; - const direction = normalizeDirection(hit.direction); - return ( - - ); - })} -
+
+
+ {showingSmartMoney ? virtual.virtualItems.map(({ item, key, index, start, size }) => { + const event = item as SmartMoneyEvent; + const primaryScore = + event.profile_scores.find((score) => score.profile_id === event.primary_profile_id) ?? + event.profile_scores[0]; + const direction = normalizeDirection(event.primary_direction); + return ( + + ); + }) : virtual.virtualItems.map(({ item, key, index, start, size }) => { + const hit = item as ClassifierHitEvent; + const direction = normalizeDirection(hit.direction); + return ( + + ); + })} +
+
)}
); -}; +}); type DarkPaneProps = { + state: TerminalState; limit?: number; className?: string; }; -const DarkPane = ({ limit, className }: DarkPaneProps) => { - const state = useTerminal(); +const DarkPane = memo(({ state, limit, className }: DarkPaneProps) => { const items = limit ? state.filteredInferredDark.slice(0, limit) : state.filteredInferredDark; - const virtual = useMeasuredVirtualList(items, state.darkScroll.listRef, 44, 8, "dark"); + const virtual = useTapeVirtualList(items, state.darkScroll.listRef, getTapeVirtualConfig("dark")); useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("inferred-dark") ); @@ -7315,7 +7569,7 @@ const DarkPane = ({ limit, className }: DarkPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."} ) : ( -
+
TIME @@ -7325,53 +7579,54 @@ const DarkPane = ({ limit, className }: DarkPaneProps) => { EVIDENCE NOTE
-
- {virtual.virtualItems.map(({ item: event, key, index, start, size }) => { - const underlying = inferDarkUnderlying(event, state.equityPrintMap, state.equityJoinMap); - const evidenceCount = event.evidence_refs.length; +
+
+ {virtual.virtualItems.map(({ item: event, key, index, start, size }) => { + const underlying = inferDarkUnderlying(event, state.equityJoinMap); + const evidenceCount = event.evidence_refs.length; - return ( - - ); - })} -
+ return ( + + ); + })} +
+
)}
); -}; +}); type ChartPaneProps = { + state: TerminalState; title?: string; }; -const ChartPane = ({ title = "Chart" }: ChartPaneProps) => { - const state = useTerminal(); +const ChartPane = memo(({ state, title = "Chart" }: ChartPaneProps) => { return ( { /> ); -}; +}); -const FocusPane = () => { - const state = useTerminal(); +const FocusPane = memo(({ state }: { state: TerminalState }) => { const hits = state.chartSmartMoneyEvents.slice(-10).reverse(); const dark = state.chartInferredDark.slice(-10).reverse(); @@ -7477,10 +7731,9 @@ const FocusPane = () => { ); -}; +}); -const ReplayConsole = () => { - const state = useTerminal(); +const ReplayConsole = memo(({ state }: { state: TerminalState }) => { const replayActive = state.mode === "replay"; return ( @@ -7512,7 +7765,7 @@ const ReplayConsole = () => { ); -}; +}); export function TerminalAppShell({ children }: { children: ReactNode }) { const state = useTerminalState(); @@ -7632,68 +7885,89 @@ export function TerminalAppShell({ children }: { children: ReactNode }) { } export function OverviewRoute() { + const state = useTerminal(); return (
- - - + + +
); } export function TapeRoute() { + const state = useTerminal(); return ( - - + + } >
- - - + + +
); } export function SignalsRoute() { + const state = useTerminal(); return (
- - - + + +
); } export function ChartsRoute() { + const state = useTerminal(); return (
- - + +
); } export function ReplayRoute() { + const state = useTerminal(); return (
- - - - + + + +
); diff --git a/deploy-branch.sh b/deploy-branch.sh new file mode 100755 index 0000000..c5961b8 --- /dev/null +++ b/deploy-branch.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +git fetch +git pull +docker compose up -d --build --force-recreate diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..9ea97a6 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +git fetch +git switch deployment +git pull +docker compose up -d --build --force-recreate From de9a965a6c0eed85f8455a60da24db129e92aa81 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Thu, 7 May 2026 22:16:21 -0400 Subject: [PATCH 11/15] Fix table flex sizing and move deploy scripts - Make data tables fill available height so scrolling behaves correctly - Relocate deploy scripts under `deploy/docker` --- apps/web/app/globals.css | 3 +++ deploy-branch.sh => deploy/docker/deploy-branch.sh | 0 deploy.sh => deploy/docker/deploy.sh | 0 3 files changed, 3 insertions(+) rename deploy-branch.sh => deploy/docker/deploy-branch.sh (100%) rename deploy.sh => deploy/docker/deploy.sh (100%) diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index ed3edc2..8cf07a3 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -954,6 +954,7 @@ h3 { } .data-table-wrap { + display: flex; flex: 1 1 auto; min-height: 0; overflow-x: auto; @@ -965,7 +966,9 @@ h3 { .data-table { display: flex; + flex: 1 1 auto; flex-direction: column; + height: 100%; min-height: 0; min-width: 980px; } diff --git a/deploy-branch.sh b/deploy/docker/deploy-branch.sh similarity index 100% rename from deploy-branch.sh rename to deploy/docker/deploy-branch.sh diff --git a/deploy.sh b/deploy/docker/deploy.sh similarity index 100% rename from deploy.sh rename to deploy/docker/deploy.sh From 73e25ddf7090bfc24f21ce911791c3ca4def7dfa Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Thu, 7 May 2026 22:17:52 -0400 Subject: [PATCH 12/15] Rename docker deployment scripts - Move branch and main deploy scripts under `deployment/docker` - Keep script contents unchanged --- {deploy => deployment}/docker/deploy-branch.sh | 0 {deploy => deployment}/docker/deploy.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {deploy => deployment}/docker/deploy-branch.sh (100%) rename {deploy => deployment}/docker/deploy.sh (100%) diff --git a/deploy/docker/deploy-branch.sh b/deployment/docker/deploy-branch.sh similarity index 100% rename from deploy/docker/deploy-branch.sh rename to deployment/docker/deploy-branch.sh diff --git a/deploy/docker/deploy.sh b/deployment/docker/deploy.sh similarity index 100% rename from deploy/docker/deploy.sh rename to deployment/docker/deploy.sh From b73e62bdba3164623d7efd742400f5b8631a862d Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Thu, 7 May 2026 23:37:32 -0400 Subject: [PATCH 13/15] Fix contract-focused options tape hydration --- .beads/issues.jsonl | 1 + apps/web/app/terminal.test.ts | 173 +++++++++++++ apps/web/app/terminal.tsx | 295 ++++++++++++++++------ services/api/src/index.ts | 87 +------ services/api/src/live.ts | 37 ++- services/api/src/option-queries.ts | 107 ++++++++ services/api/tests/live.test.ts | 69 +++++ services/api/tests/option-queries.test.ts | 59 +++++ 8 files changed, 657 insertions(+), 171 deletions(-) create mode 100644 services/api/src/option-queries.ts create mode 100644 services/api/tests/option-queries.test.ts diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index c755228..5ca3a9f 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-pre","title":"Fix contract-focused options tape hydration","description":"Implement contract-focused options tape hydration so focused contract views preserve the clicked seed row, stop reapplying broad flow filters in the Options pane, and use raw contract-scoped ClickHouse queries consistently across live snapshots, history, and replay. Includes frontend replay source-grouping changes and regression tests for focus seed durability, focused filtering, and contract-scoped API behavior.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T03:27:31Z","created_by":"dirtydishes","updated_at":"2026-05-08T03:37:18Z","started_at":"2026-05-08T03:27:35Z","closed_at":"2026-05-08T03:37:18Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-9xs","title":"Fix terminal hydration and virtual row measurement crash","description":"Fix client crash caused by options-support hydration on non-JSON/404 responses and satisfy tanstack virtual measured-row data-index requirement across virtualized tables.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T06:14:33Z","created_by":"dirtydishes","updated_at":"2026-05-07T06:17:09Z","started_at":"2026-05-07T06:14:43Z","closed_at":"2026-05-07T06:17:09Z","close_reason":"Completed: added data-index attributes on measured virtual rows, hardened options-support hydration error handling/content-type validation, and guarded trace-id hydration loops against malformed payload entries.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-35g","title":"Fix Docker deployment workspace lockfile drift","description":"Refresh deployment/docker workspace lockfile for Docker builds, add a drift guard for Docker-built workspaces, and document the separate deployment snapshot so frozen Bun installs cannot fail when repo dependencies change.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T06:02:06Z","created_by":"dirtydishes","updated_at":"2026-05-07T06:07:50Z","started_at":"2026-05-07T06:02:15Z","closed_at":"2026-05-07T06:07:50Z","close_reason":"Completed: synced deployment Docker workspace snapshot from repo root, refreshed deployment bun.lock, added sync/check scripts, and documented maintenance workflow. Local docker compose build validation is blocked here because Docker daemon is unavailable.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-2ij","title":"Harden tape virtualization, scoped focus, and live feed health","description":"Implement the coordinated tape stability plan across web and API.\n\nScope:\n- replace fixed-height tape virtualization with measured virtualization and virtual-end history loading\n- replace scrollHeight anchoring with key-based anchor restore\n- compose canonical tape lists across seed/live/history sources\n- preserve clicked contract/ticker context during scoped focus transitions\n- separate backend hot-channel health from scoped quiet empty states\n- shrink browser hot windows and modestly reduce server cache limits\n- add regression tests and development instrumentation\n\nAcceptance:\n- no giant blank spacer gaps during tape scrolling\n- scroll remains stable while live data and history mutate the list\n- clicked deep-history option/equity rows remain visible immediately after focus\n- narrow scopes do not surface Feed behind unless backend channel health is stale\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T05:35:18Z","created_by":"dirtydishes","updated_at":"2026-05-07T05:52:14Z","started_at":"2026-05-07T05:35:21Z","closed_at":"2026-05-07T05:52:14Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index e4d9a52..2ada99a 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -4,19 +4,23 @@ import { NAV_ITEMS, appendHistoryTail, buildDefaultFlowFilters, + buildOptionTapeQueryParams, classifierToneForFamily, composeTapeItems, deriveAlertDirection, countActiveFlowFilterGroups, + filterOptionTapeItems, findAnchorRestoreIndex, formatCompactUsd, formatOptionContractLabel, flushPausableTapeData, + getEffectiveOptionPrintFilters, getAlertWindowAnchorTs, getHotChannelFeedStatus, getScopedLiveAutoHydrationChannels, getLiveHistoryRetentionCap, getOptionTableSnapshot, + getOptionScope, getLiveFeedStatus, getLiveManifest, getRouteFeatures, @@ -30,6 +34,7 @@ import { shouldIncludeEquitiesForDarkUnderlyingFallback, shouldShowEquitiesSilentFeedWarning, selectPrimaryClassifierHit, + shouldClearOptionFocusSeed, smartMoneyProfileLabel, smartMoneyToneForProfile, statusLabel, @@ -42,6 +47,25 @@ const makeItem = (traceId: string, seq: number, ts: number) => ({ ts }); +const makeOptionPrint = (overrides: Record = {}) => + ({ + trace_id: "opt-1", + seq: 1, + ts: 1_000, + source_ts: 1_000, + ingest_ts: 1_001, + option_contract_id: "AAPL-2025-01-17-200-C", + underlying_id: "AAPL", + option_type: "call", + nbbo_side: "A", + notional: 250_000, + signal_pass: true, + price: 1, + size: 10, + exchange: "X", + ...overrides + }) as any; + const makeAlert = (overrides: Record = {}) => ({ trace_id: "alert-1", @@ -125,6 +149,31 @@ describe("live manifest", () => { expect(equitiesSubscription?.underlying_ids).toEqual(["AAPL"]); }); + it("drops option-print filters for contract-focused options subscriptions but keeps flow filters", () => { + const filters = { + ...buildDefaultFlowFilters(), + minNotional: 500_000, + optionTypes: ["put"] as const + }; + const manifest = getLiveManifest( + "/tape", + "AAPL", + 60000, + filters, + { + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }, + { underlying_ids: ["AAPL"] }, + undefined + ); + const optionsSubscription = manifest.find((subscription) => subscription.channel === "options"); + const flowSubscription = manifest.find((subscription) => subscription.channel === "flow"); + + expect(optionsSubscription?.filters).toBeUndefined(); + expect(flowSubscription?.filters).toBe(filters); + }); + it("scopes /signals subscriptions to signals channels only", () => { const channels = getLiveManifest("/signals", "SPY", 60000, buildDefaultFlowFilters()).map( (subscription) => subscription.channel @@ -154,6 +203,130 @@ describe("live manifest", () => { }); }); +describe("contract-focused option helpers", () => { + it("uses the focused contract underlying for option scope even when ticker input differs", () => { + expect( + getOptionScope(["MSFT"], "AAPL", { + kind: "option-contract", + contractId: "AAPL-2025-01-17-200-C", + underlyingId: "AAPL" + }) + ).toEqual({ + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }); + }); + + it("ignores broad flow filters for focused contract options", () => { + const filters = { + ...buildDefaultFlowFilters(), + minNotional: 500_000 + }; + const items = [ + makeOptionPrint({ + trace_id: "focused-low", + option_contract_id: "AAPL-2025-01-17-200-C", + notional: 100_000, + signal_pass: false + }), + makeOptionPrint({ + trace_id: "focused-high", + seq: 2, + ts: 2_000, + option_contract_id: "AAPL-2025-01-17-200-C", + notional: 750_000 + }), + makeOptionPrint({ + trace_id: "other-contract", + seq: 3, + ts: 3_000, + option_contract_id: "MSFT-2025-01-17-300-C", + underlying_id: "MSFT", + notional: 900_000 + }) + ]; + + expect( + filterOptionTapeItems( + items, + getEffectiveOptionPrintFilters(filters, true), + { + kind: "option-contract", + contractId: "AAPL-2025-01-17-200-C", + underlyingId: "AAPL" + }, + new Set(["MSFT"]), + "AAPL" + ).map((item) => item.trace_id) + ).toEqual(["focused-low", "focused-high"]); + }); + + it("includes option_contract_id and drops broad filters in focused replay query params", () => { + const filters = { + ...buildDefaultFlowFilters(), + minNotional: 500_000, + optionTypes: ["put"] as const + }; + + expect( + buildOptionTapeQueryParams(getEffectiveOptionPrintFilters(filters, true), { + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }) + ).toEqual({ + underlying_ids: "AAPL", + option_contract_id: "AAPL-2025-01-17-200-C" + }); + }); + + it("keeps the focus seed until the matching scoped subscription has loaded it", () => { + const seedItem = makeOptionPrint({ + trace_id: "focused-seed", + option_contract_id: "AAPL-2025-01-17-200-C" + }); + const seed = { + scopeKey: "option-contract:AAPL-2025-01-17-200-C", + subscriptionKey: getLiveSubscriptionKey({ + channel: "options", + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }), + items: [seedItem] + }; + + expect( + shouldClearOptionFocusSeed( + seed, + "option-contract:AAPL-2025-01-17-200-C", + getLiveSubscriptionKey({ + channel: "options", + filters: { + ...buildDefaultFlowFilters(), + minNotional: 500_000 + }, + underlying_ids: ["AAPL"] + }), + [makeOptionPrint({ trace_id: "broad-old" })], + [] + ) + ).toBe(false); + + expect( + shouldClearOptionFocusSeed( + seed, + "option-contract:AAPL-2025-01-17-200-C", + getLiveSubscriptionKey({ + channel: "options", + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }), + [seedItem], + [] + ) + ).toBe(true); + }); +}); + describe("route feature map", () => { it("maps /tape to tape panes and dependencies", () => { const features = getRouteFeatures("/tape"); diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 01ee884..854ea85 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -350,9 +350,17 @@ type SelectedInstrument = type TapeFocusSeed = { scopeKey: string; + subscriptionKey?: string; items: T[]; }; +type OptionScope = Pick< + Extract, + "underlying_ids" | "option_contract_id" +>; + +type EquityScope = Pick, "underlying_ids">; + const formatIntervalLabel = (intervalMs: number): string => { const match = CANDLE_INTERVALS.find((interval) => interval.ms === intervalMs); if (match) { @@ -1956,6 +1964,13 @@ const useTape = ( const replaySourceKey = config.replaySourceKey ?? null; const onReplaySourceKey = config.onReplaySourceKey; const queryParams = config.queryParams; + const queryKey = useMemo( + () => + JSON.stringify( + Object.entries(queryParams ?? {}).sort(([left], [right]) => left.localeCompare(right)) + ), + [queryParams] + ); const hotWindowLimit = config.hotWindowLimit ?? LIVE_HOT_WINDOW; const [status, setStatus] = useState("connecting"); const [items, setItems] = useState([]); @@ -2046,7 +2061,7 @@ const useTape = ( pendingRef.current = []; pendingCountRef.current = 0; cancelFlush(); - }, [mode, replaySourceKey, cancelFlush]); + }, [mode, replaySourceKey, queryKey, cancelFlush]); useEffect(() => { if (mode !== "replay" || !latestPath) { @@ -2091,7 +2106,7 @@ const useTape = ( return () => { active = false; }; - }, [mode, latestPath, getItemTs, replaySourceKey, queryParams]); + }, [mode, latestPath, getItemTs, replaySourceKey, queryKey, queryParams]); useEffect(() => { if (mode !== "live" || config.liveEnabled === false) { @@ -2242,9 +2257,14 @@ const useTape = ( } } - if (onReplaySourceKey && sourcePrefix && replaySourceNotifiedRef.current !== sourcePrefix) { - replaySourceNotifiedRef.current = sourcePrefix; - onReplaySourceKey(sourcePrefix); + if (onReplaySourceKey) { + if (sourcePrefix && replaySourceNotifiedRef.current !== sourcePrefix) { + replaySourceNotifiedRef.current = sourcePrefix; + onReplaySourceKey(sourcePrefix); + } else if (!sourcePrefix && replaySourceNotifiedRef.current !== null) { + replaySourceNotifiedRef.current = null; + onReplaySourceKey(null); + } } const filtered = sourcePrefix @@ -2330,6 +2350,7 @@ const useTape = ( getReplayKey, replaySourceKey, onReplaySourceKey, + queryKey, queryParams ]); @@ -2784,6 +2805,99 @@ const appendOptionFlowFilters = (params: URLSearchParams, filters: OptionFlowFil } }; +const appendOptionScopeParams = ( + params: URLSearchParams, + optionScope: OptionScope | undefined +): void => { + if (optionScope?.underlying_ids?.length) { + params.set("underlying_ids", optionScope.underlying_ids.join(",")); + } + if (optionScope?.option_contract_id) { + params.set("option_contract_id", optionScope.option_contract_id); + } +}; + +export const getEffectiveOptionPrintFilters = ( + flowFilters: OptionFlowFilters, + isOptionContractFocused: boolean +): OptionFlowFilters | undefined => { + return isOptionContractFocused ? undefined : flowFilters; +}; + +export const getOptionScope = ( + activeTickers: string[], + instrumentUnderlying: string | null, + selectedInstrument: SelectedInstrument +): OptionScope => ({ + underlying_ids: + selectedInstrument?.kind === "option-contract" + ? instrumentUnderlying + ? [instrumentUnderlying] + : undefined + : activeTickers.length > 0 + ? activeTickers + : instrumentUnderlying + ? [instrumentUnderlying] + : undefined, + option_contract_id: + selectedInstrument?.kind === "option-contract" ? selectedInstrument.contractId : undefined +}); + +export const buildOptionTapeQueryParams = ( + filters: OptionFlowFilters | undefined, + optionScope: OptionScope | undefined +): Record => { + const params = new URLSearchParams(); + appendOptionFlowFilters(params, filters); + appendOptionScopeParams(params, optionScope); + return Object.fromEntries(params.entries()); +}; + +export const filterOptionTapeItems = ( + items: OptionPrint[], + filters: OptionFlowFilters | undefined, + selectedInstrument: SelectedInstrument, + tickerSet: Set, + instrumentUnderlying: string | null +): OptionPrint[] => { + return items.filter((print) => { + const contractId = normalizeContractId(print.option_contract_id); + if (selectedInstrument?.kind === "option-contract") { + return contractId === selectedInstrument.contractId; + } + if (!matchesOptionPrintFilters(print, filters)) { + return false; + } + const underlying = extractUnderlying(contractId); + if (tickerSet.size === 0) { + return !instrumentUnderlying || underlying === instrumentUnderlying; + } + return Boolean(underlying) && tickerSet.has(underlying.toUpperCase()); + }); +}; + +export const shouldClearOptionFocusSeed = ( + seed: TapeFocusSeed | null, + optionFocusScopeKey: string | null, + currentOptionSubscriptionKey: string | null, + liveItems: OptionPrint[], + historyItems: OptionPrint[] +): boolean => { + if (!seed) { + return false; + } + if (seed.scopeKey !== optionFocusScopeKey) { + return true; + } + if (seed.subscriptionKey && seed.subscriptionKey !== currentOptionSubscriptionKey) { + return false; + } + const liveKeys = new Set( + composeTapeItems([], liveItems, historyItems).map((item) => getTapeItemKey(item)) + ); + return seed.items.every((item) => liveKeys.has(getTapeItemKey(item))); +}; + const appendLiveScopeParams = (params: URLSearchParams, subscription: LiveSubscription): void => { if ((subscription.channel === "options" || subscription.channel === "equities") && subscription.underlying_ids?.length) { params.set("underlying_ids", subscription.underlying_ids.join(",")); @@ -2810,8 +2924,9 @@ export const getLiveManifest = ( chartTicker: string, chartIntervalMs: number, flowFilters: OptionFlowFilters, - optionScope?: Pick, "underlying_ids" | "option_contract_id">, - equityScope?: Pick, "underlying_ids"> + optionScope?: OptionScope, + equityScope?: EquityScope, + optionPrintFilters?: OptionFlowFilters ): LiveSubscription[] => { const features = getRouteFeatures(pathname); const subscriptions: LiveSubscription[] = []; @@ -2819,7 +2934,10 @@ export const getLiveManifest = ( if (features.options) { subscriptions.push({ channel: "options", - filters: flowFilters, + filters: + optionScope?.option_contract_id && optionPrintFilters === undefined + ? undefined + : optionPrintFilters ?? flowFilters, ...optionScope, snapshot_limit: LIVE_HOT_WINDOW_OPTIONS }); @@ -2868,11 +2986,7 @@ export const getLiveManifest = ( const useLiveSession = ( enabled: boolean, pathname: string, - chartTicker: string, - chartIntervalMs: number, - flowFilters: OptionFlowFilters, - optionScope?: Pick, "underlying_ids" | "option_contract_id">, - equityScope?: Pick, "underlying_ids"> + manifest: LiveSubscription[] ): LiveSessionState => { const [status, setStatus] = useState(enabled ? "connecting" : "disconnected"); const [connectedAt, setConnectedAt] = useState(null); @@ -2938,11 +3052,6 @@ const useLiveSession = ( const lastEventAtRef = useRef(null); const subscribedKeysRef = useRef>(new Set()); const subscribedMapRef = useRef>(new Map()); - const manifest = useMemo( - () => getLiveManifest(pathname, chartTicker.toUpperCase(), chartIntervalMs, flowFilters, optionScope, equityScope), - [pathname, chartTicker, chartIntervalMs, flowFilters, optionScope, equityScope] - ); - const replaceArrayState = ( setter: Dispatch>, ref: { current: T[] }, @@ -4857,20 +4966,21 @@ const useTerminalState = () => { }, [filterInput]); const tickerSet = useMemo(() => new Set(activeTickers), [activeTickers]); const instrumentUnderlying = selectedInstrument?.underlyingId.toUpperCase() ?? null; + const isOptionContractFocused = selectedInstrument?.kind === "option-contract"; + const focusedOptionContractId = + selectedInstrument?.kind === "option-contract" ? selectedInstrument.contractId : null; const optionFocusScopeKey = - selectedInstrument?.kind === "option-contract" - ? `option-contract:${selectedInstrument.contractId}` - : null; + focusedOptionContractId ? `option-contract:${focusedOptionContractId}` : null; const equityFocusScopeKey = selectedInstrument?.kind === "equity" ? `equity:${selectedInstrument.underlyingId.toUpperCase()}` : null; + const effectiveOptionPrintFilters = useMemo( + () => getEffectiveOptionPrintFilters(flowFilters, isOptionContractFocused), + [flowFilters, isOptionContractFocused] + ); const optionScope = useMemo( - () => ({ - underlying_ids: activeTickers.length > 0 ? activeTickers : instrumentUnderlying ? [instrumentUnderlying] : undefined, - option_contract_id: - selectedInstrument?.kind === "option-contract" ? selectedInstrument.contractId : undefined - }), + () => getOptionScope(activeTickers, instrumentUnderlying, selectedInstrument), [activeTickers, instrumentUnderlying, selectedInstrument] ); const equityScope = useMemo( @@ -4895,14 +5005,39 @@ const useTerminalState = () => { ? `Contract: ${display.ticker} ${display.expiration} ${display.strike}` : `Contract: ${selectedInstrument.contractId}`; }, [selectedInstrument]); - const liveSession = useLiveSession( - mode === "live", - pathname, - chartTicker, - chartIntervalMs, - flowFilters, - optionScope, - equityScope + const liveManifest = useMemo( + () => + getLiveManifest( + pathname, + chartTicker.toUpperCase(), + chartIntervalMs, + flowFilters, + optionScope, + equityScope, + effectiveOptionPrintFilters + ), + [ + pathname, + chartTicker, + chartIntervalMs, + flowFilters, + optionScope, + equityScope, + effectiveOptionPrintFilters + ] + ); + const liveSession = useLiveSession(mode === "live", pathname, liveManifest); + const currentOptionSubscription = useMemo( + () => + liveManifest.find( + (subscription): subscription is Extract => + subscription.channel === "options" + ) ?? null, + [liveManifest] + ); + const currentOptionSubscriptionKey = useMemo( + () => (currentOptionSubscription ? getLiveSubscriptionKey(currentOptionSubscription) : null), + [currentOptionSubscription] ); const equitiesLiveSubscriptionActive = routeFeatures.equities; @@ -4966,18 +5101,8 @@ const useTerminalState = () => { ); const disableReplayGrouping = useCallback(() => null, []); const optionQueryParams = useMemo>( - () => ({ - view: flowFilters.view ?? "signal", - security: - flowFilters.securityTypes?.length === 1 ? flowFilters.securityTypes[0] : undefined, - side: flowFilters.nbboSides?.length ? flowFilters.nbboSides.join(",") : undefined, - type: flowFilters.optionTypes?.length ? flowFilters.optionTypes.join(",") : undefined, - min_notional: - typeof flowFilters.minNotional === "number" - ? String(flowFilters.minNotional) - : undefined - }), - [flowFilters] + () => buildOptionTapeQueryParams(effectiveOptionPrintFilters, optionScope), + [effectiveOptionPrintFilters, optionScope] ); const options = useTape({ @@ -4992,9 +5117,10 @@ const useTerminalState = () => { pollMs: mode === "replay" ? 200 : undefined, captureScroll: optionsAnchor.capture, onNewItems: optionsScroll.onNewItems, - getReplayKey: extractReplaySource, - onReplaySourceKey: handleReplaySource, - queryParams: optionQueryParams + getReplayKey: isOptionContractFocused ? disableReplayGrouping : extractReplaySource, + onReplaySourceKey: isOptionContractFocused ? undefined : handleReplaySource, + queryParams: optionQueryParams, + replaySourceKey: isOptionContractFocused ? null : replaySource }); const equities = useTape({ @@ -5010,6 +5136,12 @@ const useTerminalState = () => { onNewItems: equitiesScroll.onNewItems }); + useEffect(() => { + if (isOptionContractFocused && replaySource !== null) { + setReplaySource(null); + } + }, [isOptionContractFocused, replaySource]); + const equityJoins = useTape({ mode, liveEnabled: false, @@ -5922,25 +6054,20 @@ const useTerminalState = () => { ); const filteredOptions = useMemo(() => { - return optionsFeed.items.filter((print) => { - if (!matchesOptionPrintFilters(print, flowFilters)) { - return false; - } - if ( - selectedInstrument?.kind === "option-contract" && - normalizeContractId(print.option_contract_id) !== selectedInstrument.contractId - ) { - return false; - } - if (tickerSet.size === 0) { - return ( - !instrumentUnderlying || - extractUnderlying(normalizeContractId(print.option_contract_id)) === instrumentUnderlying - ); - } - return matchesTicker(extractUnderlying(normalizeContractId(print.option_contract_id))); - }); - }, [flowFilters, optionsFeed.items, matchesTicker, tickerSet, selectedInstrument, instrumentUnderlying]); + return filterOptionTapeItems( + optionsFeed.items, + effectiveOptionPrintFilters, + selectedInstrument, + tickerSet, + instrumentUnderlying + ); + }, [ + effectiveOptionPrintFilters, + instrumentUnderlying, + optionsFeed.items, + selectedInstrument, + tickerSet + ]); const filteredEquities = useMemo(() => { if (tickerSet.size === 0) { @@ -5956,16 +6083,24 @@ const useTerminalState = () => { if (!optionFocusSeed) { return; } - if (optionFocusSeed.scopeKey !== optionFocusScopeKey) { - setOptionFocusSeed(null); - return; - } - const composedBaseItems = composeTapeItems([], liveOptions.liveItems ?? [], liveOptions.historyItems ?? []); - const liveKeys = new Set(composedBaseItems.map((item) => getTapeItemKey(item))); - if (optionFocusSeed.items.every((item) => liveKeys.has(getTapeItemKey(item)))) { + if ( + shouldClearOptionFocusSeed( + optionFocusSeed, + optionFocusScopeKey, + currentOptionSubscriptionKey, + liveOptions.liveItems ?? [], + liveOptions.historyItems ?? [] + ) + ) { setOptionFocusSeed(null); } - }, [liveOptions.historyItems, liveOptions.liveItems, optionFocusScopeKey, optionFocusSeed]); + }, [ + currentOptionSubscriptionKey, + liveOptions.historyItems, + liveOptions.liveItems, + optionFocusScopeKey, + optionFocusSeed + ]); useEffect(() => { if (!equityFocusSeed) { @@ -5988,15 +6123,21 @@ const useTerminalState = () => { const parsed = parseOptionContractId(contractId); const underlyingId = (print.underlying_id ?? parsed?.root ?? extractUnderlying(contractId)).toUpperCase(); const scopeKey = `option-contract:${contractId}`; + const subscriptionKey = getLiveSubscriptionKey({ + channel: "options", + underlying_ids: [underlyingId], + option_contract_id: contractId + }); const seedItems = composeTapeItems( [print], filteredOptions.filter((candidate) => normalizeContractId(candidate.option_contract_id) === contractId), [] ); - setOptionFocusSeed({ scopeKey, items: seedItems }); + setOptionFocusSeed({ scopeKey, subscriptionKey, items: seedItems }); bumpTapeDebugMetric("focusSeedRowCount", seedItems.length); logTapeDebug("option focus seed captured", { contract_id: contractId, + subscription_key: subscriptionKey, row_count: seedItems.length }); setSelectedInstrument({ diff --git a/services/api/src/index.ts b/services/api/src/index.ts index 3035897..b7af494 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -82,7 +82,7 @@ import { fetchClassifierHitsByPacketIds, fetchRecentOptionPrints } from "@islandflow/storage"; -import type { EquityPrintQueryFilters, OptionPrintQueryFilters } from "@islandflow/storage"; +import type { EquityPrintQueryFilters } from "@islandflow/storage"; import { AlertEventSchema, ClassifierHitEventSchema, @@ -99,11 +99,6 @@ import { LiveSubscriptionSchema, matchesFlowPacketFilters, matchesOptionPrintFilters, - OptionFlowFilters, - OptionFlowViewSchema, - OptionNbboSideSchema, - OptionSecurityTypeSchema, - OptionTypeSchema, FlowPacketSchema, SmartMoneyEventSchema, OptionNBBOSchema, @@ -113,6 +108,7 @@ import { import { createClient } from "redis"; import { z } from "zod"; import { HOT_LIVE_REDIS_KEYS, LiveStateManager, shouldFanoutLiveEvent } from "./live"; +import { parseOptionPrintQuery } from "./option-queries"; const service = "api"; const logger = createLogger({ service }); @@ -224,33 +220,6 @@ const equityPrintRangeSchema = z.object({ end_ts: z.coerce.number().int().nonnegative(), limit: limitSchema.optional() }); -const optionSideListSchema = z - .string() - .transform((value) => - value - .split(",") - .map((entry) => entry.trim()) - .filter(Boolean) - ) - .pipe(z.array(OptionNbboSideSchema)); -const optionTypeListSchema = z - .string() - .transform((value) => - value - .split(",") - .map((entry) => entry.trim()) - .filter(Boolean) - ) - .pipe(z.array(OptionTypeSchema)); -const optionSecuritySchema = z.enum(["stock", "etf", "all"]); -const optionFilterQuerySchema = z.object({ - view: OptionFlowViewSchema.optional(), - security: optionSecuritySchema.optional(), - side: optionSideListSchema.optional(), - type: optionTypeListSchema.optional(), - min_notional: z.coerce.number().nonnegative().optional() -}); - type Channel = | "options" | "options-nbbo" @@ -351,43 +320,6 @@ const applyDeliverPolicy = ( } }; -const parseOptionPrintFilters = ( - url: URL -): { - view: z.infer; - storageFilters: Parameters[3]; - liveFilters: OptionFlowFilters; -} => { - const parsed = optionFilterQuerySchema.parse({ - view: url.searchParams.get("view") ?? undefined, - security: url.searchParams.get("security") ?? undefined, - side: url.searchParams.get("side") ?? undefined, - type: url.searchParams.get("type") ?? undefined, - min_notional: url.searchParams.get("min_notional") ?? undefined - }); - const view = parsed.view ?? "signal"; - const security = parsed.security ?? (view === "raw" ? "all" : "stock"); - const storageFilters = { - view, - security, - minNotional: parsed.min_notional, - nbboSides: parsed.side, - optionTypes: parsed.type - } as const; - const liveFilters: OptionFlowFilters = { - view, - securityTypes: - security === "all" - ? undefined - : ([security] as Array>), - nbboSides: parsed.side, - optionTypes: parsed.type, - minNotional: parsed.min_notional - }; - - return { view, storageFilters, liveFilters }; -}; - const parseReplayParams = (url: URL): { afterTs: number; afterSeq: number; limit: number } => { const params = replayParamsSchema.parse({ after_ts: url.searchParams.get("after_ts") ?? undefined, @@ -605,15 +537,6 @@ const parseScopeList = (url: URL, ...keys: string[]): string[] | undefined => { return unique.length > 0 ? unique : undefined; }; -const parseLiveOptionPrintFilters = (url: URL): OptionPrintQueryFilters => { - const { storageFilters } = parseOptionPrintFilters(url); - return { - ...storageFilters, - underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids"), - optionContractId: url.searchParams.get("option_contract_id") ?? undefined - }; -}; - const parseLiveEquityPrintFilters = (url: URL): EquityPrintQueryFilters => ({ underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids") }); @@ -1399,7 +1322,7 @@ const run = async () => { try { const limit = parseLimit(url.searchParams.get("limit")); const source = parseReplaySource(url) ?? undefined; - const { storageFilters } = parseOptionPrintFilters(url); + const { storageFilters } = parseOptionPrintQuery(url); const data = await fetchRecentOptionPrints(clickhouse, limit, source, storageFilters); return jsonResponse({ data }); } catch (error) { @@ -1525,7 +1448,7 @@ const run = async () => { try { const { beforeTs, beforeSeq, limit } = parseBeforeParams(url); const source = parseReplaySource(url) ?? undefined; - const storageFilters = parseLiveOptionPrintFilters(url); + const { storageFilters } = parseOptionPrintQuery(url); const data = await fetchOptionPrintsBefore( clickhouse, beforeTs, @@ -1668,7 +1591,7 @@ const run = async () => { try { const { afterTs, afterSeq, limit } = parseReplayParams(url); const source = parseReplaySource(url) ?? undefined; - const { storageFilters } = parseOptionPrintFilters(url); + const { storageFilters } = parseOptionPrintQuery(url); const data = await fetchOptionPrintsAfter( clickhouse, afterTs, diff --git a/services/api/src/live.ts b/services/api/src/live.ts index bd579da..0e2ab1b 100644 --- a/services/api/src/live.ts +++ b/services/api/src/live.ts @@ -345,6 +345,30 @@ const snapshotLimitFor = (subscription: LiveSubscription, configuredLimit: numbe return Math.max(1, Math.min(configuredLimit, Math.floor(requested))); }; +export const buildOptionSnapshotFilters = ( + subscription: Extract +): OptionPrintQueryFilters => { + if (subscription.option_contract_id) { + return { + view: "raw", + optionContractId: subscription.option_contract_id + }; + } + + return { + view: subscription.filters?.view ?? "signal", + security: + subscription.filters?.securityTypes?.length === 1 + ? subscription.filters.securityTypes[0] + : "all", + nbboSides: subscription.filters?.nbboSides, + optionTypes: subscription.filters?.optionTypes, + minNotional: subscription.filters?.minNotional, + underlyingIds: subscription.underlying_ids, + optionContractId: subscription.option_contract_id + }; +}; + const candleRedisKey = (underlyingId: string, intervalMs: number): string => `live:equity-candles:${underlyingId}:${intervalMs}`; @@ -489,18 +513,7 @@ export class LiveStateManager { if (subscription.filters?.view === "raw" || scoped) { this.stats.scopedClickHouseSnapshots += 1; const limit = snapshotLimitFor(subscription, this.generic.options.limit); - const storageFilters: OptionPrintQueryFilters = { - view: subscription.filters?.view ?? "signal", - security: - subscription.filters?.securityTypes?.length === 1 - ? subscription.filters.securityTypes[0] - : "all", - nbboSides: subscription.filters?.nbboSides, - optionTypes: subscription.filters?.optionTypes, - minNotional: subscription.filters?.minNotional, - underlyingIds: subscription.underlying_ids, - optionContractId: subscription.option_contract_id - }; + const storageFilters = buildOptionSnapshotFilters(subscription); const items = await fetchRecentOptionPrints( this.clickhouse, limit, diff --git a/services/api/src/option-queries.ts b/services/api/src/option-queries.ts new file mode 100644 index 0000000..193cbb2 --- /dev/null +++ b/services/api/src/option-queries.ts @@ -0,0 +1,107 @@ +import type { OptionPrintQueryFilters } from "@islandflow/storage"; +import { + OptionFlowViewSchema, + OptionNbboSideSchema, + OptionSecurityTypeSchema, + OptionTypeSchema, + type OptionFlowFilters +} from "@islandflow/types"; +import { z } from "zod"; + +const optionSideListSchema = z + .string() + .transform((value) => + value + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean) + ) + .pipe(z.array(OptionNbboSideSchema)); + +const optionTypeListSchema = z + .string() + .transform((value) => + value + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean) + ) + .pipe(z.array(OptionTypeSchema)); + +const optionSecuritySchema = z.enum(["stock", "etf", "all"]); + +const optionFilterQuerySchema = z.object({ + view: OptionFlowViewSchema.optional(), + security: optionSecuritySchema.optional(), + side: optionSideListSchema.optional(), + type: optionTypeListSchema.optional(), + min_notional: z.coerce.number().nonnegative().optional() +}); + +export type ParsedOptionPrintQuery = { + scope: { + underlyingIds?: string[]; + optionContractId?: string; + }; + flowFilters: OptionFlowFilters; + storageFilters: OptionPrintQueryFilters; + isContractDrilldown: boolean; +}; + +const parseScopeList = (url: URL, ...keys: string[]): string[] | undefined => { + const values = keys + .flatMap((key) => url.searchParams.getAll(key)) + .flatMap((value) => value.split(",")) + .map((value) => value.trim().toUpperCase()) + .filter(Boolean); + const unique = Array.from(new Set(values)); + return unique.length > 0 ? unique : undefined; +}; + +export const parseOptionPrintQuery = (url: URL): ParsedOptionPrintQuery => { + const parsed = optionFilterQuerySchema.parse({ + view: url.searchParams.get("view") ?? undefined, + security: url.searchParams.get("security") ?? undefined, + side: url.searchParams.get("side") ?? undefined, + type: url.searchParams.get("type") ?? undefined, + min_notional: url.searchParams.get("min_notional") ?? undefined + }); + const scope = { + underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids"), + optionContractId: url.searchParams.get("option_contract_id") ?? undefined + }; + const view = parsed.view ?? "signal"; + const security = parsed.security ?? (view === "raw" ? "all" : "stock"); + const flowFilters: OptionFlowFilters = { + view, + securityTypes: + security === "all" + ? undefined + : ([security] as Array>), + nbboSides: parsed.side, + optionTypes: parsed.type, + minNotional: parsed.min_notional + }; + const isContractDrilldown = Boolean(scope.optionContractId); + const storageFilters: OptionPrintQueryFilters = isContractDrilldown + ? { + view: "raw", + optionContractId: scope.optionContractId + } + : { + view, + security, + minNotional: parsed.min_notional, + nbboSides: parsed.side, + optionTypes: parsed.type, + underlyingIds: scope.underlyingIds, + optionContractId: scope.optionContractId + }; + + return { + scope, + flowFilters, + storageFilters, + isContractDrilldown + }; +}; diff --git a/services/api/tests/live.test.ts b/services/api/tests/live.test.ts index 55232cc..3d0aa63 100644 --- a/services/api/tests/live.test.ts +++ b/services/api/tests/live.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "bun:test"; import type { ClickHouseClient } from "@islandflow/storage"; import { + buildOptionSnapshotFilters, HOT_LIVE_REDIS_KEYS, LiveStateManager, isLiveItemFresh, @@ -450,6 +451,74 @@ describe("LiveStateManager", () => { expect(isLiveItemFresh("options", snapshot.items[0], now)).toBe(false); }); + it("builds raw contract-only snapshot filters for focused option subscriptions", () => { + expect( + buildOptionSnapshotFilters({ + channel: "options", + filters: { + view: "signal", + minNotional: 500_000, + nbboSides: ["A"], + optionTypes: ["call"], + securityTypes: ["stock"] + }, + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }) + ).toEqual({ + view: "raw", + optionContractId: "AAPL-2025-01-17-200-C" + }); + }); + + it("returns raw contract rows for focused option snapshots even when broad filters would reject them", async () => { + const manager = new LiveStateManager( + makeClickHouse((query) => { + expect(query).toContain("option_contract_id = 'AAPL-2025-01-17-200-C'"); + expect(query).not.toContain("signal_pass = 1"); + expect(query).not.toContain("notional >="); + expect(query).not.toContain("nbbo_side IN"); + expect(query).not.toContain("option_type IN"); + return [ + { + source_ts: 1_000, + ingest_ts: 1_001, + seq: 1, + trace_id: "opt-raw", + ts: 1_000, + option_contract_id: "AAPL-2025-01-17-200-C", + underlying_id: "AAPL", + option_type: "put", + nbbo_side: "B", + notional: 50_000, + signal_pass: false, + price: 1, + size: 5, + exchange: "X" + } + ]; + }), + null + ); + + const snapshot = await manager.getSnapshot({ + channel: "options", + filters: { + view: "signal", + minNotional: 500_000, + nbboSides: ["A"], + optionTypes: ["call"], + securityTypes: ["stock"] + }, + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }); + + expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ + "opt-raw" + ]); + }); + it("seeds scoped equity snapshots from clickhouse rows older than 24h", async () => { const now = Date.now(); const staleTs = now - 25 * 60 * 60 * 1000; diff --git a/services/api/tests/option-queries.test.ts b/services/api/tests/option-queries.test.ts new file mode 100644 index 0000000..d189303 --- /dev/null +++ b/services/api/tests/option-queries.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "bun:test"; +import { parseOptionPrintQuery } from "../src/option-queries"; + +describe("parseOptionPrintQuery", () => { + it("keeps broad option flow filters for non-contract requests", () => { + const url = new URL( + "http://localhost/prints/options?view=signal&security=stock&side=A&type=call&min_notional=500000&underlying_ids=AAPL,MSFT" + ); + + expect(parseOptionPrintQuery(url)).toEqual({ + scope: { + underlyingIds: ["AAPL", "MSFT"], + optionContractId: undefined + }, + flowFilters: { + view: "signal", + securityTypes: ["stock"], + nbboSides: ["A"], + optionTypes: ["call"], + minNotional: 500000 + }, + storageFilters: { + view: "signal", + security: "stock", + nbboSides: ["A"], + optionTypes: ["call"], + minNotional: 500000, + underlyingIds: ["AAPL", "MSFT"], + optionContractId: undefined + }, + isContractDrilldown: false + }); + }); + + it("switches contract requests to raw contract-only storage filters", () => { + const url = new URL( + "http://localhost/replay/options?view=signal&security=stock&side=A&type=call&min_notional=500000&underlying_id=AAPL&option_contract_id=AAPL-2025-01-17-200-C" + ); + + expect(parseOptionPrintQuery(url)).toEqual({ + scope: { + underlyingIds: ["AAPL"], + optionContractId: "AAPL-2025-01-17-200-C" + }, + flowFilters: { + view: "signal", + securityTypes: ["stock"], + nbboSides: ["A"], + optionTypes: ["call"], + minNotional: 500000 + }, + storageFilters: { + view: "raw", + optionContractId: "AAPL-2025-01-17-200-C" + }, + isContractDrilldown: true + }); + }); +}); From 5d488fd7f5045d024f3b31649348fdf2f5c8eabd Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Fri, 8 May 2026 00:38:54 -0400 Subject: [PATCH 14/15] Add terminal extraction refactor plan - Document target terminal module layout and dependency rules - Outline test split, facade contract, and follow-up bd issues --- plans/terminal-extraction-refactor.md | 372 ++++++++++++++++++++++++++ 1 file changed, 372 insertions(+) create mode 100644 plans/terminal-extraction-refactor.md diff --git a/plans/terminal-extraction-refactor.md b/plans/terminal-extraction-refactor.md new file mode 100644 index 0000000..6125127 --- /dev/null +++ b/plans/terminal-extraction-refactor.md @@ -0,0 +1,372 @@ +# Terminal Extraction Plan + +## Summary + +Refactor [`apps/web/app/terminal.tsx`](/Users/kell/Cloud/dev/islandflow/apps/web/app/terminal.tsx:1) from a single 7,974-line client module into a feature folder at `apps/web/terminal/*`, while keeping `apps/web/app/terminal.tsx` as a temporary compatibility facade in the first pass. + +This first extraction is a medium-scope, behavior-preserving refactor: +- no product behavior changes +- no route behavior changes +- no visual redesign +- no data model changes +- no immediate deletion of the old import surface + +Current baseline is healthy and must remain healthy after the refactor: +- `bun test apps/web/app/terminal.test.ts apps/web/app/routes.test.ts` passes +- `bun --cwd=apps/web run build` passes + +## Target Structure + +Create this feature layout: + +```text +apps/web/terminal/ + index.ts + state.tsx + shell.tsx + routes.tsx + core/ + format.ts + filters.ts + route-config.ts + tape-data.ts + signals.ts + live-manifest.ts + hooks/ + use-tape-data.ts + use-live-session.ts + use-virtual-tape.ts + components/ + chrome.tsx + chart.tsx + drawers.tsx + panes.tsx + tests/ + core.test.ts + live-manifest.test.ts + signals.test.ts + tape-data.test.ts +``` + +Keep this file in place for the first pass: +- `apps/web/app/terminal.tsx` + +Its final first-pass role is: +- `"use client"` entrypoint +- thin re-export facade only +- no business logic +- no React state +- no websocket/session logic +- no chart implementation +- target size: under 120 lines + +## Dependency Rules + +Use this dependency direction and do not violate it: +- `core/*` may depend only on shared types and other `core/*` +- `hooks/*` may depend on `core/*` +- `components/*` may depend on `core/*` and `hooks/*` +- `state.tsx` may depend on `core/*`, `hooks/*`, and `components/*` types only as needed +- `shell.tsx` and `routes.tsx` may depend on `state.tsx` and `components/*` +- `index.ts` re-exports public feature symbols +- `app/terminal.tsx` re-exports from `apps/web/terminal/index.ts` + +Do not allow circular imports. + +## Module Mapping + +Move code out of `terminal.tsx` in this order. + +### 1. Pure helpers first + +Move non-React helpers into `apps/web/terminal/core/*`: + +- `core/route-config.ts` + - `getRouteFeatures` + - `getTapeVirtualConfig` + - `shouldIncludeEquitiesForDarkUnderlyingFallback` + +- `core/live-manifest.ts` + - `getLiveManifest` + - `getLiveHistoryRetentionCap` + - `getScopedLiveAutoHydrationChannels` + - `getLiveFeedStatus` + - `getHotChannelFeedStatus` + +- `core/tape-data.ts` + - `mergeNewestWithOverflow` + - `composeTapeItems` + - `reducePausableTapeData` + - `flushPausableTapeData` + - `appendHistoryTail` + - `projectPausableTapeState` + - `findAnchorRestoreIndex` + - `shouldRetainLiveSnapshotHistory` + - `shouldShowEquitiesSilentFeedWarning` + - tape/history support types used only by these helpers + +- `core/format.ts` + - `formatCompactUsd` + - `formatOptionContractLabel` + - `getOptionTableSnapshot` + - price/size/time/date/contract formatting helpers that support UI rendering + +- `core/signals.ts` + - `normalizeAlertSeverity` + - `deriveAlertDirection` + - `getAlertWindowAnchorTs` + - `selectPrimaryClassifierHit` + - `classifierToneForFamily` + - `smartMoneyToneForProfile` + - `smartMoneyProfileLabel` + +- `core/filters.ts` + - `buildDefaultFlowFilters` + - `countActiveFlowFilterGroups` + - `toggleFilterValue` + - `nextFlowFilterPopoverState` + +These files must not include `"use client"`. + +### 2. Extract hooks and session logic + +Move React hooks into `apps/web/terminal/hooks/*`: + +- `hooks/use-virtual-tape.ts` + - `useListScroll` + - `useScrollAnchor` + - `useVirtualHistoryGate` + - `useTapeVirtualList` + +- `hooks/use-tape-data.ts` + - `useTape` + - `usePausableTapeView` + - `useLiveStream` + - `useFlowStream` + - `statusLabel` + - internal tape state types + +- `hooks/use-live-session.ts` + - `useLiveSession` + - live history endpoint constants + - live history query builders + - subscription dedupe helpers + - session-local types + +Keep signatures stable unless a change is required to break a circular dependency. If a signature changes, update all callers in the same PR. + +### 3. Extract UI components + +Move rendering code into `apps/web/terminal/components/*`: + +- `components/chrome.tsx` + - `TapeStatus` + - `TapeControls` + - `PageFrame` + - `Pane` + - `ShellMetricStrip` + - `FlowFilterPopover` + - local filter UI helpers + +- `components/chart.tsx` + - `CandleChart` + - chart-only local types and overlay helpers + - isolate `lightweight-charts` usage here + +- `components/drawers.tsx` + - `AlertSeverityStrip` + - `AlertDrawer` + - `ClassifierHitDrawer` + - `SmartMoneyDrawer` + - `DarkDrawer` + +- `components/panes.tsx` + - `OptionsPane` + - `EquitiesPane` + - `FlowPane` + - `AlertsPane` + - `ClassifierPane` + - `DarkPane` + - `ChartPane` + - `FocusPane` + - `ReplayConsole` + +### 4. Extract state orchestration + +Create `apps/web/terminal/state.tsx` for: +- `useTerminalState` +- `TerminalContext` +- `useTerminal` + +This file owns: +- route-aware feature selection +- filter input state +- selected entity/drawer state +- scroll-anchor wiring +- assembly of hook outputs into the single terminal state object + +Keep `useTerminalState` internal. Do not export it from the feature barrel. + +### 5. Extract shell and routes + +Create: +- `apps/web/terminal/shell.tsx` + - `TerminalAppShell` + +- `apps/web/terminal/routes.tsx` + - `NAV_ITEMS` + - `OverviewRoute` + - `TapeRoute` + - `SignalsRoute` + - `ChartsRoute` + - `ReplayRoute` + +Important first-pass rule: +- keep existing route behavior exactly as-is +- [app/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/page.tsx:1), [app/layout.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/layout.tsx:4), and [app/tape/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/tape/page.tsx:1) may continue importing from `./terminal` / `../terminal` +- [app/signals/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/signals/page.tsx:1), [app/charts/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/charts/page.tsx:1), and [app/replay/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/replay/page.tsx:1) must remain redirect pages in this pass + +## Facade Contract + +Replace `apps/web/app/terminal.tsx` with a facade that re-exports from `apps/web/terminal/index.ts`. + +The facade must continue exporting these symbols in the first pass: + +- `getTapeVirtualConfig` +- `shouldIncludeEquitiesForDarkUnderlyingFallback` +- `getRouteFeatures` +- `mergeNewestWithOverflow` +- `composeTapeItems` +- `reducePausableTapeData` +- `flushPausableTapeData` +- `appendHistoryTail` +- `getLiveHistoryRetentionCap` +- `getScopedLiveAutoHydrationChannels` +- `getLiveFeedStatus` +- `getHotChannelFeedStatus` +- `findAnchorRestoreIndex` +- `formatCompactUsd` +- `formatOptionContractLabel` +- `normalizeAlertSeverity` +- `deriveAlertDirection` +- `getAlertWindowAnchorTs` +- `buildDefaultFlowFilters` +- `countActiveFlowFilterGroups` +- `toggleFilterValue` +- `nextFlowFilterPopoverState` +- `projectPausableTapeState` +- `shouldShowEquitiesSilentFeedWarning` +- `shouldRetainLiveSnapshotHistory` +- `selectPrimaryClassifierHit` +- `classifierToneForFamily` +- `smartMoneyToneForProfile` +- `smartMoneyProfileLabel` +- `getOptionTableSnapshot` +- `statusLabel` +- `getLiveManifest` +- `NAV_ITEMS` +- `FlowFilterPopover` +- `TerminalAppShell` +- `OverviewRoute` +- `TapeRoute` +- `SignalsRoute` +- `ChartsRoute` +- `ReplayRoute` + +Do not add new facade-only exports. + +## Test Plan + +Restructure tests so pure logic is tested from its final home instead of through the facade. + +### Keep +- `apps/web/app/routes.test.ts` + - still verifies redirect behavior for `/signals`, `/charts`, `/replay` + +### Split `app/terminal.test.ts` into feature tests +- `apps/web/terminal/tests/live-manifest.test.ts` + - route feature mapping + - manifest composition + - nav items if still treated as route metadata + +- `apps/web/terminal/tests/tape-data.test.ts` + - merge/dedupe logic + - pausable tape behavior + - history seam behavior + - anchor restore behavior + - retention cap behavior + - scoped history behavior + +- `apps/web/terminal/tests/core.test.ts` + - option contract formatting + - compact USD formatting + - option table snapshot formatting + - flow filter helpers + +- `apps/web/terminal/tests/signals.test.ts` + - alert severity normalization + - direction derivation + - alert window anchor + - classifier/smart-money label and tone helpers + - live status labeling if kept outside tape-data tests + +Optional and recommended: +- add one tiny `apps/web/app/terminal-facade.test.ts` that imports the facade and asserts a few critical exports exist, so we notice accidental facade breakage during the transition + +## Validation Gates + +Implementation is not complete unless all of these pass: + +1. `bun test apps/web/terminal/tests apps/web/app/routes.test.ts` +2. `bun --cwd=apps/web run build` +3. Existing behavior smoke check: + - `/` still renders the shell and overview + - `/tape` still renders shell and tape panes + - `/signals`, `/charts`, `/replay` still redirect to `/` +4. `apps/web/app/terminal.tsx` is a facade only and contains no moved logic +5. No extracted pure helper file contains React imports +6. No new circular imports are introduced + +## Non-Goals For This Pass + +Do not do these in the first extraction: +- redesign panes or drawers +- change websocket or replay behavior +- change route inventory +- remove unused legacy route exports +- change CSS structure beyond import fixes +- optimize bundle size as a separate objective +- rewrite tests to different testing tools + +## Beads Follow-Up Issues To File + +Create these `bd` issues during implementation if they do not already exist: + +1. `task`, priority `2` + Title: `Remove temporary apps/web/app/terminal.tsx facade after terminal imports are migrated` + Description: track deletion of the compatibility facade once route/layout/test imports point at final `apps/web/terminal/*` modules + +2. `task`, priority `3` + Title: `Audit and remove dead terminal route exports no longer used by app redirects` + Description: verify whether `SignalsRoute`, `ChartsRoute`, and `ReplayRoute` should be deleted since App Router pages now redirect to `/` + +If additional cleanup is discovered during extraction, create linked `bd` tasks with `discovered-from` dependencies rather than expanding this refactor mid-flight. + +## Acceptance Criteria + +The first extraction is successful when: +- terminal logic is split into the target `apps/web/terminal/*` structure +- `apps/web/app/terminal.tsx` remains only as a thin compatibility layer +- app entrypoints continue to work without behavior changes +- tests target the new module homes for pure logic +- build and tests pass +- follow-up `bd` issues exist for facade removal and dead-export cleanup + +## Assumptions And Defaults + +- Chosen scope: medium slice, not full architectural rewrite +- Chosen transition: keep `apps/web/app/terminal.tsx` as a temporary facade +- Chosen module home: `apps/web/terminal/*`, not `apps/web/app/terminal/*` +- Default behavior requirement: strict behavioral parity +- Default testing approach: split existing monolithic helper tests by concern and colocate them under `apps/web/terminal/tests` +- Default routing approach: keep redirect pages untouched in the first pass From e7f4805ccc58a8d337ce544dc2e8b76e16b8de8c Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Fri, 8 May 2026 02:46:41 -0400 Subject: [PATCH 15/15] Implement first-pass load reduction controls --- .beads/issues.jsonl | 3 + .env.example | 40 +- deployment/docker/.env.example | 31 ++ deployment/docker/docker-compose.yml | 4 + packages/bus/src/jetstream.ts | 44 ++ packages/observability/src/logger.ts | 32 +- packages/storage/src/clickhouse.ts | 151 +++++++ services/api/src/index.ts | 187 ++------ services/api/src/live.ts | 439 ++++++++++++++----- services/api/tests/live.test.ts | 116 ++++- services/candles/src/index.ts | 28 +- services/compute/src/index.ts | 395 +++++++++-------- services/compute/src/rolling-stats.ts | 143 +++++- services/compute/tests/rolling-stats.test.ts | 16 +- services/ingest-equities/src/index.ts | 33 +- services/ingest-options/src/index.ts | 119 +++-- services/replay/src/index.ts | 18 +- 17 files changed, 1191 insertions(+), 608 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 5ca3a9f..704be02 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,5 @@ +{"_type":"issue","id":"islandflow-dil","title":"Run production baseline and post-rollout verification for load reduction","description":"Run the production verification checklist from the load-reduction plan on the VPS, capture baseline container/resource stats, validate replay remains disabled, and confirm JetStream/Redis behavior after rollout.\n\nThis follow-up is operational rather than code-local and could not be executed from the current workspace. It should compare pre/post CPU, RSS, Redis memory, and retention growth using the documented commands.\n","status":"open","priority":1,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:45:06Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:45:06Z","dependencies":[{"issue_id":"islandflow-dil","depends_on_id":"islandflow-1ln","type":"discovered-from","created_at":"2026-05-08T02:45:06Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-1ln","title":"Implement VPS load reduction plan","description":"Implement load-reduction plan across API, compute, logging, retention, and cache pruning.\n\nThis issue tracks the first-pass implementation of VPS load mitigations: lower live cache limits, async Redis write-behind in API live state, scoped cache eviction, reduced hot-path logging, bounded JetStream retention via shared config, in-memory rolling stats with async Redis snapshots, batched ClickHouse inserts for derived tables, and TTL/cardinality pruning for long-lived in-process maps.\n\nAcceptance:\n- Config surface for live limits, logging, rolling cache, and stream retention added\n- API live ingest avoids per-event full resort in monotonic case and avoids synchronous Redis writes per event\n- Compute rolling stats leave Redis hot path and derived ClickHouse writes batch\n- Long-lived caches/maps are pruned by TTL/cardinality\n- Tests cover monotonic/out-of-order live ingest, scoped eviction, rolling stats, and pruning behavior\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:27:41Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:46:23Z","started_at":"2026-05-08T06:27:54Z","closed_at":"2026-05-08T06:46:23Z","close_reason":"Implemented in code; rollout verification follow-up is islandflow-dil and Redis durability decision follow-up is islandflow-ybs","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-pre","title":"Fix contract-focused options tape hydration","description":"Implement contract-focused options tape hydration so focused contract views preserve the clicked seed row, stop reapplying broad flow filters in the Options pane, and use raw contract-scoped ClickHouse queries consistently across live snapshots, history, and replay. Includes frontend replay source-grouping changes and regression tests for focus seed durability, focused filtering, and contract-scoped API behavior.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T03:27:31Z","created_by":"dirtydishes","updated_at":"2026-05-08T03:37:18Z","started_at":"2026-05-08T03:27:35Z","closed_at":"2026-05-08T03:37:18Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-9xs","title":"Fix terminal hydration and virtual row measurement crash","description":"Fix client crash caused by options-support hydration on non-JSON/404 responses and satisfy tanstack virtual measured-row data-index requirement across virtualized tables.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T06:14:33Z","created_by":"dirtydishes","updated_at":"2026-05-07T06:17:09Z","started_at":"2026-05-07T06:14:43Z","closed_at":"2026-05-07T06:17:09Z","close_reason":"Completed: added data-index attributes on measured virtual rows, hardened options-support hydration error handling/content-type validation, and guarded trace-id hydration loops against malformed payload entries.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-35g","title":"Fix Docker deployment workspace lockfile drift","description":"Refresh deployment/docker workspace lockfile for Docker builds, add a drift guard for Docker-built workspaces, and document the separate deployment snapshot so frozen Bun installs cannot fail when repo dependencies change.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T06:02:06Z","created_by":"dirtydishes","updated_at":"2026-05-07T06:07:50Z","started_at":"2026-05-07T06:02:15Z","closed_at":"2026-05-07T06:07:50Z","close_reason":"Completed: synced deployment Docker workspace snapshot from repo root, refreshed deployment bun.lock, added sync/check scripts, and documented maintenance workflow. Local docker compose build validation is blocked here because Docker daemon is unavailable.","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -10,6 +12,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-ybs","title":"Decide Redis AOF and cache/durable split after load rollout","description":"Decide whether the deployment Redis should keep AOF enabled or be split into cache vs durable roles after the first rollout data is available.\n\nThe current code changes reduce cache churn, but the operational durability/caching tradeoff still needs a production decision.\n","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:45:05Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:45:05Z","dependencies":[{"issue_id":"islandflow-ybs","depends_on_id":"islandflow-1ln","type":"discovered-from","created_at":"2026-05-08T02:45:04Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-vbk","title":"Remove deprecated Alpaca key-pair auth","description":"Remove legacy Alpaca key-pair authentication support and keep ALPACA_API_KEY as the only supported auth method across options/equities ingest and docs.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:19:51Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:21:10Z","started_at":"2026-05-05T07:19:54Z","closed_at":"2026-05-05T07:21:10Z","close_reason":"Removed key-pair auth and kept ALPACA_API_KEY only","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-h47","title":"Support single-token Alpaca auth","description":"Support single-token Alpaca authentication across ingest adapters using ALPACA_API_KEY with fallback to ALPACA_KEY_ID/ALPACA_SECRET_KEY, and document env usage.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:12:22Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:13:54Z","started_at":"2026-05-05T07:12:25Z","closed_at":"2026-05-05T07:13:54Z","close_reason":"Added ALPACA_API_KEY support with key-pair fallback","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-neu","title":"Add Alpha Vantage event calendar provider","description":"Add an Alpha Vantage earnings-calendar provider to services/refdata that fetches CSV, normalizes entries, writes the JSON cache consumed by compute, and documents the required env variables.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:00:31Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:02:30Z","started_at":"2026-05-05T07:00:37Z","closed_at":"2026-05-05T07:02:30Z","close_reason":"Added Alpha Vantage event-calendar provider","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/.env.example b/.env.example index 4d5ac1b..5442eac 100644 --- a/.env.example +++ b/.env.example @@ -91,6 +91,7 @@ ALPHA_VANTAGE_EARNINGS_SYMBOL= REFDATA_EVENT_CALENDAR_REFRESH_MS=86400000 # Replay service +LOG_LEVEL=info REPLAY_ENABLED=false REPLAY_STREAMS=options,nbbo,equities,equity-quotes REPLAY_START_TS=0 @@ -100,12 +101,33 @@ REPLAY_BATCH_SIZE=200 REPLAY_LOG_EVERY=1000 # API live retention (generic channels) -LIVE_LIMIT_OPTIONS=2000 -LIVE_LIMIT_NBBO=10000 -LIVE_LIMIT_EQUITIES=2000 -LIVE_LIMIT_EQUITY_QUOTES=10000 -LIVE_LIMIT_EQUITY_JOINS=10000 -LIVE_LIMIT_FLOW=2000 -LIVE_LIMIT_CLASSIFIER_HITS=10000 -LIVE_LIMIT_ALERTS=10000 -LIVE_LIMIT_INFERRED_DARK=10000 +LIVE_LIMIT_DEFAULT=1000 +LIVE_LIMIT_OPTIONS=1000 +LIVE_LIMIT_NBBO=1000 +LIVE_LIMIT_EQUITIES=1000 +LIVE_LIMIT_EQUITY_QUOTES=500 +LIVE_LIMIT_EQUITY_JOINS=500 +LIVE_LIMIT_FLOW=500 +LIVE_LIMIT_SMART_MONEY=300 +LIVE_LIMIT_CLASSIFIER_HITS=300 +LIVE_LIMIT_ALERTS=300 +LIVE_LIMIT_INFERRED_DARK=300 +LIVE_SCOPED_CACHE_MAX_KEYS=32 +LIVE_REDIS_FLUSH_INTERVAL_MS=250 +LIVE_REDIS_FLUSH_MAX_ITEMS=100 + +# Compute rolling/cache retention +ROLLING_CACHE_FLUSH_INTERVAL_MS=30000 +ROLLING_CACHE_MAX_KEYS=20000 +COMPUTE_NBBO_CACHE_MAX_KEYS=20000 +COMPUTE_NBBO_CACHE_TTL_MS=900000 + +# Ingest context retention +OPTION_CONTEXT_MAX_KEYS=20000 +OPTION_CONTEXT_TTL_MS=900000 + +# JetStream retention +STREAM_RAW_MAX_AGE_MS=7200000 +STREAM_RAW_MAX_BYTES=1073741824 +STREAM_DERIVED_MAX_AGE_MS=86400000 +STREAM_DERIVED_MAX_BYTES=536870912 diff --git a/deployment/docker/.env.example b/deployment/docker/.env.example index 2512e0e..986968c 100644 --- a/deployment/docker/.env.example +++ b/deployment/docker/.env.example @@ -98,6 +98,7 @@ CLASSIFIER_0DTE_MIN_PREMIUM=20000 CLASSIFIER_0DTE_MIN_SIZE=400 # Smart money refdata +LOG_LEVEL=warn SMART_MONEY_EVENT_CALENDAR_PATH=data/event-calendar.json REFDATA_EVENT_CALENDAR_PATH= REFDATA_EVENT_CALENDAR_PROVIDER= @@ -120,3 +121,33 @@ REPLAY_END_TS=0 REPLAY_SPEED=1 REPLAY_BATCH_SIZE=200 REPLAY_LOG_EVERY=1000 + +# API live retention +LIVE_LIMIT_DEFAULT=1000 +LIVE_LIMIT_OPTIONS=1000 +LIVE_LIMIT_NBBO=1000 +LIVE_LIMIT_EQUITIES=1000 +LIVE_LIMIT_EQUITY_QUOTES=500 +LIVE_LIMIT_EQUITY_JOINS=500 +LIVE_LIMIT_FLOW=500 +LIVE_LIMIT_SMART_MONEY=300 +LIVE_LIMIT_CLASSIFIER_HITS=300 +LIVE_LIMIT_ALERTS=300 +LIVE_LIMIT_INFERRED_DARK=300 +LIVE_SCOPED_CACHE_MAX_KEYS=32 +LIVE_REDIS_FLUSH_INTERVAL_MS=250 +LIVE_REDIS_FLUSH_MAX_ITEMS=100 + +# Compute and ingest cache retention +ROLLING_CACHE_FLUSH_INTERVAL_MS=30000 +ROLLING_CACHE_MAX_KEYS=20000 +OPTION_CONTEXT_MAX_KEYS=20000 +OPTION_CONTEXT_TTL_MS=900000 +COMPUTE_NBBO_CACHE_MAX_KEYS=20000 +COMPUTE_NBBO_CACHE_TTL_MS=900000 + +# JetStream retention +STREAM_RAW_MAX_AGE_MS=7200000 +STREAM_RAW_MAX_BYTES=1073741824 +STREAM_DERIVED_MAX_AGE_MS=86400000 +STREAM_DERIVED_MAX_BYTES=536870912 diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index a3ed7a4..96598ba 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -14,6 +14,8 @@ x-service-common: &service-common dockerfile: Dockerfile.service env_file: - ./.env + environment: + LOG_LEVEL: ${LOG_LEVEL:-warn} restart: unless-stopped init: true extra_hosts: @@ -94,6 +96,8 @@ services: dockerfile: Dockerfile.ingest-options env_file: - ./.env + environment: + LOG_LEVEL: ${LOG_LEVEL:-warn} restart: unless-stopped init: true extra_hosts: diff --git a/packages/bus/src/jetstream.ts b/packages/bus/src/jetstream.ts index d79daba..204395e 100644 --- a/packages/bus/src/jetstream.ts +++ b/packages/bus/src/jetstream.ts @@ -84,6 +84,50 @@ export const ensureStream = async ( } }; +const parseBoundedNumber = (value: string | undefined, fallback: number): number => { + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed < 0) { + return fallback; + } + return Math.floor(parsed); +}; + +export type StreamRetentionClass = "raw" | "derived"; + +export const resolveStreamRetention = ( + streamClass: StreamRetentionClass, + env: Record = process.env +): Pick => { + if (streamClass === "raw") { + return { + max_age: parseBoundedNumber(env.STREAM_RAW_MAX_AGE_MS, 7_200_000), + max_bytes: parseBoundedNumber(env.STREAM_RAW_MAX_BYTES, 1_073_741_824) + }; + } + + return { + max_age: parseBoundedNumber(env.STREAM_DERIVED_MAX_AGE_MS, 86_400_000), + max_bytes: parseBoundedNumber(env.STREAM_DERIVED_MAX_BYTES, 536_870_912) + }; +}; + +export const buildStreamConfig = ( + name: string, + subject: string, + streamClass: StreamRetentionClass, + env: Record = process.env +): StreamConfig => ({ + name, + subjects: [subject], + retention: "limits", + storage: "file", + discard: "old", + max_msgs_per_subject: -1, + max_msgs: -1, + ...resolveStreamRetention(streamClass, env), + num_replicas: 1 +}); + export const buildDurableConsumer = ( durableName: string, deliverSubject: string = createInbox() diff --git a/packages/observability/src/logger.ts b/packages/observability/src/logger.ts index 0c4b437..b883695 100644 --- a/packages/observability/src/logger.ts +++ b/packages/observability/src/logger.ts @@ -22,20 +22,46 @@ export type LoggerOptions = { service: string; now?: () => string; sink?: (record: LogRecord) => void; + level?: LogLevel; }; const defaultSink = (record: LogRecord) => { console.log(JSON.stringify(record)); }; +const LOG_LEVEL_ORDER: Record = { + debug: 10, + info: 20, + warn: 30, + error: 40 +}; + +const resolveLogLevel = (value: string | undefined): LogLevel => { + switch ((value ?? "").trim().toLowerCase()) { + case "debug": + case "info": + case "warn": + case "error": + return value!.trim().toLowerCase() as LogLevel; + default: + return "info"; + } +}; + export const createLogger = ({ service, now = () => new Date().toISOString(), - sink = defaultSink + sink = defaultSink, + level = resolveLogLevel(process.env.LOG_LEVEL) }: LoggerOptions): Logger => { - const write = (level: LogLevel, msg: string, context?: LogContext) => { + const levelThreshold = resolveLogLevel(level); + + const write = (recordLevel: LogLevel, msg: string, context?: LogContext) => { + if (LOG_LEVEL_ORDER[recordLevel] < LOG_LEVEL_ORDER[levelThreshold]) { + return; + } const record: LogRecord = { - level, + level: recordLevel, service, msg, ts: now(), diff --git a/packages/storage/src/clickhouse.ts b/packages/storage/src/clickhouse.ts index d26b046..b5b0484 100644 --- a/packages/storage/src/clickhouse.ts +++ b/packages/storage/src/clickhouse.ts @@ -449,6 +449,157 @@ export const insertAlert = async (client: ClickHouseClient, alert: AlertEvent): }); }; +export type ClickHouseBatchWriterOptions = { + flushIntervalMs?: number; + maxRows?: number; + onError?: (table: string, error: unknown, rowCount: number) => void; +}; + +type BatchState = { + rows: unknown[]; + timer: ReturnType | null; + flushing: Promise | null; +}; + +const createBatchState = (): BatchState => ({ + rows: [], + timer: null, + flushing: null +}); + +export class ClickHouseBatchWriter { + private readonly flushIntervalMs: number; + private readonly maxRows: number; + private readonly states = new Map(); + + constructor( + private readonly client: ClickHouseClient, + options: ClickHouseBatchWriterOptions = {} + ) { + this.flushIntervalMs = Math.max(1, Math.floor(options.flushIntervalMs ?? 100)); + this.maxRows = Math.max(1, Math.floor(options.maxRows ?? 250)); + this.onError = options.onError; + } + + private readonly onError?: (table: string, error: unknown, rowCount: number) => void; + + enqueue(table: string, row: unknown): void { + const state = this.states.get(table) ?? createBatchState(); + if (!this.states.has(table)) { + this.states.set(table, state); + } + + state.rows.push(row); + + if (state.rows.length >= this.maxRows) { + void this.flush(table); + return; + } + + if (!state.timer) { + state.timer = setTimeout(() => { + state.timer = null; + void this.flush(table); + }, this.flushIntervalMs); + } + } + + async flush(table: string): Promise { + const state = this.states.get(table); + if (!state) { + return; + } + + if (state.flushing) { + await state.flushing; + return; + } + + if (state.timer) { + clearTimeout(state.timer); + state.timer = null; + } + + if (state.rows.length === 0) { + return; + } + + const rows = state.rows.splice(0, state.rows.length); + state.flushing = this.client + .insert({ + table, + values: rows, + format: "JSONEachRow" + }) + .catch((error) => { + this.onError?.(table, error, rows.length); + }) + .finally(() => { + state.flushing = null; + }); + + await state.flushing; + } + + async flushAll(): Promise { + for (const table of this.states.keys()) { + await this.flush(table); + } + } + + async close(): Promise { + for (const state of this.states.values()) { + if (state.timer) { + clearTimeout(state.timer); + state.timer = null; + } + } + await this.flushAll(); + } +} + +export const enqueueEquityPrintJoinInsert = ( + writer: ClickHouseBatchWriter, + join: EquityPrintJoin +): void => { + writer.enqueue(EQUITY_PRINT_JOINS_TABLE, toEquityPrintJoinRecord(join)); +}; + +export const enqueueInferredDarkInsert = ( + writer: ClickHouseBatchWriter, + event: InferredDarkEvent +): void => { + writer.enqueue(INFERRED_DARK_TABLE, toInferredDarkRecord(event)); +}; + +export const enqueueFlowPacketInsert = ( + writer: ClickHouseBatchWriter, + packet: FlowPacket +): void => { + writer.enqueue(FLOW_PACKETS_TABLE, toFlowPacketRecord(packet)); +}; + +export const enqueueSmartMoneyEventInsert = ( + writer: ClickHouseBatchWriter, + event: SmartMoneyEvent +): void => { + writer.enqueue(SMART_MONEY_EVENTS_TABLE, toSmartMoneyEventRecord(event)); +}; + +export const enqueueClassifierHitInsert = ( + writer: ClickHouseBatchWriter, + hit: ClassifierHitEvent +): void => { + writer.enqueue(CLASSIFIER_HITS_TABLE, toClassifierHitRecord(hit)); +}; + +export const enqueueAlertInsert = ( + writer: ClickHouseBatchWriter, + alert: AlertEvent +): void => { + writer.enqueue(ALERTS_TABLE, toAlertRecord(alert)); +}; + const clampLimit = (limit: number): number => { if (!Number.isFinite(limit)) { return 100; diff --git a/services/api/src/index.ts b/services/api/src/index.ts index b7af494..31f861a 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -1,5 +1,5 @@ import { readEnv } from "@islandflow/config"; -import { createLogger } from "@islandflow/observability"; +import { createLogger, createMetrics } from "@islandflow/observability"; import { SUBJECT_ALERTS, SUBJECT_CLASSIFIER_HITS, @@ -23,6 +23,7 @@ import { STREAM_SMART_MONEY_EVENTS, STREAM_OPTION_NBBO, STREAM_OPTION_SIGNAL_PRINTS, + buildStreamConfig, buildDurableConsumer, connectJetStreamWithRetry, ensureStream, @@ -107,11 +108,17 @@ import { } from "@islandflow/types"; import { createClient } from "redis"; import { z } from "zod"; -import { HOT_LIVE_REDIS_KEYS, LiveStateManager, shouldFanoutLiveEvent } from "./live"; +import { + HOT_LIVE_REDIS_KEYS, + LiveStateManager, + resolveLiveStateConfig, + shouldFanoutLiveEvent +} from "./live"; import { parseOptionPrintQuery } from "./option-queries"; const service = "api"; const logger = createLogger({ service }); +const metrics = createMetrics({ service }); const DeliverPolicySchema = z.enum(["new", "all", "last", "last_per_subject"]); @@ -617,148 +624,17 @@ const run = async () => { { attempts: 120, delayMs: 500 } ); - await ensureStream(jsm, { - name: STREAM_OPTION_SIGNAL_PRINTS, - subjects: [SUBJECT_OPTION_SIGNAL_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_OPTION_NBBO, - subjects: [SUBJECT_OPTION_NBBO], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_PRINTS, - subjects: [SUBJECT_EQUITY_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_QUOTES, - subjects: [SUBJECT_EQUITY_QUOTES], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_CANDLES, - subjects: [SUBJECT_EQUITY_CANDLES], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_JOINS, - subjects: [SUBJECT_EQUITY_JOINS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_INFERRED_DARK, - subjects: [SUBJECT_INFERRED_DARK], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_FLOW_PACKETS, - subjects: [SUBJECT_FLOW_PACKETS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_SMART_MONEY_EVENTS, - subjects: [SUBJECT_SMART_MONEY_EVENTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_CLASSIFIER_HITS, - subjects: [SUBJECT_CLASSIFIER_HITS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_ALERTS, - subjects: [SUBJECT_ALERTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_CANDLES, SUBJECT_EQUITY_CANDLES, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_JOINS, SUBJECT_EQUITY_JOINS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_INFERRED_DARK, SUBJECT_INFERRED_DARK, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_FLOW_PACKETS, SUBJECT_FLOW_PACKETS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_SMART_MONEY_EVENTS, SUBJECT_SMART_MONEY_EVENTS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_CLASSIFIER_HITS, SUBJECT_CLASSIFIER_HITS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_ALERTS, SUBJECT_ALERTS, "derived")); const clickhouse = createClickHouseClient({ url: env.CLICKHOUSE_URL, @@ -804,7 +680,7 @@ const run = async () => { redis = null; } - const liveState = new LiveStateManager(clickhouse, redis); + const liveState = new LiveStateManager(clickhouse, redis, resolveLiveStateConfig()); await liveState.hydrate(); const warnLiveLag = ( channel: keyof typeof HOT_LIVE_REDIS_KEYS, @@ -1069,6 +945,11 @@ const run = async () => { return; } + const optionItem = ingestChannel === "options" ? (item as Parameters[0]) : null; + const equityItem = ingestChannel === "equities" ? (item as Parameters[0]) : null; + const flowItem = ingestChannel === "flow" ? (item as Parameters[0]) : null; + let matchedSubscriptions = 0; + for (const [key, candidate] of matchingSubscriptions) { const sockets = subscriptionSockets.get(key); if (!sockets || sockets.size === 0) { @@ -1077,26 +958,29 @@ const run = async () => { if ( candidate.channel === "options" && - (!matchesOptionPrintFilters(OptionPrintSchema.parse(item), candidate.filters) || - !matchesScopedOptionSubscription(OptionPrintSchema.parse(item), candidate)) + (!optionItem || + !matchesOptionPrintFilters(optionItem, candidate.filters) || + !matchesScopedOptionSubscription(optionItem, candidate)) ) { continue; } if ( candidate.channel === "equities" && - !matchesScopedEquitySubscription(EquityPrintSchema.parse(item), candidate) + (!equityItem || !matchesScopedEquitySubscription(equityItem, candidate)) ) { continue; } if ( candidate.channel === "flow" && - !matchesFlowPacketFilters(FlowPacketSchema.parse(item), candidate.filters) + (!flowItem || !matchesFlowPacketFilters(flowItem, candidate.filters)) ) { continue; } + matchedSubscriptions += 1; + for (const socket of sockets) { sendLiveMessage(socket, { op: "event", @@ -1106,6 +990,10 @@ const run = async () => { }); } } + + if (matchedSubscriptions > 0) { + metrics.count("api.live.subscription_match_count", matchedSubscriptions); + } }; const pumpOptions = async () => { @@ -1931,6 +1819,7 @@ const run = async () => { logger.info("service stopping", { signal }); server.stop(); clearInterval(liveStateMetricsTimer); + await liveState.close(); if (redis && redis.isOpen) { try { diff --git a/services/api/src/live.ts b/services/api/src/live.ts index 0e2ab1b..ca228fc 100644 --- a/services/api/src/live.ts +++ b/services/api/src/live.ts @@ -41,12 +41,15 @@ import { type EquityPrint, type LiveChannel } from "@islandflow/types"; +import { createMetrics } from "@islandflow/observability"; import type { RedisClientType } from "redis"; const CURSOR_HASH_KEY = "live:cursors"; export const LIVE_FEED_LOOKBACK_MS = 24 * 60 * 60 * 1000; -const DEFAULT_GENERIC_LIMIT = 10000; +const metrics = createMetrics({ service: "api" }); + +const DEFAULT_GENERIC_LIMIT = 1000; const MAX_GENERIC_LIMIT = 100000; const MIN_GENERIC_LIMIT = 1; const GENERIC_LIMIT_ENV_KEYS: Record = { @@ -67,6 +70,23 @@ const CHART_LIMITS = { overlay: 1500 } as const; +const DEFAULT_LIVE_LIMITS: GenericLiveLimits = { + options: 1000, + nbbo: 1000, + equities: 1000, + "equity-quotes": 500, + "equity-joins": 500, + flow: 500, + "smart-money": 300, + "classifier-hits": 300, + alerts: 300, + "inferred-dark": 300 +}; + +const DEFAULT_SCOPED_CACHE_MAX_KEYS = 32; +const DEFAULT_REDIS_FLUSH_INTERVAL_MS = 250; +const DEFAULT_REDIS_FLUSH_MAX_ITEMS = 100; + type GenericFeedConfig = { redisKey: string; cursorField: string; @@ -93,6 +113,13 @@ export const HOT_LIVE_REDIS_KEYS = { export type GenericLiveLimits = Record; +type LiveStateConfig = { + limits: GenericLiveLimits; + scopedCacheMaxKeys: number; + redisFlushIntervalMs: number; + redisFlushMaxItems: number; +}; + const parseGenericLimit = ( env: NodeJS.ProcessEnv, channel: LiveGenericChannel, @@ -117,17 +144,77 @@ const parseGenericLimit = ( return bounded; }; -export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env): GenericLiveLimits => ({ - options: parseGenericLimit(env, "options", DEFAULT_GENERIC_LIMIT), - nbbo: parseGenericLimit(env, "nbbo", DEFAULT_GENERIC_LIMIT), - equities: parseGenericLimit(env, "equities", DEFAULT_GENERIC_LIMIT), - "equity-quotes": parseGenericLimit(env, "equity-quotes", DEFAULT_GENERIC_LIMIT), - "equity-joins": parseGenericLimit(env, "equity-joins", DEFAULT_GENERIC_LIMIT), - flow: parseGenericLimit(env, "flow", DEFAULT_GENERIC_LIMIT), - "smart-money": parseGenericLimit(env, "smart-money", DEFAULT_GENERIC_LIMIT), - "classifier-hits": parseGenericLimit(env, "classifier-hits", DEFAULT_GENERIC_LIMIT), - alerts: parseGenericLimit(env, "alerts", DEFAULT_GENERIC_LIMIT), - "inferred-dark": parseGenericLimit(env, "inferred-dark", DEFAULT_GENERIC_LIMIT) +const parseGenericLimitFallback = (env: NodeJS.ProcessEnv, fallback: number): number => { + const raw = env.LIVE_LIMIT_DEFAULT; + if (!raw || raw.trim().length === 0) { + return fallback; + } + + const parsed = Number(raw); + if (!Number.isFinite(parsed)) { + console.warn(`Invalid LIVE_LIMIT_DEFAULT="${raw}", using ${fallback}`); + return fallback; + } + + return Math.max(MIN_GENERIC_LIMIT, Math.min(MAX_GENERIC_LIMIT, Math.floor(parsed))); +}; + +export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env): GenericLiveLimits => { + const liveLimitDefault = parseGenericLimitFallback(env, DEFAULT_GENERIC_LIMIT); + return { + options: parseGenericLimit(env, "options", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.options), + nbbo: parseGenericLimit(env, "nbbo", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.nbbo), + equities: parseGenericLimit( + env, + "equities", + env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.equities + ), + "equity-quotes": parseGenericLimit( + env, + "equity-quotes", + env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["equity-quotes"] + ), + "equity-joins": parseGenericLimit( + env, + "equity-joins", + env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["equity-joins"] + ), + flow: parseGenericLimit(env, "flow", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.flow), + "smart-money": parseGenericLimit( + env, + "smart-money", + env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["smart-money"] + ), + "classifier-hits": parseGenericLimit( + env, + "classifier-hits", + env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["classifier-hits"] + ), + alerts: parseGenericLimit(env, "alerts", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.alerts), + "inferred-dark": parseGenericLimit( + env, + "inferred-dark", + env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["inferred-dark"] + ) + }; +}; + +const parsePositiveInt = (value: string | undefined, fallback: number): number => { + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + return fallback; + } + return Math.max(1, Math.floor(parsed)); +}; + +export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): LiveStateConfig => ({ + limits: resolveGenericLiveLimits(env), + scopedCacheMaxKeys: parsePositiveInt(env.LIVE_SCOPED_CACHE_MAX_KEYS, DEFAULT_SCOPED_CACHE_MAX_KEYS), + redisFlushIntervalMs: parsePositiveInt( + env.LIVE_REDIS_FLUSH_INTERVAL_MS, + DEFAULT_REDIS_FLUSH_INTERVAL_MS + ), + redisFlushMaxItems: parsePositiveInt(env.LIVE_REDIS_FLUSH_MAX_ITEMS, DEFAULT_REDIS_FLUSH_MAX_ITEMS) }); type RedisLike = Pick< @@ -378,7 +465,50 @@ const candleCursorField = (underlyingId: string, intervalMs: number): string => const overlayRedisKey = (underlyingId: string): string => `live:equity-overlay:${underlyingId}`; const overlayCursorField = (underlyingId: string): string => `equities:${underlyingId}`; +const dropMatchingCursor = ( + items: T[], + target: Cursor, + cursorOf: (item: T) => Cursor +): T[] => items.filter((item) => compareCursors(cursorOf(item), target) !== 0); + +const insertNewestFirst = ( + items: T[], + item: T, + cursorOf: (item: T) => Cursor, + limit: number +): { items: T[]; outOfOrder: boolean } => { + const cursor = cursorOf(item); + const deduped = dropMatchingCursor(items, cursor, cursorOf); + const head = deduped[0]; + const outOfOrder = head ? compareCursors(cursor, cursorOf(head)) > 0 : false; + + if (!outOfOrder) { + return { + items: [item, ...deduped].slice(0, limit), + outOfOrder: false + }; + } + + return { + items: sortGenericItems([...deduped, item], cursorOf).slice(0, limit), + outOfOrder: true + }; +}; + +type BufferedRedisWrite = { + listKey: string; + cursorField: string; + items: unknown[]; + limit: number; + cursor: Cursor | null; + updates: number; +}; + +const isLiveStateConfig = (value: GenericLiveLimits | LiveStateConfig): value is LiveStateConfig => + "limits" in value; + export class LiveStateManager { + private readonly config: LiveStateConfig; private readonly generic: { [K in LiveGenericChannel]: GenericFeedConfig; }; @@ -386,14 +516,22 @@ export class LiveStateManager { private readonly genericCursors = new Map(); private readonly candleItems = new Map(); private readonly candleCursors = new Map(); + private readonly candleAccess = new Map(); private readonly overlayItems = new Map(); private readonly overlayCursors = new Map(); + private readonly overlayAccess = new Map(); + private readonly pendingRedisWrites = new Map(); + private readonly redisFlushTimer: ReturnType | null; private readonly stats = { genericHydrateFromRedis: 0, genericHydrateFromClickHouse: 0, genericCacheSnapshots: 0, scopedClickHouseSnapshots: 0, trimOperations: 0, + redisFlushCount: 0, + redisFlushItems: 0, + cacheEvictions: 0, + outOfOrderEvents: 0, cacheDepthByKey: new Map(), freshnessAgeMsByKey: new Map() }; @@ -401,9 +539,31 @@ export class LiveStateManager { constructor( private readonly clickhouse: ClickHouseClient, private readonly redis: RedisLike | null, - limits: GenericLiveLimits = resolveGenericLiveLimits() + config: GenericLiveLimits | LiveStateConfig = resolveLiveStateConfig() ) { - this.generic = getGenericConfig(limits); + this.config = isLiveStateConfig(config) + ? config + : { + limits: config, + scopedCacheMaxKeys: DEFAULT_SCOPED_CACHE_MAX_KEYS, + redisFlushIntervalMs: DEFAULT_REDIS_FLUSH_INTERVAL_MS, + redisFlushMaxItems: DEFAULT_REDIS_FLUSH_MAX_ITEMS + }; + this.generic = getGenericConfig(this.config.limits); + this.redisFlushTimer = + this.redis && this.redis.isOpen + ? setInterval(() => { + void this.flushRedisWrites(); + }, this.config.redisFlushIntervalMs) + : null; + this.redisFlushTimer?.unref?.(); + } + + async close(): Promise { + if (this.redisFlushTimer) { + clearInterval(this.redisFlushTimer); + } + await this.flushRedisWrites(); } getStatsSnapshot(): { @@ -412,6 +572,10 @@ export class LiveStateManager { genericCacheSnapshots: number; scopedClickHouseSnapshots: number; trimOperations: number; + redisFlushCount: number; + redisFlushItems: number; + cacheEvictions: number; + outOfOrderEvents: number; cacheDepthByKey: Record; freshnessAgeMsByKey: Record; } { @@ -421,6 +585,10 @@ export class LiveStateManager { genericCacheSnapshots: this.stats.genericCacheSnapshots, scopedClickHouseSnapshots: this.stats.scopedClickHouseSnapshots, trimOperations: this.stats.trimOperations, + redisFlushCount: this.stats.redisFlushCount, + redisFlushItems: this.stats.redisFlushItems, + cacheEvictions: this.stats.cacheEvictions, + outOfOrderEvents: this.stats.outOfOrderEvents, cacheDepthByKey: Object.fromEntries(this.stats.cacheDepthByKey), freshnessAgeMsByKey: Object.fromEntries(this.stats.freshnessAgeMsByKey) }; @@ -435,6 +603,23 @@ export class LiveStateManager { }; } + async flushRedisWrites(): Promise { + if (!this.redis?.isOpen) { + return; + } + + const writes = Array.from(this.pendingRedisWrites.values()); + this.pendingRedisWrites.clear(); + + for (const write of writes) { + await this.persistList(write.listKey, write.cursorField, write.items, write.limit, write.cursor); + this.stats.redisFlushCount += 1; + this.stats.redisFlushItems += write.items.length; + metrics.count("api.live.redis_flush_count", 1); + metrics.count("api.live.redis_flush_items", write.items.length); + } + } + private getChannelHealth(channel: LiveHotChannel): LiveChannelHealth { const listKey = HOT_LIVE_REDIS_KEYS[channel]; const thresholdMs = LIVE_FRESHNESS_THRESHOLDS[channel]; @@ -449,6 +634,34 @@ export class LiveStateManager { }; } + private touchAccess(accessMap: Map, key: string): void { + accessMap.set(key, Date.now()); + } + + private evictScopedCachesIfNeeded( + itemsMap: Map, + cursorsMap: Map, + accessMap: Map + ): void { + while (itemsMap.size > this.config.scopedCacheMaxKeys) { + const oldest = [...accessMap.entries()].sort((a, b) => a[1] - b[1])[0]; + if (!oldest) { + break; + } + const [key] = oldest; + itemsMap.delete(key); + cursorsMap.delete( + key.startsWith("live:equity-candles:") + ? key.replace("live:", "") + : key.replace("live:equity-overlay:", "equities:") + ); + accessMap.delete(key); + this.stats.cacheDepthByKey.delete(key); + this.stats.cacheEvictions += 1; + metrics.count("api.live.cache_evictions", 1); + } + } + private updateFreshnessMetric(listKey: string, channel: LiveChannel, item: unknown, now = Date.now()): void { const ts = channel === "equity-candles" || channel === "equity-overlay" @@ -462,6 +675,32 @@ export class LiveStateManager { } } + private queueRedisWrite( + listKey: string, + cursorField: string, + items: unknown[], + limit: number, + cursor: Cursor | null + ): void { + if (!this.redis?.isOpen) { + return; + } + + const existing = this.pendingRedisWrites.get(listKey); + const write: BufferedRedisWrite = { + listKey, + cursorField, + items: [...items], + limit, + cursor, + updates: (existing?.updates ?? 0) + 1 + }; + this.pendingRedisWrites.set(listKey, write); + if (write.updates >= this.config.redisFlushMaxItems) { + void this.flushRedisWrites(); + } + } + async hydrate(): Promise { const channels = Object.keys(this.generic) as LiveGenericChannel[]; await Promise.all(channels.map((channel) => this.hydrateGeneric(channel))); @@ -477,23 +716,16 @@ export class LiveStateManager { this.stats.genericHydrateFromRedis += 1; this.stats.cacheDepthByKey.set(config.redisKey, cached.length); this.updateFreshnessMetric(config.redisKey, channel, cached[0]); - this.genericCursors.set(config.cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, config.cursorField))); - await this.persistList( - config.redisKey, + this.genericCursors.set( config.cursorField, - cached, - config.limit, - this.genericCursors.get(config.cursorField) ?? null + parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, config.cursorField)) ); + await this.persistList(config.redisKey, config.cursorField, cached, config.limit, this.genericCursors.get(config.cursorField) ?? null); return; } } - const fresh = normalizeGenericItems( - channel, - await config.fetchRecent(this.clickhouse, config.limit), - config - ); + const fresh = normalizeGenericItems(channel, await config.fetchRecent(this.clickhouse, config.limit), config); this.stats.genericHydrateFromClickHouse += 1; this.stats.cacheDepthByKey.set(config.redisKey, fresh.length); this.genericItems.set(channel, fresh); @@ -508,32 +740,26 @@ export class LiveStateManager { async getSnapshot(subscription: LiveSubscription): Promise> { switch (subscription.channel) { case "options": { - const scoped = - Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id); + const scoped = Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id); if (subscription.filters?.view === "raw" || scoped) { this.stats.scopedClickHouseSnapshots += 1; const limit = snapshotLimitFor(subscription, this.generic.options.limit); const storageFilters = buildOptionSnapshotFilters(subscription); - const items = await fetchRecentOptionPrints( - this.clickhouse, - limit, - undefined, - storageFilters - ); + const items = await fetchRecentOptionPrints(this.clickhouse, limit, undefined, storageFilters); return { subscription, items, watermark: items[0] ? { ts: items[0].ts, seq: items[0].seq } : null, - next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq })) + next_before: nextBeforeForItems(items, (entry) => ({ ts: entry.ts, seq: entry.seq })) }; } const config = this.generic.options; this.stats.genericCacheSnapshots += 1; const limit = snapshotLimitFor(subscription, config.limit); - const items = (this.genericItems.get("options") ?? []).filter((item) => - matchesOptionPrintFilters(item, subscription.filters) - ).slice(0, limit); + const items = (this.genericItems.get("options") ?? []) + .filter((entry) => matchesOptionPrintFilters(entry, subscription.filters)) + .slice(0, limit); return { subscription, items, @@ -545,9 +771,9 @@ export class LiveStateManager { const config = this.generic.flow; this.stats.genericCacheSnapshots += 1; const limit = snapshotLimitFor(subscription, config.limit); - const items = (this.genericItems.get("flow") ?? []).filter((item) => - matchesFlowPacketFilters(item, subscription.filters) - ).slice(0, limit); + const items = (this.genericItems.get("flow") ?? []) + .filter((entry) => matchesFlowPacketFilters(entry, subscription.filters)) + .slice(0, limit); return { subscription, items, @@ -560,9 +786,7 @@ export class LiveStateManager { const limit = snapshotLimitFor(subscription, config.limit); if (subscription.underlying_ids?.length) { this.stats.scopedClickHouseSnapshots += 1; - const filters: EquityPrintQueryFilters = { - underlyingIds: subscription.underlying_ids - }; + const filters: EquityPrintQueryFilters = { underlyingIds: subscription.underlying_ids }; const items = await fetchRecentEquityPrints(this.clickhouse, limit, filters); return { subscription, @@ -586,12 +810,13 @@ export class LiveStateManager { if (!this.candleItems.has(key)) { await this.hydrateCandles(subscription.underlying_id, subscription.interval_ms); } + this.touchAccess(this.candleAccess, key); const items = this.candleItems.get(key) ?? []; return { subscription, items, watermark: this.candleCursors.get(cursorField) ?? null, - next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq })) + next_before: nextBeforeForItems(items, (entry) => ({ ts: entry.ts, seq: entry.seq })) }; } case "equity-overlay": { @@ -600,12 +825,13 @@ export class LiveStateManager { if (!this.overlayItems.has(key)) { await this.hydrateOverlay(subscription.underlying_id); } + this.touchAccess(this.overlayAccess, key); const items = this.overlayItems.get(key) ?? []; return { subscription, items, watermark: this.overlayCursors.get(cursorField) ?? null, - next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq })) + next_before: nextBeforeForItems(items, (entry) => ({ ts: entry.ts, seq: entry.seq })) }; } default: { @@ -629,48 +855,52 @@ export class LiveStateManager { const candle = EquityCandleSchema.parse(item); const key = candleRedisKey(candle.underlying_id, candle.interval_ms); const cursorField = candleCursorField(candle.underlying_id, candle.interval_ms); - const previousCursor = this.candleCursors.get(cursorField) ?? null; - const items = this.candleItems.get(key) ?? []; - const next = [candle, ...items] - .sort((a, b) => (b.ts - a.ts) || (b.seq - a.seq)) - .slice(0, CHART_LIMITS.candles); - this.candleItems.set(key, next); - this.stats.cacheDepthByKey.set(key, next.length); + const nextState = insertNewestFirst( + this.candleItems.get(key) ?? [], + candle, + (entry) => ({ ts: entry.ts, seq: entry.seq }), + CHART_LIMITS.candles + ); const cursor = { ts: candle.ts, seq: candle.seq }; + this.candleItems.set(key, nextState.items); this.candleCursors.set(cursorField, cursor); - if (next.length > 0) { - this.updateFreshnessMetric(key, "equity-candles", next[0]); + this.touchAccess(this.candleAccess, key); + this.evictScopedCachesIfNeeded(this.candleItems as Map, this.candleCursors, this.candleAccess); + if (nextState.outOfOrder) { + this.stats.outOfOrderEvents += 1; + metrics.count("api.live.out_of_order_events", 1); } - const outOfOrder = previousCursor ? compareCursors(cursor, previousCursor) > 0 : false; - if (outOfOrder) { - await this.persistList(key, cursorField, next, CHART_LIMITS.candles, cursor); - } else { - await this.persistItem(key, cursorField, candle, CHART_LIMITS.candles, cursor, next.length); + this.stats.cacheDepthByKey.set(key, nextState.items.length); + if (nextState.items.length > 0) { + this.updateFreshnessMetric(key, "equity-candles", nextState.items[0]); } + this.queueRedisWrite(key, cursorField, nextState.items, CHART_LIMITS.candles, cursor); return cursor; } case "equity-overlay": { const print = EquityPrintSchema.parse(item); const key = overlayRedisKey(print.underlying_id); const cursorField = overlayCursorField(print.underlying_id); - const previousCursor = this.overlayCursors.get(cursorField) ?? null; - const items = this.overlayItems.get(key) ?? []; - const next = [print, ...items] - .sort((a, b) => (b.ts - a.ts) || (b.seq - a.seq)) - .slice(0, CHART_LIMITS.overlay); - this.overlayItems.set(key, next); - this.stats.cacheDepthByKey.set(key, next.length); + const nextState = insertNewestFirst( + this.overlayItems.get(key) ?? [], + print, + (entry) => ({ ts: entry.ts, seq: entry.seq }), + CHART_LIMITS.overlay + ); const cursor = { ts: print.ts, seq: print.seq }; + this.overlayItems.set(key, nextState.items); this.overlayCursors.set(cursorField, cursor); - if (next.length > 0) { - this.updateFreshnessMetric(key, "equity-overlay", next[0]); + this.touchAccess(this.overlayAccess, key); + this.evictScopedCachesIfNeeded(this.overlayItems as Map, this.overlayCursors, this.overlayAccess); + if (nextState.outOfOrder) { + this.stats.outOfOrderEvents += 1; + metrics.count("api.live.out_of_order_events", 1); } - const outOfOrder = previousCursor ? compareCursors(cursor, previousCursor) > 0 : false; - if (outOfOrder) { - await this.persistList(key, cursorField, next, CHART_LIMITS.overlay, cursor); - } else { - await this.persistItem(key, cursorField, print, CHART_LIMITS.overlay, cursor, next.length); + this.stats.cacheDepthByKey.set(key, nextState.items.length); + if (nextState.items.length > 0) { + this.updateFreshnessMetric(key, "equity-overlay", nextState.items[0]); } + this.queueRedisWrite(key, cursorField, nextState.items, CHART_LIMITS.overlay, cursor); return cursor; } default: { @@ -679,22 +909,28 @@ export class LiveStateManager { if (!isWithinLiveFeedLookback(channel, parsed)) { return null; } - const previousCursor = this.genericCursors.get(config.cursorField) ?? null; - const items = this.genericItems.get(channel) ?? []; - const next = normalizeGenericItems(channel, [parsed, ...items], config); - this.genericItems.set(channel, next); - this.stats.cacheDepthByKey.set(config.redisKey, next.length); + const cursor = config.cursor(parsed); + const nextState = + channel === "nbbo" + ? { + items: normalizeGenericItems(channel, [parsed, ...(this.genericItems.get(channel) ?? [])], config), + outOfOrder: false + } + : insertNewestFirst(this.genericItems.get(channel) ?? [], parsed, config.cursor, config.limit); + + if (nextState.outOfOrder) { + this.stats.outOfOrderEvents += 1; + metrics.count("api.live.out_of_order_events", 1); + } + + this.genericItems.set(channel, nextState.items); this.genericCursors.set(config.cursorField, cursor); - if (next.length > 0) { - this.updateFreshnessMetric(config.redisKey, channel, next[0]); - } - const outOfOrder = previousCursor ? compareCursors(cursor, previousCursor) > 0 : false; - if (channel === "nbbo" || outOfOrder) { - await this.persistList(config.redisKey, config.cursorField, next, config.limit, cursor); - } else { - await this.persistItem(config.redisKey, config.cursorField, parsed, config.limit, cursor, next.length); + this.stats.cacheDepthByKey.set(config.redisKey, nextState.items.length); + if (nextState.items.length > 0) { + this.updateFreshnessMetric(config.redisKey, channel, nextState.items[0]); } + this.queueRedisWrite(config.redisKey, config.cursorField, nextState.items, config.limit, cursor); return cursor; } } @@ -708,6 +944,8 @@ export class LiveStateManager { const cached = parseJsonList(payloads, (value) => EquityCandleSchema.parse(value)); if (cached.length > 0) { this.candleItems.set(key, cached); + this.touchAccess(this.candleAccess, key); + this.evictScopedCachesIfNeeded(this.candleItems as Map, this.candleCursors, this.candleAccess); this.stats.cacheDepthByKey.set(key, cached.length); this.updateFreshnessMetric(key, "equity-candles", cached[0]); this.candleCursors.set(cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField))); @@ -717,6 +955,8 @@ export class LiveStateManager { const fresh = await fetchRecentEquityCandles(this.clickhouse, underlyingId, intervalMs, CHART_LIMITS.candles); this.candleItems.set(key, fresh); + this.touchAccess(this.candleAccess, key); + this.evictScopedCachesIfNeeded(this.candleItems as Map, this.candleCursors, this.candleAccess); this.stats.cacheDepthByKey.set(key, fresh.length); if (fresh.length > 0) { this.updateFreshnessMetric(key, "equity-candles", fresh[0]); @@ -734,6 +974,8 @@ export class LiveStateManager { const cached = parseJsonList(payloads, (value) => EquityPrintSchema.parse(value)); if (cached.length > 0) { this.overlayItems.set(key, cached); + this.touchAccess(this.overlayAccess, key); + this.evictScopedCachesIfNeeded(this.overlayItems as Map, this.overlayCursors, this.overlayAccess); this.stats.cacheDepthByKey.set(key, cached.length); this.updateFreshnessMetric(key, "equity-overlay", cached[0]); this.overlayCursors.set(cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField))); @@ -742,9 +984,11 @@ export class LiveStateManager { } const fresh = (await fetchRecentEquityPrints(this.clickhouse, CHART_LIMITS.overlay)).filter( - (item) => item.underlying_id === underlyingId + (entry) => entry.underlying_id === underlyingId ); this.overlayItems.set(key, fresh); + this.touchAccess(this.overlayAccess, key); + this.evictScopedCachesIfNeeded(this.overlayItems as Map, this.overlayCursors, this.overlayAccess); this.stats.cacheDepthByKey.set(key, fresh.length); if (fresh.length > 0) { this.updateFreshnessMetric(key, "equity-overlay", fresh[0]); @@ -754,25 +998,6 @@ export class LiveStateManager { await this.persistList(key, cursorField, fresh, CHART_LIMITS.overlay, watermark); } - private async persistItem( - listKey: string, - cursorField: string, - item: T, - limit: number, - cursor: Cursor | null, - depth: number - ): Promise { - if (!this.redis?.isOpen) { - return; - } - - await this.redis.lPush(listKey, JSON.stringify(item)); - await this.redis.lTrim(listKey, 0, limit - 1); - this.stats.trimOperations += 1; - this.stats.cacheDepthByKey.set(listKey, Math.min(depth, limit)); - await this.redis.hSet(CURSOR_HASH_KEY, cursorField, JSON.stringify(cursor)); - } - private async persistList( listKey: string, cursorField: string, @@ -784,7 +1009,7 @@ export class LiveStateManager { return; } - const payloads = items.map((item) => JSON.stringify(item)); + const payloads = items.map((entry) => JSON.stringify(entry)); await this.redis.lTrim(listKey, 1, 0); this.stats.trimOperations += 1; if (payloads.length > 0) { diff --git a/services/api/tests/live.test.ts b/services/api/tests/live.test.ts index 3d0aa63..bd4d0c8 100644 --- a/services/api/tests/live.test.ts +++ b/services/api/tests/live.test.ts @@ -66,9 +66,9 @@ describe("LiveStateManager", () => { expect(limits.options).toBe(777); expect(limits.nbbo).toBe(100000); - expect(limits.flow).toBe(10000); - expect(limits["equity-quotes"]).toBe(10000); - expect(limits.alerts).toBe(10000); + expect(limits.flow).toBe(500); + expect(limits["equity-quotes"]).toBe(500); + expect(limits.alerts).toBe(300); }); it("hydrates snapshots from redis generic windows", async () => { @@ -204,13 +204,121 @@ describe("LiveStateManager", () => { ]); const persisted = await redis.lRange("live:flow", 0, 99); - expect(persisted).toHaveLength(2); + await manager.flushRedisWrites(); + const flushed = await redis.lRange("live:flow", 0, 99); + expect(persisted).toHaveLength(0); + expect(flushed).toHaveLength(2); const stats = manager.getStatsSnapshot(); expect(stats.trimOperations).toBeGreaterThan(0); + expect(stats.redisFlushCount).toBeGreaterThan(0); expect(stats.cacheDepthByKey["live:flow"]).toBe(2); }); + it("reorders out-of-order live events without dropping newest-first semantics", async () => { + const now = Date.now(); + const manager = new LiveStateManager(makeClickHouse(), null, { + limits: { + options: 1000, + nbbo: 1000, + equities: 1000, + "equity-quotes": 500, + "equity-joins": 500, + flow: 3, + "smart-money": 300, + "classifier-hits": 300, + alerts: 300, + "inferred-dark": 300 + }, + scopedCacheMaxKeys: 32, + redisFlushIntervalMs: 250, + redisFlushMaxItems: 100 + }); + + await manager.ingest("flow", { + source_ts: now, + ingest_ts: now + 1, + seq: 2, + trace_id: "flow-2", + id: "flow-2", + members: [], + features: {}, + join_quality: {} + }); + await manager.ingest("flow", { + source_ts: now - 1_000, + ingest_ts: now - 999, + seq: 1, + trace_id: "flow-1", + id: "flow-1", + members: [], + features: {}, + join_quality: {} + }); + + const snapshot = await manager.getSnapshot({ channel: "flow" }); + expect((snapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([ + "flow-2", + "flow-1" + ]); + expect(manager.getStatsSnapshot().outOfOrderEvents).toBe(1); + }); + + it("evicts least-recently-used scoped candle caches past the configured key limit", async () => { + const manager = new LiveStateManager(makeClickHouse(), null, { + limits: resolveGenericLiveLimits(), + scopedCacheMaxKeys: 1, + redisFlushIntervalMs: 250, + redisFlushMaxItems: 100 + }); + + await manager.ingest("equity-candles", { + source_ts: 100, + ingest_ts: 101, + seq: 1, + trace_id: "candle:SPY:60000:100", + ts: 100, + interval_ms: 60000, + underlying_id: "SPY", + open: 1, + high: 2, + low: 1, + close: 2, + volume: 10, + trade_count: 1 + }); + await manager.ingest("equity-candles", { + source_ts: 200, + ingest_ts: 201, + seq: 2, + trace_id: "candle:QQQ:60000:200", + ts: 200, + interval_ms: 60000, + underlying_id: "QQQ", + open: 3, + high: 4, + low: 3, + close: 4, + volume: 20, + trade_count: 2 + }); + + const qqqSnapshot = await manager.getSnapshot({ + channel: "equity-candles", + underlying_id: "QQQ", + interval_ms: 60000 + }); + const spySnapshot = await manager.getSnapshot({ + channel: "equity-candles", + underlying_id: "SPY", + interval_ms: 60000 + }); + + expect(qqqSnapshot.items).toHaveLength(1); + expect(spySnapshot.items).toEqual([]); + expect(manager.getStatsSnapshot().cacheEvictions).toBeGreaterThan(0); + }); + it("filters option and flow snapshots using subscription filters", async () => { const manager = new LiveStateManager(makeClickHouse(), null); const now = Date.now(); diff --git a/services/candles/src/index.ts b/services/candles/src/index.ts index 39e6609..86f0dfa 100644 --- a/services/candles/src/index.ts +++ b/services/candles/src/index.ts @@ -5,6 +5,7 @@ import { SUBJECT_EQUITY_PRINTS, STREAM_EQUITY_CANDLES, STREAM_EQUITY_PRINTS, + buildStreamConfig, buildDurableConsumer, connectJetStreamWithRetry, ensureStream, @@ -240,31 +241,8 @@ const run = async () => { { attempts: 120, delayMs: 500 } ); - await ensureStream(jsm, { - name: STREAM_EQUITY_PRINTS, - subjects: [SUBJECT_EQUITY_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_CANDLES, - subjects: [SUBJECT_EQUITY_CANDLES], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_CANDLES, SUBJECT_EQUITY_CANDLES, "derived")); const clickhouse = createClickHouseClient({ url: env.CLICKHOUSE_URL, diff --git a/services/compute/src/index.ts b/services/compute/src/index.ts index 65c6a1e..8e561c3 100644 --- a/services/compute/src/index.ts +++ b/services/compute/src/index.ts @@ -26,6 +26,7 @@ import { STREAM_SMART_MONEY_EVENTS, STREAM_OPTION_NBBO, STREAM_OPTION_SIGNAL_PRINTS, + buildStreamConfig, buildDurableConsumer, connectJetStreamWithRetry, ensureStream, @@ -40,12 +41,13 @@ import { ensureInferredDarkTable, ensureFlowPacketsTable, ensureSmartMoneyEventsTable, - insertAlert, - insertClassifierHit, - insertEquityPrintJoin, - insertInferredDark, - insertFlowPacket, - insertSmartMoneyEvent + ClickHouseBatchWriter, + enqueueAlertInsert, + enqueueClassifierHitInsert, + enqueueEquityPrintJoinInsert, + enqueueFlowPacketInsert, + enqueueInferredDarkInsert, + enqueueSmartMoneyEventInsert, } from "@islandflow/storage"; import { AlertEventSchema, @@ -82,7 +84,12 @@ import { type DarkInferenceConfig } from "./dark-inference"; import { buildEquityPrintJoin, type EquityQuoteJoin } from "./equity-joins"; -import { createRedisClient, updateRollingStats, type RollingStatsConfig } from "./rolling-stats"; +import { + createRedisClient, + RollingWindowStore, + type RollingStatsConfig, + type RollingWindowStoreConfig +} from "./rolling-stats"; import { summarizeStructure, type ContractLeg } from "./structures"; import { buildStructureFlowPacket, @@ -103,6 +110,8 @@ const envSchema = z.object({ CLUSTER_WINDOW_MS: z.coerce.number().int().positive().default(500), ROLLING_WINDOW_SIZE: z.coerce.number().int().positive().default(50), ROLLING_TTL_SEC: z.coerce.number().int().nonnegative().default(86400), + ROLLING_CACHE_FLUSH_INTERVAL_MS: z.coerce.number().int().positive().default(30_000), + ROLLING_CACHE_MAX_KEYS: z.coerce.number().int().positive().default(20_000), COMPUTE_DELIVER_POLICY: z.enum(["new", "all", "last", "last_per_subject"]).default("new"), COMPUTE_CONSUMER_RESET: z .preprocess((value) => { @@ -119,6 +128,8 @@ const envSchema = z.object({ }, z.boolean()) .default(false), NBBO_MAX_AGE_MS: z.coerce.number().int().positive().default(1000), + COMPUTE_NBBO_CACHE_MAX_KEYS: z.coerce.number().int().positive().default(20_000), + COMPUTE_NBBO_CACHE_TTL_MS: z.coerce.number().int().positive().default(900_000), EQUITY_QUOTE_MAX_AGE_MS: z.coerce.number().int().positive().default(1000), DARK_INFER_WINDOW_MS: z.coerce.number().int().positive().default(60000), DARK_INFER_COOLDOWN_MS: z.coerce.number().int().nonnegative().default(30000), @@ -269,6 +280,9 @@ const clusters = new Map(); const nbboCache = new Map(); const equityQuoteCache = new Map(); const darkInferenceState = createDarkInferenceState(); +const nbboCacheTouchedAt = new Map(); +const equityQuoteCacheTouchedAt = new Map(); +const darkInferenceTouchedAt = new Map(); const recentLegsByKey = new Map(); const recentLegsByRoot = new Map(); const recentStructureEmits = new Map(); @@ -278,6 +292,20 @@ const runtimeState = { }; const MAX_RECENT_LEGS = 20; +const EQUITY_QUOTE_CACHE_MAX_KEYS = 2_000; +const EQUITY_QUOTE_CACHE_TTL_MS = 900_000; +const DARK_INFERENCE_TTL_MS = 900_000; +const CACHE_PRUNE_INTERVAL_MS = 60_000; + +const emitCounters = { + flowPackets: 0, + structurePackets: 0, + smartMoneyEvents: 0, + classifierHits: 0, + alerts: 0, + equityJoins: 0, + darkEvents: 0 +}; const rollingKey = (metric: string, contractId: string): string => { return `rolling:${metric}:${contractId}`; @@ -479,8 +507,8 @@ const pruneRecentStructureEmits = (anchorTs: number): void => { }; const emitStructurePacketIfNeeded = async ( - clickhouse: ReturnType, js: Awaited>["js"], + batchWriter: ClickHouseBatchWriter, legs: LegEvidence[], summary: ReturnType, currentContractId: string @@ -512,16 +540,11 @@ const emitStructurePacketIfNeeded = async ( const packet = buildStructureFlowPacket(plan, summary); const validated = FlowPacketSchema.parse(packet); - await insertFlowPacket(clickhouse, validated); + enqueueFlowPacketInsert(batchWriter, validated); await publishJson(js, SUBJECT_FLOW_PACKETS, validated); - await emitClassifiers(clickhouse, js, validated); - - logger.info("emitted structure flow packet", { - id: validated.id, - type: summary.type, - legs: summary.legs, - strikes: summary.strikes - }); + emitCounters.flowPackets += 1; + emitCounters.structurePackets += 1; + await emitClassifiers(js, batchWriter, validated); }; const applyDeliverPolicy = ( @@ -606,6 +629,7 @@ const updateNbboCache = (nbbo: OptionNBBO): void => { (nbbo.ts === existing.ts && nbbo.seq >= existing.seq) ) { nbboCache.set(nbbo.option_contract_id, nbbo); + nbboCacheTouchedAt.set(nbbo.option_contract_id, Date.now()); } }; @@ -617,6 +641,7 @@ const updateEquityQuoteCache = (quote: EquityQuote): void => { (quote.ts === existing.ts && quote.seq >= existing.seq) ) { equityQuoteCache.set(quote.underlying_id, quote); + equityQuoteCacheTouchedAt.set(quote.underlying_id, Date.now()); } }; @@ -626,6 +651,7 @@ const selectNbbo = (contractId: string, ts: number): NbboJoin => { return { nbbo: null, ageMs: env.NBBO_MAX_AGE_MS + 1, stale: true }; } + nbboCacheTouchedAt.set(contractId, Date.now()); const ageMs = Math.abs(ts - nbbo.ts); const stale = ageMs > env.NBBO_MAX_AGE_MS; return { nbbo, ageMs, stale }; @@ -637,11 +663,77 @@ const selectEquityQuote = (underlyingId: string, ts: number): EquityQuoteJoin => return { quote: null, ageMs: env.EQUITY_QUOTE_MAX_AGE_MS + 1, stale: true }; } + equityQuoteCacheTouchedAt.set(underlyingId, Date.now()); const ageMs = Math.abs(ts - quote.ts); const stale = ageMs > env.EQUITY_QUOTE_MAX_AGE_MS; return { quote, ageMs, stale }; }; +const pruneTimedMap = ( + values: Map, + touchedAt: Map, + maxKeys: number, + ttlMs: number, + now = Date.now() +): number => { + let removed = 0; + + for (const [key, touched] of touchedAt) { + if (now - touched > ttlMs) { + touchedAt.delete(key); + values.delete(key); + removed += 1; + } + } + + if (values.size <= maxKeys) { + return removed; + } + + const overflow = values.size - maxKeys; + const oldest = [...touchedAt.entries()].sort((a, b) => a[1] - b[1]).slice(0, overflow); + for (const [key] of oldest) { + touchedAt.delete(key); + values.delete(key); + removed += 1; + } + + return removed; +}; + +const pruneComputeCaches = (rollingStore: RollingWindowStore, now = Date.now()) => { + const nbboRemoved = pruneTimedMap( + nbboCache, + nbboCacheTouchedAt, + env.COMPUTE_NBBO_CACHE_MAX_KEYS, + env.COMPUTE_NBBO_CACHE_TTL_MS, + now + ); + const quoteRemoved = pruneTimedMap( + equityQuoteCache, + equityQuoteCacheTouchedAt, + EQUITY_QUOTE_CACHE_MAX_KEYS, + EQUITY_QUOTE_CACHE_TTL_MS, + now + ); + const darkRemoved = pruneTimedMap( + darkInferenceState.lastEmittedByUnderlying, + darkInferenceTouchedAt, + EQUITY_QUOTE_CACHE_MAX_KEYS, + DARK_INFERENCE_TTL_MS, + now + ); + const rollingRemoved = rollingStore.prune(now); + + logger.info("compute cache summary", { + nbbo_cache_size: nbboCache.size, + equity_quote_cache_size: equityQuoteCache.size, + dark_inference_cache_size: darkInferenceState.lastEmittedByUnderlying.size, + rolling_cache_size: rollingStore.size, + removed: nbboRemoved + quoteRemoved + darkRemoved + rollingRemoved + }); +}; + const classifyPlacement = (price: number, join: NbboJoin): NbboPlacement => { if (!Number.isFinite(price)) { return "MISSING"; @@ -679,10 +771,9 @@ const classifyPlacement = (price: number, join: NbboJoin): NbboPlacement => { }; const flushCluster = async ( - clickhouse: ReturnType, js: Awaited>["js"], - redis: ReturnType, - rollingConfig: RollingStatsConfig, + batchWriter: ClickHouseBatchWriter, + rollingStore: RollingWindowStore, cluster: ClusterState ): Promise => { if (cluster.flushed) { @@ -784,12 +875,7 @@ const flushCluster = async ( prefix: string ): Promise => { try { - const snapshot = await updateRollingStats( - redis, - rollingKey(metric, cluster.contractId), - value, - rollingConfig - ); + const snapshot = rollingStore.update(rollingKey(metric, cluster.contractId), value); features[`${prefix}_mean`] = roundTo(snapshot.mean); features[`${prefix}_std`] = roundTo(snapshot.stddev); features[`${prefix}_z`] = roundTo(snapshot.zscore); @@ -824,7 +910,7 @@ const flushCluster = async ( features.structure_rights = summary.rights; } - await emitStructurePacketIfNeeded(clickhouse, js, legs, summary, currentLeg.contractId); + await emitStructurePacketIfNeeded(js, batchWriter, legs, summary, currentLeg.contractId); const rootKey = buildRootKey(currentLeg); const rootCandidates = [ @@ -834,7 +920,7 @@ const flushCluster = async ( const rollLegs = [currentLeg, ...rootCandidates]; const rollSummary = summarizeStructure(rollLegs); if (rollSummary?.type === "roll") { - await emitStructurePacketIfNeeded(clickhouse, js, rollLegs, rollSummary, currentLeg.contractId); + await emitStructurePacketIfNeeded(js, batchWriter, rollLegs, rollSummary, currentLeg.contractId); } storeRecentLeg(currentLeg, anchorTs); @@ -873,16 +959,10 @@ const flushCluster = async ( const validated = FlowPacketSchema.parse(packet); try { - await insertFlowPacket(clickhouse, validated); + enqueueFlowPacketInsert(batchWriter, validated); await publishJson(js, SUBJECT_FLOW_PACKETS, validated); - - await emitClassifiers(clickhouse, js, validated); - - logger.info("emitted flow packet", { - id: validated.id, - contract: cluster.contractId, - count: cluster.members.length - }); + emitCounters.flowPackets += 1; + await emitClassifiers(js, batchWriter, validated); } catch (error) { if (isExpectedShutdownNatsError(error)) { logger.info("skipped flow packet publish during shutdown", { @@ -899,8 +979,8 @@ const flushCluster = async ( }; const emitClassifiers = async ( - clickhouse: ReturnType, js: Awaited>["js"], + batchWriter: ClickHouseBatchWriter, packet: FlowPacket ): Promise => { let smartMoneyEvent: SmartMoneyEvent; @@ -915,8 +995,9 @@ const emitClassifiers = async ( : packet.source_ts; const eventCalendarMatch = underlyingId ? eventCalendarProvider.findNextEvent(underlyingId, referenceTs) : null; smartMoneyEvent = SmartMoneyEventSchema.parse(buildSmartMoneyEventFromPacket(packet, { eventCalendarMatch })); - await insertSmartMoneyEvent(clickhouse, smartMoneyEvent); + enqueueSmartMoneyEventInsert(batchWriter, smartMoneyEvent); await publishJson(js, SUBJECT_SMART_MONEY_EVENTS, smartMoneyEvent); + emitCounters.smartMoneyEvents += 1; } catch (error) { if (isExpectedShutdownNatsError(error)) { return; @@ -945,8 +1026,9 @@ const emitClassifiers = async ( for (const hit of hitEvents) { try { - await insertClassifierHit(clickhouse, hit); + enqueueClassifierHitInsert(batchWriter, hit); await publishJson(js, SUBJECT_CLASSIFIER_HITS, hit); + emitCounters.classifierHits += 1; } catch (error) { if (isExpectedShutdownNatsError(error)) { continue; @@ -981,8 +1063,9 @@ const emitClassifiers = async ( }); try { - await insertAlert(clickhouse, alert); + enqueueAlertInsert(batchWriter, alert); await publishJson(js, SUBJECT_ALERTS, alert); + emitCounters.alerts += 1; } catch (error) { if (isExpectedShutdownNatsError(error)) { return; @@ -995,17 +1078,21 @@ const emitClassifiers = async ( }; const emitEquityJoin = async ( - clickhouse: ReturnType, js: Awaited>["js"], + batchWriter: ClickHouseBatchWriter, print: EquityPrint ): Promise => { const join = selectEquityQuote(print.underlying_id, print.ts); const payload: EquityPrintJoin = EquityPrintJoinSchema.parse(buildEquityPrintJoin(print, join)); try { - await insertEquityPrintJoin(clickhouse, payload); + enqueueEquityPrintJoinInsert(batchWriter, payload); } catch (error) { - logger.error("failed to emit equity print join", { + if (isExpectedShutdownNatsError(error)) { + return; + } + + logger.error("failed to queue equity print join", { error: error instanceof Error ? error.message : String(error), trace_id: payload.trace_id }); @@ -1014,6 +1101,7 @@ const emitEquityJoin = async ( try { await publishJson(js, SUBJECT_EQUITY_JOINS, payload); + emitCounters.equityJoins += 1; } catch (error) { if (isExpectedShutdownNatsError(error)) { return; @@ -1024,20 +1112,26 @@ const emitEquityJoin = async ( }); } - await emitDarkInferences(clickhouse, js, payload); + await emitDarkInferences(js, batchWriter, payload); }; const emitDarkInferences = async ( - clickhouse: ReturnType, js: Awaited>["js"], + batchWriter: ClickHouseBatchWriter, join: EquityPrintJoin ): Promise => { const events = evaluateDarkInferences(join, darkInferenceConfig, darkInferenceState); for (const event of events) { const validated: InferredDarkEvent = InferredDarkEventSchema.parse(event); try { - await insertInferredDark(clickhouse, validated); + enqueueInferredDarkInsert(batchWriter, validated); await publishJson(js, SUBJECT_INFERRED_DARK, validated); + emitCounters.darkEvents += 1; + const underlyingId = + typeof join.features?.underlying_id === "string" ? join.features.underlying_id : null; + if (underlyingId) { + darkInferenceTouchedAt.set(underlyingId, Date.now()); + } } catch (error) { if (isExpectedShutdownNatsError(error)) { continue; @@ -1051,10 +1145,9 @@ const emitDarkInferences = async ( }; const flushEligibleClusters = async ( - clickhouse: ReturnType, js: Awaited>["js"], - redis: ReturnType, - rollingConfig: RollingStatsConfig, + batchWriter: ClickHouseBatchWriter, + rollingStore: RollingWindowStore, currentTs: number, skipContractId: string ): Promise => { @@ -1065,7 +1158,7 @@ const flushEligibleClusters = async ( if (currentTs - cluster.endTs > env.CLUSTER_WINDOW_MS) { clusters.delete(contractId); - await flushCluster(clickhouse, js, redis, rollingConfig, cluster); + await flushCluster(js, batchWriter, rollingStore, cluster); } } }; @@ -1081,135 +1174,16 @@ const run = async () => { { attempts: 120, delayMs: 500 } ); - await ensureStream(jsm, { - name: STREAM_OPTION_SIGNAL_PRINTS, - subjects: [SUBJECT_OPTION_SIGNAL_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_OPTION_NBBO, - subjects: [SUBJECT_OPTION_NBBO], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_PRINTS, - subjects: [SUBJECT_EQUITY_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_QUOTES, - subjects: [SUBJECT_EQUITY_QUOTES], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_FLOW_PACKETS, - subjects: [SUBJECT_FLOW_PACKETS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_SMART_MONEY_EVENTS, - subjects: [SUBJECT_SMART_MONEY_EVENTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_JOINS, - subjects: [SUBJECT_EQUITY_JOINS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_INFERRED_DARK, - subjects: [SUBJECT_INFERRED_DARK], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_CLASSIFIER_HITS, - subjects: [SUBJECT_CLASSIFIER_HITS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_ALERTS, - subjects: [SUBJECT_ALERTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_FLOW_PACKETS, SUBJECT_FLOW_PACKETS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_SMART_MONEY_EVENTS, SUBJECT_SMART_MONEY_EVENTS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_JOINS, SUBJECT_EQUITY_JOINS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_INFERRED_DARK, SUBJECT_INFERRED_DARK, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_CLASSIFIER_HITS, SUBJECT_CLASSIFIER_HITS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_ALERTS, SUBJECT_ALERTS, "derived")); const clickhouse = createClickHouseClient({ url: env.CLICKHOUSE_URL, @@ -1242,6 +1216,51 @@ const run = async () => { windowSize: env.ROLLING_WINDOW_SIZE, ttlSeconds: env.ROLLING_TTL_SEC }; + const rollingStore = new RollingWindowStore({ + ...rollingConfig, + flushIntervalMs: env.ROLLING_CACHE_FLUSH_INTERVAL_MS, + maxKeys: env.ROLLING_CACHE_MAX_KEYS + } satisfies RollingWindowStoreConfig); + const batchWriter = new ClickHouseBatchWriter(clickhouse, { + flushIntervalMs: 100, + maxRows: 250, + onError: (table, error, rowCount) => { + logger.error("batched clickhouse insert failed", { + table, + row_count: rowCount, + error: error instanceof Error ? error.message : String(error), + action: "dropped" + }); + } + }); + const rollingFlushTimer = setInterval(() => { + void rollingStore.flushToRedis(redis); + }, env.ROLLING_CACHE_FLUSH_INTERVAL_MS); + const pruneTimer = setInterval(() => { + pruneComputeCaches(rollingStore); + }, CACHE_PRUNE_INTERVAL_MS); + const summaryTimer = setInterval(() => { + logger.info("compute minute summary", { + flow_packets_emitted: emitCounters.flowPackets, + structure_packets_emitted: emitCounters.structurePackets, + smart_money_events_emitted: emitCounters.smartMoneyEvents, + classifier_hits_emitted: emitCounters.classifierHits, + alerts_emitted: emitCounters.alerts, + equity_joins_emitted: emitCounters.equityJoins, + dark_events_emitted: emitCounters.darkEvents, + rolling_stats_cache_size: rollingStore.size + }); + emitCounters.flowPackets = 0; + emitCounters.structurePackets = 0; + emitCounters.smartMoneyEvents = 0; + emitCounters.classifierHits = 0; + emitCounters.alerts = 0; + emitCounters.equityJoins = 0; + emitCounters.darkEvents = 0; + }, 60_000); + rollingFlushTimer.unref?.(); + pruneTimer.unref?.(); + summaryTimer.unref?.(); await retry("clickhouse table init", 120, 500, async () => { await ensureFlowPacketsTable(clickhouse); @@ -1578,7 +1597,7 @@ const run = async () => { try { const print = EquityPrintSchema.parse(equitySubscription.decode(msg)); - await emitEquityJoin(clickhouse, js, print); + await emitEquityJoin(js, batchWriter, print); msg.ack(); } catch (error) { logger.error("failed to process equity print", { @@ -1602,11 +1621,16 @@ const run = async () => { runtimeState.shuttingDown = true; runtimeState.shutdownPromise = (async () => { logger.info("service stopping", { signal }); + clearInterval(rollingFlushTimer); + clearInterval(pruneTimer); + clearInterval(summaryTimer); for (const cluster of [...clusters.values()]) { - await flushCluster(clickhouse, js, redis, rollingConfig, cluster); + await flushCluster(js, batchWriter, rollingStore, cluster); } clusters.clear(); + await batchWriter.close(); + await rollingStore.flushToRedis(redis); try { await nc.drain(); @@ -1655,10 +1679,9 @@ const run = async () => { try { const print = OptionPrintSchema.parse(subscription.decode(msg)); await flushEligibleClusters( - clickhouse, js, - redis, - rollingConfig, + batchWriter, + rollingStore, print.ts, print.option_contract_id ); @@ -1674,7 +1697,7 @@ const run = async () => { updateCluster(existing, print); } else { clusters.delete(print.option_contract_id); - await flushCluster(clickhouse, js, redis, rollingConfig, existing); + await flushCluster(js, batchWriter, rollingStore, existing); clusters.set(print.option_contract_id, buildCluster(print)); } diff --git a/services/compute/src/rolling-stats.ts b/services/compute/src/rolling-stats.ts index 63c6caa..d30b930 100644 --- a/services/compute/src/rolling-stats.ts +++ b/services/compute/src/rolling-stats.ts @@ -5,6 +5,11 @@ export type RollingStatsConfig = { ttlSeconds: number; }; +export type RollingWindowStoreConfig = RollingStatsConfig & { + flushIntervalMs: number; + maxKeys: number; +}; + export type RollingSnapshot = { baselineCount: number; mean: number; @@ -12,6 +17,12 @@ export type RollingSnapshot = { zscore: number; }; +type RollingWindowEntry = { + values: number[]; + updatedAt: number; + dirty: boolean; +}; + const toNumbers = (values: string[]): number[] => { return values .map((value) => Number(value)) @@ -49,26 +60,120 @@ export const createRedisClient = (url: string) => { return createClient({ url }); }; -export const updateRollingStats = async ( - client: ReturnType, - key: string, - value: number, - config: RollingStatsConfig -): Promise => { - const limit = Math.max(0, config.windowSize - 1); - const existing = await client.lRange(key, 0, limit); - const baseline = toNumbers(existing); - const snapshot = computeSnapshot(baseline, value); +const getOldestKey = (store: Map): string | null => { + let oldestKey: string | null = null; + let oldestUpdatedAt = Number.POSITIVE_INFINITY; - const multi = client.multi(); - multi.lPush(key, value.toString()); - if (config.windowSize > 0) { - multi.lTrim(key, 0, config.windowSize - 1); + for (const [key, entry] of store) { + if (entry.updatedAt < oldestUpdatedAt) { + oldestUpdatedAt = entry.updatedAt; + oldestKey = key; + } } - if (config.ttlSeconds > 0) { - multi.expire(key, config.ttlSeconds); - } - await multi.exec(); - return snapshot; + return oldestKey; }; + +export class RollingWindowStore { + private readonly store = new Map(); + private readonly ttlMs: number; + private readonly windowSize: number; + private readonly maxKeys: number; + + constructor(private readonly config: RollingWindowStoreConfig) { + this.ttlMs = Math.max(0, config.ttlSeconds * 1000); + this.windowSize = Math.max(1, config.windowSize); + this.maxKeys = Math.max(1, config.maxKeys); + } + + get size(): number { + return this.store.size; + } + + update(key: string, value: number, now = Date.now()): RollingSnapshot { + this.prune(now); + + const existing = this.store.get(key); + const baseline = existing?.values ?? []; + const snapshot = computeSnapshot(baseline, value); + const nextValues = [value, ...baseline].slice(0, this.windowSize); + + this.store.set(key, { + values: nextValues, + updatedAt: now, + dirty: true + }); + + this.enforceMaxKeys(); + return snapshot; + } + + prune(now = Date.now()): number { + if (this.ttlMs <= 0) { + return 0; + } + + let removed = 0; + for (const [key, entry] of this.store) { + if (now - entry.updatedAt > this.ttlMs) { + this.store.delete(key); + removed += 1; + } + } + return removed; + } + + async hydrateFromRedis( + client: ReturnType, + keys: string[], + now = Date.now() + ): Promise { + for (const key of keys) { + const values = toNumbers(await client.lRange(key, 0, this.windowSize - 1)); + if (values.length === 0) { + continue; + } + this.store.set(key, { + values, + updatedAt: now, + dirty: false + }); + } + this.enforceMaxKeys(); + } + + async flushToRedis(client: ReturnType): Promise { + let flushed = 0; + for (const [key, entry] of this.store) { + if (!entry.dirty) { + continue; + } + + const multi = client.multi(); + multi.lTrim(key, 1, 0); + for (let idx = entry.values.length - 1; idx >= 0; idx -= 1) { + const value = entry.values[idx]; + if (typeof value === "number" && Number.isFinite(value)) { + multi.lPush(key, value.toString()); + } + } + if (this.config.ttlSeconds > 0) { + multi.expire(key, this.config.ttlSeconds); + } + await multi.exec(); + entry.dirty = false; + flushed += 1; + } + return flushed; + } + + private enforceMaxKeys(): void { + while (this.store.size > this.maxKeys) { + const oldestKey = getOldestKey(this.store); + if (!oldestKey) { + break; + } + this.store.delete(oldestKey); + } + } +} diff --git a/services/compute/tests/rolling-stats.test.ts b/services/compute/tests/rolling-stats.test.ts index 555d77c..aa9d738 100644 --- a/services/compute/tests/rolling-stats.test.ts +++ b/services/compute/tests/rolling-stats.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { computeSnapshot, computeStats } from "../src/rolling-stats"; +import { computeSnapshot, computeStats, RollingWindowStore } from "../src/rolling-stats"; describe("rolling stats helpers", () => { test("computeStats handles empty baseline", () => { @@ -21,4 +21,18 @@ describe("rolling stats helpers", () => { expect(snapshot.baselineCount).toBe(3); expect(snapshot.zscore).toBeCloseTo(1.84, 2); }); + + test("RollingWindowStore prunes stale keys by ttl", () => { + const store = new RollingWindowStore({ + windowSize: 3, + ttlSeconds: 1, + flushIntervalMs: 30_000, + maxKeys: 10 + }); + + store.update("rolling:premium:ABC", 10, 0); + expect(store.size).toBe(1); + expect(store.prune(1_500)).toBe(1); + expect(store.size).toBe(0); + }); }); diff --git a/services/ingest-equities/src/index.ts b/services/ingest-equities/src/index.ts index 3b77642..15dff9e 100644 --- a/services/ingest-equities/src/index.ts +++ b/services/ingest-equities/src/index.ts @@ -5,6 +5,7 @@ import { SUBJECT_EQUITY_QUOTES, STREAM_EQUITY_PRINTS, STREAM_EQUITY_QUOTES, + buildStreamConfig, connectJetStreamWithRetry, ensureStream, publishJson @@ -194,31 +195,8 @@ const run = async () => { { attempts: 120, delayMs: 500 } ); - await ensureStream(jsm, { - name: STREAM_EQUITY_PRINTS, - subjects: [SUBJECT_EQUITY_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_QUOTES, - subjects: [SUBJECT_EQUITY_QUOTES], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw")); const clickhouse = createClickHouseClient({ url: env.CLICKHOUSE_URL, @@ -251,11 +229,6 @@ const run = async () => { try { await insertEquityPrint(clickhouse, print); await publishJson(js, SUBJECT_EQUITY_PRINTS, print); - logger.info("published equity print", { - trace_id: print.trace_id, - seq: print.seq, - underlying_id: print.underlying_id - }); } catch (error) { if (isExpectedShutdownError(error)) { return; diff --git a/services/ingest-options/src/index.ts b/services/ingest-options/src/index.ts index bf50431..8e2bf41 100644 --- a/services/ingest-options/src/index.ts +++ b/services/ingest-options/src/index.ts @@ -9,6 +9,7 @@ import { STREAM_OPTION_NBBO, STREAM_OPTION_PRINTS, STREAM_OPTION_SIGNAL_PRINTS, + buildStreamConfig, buildDurableConsumer, connectJetStreamWithRetry, ensureStream, @@ -109,7 +110,9 @@ const envSchema = z.object({ return value; }, z.boolean()) .default(false), - TESTING_THROTTLE_MS: z.coerce.number().int().nonnegative().default(200) + TESTING_THROTTLE_MS: z.coerce.number().int().nonnegative().default(200), + OPTION_CONTEXT_MAX_KEYS: z.coerce.number().int().positive().default(20_000), + OPTION_CONTEXT_TTL_MS: z.coerce.number().int().positive().default(900_000) }); const env = readEnv(envSchema); @@ -143,6 +146,44 @@ const state = { const nbboHistoryByContract: ContextHistory = new Map(); const equityQuoteHistoryByUnderlying: ContextHistory = new Map(); +const OPTION_CONTEXT_PRUNE_INTERVAL_MS = 60_000; + +const pruneContextHistory = ( + history: ContextHistory, + maxKeys: number, + ttlMs: number, + now = Date.now() +): number => { + let removed = 0; + for (const [key, items] of history) { + const filtered = items.filter((item) => now - item.ts <= ttlMs); + if (filtered.length === 0) { + history.delete(key); + removed += 1; + continue; + } + if (filtered.length !== items.length) { + history.set(key, filtered); + } + } + + if (history.size <= maxKeys) { + return removed; + } + + const overflow = history.size - maxKeys; + const oldestKeys = [...history.entries()] + .map(([key, items]) => [key, items.at(-1)?.ts ?? Number.NEGATIVE_INFINITY] as const) + .sort((a, b) => a[1] - b[1]) + .slice(0, overflow); + + for (const [key] of oldestKeys) { + history.delete(key); + removed += 1; + } + + return removed; +}; const getErrorMessage = (error: unknown): string => { return error instanceof Error ? error.message : String(error); @@ -305,57 +346,10 @@ const run = async () => { { attempts: 120, delayMs: 500 } ); - await ensureStream(jsm, { - name: STREAM_OPTION_PRINTS, - subjects: [SUBJECT_OPTION_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_OPTION_NBBO, - subjects: [SUBJECT_OPTION_NBBO], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_OPTION_SIGNAL_PRINTS, - subjects: [SUBJECT_OPTION_SIGNAL_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_QUOTES, - subjects: [SUBJECT_EQUITY_QUOTES], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_PRINTS, SUBJECT_OPTION_PRINTS, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw")); const clickhouse = createClickHouseClient({ url: env.CLICKHOUSE_URL, @@ -400,14 +394,6 @@ const run = async () => { if (print.signal_pass) { await publishJson(js, SUBJECT_OPTION_SIGNAL_PRINTS, print); } - logger.info("published option print", { - trace_id: print.trace_id, - seq: print.seq, - option_contract_id: print.option_contract_id, - signal_pass: print.signal_pass, - nbbo_side: print.nbbo_side, - notional: print.notional - }); } catch (error) { if (isExpectedShutdownError(error)) { return; @@ -475,6 +461,18 @@ const run = async () => { } })(); + const pruneTimer = setInterval(() => { + const removed = + pruneContextHistory(nbboHistoryByContract, env.OPTION_CONTEXT_MAX_KEYS, env.OPTION_CONTEXT_TTL_MS) + + pruneContextHistory(equityQuoteHistoryByUnderlying, env.OPTION_CONTEXT_MAX_KEYS, env.OPTION_CONTEXT_TTL_MS); + logger.info("option context cache summary", { + nbbo_context_keys: nbboHistoryByContract.size, + equity_quote_context_keys: equityQuoteHistoryByUnderlying.size, + removed + }); + }, OPTION_CONTEXT_PRUNE_INTERVAL_MS); + pruneTimer.unref?.(); + const shutdown = async (signal: string) => { if (state.shutdownPromise) { return state.shutdownPromise; @@ -483,6 +481,7 @@ const run = async () => { state.shuttingDown = true; state.shutdownPromise = (async () => { logger.info("service stopping", { signal }); + clearInterval(pruneTimer); await stopAdapter(); try { diff --git a/services/replay/src/index.ts b/services/replay/src/index.ts index 1ba8342..21e4981 100644 --- a/services/replay/src/index.ts +++ b/services/replay/src/index.ts @@ -11,6 +11,7 @@ import { STREAM_OPTION_NBBO, STREAM_OPTION_PRINTS, STREAM_OPTION_SIGNAL_PRINTS, + buildStreamConfig, connectJetStreamWithRetry, ensureStream, publishJson @@ -180,19 +181,6 @@ const parseStreamList = (value: string): ReplayStreamKind[] => { return result; }; -const buildStreamConfig = (name: string, subject: string) => ({ - name, - subjects: [subject], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 -}); - const buildStartCursor = (startTs: number): ReplayCursor => { if (startTs <= 0) { return { ts: 0, seq: 0 }; @@ -304,10 +292,10 @@ const run = async () => { for (const kind of streamKinds) { const def = STREAM_DEFS[kind]; - await ensureStream(jsm, buildStreamConfig(def.streamName, def.subject)); + await ensureStream(jsm, buildStreamConfig(def.streamName, def.subject, "raw")); } if (streamKinds.includes("options")) { - await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS)); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived")); } const clickhouse = createClickHouseClient({