diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index a281161..704be02 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,10 @@ +{"_type":"issue","id":"islandflow-dil","title":"Run production baseline and post-rollout verification for load reduction","description":"Run the production verification checklist from the load-reduction plan on the VPS, capture baseline container/resource stats, validate replay remains disabled, and confirm JetStream/Redis behavior after rollout.\n\nThis follow-up is operational rather than code-local and could not be executed from the current workspace. It should compare pre/post CPU, RSS, Redis memory, and retention growth using the documented commands.\n","status":"open","priority":1,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:45:06Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:45:06Z","dependencies":[{"issue_id":"islandflow-dil","depends_on_id":"islandflow-1ln","type":"discovered-from","created_at":"2026-05-08T02:45:06Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-1ln","title":"Implement VPS load reduction plan","description":"Implement load-reduction plan across API, compute, logging, retention, and cache pruning.\n\nThis issue tracks the first-pass implementation of VPS load mitigations: lower live cache limits, async Redis write-behind in API live state, scoped cache eviction, reduced hot-path logging, bounded JetStream retention via shared config, in-memory rolling stats with async Redis snapshots, batched ClickHouse inserts for derived tables, and TTL/cardinality pruning for long-lived in-process maps.\n\nAcceptance:\n- Config surface for live limits, logging, rolling cache, and stream retention added\n- API live ingest avoids per-event full resort in monotonic case and avoids synchronous Redis writes per event\n- Compute rolling stats leave Redis hot path and derived ClickHouse writes batch\n- Long-lived caches/maps are pruned by TTL/cardinality\n- Tests cover monotonic/out-of-order live ingest, scoped eviction, rolling stats, and pruning behavior\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:27:41Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:46:23Z","started_at":"2026-05-08T06:27:54Z","closed_at":"2026-05-08T06:46:23Z","close_reason":"Implemented in code; rollout verification follow-up is islandflow-dil and Redis durability decision follow-up is islandflow-ybs","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-pre","title":"Fix contract-focused options tape hydration","description":"Implement contract-focused options tape hydration so focused contract views preserve the clicked seed row, stop reapplying broad flow filters in the Options pane, and use raw contract-scoped ClickHouse queries consistently across live snapshots, history, and replay. Includes frontend replay source-grouping changes and regression tests for focus seed durability, focused filtering, and contract-scoped API behavior.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T03:27:31Z","created_by":"dirtydishes","updated_at":"2026-05-08T03:37:18Z","started_at":"2026-05-08T03:27:35Z","closed_at":"2026-05-08T03:37:18Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-9xs","title":"Fix terminal hydration and virtual row measurement crash","description":"Fix client crash caused by options-support hydration on non-JSON/404 responses and satisfy tanstack virtual measured-row data-index requirement across virtualized tables.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T06:14:33Z","created_by":"dirtydishes","updated_at":"2026-05-07T06:17:09Z","started_at":"2026-05-07T06:14:43Z","closed_at":"2026-05-07T06:17:09Z","close_reason":"Completed: added data-index attributes on measured virtual rows, hardened options-support hydration error handling/content-type validation, and guarded trace-id hydration loops against malformed payload entries.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-35g","title":"Fix Docker deployment workspace lockfile drift","description":"Refresh deployment/docker workspace lockfile for Docker builds, add a drift guard for Docker-built workspaces, and document the separate deployment snapshot so frozen Bun installs cannot fail when repo dependencies change.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T06:02:06Z","created_by":"dirtydishes","updated_at":"2026-05-07T06:07:50Z","started_at":"2026-05-07T06:02:15Z","closed_at":"2026-05-07T06:07:50Z","close_reason":"Completed: synced deployment Docker workspace snapshot from repo root, refreshed deployment bun.lock, added sync/check scripts, and documented maintenance workflow. Local docker compose build validation is blocked here because Docker daemon is unavailable.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-2ij","title":"Harden tape virtualization, scoped focus, and live feed health","description":"Implement the coordinated tape stability plan across web and API.\n\nScope:\n- replace fixed-height tape virtualization with measured virtualization and virtual-end history loading\n- replace scrollHeight anchoring with key-based anchor restore\n- compose canonical tape lists across seed/live/history sources\n- preserve clicked contract/ticker context during scoped focus transitions\n- separate backend hot-channel health from scoped quiet empty states\n- shrink browser hot windows and modestly reduce server cache limits\n- add regression tests and development instrumentation\n\nAcceptance:\n- no giant blank spacer gaps during tape scrolling\n- scroll remains stable while live data and history mutate the list\n- clicked deep-history option/equity rows remain visible immediately after focus\n- narrow scopes do not surface Feed behind unless backend channel health is stale\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T05:35:18Z","created_by":"dirtydishes","updated_at":"2026-05-07T05:52:14Z","started_at":"2026-05-07T05:35:21Z","closed_at":"2026-05-07T05:52:14Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-uj7","title":"Fix home to tape navigation","description":"Home rail Tape navigation was not reliably switching to the tape route. Use browser-native top-level navigation for Home/Tape rail links so /tape remains reachable even if client router handling stalls.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T03:18:14Z","created_by":"dirtydishes","updated_at":"2026-05-07T03:18:21Z","started_at":"2026-05-07T03:18:20Z","closed_at":"2026-05-07T03:18:21Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-84s","title":"Implement seamless /tape live-to-history scroll gate","description":"Implement seamless live-to-ClickHouse scroll-gated history for /tape panes, including split live/history buffers in the web client, snapshot_limit support on live subscriptions, a bundled options support lookup endpoint, ClickHouse helpers for parity hydration, and test coverage for live head retention, background history loading, scoped options deep-hydration, and historical options decor restoration.\n","status":"in_progress","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T02:10:43Z","created_by":"dirtydishes","updated_at":"2026-05-07T02:10:47Z","started_at":"2026-05-07T02:10:47Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-sh1","title":"Fix live websocket stale lag and reconnect loop","description":"Investigate and fix API live consumer lag causing stale timestamps, feed-behind status, and reconnect loops. Optimize live cache persistence path, add lag telemetry/alerts, and validate in runtime.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T17:04:34Z","created_by":"dirtydishes","updated_at":"2026-05-04T17:09:44Z","started_at":"2026-05-04T17:04:38Z","closed_at":"2026-05-04T17:09:44Z","close_reason":"Completed: optimized live cache persistence path, added lag telemetry, deployed api via docker compose on di, verified ws freshness and low hotFeedLagMs","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-b3o","title":"Implement options tape table with execution spot","description":"Redesign OptionsPane into a dense classifier-colored table and preserve execution-time underlying spot on option prints from equity quote mid.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:41:59Z","created_by":"dirtydishes","updated_at":"2026-05-04T05:14:26Z","started_at":"2026-05-04T04:42:08Z","closed_at":"2026-05-04T05:14:26Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -5,6 +12,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-ybs","title":"Decide Redis AOF and cache/durable split after load rollout","description":"Decide whether the deployment Redis should keep AOF enabled or be split into cache vs durable roles after the first rollout data is available.\n\nThe current code changes reduce cache churn, but the operational durability/caching tradeoff still needs a production decision.\n","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:45:05Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:45:05Z","dependencies":[{"issue_id":"islandflow-ybs","depends_on_id":"islandflow-1ln","type":"discovered-from","created_at":"2026-05-08T02:45:04Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-vbk","title":"Remove deprecated Alpaca key-pair auth","description":"Remove legacy Alpaca key-pair authentication support and keep ALPACA_API_KEY as the only supported auth method across options/equities ingest and docs.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:19:51Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:21:10Z","started_at":"2026-05-05T07:19:54Z","closed_at":"2026-05-05T07:21:10Z","close_reason":"Removed key-pair auth and kept ALPACA_API_KEY only","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-h47","title":"Support single-token Alpaca auth","description":"Support single-token Alpaca authentication across ingest adapters using ALPACA_API_KEY with fallback to ALPACA_KEY_ID/ALPACA_SECRET_KEY, and document env usage.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:12:22Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:13:54Z","started_at":"2026-05-05T07:12:25Z","closed_at":"2026-05-05T07:13:54Z","close_reason":"Added ALPACA_API_KEY support with key-pair fallback","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-neu","title":"Add Alpha Vantage event calendar provider","description":"Add an Alpha Vantage earnings-calendar provider to services/refdata that fetches CSV, normalizes entries, writes the JSON cache consumed by compute, and documents the required env variables.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:00:31Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:02:30Z","started_at":"2026-05-05T07:00:37Z","closed_at":"2026-05-05T07:02:30Z","close_reason":"Added Alpha Vantage event-calendar provider","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/.env.example b/.env.example index 50f9c5a..5442eac 100644 --- a/.env.example +++ b/.env.example @@ -58,8 +58,8 @@ API_DELIVER_POLICY=new API_CONSUMER_RESET=false NBBO_MAX_AGE_MS=1000 NEXT_PUBLIC_NBBO_MAX_AGE_MS=1000 -NEXT_PUBLIC_LIVE_HOT_WINDOW=2000 -NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS=25000 +NEXT_PUBLIC_LIVE_HOT_WINDOW=600 +NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS=1200 NEXT_PUBLIC_PINNED_EVIDENCE_TTL_MS=1200000 NEXT_PUBLIC_PINNED_EVIDENCE_MAX_ITEMS=4000 ROLLING_WINDOW_SIZE=50 @@ -91,6 +91,7 @@ ALPHA_VANTAGE_EARNINGS_SYMBOL= REFDATA_EVENT_CALENDAR_REFRESH_MS=86400000 # Replay service +LOG_LEVEL=info REPLAY_ENABLED=false REPLAY_STREAMS=options,nbbo,equities,equity-quotes REPLAY_START_TS=0 @@ -100,12 +101,33 @@ REPLAY_BATCH_SIZE=200 REPLAY_LOG_EVERY=1000 # API live retention (generic channels) -LIVE_LIMIT_OPTIONS=10000 -LIVE_LIMIT_NBBO=10000 -LIVE_LIMIT_EQUITIES=10000 -LIVE_LIMIT_EQUITY_QUOTES=10000 -LIVE_LIMIT_EQUITY_JOINS=10000 -LIVE_LIMIT_FLOW=10000 -LIVE_LIMIT_CLASSIFIER_HITS=10000 -LIVE_LIMIT_ALERTS=10000 -LIVE_LIMIT_INFERRED_DARK=10000 +LIVE_LIMIT_DEFAULT=1000 +LIVE_LIMIT_OPTIONS=1000 +LIVE_LIMIT_NBBO=1000 +LIVE_LIMIT_EQUITIES=1000 +LIVE_LIMIT_EQUITY_QUOTES=500 +LIVE_LIMIT_EQUITY_JOINS=500 +LIVE_LIMIT_FLOW=500 +LIVE_LIMIT_SMART_MONEY=300 +LIVE_LIMIT_CLASSIFIER_HITS=300 +LIVE_LIMIT_ALERTS=300 +LIVE_LIMIT_INFERRED_DARK=300 +LIVE_SCOPED_CACHE_MAX_KEYS=32 +LIVE_REDIS_FLUSH_INTERVAL_MS=250 +LIVE_REDIS_FLUSH_MAX_ITEMS=100 + +# Compute rolling/cache retention +ROLLING_CACHE_FLUSH_INTERVAL_MS=30000 +ROLLING_CACHE_MAX_KEYS=20000 +COMPUTE_NBBO_CACHE_MAX_KEYS=20000 +COMPUTE_NBBO_CACHE_TTL_MS=900000 + +# Ingest context retention +OPTION_CONTEXT_MAX_KEYS=20000 +OPTION_CONTEXT_TTL_MS=900000 + +# JetStream retention +STREAM_RAW_MAX_AGE_MS=7200000 +STREAM_RAW_MAX_BYTES=1073741824 +STREAM_DERIVED_MAX_AGE_MS=86400000 +STREAM_DERIVED_MAX_BYTES=536870912 diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index 5af91c1..8cf07a3 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -954,19 +954,37 @@ h3 { } .data-table-wrap { + display: flex; flex: 1 1 auto; min-height: 0; - overflow: auto; + overflow-x: auto; + overflow-y: hidden; border-top: 1px solid var(--border); border-bottom: 1px solid var(--border); background: rgba(5, 8, 12, 0.42); } .data-table { - display: block; + display: flex; + flex: 1 1 auto; + flex-direction: column; + height: 100%; + min-height: 0; min-width: 980px; } +.data-table-scroll { + flex: 1 1 auto; + min-height: 0; + overflow-y: auto; + overflow-x: hidden; +} + +.data-table-body { + position: relative; + min-width: 100%; +} + .data-table-options { min-width: 1280px; } @@ -999,10 +1017,8 @@ h3 { } .data-table-head { - position: sticky; - top: 0; - z-index: 2; - min-height: 30px; + flex: 0 0 auto; + height: 30px; padding: 0 10px; border-bottom: 1px solid rgba(255, 255, 255, 0.095); background: rgba(8, 11, 16, 0.98); @@ -1014,7 +1030,7 @@ h3 { .data-table-row { width: 100%; - min-height: 40px; + height: 40px; padding: 0 10px; border: 0; border-bottom: 1px solid rgba(255, 255, 255, 0.055); @@ -1024,10 +1040,17 @@ h3 { text-align: left; } -.data-table-row:nth-child(even) { +.data-table-row.is-even { background: rgba(255, 255, 255, 0.022); } +.data-table-virtual-row { + position: absolute; + top: 0; + left: 0; + width: 100%; +} + .data-table-row:hover, .data-table-row:focus-visible { outline: none; @@ -1039,18 +1062,18 @@ h3 { } .data-table-row-options { - min-height: 36px; + height: 36px; } .data-table-row-equities { - min-height: 34px; + height: 36px; } .data-table-row-flow, .data-table-row-alerts, .data-table-row-classifier, .data-table-row-dark { - min-height: 44px; + height: 44px; } .data-table-row-classified { diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index 2071762..2ada99a 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -1,24 +1,40 @@ import { describe, expect, it } from "bun:test"; +import { getSubscriptionKey as getLiveSubscriptionKey } from "@islandflow/types"; import { NAV_ITEMS, + appendHistoryTail, buildDefaultFlowFilters, + buildOptionTapeQueryParams, classifierToneForFamily, + composeTapeItems, deriveAlertDirection, countActiveFlowFilterGroups, + filterOptionTapeItems, + findAnchorRestoreIndex, formatCompactUsd, formatOptionContractLabel, flushPausableTapeData, + getEffectiveOptionPrintFilters, getAlertWindowAnchorTs, + getHotChannelFeedStatus, + getScopedLiveAutoHydrationChannels, + getLiveHistoryRetentionCap, getOptionTableSnapshot, + getOptionScope, getLiveFeedStatus, getLiveManifest, + getRouteFeatures, + getTapeVirtualConfig, + mergeNewestWithOverflow, normalizeAlertSeverity, nextFlowFilterPopoverState, projectPausableTapeState, reducePausableTapeData, shouldRetainLiveSnapshotHistory, + shouldIncludeEquitiesForDarkUnderlyingFallback, shouldShowEquitiesSilentFeedWarning, selectPrimaryClassifierHit, + shouldClearOptionFocusSeed, smartMoneyProfileLabel, smartMoneyToneForProfile, statusLabel, @@ -31,6 +47,25 @@ const makeItem = (traceId: string, seq: number, ts: number) => ({ ts }); +const makeOptionPrint = (overrides: Record = {}) => + ({ + trace_id: "opt-1", + seq: 1, + ts: 1_000, + source_ts: 1_000, + ingest_ts: 1_001, + option_contract_id: "AAPL-2025-01-17-200-C", + underlying_id: "AAPL", + option_type: "call", + nbbo_side: "A", + notional: 250_000, + signal_pass: true, + price: 1, + size: 10, + exchange: "X", + ...overrides + }) as any; + const makeAlert = (overrides: Record = {}) => ({ trace_id: "alert-1", @@ -43,15 +78,13 @@ const makeAlert = (overrides: Record = {}) => }) as any; describe("live manifest", () => { - it("includes options on home and tape", () => { + it("includes only tape channels on /tape", () => { const filters = buildDefaultFlowFilters(); - for (const pathname of ["/", "/tape"]) { - expect( - getLiveManifest(pathname, "SPY", 60000, filters).some( - (subscription) => subscription.channel === "options" - ) - ).toBe(true); - } + const channels = getLiveManifest("/tape", "SPY", 60000, filters).map( + (subscription) => subscription.channel + ); + + expect(channels).toEqual(["options", "nbbo", "equities", "flow"]); }); it("dedupes tape options subscription", () => { @@ -64,37 +97,29 @@ describe("live manifest", () => { expect(tapeOptionsSubscriptions).toHaveLength(1); }); - it("keeps option filters on baseline subscription across page changes", () => { + it("keeps option filters on /tape options subscriptions", () => { const filters = { ...buildDefaultFlowFilters(), minNotional: 125_000 }; - const homeOptionsSubscription = getLiveManifest("/", "SPY", 60000, filters).find( - (subscription) => subscription.channel === "options" - ); const tapeOptionsSubscription = getLiveManifest("/tape", "SPY", 60000, filters).find( (subscription) => subscription.channel === "options" ); - expect(homeOptionsSubscription?.filters).toBe(filters); expect(tapeOptionsSubscription?.filters).toBe(filters); }); - it("applies global flow filters to flow subscriptions on home and tape", () => { + it("applies global flow filters to flow subscriptions on /tape", () => { const filters = { ...buildDefaultFlowFilters(), minNotional: 50_000 }; - const homeFlowSubscription = getLiveManifest("/", "SPY", 60000, filters).find( - (subscription) => subscription.channel === "flow" - ); const tapeFlowSubscription = getLiveManifest("/tape", "SPY", 60000, filters).find( (subscription) => subscription.channel === "flow" ); - expect(homeFlowSubscription?.filters).toBe(filters); expect(tapeFlowSubscription?.filters).toBe(filters); }); @@ -123,6 +148,239 @@ describe("live manifest", () => { expect(optionsSubscription?.option_contract_id).toBe("AAPL-2025-01-17-200-C"); expect(equitiesSubscription?.underlying_ids).toEqual(["AAPL"]); }); + + it("drops option-print filters for contract-focused options subscriptions but keeps flow filters", () => { + const filters = { + ...buildDefaultFlowFilters(), + minNotional: 500_000, + optionTypes: ["put"] as const + }; + const manifest = getLiveManifest( + "/tape", + "AAPL", + 60000, + filters, + { + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }, + { underlying_ids: ["AAPL"] }, + undefined + ); + const optionsSubscription = manifest.find((subscription) => subscription.channel === "options"); + const flowSubscription = manifest.find((subscription) => subscription.channel === "flow"); + + expect(optionsSubscription?.filters).toBeUndefined(); + expect(flowSubscription?.filters).toBe(filters); + }); + + it("scopes /signals subscriptions to signals channels only", () => { + const channels = getLiveManifest("/signals", "SPY", 60000, buildDefaultFlowFilters()).map( + (subscription) => subscription.channel + ); + + expect(channels).toEqual([ + "alerts", + "smart-money", + "classifier-hits", + "inferred-dark", + "equity-joins" + ]); + }); + + it("scopes /charts subscriptions to chart channels only", () => { + const channels = getLiveManifest("/charts", "SPY", 60000, buildDefaultFlowFilters()).map( + (subscription) => subscription.channel + ); + + expect(channels).toEqual([ + "smart-money", + "inferred-dark", + "equity-joins", + "equity-candles", + "equity-overlay" + ]); + }); +}); + +describe("contract-focused option helpers", () => { + it("uses the focused contract underlying for option scope even when ticker input differs", () => { + expect( + getOptionScope(["MSFT"], "AAPL", { + kind: "option-contract", + contractId: "AAPL-2025-01-17-200-C", + underlyingId: "AAPL" + }) + ).toEqual({ + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }); + }); + + it("ignores broad flow filters for focused contract options", () => { + const filters = { + ...buildDefaultFlowFilters(), + minNotional: 500_000 + }; + const items = [ + makeOptionPrint({ + trace_id: "focused-low", + option_contract_id: "AAPL-2025-01-17-200-C", + notional: 100_000, + signal_pass: false + }), + makeOptionPrint({ + trace_id: "focused-high", + seq: 2, + ts: 2_000, + option_contract_id: "AAPL-2025-01-17-200-C", + notional: 750_000 + }), + makeOptionPrint({ + trace_id: "other-contract", + seq: 3, + ts: 3_000, + option_contract_id: "MSFT-2025-01-17-300-C", + underlying_id: "MSFT", + notional: 900_000 + }) + ]; + + expect( + filterOptionTapeItems( + items, + getEffectiveOptionPrintFilters(filters, true), + { + kind: "option-contract", + contractId: "AAPL-2025-01-17-200-C", + underlyingId: "AAPL" + }, + new Set(["MSFT"]), + "AAPL" + ).map((item) => item.trace_id) + ).toEqual(["focused-low", "focused-high"]); + }); + + it("includes option_contract_id and drops broad filters in focused replay query params", () => { + const filters = { + ...buildDefaultFlowFilters(), + minNotional: 500_000, + optionTypes: ["put"] as const + }; + + expect( + buildOptionTapeQueryParams(getEffectiveOptionPrintFilters(filters, true), { + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }) + ).toEqual({ + underlying_ids: "AAPL", + option_contract_id: "AAPL-2025-01-17-200-C" + }); + }); + + it("keeps the focus seed until the matching scoped subscription has loaded it", () => { + const seedItem = makeOptionPrint({ + trace_id: "focused-seed", + option_contract_id: "AAPL-2025-01-17-200-C" + }); + const seed = { + scopeKey: "option-contract:AAPL-2025-01-17-200-C", + subscriptionKey: getLiveSubscriptionKey({ + channel: "options", + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }), + items: [seedItem] + }; + + expect( + shouldClearOptionFocusSeed( + seed, + "option-contract:AAPL-2025-01-17-200-C", + getLiveSubscriptionKey({ + channel: "options", + filters: { + ...buildDefaultFlowFilters(), + minNotional: 500_000 + }, + underlying_ids: ["AAPL"] + }), + [makeOptionPrint({ trace_id: "broad-old" })], + [] + ) + ).toBe(false); + + expect( + shouldClearOptionFocusSeed( + seed, + "option-contract:AAPL-2025-01-17-200-C", + getLiveSubscriptionKey({ + channel: "options", + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }), + [seedItem], + [] + ) + ).toBe(true); + }); +}); + +describe("route feature map", () => { + it("maps /tape to tape panes and dependencies", () => { + const features = getRouteFeatures("/tape"); + expect(features.showOptionsPane).toBe(true); + expect(features.showEquitiesPane).toBe(true); + expect(features.showFlowPane).toBe(true); + expect(features.needsClassifierDecor).toBe(true); + expect(features.alerts).toBe(false); + }); + + it("maps /signals to signal panes and dependencies", () => { + const features = getRouteFeatures("/signals"); + expect(features.showAlertsPane).toBe(true); + expect(features.showClassifierPane).toBe(true); + expect(features.showDarkPane).toBe(true); + expect(features.options).toBe(false); + expect(features.equityJoins).toBe(true); + }); + + it("maps /charts to chart panes and dependencies", () => { + const features = getRouteFeatures("/charts"); + expect(features.showChartPane).toBe(true); + expect(features.showFocusPane).toBe(true); + expect(features.equityCandles).toBe(true); + expect(features.equityOverlay).toBe(true); + expect(features.alerts).toBe(false); + }); +}); + +describe("fixed tape virtualization config", () => { + it("uses expected fixed row heights and overscan by table", () => { + expect(getTapeVirtualConfig("options")).toEqual({ rowHeight: 36, overscan: 24, debugLabel: "options" }); + expect(getTapeVirtualConfig("equities")).toEqual({ rowHeight: 36, overscan: 20, debugLabel: "equities" }); + expect(getTapeVirtualConfig("flow")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "flow" }); + expect(getTapeVirtualConfig("alerts")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "alerts" }); + expect(getTapeVirtualConfig("classifier")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "classifier" }); + expect(getTapeVirtualConfig("dark")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "dark" }); + }); +}); + +describe("dark underlying route dependency helper", () => { + it("does not keep extra equities subscriptions when joins+trace fallback are sufficient", () => { + expect(shouldIncludeEquitiesForDarkUnderlyingFallback()).toBe(false); + expect( + getLiveManifest("/signals", "SPY", 60000, buildDefaultFlowFilters()).some( + (subscription) => subscription.channel === "equities" + ) + ).toBe(false); + expect( + getLiveManifest("/charts", "SPY", 60000, buildDefaultFlowFilters()).some( + (subscription) => subscription.channel === "equities" + ) + ).toBe(false); + }); }); describe("terminal navigation", () => { @@ -240,6 +498,171 @@ describe("live tape pausable helpers", () => { }); }); +describe("live tape history helpers", () => { + it("composes tape items across seed, live, and history without seam duplicates", () => { + const seed = [makeItem("seed", 1, 100), makeItem("dup", 2, 200)]; + const live = [makeItem("live", 5, 500), makeItem("dup", 2, 200)]; + const history = [makeItem("old", 0, 50), makeItem("mid", 3, 300)]; + + expect(composeTapeItems(seed, live, history).map((item) => item.trace_id)).toEqual([ + "live", + "mid", + "dup", + "seed", + "old" + ]); + }); + + it("keeps a clicked seed row visible before scoped live and history arrive", () => { + const clicked = makeItem("clicked", 3, 300); + + expect(composeTapeItems([clicked], [], []).map((item) => item.trace_id)).toEqual(["clicked"]); + }); + + it("drops focus seed duplicates once equivalent live or history rows arrive", () => { + const clicked = makeItem("clicked", 3, 300); + const live = [makeItem("new", 4, 400)]; + const history = [makeItem("clicked", 3, 300)]; + + expect(composeTapeItems([clicked], live, history).map((item) => item.trace_id)).toEqual([ + "new", + "clicked" + ]); + }); + + it("promotes hot-window overflow into the history tail", () => { + const currentHot = [makeItem("hot-3", 3, 300), makeItem("hot-2", 2, 200), makeItem("hot-1", 1, 100)]; + const incoming = [makeItem("hot-4", 4, 400)]; + + const { kept, evicted } = mergeNewestWithOverflow(incoming, currentHot, 3); + const nextHistory = appendHistoryTail([], evicted, kept, 5000); + + expect(kept.map((item) => item.trace_id)).toEqual(["hot-4", "hot-3", "hot-2"]); + expect(nextHistory.map((item) => item.trace_id)).toEqual(["hot-1"]); + }); + + it("keeps the combined tape continuous beyond the hot live window", () => { + let hot: Array> = []; + let history: Array> = []; + + for (let seq = 1; seq <= 5; seq += 1) { + const { kept, evicted } = mergeNewestWithOverflow([makeItem(`row-${seq}`, seq, seq * 100)], hot, 2); + hot = kept; + history = appendHistoryTail(history, evicted, hot, 5000); + } + + expect([...hot, ...history].map((item) => item.trace_id)).toEqual([ + "row-5", + "row-4", + "row-3", + "row-2", + "row-1" + ]); + }); + + it("appends older scoped rows behind the hot live head", () => { + const liveHead = Array.from({ length: 100 }, (_, idx) => + makeItem(`hot-${idx}`, 200 - idx, 2_000 - idx) + ); + const older = [makeItem("older-1", 99, 999), makeItem("older-2", 98, 998)]; + + const next = appendHistoryTail([], older, liveHead, 5000); + + expect(next.map((item) => item.trace_id)).toEqual(["older-1", "older-2"]); + }); + + it("skips duplicates already present in the live head", () => { + const liveHead = [makeItem("latest", 3, 300), makeItem("duplicate", 2, 200)]; + const older = [makeItem("duplicate", 2, 200), makeItem("older", 1, 100)]; + + const next = appendHistoryTail([], older, liveHead, 5000); + + expect(next.map((item) => item.trace_id)).toEqual(["older"]); + }); + + it("dedupes the seam between promoted overflow and fetched history", () => { + const currentHot = [makeItem("hot-3", 3, 300), makeItem("hot-2", 2, 200), makeItem("hot-1", 1, 100)]; + const { kept, evicted } = mergeNewestWithOverflow([makeItem("hot-4", 4, 400)], currentHot, 3); + const promoted = appendHistoryTail([], evicted, kept, 5000); + const merged = appendHistoryTail(promoted, [makeItem("hot-1", 1, 100), makeItem("older", 0, 50)], kept, 5000); + + expect(merged.map((item) => item.trace_id)).toEqual(["hot-1", "older"]); + expect(new Set([...kept, ...merged].map((item) => item.trace_id)).size).toBe(kept.length + merged.length); + }); + + it("trims the history tail to the soft cap", () => { + const current = [makeItem("existing", 4, 400)]; + const older = [makeItem("older-1", 3, 300), makeItem("older-2", 2, 200)]; + + const next = appendHistoryTail(current, older, [], 2); + + expect(next.map((item) => item.trace_id)).toEqual(["existing", "older-1"]); + }); + + it("keeps scoped option and equity history on the normal retention cap", () => { + expect( + getLiveHistoryRetentionCap({ + channel: "options", + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + } as any) + ).toBeGreaterThan(0); + expect( + getLiveHistoryRetentionCap({ + channel: "equities", + underlying_ids: ["AAPL"] + } as any) + ).toBeGreaterThan(0); + }); + + it("keeps auto-hydrating scoped live history while next_before exists", () => { + const manifest = getLiveManifest( + "/tape", + "AAPL", + 60000, + buildDefaultFlowFilters(), + { + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }, + { underlying_ids: ["AAPL"] } + ); + const historyCursors = Object.fromEntries( + manifest.map((subscription) => [getLiveSubscriptionKey(subscription), { ts: 1, seq: 1 }]) + ); + + expect( + getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {}) + ).toEqual(["options", "equities"]); + expect( + getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, { + [getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "options")!)]: true + }) + ).toEqual(["equities"]); + expect( + getScopedLiveAutoHydrationChannels(true, "/tape", manifest, { + ...historyCursors, + [getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "equities")!)]: null + }, {}) + ).toEqual(["options"]); + }); + + it("restores the same anchor key after live insertions at the top", () => { + const nextKeys = ["new-1", "new-2", "anchor", "after-1", "after-2"]; + expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(2); + }); + + it("falls forward to the nearest surviving key when the anchor is evicted", () => { + const nextKeys = ["new-1", "after-1", "after-2"]; + expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(1); + }); + + it("keeps the same anchor when history is appended at the bottom", () => { + const nextKeys = ["anchor", "after-1", "after-2", "older-1", "older-2"]; + expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(0); + }); +}); + describe("options display formatters", () => { it("formats dashed option contracts as ticker strike expiry", () => { expect(formatOptionContractLabel("SPY-2025-01-17-450-C")).toEqual({ @@ -409,4 +832,13 @@ describe("signals helpers", () => { expect(statusLabel("connected", false, "live")).toBe("Connected"); expect(statusLabel("stale", false, "live")).toBe("Feed behind"); }); + + it("treats healthy scoped channels as connected even when no matching rows are visible", () => { + expect(getHotChannelFeedStatus("connected", { healthy: true })).toBe("connected"); + }); + + it("surfaces feed behind only when the backend channel health is stale", () => { + expect(getHotChannelFeedStatus("connected", { healthy: false })).toBe("stale"); + expect(getHotChannelFeedStatus("disconnected", { healthy: true })).toBe("disconnected"); + }); }); diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index e4c67a6..854ea85 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -4,6 +4,7 @@ import Link from "next/link"; import { usePathname } from "next/navigation"; import { createContext, + memo, useCallback, useContext, useEffect, @@ -17,6 +18,7 @@ import { type ReactNode, type SetStateAction } from "react"; +import { useVirtualizer } from "@tanstack/react-virtual"; import type { AlertEvent, ClassifierHitEvent, @@ -28,6 +30,7 @@ import type { FlowPacket, InferredDarkEvent, LiveServerMessage, + LiveHotChannelHealthMap, LiveSubscription, OptionFlowFilters, OptionNbboSide, @@ -62,10 +65,10 @@ const parseBoundedInt = ( return Math.max(min, Math.min(max, Math.floor(parsed))); }; -const LIVE_HOT_WINDOW = parseBoundedInt(process.env.NEXT_PUBLIC_LIVE_HOT_WINDOW, 100, 1, 100000); +const LIVE_HOT_WINDOW = parseBoundedInt(process.env.NEXT_PUBLIC_LIVE_HOT_WINDOW, 600, 1, 100000); const LIVE_HOT_WINDOW_OPTIONS = parseBoundedInt( process.env.NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS, - 100, + 1200, 1, 100000 ); @@ -123,6 +126,206 @@ const LIVE_SESSION_HOT_CHANNELS = new Set([ "equity-overlay" ]); +type TapeVirtualPane = "options" | "equities" | "flow" | "alerts" | "classifier" | "dark"; + +type TapeVirtualListConfig = { + rowHeight: number; + overscan: number; + debugLabel: TapeVirtualPane; +}; + +const TAPE_VIRTUAL_CONFIG: Record = { + options: { rowHeight: 36, overscan: 24, debugLabel: "options" }, + equities: { rowHeight: 36, overscan: 20, debugLabel: "equities" }, + flow: { rowHeight: 44, overscan: 16, debugLabel: "flow" }, + alerts: { rowHeight: 44, overscan: 16, debugLabel: "alerts" }, + classifier: { rowHeight: 44, overscan: 16, debugLabel: "classifier" }, + dark: { rowHeight: 44, overscan: 16, debugLabel: "dark" } +}; + +export const getTapeVirtualConfig = (pane: TapeVirtualPane): TapeVirtualListConfig => + TAPE_VIRTUAL_CONFIG[pane]; + +type RouteFeatures = { + options: boolean; + nbbo: boolean; + equities: boolean; + flow: boolean; + alerts: boolean; + smartMoney: boolean; + classifierHits: boolean; + inferredDark: boolean; + equityJoins: boolean; + equityCandles: boolean; + equityOverlay: boolean; + showOptionsPane: boolean; + showEquitiesPane: boolean; + showFlowPane: boolean; + showAlertsPane: boolean; + showClassifierPane: boolean; + showDarkPane: boolean; + showChartPane: boolean; + showFocusPane: boolean; + showReplayConsole: boolean; + needsClassifierDecor: boolean; + needsAlertEvidencePrefetch: boolean; + needsDarkUnderlying: boolean; +}; + +export const shouldIncludeEquitiesForDarkUnderlyingFallback = (): boolean => { + return false; +}; + +export const getRouteFeatures = (pathname: string): RouteFeatures => { + const includeEquitiesFallback = shouldIncludeEquitiesForDarkUnderlyingFallback(); + const normalizedPath = + pathname === "/tape" || + pathname === "/signals" || + pathname === "/charts" || + pathname === "/replay" + ? pathname + : "/"; + + switch (normalizedPath) { + case "/tape": + return { + options: true, + nbbo: true, + equities: true, + flow: true, + alerts: false, + smartMoney: false, + classifierHits: false, + inferredDark: false, + equityJoins: false, + equityCandles: false, + equityOverlay: false, + showOptionsPane: true, + showEquitiesPane: true, + showFlowPane: true, + showAlertsPane: false, + showClassifierPane: false, + showDarkPane: false, + showChartPane: false, + showFocusPane: false, + showReplayConsole: false, + needsClassifierDecor: true, + needsAlertEvidencePrefetch: false, + needsDarkUnderlying: false + }; + case "/signals": + return { + options: false, + nbbo: false, + equities: includeEquitiesFallback, + flow: false, + alerts: true, + smartMoney: true, + classifierHits: true, + inferredDark: true, + equityJoins: true, + equityCandles: false, + equityOverlay: false, + showOptionsPane: false, + showEquitiesPane: false, + showFlowPane: false, + showAlertsPane: true, + showClassifierPane: true, + showDarkPane: true, + showChartPane: false, + showFocusPane: false, + showReplayConsole: false, + needsClassifierDecor: false, + needsAlertEvidencePrefetch: true, + needsDarkUnderlying: true + }; + case "/charts": + return { + options: false, + nbbo: false, + equities: includeEquitiesFallback, + flow: false, + alerts: false, + smartMoney: true, + classifierHits: false, + inferredDark: true, + equityJoins: true, + equityCandles: true, + equityOverlay: true, + showOptionsPane: false, + showEquitiesPane: false, + showFlowPane: false, + showAlertsPane: false, + showClassifierPane: false, + showDarkPane: false, + showChartPane: true, + showFocusPane: true, + showReplayConsole: false, + needsClassifierDecor: false, + needsAlertEvidencePrefetch: false, + needsDarkUnderlying: true + }; + case "/replay": + return { + options: false, + nbbo: false, + equities: false, + flow: false, + alerts: false, + smartMoney: false, + classifierHits: false, + inferredDark: false, + equityJoins: false, + equityCandles: false, + equityOverlay: false, + showOptionsPane: true, + showEquitiesPane: false, + showFlowPane: true, + showAlertsPane: true, + showClassifierPane: false, + showDarkPane: false, + showChartPane: false, + showFocusPane: false, + showReplayConsole: true, + needsClassifierDecor: true, + needsAlertEvidencePrefetch: true, + needsDarkUnderlying: false + }; + case "/": + default: + return { + options: false, + nbbo: false, + equities: true, + flow: false, + alerts: true, + smartMoney: true, + classifierHits: false, + inferredDark: true, + equityJoins: true, + equityCandles: true, + equityOverlay: true, + showOptionsPane: false, + showEquitiesPane: true, + showFlowPane: false, + showAlertsPane: true, + showClassifierPane: false, + showDarkPane: false, + showChartPane: true, + showFocusPane: false, + showReplayConsole: false, + needsClassifierDecor: false, + needsAlertEvidencePrefetch: true, + needsDarkUnderlying: true + }; + } +}; + +const EMPTY_ALERT_EVENTS: AlertEvent[] = []; +const EMPTY_CLASSIFIER_HIT_EVENTS: ClassifierHitEvent[] = []; +const EMPTY_SMART_MONEY_EVENTS: SmartMoneyEvent[] = []; +const EMPTY_INFERRED_DARK_EVENTS: InferredDarkEvent[] = []; + type CandlestickSeries = ReturnType; type EquityOverlayPoint = { @@ -145,6 +348,19 @@ type SelectedInstrument = | { kind: "equity"; underlyingId: string } | { kind: "option-contract"; contractId: string; underlyingId: string }; +type TapeFocusSeed = { + scopeKey: string; + subscriptionKey?: string; + items: T[]; +}; + +type OptionScope = Pick< + Extract, + "underlying_ids" | "option_contract_id" +>; + +type EquityScope = Pick, "underlying_ids">; + const formatIntervalLabel = (intervalMs: number): string => { const match = CANDLE_INTERVALS.find((interval) => interval.ms === intervalMs); if (match) { @@ -227,19 +443,31 @@ const sampleToLimit = (items: T[], limit: number): T[] => { }; const readErrorDetail = async (response: Response): Promise => { + const statusLabel = `HTTP ${response.status}${response.statusText ? ` ${response.statusText}` : ""}`; const text = await response.text(); if (!text) { - return ""; + return statusLabel; } + const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""; + const trimmed = text.trimStart(); + const truncated = text.length > 600 ? `${text.slice(0, 600)}...` : text; + + if (!contentType.includes("application/json")) { + if (/^ = { pinnedStoreSize: 0 }; +const DEV_TAPE_DEBUG = process.env.NODE_ENV !== "production"; + +type TapeDebugMetricKey = + | "anchorRestoreCount" + | "anchorRestoreFallbackCount" + | "virtualRowMeasurementCount" + | "focusSeedRowCount" + | "scopedQuietTransitions"; + +const frontendTapeDebugMetrics: Record = { + anchorRestoreCount: 0, + anchorRestoreFallbackCount: 0, + virtualRowMeasurementCount: 0, + focusSeedRowCount: 0, + scopedQuietTransitions: 0 +}; + +const bumpTapeDebugMetric = (key: TapeDebugMetricKey, count = 1): void => { + frontendTapeDebugMetrics[key] += count; + if (DEV_TAPE_DEBUG && typeof window !== "undefined") { + (window as typeof window & { __IF_TAPE_DEBUG__?: Record }).__IF_TAPE_DEBUG__ = + frontendTapeDebugMetrics; + } +}; + +const logTapeDebug = (message: string, payload?: Record): void => { + if (!DEV_TAPE_DEBUG) { + return; + } + if (payload) { + console.debug(`[tape] ${message}`, payload); + return; + } + console.debug(`[tape] ${message}`); +}; + const incrementRetentionMetric = (key: RetentionMetricKey, count = 1): void => { frontendRetentionMetrics[key] += count; }; @@ -368,15 +632,15 @@ const buildItemKey = (item: SortableItem): string | null => { return null; }; -const mergeNewest = ( +export const mergeNewestWithOverflow = ( incoming: T[], existing: T[], limit = LIVE_HOT_WINDOW, onTrim?: (evicted: number) => void -): T[] => { +): { kept: T[]; evicted: T[] } => { const combined = [...incoming, ...existing]; if (combined.length === 0) { - return combined; + return { kept: combined, evicted: [] }; } const seen = new Set(); @@ -402,18 +666,48 @@ const mergeNewest = ( }); const safeLimit = Math.max(1, Math.floor(limit)); - const evicted = Math.max(0, deduped.length - safeLimit); - if (evicted > 0) { - onTrim?.(evicted); + const evicted = deduped.slice(safeLimit); + if (evicted.length > 0) { + onTrim?.(evicted.length); } - return deduped.slice(0, safeLimit); + return { + kept: deduped.slice(0, safeLimit), + evicted + }; +}; + +const mergeNewest = ( + incoming: T[], + existing: T[], + limit = LIVE_HOT_WINDOW, + onTrim?: (evicted: number) => void +): T[] => { + return mergeNewestWithOverflow(incoming, existing, limit, onTrim).kept; }; const getTapeItemKey = (item: SortableItem): string => { return buildItemKey(item) ?? `${extractSortTs(item)}:${extractSortSeq(item)}`; }; +export const composeTapeItems = ( + seedItems: T[], + liveItems: T[], + historyItems: T[] +): T[] => { + const deduped = new Map(); + for (const item of [...seedItems, ...liveItems, ...historyItems]) { + deduped.set(getTapeItemKey(item), item); + } + return Array.from(deduped.values()).sort((a, b) => { + const delta = extractSortTs(b) - extractSortTs(a); + if (delta !== 0) { + return delta; + } + return extractSortSeq(b) - extractSortSeq(a); + }); +}; + type PausableTapeData = { visible: T[]; queued: T[]; @@ -510,7 +804,7 @@ const EMPTY_PAUSABLE_TAPE = { dropped: 0 }; -const appendHistoryTail = ( +export const appendHistoryTail = ( current: T[], incoming: T[], liveHead: T[], @@ -520,25 +814,67 @@ const appendHistoryTail = ( return current; } - const seen = new Set(); - for (const item of liveHead) { - seen.add(getTapeItemKey(item)); - } - for (const item of current) { - seen.add(getTapeItemKey(item)); - } + const seen = new Set(liveHead.map((item) => getTapeItemKey(item))); + const combined: T[] = []; - const appended = [...current]; - for (const item of incoming) { + for (const item of [...current, ...incoming]) { const key = getTapeItemKey(item); if (seen.has(key)) { continue; } seen.add(key); - appended.push(item); + combined.push(item); } - return cap > 0 ? appended.slice(0, cap) : appended; + combined.sort((a, b) => { + const delta = extractSortTs(b) - extractSortTs(a); + if (delta !== 0) { + return delta; + } + return extractSortSeq(b) - extractSortSeq(a); + }); + + return cap > 0 ? combined.slice(0, cap) : combined; +}; + +export const getLiveHistoryRetentionCap = (subscription: LiveSubscription): number => { + switch (subscription.channel) { + case "options": + case "equities": + return LIVE_HISTORY_SOFT_CAP; + default: + return LIVE_HISTORY_SOFT_CAP; + } +}; + +export const getScopedLiveAutoHydrationChannels = ( + enabled: boolean, + pathname: string, + manifest: LiveSubscription[], + historyCursors: Partial>, + historyLoading: Partial> +): Array> => { + if (!enabled || pathname !== "/tape") { + return []; + } + + const channels: Array> = []; + for (const subscription of manifest) { + const scoped = + (subscription.channel === "options" && + (subscription.underlying_ids?.length || subscription.option_contract_id)) || + (subscription.channel === "equities" && subscription.underlying_ids?.length); + if (!scoped) { + continue; + } + + const key = getLiveSubscriptionKey(subscription); + if (historyCursors[key] && !historyLoading[key]) { + channels.push(subscription.channel); + } + } + + return channels; }; export const getLiveFeedStatus = ( @@ -564,9 +900,45 @@ export const getLiveFeedStatus = ( return behindMs > behindDelayMs ? "stale" : "connected"; }; +export const getHotChannelFeedStatus = ( + sourceStatus: WsStatus, + health: { healthy: boolean } | null | undefined +): WsStatus => { + if (sourceStatus !== "connected") { + return sourceStatus; + } + if (!health) { + return "connected"; + } + return health.healthy ? "connected" : "stale"; +}; + +export const findAnchorRestoreIndex = ( + keys: string[], + anchorKey: string, + fallbackKeys: string[] +): number => { + const directIndex = keys.indexOf(anchorKey); + if (directIndex >= 0) { + return directIndex; + } + + const indexByKey = new Map(keys.map((key, index) => [key, index])); + for (const key of fallbackKeys) { + const index = indexByKey.get(key); + if (typeof index === "number") { + return index; + } + } + + return -1; +}; + type TapeState = { status: WsStatus; items: T[]; + liveItems?: T[]; + historyItems?: T[]; lastUpdate: number | null; replayTime: number | null; replayComplete: boolean; @@ -818,14 +1190,6 @@ const extractUnderlying = (contractId: string): string => { return contractId.split("-")[0]?.toUpperCase() ?? contractId.toUpperCase(); }; -const extractEquityTraceFromJoin = (joinId: string): string | null => { - const match = joinId.match(/^equityjoin:(.+)$/); - if (match?.[1]) { - return match[1]; - } - return joinId.trim().length > 0 ? joinId.trim() : null; -}; - const normalizeJoinRefCandidates = (value: string): string[] => { const ref = value.trim(); if (!ref) { @@ -879,7 +1243,6 @@ const formatDarkTrace = (traceId: string): string => { const inferDarkUnderlying = ( event: InferredDarkEvent, - equityPrints: Map, equityJoins: Map ): string | null => { for (const ref of event.evidence_refs) { @@ -898,17 +1261,6 @@ const inferDarkUnderlying = ( return match[1].toUpperCase(); } - for (const ref of event.evidence_refs) { - const traceId = extractEquityTraceFromJoin(ref); - if (!traceId) { - continue; - } - const print = equityPrints.get(traceId); - if (print) { - return print.underlying_id.toUpperCase(); - } - } - return null; }; @@ -1123,6 +1475,10 @@ type ClassifierDecor = { intensity: number; }; +const EMPTY_CLASSIFIER_HITS_BY_PACKET_ID = new Map(); +const EMPTY_PACKET_ID_BY_OPTION_TRACE_ID = new Map(); +const EMPTY_CLASSIFIER_DECOR_BY_OPTION_TRACE_ID = new Map(); + const SMART_MONEY_PROFILE_TONES: Record = { institutional_directional: "green", retail_whale: "amber", @@ -1219,6 +1575,7 @@ export const getOptionTableSnapshot = ( type ListScrollState = { listRef: React.RefObject; + listNode: HTMLDivElement | null; setListRef: (node: HTMLDivElement | null) => void; isAtTop: boolean; isAtTopRef: React.MutableRefObject; @@ -1310,6 +1667,7 @@ const useListScroll = (): ListScrollState => { return { listRef, + listNode, setListRef, isAtTop, isAtTopRef, @@ -1324,7 +1682,26 @@ const useScrollAnchor = ( listRef: React.RefObject, isAtTopRef: React.MutableRefObject ) => { - const pendingRef = useRef<{ height: number } | null>(null); + const pendingRef = useRef<{ + key: string; + offset: number; + fallbackKeys: string[]; + } | null>(null); + + const readRenderedRows = useCallback((element: HTMLDivElement) => { + return Array.from(element.querySelectorAll("[data-tape-key][data-row-start][data-row-size]")) + .map((node) => { + const key = node.dataset.tapeKey; + const start = Number(node.dataset.rowStart); + const size = Number(node.dataset.rowSize); + if (!key || !Number.isFinite(start) || !Number.isFinite(size)) { + return null; + } + return { key, start, size }; + }) + .filter((row): row is { key: string; start: number; size: number } => row !== null) + .sort((a, b) => a.start - b.start); + }, []); const capture = useCallback(() => { if (isAtTopRef.current) { @@ -1337,10 +1714,27 @@ const useScrollAnchor = ( return; } + const rows = readRenderedRows(el); + if (rows.length === 0) { + pendingRef.current = null; + return; + } + + const scrollTop = el.scrollTop; + const anchorIndex = rows.findIndex((row) => row.start + row.size > scrollTop); + const resolvedIndex = anchorIndex >= 0 ? anchorIndex : 0; + const anchorRow = rows[resolvedIndex]; + if (!anchorRow) { + pendingRef.current = null; + return; + } + pendingRef.current = { - height: el.scrollHeight + key: anchorRow.key, + offset: Math.max(0, scrollTop - anchorRow.start), + fallbackKeys: rows.slice(resolvedIndex).map((row) => row.key) }; - }, [isAtTopRef, listRef]); + }, [isAtTopRef, listRef, readRenderedRows]); const apply = useCallback(() => { const pending = pendingRef.current; @@ -1358,19 +1752,41 @@ const useScrollAnchor = ( return; } - const delta = el.scrollHeight - pending.height; - if (delta !== 0) { - el.scrollTop = Math.max(0, el.scrollTop + delta); + const rows = readRenderedRows(el); + if (rows.length === 0) { + return; + } + + const keys = rows.map((row) => row.key); + const restoreIndex = findAnchorRestoreIndex(keys, pending.key, pending.fallbackKeys); + if (restoreIndex < 0) { + return; + } + + const row = rows[restoreIndex]; + if (!row) { + return; + } + + el.scrollTop = Math.max(0, row.start + pending.offset); + bumpTapeDebugMetric("anchorRestoreCount", 1); + if (row.key !== pending.key) { + bumpTapeDebugMetric("anchorRestoreFallbackCount", 1); + logTapeDebug("anchor restore fallback", { + requested_key: pending.key, + restored_key: row.key + }); } pendingRef.current = null; - }, [isAtTopRef, listRef]); + }, [isAtTopRef, listRef, readRenderedRows]); return { capture, apply }; }; -const useBottomHistoryGate = ( - listRef: React.RefObject, +const useVirtualHistoryGate = ( enabled: boolean, + itemCount: number, + lastVirtualIndex: number, onLoadOlder: () => void ): void => { const loadRef = useRef(onLoadOlder); @@ -1379,107 +1795,90 @@ const useBottomHistoryGate = ( }, [onLoadOlder]); useEffect(() => { - if (!enabled) { + if (!enabled || itemCount === 0) { return; } - const element = listRef.current; - if (!element) { + if (lastVirtualIndex < itemCount - 1) { return; } - - const maybeLoad = () => { - const threshold = Math.max(240, element.clientHeight * 0.5); - if (element.scrollTop + element.clientHeight >= element.scrollHeight - threshold) { - loadRef.current(); - } - }; - - maybeLoad(); - element.addEventListener("scroll", maybeLoad); - return () => { - element.removeEventListener("scroll", maybeLoad); - }; - }, [enabled, listRef]); + loadRef.current(); + }, [enabled, itemCount, lastVirtualIndex]); }; -type VirtualListResult = { - visibleItems: T[]; - topSpacerHeight: number; - bottomSpacerHeight: number; +type TapeVirtualListResult = { + totalSize: number; + virtualItems: TapeVirtualRow[]; }; -const useVirtualList = ( +type TapeVirtualRow = { + item: T; + key: string; + index: number; + start: number; + size: number; + end: number; +}; + +const useTapeVirtualList = ( items: T[], listRef: React.RefObject, - enabled: boolean, - rowHeight: number, - overscan = 8 -): VirtualListResult => { - const [range, setRange] = useState<{ start: number; end: number }>({ - start: 0, - end: items.length + config: TapeVirtualListConfig +): TapeVirtualListResult => { + const virtualizer = useVirtualizer({ + count: items.length, + getScrollElement: () => listRef.current, + estimateSize: () => config.rowHeight, + overscan: config.overscan, + getItemKey: (index) => getTapeItemKey(items[index] as SortableItem) }); - const recompute = useCallback(() => { - if (!enabled) { - setRange({ start: 0, end: items.length }); - return; - } - - const element = listRef.current; - if (!element) { - setRange({ start: 0, end: Math.min(items.length, 80) }); - return; - } - - const viewportHeight = Math.max(rowHeight, element.clientHeight); - const visibleCount = Math.ceil(viewportHeight / rowHeight); - const start = Math.max(0, Math.floor(element.scrollTop / rowHeight) - overscan); - const end = Math.min(items.length, start + visibleCount + overscan * 2); - setRange({ start, end }); - }, [enabled, items.length, listRef, overscan, rowHeight]); + const virtualItems: TapeVirtualRow[] = virtualizer + .getVirtualItems() + .map((virtualItem) => { + const item = items[virtualItem.index] as T | undefined; + if (!item) { + return null; + } + return { + item, + key: getTapeItemKey(item), + index: virtualItem.index, + start: virtualItem.start, + size: virtualItem.size, + end: virtualItem.end + }; + }) + .filter((virtualItem): virtualItem is TapeVirtualRow => virtualItem !== null); useEffect(() => { - recompute(); - }, [items.length, recompute]); - - useEffect(() => { - if (!enabled) { + if (!DEV_TAPE_DEBUG || items.length === 0) { return; } - const element = listRef.current; if (!element) { return; } - - const onScroll = () => recompute(); - const onResize = () => recompute(); - - element.addEventListener("scroll", onScroll); - window.addEventListener("resize", onResize); - - return () => { - element.removeEventListener("scroll", onScroll); - window.removeEventListener("resize", onResize); - }; - }, [enabled, listRef, recompute]); - - if (!enabled) { - return { - visibleItems: items, - topSpacerHeight: 0, - bottomSpacerHeight: 0 - }; - } - - const start = Math.min(range.start, items.length); - const end = Math.min(Math.max(range.end, start), items.length); + const first = virtualItems[0]; + const last = virtualItems.at(-1); + if (!first || !last) { + return; + } + const visibleTopGap = Math.max(0, first.start - element.scrollTop); + const visibleBottomGap = Math.max(0, element.scrollTop + element.clientHeight - last.end); + if (visibleTopGap > element.clientHeight || visibleBottomGap > element.clientHeight) { + console.warn("[tape] false-gap watchdog", { + pane: config.debugLabel, + item_count: items.length, + visible_top_gap: visibleTopGap, + visible_bottom_gap: visibleBottomGap, + viewport_height: element.clientHeight + }); + } + }, [config.debugLabel, items.length, listRef, virtualItems]); return { - visibleItems: items.slice(start, end), - topSpacerHeight: start * rowHeight, - bottomSpacerHeight: Math.max(0, (items.length - end) * rowHeight) + totalSize: virtualizer.getTotalSize(), + virtualItems }; }; @@ -1565,6 +1964,13 @@ const useTape = ( const replaySourceKey = config.replaySourceKey ?? null; const onReplaySourceKey = config.onReplaySourceKey; const queryParams = config.queryParams; + const queryKey = useMemo( + () => + JSON.stringify( + Object.entries(queryParams ?? {}).sort(([left], [right]) => left.localeCompare(right)) + ), + [queryParams] + ); const hotWindowLimit = config.hotWindowLimit ?? LIVE_HOT_WINDOW; const [status, setStatus] = useState("connecting"); const [items, setItems] = useState([]); @@ -1655,7 +2061,7 @@ const useTape = ( pendingRef.current = []; pendingCountRef.current = 0; cancelFlush(); - }, [mode, replaySourceKey, cancelFlush]); + }, [mode, replaySourceKey, queryKey, cancelFlush]); useEffect(() => { if (mode !== "replay" || !latestPath) { @@ -1700,7 +2106,7 @@ const useTape = ( return () => { active = false; }; - }, [mode, latestPath, getItemTs, replaySourceKey, queryParams]); + }, [mode, latestPath, getItemTs, replaySourceKey, queryKey, queryParams]); useEffect(() => { if (mode !== "live" || config.liveEnabled === false) { @@ -1851,9 +2257,14 @@ const useTape = ( } } - if (onReplaySourceKey && sourcePrefix && replaySourceNotifiedRef.current !== sourcePrefix) { - replaySourceNotifiedRef.current = sourcePrefix; - onReplaySourceKey(sourcePrefix); + if (onReplaySourceKey) { + if (sourcePrefix && replaySourceNotifiedRef.current !== sourcePrefix) { + replaySourceNotifiedRef.current = sourcePrefix; + onReplaySourceKey(sourcePrefix); + } else if (!sourcePrefix && replaySourceNotifiedRef.current !== null) { + replaySourceNotifiedRef.current = null; + onReplaySourceKey(null); + } } const filtered = sourcePrefix @@ -1939,6 +2350,7 @@ const useTape = ( getReplayKey, replaySourceKey, onReplaySourceKey, + queryKey, queryParams ]); @@ -1961,6 +2373,8 @@ const toStaticTapeState = ( ): TapeState => ({ status, items, + liveItems: items, + historyItems: [], lastUpdate, replayTime: null, replayComplete: false, @@ -1975,10 +2389,8 @@ type PausableTapeViewConfig = { sourceItems: T[]; historyTail?: T[]; lastUpdate: number | null; - freshnessMs: number; onNewItems?: (count: number) => void; captureScroll?: () => void; - getItemTs?: (item: T) => number; retentionLimit?: number; shouldHold?: () => boolean; resumeSignal?: number; @@ -1989,17 +2401,6 @@ const usePausableTapeView = ( ): TapeState => { const [paused, setPaused] = useState(false); const [data, setData] = useState>(EMPTY_PAUSABLE_TAPE); - const [clock, setClock] = useState(() => Date.now()); - - useEffect(() => { - const handle = window.setInterval(() => { - setClock(Date.now()); - }, 1000); - - return () => { - window.clearInterval(handle); - }; - }, []); useEffect(() => { if (!config.enabled) { @@ -2075,38 +2476,16 @@ const usePausableTapeView = ( setPaused((current) => !current); }, []); - const getItemTs = config.getItemTs ?? extractSortTs; - const freshestTs = useMemo(() => { - if (config.sourceItems.length === 0) { - return null; - } - - let newest = Number.NEGATIVE_INFINITY; - for (const item of config.sourceItems) { - newest = Math.max(newest, getItemTs(item)); - } - - return Number.isFinite(newest) ? newest : null; - }, [config.sourceItems, getItemTs]); - - const status = config.enabled - ? getLiveFeedStatus( - config.sourceStatus, - freshestTs, - config.freshnessMs, - clock, - LIVE_FEED_BEHIND_DELAY_MS - ) - : "disconnected"; + const status = config.enabled ? config.sourceStatus : "disconnected"; const projected = projectPausableTapeState(data.visible, status, config.lastUpdate); - const items = useMemo( - () => [...projected.items, ...(config.historyTail ?? [])], - [projected.items, config.historyTail] - ); + const historyItems = config.historyTail ?? []; + const items = useMemo(() => composeTapeItems([], projected.items, historyItems), [projected.items, historyItems]); return { status, items, + liveItems: projected.items, + historyItems, lastUpdate: projected.lastUpdate, replayTime: null, replayComplete: false, @@ -2355,6 +2734,7 @@ type LiveSessionState = { status: WsStatus; connectedAt: number | null; lastUpdate: number | null; + channelHealth: LiveHotChannelHealthMap; lastEventByChannel: Partial>; manifest: LiveSubscription[]; historyCursors: Partial>; @@ -2425,6 +2805,99 @@ const appendOptionFlowFilters = (params: URLSearchParams, filters: OptionFlowFil } }; +const appendOptionScopeParams = ( + params: URLSearchParams, + optionScope: OptionScope | undefined +): void => { + if (optionScope?.underlying_ids?.length) { + params.set("underlying_ids", optionScope.underlying_ids.join(",")); + } + if (optionScope?.option_contract_id) { + params.set("option_contract_id", optionScope.option_contract_id); + } +}; + +export const getEffectiveOptionPrintFilters = ( + flowFilters: OptionFlowFilters, + isOptionContractFocused: boolean +): OptionFlowFilters | undefined => { + return isOptionContractFocused ? undefined : flowFilters; +}; + +export const getOptionScope = ( + activeTickers: string[], + instrumentUnderlying: string | null, + selectedInstrument: SelectedInstrument +): OptionScope => ({ + underlying_ids: + selectedInstrument?.kind === "option-contract" + ? instrumentUnderlying + ? [instrumentUnderlying] + : undefined + : activeTickers.length > 0 + ? activeTickers + : instrumentUnderlying + ? [instrumentUnderlying] + : undefined, + option_contract_id: + selectedInstrument?.kind === "option-contract" ? selectedInstrument.contractId : undefined +}); + +export const buildOptionTapeQueryParams = ( + filters: OptionFlowFilters | undefined, + optionScope: OptionScope | undefined +): Record => { + const params = new URLSearchParams(); + appendOptionFlowFilters(params, filters); + appendOptionScopeParams(params, optionScope); + return Object.fromEntries(params.entries()); +}; + +export const filterOptionTapeItems = ( + items: OptionPrint[], + filters: OptionFlowFilters | undefined, + selectedInstrument: SelectedInstrument, + tickerSet: Set, + instrumentUnderlying: string | null +): OptionPrint[] => { + return items.filter((print) => { + const contractId = normalizeContractId(print.option_contract_id); + if (selectedInstrument?.kind === "option-contract") { + return contractId === selectedInstrument.contractId; + } + if (!matchesOptionPrintFilters(print, filters)) { + return false; + } + const underlying = extractUnderlying(contractId); + if (tickerSet.size === 0) { + return !instrumentUnderlying || underlying === instrumentUnderlying; + } + return Boolean(underlying) && tickerSet.has(underlying.toUpperCase()); + }); +}; + +export const shouldClearOptionFocusSeed = ( + seed: TapeFocusSeed | null, + optionFocusScopeKey: string | null, + currentOptionSubscriptionKey: string | null, + liveItems: OptionPrint[], + historyItems: OptionPrint[] +): boolean => { + if (!seed) { + return false; + } + if (seed.scopeKey !== optionFocusScopeKey) { + return true; + } + if (seed.subscriptionKey && seed.subscriptionKey !== currentOptionSubscriptionKey) { + return false; + } + const liveKeys = new Set( + composeTapeItems([], liveItems, historyItems).map((item) => getTapeItemKey(item)) + ); + return seed.items.every((item) => liveKeys.has(getTapeItemKey(item))); +}; + const appendLiveScopeParams = (params: URLSearchParams, subscription: LiveSubscription): void => { if ((subscription.channel === "options" || subscription.channel === "equities") && subscription.underlying_ids?.length) { params.set("underlying_ids", subscription.underlying_ids.join(",")); @@ -2451,59 +2924,79 @@ export const getLiveManifest = ( chartTicker: string, chartIntervalMs: number, flowFilters: OptionFlowFilters, - optionScope?: Pick, "underlying_ids" | "option_contract_id">, - equityScope?: Pick, "underlying_ids"> + optionScope?: OptionScope, + equityScope?: EquityScope, + optionPrintFilters?: OptionFlowFilters ): LiveSubscription[] => { - const baselineSubs: LiveSubscription[] = [{ channel: "options", filters: flowFilters, ...optionScope }]; - const chartSubs: LiveSubscription[] = [ - { channel: "equity-candles", underlying_id: chartTicker, interval_ms: chartIntervalMs }, - { channel: "equity-overlay", underlying_id: chartTicker } - ]; + const features = getRouteFeatures(pathname); + const subscriptions: LiveSubscription[] = []; - if (pathname === "/tape") { - const optionsSub: Extract = { + if (features.options) { + subscriptions.push({ channel: "options", - filters: flowFilters, + filters: + optionScope?.option_contract_id && optionPrintFilters === undefined + ? undefined + : optionPrintFilters ?? flowFilters, ...optionScope, snapshot_limit: LIVE_HOT_WINDOW_OPTIONS - }; - const tapeSubs: LiveSubscription[] = [ - optionsSub, - { channel: "nbbo", snapshot_limit: LIVE_HOT_WINDOW }, - { channel: "equities", ...equityScope, snapshot_limit: LIVE_HOT_WINDOW }, - { channel: "flow", filters: flowFilters, snapshot_limit: LIVE_HOT_WINDOW }, - { channel: "smart-money", snapshot_limit: LIVE_HOT_WINDOW }, - { channel: "classifier-hits", snapshot_limit: LIVE_HOT_WINDOW }, - { channel: "alerts", snapshot_limit: LIVE_HOT_WINDOW }, - { channel: "inferred-dark", snapshot_limit: LIVE_HOT_WINDOW } - ]; - return dedupeLiveSubscriptions(tapeSubs); + }); + } + if (features.nbbo) { + subscriptions.push({ channel: "nbbo", snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.equities) { + subscriptions.push({ channel: "equities", ...equityScope, snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.flow) { + subscriptions.push({ channel: "flow", filters: flowFilters, snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.alerts) { + subscriptions.push({ channel: "alerts", snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.smartMoney) { + subscriptions.push({ channel: "smart-money", snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.classifierHits) { + subscriptions.push({ channel: "classifier-hits", snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.inferredDark) { + subscriptions.push({ channel: "inferred-dark", snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.equityJoins) { + subscriptions.push({ channel: "equity-joins", snapshot_limit: LIVE_HOT_WINDOW }); + } + if (features.equityCandles) { + subscriptions.push({ + channel: "equity-candles", + underlying_id: chartTicker, + interval_ms: chartIntervalMs + }); + } + if (features.equityOverlay) { + subscriptions.push({ + channel: "equity-overlay", + underlying_id: chartTicker + }); } - return dedupeLiveSubscriptions([ - ...baselineSubs, - { channel: "equities", ...equityScope }, - { channel: "flow", filters: flowFilters }, - { channel: "alerts" }, - { channel: "smart-money" }, - { channel: "classifier-hits" }, - { channel: "inferred-dark" }, - ...chartSubs - ]); + return dedupeLiveSubscriptions(subscriptions); }; const useLiveSession = ( enabled: boolean, pathname: string, - chartTicker: string, - chartIntervalMs: number, - flowFilters: OptionFlowFilters, - optionScope?: Pick, "underlying_ids" | "option_contract_id">, - equityScope?: Pick, "underlying_ids"> + manifest: LiveSubscription[] ): LiveSessionState => { const [status, setStatus] = useState(enabled ? "connecting" : "disconnected"); const [connectedAt, setConnectedAt] = useState(null); const [lastUpdate, setLastUpdate] = useState(null); + const [channelHealth, setChannelHealth] = useState({ + options: { freshness_age_ms: null, healthy: false }, + nbbo: { freshness_age_ms: null, healthy: false }, + equities: { freshness_age_ms: null, healthy: false }, + flow: { freshness_age_ms: null, healthy: false } + }); const [lastEventByChannel, setLastEventByChannel] = useState< Partial> >({}); @@ -2531,6 +3024,27 @@ const useLiveSession = ( const [inferredDarkHistory, setInferredDarkHistory] = useState([]); const [chartCandles, setChartCandles] = useState([]); const [chartOverlay, setChartOverlay] = useState([]); + const optionsRef = useRef([]); + const nbboRef = useRef([]); + const equitiesRef = useRef([]); + const equityQuotesRef = useRef([]); + const equityJoinsRef = useRef([]); + const flowRef = useRef([]); + const smartMoneyRef = useRef([]); + const classifierHitsRef = useRef([]); + const alertsRef = useRef([]); + const inferredDarkRef = useRef([]); + const chartCandlesRef = useRef([]); + const chartOverlayRef = useRef([]); + const optionsHistoryRef = useRef([]); + const nbboHistoryRef = useRef([]); + const equitiesHistoryRef = useRef([]); + const equityJoinsHistoryRef = useRef([]); + const flowHistoryRef = useRef([]); + const smartMoneyHistoryRef = useRef([]); + const classifierHitsHistoryRef = useRef([]); + const alertsHistoryRef = useRef([]); + const inferredDarkHistoryRef = useRef([]); const socketRef = useRef(null); const reconnectRef = useRef(null); const idleWatchdogRef = useRef(null); @@ -2538,16 +3052,38 @@ const useLiveSession = ( const lastEventAtRef = useRef(null); const subscribedKeysRef = useRef>(new Set()); const subscribedMapRef = useRef>(new Map()); - const manifest = useMemo( - () => getLiveManifest(pathname, chartTicker.toUpperCase(), chartIntervalMs, flowFilters, optionScope, equityScope), - [pathname, chartTicker, chartIntervalMs, flowFilters, optionScope, equityScope] - ); + const replaceArrayState = ( + setter: Dispatch>, + ref: { current: T[] }, + next: T[] + ): void => { + ref.current = next; + setter(next); + }; + + const mergeHistoryState = ( + setter: Dispatch>, + ref: { current: T[] }, + incoming: T[], + liveHead: T[], + cap = LIVE_HISTORY_SOFT_CAP + ): void => { + const next = appendHistoryTail(ref.current, incoming, liveHead, cap); + ref.current = next; + setter(next); + }; useEffect(() => { if (!enabled) { setStatus("disconnected"); setConnectedAt(null); setLastUpdate(null); + setChannelHealth({ + options: { freshness_age_ms: null, healthy: false }, + nbbo: { freshness_age_ms: null, healthy: false }, + equities: { freshness_age_ms: null, healthy: false }, + flow: { freshness_age_ms: null, healthy: false } + }); setLastEventByChannel({}); setHistoryCursors({}); setHistoryLoading({}); @@ -2573,6 +3109,27 @@ const useLiveSession = ( setInferredDarkHistory([]); setChartCandles([]); setChartOverlay([]); + optionsRef.current = []; + nbboRef.current = []; + equitiesRef.current = []; + equityQuotesRef.current = []; + equityJoinsRef.current = []; + flowRef.current = []; + smartMoneyRef.current = []; + classifierHitsRef.current = []; + alertsRef.current = []; + inferredDarkRef.current = []; + chartCandlesRef.current = []; + chartOverlayRef.current = []; + optionsHistoryRef.current = []; + nbboHistoryRef.current = []; + equitiesHistoryRef.current = []; + equityJoinsHistoryRef.current = []; + flowHistoryRef.current = []; + smartMoneyHistoryRef.current = []; + classifierHitsHistoryRef.current = []; + alertsHistoryRef.current = []; + inferredDarkHistoryRef.current = []; subscribedKeysRef.current = new Set(); subscribedMapRef.current = new Map(); if (socketRef.current) { @@ -2616,6 +3173,7 @@ const useLiveSession = ( const handleMessage = (message: LiveServerMessage) => { if (message.op === "ready" || message.op === "heartbeat") { + setChannelHealth(message.channel_health); return; } if (message.op === "error") { @@ -2629,62 +3187,112 @@ const useLiveSession = ( const updateAt = Date.now(); const mergeItems = ( - setter: React.Dispatch>, + setter: Dispatch>, + ref: { current: T[] }, nextItems: T[], - retentionLimit = LIVE_HOT_WINDOW + retentionLimit = LIVE_HOT_WINDOW, + history?: { + setter: Dispatch>; + ref: { current: T[] }; + cap?: number; + } ) => { - setter((prev) => - message.op === "snapshot" - ? shouldRetainLiveSnapshotHistory( - subscription.channel, - true, - nextItems.length, - prev.length - ) - ? prev - : (nextItems as T[]) - : mergeNewest(nextItems as T[], prev, retentionLimit, (evicted) => - incrementRetentionMetric("hotWindowEvictions", evicted) - ) + if (message.op === "snapshot") { + const next = shouldRetainLiveSnapshotHistory( + subscription.channel, + true, + nextItems.length, + ref.current.length + ) + ? ref.current + : nextItems; + replaceArrayState(setter, ref, next); + return; + } + + const { kept, evicted } = mergeNewestWithOverflow( + nextItems, + ref.current, + retentionLimit, + (evictedCount) => incrementRetentionMetric("hotWindowEvictions", evictedCount) ); + replaceArrayState(setter, ref, kept); + if (history && evicted.length > 0) { + mergeHistoryState(history.setter, history.ref, evicted, kept, history.cap); + } }; switch (subscription.channel) { case "options": - mergeItems(setOptions, items as OptionPrint[], LIVE_HOT_WINDOW_OPTIONS); + mergeItems(setOptions, optionsRef, items as OptionPrint[], LIVE_HOT_WINDOW_OPTIONS, { + setter: setOptionsHistory, + ref: optionsHistoryRef, + cap: getLiveHistoryRetentionCap(subscription) + }); break; case "nbbo": - mergeItems(setNbbo, items as OptionNBBO[]); + mergeItems(setNbbo, nbboRef, items as OptionNBBO[], LIVE_HOT_WINDOW, { + setter: setNbboHistory, + ref: nbboHistoryRef + }); break; case "equities": - mergeItems(setEquities, items as EquityPrint[]); + mergeItems(setEquities, equitiesRef, items as EquityPrint[], LIVE_HOT_WINDOW, { + setter: setEquitiesHistory, + ref: equitiesHistoryRef, + cap: getLiveHistoryRetentionCap(subscription) + }); break; case "equity-quotes": - mergeItems(setEquityQuotes, items as EquityQuote[]); + mergeItems(setEquityQuotes, equityQuotesRef, items as EquityQuote[]); break; case "equity-joins": - mergeItems(setEquityJoins, items as EquityPrintJoin[]); + mergeItems(setEquityJoins, equityJoinsRef, items as EquityPrintJoin[], LIVE_HOT_WINDOW, { + setter: setEquityJoinsHistory, + ref: equityJoinsHistoryRef + }); break; case "flow": - mergeItems(setFlow, items as FlowPacket[]); + mergeItems(setFlow, flowRef, items as FlowPacket[], LIVE_HOT_WINDOW, { + setter: setFlowHistory, + ref: flowHistoryRef + }); break; case "smart-money": - mergeItems(setSmartMoney, items as SmartMoneyEvent[]); + mergeItems(setSmartMoney, smartMoneyRef, items as SmartMoneyEvent[], LIVE_HOT_WINDOW, { + setter: setSmartMoneyHistory, + ref: smartMoneyHistoryRef + }); break; case "classifier-hits": - mergeItems(setClassifierHits, items as ClassifierHitEvent[]); + mergeItems( + setClassifierHits, + classifierHitsRef, + items as ClassifierHitEvent[], + LIVE_HOT_WINDOW, + { + setter: setClassifierHitsHistory, + ref: classifierHitsHistoryRef + } + ); break; case "alerts": - mergeItems(setAlerts, items as AlertEvent[]); + mergeItems(setAlerts, alertsRef, items as AlertEvent[], LIVE_HOT_WINDOW, { + setter: setAlertsHistory, + ref: alertsHistoryRef + }); break; case "inferred-dark": - mergeItems(setInferredDark, items as InferredDarkEvent[]); + mergeItems(setInferredDark, inferredDarkRef, items as InferredDarkEvent[], LIVE_HOT_WINDOW, { + setter: setInferredDarkHistory, + ref: inferredDarkHistoryRef + }); break; case "equity-candles": - mergeItems(setChartCandles, items as EquityCandle[]); + mergeItems(setChartCandles, chartCandlesRef, items as EquityCandle[]); break; case "equity-overlay": - mergeItems(setChartOverlay, items as EquityPrint[]); + mergeItems(setChartOverlay, chartOverlayRef, items as EquityPrint[]); break; } @@ -2826,10 +3434,14 @@ const useLiveSession = ( .filter((channel) => channel === "options" || channel === "equities") ); if (resetScopedChannels.has("options")) { + optionsRef.current = []; + optionsHistoryRef.current = []; setOptions([]); setOptionsHistory([]); } if (resetScopedChannels.has("equities")) { + equitiesRef.current = []; + equitiesHistoryRef.current = []; setEquities([]); setEquitiesHistory([]); } @@ -2913,49 +3525,56 @@ const useLiveSession = ( const mergeOlder = ( setter: Dispatch>, + ref: { current: T[] }, liveHead: T[], cap = LIVE_HISTORY_SOFT_CAP ) => { - setter((prev) => appendHistoryTail(prev, older as T[], liveHead, cap)); + mergeHistoryState(setter, ref, older as T[], liveHead, cap); }; switch (subscription.channel) { case "options": mergeOlder( setOptionsHistory, - options, - subscription.underlying_ids?.length || subscription.option_contract_id ? 0 : LIVE_HISTORY_SOFT_CAP + optionsHistoryRef, + optionsRef.current, + getLiveHistoryRetentionCap(subscription) ); break; case "nbbo": - mergeOlder(setNbboHistory, nbbo); + mergeOlder(setNbboHistory, nbboHistoryRef, nbboRef.current); break; case "equities": mergeOlder( setEquitiesHistory, - equities, - subscription.underlying_ids?.length ? 0 : LIVE_HISTORY_SOFT_CAP + equitiesHistoryRef, + equitiesRef.current, + getLiveHistoryRetentionCap(subscription) ); break; case "equity-quotes": break; case "equity-joins": - mergeOlder(setEquityJoinsHistory, equityJoins); + mergeOlder(setEquityJoinsHistory, equityJoinsHistoryRef, equityJoinsRef.current); break; case "flow": - mergeOlder(setFlowHistory, flow); + mergeOlder(setFlowHistory, flowHistoryRef, flowRef.current); break; case "smart-money": - mergeOlder(setSmartMoneyHistory, smartMoney); + mergeOlder(setSmartMoneyHistory, smartMoneyHistoryRef, smartMoneyRef.current); break; case "classifier-hits": - mergeOlder(setClassifierHitsHistory, classifierHits); + mergeOlder( + setClassifierHitsHistory, + classifierHitsHistoryRef, + classifierHitsRef.current + ); break; case "alerts": - mergeOlder(setAlertsHistory, alerts); + mergeOlder(setAlertsHistory, alertsHistoryRef, alertsRef.current); break; case "inferred-dark": - mergeOlder(setInferredDarkHistory, inferredDark); + mergeOlder(setInferredDarkHistory, inferredDarkHistoryRef, inferredDarkRef.current); break; } @@ -2973,41 +3592,18 @@ const useLiveSession = ( setHistoryLoading((current) => ({ ...current, [key]: false })); } }, - [ - enabled, - manifest, - historyCursors, - historyLoading, - options, - nbbo, - equities, - equityJoins, - flow, - smartMoney, - classifierHits, - alerts, - inferredDark - ] + [enabled, manifest, historyCursors, historyLoading] ); useEffect(() => { - if (!enabled || pathname !== "/tape") { - return; - } - const scoped = manifest.filter( - (subscription) => - (subscription.channel === "options" && - (subscription.underlying_ids?.length || subscription.option_contract_id)) || - (subscription.channel === "equities" && subscription.underlying_ids?.length) - ); - if (scoped.length === 0) { - return; - } - for (const subscription of scoped) { - const key = getLiveSubscriptionKey(subscription); - if (historyCursors[key] && !historyLoading[key]) { - void loadOlder(subscription.channel); - } + for (const channel of getScopedLiveAutoHydrationChannels( + enabled, + pathname, + manifest, + historyCursors, + historyLoading + )) { + void loadOlder(channel); } }, [enabled, pathname, manifest, historyCursors, historyLoading, loadOlder]); @@ -3015,6 +3611,7 @@ const useLiveSession = ( status, connectedAt, lastUpdate, + channelHealth, lastEventByChannel, manifest, historyCursors, @@ -4347,6 +4944,7 @@ const formatFlowMetric = (value: number, suffix?: string): string => { const useTerminalState = () => { const pathname = usePathname(); + const routeFeatures = useMemo(() => getRouteFeatures(pathname), [pathname]); const [mode, setMode] = useState("live"); const [replaySource, setReplaySource] = useState(null); const [selectedAlert, setSelectedAlert] = useState(null); @@ -4354,6 +4952,8 @@ const useTerminalState = () => { const [selectedClassifierHit, setSelectedClassifierHit] = useState(null); const [selectedSmartMoneyEvent, setSelectedSmartMoneyEvent] = useState(null); const [selectedInstrument, setSelectedInstrument] = useState(null); + const [optionFocusSeed, setOptionFocusSeed] = useState | null>(null); + const [equityFocusSeed, setEquityFocusSeed] = useState | null>(null); const [filterInput, setFilterInput] = useState(""); const [flowFilters, setFlowFilters] = useState(() => buildDefaultFlowFilters()); const [chartIntervalMs, setChartIntervalMs] = useState(CANDLE_INTERVALS[0].ms); @@ -4366,12 +4966,21 @@ const useTerminalState = () => { }, [filterInput]); const tickerSet = useMemo(() => new Set(activeTickers), [activeTickers]); const instrumentUnderlying = selectedInstrument?.underlyingId.toUpperCase() ?? null; + const isOptionContractFocused = selectedInstrument?.kind === "option-contract"; + const focusedOptionContractId = + selectedInstrument?.kind === "option-contract" ? selectedInstrument.contractId : null; + const optionFocusScopeKey = + focusedOptionContractId ? `option-contract:${focusedOptionContractId}` : null; + const equityFocusScopeKey = + selectedInstrument?.kind === "equity" + ? `equity:${selectedInstrument.underlyingId.toUpperCase()}` + : null; + const effectiveOptionPrintFilters = useMemo( + () => getEffectiveOptionPrintFilters(flowFilters, isOptionContractFocused), + [flowFilters, isOptionContractFocused] + ); const optionScope = useMemo( - () => ({ - underlying_ids: activeTickers.length > 0 ? activeTickers : instrumentUnderlying ? [instrumentUnderlying] : undefined, - option_contract_id: - selectedInstrument?.kind === "option-contract" ? selectedInstrument.contractId : undefined - }), + () => getOptionScope(activeTickers, instrumentUnderlying, selectedInstrument), [activeTickers, instrumentUnderlying, selectedInstrument] ); const equityScope = useMemo( @@ -4396,22 +5005,41 @@ const useTerminalState = () => { ? `Contract: ${display.ticker} ${display.expiration} ${display.strike}` : `Contract: ${selectedInstrument.contractId}`; }, [selectedInstrument]); - const liveSession = useLiveSession( - mode === "live", - pathname, - chartTicker, - chartIntervalMs, - flowFilters, - optionScope, - equityScope - ); - const equitiesLiveSubscriptionActive = useMemo( + const liveManifest = useMemo( () => - getLiveManifest(pathname, chartTicker.toUpperCase(), chartIntervalMs, flowFilters, optionScope, equityScope).some( - (sub) => sub.channel === "equities" + getLiveManifest( + pathname, + chartTicker.toUpperCase(), + chartIntervalMs, + flowFilters, + optionScope, + equityScope, + effectiveOptionPrintFilters ), - [pathname, chartTicker, chartIntervalMs, flowFilters, optionScope, equityScope] + [ + pathname, + chartTicker, + chartIntervalMs, + flowFilters, + optionScope, + equityScope, + effectiveOptionPrintFilters + ] ); + const liveSession = useLiveSession(mode === "live", pathname, liveManifest); + const currentOptionSubscription = useMemo( + () => + liveManifest.find( + (subscription): subscription is Extract => + subscription.channel === "options" + ) ?? null, + [liveManifest] + ); + const currentOptionSubscriptionKey = useMemo( + () => (currentOptionSubscription ? getLiveSubscriptionKey(currentOptionSubscription) : null), + [currentOptionSubscription] + ); + const equitiesLiveSubscriptionActive = routeFeatures.equities; const handleReplaySource = useCallback((value: string | null) => { setReplaySource(value); @@ -4473,18 +5101,8 @@ const useTerminalState = () => { ); const disableReplayGrouping = useCallback(() => null, []); const optionQueryParams = useMemo>( - () => ({ - view: flowFilters.view ?? "signal", - security: - flowFilters.securityTypes?.length === 1 ? flowFilters.securityTypes[0] : undefined, - side: flowFilters.nbboSides?.length ? flowFilters.nbboSides.join(",") : undefined, - type: flowFilters.optionTypes?.length ? flowFilters.optionTypes.join(",") : undefined, - min_notional: - typeof flowFilters.minNotional === "number" - ? String(flowFilters.minNotional) - : undefined - }), - [flowFilters] + () => buildOptionTapeQueryParams(effectiveOptionPrintFilters, optionScope), + [effectiveOptionPrintFilters, optionScope] ); const options = useTape({ @@ -4499,9 +5117,10 @@ const useTerminalState = () => { pollMs: mode === "replay" ? 200 : undefined, captureScroll: optionsAnchor.capture, onNewItems: optionsScroll.onNewItems, - getReplayKey: extractReplaySource, - onReplaySourceKey: handleReplaySource, - queryParams: optionQueryParams + getReplayKey: isOptionContractFocused ? disableReplayGrouping : extractReplaySource, + onReplaySourceKey: isOptionContractFocused ? undefined : handleReplaySource, + queryParams: optionQueryParams, + replaySourceKey: isOptionContractFocused ? null : replaySource }); const equities = useTape({ @@ -4517,6 +5136,12 @@ const useTerminalState = () => { onNewItems: equitiesScroll.onNewItems }); + useEffect(() => { + if (isOptionContractFocused && replaySource !== null) { + setReplaySource(null); + } + }, [isOptionContractFocused, replaySource]); + const equityJoins = useTape({ mode, liveEnabled: false, @@ -4609,13 +5234,19 @@ const useTerminalState = () => { getReplayKey: disableReplayGrouping }); + const optionsChannelStatus = getHotChannelFeedStatus(liveSession.status, liveSession.channelHealth.options); + const equitiesChannelStatus = getHotChannelFeedStatus( + liveSession.status, + liveSession.channelHealth.equities + ); + const flowChannelStatus = getHotChannelFeedStatus(liveSession.status, liveSession.channelHealth.flow); + const liveOptions = usePausableTapeView({ enabled: mode === "live", - sourceStatus: liveSession.status, + sourceStatus: optionsChannelStatus, sourceItems: liveSession.options, historyTail: liveSession.optionsHistory, lastUpdate: liveSession.lastUpdate, - freshnessMs: LIVE_OPTIONS_STALE_MS, retentionLimit: LIVE_HOT_WINDOW_OPTIONS, captureScroll: optionsAnchor.capture, onNewItems: optionsScroll.onNewItems, @@ -4624,11 +5255,10 @@ const useTerminalState = () => { }); const liveEquities = usePausableTapeView({ enabled: mode === "live", - sourceStatus: liveSession.status, + sourceStatus: equitiesChannelStatus, sourceItems: liveSession.equities, historyTail: liveSession.equitiesHistory, lastUpdate: liveSession.lastUpdate, - freshnessMs: LIVE_EQUITIES_STALE_MS, captureScroll: equitiesAnchor.capture, onNewItems: equitiesScroll.onNewItems, shouldHold: () => !equitiesScroll.isAtTopRef.current, @@ -4636,40 +5266,87 @@ const useTerminalState = () => { }); const liveFlow = usePausableTapeView({ enabled: mode === "live", - sourceStatus: liveSession.status, + sourceStatus: flowChannelStatus, sourceItems: liveSession.flow, historyTail: liveSession.flowHistory, lastUpdate: liveSession.lastUpdate, - freshnessMs: LIVE_FLOW_STALE_MS, captureScroll: flowAnchor.capture, onNewItems: flowScroll.onNewItems, shouldHold: () => !flowScroll.isAtTopRef.current, - resumeSignal: flowScroll.resumeTick, - getItemTs: (item) => item.source_ts + resumeSignal: flowScroll.resumeTick }); - const optionsFeed = mode === "live" ? liveOptions : options; + const seededLiveOptionsItems = useMemo( + () => + composeTapeItems( + optionFocusSeed?.scopeKey === optionFocusScopeKey ? optionFocusSeed.items : [], + liveOptions.liveItems ?? [], + liveOptions.historyItems ?? [] + ), + [liveOptions.historyItems, liveOptions.liveItems, optionFocusScopeKey, optionFocusSeed] + ); + const seededLiveEquitiesItems = useMemo( + () => + composeTapeItems( + equityFocusSeed?.scopeKey === equityFocusScopeKey ? equityFocusSeed.items : [], + liveEquities.liveItems ?? [], + liveEquities.historyItems ?? [] + ), + [equityFocusScopeKey, equityFocusSeed, liveEquities.historyItems, liveEquities.liveItems] + ); + + const optionsFeed = + mode === "live" ? { ...liveOptions, items: seededLiveOptionsItems } : options; const nbboFeed = - mode === "live" ? toStaticTapeState(liveSession.status, [...liveSession.nbbo, ...liveSession.nbboHistory], liveSession.lastUpdate) : nbbo; - const equitiesFeed = mode === "live" ? liveEquities : equities; + mode === "live" + ? toStaticTapeState( + getHotChannelFeedStatus(liveSession.status, liveSession.channelHealth.nbbo), + composeTapeItems([], liveSession.nbbo, liveSession.nbboHistory), + liveSession.lastUpdate + ) + : nbbo; + const equitiesFeed = + mode === "live" ? { ...liveEquities, items: seededLiveEquitiesItems } : equities; const equityJoinsFeed = mode === "live" - ? toStaticTapeState(liveSession.status, [...liveSession.equityJoins, ...liveSession.equityJoinsHistory], liveSession.lastUpdate) + ? toStaticTapeState( + liveSession.status, + composeTapeItems([], liveSession.equityJoins, liveSession.equityJoinsHistory), + liveSession.lastUpdate + ) : equityJoins; const flowFeed = mode === "live" ? liveFlow : flow; const alertsFeed = - mode === "live" ? toStaticTapeState(liveSession.status, [...liveSession.alerts, ...liveSession.alertsHistory], liveSession.lastUpdate) : alerts; + mode === "live" + ? toStaticTapeState( + liveSession.status, + composeTapeItems([], liveSession.alerts, liveSession.alertsHistory), + liveSession.lastUpdate + ) + : alerts; const classifierHitsFeed = mode === "live" - ? toStaticTapeState(liveSession.status, [...liveSession.classifierHits, ...liveSession.classifierHitsHistory], liveSession.lastUpdate) + ? toStaticTapeState( + liveSession.status, + composeTapeItems([], liveSession.classifierHits, liveSession.classifierHitsHistory), + liveSession.lastUpdate + ) : classifierHits; const smartMoneyFeed = mode === "live" - ? toStaticTapeState(liveSession.status, [...liveSession.smartMoney, ...liveSession.smartMoneyHistory], liveSession.lastUpdate) + ? toStaticTapeState( + liveSession.status, + composeTapeItems([], liveSession.smartMoney, liveSession.smartMoneyHistory), + liveSession.lastUpdate + ) : smartMoney; const inferredDarkFeed = mode === "live" - ? toStaticTapeState(liveSession.status, [...liveSession.inferredDark, ...liveSession.inferredDarkHistory], liveSession.lastUpdate) + ? toStaticTapeState( + liveSession.status, + composeTapeItems([], liveSession.inferredDark, liveSession.inferredDarkHistory), + liveSession.lastUpdate + ) : inferredDark; useLayoutEffect(() => { @@ -4722,16 +5399,6 @@ const useTerminalState = () => { return map; }, [optionsFeed.items]); - const equityPrintMap = useMemo(() => { - const map = new Map(); - for (const print of equitiesFeed.items) { - if (print.trace_id) { - map.set(print.trace_id, print); - } - } - return map; - }, [equitiesFeed.items]); - const equityJoinMap = useMemo(() => { const map = new Map(); for (const join of equityJoinsFeed.items) { @@ -4848,6 +5515,9 @@ const useTerminalState = () => { .then((payload: { data?: OptionPrint[] }) => { const next = new Map(); for (const item of payload.data ?? []) { + if (!item || !item.trace_id) { + continue; + } next.set(item.trace_id, item); } if (next.size > 0) { @@ -4895,6 +5565,9 @@ const useTerminalState = () => { .then((payload: { data?: EquityPrintJoin[] }) => { const next = new Map(); for (const item of payload.data ?? []) { + if (!item || !item.id || !item.trace_id) { + continue; + } next.set(item.id, item); next.set(item.trace_id, item); if (item.print_trace_id) { @@ -4953,11 +5626,11 @@ const useTerminalState = () => { }, [selectedDarkEvent, resolvedEquityJoinMap]); const selectedDarkUnderlying = useMemo(() => { - if (!selectedDarkEvent) { + if (!routeFeatures.needsDarkUnderlying || !selectedDarkEvent) { return null; } - return inferDarkUnderlying(selectedDarkEvent, equityPrintMap, resolvedEquityJoinMap); - }, [selectedDarkEvent, resolvedEquityJoinMap, equityPrintMap]); + return inferDarkUnderlying(selectedDarkEvent, resolvedEquityJoinMap); + }, [routeFeatures.needsDarkUnderlying, selectedDarkEvent, resolvedEquityJoinMap]); useEffect(() => { if (mode !== "live") { @@ -4994,6 +5667,9 @@ const useTerminalState = () => { }, []); const classifierHitsByPacketId = useMemo(() => { + if (!routeFeatures.needsClassifierDecor) { + return EMPTY_CLASSIFIER_HITS_BY_PACKET_ID; + } const map = new Map(); for (const hit of [...classifierHitsFeed.items, ...optionSupportClassifierHits]) { const packetId = extractPacketIdFromClassifierHitTrace(hit.trace_id); @@ -5003,9 +5679,17 @@ const useTerminalState = () => { map.set(packetId, [...(map.get(packetId) ?? []), hit]); } return map; - }, [classifierHitsFeed.items, optionSupportClassifierHits, extractPacketIdFromClassifierHitTrace]); + }, [ + classifierHitsFeed.items, + optionSupportClassifierHits, + extractPacketIdFromClassifierHitTrace, + routeFeatures.needsClassifierDecor + ]); const smartMoneyByPacketId = useMemo(() => { + if (!routeFeatures.needsClassifierDecor) { + return new Map(); + } const map = new Map(); for (const event of [...smartMoneyFeed.items, ...optionSupportSmartMoney]) { for (const packetId of event.packet_ids) { @@ -5016,9 +5700,12 @@ const useTerminalState = () => { } } return map; - }, [smartMoneyFeed.items, optionSupportSmartMoney]); + }, [smartMoneyFeed.items, optionSupportSmartMoney, routeFeatures.needsClassifierDecor]); const packetIdByOptionTraceId = useMemo(() => { + if (!routeFeatures.needsClassifierDecor) { + return EMPTY_PACKET_ID_BY_OPTION_TRACE_ID; + } const map = new Map(); for (const packet of resolvedFlowPacketMap.values()) { for (const member of packet.members) { @@ -5026,9 +5713,12 @@ const useTerminalState = () => { } } return map; - }, [resolvedFlowPacketMap]); + }, [resolvedFlowPacketMap, routeFeatures.needsClassifierDecor]); const classifierDecorByOptionTraceId = useMemo(() => { + if (!routeFeatures.needsClassifierDecor) { + return EMPTY_CLASSIFIER_DECOR_BY_OPTION_TRACE_ID; + } const map = new Map(); for (const [traceId, packetId] of packetIdByOptionTraceId) { const smartMoneyEvent = smartMoneyByPacketId.get(packetId); @@ -5042,10 +5732,15 @@ const useTerminalState = () => { } } return map; - }, [classifierHitsByPacketId, packetIdByOptionTraceId, smartMoneyByPacketId]); + }, [ + classifierHitsByPacketId, + packetIdByOptionTraceId, + smartMoneyByPacketId, + routeFeatures.needsClassifierDecor + ]); useEffect(() => { - if (mode !== "live" || optionsFeed.items.length === 0) { + if (!routeFeatures.needsClassifierDecor || mode !== "live" || optionsFeed.items.length === 0) { return; } @@ -5095,6 +5790,12 @@ const useTerminalState = () => { if (!response.ok) { throw new Error(await readErrorDetail(response)); } + const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""; + if (!contentType.includes("application/json")) { + throw new Error( + `Unexpected content type from /lookup/options-support: ${contentType || "unknown"}` + ); + } return response.json() as Promise<{ packets?: FlowPacket[]; smart_money?: SmartMoneyEvent[]; @@ -5109,19 +5810,28 @@ const useTerminalState = () => { const now = Date.now(); const packetMap = new Map(); for (const packet of payload.packets ?? []) { + if (!packet || !packet.id) { + continue; + } packetMap.set(packet.id, packet); } if (packetMap.size > 0) { setPinnedFlowPacketMap((prev) => upsertPinnedEntries(prev, packetMap, now)); } if (payload.smart_money?.length) { + const filtered = payload.smart_money.filter((item): item is SmartMoneyEvent => + Boolean(item && item.trace_id) + ); setOptionSupportSmartMoney((prev) => - mergeNewest(payload.smart_money ?? [], prev, PINNED_EVIDENCE_MAX_ITEMS) + mergeNewest(filtered, prev, PINNED_EVIDENCE_MAX_ITEMS) ); } if (payload.classifier_hits?.length) { + const filtered = payload.classifier_hits.filter((item): item is ClassifierHitEvent => + Boolean(item && item.trace_id) + ); setOptionSupportClassifierHits((prev) => - mergeNewest(payload.classifier_hits ?? [], prev, PINNED_EVIDENCE_MAX_ITEMS) + mergeNewest(filtered, prev, PINNED_EVIDENCE_MAX_ITEMS) ); } if (payload.nbbo_by_trace_id) { @@ -5146,7 +5856,8 @@ const useTerminalState = () => { optionsFeed.items, classifierDecorByOptionTraceId, packetIdByOptionTraceId, - historicalNbboByTraceId + historicalNbboByTraceId, + routeFeatures.needsClassifierDecor ]); const selectedClassifierPacketId = useMemo(() => { @@ -5287,6 +5998,9 @@ const useTerminalState = () => { .then((payload: { data?: OptionPrint[] }) => { const next = new Map(); for (const item of payload.data ?? []) { + if (!item || !item.trace_id) { + continue; + } next.set(item.trace_id, item); } if (next.size > 0) { @@ -5340,25 +6054,20 @@ const useTerminalState = () => { ); const filteredOptions = useMemo(() => { - return optionsFeed.items.filter((print) => { - if (!matchesOptionPrintFilters(print, flowFilters)) { - return false; - } - if ( - selectedInstrument?.kind === "option-contract" && - normalizeContractId(print.option_contract_id) !== selectedInstrument.contractId - ) { - return false; - } - if (tickerSet.size === 0) { - return ( - !instrumentUnderlying || - extractUnderlying(normalizeContractId(print.option_contract_id)) === instrumentUnderlying - ); - } - return matchesTicker(extractUnderlying(normalizeContractId(print.option_contract_id))); - }); - }, [flowFilters, optionsFeed.items, matchesTicker, tickerSet, selectedInstrument, instrumentUnderlying]); + return filterOptionTapeItems( + optionsFeed.items, + effectiveOptionPrintFilters, + selectedInstrument, + tickerSet, + instrumentUnderlying + ); + }, [ + effectiveOptionPrintFilters, + instrumentUnderlying, + optionsFeed.items, + selectedInstrument, + tickerSet + ]); const filteredEquities = useMemo(() => { if (tickerSet.size === 0) { @@ -5370,22 +6079,159 @@ const useTerminalState = () => { return equitiesFeed.items.filter((print) => matchesTicker(print.underlying_id)); }, [equitiesFeed.items, matchesTicker, tickerSet, instrumentUnderlying]); + useEffect(() => { + if (!optionFocusSeed) { + return; + } + if ( + shouldClearOptionFocusSeed( + optionFocusSeed, + optionFocusScopeKey, + currentOptionSubscriptionKey, + liveOptions.liveItems ?? [], + liveOptions.historyItems ?? [] + ) + ) { + setOptionFocusSeed(null); + } + }, [ + currentOptionSubscriptionKey, + liveOptions.historyItems, + liveOptions.liveItems, + optionFocusScopeKey, + optionFocusSeed + ]); + + useEffect(() => { + if (!equityFocusSeed) { + return; + } + if (equityFocusSeed.scopeKey !== equityFocusScopeKey) { + setEquityFocusSeed(null); + return; + } + const composedBaseItems = composeTapeItems([], liveEquities.liveItems ?? [], liveEquities.historyItems ?? []); + const liveKeys = new Set(composedBaseItems.map((item) => getTapeItemKey(item))); + if (equityFocusSeed.items.every((item) => liveKeys.has(getTapeItemKey(item)))) { + setEquityFocusSeed(null); + } + }, [equityFocusScopeKey, equityFocusSeed, liveEquities.historyItems, liveEquities.liveItems]); + + const focusOptionContract = useCallback( + (print: OptionPrint) => { + const contractId = normalizeContractId(print.option_contract_id); + const parsed = parseOptionContractId(contractId); + const underlyingId = (print.underlying_id ?? parsed?.root ?? extractUnderlying(contractId)).toUpperCase(); + const scopeKey = `option-contract:${contractId}`; + const subscriptionKey = getLiveSubscriptionKey({ + channel: "options", + underlying_ids: [underlyingId], + option_contract_id: contractId + }); + const seedItems = composeTapeItems( + [print], + filteredOptions.filter((candidate) => normalizeContractId(candidate.option_contract_id) === contractId), + [] + ); + setOptionFocusSeed({ scopeKey, subscriptionKey, items: seedItems }); + bumpTapeDebugMetric("focusSeedRowCount", seedItems.length); + logTapeDebug("option focus seed captured", { + contract_id: contractId, + subscription_key: subscriptionKey, + row_count: seedItems.length + }); + setSelectedInstrument({ + kind: "option-contract", + contractId, + underlyingId + }); + }, + [filteredOptions] + ); + + const focusEquityTicker = useCallback( + (print: EquityPrint) => { + const underlyingId = print.underlying_id.toUpperCase(); + const scopeKey = `equity:${underlyingId}`; + const seedItems = composeTapeItems( + [print], + filteredEquities.filter((candidate) => candidate.underlying_id.toUpperCase() === underlyingId), + [] + ); + setEquityFocusSeed({ scopeKey, items: seedItems }); + bumpTapeDebugMetric("focusSeedRowCount", seedItems.length); + logTapeDebug("equity focus seed captured", { + underlying_id: underlyingId, + row_count: seedItems.length + }); + setSelectedInstrument({ + kind: "equity", + underlyingId + }); + }, + [filteredEquities] + ); + const equitiesSilentWarning = shouldShowEquitiesSilentFeedWarning({ wsStatus: liveSession.status, equitiesSubscribed: mode === "live" && equitiesLiveSubscriptionActive, connectedAt: liveSession.connectedAt, lastEquitiesEventAt: liveSession.lastEventByChannel.equities ?? null }); + const optionsScopeActive = Boolean( + optionScope.option_contract_id || optionScope.underlying_ids?.length + ); + const equitiesScopeActive = Boolean(equityScope.underlying_ids?.length); + const optionsScopedQuiet = + mode === "live" && + optionsScopeActive && + optionsChannelStatus === "connected" && + filteredOptions.length === 0; + const equitiesScopedQuiet = + mode === "live" && + equitiesScopeActive && + equitiesChannelStatus === "connected" && + filteredEquities.length === 0; + + const previousScopedQuietRef = useRef({ + options: optionsScopedQuiet, + equities: equitiesScopedQuiet + }); + + useEffect(() => { + const previous = previousScopedQuietRef.current; + if (previous.options !== optionsScopedQuiet) { + bumpTapeDebugMetric("scopedQuietTransitions", 1); + logTapeDebug("options scoped quiet transition", { active: optionsScopedQuiet }); + } + if (previous.equities !== equitiesScopedQuiet) { + bumpTapeDebugMetric("scopedQuietTransitions", 1); + logTapeDebug("equities scoped quiet transition", { active: equitiesScopedQuiet }); + } + previousScopedQuietRef.current = { + options: optionsScopedQuiet, + equities: equitiesScopedQuiet + }; + }, [equitiesScopedQuiet, optionsScopedQuiet]); const filteredInferredDark = useMemo(() => { + if (!routeFeatures.inferredDark) { + return EMPTY_INFERRED_DARK_EVENTS; + } if (tickerSet.size === 0) { return inferredDarkFeed.items; } return inferredDarkFeed.items.filter((event) => { - const underlying = inferDarkUnderlying(event, equityPrintMap, resolvedEquityJoinMap); + const underlying = inferDarkUnderlying(event, resolvedEquityJoinMap); return matchesTicker(underlying); }); - }, [resolvedEquityJoinMap, equityPrintMap, inferredDarkFeed.items, matchesTicker, tickerSet]); + }, [ + resolvedEquityJoinMap, + inferredDarkFeed.items, + matchesTicker, + tickerSet, + routeFeatures.inferredDark + ]); const filteredFlow = useMemo(() => { return flowFeed.items.filter((packet) => { @@ -5400,13 +6246,31 @@ const useTerminalState = () => { }, [flowFeed.items, flowFilters, extractPacketContract, matchesTicker, tickerSet]); const filteredAlerts = useMemo(() => { + if (!routeFeatures.showAlertsPane && !routeFeatures.needsAlertEvidencePrefetch) { + return EMPTY_ALERT_EVENTS; + } if (tickerSet.size === 0) { return alertsFeed.items; } return alertsFeed.items.filter((alert) => matchesTicker(inferAlertUnderlying(alert))); - }, [alertsFeed.items, inferAlertUnderlying, matchesTicker, tickerSet]); + }, [ + alertsFeed.items, + inferAlertUnderlying, + matchesTicker, + tickerSet, + routeFeatures.showAlertsPane, + routeFeatures.needsAlertEvidencePrefetch + ]); - const visibleAlerts = useMemo(() => filteredAlerts.slice(0, 12), [filteredAlerts]); + const visibleAlerts = useMemo(() => { + if (routeFeatures.needsAlertEvidencePrefetch) { + return filteredAlerts.slice(0, 12); + } + if (routeFeatures.showAlertsPane) { + return filteredAlerts.slice(0, 12); + } + return EMPTY_ALERT_EVENTS; + }, [filteredAlerts, routeFeatures.needsAlertEvidencePrefetch, routeFeatures.showAlertsPane]); const visibleAlertEvidenceRefs = useMemo(() => { const refs = new Set(); @@ -5419,7 +6283,7 @@ const useTerminalState = () => { }, [visibleAlerts]); useEffect(() => { - if (mode !== "live" || visibleAlerts.length === 0) { + if (!routeFeatures.needsAlertEvidencePrefetch || mode !== "live" || visibleAlerts.length === 0) { return; } @@ -5482,6 +6346,9 @@ const useTerminalState = () => { .then((payload: { data?: OptionPrint[] }) => { const next = new Map(); for (const item of payload.data ?? []) { + if (!item || !item.trace_id) { + continue; + } next.set(item.trace_id, item); } if (next.size > 0) { @@ -5498,7 +6365,8 @@ const useTerminalState = () => { visibleAlerts, visibleAlertEvidenceRefs, resolvedFlowPacketMap, - resolvedOptionPrintMap + resolvedOptionPrintMap, + routeFeatures.needsAlertEvidencePrefetch ]); const activePinnedFlowKeys = useMemo(() => { @@ -5584,6 +6452,9 @@ const useTerminalState = () => { }, []); const filteredClassifierHits = useMemo(() => { + if (!routeFeatures.classifierHits) { + return EMPTY_CLASSIFIER_HIT_EVENTS; + } if (tickerSet.size === 0) { return classifierHitsFeed.items; } @@ -5591,16 +6462,28 @@ const useTerminalState = () => { const underlying = extractUnderlyingFromTrace(hit.trace_id); return matchesTicker(underlying); }); - }, [classifierHitsFeed.items, extractUnderlyingFromTrace, matchesTicker, tickerSet]); + }, [ + classifierHitsFeed.items, + extractUnderlyingFromTrace, + matchesTicker, + tickerSet, + routeFeatures.classifierHits + ]); const filteredSmartMoneyEvents = useMemo(() => { + if (!routeFeatures.smartMoney) { + return EMPTY_SMART_MONEY_EVENTS; + } if (tickerSet.size === 0) { return smartMoneyFeed.items; } return smartMoneyFeed.items.filter((event) => matchesTicker(event.underlying_id)); - }, [matchesTicker, smartMoneyFeed.items, tickerSet]); + }, [matchesTicker, smartMoneyFeed.items, tickerSet, routeFeatures.smartMoney]); const chartSmartMoneyEvents = useMemo(() => { + if (!routeFeatures.showChartPane && !routeFeatures.showFocusPane) { + return EMPTY_SMART_MONEY_EVENTS; + } const desired = chartTicker.toUpperCase(); return smartMoneyFeed.items .filter((event) => event.underlying_id.toUpperCase() === desired) @@ -5611,12 +6494,15 @@ const useTerminalState = () => { } return a.seq - b.seq; }); - }, [chartTicker, smartMoneyFeed.items]); + }, [chartTicker, smartMoneyFeed.items, routeFeatures.showChartPane, routeFeatures.showFocusPane]); const chartInferredDark = useMemo(() => { + if (!routeFeatures.showChartPane && !routeFeatures.showFocusPane) { + return EMPTY_INFERRED_DARK_EVENTS; + } const desired = chartTicker.toUpperCase(); return inferredDarkFeed.items - .filter((event) => inferDarkUnderlying(event, equityPrintMap, resolvedEquityJoinMap) === desired) + .filter((event) => inferDarkUnderlying(event, resolvedEquityJoinMap) === desired) .sort((a, b) => { const delta = a.source_ts - b.source_ts; if (delta !== 0) { @@ -5624,7 +6510,13 @@ const useTerminalState = () => { } return a.seq - b.seq; }); - }, [chartTicker, inferredDarkFeed.items, resolvedEquityJoinMap, equityPrintMap]); + }, [ + chartTicker, + inferredDarkFeed.items, + resolvedEquityJoinMap, + routeFeatures.showChartPane, + routeFeatures.showFocusPane + ]); const findAlertForClassifierHit = useCallback( (hit: ClassifierHitEvent): AlertEvent | null => { @@ -5684,18 +6576,47 @@ const useTerminalState = () => { }, []); const lastSeen = useMemo(() => { - return [ - optionsFeed.lastUpdate, - equitiesFeed.lastUpdate, - inferredDarkFeed.lastUpdate, - flowFeed.lastUpdate, - alertsFeed.lastUpdate, - smartMoneyFeed.lastUpdate, - classifierHitsFeed.lastUpdate - ] + const updates: Array = []; + if (routeFeatures.options || routeFeatures.showOptionsPane) { + updates.push(optionsFeed.lastUpdate); + } + if (routeFeatures.equities || routeFeatures.showEquitiesPane) { + updates.push(equitiesFeed.lastUpdate); + } + if (routeFeatures.inferredDark || routeFeatures.showDarkPane || routeFeatures.showFocusPane) { + updates.push(inferredDarkFeed.lastUpdate); + } + if (routeFeatures.flow || routeFeatures.showFlowPane) { + updates.push(flowFeed.lastUpdate); + } + if (routeFeatures.alerts || routeFeatures.showAlertsPane) { + updates.push(alertsFeed.lastUpdate); + } + if (routeFeatures.smartMoney || routeFeatures.showClassifierPane || routeFeatures.showChartPane || routeFeatures.showFocusPane) { + updates.push(smartMoneyFeed.lastUpdate); + } + if (routeFeatures.classifierHits || routeFeatures.showClassifierPane) { + updates.push(classifierHitsFeed.lastUpdate); + } + return updates .filter((value): value is number => value !== null) .sort((a, b) => b - a)[0] ?? null; }, [ + routeFeatures.options, + routeFeatures.showOptionsPane, + routeFeatures.equities, + routeFeatures.showEquitiesPane, + routeFeatures.inferredDark, + routeFeatures.showDarkPane, + routeFeatures.showFocusPane, + routeFeatures.flow, + routeFeatures.showFlowPane, + routeFeatures.alerts, + routeFeatures.showAlertsPane, + routeFeatures.smartMoney, + routeFeatures.showClassifierPane, + routeFeatures.showChartPane, + routeFeatures.classifierHits, optionsFeed.lastUpdate, equitiesFeed.lastUpdate, inferredDarkFeed.lastUpdate, @@ -5743,13 +6664,13 @@ const useTerminalState = () => { smartMoney: smartMoneyFeed, classifierHits: classifierHitsFeed, liveSession, + routeFeatures, activeTickers, tickerSet, chartTicker, nbboMap, historicalNbboByTraceId, optionPrintMap: resolvedOptionPrintMap, - equityPrintMap, equityJoinMap: resolvedEquityJoinMap, flowPacketMap: resolvedFlowPacketMap, classifierHitsByPacketId, @@ -5766,6 +6687,8 @@ const useTerminalState = () => { selectedSmartMoneyEvidence, filteredOptions, filteredEquities, + optionsScopedQuiet, + equitiesScopedQuiet, equitiesSilentWarning, filteredInferredDark, filteredFlow, @@ -5774,6 +6697,8 @@ const useTerminalState = () => { filteredClassifierHits, chartSmartMoneyEvents, chartInferredDark, + focusOptionContract, + focusEquityTicker, openFromSmartMoneyEvent, openFromClassifierHit, handleSmartMoneyMarkerClick, @@ -6009,36 +6934,6 @@ export const FlowFilterPopover = ({ filters, onChange }: FlowFilterPopoverProps) ); }; -const FlowFilterControls = () => { - const state = useTerminal(); - - return ; -}; - -const ContractFilterControl = () => { - const state = useTerminal(); - const selected = state.selectedInstrument; - const isContractFilterActive = selected?.kind === "option-contract"; - - return ( - - ); -}; - type PaneProps = { title: string; status?: ReactNode; @@ -6093,14 +6988,14 @@ const ShellMetricStrip = () => { }; type OptionsPaneProps = { + state: TerminalState; limit?: number; }; -const OptionsPane = ({ limit }: OptionsPaneProps) => { - const state = useTerminal(); +const OptionsPane = memo(({ state, limit }: OptionsPaneProps) => { const items = limit ? state.filteredOptions.slice(0, limit) : state.filteredOptions; - const virtual = useVirtualList(items, state.optionsScroll.listRef, !limit, 36); - useBottomHistoryGate(state.optionsScroll.listRef, state.mode === "live" && !limit, () => + const virtual = useTapeVirtualList(items, state.optionsScroll.listRef, getTapeVirtualConfig("options")); + useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("options") ); @@ -6131,16 +7026,20 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => {
{items.length === 0 ? (
- {state.tickerSet.size > 0 - ? "No option prints match the current filter." - : state.mode === "live" - ? state.options.status === "stale" - ? "Feed behind. Waiting for fresh option prints." - : "No option prints yet. Start ingest-options." + {state.mode === "live" + ? state.options.status === "stale" + ? "Feed behind. Waiting for fresh option prints." + : state.optionsScopedQuiet + ? "No recent option prints for this scope yet." + : state.tickerSet.size > 0 + ? "No option prints match the current filter." + : "No option prints yet. Start ingest-options." + : state.tickerSet.size > 0 + ? "No option prints match the current filter." : "Replay queue empty. Ensure ClickHouse has data."}
) : ( -
+
TIME @@ -6156,128 +7055,135 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { IV CLASSIFIER
- {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {virtual.visibleItems.map((print) => { - const contractId = normalizeContractId(print.option_contract_id); - const parsed = parseOptionContractId(contractId); - const contractDisplay = formatOptionContractLabel(contractId); - const quote = state.historicalNbboByTraceId.get(print.trace_id) ?? state.nbboMap.get(contractId); - const hasPreservedNbbo = typeof print.execution_nbbo_side === "string"; - const nbboSide = - print.execution_nbbo_side ?? - print.nbbo_side ?? - (!hasPreservedNbbo ? classifyNbboSide(print.price, quote) : null); - const notional = print.notional ?? print.price * print.size * 100; - const spot = print.execution_underlying_spot; - const iv = print.execution_iv; - const decor = state.classifierDecorByOptionTraceId.get(print.trace_id); - const underlyingId = (print.underlying_id ?? parsed?.root ?? extractUnderlying(contractId)).toUpperCase(); - const focusContract = (event: ReactMouseEvent) => { - event.stopPropagation(); - state.setSelectedInstrument({ - kind: "option-contract", - contractId, - underlyingId - }); - }; - const commonProps = { - className: `data-table-row data-table-row-button data-table-row-classified data-table-row-options${decor ? ` is-classified classifier-${decor.tone}` : ""}`, - style: decor ? ({ "--classifier-intensity": decor.intensity } as CSSProperties) : undefined - }; - const cells = ( - <> - {formatTime(print.ts)} - - - - - - - - - - - - - {typeof spot === "number" ? formatPrice(spot) : "--"} - - {formatSize(print.size)}@{formatPrice(print.price)}_{nbboSide ?? "--"} - - {print.option_type ?? "--"} - ${formatCompactUsd(notional)} - - {nbboSide ? ( - {nbboSide} - ) : ( - "--" - )} - - {typeof iv === "number" ? formatPct(iv) : "--"} - {decor ? humanizeClassifierId(decor.family) : "--"} - - ); - - return decor ? ( - - ) : ( -
- {cells} + {virtual.virtualItems.map(({ item: print, key, index, start, size }) => { + const contractId = normalizeContractId(print.option_contract_id); + const parsed = parseOptionContractId(contractId); + const contractDisplay = formatOptionContractLabel(contractId); + const quote = state.historicalNbboByTraceId.get(print.trace_id) ?? state.nbboMap.get(contractId); + const hasPreservedNbbo = typeof print.execution_nbbo_side === "string"; + const nbboSide = + print.execution_nbbo_side ?? + print.nbbo_side ?? + (!hasPreservedNbbo ? classifyNbboSide(print.price, quote) : null); + const notional = print.notional ?? print.price * print.size * 100; + const spot = print.execution_underlying_spot; + const iv = print.execution_iv; + const decor = state.classifierDecorByOptionTraceId.get(print.trace_id); + const focusContract = (event: ReactMouseEvent) => { + event.stopPropagation(); + state.focusOptionContract(print); + }; + const rowStyle = { + ...(decor + ? ({ "--classifier-intensity": decor.intensity } as CSSProperties) + : undefined), + transform: `translateY(${start}px)` + } as CSSProperties; + const commonProps = { + className: `data-table-row data-table-row-button data-table-row-classified data-table-row-options data-table-virtual-row${index % 2 === 1 ? " is-even" : ""}${decor ? ` is-classified classifier-${decor.tone}` : ""}`, + style: rowStyle, + "data-index": index, + "data-row-start": String(start), + "data-row-size": String(size), + "data-tape-key": key + }; + const cells = ( + <> + {formatTime(print.ts)} + + + + + + + + + + + + + {typeof spot === "number" ? formatPrice(spot) : "--"} + + {formatSize(print.size)}@{formatPrice(print.price)}_{nbboSide ?? "--"} + + {print.option_type ?? "--"} + ${formatCompactUsd(notional)} + + {nbboSide ? ( + {nbboSide} + ) : ( + "--" + )} + + {typeof iv === "number" ? formatPct(iv) : "--"} + {decor ? humanizeClassifierId(decor.family) : "--"} + + ); + + return decor ? ( + + ) : ( +
+ {cells} +
+ ); + })}
- ); - })} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null} +
)}
); -}; +}); type EquitiesPaneProps = { + state: TerminalState; limit?: number; }; -const EquitiesPane = ({ limit }: EquitiesPaneProps) => { - const state = useTerminal(); +const EquitiesPane = memo(({ state, limit }: EquitiesPaneProps) => { const items = limit ? state.filteredEquities.slice(0, limit) : state.filteredEquities; - const virtual = useVirtualList(items, state.equitiesScroll.listRef, !limit, 36); - useBottomHistoryGate(state.equitiesScroll.listRef, state.mode === "live" && !limit, () => + const virtual = useTapeVirtualList(items, state.equitiesScroll.listRef, getTapeVirtualConfig("equities")); + useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("equities") ); @@ -6308,18 +7214,22 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => {
{items.length === 0 ? (
- {state.tickerSet.size > 0 - ? "No equity prints match the current filter." - : state.mode === "live" - ? state.equitiesSilentWarning - ? "Connected but no equity prints received. Check ingest-equities." - : state.equities.status === "stale" - ? "Feed behind. Waiting for fresh equity prints." - : "No equity prints yet. Start ingest-equities." + {state.mode === "live" + ? state.equities.status === "stale" + ? "Feed behind. Waiting for fresh equity prints." + : state.equitiesScopedQuiet + ? "No recent equity prints for this scope yet." + : state.tickerSet.size > 0 + ? "No equity prints match the current filter." + : state.equitiesSilentWarning + ? "Connected but no equity prints received. Check ingest-equities." + : "No equity prints yet. Start ingest-equities." + : state.tickerSet.size > 0 + ? "No equity prints match the current filter." : "Replay queue empty. Ensure ClickHouse has data."}
) : ( -
+
TIME @@ -6329,53 +7239,54 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => { VENUE TAPE
- {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {virtual.visibleItems.map((print) => ( -
- {formatTime(print.ts)} - - - - ${formatPrice(print.price)} - {formatSize(print.size)}x - {print.exchange} - {print.offExchangeFlag ? "Off-Ex" : "Lit"} +
+
+ {virtual.virtualItems.map(({ item: print, key, index, start, size }) => ( +
+ {formatTime(print.ts)} + + + + ${formatPrice(print.price)} + {formatSize(print.size)}x + {print.exchange} + {print.offExchangeFlag ? "Off-Ex" : "Lit"} +
+ ))} +
- ))} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null}
)}
); -}; +}); type FlowPaneProps = { + state: TerminalState; limit?: number; title?: string; }; -const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { - const state = useTerminal(); +const FlowPane = memo(({ state, limit, title = "Flow" }: FlowPaneProps) => { const items = limit ? state.filteredFlow.slice(0, limit) : state.filteredFlow; - const virtual = useVirtualList(items, state.flowScroll.listRef, !limit, 44); - useBottomHistoryGate(state.flowScroll.listRef, state.mode === "live" && !limit, () => + const virtual = useTapeVirtualList(items, state.flowScroll.listRef, getTapeVirtualConfig("flow")); + useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("flow") ); @@ -6415,7 +7326,7 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."}
) : ( -
+
TIME @@ -6428,96 +7339,102 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { NBBO QUALITY
- {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {virtual.visibleItems.map((packet) => { - const features = packet.features ?? {}; - const contract = String(features.option_contract_id ?? packet.id ?? "unknown"); - const count = parseNumber(features.count, packet.members.length); - const totalSize = parseNumber(features.total_size, 0); - const totalNotional = parseNumber(features.total_notional, Number.NaN); - const notional = Number.isFinite(totalNotional) - ? totalNotional - : parseNumber(features.total_premium, 0) * 100; - const startTs = parseNumber(features.start_ts, packet.source_ts); - const endTs = parseNumber(features.end_ts, startTs); - const windowMs = parseNumber(features.window_ms, 0); - const structureType = - typeof features.structure_type === "string" ? features.structure_type : ""; - const structureLegs = parseNumber(features.structure_legs, 0); - const structureRights = - typeof features.structure_rights === "string" ? features.structure_rights : ""; - const structureStrikes = parseNumber(features.structure_strikes, 0); - const nbboBid = parseNumber(features.nbbo_bid, Number.NaN); - const nbboAsk = parseNumber(features.nbbo_ask, Number.NaN); - const nbboMid = parseNumber(features.nbbo_mid, Number.NaN); - const nbboSpread = parseNumber(features.nbbo_spread, Number.NaN); - const aggressiveBuyRatio = parseNumber(features.nbbo_aggressive_buy_ratio, Number.NaN); - const aggressiveSellRatio = parseNumber( - features.nbbo_aggressive_sell_ratio, - Number.NaN - ); - const aggressiveCoverage = parseNumber(features.nbbo_coverage_ratio, Number.NaN); - const insideRatio = parseNumber(features.nbbo_inside_ratio, Number.NaN); - const nbboAge = parseNumber(packet.join_quality.nbbo_age_ms, Number.NaN); - const nbboStale = parseNumber(packet.join_quality.nbbo_stale, 0) > 0; - const nbboMissing = parseNumber(packet.join_quality.nbbo_missing, 0) > 0; - const structureLabel = structureType - ? `${structureType.replace(/_/g, " ")}${structureRights ? ` ${structureRights}` : ""}${structureLegs > 0 ? ` ${structureLegs}L` : ""}${structureStrikes > 0 ? ` ${structureStrikes}K` : ""}` - : "--"; - const nbboLabel = Number.isFinite(nbboBid) && Number.isFinite(nbboAsk) - ? `${formatPrice(nbboBid)} x ${formatPrice(nbboAsk)}` - : Number.isFinite(nbboMid) - ? `Mid ${formatPrice(nbboMid)}` - : "--"; - const qualityLabel = [ - Number.isFinite(aggressiveCoverage) && aggressiveCoverage > 0 - ? `Agg ${formatPct(aggressiveBuyRatio)}/${formatPct(aggressiveSellRatio)} ${formatPct(aggressiveCoverage)} cov` - : null, - Number.isFinite(insideRatio) && insideRatio > 0 ? `In ${formatPct(insideRatio)}` : null, - Number.isFinite(nbboSpread) ? `Spr ${formatPrice(nbboSpread)}` : null, - Number.isFinite(nbboAge) ? `${Math.round(nbboAge)}ms` : null, - nbboStale ? "Stale" : null, - nbboMissing ? "Missing" : null - ].filter(Boolean).join(" | "); +
+
+ {virtual.virtualItems.map(({ item: packet, key, index, start, size }) => { + const features = packet.features ?? {}; + const contract = String(features.option_contract_id ?? packet.id ?? "unknown"); + const count = parseNumber(features.count, packet.members.length); + const totalSize = parseNumber(features.total_size, 0); + const totalNotional = parseNumber(features.total_notional, Number.NaN); + const notional = Number.isFinite(totalNotional) + ? totalNotional + : parseNumber(features.total_premium, 0) * 100; + const startTs = parseNumber(features.start_ts, packet.source_ts); + const endTs = parseNumber(features.end_ts, startTs); + const windowMs = parseNumber(features.window_ms, 0); + const structureType = + typeof features.structure_type === "string" ? features.structure_type : ""; + const structureLegs = parseNumber(features.structure_legs, 0); + const structureRights = + typeof features.structure_rights === "string" ? features.structure_rights : ""; + const structureStrikes = parseNumber(features.structure_strikes, 0); + const nbboBid = parseNumber(features.nbbo_bid, Number.NaN); + const nbboAsk = parseNumber(features.nbbo_ask, Number.NaN); + const nbboMid = parseNumber(features.nbbo_mid, Number.NaN); + const nbboSpread = parseNumber(features.nbbo_spread, Number.NaN); + const aggressiveBuyRatio = parseNumber(features.nbbo_aggressive_buy_ratio, Number.NaN); + const aggressiveSellRatio = parseNumber( + features.nbbo_aggressive_sell_ratio, + Number.NaN + ); + const aggressiveCoverage = parseNumber(features.nbbo_coverage_ratio, Number.NaN); + const insideRatio = parseNumber(features.nbbo_inside_ratio, Number.NaN); + const nbboAge = parseNumber(packet.join_quality.nbbo_age_ms, Number.NaN); + const nbboStale = parseNumber(packet.join_quality.nbbo_stale, 0) > 0; + const nbboMissing = parseNumber(packet.join_quality.nbbo_missing, 0) > 0; + const structureLabel = structureType + ? `${structureType.replace(/_/g, " ")}${structureRights ? ` ${structureRights}` : ""}${structureLegs > 0 ? ` ${structureLegs}L` : ""}${structureStrikes > 0 ? ` ${structureStrikes}K` : ""}` + : "--"; + const nbboLabel = Number.isFinite(nbboBid) && Number.isFinite(nbboAsk) + ? `${formatPrice(nbboBid)} x ${formatPrice(nbboAsk)}` + : Number.isFinite(nbboMid) + ? `Mid ${formatPrice(nbboMid)}` + : "--"; + const qualityLabel = [ + Number.isFinite(aggressiveCoverage) && aggressiveCoverage > 0 + ? `Agg ${formatPct(aggressiveBuyRatio)}/${formatPct(aggressiveSellRatio)} ${formatPct(aggressiveCoverage)} cov` + : null, + Number.isFinite(insideRatio) && insideRatio > 0 ? `In ${formatPct(insideRatio)}` : null, + Number.isFinite(nbboSpread) ? `Spr ${formatPrice(nbboSpread)}` : null, + Number.isFinite(nbboAge) ? `${Math.round(nbboAge)}ms` : null, + nbboStale ? "Stale" : null, + nbboMissing ? "Missing" : null + ].filter(Boolean).join(" | "); - return ( -
- {formatTime(startTs)} → {formatTime(endTs)} - {contract} - {formatFlowMetric(count)} - {formatFlowMetric(totalSize)} - ${formatUsd(notional)} - {windowMs > 0 ? formatFlowMetric(windowMs, "ms") : "--"} - {structureLabel} - {nbboLabel} - {qualityLabel || "--"} + return ( +
+ {formatTime(startTs)} → {formatTime(endTs)} + {contract} + {formatFlowMetric(count)} + {formatFlowMetric(totalSize)} + ${formatUsd(notional)} + {windowMs > 0 ? formatFlowMetric(windowMs, "ms") : "--"} + {structureLabel} + {nbboLabel} + {qualityLabel || "--"} +
+ ); + })} +
- ); - })} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null}
)}
); -}; +}); type AlertsPaneProps = { + state: TerminalState; limit?: number; withStrip?: boolean; className?: string; }; -const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => { - const state = useTerminal(); +const AlertsPane = memo(({ state, limit, withStrip = false, className }: AlertsPaneProps) => { const items = limit ? state.filteredAlerts.slice(0, limit) : state.filteredAlerts; - const virtual = useVirtualList(items, state.alertsScroll.listRef, !limit, 46); - useBottomHistoryGate(state.alertsScroll.listRef, state.mode === "live" && !limit, () => + const virtual = useTapeVirtualList(items, state.alertsScroll.listRef, getTapeVirtualConfig("alerts")); + useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("alerts") ); @@ -6557,7 +7474,7 @@ const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => : "Replay queue empty. Ensure ClickHouse has data."}
) : ( -
+
TIME @@ -6568,58 +7485,57 @@ const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => DIR NOTE
- {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {virtual.visibleItems.map((alert) => { - const primary = alert.hits[0]; - const direction = deriveAlertDirection(alert); - const severity = normalizeAlertSeverity(alert); +
+
+ {virtual.virtualItems.map(({ item: alert, key, index, start, size }) => { + const primary = alert.hits[0]; + const direction = deriveAlertDirection(alert); + const severity = normalizeAlertSeverity(alert); - return ( - - ); - })} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null} + return ( + + ); + })} +
+
)}
); -}; +}); type ClassifierPaneProps = { + state: TerminalState; limit?: number; className?: string; }; -const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { - const state = useTerminal(); - useBottomHistoryGate(state.classifierScroll.listRef, state.mode === "live" && !limit, () => { - void state.liveSession.loadOlder("smart-money"); - void state.liveSession.loadOlder("classifier-hits"); - }); +const ClassifierPane = memo(({ state, limit, className }: ClassifierPaneProps) => { const smartMoneyItems = limit ? state.filteredSmartMoneyEvents.slice(0, limit) : state.filteredSmartMoneyEvents; const legacyItems = smartMoneyItems.length === 0 @@ -6629,12 +7545,11 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { : []; const items: Array = smartMoneyItems.length > 0 ? smartMoneyItems : legacyItems; - const virtual = useVirtualList( - items, - state.classifierScroll.listRef, - !limit, - 44 - ); + const virtual = useTapeVirtualList(items, state.classifierScroll.listRef, getTapeVirtualConfig("classifier")); + useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => { + void state.liveSession.loadOlder("smart-money"); + void state.liveSession.loadOlder("classifier-hits"); + }); const showingSmartMoney = smartMoneyItems.length > 0; return ( @@ -6672,7 +7587,7 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."}
) : ( -
+
TIME @@ -6681,72 +7596,82 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { PROB NOTE
- {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {showingSmartMoney ? (virtual.visibleItems as SmartMoneyEvent[]).map((event) => { - const primaryScore = - event.profile_scores.find((score) => score.profile_id === event.primary_profile_id) ?? - event.profile_scores[0]; - const direction = normalizeDirection(event.primary_direction); - return ( - - ); - }) : (virtual.visibleItems as ClassifierHitEvent[]).map((hit) => { - const direction = normalizeDirection(hit.direction); - return ( - - ); - })} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null} +
+
+ {showingSmartMoney ? virtual.virtualItems.map(({ item, key, index, start, size }) => { + const event = item as SmartMoneyEvent; + const primaryScore = + event.profile_scores.find((score) => score.profile_id === event.primary_profile_id) ?? + event.profile_scores[0]; + const direction = normalizeDirection(event.primary_direction); + return ( + + ); + }) : virtual.virtualItems.map(({ item, key, index, start, size }) => { + const hit = item as ClassifierHitEvent; + const direction = normalizeDirection(hit.direction); + return ( + + ); + })} +
+
)}
); -}; +}); type DarkPaneProps = { + state: TerminalState; limit?: number; className?: string; }; -const DarkPane = ({ limit, className }: DarkPaneProps) => { - const state = useTerminal(); +const DarkPane = memo(({ state, limit, className }: DarkPaneProps) => { const items = limit ? state.filteredInferredDark.slice(0, limit) : state.filteredInferredDark; - const virtual = useVirtualList(items, state.darkScroll.listRef, !limit, 44); - useBottomHistoryGate(state.darkScroll.listRef, state.mode === "live" && !limit, () => + const virtual = useTapeVirtualList(items, state.darkScroll.listRef, getTapeVirtualConfig("dark")); + useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () => void state.liveSession.loadOlder("inferred-dark") ); @@ -6785,7 +7710,7 @@ const DarkPane = ({ limit, className }: DarkPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."}
) : ( -
+
TIME @@ -6795,51 +7720,54 @@ const DarkPane = ({ limit, className }: DarkPaneProps) => { EVIDENCE NOTE
- {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {virtual.visibleItems.map((event) => { - const underlying = inferDarkUnderlying(event, state.equityPrintMap, state.equityJoinMap); - const evidenceCount = event.evidence_refs.length; +
+
+ {virtual.virtualItems.map(({ item: event, key, index, start, size }) => { + const underlying = inferDarkUnderlying(event, state.equityJoinMap); + const evidenceCount = event.evidence_refs.length; - return ( - - ); - })} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null} + return ( + + ); + })} +
+
)}
); -}; +}); type ChartPaneProps = { + state: TerminalState; title?: string; }; -const ChartPane = ({ title = "Chart" }: ChartPaneProps) => { - const state = useTerminal(); +const ChartPane = memo(({ state, title = "Chart" }: ChartPaneProps) => { return ( { /> ); -}; +}); -const FocusPane = () => { - const state = useTerminal(); +const FocusPane = memo(({ state }: { state: TerminalState }) => { const hits = state.chartSmartMoneyEvents.slice(-10).reverse(); const dark = state.chartInferredDark.slice(-10).reverse(); @@ -6945,10 +7872,9 @@ const FocusPane = () => {
); -}; +}); -const ReplayConsole = () => { - const state = useTerminal(); +const ReplayConsole = memo(({ state }: { state: TerminalState }) => { const replayActive = state.mode === "replay"; return ( @@ -6980,7 +7906,7 @@ const ReplayConsole = () => {
); -}; +}); export function TerminalAppShell({ children }: { children: ReactNode }) { const state = useTerminalState(); @@ -7100,68 +8026,89 @@ export function TerminalAppShell({ children }: { children: ReactNode }) { } export function OverviewRoute() { + const state = useTerminal(); return (
- - - + + +
); } export function TapeRoute() { + const state = useTerminal(); return ( - - + + } >
- - - + + +
); } export function SignalsRoute() { + const state = useTerminal(); return (
- - - + + +
); } export function ChartsRoute() { + const state = useTerminal(); return (
- - + +
); } export function ReplayRoute() { + const state = useTerminal(); return (
- - - - + + + +
); diff --git a/apps/web/package.json b/apps/web/package.json index b61eb2e..8ab6906 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -9,6 +9,7 @@ }, "dependencies": { "@islandflow/types": "workspace:*", + "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", "next": "^14.2.4", "react": "^18.3.1", diff --git a/bun.lock b/bun.lock index de67cb2..47fc572 100644 --- a/bun.lock +++ b/bun.lock @@ -12,6 +12,7 @@ "name": "@islandflow/web", "dependencies": { "@islandflow/types": "workspace:*", + "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", "next": "^14.2.4", "react": "^18.3.1", @@ -208,6 +209,10 @@ "@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="], + "@tanstack/react-virtual": ["@tanstack/react-virtual@3.13.24", "", { "dependencies": { "@tanstack/virtual-core": "3.14.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aIJvz5OSkhNIhZIpYivrxrPTKYsjW9Uzy+sP/mx0S3sev2HyvPb7xmjbYvokzEpfgYHy/HjzJ2zFAETuUfgCpg=="], + + "@tanstack/virtual-core": ["@tanstack/virtual-core@3.14.0", "", {}, "sha512-JLANqGy/D6k4Ujmh8Tr25lGimuOXNiaVyXaCAZS0W+1390sADdGnyUdSWNIfd49gebtIxGMij4IktRVzrdr12Q=="], + "@types/node": ["@types/node@20.19.27", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-N2clP5pJhB2YnZJ3PIHFk5RkygRX5WO/5f0WC08tp0wd+sv0rsJk3MqWn3CbNmT2J505a5336jaQj4ph1AdMug=="], "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], diff --git a/deployment/docker/.env.example b/deployment/docker/.env.example index 2512e0e..986968c 100644 --- a/deployment/docker/.env.example +++ b/deployment/docker/.env.example @@ -98,6 +98,7 @@ CLASSIFIER_0DTE_MIN_PREMIUM=20000 CLASSIFIER_0DTE_MIN_SIZE=400 # Smart money refdata +LOG_LEVEL=warn SMART_MONEY_EVENT_CALENDAR_PATH=data/event-calendar.json REFDATA_EVENT_CALENDAR_PATH= REFDATA_EVENT_CALENDAR_PROVIDER= @@ -120,3 +121,33 @@ REPLAY_END_TS=0 REPLAY_SPEED=1 REPLAY_BATCH_SIZE=200 REPLAY_LOG_EVERY=1000 + +# API live retention +LIVE_LIMIT_DEFAULT=1000 +LIVE_LIMIT_OPTIONS=1000 +LIVE_LIMIT_NBBO=1000 +LIVE_LIMIT_EQUITIES=1000 +LIVE_LIMIT_EQUITY_QUOTES=500 +LIVE_LIMIT_EQUITY_JOINS=500 +LIVE_LIMIT_FLOW=500 +LIVE_LIMIT_SMART_MONEY=300 +LIVE_LIMIT_CLASSIFIER_HITS=300 +LIVE_LIMIT_ALERTS=300 +LIVE_LIMIT_INFERRED_DARK=300 +LIVE_SCOPED_CACHE_MAX_KEYS=32 +LIVE_REDIS_FLUSH_INTERVAL_MS=250 +LIVE_REDIS_FLUSH_MAX_ITEMS=100 + +# Compute and ingest cache retention +ROLLING_CACHE_FLUSH_INTERVAL_MS=30000 +ROLLING_CACHE_MAX_KEYS=20000 +OPTION_CONTEXT_MAX_KEYS=20000 +OPTION_CONTEXT_TTL_MS=900000 +COMPUTE_NBBO_CACHE_MAX_KEYS=20000 +COMPUTE_NBBO_CACHE_TTL_MS=900000 + +# JetStream retention +STREAM_RAW_MAX_AGE_MS=7200000 +STREAM_RAW_MAX_BYTES=1073741824 +STREAM_DERIVED_MAX_AGE_MS=86400000 +STREAM_DERIVED_MAX_BYTES=536870912 diff --git a/deployment/docker/README.md b/deployment/docker/README.md index de7c805..dca5fbe 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -21,6 +21,7 @@ It is separate from the repo-root `docker-compose.yml`, which is still the light - `deployment/docker/Dockerfile.service`: shared Bun runtime image for most services - `deployment/docker/Dockerfile.ingest-options`: Bun runtime plus Python dependencies for Databento and IBKR adapters - `deployment/docker/Dockerfile.web`: multi-stage build for the Next.js web app +- `deployment/docker/workspace-root/`: deployment-specific workspace snapshot (`package.json`, `tsconfig.base.json`, `bun.lock`) used by Docker builds - `deployment/docker/clickhouse/listen.xml`: forces ClickHouse to listen on IPv4 for other containers on the Docker network - `deployment/docker/.env.example`: container-oriented environment template @@ -185,6 +186,22 @@ If NPM is on multiple networks and names collide (for example another stack also ## Updating the deployment +This deployment installs dependencies from `deployment/docker/workspace-root/bun.lock` (not the repo-root lockfile). + +When dependencies change in any workspace used by Docker builds, refresh and validate the deployment snapshot first: + +```bash +bun run sync:docker-workspace +bun run check:docker-workspace +``` + +Then validate the VPS build path: + +```bash +cd deployment/docker +docker compose build web +``` + When you pull new code: ```bash diff --git a/deployment/docker/deploy-branch.sh b/deployment/docker/deploy-branch.sh new file mode 100755 index 0000000..c5961b8 --- /dev/null +++ b/deployment/docker/deploy-branch.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +git fetch +git pull +docker compose up -d --build --force-recreate diff --git a/deployment/docker/deploy.sh b/deployment/docker/deploy.sh new file mode 100755 index 0000000..9ea97a6 --- /dev/null +++ b/deployment/docker/deploy.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +git fetch +git switch deployment +git pull +docker compose up -d --build --force-recreate diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index a3ed7a4..96598ba 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -14,6 +14,8 @@ x-service-common: &service-common dockerfile: Dockerfile.service env_file: - ./.env + environment: + LOG_LEVEL: ${LOG_LEVEL:-warn} restart: unless-stopped init: true extra_hosts: @@ -94,6 +96,8 @@ services: dockerfile: Dockerfile.ingest-options env_file: - ./.env + environment: + LOG_LEVEL: ${LOG_LEVEL:-warn} restart: unless-stopped init: true extra_hosts: diff --git a/deployment/docker/workspace-root/bun.lock b/deployment/docker/workspace-root/bun.lock index d6e99c6..47fc572 100644 --- a/deployment/docker/workspace-root/bun.lock +++ b/deployment/docker/workspace-root/bun.lock @@ -12,6 +12,7 @@ "name": "@islandflow/web", "dependencies": { "@islandflow/types": "workspace:*", + "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", "next": "^14.2.4", "react": "^18.3.1", @@ -81,6 +82,7 @@ "@islandflow/bus": "workspace:*", "@islandflow/config": "workspace:*", "@islandflow/observability": "workspace:*", + "@islandflow/refdata": "workspace:*", "@islandflow/storage": "workspace:*", "@islandflow/types": "workspace:*", "redis": "^5.10.0", @@ -207,6 +209,10 @@ "@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="], + "@tanstack/react-virtual": ["@tanstack/react-virtual@3.13.24", "", { "dependencies": { "@tanstack/virtual-core": "3.14.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aIJvz5OSkhNIhZIpYivrxrPTKYsjW9Uzy+sP/mx0S3sev2HyvPb7xmjbYvokzEpfgYHy/HjzJ2zFAETuUfgCpg=="], + + "@tanstack/virtual-core": ["@tanstack/virtual-core@3.14.0", "", {}, "sha512-JLANqGy/D6k4Ujmh8Tr25lGimuOXNiaVyXaCAZS0W+1390sADdGnyUdSWNIfd49gebtIxGMij4IktRVzrdr12Q=="], + "@types/node": ["@types/node@20.19.27", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-N2clP5pJhB2YnZJ3PIHFk5RkygRX5WO/5f0WC08tp0wd+sv0rsJk3MqWn3CbNmT2J505a5336jaQj4ph1AdMug=="], "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], diff --git a/deployment/docker/workspace-root/package.json b/deployment/docker/workspace-root/package.json index 0d570a9..8240012 100644 --- a/deployment/docker/workspace-root/package.json +++ b/deployment/docker/workspace-root/package.json @@ -12,7 +12,9 @@ "dev:infra": "docker compose up", "dev:infra:down": "docker compose down", "dev:web": "bun --cwd=apps/web run dev", - "dev:services": "bun run scripts/dev-services.ts" + "dev:services": "bun run scripts/dev-services.ts", + "sync:docker-workspace": "bun run scripts/sync-docker-workspace.ts", + "check:docker-workspace": "bun run scripts/check-docker-workspace.ts" }, "devDependencies": { "typescript-language-server": "^5.1.3" diff --git a/deployment/docker/workspace-root/tsconfig.base.json b/deployment/docker/workspace-root/tsconfig.base.json index f98f46a..34b15d2 100644 --- a/deployment/docker/workspace-root/tsconfig.base.json +++ b/deployment/docker/workspace-root/tsconfig.base.json @@ -8,6 +8,6 @@ "isolatedModules": true, "resolveJsonModule": true, "skipLibCheck": true, - "noEmit": true - } + "noEmit": true, + }, } diff --git a/package.json b/package.json index 0d570a9..8240012 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,9 @@ "dev:infra": "docker compose up", "dev:infra:down": "docker compose down", "dev:web": "bun --cwd=apps/web run dev", - "dev:services": "bun run scripts/dev-services.ts" + "dev:services": "bun run scripts/dev-services.ts", + "sync:docker-workspace": "bun run scripts/sync-docker-workspace.ts", + "check:docker-workspace": "bun run scripts/check-docker-workspace.ts" }, "devDependencies": { "typescript-language-server": "^5.1.3" diff --git a/packages/bus/src/jetstream.ts b/packages/bus/src/jetstream.ts index d79daba..204395e 100644 --- a/packages/bus/src/jetstream.ts +++ b/packages/bus/src/jetstream.ts @@ -84,6 +84,50 @@ export const ensureStream = async ( } }; +const parseBoundedNumber = (value: string | undefined, fallback: number): number => { + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed < 0) { + return fallback; + } + return Math.floor(parsed); +}; + +export type StreamRetentionClass = "raw" | "derived"; + +export const resolveStreamRetention = ( + streamClass: StreamRetentionClass, + env: Record = process.env +): Pick => { + if (streamClass === "raw") { + return { + max_age: parseBoundedNumber(env.STREAM_RAW_MAX_AGE_MS, 7_200_000), + max_bytes: parseBoundedNumber(env.STREAM_RAW_MAX_BYTES, 1_073_741_824) + }; + } + + return { + max_age: parseBoundedNumber(env.STREAM_DERIVED_MAX_AGE_MS, 86_400_000), + max_bytes: parseBoundedNumber(env.STREAM_DERIVED_MAX_BYTES, 536_870_912) + }; +}; + +export const buildStreamConfig = ( + name: string, + subject: string, + streamClass: StreamRetentionClass, + env: Record = process.env +): StreamConfig => ({ + name, + subjects: [subject], + retention: "limits", + storage: "file", + discard: "old", + max_msgs_per_subject: -1, + max_msgs: -1, + ...resolveStreamRetention(streamClass, env), + num_replicas: 1 +}); + export const buildDurableConsumer = ( durableName: string, deliverSubject: string = createInbox() diff --git a/packages/observability/src/logger.ts b/packages/observability/src/logger.ts index 0c4b437..b883695 100644 --- a/packages/observability/src/logger.ts +++ b/packages/observability/src/logger.ts @@ -22,20 +22,46 @@ export type LoggerOptions = { service: string; now?: () => string; sink?: (record: LogRecord) => void; + level?: LogLevel; }; const defaultSink = (record: LogRecord) => { console.log(JSON.stringify(record)); }; +const LOG_LEVEL_ORDER: Record = { + debug: 10, + info: 20, + warn: 30, + error: 40 +}; + +const resolveLogLevel = (value: string | undefined): LogLevel => { + switch ((value ?? "").trim().toLowerCase()) { + case "debug": + case "info": + case "warn": + case "error": + return value!.trim().toLowerCase() as LogLevel; + default: + return "info"; + } +}; + export const createLogger = ({ service, now = () => new Date().toISOString(), - sink = defaultSink + sink = defaultSink, + level = resolveLogLevel(process.env.LOG_LEVEL) }: LoggerOptions): Logger => { - const write = (level: LogLevel, msg: string, context?: LogContext) => { + const levelThreshold = resolveLogLevel(level); + + const write = (recordLevel: LogLevel, msg: string, context?: LogContext) => { + if (LOG_LEVEL_ORDER[recordLevel] < LOG_LEVEL_ORDER[levelThreshold]) { + return; + } const record: LogRecord = { - level, + level: recordLevel, service, msg, ts: now(), diff --git a/packages/storage/src/clickhouse.ts b/packages/storage/src/clickhouse.ts index d26b046..b5b0484 100644 --- a/packages/storage/src/clickhouse.ts +++ b/packages/storage/src/clickhouse.ts @@ -449,6 +449,157 @@ export const insertAlert = async (client: ClickHouseClient, alert: AlertEvent): }); }; +export type ClickHouseBatchWriterOptions = { + flushIntervalMs?: number; + maxRows?: number; + onError?: (table: string, error: unknown, rowCount: number) => void; +}; + +type BatchState = { + rows: unknown[]; + timer: ReturnType | null; + flushing: Promise | null; +}; + +const createBatchState = (): BatchState => ({ + rows: [], + timer: null, + flushing: null +}); + +export class ClickHouseBatchWriter { + private readonly flushIntervalMs: number; + private readonly maxRows: number; + private readonly states = new Map(); + + constructor( + private readonly client: ClickHouseClient, + options: ClickHouseBatchWriterOptions = {} + ) { + this.flushIntervalMs = Math.max(1, Math.floor(options.flushIntervalMs ?? 100)); + this.maxRows = Math.max(1, Math.floor(options.maxRows ?? 250)); + this.onError = options.onError; + } + + private readonly onError?: (table: string, error: unknown, rowCount: number) => void; + + enqueue(table: string, row: unknown): void { + const state = this.states.get(table) ?? createBatchState(); + if (!this.states.has(table)) { + this.states.set(table, state); + } + + state.rows.push(row); + + if (state.rows.length >= this.maxRows) { + void this.flush(table); + return; + } + + if (!state.timer) { + state.timer = setTimeout(() => { + state.timer = null; + void this.flush(table); + }, this.flushIntervalMs); + } + } + + async flush(table: string): Promise { + const state = this.states.get(table); + if (!state) { + return; + } + + if (state.flushing) { + await state.flushing; + return; + } + + if (state.timer) { + clearTimeout(state.timer); + state.timer = null; + } + + if (state.rows.length === 0) { + return; + } + + const rows = state.rows.splice(0, state.rows.length); + state.flushing = this.client + .insert({ + table, + values: rows, + format: "JSONEachRow" + }) + .catch((error) => { + this.onError?.(table, error, rows.length); + }) + .finally(() => { + state.flushing = null; + }); + + await state.flushing; + } + + async flushAll(): Promise { + for (const table of this.states.keys()) { + await this.flush(table); + } + } + + async close(): Promise { + for (const state of this.states.values()) { + if (state.timer) { + clearTimeout(state.timer); + state.timer = null; + } + } + await this.flushAll(); + } +} + +export const enqueueEquityPrintJoinInsert = ( + writer: ClickHouseBatchWriter, + join: EquityPrintJoin +): void => { + writer.enqueue(EQUITY_PRINT_JOINS_TABLE, toEquityPrintJoinRecord(join)); +}; + +export const enqueueInferredDarkInsert = ( + writer: ClickHouseBatchWriter, + event: InferredDarkEvent +): void => { + writer.enqueue(INFERRED_DARK_TABLE, toInferredDarkRecord(event)); +}; + +export const enqueueFlowPacketInsert = ( + writer: ClickHouseBatchWriter, + packet: FlowPacket +): void => { + writer.enqueue(FLOW_PACKETS_TABLE, toFlowPacketRecord(packet)); +}; + +export const enqueueSmartMoneyEventInsert = ( + writer: ClickHouseBatchWriter, + event: SmartMoneyEvent +): void => { + writer.enqueue(SMART_MONEY_EVENTS_TABLE, toSmartMoneyEventRecord(event)); +}; + +export const enqueueClassifierHitInsert = ( + writer: ClickHouseBatchWriter, + hit: ClassifierHitEvent +): void => { + writer.enqueue(CLASSIFIER_HITS_TABLE, toClassifierHitRecord(hit)); +}; + +export const enqueueAlertInsert = ( + writer: ClickHouseBatchWriter, + alert: AlertEvent +): void => { + writer.enqueue(ALERTS_TABLE, toAlertRecord(alert)); +}; + const clampLimit = (limit: number): number => { if (!Number.isFinite(limit)) { return 100; diff --git a/packages/types/src/live.ts b/packages/types/src/live.ts index 01fe4af..0787c84 100644 --- a/packages/types/src/live.ts +++ b/packages/types/src/live.ts @@ -54,6 +54,24 @@ export const LiveChannelSchema = z.enum([ export type LiveChannel = z.infer; export type LiveGenericChannel = z.infer; +export const LiveHotChannelSchema = z.enum(["options", "nbbo", "equities", "flow"]); +export type LiveHotChannel = z.infer; + +export const LiveChannelHealthSchema = z.object({ + freshness_age_ms: z.number().int().nonnegative().nullable(), + healthy: z.boolean() +}); + +export type LiveChannelHealth = z.infer; + +export const LiveHotChannelHealthSchema = z.object({ + options: LiveChannelHealthSchema, + nbbo: LiveChannelHealthSchema, + equities: LiveChannelHealthSchema, + flow: LiveChannelHealthSchema +}); + +export type LiveHotChannelHealthMap = z.infer; export const LiveSubscriptionSchema = z.discriminatedUnion("channel", [ z.object({ @@ -152,7 +170,8 @@ export const LiveClientMessageSchema = z.discriminatedUnion("op", [ export type LiveClientMessage = z.infer; export const LiveReadyMessageSchema = z.object({ - op: z.literal("ready") + op: z.literal("ready"), + channel_health: LiveHotChannelHealthSchema }); export type LiveReadyMessage = z.infer; @@ -175,7 +194,8 @@ export type LiveEventMessage = z.infer; export const LiveHeartbeatMessageSchema = z.object({ op: z.literal("heartbeat"), - ts: z.number().int().nonnegative() + ts: z.number().int().nonnegative(), + channel_health: LiveHotChannelHealthSchema }); export type LiveHeartbeatMessage = z.infer; diff --git a/plans/terminal-extraction-refactor.md b/plans/terminal-extraction-refactor.md new file mode 100644 index 0000000..6125127 --- /dev/null +++ b/plans/terminal-extraction-refactor.md @@ -0,0 +1,372 @@ +# Terminal Extraction Plan + +## Summary + +Refactor [`apps/web/app/terminal.tsx`](/Users/kell/Cloud/dev/islandflow/apps/web/app/terminal.tsx:1) from a single 7,974-line client module into a feature folder at `apps/web/terminal/*`, while keeping `apps/web/app/terminal.tsx` as a temporary compatibility facade in the first pass. + +This first extraction is a medium-scope, behavior-preserving refactor: +- no product behavior changes +- no route behavior changes +- no visual redesign +- no data model changes +- no immediate deletion of the old import surface + +Current baseline is healthy and must remain healthy after the refactor: +- `bun test apps/web/app/terminal.test.ts apps/web/app/routes.test.ts` passes +- `bun --cwd=apps/web run build` passes + +## Target Structure + +Create this feature layout: + +```text +apps/web/terminal/ + index.ts + state.tsx + shell.tsx + routes.tsx + core/ + format.ts + filters.ts + route-config.ts + tape-data.ts + signals.ts + live-manifest.ts + hooks/ + use-tape-data.ts + use-live-session.ts + use-virtual-tape.ts + components/ + chrome.tsx + chart.tsx + drawers.tsx + panes.tsx + tests/ + core.test.ts + live-manifest.test.ts + signals.test.ts + tape-data.test.ts +``` + +Keep this file in place for the first pass: +- `apps/web/app/terminal.tsx` + +Its final first-pass role is: +- `"use client"` entrypoint +- thin re-export facade only +- no business logic +- no React state +- no websocket/session logic +- no chart implementation +- target size: under 120 lines + +## Dependency Rules + +Use this dependency direction and do not violate it: +- `core/*` may depend only on shared types and other `core/*` +- `hooks/*` may depend on `core/*` +- `components/*` may depend on `core/*` and `hooks/*` +- `state.tsx` may depend on `core/*`, `hooks/*`, and `components/*` types only as needed +- `shell.tsx` and `routes.tsx` may depend on `state.tsx` and `components/*` +- `index.ts` re-exports public feature symbols +- `app/terminal.tsx` re-exports from `apps/web/terminal/index.ts` + +Do not allow circular imports. + +## Module Mapping + +Move code out of `terminal.tsx` in this order. + +### 1. Pure helpers first + +Move non-React helpers into `apps/web/terminal/core/*`: + +- `core/route-config.ts` + - `getRouteFeatures` + - `getTapeVirtualConfig` + - `shouldIncludeEquitiesForDarkUnderlyingFallback` + +- `core/live-manifest.ts` + - `getLiveManifest` + - `getLiveHistoryRetentionCap` + - `getScopedLiveAutoHydrationChannels` + - `getLiveFeedStatus` + - `getHotChannelFeedStatus` + +- `core/tape-data.ts` + - `mergeNewestWithOverflow` + - `composeTapeItems` + - `reducePausableTapeData` + - `flushPausableTapeData` + - `appendHistoryTail` + - `projectPausableTapeState` + - `findAnchorRestoreIndex` + - `shouldRetainLiveSnapshotHistory` + - `shouldShowEquitiesSilentFeedWarning` + - tape/history support types used only by these helpers + +- `core/format.ts` + - `formatCompactUsd` + - `formatOptionContractLabel` + - `getOptionTableSnapshot` + - price/size/time/date/contract formatting helpers that support UI rendering + +- `core/signals.ts` + - `normalizeAlertSeverity` + - `deriveAlertDirection` + - `getAlertWindowAnchorTs` + - `selectPrimaryClassifierHit` + - `classifierToneForFamily` + - `smartMoneyToneForProfile` + - `smartMoneyProfileLabel` + +- `core/filters.ts` + - `buildDefaultFlowFilters` + - `countActiveFlowFilterGroups` + - `toggleFilterValue` + - `nextFlowFilterPopoverState` + +These files must not include `"use client"`. + +### 2. Extract hooks and session logic + +Move React hooks into `apps/web/terminal/hooks/*`: + +- `hooks/use-virtual-tape.ts` + - `useListScroll` + - `useScrollAnchor` + - `useVirtualHistoryGate` + - `useTapeVirtualList` + +- `hooks/use-tape-data.ts` + - `useTape` + - `usePausableTapeView` + - `useLiveStream` + - `useFlowStream` + - `statusLabel` + - internal tape state types + +- `hooks/use-live-session.ts` + - `useLiveSession` + - live history endpoint constants + - live history query builders + - subscription dedupe helpers + - session-local types + +Keep signatures stable unless a change is required to break a circular dependency. If a signature changes, update all callers in the same PR. + +### 3. Extract UI components + +Move rendering code into `apps/web/terminal/components/*`: + +- `components/chrome.tsx` + - `TapeStatus` + - `TapeControls` + - `PageFrame` + - `Pane` + - `ShellMetricStrip` + - `FlowFilterPopover` + - local filter UI helpers + +- `components/chart.tsx` + - `CandleChart` + - chart-only local types and overlay helpers + - isolate `lightweight-charts` usage here + +- `components/drawers.tsx` + - `AlertSeverityStrip` + - `AlertDrawer` + - `ClassifierHitDrawer` + - `SmartMoneyDrawer` + - `DarkDrawer` + +- `components/panes.tsx` + - `OptionsPane` + - `EquitiesPane` + - `FlowPane` + - `AlertsPane` + - `ClassifierPane` + - `DarkPane` + - `ChartPane` + - `FocusPane` + - `ReplayConsole` + +### 4. Extract state orchestration + +Create `apps/web/terminal/state.tsx` for: +- `useTerminalState` +- `TerminalContext` +- `useTerminal` + +This file owns: +- route-aware feature selection +- filter input state +- selected entity/drawer state +- scroll-anchor wiring +- assembly of hook outputs into the single terminal state object + +Keep `useTerminalState` internal. Do not export it from the feature barrel. + +### 5. Extract shell and routes + +Create: +- `apps/web/terminal/shell.tsx` + - `TerminalAppShell` + +- `apps/web/terminal/routes.tsx` + - `NAV_ITEMS` + - `OverviewRoute` + - `TapeRoute` + - `SignalsRoute` + - `ChartsRoute` + - `ReplayRoute` + +Important first-pass rule: +- keep existing route behavior exactly as-is +- [app/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/page.tsx:1), [app/layout.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/layout.tsx:4), and [app/tape/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/tape/page.tsx:1) may continue importing from `./terminal` / `../terminal` +- [app/signals/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/signals/page.tsx:1), [app/charts/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/charts/page.tsx:1), and [app/replay/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/replay/page.tsx:1) must remain redirect pages in this pass + +## Facade Contract + +Replace `apps/web/app/terminal.tsx` with a facade that re-exports from `apps/web/terminal/index.ts`. + +The facade must continue exporting these symbols in the first pass: + +- `getTapeVirtualConfig` +- `shouldIncludeEquitiesForDarkUnderlyingFallback` +- `getRouteFeatures` +- `mergeNewestWithOverflow` +- `composeTapeItems` +- `reducePausableTapeData` +- `flushPausableTapeData` +- `appendHistoryTail` +- `getLiveHistoryRetentionCap` +- `getScopedLiveAutoHydrationChannels` +- `getLiveFeedStatus` +- `getHotChannelFeedStatus` +- `findAnchorRestoreIndex` +- `formatCompactUsd` +- `formatOptionContractLabel` +- `normalizeAlertSeverity` +- `deriveAlertDirection` +- `getAlertWindowAnchorTs` +- `buildDefaultFlowFilters` +- `countActiveFlowFilterGroups` +- `toggleFilterValue` +- `nextFlowFilterPopoverState` +- `projectPausableTapeState` +- `shouldShowEquitiesSilentFeedWarning` +- `shouldRetainLiveSnapshotHistory` +- `selectPrimaryClassifierHit` +- `classifierToneForFamily` +- `smartMoneyToneForProfile` +- `smartMoneyProfileLabel` +- `getOptionTableSnapshot` +- `statusLabel` +- `getLiveManifest` +- `NAV_ITEMS` +- `FlowFilterPopover` +- `TerminalAppShell` +- `OverviewRoute` +- `TapeRoute` +- `SignalsRoute` +- `ChartsRoute` +- `ReplayRoute` + +Do not add new facade-only exports. + +## Test Plan + +Restructure tests so pure logic is tested from its final home instead of through the facade. + +### Keep +- `apps/web/app/routes.test.ts` + - still verifies redirect behavior for `/signals`, `/charts`, `/replay` + +### Split `app/terminal.test.ts` into feature tests +- `apps/web/terminal/tests/live-manifest.test.ts` + - route feature mapping + - manifest composition + - nav items if still treated as route metadata + +- `apps/web/terminal/tests/tape-data.test.ts` + - merge/dedupe logic + - pausable tape behavior + - history seam behavior + - anchor restore behavior + - retention cap behavior + - scoped history behavior + +- `apps/web/terminal/tests/core.test.ts` + - option contract formatting + - compact USD formatting + - option table snapshot formatting + - flow filter helpers + +- `apps/web/terminal/tests/signals.test.ts` + - alert severity normalization + - direction derivation + - alert window anchor + - classifier/smart-money label and tone helpers + - live status labeling if kept outside tape-data tests + +Optional and recommended: +- add one tiny `apps/web/app/terminal-facade.test.ts` that imports the facade and asserts a few critical exports exist, so we notice accidental facade breakage during the transition + +## Validation Gates + +Implementation is not complete unless all of these pass: + +1. `bun test apps/web/terminal/tests apps/web/app/routes.test.ts` +2. `bun --cwd=apps/web run build` +3. Existing behavior smoke check: + - `/` still renders the shell and overview + - `/tape` still renders shell and tape panes + - `/signals`, `/charts`, `/replay` still redirect to `/` +4. `apps/web/app/terminal.tsx` is a facade only and contains no moved logic +5. No extracted pure helper file contains React imports +6. No new circular imports are introduced + +## Non-Goals For This Pass + +Do not do these in the first extraction: +- redesign panes or drawers +- change websocket or replay behavior +- change route inventory +- remove unused legacy route exports +- change CSS structure beyond import fixes +- optimize bundle size as a separate objective +- rewrite tests to different testing tools + +## Beads Follow-Up Issues To File + +Create these `bd` issues during implementation if they do not already exist: + +1. `task`, priority `2` + Title: `Remove temporary apps/web/app/terminal.tsx facade after terminal imports are migrated` + Description: track deletion of the compatibility facade once route/layout/test imports point at final `apps/web/terminal/*` modules + +2. `task`, priority `3` + Title: `Audit and remove dead terminal route exports no longer used by app redirects` + Description: verify whether `SignalsRoute`, `ChartsRoute`, and `ReplayRoute` should be deleted since App Router pages now redirect to `/` + +If additional cleanup is discovered during extraction, create linked `bd` tasks with `discovered-from` dependencies rather than expanding this refactor mid-flight. + +## Acceptance Criteria + +The first extraction is successful when: +- terminal logic is split into the target `apps/web/terminal/*` structure +- `apps/web/app/terminal.tsx` remains only as a thin compatibility layer +- app entrypoints continue to work without behavior changes +- tests target the new module homes for pure logic +- build and tests pass +- follow-up `bd` issues exist for facade removal and dead-export cleanup + +## Assumptions And Defaults + +- Chosen scope: medium slice, not full architectural rewrite +- Chosen transition: keep `apps/web/app/terminal.tsx` as a temporary facade +- Chosen module home: `apps/web/terminal/*`, not `apps/web/app/terminal/*` +- Default behavior requirement: strict behavioral parity +- Default testing approach: split existing monolithic helper tests by concern and colocate them under `apps/web/terminal/tests` +- Default routing approach: keep redirect pages untouched in the first pass diff --git a/scripts/check-docker-workspace.ts b/scripts/check-docker-workspace.ts new file mode 100644 index 0000000..bc0d33e --- /dev/null +++ b/scripts/check-docker-workspace.ts @@ -0,0 +1,244 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +type DependencyMap = Record; + +type LockWorkspace = { + name?: string; + dependencies?: DependencyMap; + devDependencies?: DependencyMap; + optionalDependencies?: DependencyMap; + peerDependencies?: DependencyMap; +}; + +type BunLock = { + lockfileVersion?: number; + configVersion?: number; + workspaces?: Record; + packages?: Record; +}; + +type RootPackageManifest = { + workspaces?: string[]; +}; + +const repoRoot = path.resolve(import.meta.dir, ".."); +const deploymentRoot = path.join(repoRoot, "deployment/docker/workspace-root"); + +const rootPackagePath = path.join(repoRoot, "package.json"); +const deploymentPackagePath = path.join(deploymentRoot, "package.json"); +const rootTsconfigPath = path.join(repoRoot, "tsconfig.base.json"); +const deploymentTsconfigPath = path.join(deploymentRoot, "tsconfig.base.json"); +const rootLockPath = path.join(repoRoot, "bun.lock"); +const deploymentLockPath = path.join(deploymentRoot, "bun.lock"); + +const readUtf8 = async (filePath: string): Promise => { + return readFile(filePath, "utf8"); +}; + +const parseObjectLiteral = async (filePath: string): Promise => { + const raw = await readUtf8(filePath); + try { + const parsed = Function(`"use strict"; return (${raw});`)() as T; + return parsed; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to parse ${filePath}: ${message}`); + } +}; + +const stableSortObject = (value: unknown): unknown => { + if (Array.isArray(value)) { + return value.map(stableSortObject); + } + if (value && typeof value === "object") { + const entries = Object.entries(value as Record) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, nested]) => [key, stableSortObject(nested)] as const); + return Object.fromEntries(entries); + } + return value; +}; + +const stableStringify = (value: unknown): string => { + return JSON.stringify(stableSortObject(value)); +}; + +const listWorkspacePaths = async (workspacePatterns: string[]): Promise => { + const paths = new Set(); + + for (const pattern of workspacePatterns) { + const globPattern = pattern.endsWith("/") ? `${pattern}package.json` : `${pattern}/package.json`; + const glob = new Bun.Glob(globPattern); + for await (const match of glob.scan({ cwd: repoRoot })) { + const normalized = match.replaceAll("\\", "/"); + paths.add(path.posix.dirname(normalized)); + } + } + + return Array.from(paths).sort((a, b) => a.localeCompare(b)); +}; + +const normalizedDependencyMap = (input: DependencyMap | undefined): DependencyMap => { + if (!input) { + return {}; + } + return Object.fromEntries( + Object.entries(input) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, version]) => [name, version]) + ); +}; + +const formatDependencyDiff = ( + workspacePath: string, + section: string, + expected: DependencyMap, + actual: DependencyMap +): string[] => { + const issues: string[] = []; + const expectedKeys = new Set(Object.keys(expected)); + const actualKeys = new Set(Object.keys(actual)); + + for (const key of expectedKeys) { + if (!actualKeys.has(key)) { + issues.push(`${workspacePath} ${section}: missing ${key}@${expected[key]}`); + continue; + } + if (expected[key] !== actual[key]) { + issues.push( + `${workspacePath} ${section}: ${key} expected ${expected[key]} but found ${actual[key]}` + ); + } + } + + for (const key of actualKeys) { + if (!expectedKeys.has(key)) { + issues.push(`${workspacePath} ${section}: extra ${key}@${actual[key]}`); + } + } + + return issues; +}; + +const check = async (): Promise => { + const issues: string[] = []; + + const [rootPackage, deploymentPackage, rootTsconfig, deploymentTsconfig, rootLock, deploymentLock] = + await Promise.all([ + parseObjectLiteral(rootPackagePath), + parseObjectLiteral(deploymentPackagePath), + parseObjectLiteral(rootTsconfigPath), + parseObjectLiteral(deploymentTsconfigPath), + parseObjectLiteral(rootLockPath), + parseObjectLiteral(deploymentLockPath) + ]); + + const rootPackageSnapshot = stableStringify(rootPackage); + const deploymentPackageSnapshot = stableStringify(deploymentPackage); + if (rootPackageSnapshot !== deploymentPackageSnapshot) { + issues.push( + "deployment/docker/workspace-root/package.json does not match repo-root package.json" + ); + } + + const rootTsconfigSnapshot = stableStringify(rootTsconfig); + const deploymentTsconfigSnapshot = stableStringify(deploymentTsconfig); + if (rootTsconfigSnapshot !== deploymentTsconfigSnapshot) { + issues.push( + "deployment/docker/workspace-root/tsconfig.base.json does not match repo-root tsconfig.base.json" + ); + } + + const rootWorkspaces = rootLock.workspaces ?? {}; + const deploymentWorkspaces = deploymentLock.workspaces ?? {}; + + const workspacePatterns = rootPackage.workspaces ?? []; + const workspacePackagePaths = await listWorkspacePaths(workspacePatterns); + for (const workspacePath of workspacePackagePaths) { + const packageJsonPath = path.join(repoRoot, workspacePath, "package.json"); + const workspacePackage = (await parseObjectLiteral(packageJsonPath)) as LockWorkspace; + const deploymentWorkspace = deploymentWorkspaces[workspacePath]; + + if (!deploymentWorkspace) { + issues.push(`deployment lock is missing workspace entry: ${workspacePath}`); + continue; + } + + const sections: Array = [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies" + ]; + for (const section of sections) { + const expectedMap = normalizedDependencyMap(workspacePackage[section] as DependencyMap | undefined); + const actualMap = normalizedDependencyMap( + deploymentWorkspace[section] as DependencyMap | undefined + ); + issues.push(...formatDependencyDiff(workspacePath, section, expectedMap, actualMap)); + } + } + + const workspacePaths = Array.from( + new Set([...Object.keys(rootWorkspaces), ...Object.keys(deploymentWorkspaces)]) + ).sort((a, b) => a.localeCompare(b)); + + for (const workspacePath of workspacePaths) { + const rootWorkspace = rootWorkspaces[workspacePath]; + const deploymentWorkspace = deploymentWorkspaces[workspacePath]; + + if (!rootWorkspace) { + issues.push(`deployment lock has unexpected workspace entry: ${workspacePath}`); + continue; + } + if (!deploymentWorkspace) { + issues.push(`deployment lock is missing workspace entry: ${workspacePath}`); + continue; + } + + if ((rootWorkspace.name ?? "") !== (deploymentWorkspace.name ?? "")) { + issues.push( + `${workspacePath} name mismatch: expected ${rootWorkspace.name ?? "(none)"} but found ${ + deploymentWorkspace.name ?? "(none)" + }` + ); + } + + const sections: Array = [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies" + ]; + for (const section of sections) { + const expectedMap = normalizedDependencyMap(rootWorkspace[section] as DependencyMap | undefined); + const actualMap = normalizedDependencyMap( + deploymentWorkspace[section] as DependencyMap | undefined + ); + issues.push(...formatDependencyDiff(workspacePath, section, expectedMap, actualMap)); + } + } + + const rootPackagesSnapshot = stableStringify(rootLock.packages ?? {}); + const deploymentPackagesSnapshot = stableStringify(deploymentLock.packages ?? {}); + if (rootPackagesSnapshot !== deploymentPackagesSnapshot) { + issues.push( + "deployment/docker/workspace-root/bun.lock package resolutions differ from repo-root bun.lock" + ); + } + + if (issues.length > 0) { + console.error("Docker workspace snapshot is out of sync:"); + for (const issue of issues) { + console.error(`- ${issue}`); + } + console.error("Run: bun run sync:docker-workspace"); + return 1; + } + + console.log("Docker workspace snapshot is in sync."); + return 0; +}; + +process.exitCode = await check(); diff --git a/scripts/sync-docker-workspace.ts b/scripts/sync-docker-workspace.ts new file mode 100644 index 0000000..e20b293 --- /dev/null +++ b/scripts/sync-docker-workspace.ts @@ -0,0 +1,19 @@ +import { copyFile } from "node:fs/promises"; +import path from "node:path"; + +const repoRoot = path.resolve(import.meta.dir, ".."); +const deploymentRoot = path.join(repoRoot, "deployment/docker/workspace-root"); + +const filesToSync = [ + "package.json", + "bun.lock", + "tsconfig.base.json" +] as const; + +for (const fileName of filesToSync) { + const source = path.join(repoRoot, fileName); + const destination = path.join(deploymentRoot, fileName); + await copyFile(source, destination); + console.log(`synced ${fileName}`); +} + diff --git a/services/api/src/index.ts b/services/api/src/index.ts index c450ea7..31f861a 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -1,5 +1,5 @@ import { readEnv } from "@islandflow/config"; -import { createLogger } from "@islandflow/observability"; +import { createLogger, createMetrics } from "@islandflow/observability"; import { SUBJECT_ALERTS, SUBJECT_CLASSIFIER_HITS, @@ -23,6 +23,7 @@ import { STREAM_SMART_MONEY_EVENTS, STREAM_OPTION_NBBO, STREAM_OPTION_SIGNAL_PRINTS, + buildStreamConfig, buildDurableConsumer, connectJetStreamWithRetry, ensureStream, @@ -82,7 +83,7 @@ import { fetchClassifierHitsByPacketIds, fetchRecentOptionPrints } from "@islandflow/storage"; -import type { EquityPrintQueryFilters, OptionPrintQueryFilters } from "@islandflow/storage"; +import type { EquityPrintQueryFilters } from "@islandflow/storage"; import { AlertEventSchema, ClassifierHitEventSchema, @@ -99,11 +100,6 @@ import { LiveSubscriptionSchema, matchesFlowPacketFilters, matchesOptionPrintFilters, - OptionFlowFilters, - OptionFlowViewSchema, - OptionNbboSideSchema, - OptionSecurityTypeSchema, - OptionTypeSchema, FlowPacketSchema, SmartMoneyEventSchema, OptionNBBOSchema, @@ -112,10 +108,17 @@ import { } from "@islandflow/types"; import { createClient } from "redis"; import { z } from "zod"; -import { LIVE_FEED_LOOKBACK_MS, LiveStateManager, shouldFanoutLiveEvent } from "./live"; +import { + HOT_LIVE_REDIS_KEYS, + LiveStateManager, + resolveLiveStateConfig, + shouldFanoutLiveEvent +} from "./live"; +import { parseOptionPrintQuery } from "./option-queries"; const service = "api"; const logger = createLogger({ service }); +const metrics = createMetrics({ service }); const DeliverPolicySchema = z.enum(["new", "all", "last", "last_per_subject"]); @@ -138,13 +141,6 @@ const state = { shutdownPromise: null as Promise | null }; -const HOT_LIVE_REDIS_KEYS = { - options: "live:options", - equities: "live:equities", - flow: "live:flow", - nbbo: "live:nbbo" -} as const; - const getErrorMessage = (error: unknown): string => { return error instanceof Error ? error.message : String(error); }; @@ -231,33 +227,6 @@ const equityPrintRangeSchema = z.object({ end_ts: z.coerce.number().int().nonnegative(), limit: limitSchema.optional() }); -const optionSideListSchema = z - .string() - .transform((value) => - value - .split(",") - .map((entry) => entry.trim()) - .filter(Boolean) - ) - .pipe(z.array(OptionNbboSideSchema)); -const optionTypeListSchema = z - .string() - .transform((value) => - value - .split(",") - .map((entry) => entry.trim()) - .filter(Boolean) - ) - .pipe(z.array(OptionTypeSchema)); -const optionSecuritySchema = z.enum(["stock", "etf", "all"]); -const optionFilterQuerySchema = z.object({ - view: OptionFlowViewSchema.optional(), - security: optionSecuritySchema.optional(), - side: optionSideListSchema.optional(), - type: optionTypeListSchema.optional(), - min_notional: z.coerce.number().nonnegative().optional() -}); - type Channel = | "options" | "options-nbbo" @@ -358,43 +327,6 @@ const applyDeliverPolicy = ( } }; -const parseOptionPrintFilters = ( - url: URL -): { - view: z.infer; - storageFilters: Parameters[3]; - liveFilters: OptionFlowFilters; -} => { - const parsed = optionFilterQuerySchema.parse({ - view: url.searchParams.get("view") ?? undefined, - security: url.searchParams.get("security") ?? undefined, - side: url.searchParams.get("side") ?? undefined, - type: url.searchParams.get("type") ?? undefined, - min_notional: url.searchParams.get("min_notional") ?? undefined - }); - const view = parsed.view ?? "signal"; - const security = parsed.security ?? (view === "raw" ? "all" : "stock"); - const storageFilters = { - view, - security, - minNotional: parsed.min_notional, - nbboSides: parsed.side, - optionTypes: parsed.type - } as const; - const liveFilters: OptionFlowFilters = { - view, - securityTypes: - security === "all" - ? undefined - : ([security] as Array>), - nbboSides: parsed.side, - optionTypes: parsed.type, - minNotional: parsed.min_notional - }; - - return { view, storageFilters, liveFilters }; -}; - const parseReplayParams = (url: URL): { afterTs: number; afterSeq: number; limit: number } => { const params = replayParamsSchema.parse({ after_ts: url.searchParams.get("after_ts") ?? undefined, @@ -612,19 +544,8 @@ const parseScopeList = (url: URL, ...keys: string[]): string[] | undefined => { return unique.length > 0 ? unique : undefined; }; -const parseLiveOptionPrintFilters = (url: URL): OptionPrintQueryFilters => { - const { storageFilters } = parseOptionPrintFilters(url); - return { - ...storageFilters, - underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids"), - optionContractId: url.searchParams.get("option_contract_id") ?? undefined, - sinceTs: Date.now() - LIVE_FEED_LOOKBACK_MS - }; -}; - const parseLiveEquityPrintFilters = (url: URL): EquityPrintQueryFilters => ({ - underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids"), - sinceTs: Date.now() - LIVE_FEED_LOOKBACK_MS + underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids") }); const matchesScopedOptionSubscription = ( @@ -703,148 +624,17 @@ const run = async () => { { attempts: 120, delayMs: 500 } ); - await ensureStream(jsm, { - name: STREAM_OPTION_SIGNAL_PRINTS, - subjects: [SUBJECT_OPTION_SIGNAL_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_OPTION_NBBO, - subjects: [SUBJECT_OPTION_NBBO], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_PRINTS, - subjects: [SUBJECT_EQUITY_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_QUOTES, - subjects: [SUBJECT_EQUITY_QUOTES], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_CANDLES, - subjects: [SUBJECT_EQUITY_CANDLES], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_JOINS, - subjects: [SUBJECT_EQUITY_JOINS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_INFERRED_DARK, - subjects: [SUBJECT_INFERRED_DARK], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_FLOW_PACKETS, - subjects: [SUBJECT_FLOW_PACKETS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_SMART_MONEY_EVENTS, - subjects: [SUBJECT_SMART_MONEY_EVENTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_CLASSIFIER_HITS, - subjects: [SUBJECT_CLASSIFIER_HITS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_ALERTS, - subjects: [SUBJECT_ALERTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_CANDLES, SUBJECT_EQUITY_CANDLES, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_JOINS, SUBJECT_EQUITY_JOINS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_INFERRED_DARK, SUBJECT_INFERRED_DARK, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_FLOW_PACKETS, SUBJECT_FLOW_PACKETS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_SMART_MONEY_EVENTS, SUBJECT_SMART_MONEY_EVENTS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_CLASSIFIER_HITS, SUBJECT_CLASSIFIER_HITS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_ALERTS, SUBJECT_ALERTS, "derived")); const clickhouse = createClickHouseClient({ url: env.CLICKHOUSE_URL, @@ -890,7 +680,7 @@ const run = async () => { redis = null; } - const liveState = new LiveStateManager(clickhouse, redis); + const liveState = new LiveStateManager(clickhouse, redis, resolveLiveStateConfig()); await liveState.hydrate(); const warnLiveLag = ( channel: keyof typeof HOT_LIVE_REDIS_KEYS, @@ -910,6 +700,7 @@ const run = async () => { }; const liveStateMetricsTimer = setInterval(() => { const snapshot = liveState.getStatsSnapshot(); + const hotFeedHealth = liveState.getHotChannelHealth(); const hotFeedLagMs = { options: snapshot.freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.options] ?? null, equities: snapshot.freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.equities] ?? null, @@ -918,7 +709,12 @@ const run = async () => { }; logger.info("live cache metrics", { ...snapshot, - hotFeedLagMs + hotFeedLagMs, + hotFeedHealth, + snapshotSourceCounts: { + generic_cache_snapshot: snapshot.genericCacheSnapshots, + scoped_clickhouse_snapshot: snapshot.scopedClickHouseSnapshots + } }); warnLiveLag("options", hotFeedLagMs.options); warnLiveLag("equities", hotFeedLagMs.equities); @@ -1149,6 +945,11 @@ const run = async () => { return; } + const optionItem = ingestChannel === "options" ? (item as Parameters[0]) : null; + const equityItem = ingestChannel === "equities" ? (item as Parameters[0]) : null; + const flowItem = ingestChannel === "flow" ? (item as Parameters[0]) : null; + let matchedSubscriptions = 0; + for (const [key, candidate] of matchingSubscriptions) { const sockets = subscriptionSockets.get(key); if (!sockets || sockets.size === 0) { @@ -1157,26 +958,29 @@ const run = async () => { if ( candidate.channel === "options" && - (!matchesOptionPrintFilters(OptionPrintSchema.parse(item), candidate.filters) || - !matchesScopedOptionSubscription(OptionPrintSchema.parse(item), candidate)) + (!optionItem || + !matchesOptionPrintFilters(optionItem, candidate.filters) || + !matchesScopedOptionSubscription(optionItem, candidate)) ) { continue; } if ( candidate.channel === "equities" && - !matchesScopedEquitySubscription(EquityPrintSchema.parse(item), candidate) + (!equityItem || !matchesScopedEquitySubscription(equityItem, candidate)) ) { continue; } if ( candidate.channel === "flow" && - !matchesFlowPacketFilters(FlowPacketSchema.parse(item), candidate.filters) + (!flowItem || !matchesFlowPacketFilters(flowItem, candidate.filters)) ) { continue; } + matchedSubscriptions += 1; + for (const socket of sockets) { sendLiveMessage(socket, { op: "event", @@ -1186,6 +990,10 @@ const run = async () => { }); } } + + if (matchedSubscriptions > 0) { + metrics.count("api.live.subscription_match_count", matchedSubscriptions); + } }; const pumpOptions = async () => { @@ -1402,7 +1210,7 @@ const run = async () => { try { const limit = parseLimit(url.searchParams.get("limit")); const source = parseReplaySource(url) ?? undefined; - const { storageFilters } = parseOptionPrintFilters(url); + const { storageFilters } = parseOptionPrintQuery(url); const data = await fetchRecentOptionPrints(clickhouse, limit, source, storageFilters); return jsonResponse({ data }); } catch (error) { @@ -1528,7 +1336,7 @@ const run = async () => { try { const { beforeTs, beforeSeq, limit } = parseBeforeParams(url); const source = parseReplaySource(url) ?? undefined; - const storageFilters = parseLiveOptionPrintFilters(url); + const { storageFilters } = parseOptionPrintQuery(url); const data = await fetchOptionPrintsBefore( clickhouse, beforeTs, @@ -1671,7 +1479,7 @@ const run = async () => { try { const { afterTs, afterSeq, limit } = parseReplayParams(url); const source = parseReplaySource(url) ?? undefined; - const { storageFilters } = parseOptionPrintFilters(url); + const { storageFilters } = parseOptionPrintQuery(url); const data = await fetchOptionPrintsAfter( clickhouse, afterTs, @@ -1894,9 +1702,13 @@ const run = async () => { websocket: { open: (socket: any) => { if (socket.data.channel === "live") { - sendLiveMessage(socket, { op: "ready" }); + sendLiveMessage(socket, { op: "ready", channel_health: liveState.getHotChannelHealth() }); const heartbeat = setInterval(() => { - sendLiveMessage(socket, { op: "heartbeat", ts: Date.now() }); + sendLiveMessage(socket, { + op: "heartbeat", + ts: Date.now(), + channel_health: liveState.getHotChannelHealth() + }); }, 15000); liveHeartbeats.set(socket, heartbeat); } else if (socket.data.channel === "options") { @@ -1937,7 +1749,11 @@ const run = async () => { : new TextDecoder().decode(message instanceof Uint8Array ? message : new Uint8Array(message)); const parsed = LiveClientMessageSchema.parse(JSON.parse(payload)); if (parsed.op === "ping") { - sendLiveMessage(socket, { op: "heartbeat", ts: Date.now() }); + sendLiveMessage(socket, { + op: "heartbeat", + ts: Date.now(), + channel_health: liveState.getHotChannelHealth() + }); return; } @@ -2003,6 +1819,7 @@ const run = async () => { logger.info("service stopping", { signal }); server.stop(); clearInterval(liveStateMetricsTimer); + await liveState.close(); if (redis && redis.isOpen) { try { diff --git a/services/api/src/live.ts b/services/api/src/live.ts index aa4281c..ca228fc 100644 --- a/services/api/src/live.ts +++ b/services/api/src/live.ts @@ -25,7 +25,10 @@ import { FeedSnapshot, FlowPacketSchema, InferredDarkEventSchema, + LiveChannelHealth, LiveGenericChannel, + LiveHotChannel, + LiveHotChannelHealthMap, LiveSubscription, matchesFlowPacketFilters, matchesOptionPrintFilters, @@ -38,12 +41,15 @@ import { type EquityPrint, type LiveChannel } from "@islandflow/types"; +import { createMetrics } from "@islandflow/observability"; import type { RedisClientType } from "redis"; const CURSOR_HASH_KEY = "live:cursors"; export const LIVE_FEED_LOOKBACK_MS = 24 * 60 * 60 * 1000; -const DEFAULT_GENERIC_LIMIT = 10000; +const metrics = createMetrics({ service: "api" }); + +const DEFAULT_GENERIC_LIMIT = 1000; const MAX_GENERIC_LIMIT = 100000; const MIN_GENERIC_LIMIT = 1; const GENERIC_LIMIT_ENV_KEYS: Record = { @@ -64,6 +70,23 @@ const CHART_LIMITS = { overlay: 1500 } as const; +const DEFAULT_LIVE_LIMITS: GenericLiveLimits = { + options: 1000, + nbbo: 1000, + equities: 1000, + "equity-quotes": 500, + "equity-joins": 500, + flow: 500, + "smart-money": 300, + "classifier-hits": 300, + alerts: 300, + "inferred-dark": 300 +}; + +const DEFAULT_SCOPED_CACHE_MAX_KEYS = 32; +const DEFAULT_REDIS_FLUSH_INTERVAL_MS = 250; +const DEFAULT_REDIS_FLUSH_MAX_ITEMS = 100; + type GenericFeedConfig = { redisKey: string; cursorField: string; @@ -81,8 +104,22 @@ export const LIVE_FRESHNESS_THRESHOLDS: Partial; + export type GenericLiveLimits = Record; +type LiveStateConfig = { + limits: GenericLiveLimits; + scopedCacheMaxKeys: number; + redisFlushIntervalMs: number; + redisFlushMaxItems: number; +}; + const parseGenericLimit = ( env: NodeJS.ProcessEnv, channel: LiveGenericChannel, @@ -107,17 +144,77 @@ const parseGenericLimit = ( return bounded; }; -export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env): GenericLiveLimits => ({ - options: parseGenericLimit(env, "options", DEFAULT_GENERIC_LIMIT), - nbbo: parseGenericLimit(env, "nbbo", DEFAULT_GENERIC_LIMIT), - equities: parseGenericLimit(env, "equities", DEFAULT_GENERIC_LIMIT), - "equity-quotes": parseGenericLimit(env, "equity-quotes", DEFAULT_GENERIC_LIMIT), - "equity-joins": parseGenericLimit(env, "equity-joins", DEFAULT_GENERIC_LIMIT), - flow: parseGenericLimit(env, "flow", DEFAULT_GENERIC_LIMIT), - "smart-money": parseGenericLimit(env, "smart-money", DEFAULT_GENERIC_LIMIT), - "classifier-hits": parseGenericLimit(env, "classifier-hits", DEFAULT_GENERIC_LIMIT), - alerts: parseGenericLimit(env, "alerts", DEFAULT_GENERIC_LIMIT), - "inferred-dark": parseGenericLimit(env, "inferred-dark", DEFAULT_GENERIC_LIMIT) +const parseGenericLimitFallback = (env: NodeJS.ProcessEnv, fallback: number): number => { + const raw = env.LIVE_LIMIT_DEFAULT; + if (!raw || raw.trim().length === 0) { + return fallback; + } + + const parsed = Number(raw); + if (!Number.isFinite(parsed)) { + console.warn(`Invalid LIVE_LIMIT_DEFAULT="${raw}", using ${fallback}`); + return fallback; + } + + return Math.max(MIN_GENERIC_LIMIT, Math.min(MAX_GENERIC_LIMIT, Math.floor(parsed))); +}; + +export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env): GenericLiveLimits => { + const liveLimitDefault = parseGenericLimitFallback(env, DEFAULT_GENERIC_LIMIT); + return { + options: parseGenericLimit(env, "options", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.options), + nbbo: parseGenericLimit(env, "nbbo", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.nbbo), + equities: parseGenericLimit( + env, + "equities", + env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.equities + ), + "equity-quotes": parseGenericLimit( + env, + "equity-quotes", + env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["equity-quotes"] + ), + "equity-joins": parseGenericLimit( + env, + "equity-joins", + env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["equity-joins"] + ), + flow: parseGenericLimit(env, "flow", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.flow), + "smart-money": parseGenericLimit( + env, + "smart-money", + env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["smart-money"] + ), + "classifier-hits": parseGenericLimit( + env, + "classifier-hits", + env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["classifier-hits"] + ), + alerts: parseGenericLimit(env, "alerts", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.alerts), + "inferred-dark": parseGenericLimit( + env, + "inferred-dark", + env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["inferred-dark"] + ) + }; +}; + +const parsePositiveInt = (value: string | undefined, fallback: number): number => { + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + return fallback; + } + return Math.max(1, Math.floor(parsed)); +}; + +export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): LiveStateConfig => ({ + limits: resolveGenericLiveLimits(env), + scopedCacheMaxKeys: parsePositiveInt(env.LIVE_SCOPED_CACHE_MAX_KEYS, DEFAULT_SCOPED_CACHE_MAX_KEYS), + redisFlushIntervalMs: parsePositiveInt( + env.LIVE_REDIS_FLUSH_INTERVAL_MS, + DEFAULT_REDIS_FLUSH_INTERVAL_MS + ), + redisFlushMaxItems: parsePositiveInt(env.LIVE_REDIS_FLUSH_MAX_ITEMS, DEFAULT_REDIS_FLUSH_MAX_ITEMS) }); type RedisLike = Pick< @@ -335,6 +432,30 @@ const snapshotLimitFor = (subscription: LiveSubscription, configuredLimit: numbe return Math.max(1, Math.min(configuredLimit, Math.floor(requested))); }; +export const buildOptionSnapshotFilters = ( + subscription: Extract +): OptionPrintQueryFilters => { + if (subscription.option_contract_id) { + return { + view: "raw", + optionContractId: subscription.option_contract_id + }; + } + + return { + view: subscription.filters?.view ?? "signal", + security: + subscription.filters?.securityTypes?.length === 1 + ? subscription.filters.securityTypes[0] + : "all", + nbboSides: subscription.filters?.nbboSides, + optionTypes: subscription.filters?.optionTypes, + minNotional: subscription.filters?.minNotional, + underlyingIds: subscription.underlying_ids, + optionContractId: subscription.option_contract_id + }; +}; + const candleRedisKey = (underlyingId: string, intervalMs: number): string => `live:equity-candles:${underlyingId}:${intervalMs}`; @@ -344,7 +465,50 @@ const candleCursorField = (underlyingId: string, intervalMs: number): string => const overlayRedisKey = (underlyingId: string): string => `live:equity-overlay:${underlyingId}`; const overlayCursorField = (underlyingId: string): string => `equities:${underlyingId}`; +const dropMatchingCursor = ( + items: T[], + target: Cursor, + cursorOf: (item: T) => Cursor +): T[] => items.filter((item) => compareCursors(cursorOf(item), target) !== 0); + +const insertNewestFirst = ( + items: T[], + item: T, + cursorOf: (item: T) => Cursor, + limit: number +): { items: T[]; outOfOrder: boolean } => { + const cursor = cursorOf(item); + const deduped = dropMatchingCursor(items, cursor, cursorOf); + const head = deduped[0]; + const outOfOrder = head ? compareCursors(cursor, cursorOf(head)) > 0 : false; + + if (!outOfOrder) { + return { + items: [item, ...deduped].slice(0, limit), + outOfOrder: false + }; + } + + return { + items: sortGenericItems([...deduped, item], cursorOf).slice(0, limit), + outOfOrder: true + }; +}; + +type BufferedRedisWrite = { + listKey: string; + cursorField: string; + items: unknown[]; + limit: number; + cursor: Cursor | null; + updates: number; +}; + +const isLiveStateConfig = (value: GenericLiveLimits | LiveStateConfig): value is LiveStateConfig => + "limits" in value; + export class LiveStateManager { + private readonly config: LiveStateConfig; private readonly generic: { [K in LiveGenericChannel]: GenericFeedConfig; }; @@ -352,12 +516,22 @@ export class LiveStateManager { private readonly genericCursors = new Map(); private readonly candleItems = new Map(); private readonly candleCursors = new Map(); + private readonly candleAccess = new Map(); private readonly overlayItems = new Map(); private readonly overlayCursors = new Map(); + private readonly overlayAccess = new Map(); + private readonly pendingRedisWrites = new Map(); + private readonly redisFlushTimer: ReturnType | null; private readonly stats = { genericHydrateFromRedis: 0, genericHydrateFromClickHouse: 0, + genericCacheSnapshots: 0, + scopedClickHouseSnapshots: 0, trimOperations: 0, + redisFlushCount: 0, + redisFlushItems: 0, + cacheEvictions: 0, + outOfOrderEvents: 0, cacheDepthByKey: new Map(), freshnessAgeMsByKey: new Map() }; @@ -365,27 +539,129 @@ export class LiveStateManager { constructor( private readonly clickhouse: ClickHouseClient, private readonly redis: RedisLike | null, - limits: GenericLiveLimits = resolveGenericLiveLimits() + config: GenericLiveLimits | LiveStateConfig = resolveLiveStateConfig() ) { - this.generic = getGenericConfig(limits); + this.config = isLiveStateConfig(config) + ? config + : { + limits: config, + scopedCacheMaxKeys: DEFAULT_SCOPED_CACHE_MAX_KEYS, + redisFlushIntervalMs: DEFAULT_REDIS_FLUSH_INTERVAL_MS, + redisFlushMaxItems: DEFAULT_REDIS_FLUSH_MAX_ITEMS + }; + this.generic = getGenericConfig(this.config.limits); + this.redisFlushTimer = + this.redis && this.redis.isOpen + ? setInterval(() => { + void this.flushRedisWrites(); + }, this.config.redisFlushIntervalMs) + : null; + this.redisFlushTimer?.unref?.(); + } + + async close(): Promise { + if (this.redisFlushTimer) { + clearInterval(this.redisFlushTimer); + } + await this.flushRedisWrites(); } getStatsSnapshot(): { genericHydrateFromRedis: number; genericHydrateFromClickHouse: number; + genericCacheSnapshots: number; + scopedClickHouseSnapshots: number; trimOperations: number; + redisFlushCount: number; + redisFlushItems: number; + cacheEvictions: number; + outOfOrderEvents: number; cacheDepthByKey: Record; freshnessAgeMsByKey: Record; } { return { genericHydrateFromRedis: this.stats.genericHydrateFromRedis, genericHydrateFromClickHouse: this.stats.genericHydrateFromClickHouse, + genericCacheSnapshots: this.stats.genericCacheSnapshots, + scopedClickHouseSnapshots: this.stats.scopedClickHouseSnapshots, trimOperations: this.stats.trimOperations, + redisFlushCount: this.stats.redisFlushCount, + redisFlushItems: this.stats.redisFlushItems, + cacheEvictions: this.stats.cacheEvictions, + outOfOrderEvents: this.stats.outOfOrderEvents, cacheDepthByKey: Object.fromEntries(this.stats.cacheDepthByKey), freshnessAgeMsByKey: Object.fromEntries(this.stats.freshnessAgeMsByKey) }; } + getHotChannelHealth(): LiveHotChannelHealthMap { + return { + options: this.getChannelHealth("options"), + nbbo: this.getChannelHealth("nbbo"), + equities: this.getChannelHealth("equities"), + flow: this.getChannelHealth("flow") + }; + } + + async flushRedisWrites(): Promise { + if (!this.redis?.isOpen) { + return; + } + + const writes = Array.from(this.pendingRedisWrites.values()); + this.pendingRedisWrites.clear(); + + for (const write of writes) { + await this.persistList(write.listKey, write.cursorField, write.items, write.limit, write.cursor); + this.stats.redisFlushCount += 1; + this.stats.redisFlushItems += write.items.length; + metrics.count("api.live.redis_flush_count", 1); + metrics.count("api.live.redis_flush_items", write.items.length); + } + } + + private getChannelHealth(channel: LiveHotChannel): LiveChannelHealth { + const listKey = HOT_LIVE_REDIS_KEYS[channel]; + const thresholdMs = LIVE_FRESHNESS_THRESHOLDS[channel]; + const freshnessAgeMs = this.stats.freshnessAgeMsByKey.get(listKey) ?? null; + return { + freshness_age_ms: freshnessAgeMs, + healthy: + freshnessAgeMs !== null && + typeof thresholdMs === "number" && + Number.isFinite(freshnessAgeMs) && + freshnessAgeMs <= thresholdMs + }; + } + + private touchAccess(accessMap: Map, key: string): void { + accessMap.set(key, Date.now()); + } + + private evictScopedCachesIfNeeded( + itemsMap: Map, + cursorsMap: Map, + accessMap: Map + ): void { + while (itemsMap.size > this.config.scopedCacheMaxKeys) { + const oldest = [...accessMap.entries()].sort((a, b) => a[1] - b[1])[0]; + if (!oldest) { + break; + } + const [key] = oldest; + itemsMap.delete(key); + cursorsMap.delete( + key.startsWith("live:equity-candles:") + ? key.replace("live:", "") + : key.replace("live:equity-overlay:", "equities:") + ); + accessMap.delete(key); + this.stats.cacheDepthByKey.delete(key); + this.stats.cacheEvictions += 1; + metrics.count("api.live.cache_evictions", 1); + } + } + private updateFreshnessMetric(listKey: string, channel: LiveChannel, item: unknown, now = Date.now()): void { const ts = channel === "equity-candles" || channel === "equity-overlay" @@ -399,6 +675,32 @@ export class LiveStateManager { } } + private queueRedisWrite( + listKey: string, + cursorField: string, + items: unknown[], + limit: number, + cursor: Cursor | null + ): void { + if (!this.redis?.isOpen) { + return; + } + + const existing = this.pendingRedisWrites.get(listKey); + const write: BufferedRedisWrite = { + listKey, + cursorField, + items: [...items], + limit, + cursor, + updates: (existing?.updates ?? 0) + 1 + }; + this.pendingRedisWrites.set(listKey, write); + if (write.updates >= this.config.redisFlushMaxItems) { + void this.flushRedisWrites(); + } + } + async hydrate(): Promise { const channels = Object.keys(this.generic) as LiveGenericChannel[]; await Promise.all(channels.map((channel) => this.hydrateGeneric(channel))); @@ -408,37 +710,22 @@ export class LiveStateManager { const config = this.generic[channel]; if (this.redis?.isOpen) { const payloads = await this.redis.lRange(config.redisKey, 0, config.limit - 1); - const cached = normalizeGenericItems( - channel, - parseJsonList(payloads, config.parse).filter((item) => - isWithinLiveFeedLookback(channel, item) - ), - config - ); + const cached = normalizeGenericItems(channel, parseJsonList(payloads, config.parse), config); if (cached.length > 0) { this.genericItems.set(channel, cached); this.stats.genericHydrateFromRedis += 1; this.stats.cacheDepthByKey.set(config.redisKey, cached.length); this.updateFreshnessMetric(config.redisKey, channel, cached[0]); - this.genericCursors.set(config.cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, config.cursorField))); - await this.persistList( - config.redisKey, + this.genericCursors.set( config.cursorField, - cached, - config.limit, - this.genericCursors.get(config.cursorField) ?? null + parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, config.cursorField)) ); + await this.persistList(config.redisKey, config.cursorField, cached, config.limit, this.genericCursors.get(config.cursorField) ?? null); return; } } - const fresh = normalizeGenericItems( - channel, - (await config.fetchRecent(this.clickhouse, config.limit)).filter((item) => - isWithinLiveFeedLookback(channel, item) - ), - config - ); + const fresh = normalizeGenericItems(channel, await config.fetchRecent(this.clickhouse, config.limit), config); this.stats.genericHydrateFromClickHouse += 1; this.stats.cacheDepthByKey.set(config.redisKey, fresh.length); this.genericItems.set(channel, fresh); @@ -453,43 +740,26 @@ export class LiveStateManager { async getSnapshot(subscription: LiveSubscription): Promise> { switch (subscription.channel) { case "options": { - const scoped = - Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id); + const scoped = Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id); if (subscription.filters?.view === "raw" || scoped) { + this.stats.scopedClickHouseSnapshots += 1; const limit = snapshotLimitFor(subscription, this.generic.options.limit); - const storageFilters: OptionPrintQueryFilters = { - view: subscription.filters?.view ?? "signal", - security: - subscription.filters?.securityTypes?.length === 1 - ? subscription.filters.securityTypes[0] - : "all", - nbboSides: subscription.filters?.nbboSides, - optionTypes: subscription.filters?.optionTypes, - minNotional: subscription.filters?.minNotional, - underlyingIds: subscription.underlying_ids, - optionContractId: subscription.option_contract_id, - sinceTs: Date.now() - LIVE_FEED_LOOKBACK_MS - }; - const items = await fetchRecentOptionPrints( - this.clickhouse, - limit, - undefined, - storageFilters - ); + const storageFilters = buildOptionSnapshotFilters(subscription); + const items = await fetchRecentOptionPrints(this.clickhouse, limit, undefined, storageFilters); return { subscription, items, watermark: items[0] ? { ts: items[0].ts, seq: items[0].seq } : null, - next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq })) + next_before: nextBeforeForItems(items, (entry) => ({ ts: entry.ts, seq: entry.seq })) }; } const config = this.generic.options; + this.stats.genericCacheSnapshots += 1; const limit = snapshotLimitFor(subscription, config.limit); - const items = (this.genericItems.get("options") ?? []).filter((item) => - isWithinLiveFeedLookback("options", item) && - matchesOptionPrintFilters(item, subscription.filters) - ).slice(0, limit); + const items = (this.genericItems.get("options") ?? []) + .filter((entry) => matchesOptionPrintFilters(entry, subscription.filters)) + .slice(0, limit); return { subscription, items, @@ -499,11 +769,11 @@ export class LiveStateManager { } case "flow": { const config = this.generic.flow; + this.stats.genericCacheSnapshots += 1; const limit = snapshotLimitFor(subscription, config.limit); - const items = (this.genericItems.get("flow") ?? []).filter((item) => - isWithinLiveFeedLookback("flow", item) && - matchesFlowPacketFilters(item, subscription.filters) - ).slice(0, limit); + const items = (this.genericItems.get("flow") ?? []) + .filter((entry) => matchesFlowPacketFilters(entry, subscription.filters)) + .slice(0, limit); return { subscription, items, @@ -515,10 +785,8 @@ export class LiveStateManager { const config = this.generic.equities; const limit = snapshotLimitFor(subscription, config.limit); if (subscription.underlying_ids?.length) { - const filters: EquityPrintQueryFilters = { - underlyingIds: subscription.underlying_ids, - sinceTs: Date.now() - LIVE_FEED_LOOKBACK_MS - }; + this.stats.scopedClickHouseSnapshots += 1; + const filters: EquityPrintQueryFilters = { underlyingIds: subscription.underlying_ids }; const items = await fetchRecentEquityPrints(this.clickhouse, limit, filters); return { subscription, @@ -527,9 +795,8 @@ export class LiveStateManager { next_before: nextBeforeForItems(items, config.cursor) }; } - const items = (this.genericItems.get("equities") ?? []).filter((item) => - isWithinLiveFeedLookback("equities", item) - ).slice(0, limit); + this.stats.genericCacheSnapshots += 1; + const items = (this.genericItems.get("equities") ?? []).slice(0, limit); return { subscription, items, @@ -543,12 +810,13 @@ export class LiveStateManager { if (!this.candleItems.has(key)) { await this.hydrateCandles(subscription.underlying_id, subscription.interval_ms); } + this.touchAccess(this.candleAccess, key); const items = this.candleItems.get(key) ?? []; return { subscription, items, watermark: this.candleCursors.get(cursorField) ?? null, - next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq })) + next_before: nextBeforeForItems(items, (entry) => ({ ts: entry.ts, seq: entry.seq })) }; } case "equity-overlay": { @@ -557,20 +825,20 @@ export class LiveStateManager { if (!this.overlayItems.has(key)) { await this.hydrateOverlay(subscription.underlying_id); } + this.touchAccess(this.overlayAccess, key); const items = this.overlayItems.get(key) ?? []; return { subscription, items, watermark: this.overlayCursors.get(cursorField) ?? null, - next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq })) + next_before: nextBeforeForItems(items, (entry) => ({ ts: entry.ts, seq: entry.seq })) }; } default: { const config = this.generic[subscription.channel]; + this.stats.genericCacheSnapshots += 1; const limit = snapshotLimitFor(subscription, config.limit); - const items = (this.genericItems.get(subscription.channel) ?? []).filter((item) => - isWithinLiveFeedLookback(subscription.channel, item) - ).slice(0, limit); + const items = (this.genericItems.get(subscription.channel) ?? []).slice(0, limit); return { subscription, items, @@ -587,48 +855,52 @@ export class LiveStateManager { const candle = EquityCandleSchema.parse(item); const key = candleRedisKey(candle.underlying_id, candle.interval_ms); const cursorField = candleCursorField(candle.underlying_id, candle.interval_ms); - const previousCursor = this.candleCursors.get(cursorField) ?? null; - const items = this.candleItems.get(key) ?? []; - const next = [candle, ...items] - .sort((a, b) => (b.ts - a.ts) || (b.seq - a.seq)) - .slice(0, CHART_LIMITS.candles); - this.candleItems.set(key, next); - this.stats.cacheDepthByKey.set(key, next.length); + const nextState = insertNewestFirst( + this.candleItems.get(key) ?? [], + candle, + (entry) => ({ ts: entry.ts, seq: entry.seq }), + CHART_LIMITS.candles + ); const cursor = { ts: candle.ts, seq: candle.seq }; + this.candleItems.set(key, nextState.items); this.candleCursors.set(cursorField, cursor); - if (next.length > 0) { - this.updateFreshnessMetric(key, "equity-candles", next[0]); + this.touchAccess(this.candleAccess, key); + this.evictScopedCachesIfNeeded(this.candleItems as Map, this.candleCursors, this.candleAccess); + if (nextState.outOfOrder) { + this.stats.outOfOrderEvents += 1; + metrics.count("api.live.out_of_order_events", 1); } - const outOfOrder = previousCursor ? compareCursors(cursor, previousCursor) > 0 : false; - if (outOfOrder) { - await this.persistList(key, cursorField, next, CHART_LIMITS.candles, cursor); - } else { - await this.persistItem(key, cursorField, candle, CHART_LIMITS.candles, cursor, next.length); + this.stats.cacheDepthByKey.set(key, nextState.items.length); + if (nextState.items.length > 0) { + this.updateFreshnessMetric(key, "equity-candles", nextState.items[0]); } + this.queueRedisWrite(key, cursorField, nextState.items, CHART_LIMITS.candles, cursor); return cursor; } case "equity-overlay": { const print = EquityPrintSchema.parse(item); const key = overlayRedisKey(print.underlying_id); const cursorField = overlayCursorField(print.underlying_id); - const previousCursor = this.overlayCursors.get(cursorField) ?? null; - const items = this.overlayItems.get(key) ?? []; - const next = [print, ...items] - .sort((a, b) => (b.ts - a.ts) || (b.seq - a.seq)) - .slice(0, CHART_LIMITS.overlay); - this.overlayItems.set(key, next); - this.stats.cacheDepthByKey.set(key, next.length); + const nextState = insertNewestFirst( + this.overlayItems.get(key) ?? [], + print, + (entry) => ({ ts: entry.ts, seq: entry.seq }), + CHART_LIMITS.overlay + ); const cursor = { ts: print.ts, seq: print.seq }; + this.overlayItems.set(key, nextState.items); this.overlayCursors.set(cursorField, cursor); - if (next.length > 0) { - this.updateFreshnessMetric(key, "equity-overlay", next[0]); + this.touchAccess(this.overlayAccess, key); + this.evictScopedCachesIfNeeded(this.overlayItems as Map, this.overlayCursors, this.overlayAccess); + if (nextState.outOfOrder) { + this.stats.outOfOrderEvents += 1; + metrics.count("api.live.out_of_order_events", 1); } - const outOfOrder = previousCursor ? compareCursors(cursor, previousCursor) > 0 : false; - if (outOfOrder) { - await this.persistList(key, cursorField, next, CHART_LIMITS.overlay, cursor); - } else { - await this.persistItem(key, cursorField, print, CHART_LIMITS.overlay, cursor, next.length); + this.stats.cacheDepthByKey.set(key, nextState.items.length); + if (nextState.items.length > 0) { + this.updateFreshnessMetric(key, "equity-overlay", nextState.items[0]); } + this.queueRedisWrite(key, cursorField, nextState.items, CHART_LIMITS.overlay, cursor); return cursor; } default: { @@ -637,22 +909,28 @@ export class LiveStateManager { if (!isWithinLiveFeedLookback(channel, parsed)) { return null; } - const previousCursor = this.genericCursors.get(config.cursorField) ?? null; - const items = this.genericItems.get(channel) ?? []; - const next = normalizeGenericItems(channel, [parsed, ...items], config); - this.genericItems.set(channel, next); - this.stats.cacheDepthByKey.set(config.redisKey, next.length); + const cursor = config.cursor(parsed); + const nextState = + channel === "nbbo" + ? { + items: normalizeGenericItems(channel, [parsed, ...(this.genericItems.get(channel) ?? [])], config), + outOfOrder: false + } + : insertNewestFirst(this.genericItems.get(channel) ?? [], parsed, config.cursor, config.limit); + + if (nextState.outOfOrder) { + this.stats.outOfOrderEvents += 1; + metrics.count("api.live.out_of_order_events", 1); + } + + this.genericItems.set(channel, nextState.items); this.genericCursors.set(config.cursorField, cursor); - if (next.length > 0) { - this.updateFreshnessMetric(config.redisKey, channel, next[0]); - } - const outOfOrder = previousCursor ? compareCursors(cursor, previousCursor) > 0 : false; - if (channel === "nbbo" || outOfOrder) { - await this.persistList(config.redisKey, config.cursorField, next, config.limit, cursor); - } else { - await this.persistItem(config.redisKey, config.cursorField, parsed, config.limit, cursor, next.length); + this.stats.cacheDepthByKey.set(config.redisKey, nextState.items.length); + if (nextState.items.length > 0) { + this.updateFreshnessMetric(config.redisKey, channel, nextState.items[0]); } + this.queueRedisWrite(config.redisKey, config.cursorField, nextState.items, config.limit, cursor); return cursor; } } @@ -666,6 +944,8 @@ export class LiveStateManager { const cached = parseJsonList(payloads, (value) => EquityCandleSchema.parse(value)); if (cached.length > 0) { this.candleItems.set(key, cached); + this.touchAccess(this.candleAccess, key); + this.evictScopedCachesIfNeeded(this.candleItems as Map, this.candleCursors, this.candleAccess); this.stats.cacheDepthByKey.set(key, cached.length); this.updateFreshnessMetric(key, "equity-candles", cached[0]); this.candleCursors.set(cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField))); @@ -675,6 +955,8 @@ export class LiveStateManager { const fresh = await fetchRecentEquityCandles(this.clickhouse, underlyingId, intervalMs, CHART_LIMITS.candles); this.candleItems.set(key, fresh); + this.touchAccess(this.candleAccess, key); + this.evictScopedCachesIfNeeded(this.candleItems as Map, this.candleCursors, this.candleAccess); this.stats.cacheDepthByKey.set(key, fresh.length); if (fresh.length > 0) { this.updateFreshnessMetric(key, "equity-candles", fresh[0]); @@ -692,6 +974,8 @@ export class LiveStateManager { const cached = parseJsonList(payloads, (value) => EquityPrintSchema.parse(value)); if (cached.length > 0) { this.overlayItems.set(key, cached); + this.touchAccess(this.overlayAccess, key); + this.evictScopedCachesIfNeeded(this.overlayItems as Map, this.overlayCursors, this.overlayAccess); this.stats.cacheDepthByKey.set(key, cached.length); this.updateFreshnessMetric(key, "equity-overlay", cached[0]); this.overlayCursors.set(cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField))); @@ -700,9 +984,11 @@ export class LiveStateManager { } const fresh = (await fetchRecentEquityPrints(this.clickhouse, CHART_LIMITS.overlay)).filter( - (item) => item.underlying_id === underlyingId + (entry) => entry.underlying_id === underlyingId ); this.overlayItems.set(key, fresh); + this.touchAccess(this.overlayAccess, key); + this.evictScopedCachesIfNeeded(this.overlayItems as Map, this.overlayCursors, this.overlayAccess); this.stats.cacheDepthByKey.set(key, fresh.length); if (fresh.length > 0) { this.updateFreshnessMetric(key, "equity-overlay", fresh[0]); @@ -712,25 +998,6 @@ export class LiveStateManager { await this.persistList(key, cursorField, fresh, CHART_LIMITS.overlay, watermark); } - private async persistItem( - listKey: string, - cursorField: string, - item: T, - limit: number, - cursor: Cursor | null, - depth: number - ): Promise { - if (!this.redis?.isOpen) { - return; - } - - await this.redis.lPush(listKey, JSON.stringify(item)); - await this.redis.lTrim(listKey, 0, limit - 1); - this.stats.trimOperations += 1; - this.stats.cacheDepthByKey.set(listKey, Math.min(depth, limit)); - await this.redis.hSet(CURSOR_HASH_KEY, cursorField, JSON.stringify(cursor)); - } - private async persistList( listKey: string, cursorField: string, @@ -742,7 +1009,7 @@ export class LiveStateManager { return; } - const payloads = items.map((item) => JSON.stringify(item)); + const payloads = items.map((entry) => JSON.stringify(entry)); await this.redis.lTrim(listKey, 1, 0); this.stats.trimOperations += 1; if (payloads.length > 0) { diff --git a/services/api/src/option-queries.ts b/services/api/src/option-queries.ts new file mode 100644 index 0000000..193cbb2 --- /dev/null +++ b/services/api/src/option-queries.ts @@ -0,0 +1,107 @@ +import type { OptionPrintQueryFilters } from "@islandflow/storage"; +import { + OptionFlowViewSchema, + OptionNbboSideSchema, + OptionSecurityTypeSchema, + OptionTypeSchema, + type OptionFlowFilters +} from "@islandflow/types"; +import { z } from "zod"; + +const optionSideListSchema = z + .string() + .transform((value) => + value + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean) + ) + .pipe(z.array(OptionNbboSideSchema)); + +const optionTypeListSchema = z + .string() + .transform((value) => + value + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean) + ) + .pipe(z.array(OptionTypeSchema)); + +const optionSecuritySchema = z.enum(["stock", "etf", "all"]); + +const optionFilterQuerySchema = z.object({ + view: OptionFlowViewSchema.optional(), + security: optionSecuritySchema.optional(), + side: optionSideListSchema.optional(), + type: optionTypeListSchema.optional(), + min_notional: z.coerce.number().nonnegative().optional() +}); + +export type ParsedOptionPrintQuery = { + scope: { + underlyingIds?: string[]; + optionContractId?: string; + }; + flowFilters: OptionFlowFilters; + storageFilters: OptionPrintQueryFilters; + isContractDrilldown: boolean; +}; + +const parseScopeList = (url: URL, ...keys: string[]): string[] | undefined => { + const values = keys + .flatMap((key) => url.searchParams.getAll(key)) + .flatMap((value) => value.split(",")) + .map((value) => value.trim().toUpperCase()) + .filter(Boolean); + const unique = Array.from(new Set(values)); + return unique.length > 0 ? unique : undefined; +}; + +export const parseOptionPrintQuery = (url: URL): ParsedOptionPrintQuery => { + const parsed = optionFilterQuerySchema.parse({ + view: url.searchParams.get("view") ?? undefined, + security: url.searchParams.get("security") ?? undefined, + side: url.searchParams.get("side") ?? undefined, + type: url.searchParams.get("type") ?? undefined, + min_notional: url.searchParams.get("min_notional") ?? undefined + }); + const scope = { + underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids"), + optionContractId: url.searchParams.get("option_contract_id") ?? undefined + }; + const view = parsed.view ?? "signal"; + const security = parsed.security ?? (view === "raw" ? "all" : "stock"); + const flowFilters: OptionFlowFilters = { + view, + securityTypes: + security === "all" + ? undefined + : ([security] as Array>), + nbboSides: parsed.side, + optionTypes: parsed.type, + minNotional: parsed.min_notional + }; + const isContractDrilldown = Boolean(scope.optionContractId); + const storageFilters: OptionPrintQueryFilters = isContractDrilldown + ? { + view: "raw", + optionContractId: scope.optionContractId + } + : { + view, + security, + minNotional: parsed.min_notional, + nbboSides: parsed.side, + optionTypes: parsed.type, + underlyingIds: scope.underlyingIds, + optionContractId: scope.optionContractId + }; + + return { + scope, + flowFilters, + storageFilters, + isContractDrilldown + }; +}; diff --git a/services/api/tests/live.test.ts b/services/api/tests/live.test.ts index 3cb789e..bd4d0c8 100644 --- a/services/api/tests/live.test.ts +++ b/services/api/tests/live.test.ts @@ -1,21 +1,25 @@ import { describe, expect, it } from "bun:test"; import type { ClickHouseClient } from "@islandflow/storage"; import { + buildOptionSnapshotFilters, + HOT_LIVE_REDIS_KEYS, LiveStateManager, isLiveItemFresh, resolveGenericLiveLimits, shouldFanoutLiveEvent } from "../src/live"; -const makeClickHouse = (): ClickHouseClient => +const makeClickHouse = ( + queryResolver?: (query: string) => unknown[] +): ClickHouseClient => ({ exec: async () => {}, insert: async () => {}, ping: async () => ({ success: true }), close: async () => {}, - query: async () => ({ + query: async ({ query }: { query: string }) => ({ async json() { - return [] as T; + return (queryResolver?.(query) ?? []) as T; } }) }) as ClickHouseClient; @@ -62,9 +66,9 @@ describe("LiveStateManager", () => { expect(limits.options).toBe(777); expect(limits.nbbo).toBe(100000); - expect(limits.flow).toBe(10000); - expect(limits["equity-quotes"]).toBe(10000); - expect(limits.alerts).toBe(10000); + expect(limits.flow).toBe(500); + expect(limits["equity-quotes"]).toBe(500); + expect(limits.alerts).toBe(300); }); it("hydrates snapshots from redis generic windows", async () => { @@ -200,13 +204,121 @@ describe("LiveStateManager", () => { ]); const persisted = await redis.lRange("live:flow", 0, 99); - expect(persisted).toHaveLength(2); + await manager.flushRedisWrites(); + const flushed = await redis.lRange("live:flow", 0, 99); + expect(persisted).toHaveLength(0); + expect(flushed).toHaveLength(2); const stats = manager.getStatsSnapshot(); expect(stats.trimOperations).toBeGreaterThan(0); + expect(stats.redisFlushCount).toBeGreaterThan(0); expect(stats.cacheDepthByKey["live:flow"]).toBe(2); }); + it("reorders out-of-order live events without dropping newest-first semantics", async () => { + const now = Date.now(); + const manager = new LiveStateManager(makeClickHouse(), null, { + limits: { + options: 1000, + nbbo: 1000, + equities: 1000, + "equity-quotes": 500, + "equity-joins": 500, + flow: 3, + "smart-money": 300, + "classifier-hits": 300, + alerts: 300, + "inferred-dark": 300 + }, + scopedCacheMaxKeys: 32, + redisFlushIntervalMs: 250, + redisFlushMaxItems: 100 + }); + + await manager.ingest("flow", { + source_ts: now, + ingest_ts: now + 1, + seq: 2, + trace_id: "flow-2", + id: "flow-2", + members: [], + features: {}, + join_quality: {} + }); + await manager.ingest("flow", { + source_ts: now - 1_000, + ingest_ts: now - 999, + seq: 1, + trace_id: "flow-1", + id: "flow-1", + members: [], + features: {}, + join_quality: {} + }); + + const snapshot = await manager.getSnapshot({ channel: "flow" }); + expect((snapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([ + "flow-2", + "flow-1" + ]); + expect(manager.getStatsSnapshot().outOfOrderEvents).toBe(1); + }); + + it("evicts least-recently-used scoped candle caches past the configured key limit", async () => { + const manager = new LiveStateManager(makeClickHouse(), null, { + limits: resolveGenericLiveLimits(), + scopedCacheMaxKeys: 1, + redisFlushIntervalMs: 250, + redisFlushMaxItems: 100 + }); + + await manager.ingest("equity-candles", { + source_ts: 100, + ingest_ts: 101, + seq: 1, + trace_id: "candle:SPY:60000:100", + ts: 100, + interval_ms: 60000, + underlying_id: "SPY", + open: 1, + high: 2, + low: 1, + close: 2, + volume: 10, + trade_count: 1 + }); + await manager.ingest("equity-candles", { + source_ts: 200, + ingest_ts: 201, + seq: 2, + trace_id: "candle:QQQ:60000:200", + ts: 200, + interval_ms: 60000, + underlying_id: "QQQ", + open: 3, + high: 4, + low: 3, + close: 4, + volume: 20, + trade_count: 2 + }); + + const qqqSnapshot = await manager.getSnapshot({ + channel: "equity-candles", + underlying_id: "QQQ", + interval_ms: 60000 + }); + const spySnapshot = await manager.getSnapshot({ + channel: "equity-candles", + underlying_id: "SPY", + interval_ms: 60000 + }); + + expect(qqqSnapshot.items).toHaveLength(1); + expect(spySnapshot.items).toEqual([]); + expect(manager.getStatsSnapshot().cacheEvictions).toBeGreaterThan(0); + }); + it("filters option and flow snapshots using subscription filters", async () => { const manager = new LiveStateManager(makeClickHouse(), null); const now = Date.now(); @@ -408,6 +520,228 @@ describe("LiveStateManager", () => { ]); }); + it("seeds scoped option snapshots from clickhouse rows older than 24h", async () => { + const now = Date.now(); + const staleTs = now - 25 * 60 * 60 * 1000; + const manager = new LiveStateManager( + makeClickHouse((query) => + query.includes("FROM option_prints") + ? [ + { + source_ts: staleTs, + ingest_ts: staleTs + 1, + seq: 1, + trace_id: "opt-ancient", + ts: staleTs, + option_contract_id: "AAPL-2025-01-17-200-C", + underlying_id: "AAPL", + price: 1, + size: 10, + exchange: "X", + signal_pass: true + } + ] + : [] + ), + null + ); + + const snapshot = await manager.getSnapshot({ + channel: "options", + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }); + + expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ + "opt-ancient" + ]); + expect(snapshot.next_before).toEqual({ ts: staleTs, seq: 1 }); + expect(isLiveItemFresh("options", snapshot.items[0], now)).toBe(false); + }); + + it("builds raw contract-only snapshot filters for focused option subscriptions", () => { + expect( + buildOptionSnapshotFilters({ + channel: "options", + filters: { + view: "signal", + minNotional: 500_000, + nbboSides: ["A"], + optionTypes: ["call"], + securityTypes: ["stock"] + }, + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }) + ).toEqual({ + view: "raw", + optionContractId: "AAPL-2025-01-17-200-C" + }); + }); + + it("returns raw contract rows for focused option snapshots even when broad filters would reject them", async () => { + const manager = new LiveStateManager( + makeClickHouse((query) => { + expect(query).toContain("option_contract_id = 'AAPL-2025-01-17-200-C'"); + expect(query).not.toContain("signal_pass = 1"); + expect(query).not.toContain("notional >="); + expect(query).not.toContain("nbbo_side IN"); + expect(query).not.toContain("option_type IN"); + return [ + { + source_ts: 1_000, + ingest_ts: 1_001, + seq: 1, + trace_id: "opt-raw", + ts: 1_000, + option_contract_id: "AAPL-2025-01-17-200-C", + underlying_id: "AAPL", + option_type: "put", + nbbo_side: "B", + notional: 50_000, + signal_pass: false, + price: 1, + size: 5, + exchange: "X" + } + ]; + }), + null + ); + + const snapshot = await manager.getSnapshot({ + channel: "options", + filters: { + view: "signal", + minNotional: 500_000, + nbboSides: ["A"], + optionTypes: ["call"], + securityTypes: ["stock"] + }, + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }); + + expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ + "opt-raw" + ]); + }); + + it("seeds scoped equity snapshots from clickhouse rows older than 24h", async () => { + const now = Date.now(); + const staleTs = now - 25 * 60 * 60 * 1000; + const manager = new LiveStateManager( + makeClickHouse((query) => + query.includes("FROM equity_prints") + ? [ + { + source_ts: staleTs, + ingest_ts: staleTs + 1, + seq: 1, + trace_id: "eq-ancient", + ts: staleTs, + underlying_id: "AAPL", + price: 100, + size: 10, + exchange: "X", + offExchangeFlag: false + } + ] + : [] + ), + null + ); + + const snapshot = await manager.getSnapshot({ + channel: "equities", + underlying_ids: ["AAPL"] + }); + + expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ + "eq-ancient" + ]); + expect(snapshot.next_before).toEqual({ ts: staleTs, seq: 1 }); + expect(isLiveItemFresh("equities", snapshot.items[0], now)).toBe(false); + }); + + it("hydrates retained rows older than 24h into generic live snapshots and keeps them stale", async () => { + const redis = makeRedis(); + const now = Date.now(); + const staleTs = now - 25 * 60 * 60 * 1000; + + await redis.lPush( + "live:options", + JSON.stringify({ + source_ts: staleTs, + ingest_ts: staleTs + 1, + seq: 1, + trace_id: "opt-retained", + ts: staleTs, + option_contract_id: "AAPL-2025-01-17-200-C", + underlying_id: "AAPL", + price: 1, + size: 10, + exchange: "X", + signal_pass: true + }) + ); + await redis.hSet("live:cursors", "options", JSON.stringify({ ts: staleTs, seq: 1 })); + + await redis.lPush( + "live:equities", + JSON.stringify({ + source_ts: staleTs, + ingest_ts: staleTs + 1, + seq: 2, + trace_id: "eq-retained", + ts: staleTs, + underlying_id: "AAPL", + price: 100, + size: 10, + exchange: "X", + offExchangeFlag: false + }) + ); + await redis.hSet("live:cursors", "equities", JSON.stringify({ ts: staleTs, seq: 2 })); + + await redis.lPush( + "live:flow", + JSON.stringify({ + source_ts: staleTs, + ingest_ts: staleTs + 1, + seq: 3, + trace_id: "flow-retained", + id: "flow-retained", + members: ["opt-retained"], + features: {}, + join_quality: {} + }) + ); + await redis.hSet("live:cursors", "flow", JSON.stringify({ ts: staleTs, seq: 3 })); + + const manager = new LiveStateManager(makeClickHouse(), redis as never); + await manager.hydrate(); + + const [optionsSnapshot, equitiesSnapshot, flowSnapshot] = await Promise.all([ + manager.getSnapshot({ channel: "options" }), + manager.getSnapshot({ channel: "equities" }), + manager.getSnapshot({ channel: "flow" }) + ]); + + expect((optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ + "opt-retained" + ]); + expect((equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ + "eq-retained" + ]); + expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([ + "flow-retained" + ]); + expect(isLiveItemFresh("options", optionsSnapshot.items[0], now)).toBe(false); + expect(isLiveItemFresh("equities", equitiesSnapshot.items[0], now)).toBe(false); + expect(isLiveItemFresh("flow", flowSnapshot.items[0], now)).toBe(false); + }); + it("keeps only the newest NBBO quote per contract across hydrate and ingest", async () => { const redis = makeRedis(); const now = Date.now(); @@ -573,6 +907,122 @@ describe("LiveStateManager", () => { expect(persisted).toHaveLength(1); }); + it("includes hot-channel health for options, nbbo, equities, and flow", async () => { + const manager = new LiveStateManager(makeClickHouse(), null); + const now = Date.now(); + + await manager.ingest("options", { + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "opt-health", + ts: now, + option_contract_id: "AAPL-2025-01-17-200-C", + price: 1, + size: 10, + exchange: "X" + }); + await manager.ingest("nbbo", { + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "nbbo-health", + ts: now, + option_contract_id: "AAPL-2025-01-17-200-C", + bid: 1, + ask: 1.1, + bidSize: 10, + askSize: 10 + }); + await manager.ingest("equities", { + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "eq-health", + ts: now, + underlying_id: "AAPL", + price: 100, + size: 10, + exchange: "X", + offExchangeFlag: false + }); + await manager.ingest("flow", { + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "flow-health", + id: "flow-health", + members: [], + features: {}, + join_quality: {} + }); + + const health = manager.getHotChannelHealth(); + expect(health.options.healthy).toBe(true); + expect(health.nbbo.healthy).toBe(true); + expect(health.equities.healthy).toBe(true); + expect(health.flow.healthy).toBe(true); + expect(health.options.freshness_age_ms).not.toBeNull(); + expect(health.nbbo.freshness_age_ms).not.toBeNull(); + expect(health.equities.freshness_age_ms).not.toBeNull(); + expect(health.flow.freshness_age_ms).not.toBeNull(); + }); + + it("tracks generic cache and scoped clickhouse snapshot sources separately", async () => { + const manager = new LiveStateManager(makeClickHouse(() => []), null); + const now = Date.now(); + + await manager.ingest("options", { + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "opt-snapshot", + ts: now, + option_contract_id: "SPY-2025-01-17-500-C", + price: 1, + size: 10, + exchange: "X" + }); + + await manager.getSnapshot({ channel: "options" }); + await manager.getSnapshot({ + channel: "options", + underlying_ids: ["QQQ"], + option_contract_id: "QQQ-2025-01-17-400-C" + }); + + const stats = manager.getStatsSnapshot(); + expect(stats.genericCacheSnapshots).toBe(1); + expect(stats.scopedClickHouseSnapshots).toBe(1); + }); + + it("keeps backend channel health healthy when a scoped query is quiet", async () => { + const manager = new LiveStateManager(makeClickHouse(() => []), null); + const now = Date.now(); + + await manager.ingest("options", { + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "opt-global", + ts: now, + option_contract_id: "SPY-2025-01-17-500-C", + price: 1, + size: 10, + exchange: "X" + }); + + const quietSnapshot = await manager.getSnapshot({ + channel: "options", + underlying_ids: ["QQQ"], + option_contract_id: "QQQ-2025-01-17-400-C" + }); + + expect(quietSnapshot.items).toEqual([]); + expect(manager.getHotChannelHealth().options.healthy).toBe(true); + expect(manager.getStatsSnapshot().freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.options]).toBeLessThanOrEqual(50); + }); + it("exposes freshness helper for feed status", () => { expect(isLiveItemFresh("options", { ts: 1000 }, 1010)).toBe(true); expect(isLiveItemFresh("options", { ts: 1000 }, 20_001)).toBe(false); diff --git a/services/api/tests/option-queries.test.ts b/services/api/tests/option-queries.test.ts new file mode 100644 index 0000000..d189303 --- /dev/null +++ b/services/api/tests/option-queries.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "bun:test"; +import { parseOptionPrintQuery } from "../src/option-queries"; + +describe("parseOptionPrintQuery", () => { + it("keeps broad option flow filters for non-contract requests", () => { + const url = new URL( + "http://localhost/prints/options?view=signal&security=stock&side=A&type=call&min_notional=500000&underlying_ids=AAPL,MSFT" + ); + + expect(parseOptionPrintQuery(url)).toEqual({ + scope: { + underlyingIds: ["AAPL", "MSFT"], + optionContractId: undefined + }, + flowFilters: { + view: "signal", + securityTypes: ["stock"], + nbboSides: ["A"], + optionTypes: ["call"], + minNotional: 500000 + }, + storageFilters: { + view: "signal", + security: "stock", + nbboSides: ["A"], + optionTypes: ["call"], + minNotional: 500000, + underlyingIds: ["AAPL", "MSFT"], + optionContractId: undefined + }, + isContractDrilldown: false + }); + }); + + it("switches contract requests to raw contract-only storage filters", () => { + const url = new URL( + "http://localhost/replay/options?view=signal&security=stock&side=A&type=call&min_notional=500000&underlying_id=AAPL&option_contract_id=AAPL-2025-01-17-200-C" + ); + + expect(parseOptionPrintQuery(url)).toEqual({ + scope: { + underlyingIds: ["AAPL"], + optionContractId: "AAPL-2025-01-17-200-C" + }, + flowFilters: { + view: "signal", + securityTypes: ["stock"], + nbboSides: ["A"], + optionTypes: ["call"], + minNotional: 500000 + }, + storageFilters: { + view: "raw", + optionContractId: "AAPL-2025-01-17-200-C" + }, + isContractDrilldown: true + }); + }); +}); diff --git a/services/candles/src/index.ts b/services/candles/src/index.ts index 39e6609..86f0dfa 100644 --- a/services/candles/src/index.ts +++ b/services/candles/src/index.ts @@ -5,6 +5,7 @@ import { SUBJECT_EQUITY_PRINTS, STREAM_EQUITY_CANDLES, STREAM_EQUITY_PRINTS, + buildStreamConfig, buildDurableConsumer, connectJetStreamWithRetry, ensureStream, @@ -240,31 +241,8 @@ const run = async () => { { attempts: 120, delayMs: 500 } ); - await ensureStream(jsm, { - name: STREAM_EQUITY_PRINTS, - subjects: [SUBJECT_EQUITY_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_CANDLES, - subjects: [SUBJECT_EQUITY_CANDLES], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_CANDLES, SUBJECT_EQUITY_CANDLES, "derived")); const clickhouse = createClickHouseClient({ url: env.CLICKHOUSE_URL, diff --git a/services/compute/src/index.ts b/services/compute/src/index.ts index 65c6a1e..8e561c3 100644 --- a/services/compute/src/index.ts +++ b/services/compute/src/index.ts @@ -26,6 +26,7 @@ import { STREAM_SMART_MONEY_EVENTS, STREAM_OPTION_NBBO, STREAM_OPTION_SIGNAL_PRINTS, + buildStreamConfig, buildDurableConsumer, connectJetStreamWithRetry, ensureStream, @@ -40,12 +41,13 @@ import { ensureInferredDarkTable, ensureFlowPacketsTable, ensureSmartMoneyEventsTable, - insertAlert, - insertClassifierHit, - insertEquityPrintJoin, - insertInferredDark, - insertFlowPacket, - insertSmartMoneyEvent + ClickHouseBatchWriter, + enqueueAlertInsert, + enqueueClassifierHitInsert, + enqueueEquityPrintJoinInsert, + enqueueFlowPacketInsert, + enqueueInferredDarkInsert, + enqueueSmartMoneyEventInsert, } from "@islandflow/storage"; import { AlertEventSchema, @@ -82,7 +84,12 @@ import { type DarkInferenceConfig } from "./dark-inference"; import { buildEquityPrintJoin, type EquityQuoteJoin } from "./equity-joins"; -import { createRedisClient, updateRollingStats, type RollingStatsConfig } from "./rolling-stats"; +import { + createRedisClient, + RollingWindowStore, + type RollingStatsConfig, + type RollingWindowStoreConfig +} from "./rolling-stats"; import { summarizeStructure, type ContractLeg } from "./structures"; import { buildStructureFlowPacket, @@ -103,6 +110,8 @@ const envSchema = z.object({ CLUSTER_WINDOW_MS: z.coerce.number().int().positive().default(500), ROLLING_WINDOW_SIZE: z.coerce.number().int().positive().default(50), ROLLING_TTL_SEC: z.coerce.number().int().nonnegative().default(86400), + ROLLING_CACHE_FLUSH_INTERVAL_MS: z.coerce.number().int().positive().default(30_000), + ROLLING_CACHE_MAX_KEYS: z.coerce.number().int().positive().default(20_000), COMPUTE_DELIVER_POLICY: z.enum(["new", "all", "last", "last_per_subject"]).default("new"), COMPUTE_CONSUMER_RESET: z .preprocess((value) => { @@ -119,6 +128,8 @@ const envSchema = z.object({ }, z.boolean()) .default(false), NBBO_MAX_AGE_MS: z.coerce.number().int().positive().default(1000), + COMPUTE_NBBO_CACHE_MAX_KEYS: z.coerce.number().int().positive().default(20_000), + COMPUTE_NBBO_CACHE_TTL_MS: z.coerce.number().int().positive().default(900_000), EQUITY_QUOTE_MAX_AGE_MS: z.coerce.number().int().positive().default(1000), DARK_INFER_WINDOW_MS: z.coerce.number().int().positive().default(60000), DARK_INFER_COOLDOWN_MS: z.coerce.number().int().nonnegative().default(30000), @@ -269,6 +280,9 @@ const clusters = new Map(); const nbboCache = new Map(); const equityQuoteCache = new Map(); const darkInferenceState = createDarkInferenceState(); +const nbboCacheTouchedAt = new Map(); +const equityQuoteCacheTouchedAt = new Map(); +const darkInferenceTouchedAt = new Map(); const recentLegsByKey = new Map(); const recentLegsByRoot = new Map(); const recentStructureEmits = new Map(); @@ -278,6 +292,20 @@ const runtimeState = { }; const MAX_RECENT_LEGS = 20; +const EQUITY_QUOTE_CACHE_MAX_KEYS = 2_000; +const EQUITY_QUOTE_CACHE_TTL_MS = 900_000; +const DARK_INFERENCE_TTL_MS = 900_000; +const CACHE_PRUNE_INTERVAL_MS = 60_000; + +const emitCounters = { + flowPackets: 0, + structurePackets: 0, + smartMoneyEvents: 0, + classifierHits: 0, + alerts: 0, + equityJoins: 0, + darkEvents: 0 +}; const rollingKey = (metric: string, contractId: string): string => { return `rolling:${metric}:${contractId}`; @@ -479,8 +507,8 @@ const pruneRecentStructureEmits = (anchorTs: number): void => { }; const emitStructurePacketIfNeeded = async ( - clickhouse: ReturnType, js: Awaited>["js"], + batchWriter: ClickHouseBatchWriter, legs: LegEvidence[], summary: ReturnType, currentContractId: string @@ -512,16 +540,11 @@ const emitStructurePacketIfNeeded = async ( const packet = buildStructureFlowPacket(plan, summary); const validated = FlowPacketSchema.parse(packet); - await insertFlowPacket(clickhouse, validated); + enqueueFlowPacketInsert(batchWriter, validated); await publishJson(js, SUBJECT_FLOW_PACKETS, validated); - await emitClassifiers(clickhouse, js, validated); - - logger.info("emitted structure flow packet", { - id: validated.id, - type: summary.type, - legs: summary.legs, - strikes: summary.strikes - }); + emitCounters.flowPackets += 1; + emitCounters.structurePackets += 1; + await emitClassifiers(js, batchWriter, validated); }; const applyDeliverPolicy = ( @@ -606,6 +629,7 @@ const updateNbboCache = (nbbo: OptionNBBO): void => { (nbbo.ts === existing.ts && nbbo.seq >= existing.seq) ) { nbboCache.set(nbbo.option_contract_id, nbbo); + nbboCacheTouchedAt.set(nbbo.option_contract_id, Date.now()); } }; @@ -617,6 +641,7 @@ const updateEquityQuoteCache = (quote: EquityQuote): void => { (quote.ts === existing.ts && quote.seq >= existing.seq) ) { equityQuoteCache.set(quote.underlying_id, quote); + equityQuoteCacheTouchedAt.set(quote.underlying_id, Date.now()); } }; @@ -626,6 +651,7 @@ const selectNbbo = (contractId: string, ts: number): NbboJoin => { return { nbbo: null, ageMs: env.NBBO_MAX_AGE_MS + 1, stale: true }; } + nbboCacheTouchedAt.set(contractId, Date.now()); const ageMs = Math.abs(ts - nbbo.ts); const stale = ageMs > env.NBBO_MAX_AGE_MS; return { nbbo, ageMs, stale }; @@ -637,11 +663,77 @@ const selectEquityQuote = (underlyingId: string, ts: number): EquityQuoteJoin => return { quote: null, ageMs: env.EQUITY_QUOTE_MAX_AGE_MS + 1, stale: true }; } + equityQuoteCacheTouchedAt.set(underlyingId, Date.now()); const ageMs = Math.abs(ts - quote.ts); const stale = ageMs > env.EQUITY_QUOTE_MAX_AGE_MS; return { quote, ageMs, stale }; }; +const pruneTimedMap = ( + values: Map, + touchedAt: Map, + maxKeys: number, + ttlMs: number, + now = Date.now() +): number => { + let removed = 0; + + for (const [key, touched] of touchedAt) { + if (now - touched > ttlMs) { + touchedAt.delete(key); + values.delete(key); + removed += 1; + } + } + + if (values.size <= maxKeys) { + return removed; + } + + const overflow = values.size - maxKeys; + const oldest = [...touchedAt.entries()].sort((a, b) => a[1] - b[1]).slice(0, overflow); + for (const [key] of oldest) { + touchedAt.delete(key); + values.delete(key); + removed += 1; + } + + return removed; +}; + +const pruneComputeCaches = (rollingStore: RollingWindowStore, now = Date.now()) => { + const nbboRemoved = pruneTimedMap( + nbboCache, + nbboCacheTouchedAt, + env.COMPUTE_NBBO_CACHE_MAX_KEYS, + env.COMPUTE_NBBO_CACHE_TTL_MS, + now + ); + const quoteRemoved = pruneTimedMap( + equityQuoteCache, + equityQuoteCacheTouchedAt, + EQUITY_QUOTE_CACHE_MAX_KEYS, + EQUITY_QUOTE_CACHE_TTL_MS, + now + ); + const darkRemoved = pruneTimedMap( + darkInferenceState.lastEmittedByUnderlying, + darkInferenceTouchedAt, + EQUITY_QUOTE_CACHE_MAX_KEYS, + DARK_INFERENCE_TTL_MS, + now + ); + const rollingRemoved = rollingStore.prune(now); + + logger.info("compute cache summary", { + nbbo_cache_size: nbboCache.size, + equity_quote_cache_size: equityQuoteCache.size, + dark_inference_cache_size: darkInferenceState.lastEmittedByUnderlying.size, + rolling_cache_size: rollingStore.size, + removed: nbboRemoved + quoteRemoved + darkRemoved + rollingRemoved + }); +}; + const classifyPlacement = (price: number, join: NbboJoin): NbboPlacement => { if (!Number.isFinite(price)) { return "MISSING"; @@ -679,10 +771,9 @@ const classifyPlacement = (price: number, join: NbboJoin): NbboPlacement => { }; const flushCluster = async ( - clickhouse: ReturnType, js: Awaited>["js"], - redis: ReturnType, - rollingConfig: RollingStatsConfig, + batchWriter: ClickHouseBatchWriter, + rollingStore: RollingWindowStore, cluster: ClusterState ): Promise => { if (cluster.flushed) { @@ -784,12 +875,7 @@ const flushCluster = async ( prefix: string ): Promise => { try { - const snapshot = await updateRollingStats( - redis, - rollingKey(metric, cluster.contractId), - value, - rollingConfig - ); + const snapshot = rollingStore.update(rollingKey(metric, cluster.contractId), value); features[`${prefix}_mean`] = roundTo(snapshot.mean); features[`${prefix}_std`] = roundTo(snapshot.stddev); features[`${prefix}_z`] = roundTo(snapshot.zscore); @@ -824,7 +910,7 @@ const flushCluster = async ( features.structure_rights = summary.rights; } - await emitStructurePacketIfNeeded(clickhouse, js, legs, summary, currentLeg.contractId); + await emitStructurePacketIfNeeded(js, batchWriter, legs, summary, currentLeg.contractId); const rootKey = buildRootKey(currentLeg); const rootCandidates = [ @@ -834,7 +920,7 @@ const flushCluster = async ( const rollLegs = [currentLeg, ...rootCandidates]; const rollSummary = summarizeStructure(rollLegs); if (rollSummary?.type === "roll") { - await emitStructurePacketIfNeeded(clickhouse, js, rollLegs, rollSummary, currentLeg.contractId); + await emitStructurePacketIfNeeded(js, batchWriter, rollLegs, rollSummary, currentLeg.contractId); } storeRecentLeg(currentLeg, anchorTs); @@ -873,16 +959,10 @@ const flushCluster = async ( const validated = FlowPacketSchema.parse(packet); try { - await insertFlowPacket(clickhouse, validated); + enqueueFlowPacketInsert(batchWriter, validated); await publishJson(js, SUBJECT_FLOW_PACKETS, validated); - - await emitClassifiers(clickhouse, js, validated); - - logger.info("emitted flow packet", { - id: validated.id, - contract: cluster.contractId, - count: cluster.members.length - }); + emitCounters.flowPackets += 1; + await emitClassifiers(js, batchWriter, validated); } catch (error) { if (isExpectedShutdownNatsError(error)) { logger.info("skipped flow packet publish during shutdown", { @@ -899,8 +979,8 @@ const flushCluster = async ( }; const emitClassifiers = async ( - clickhouse: ReturnType, js: Awaited>["js"], + batchWriter: ClickHouseBatchWriter, packet: FlowPacket ): Promise => { let smartMoneyEvent: SmartMoneyEvent; @@ -915,8 +995,9 @@ const emitClassifiers = async ( : packet.source_ts; const eventCalendarMatch = underlyingId ? eventCalendarProvider.findNextEvent(underlyingId, referenceTs) : null; smartMoneyEvent = SmartMoneyEventSchema.parse(buildSmartMoneyEventFromPacket(packet, { eventCalendarMatch })); - await insertSmartMoneyEvent(clickhouse, smartMoneyEvent); + enqueueSmartMoneyEventInsert(batchWriter, smartMoneyEvent); await publishJson(js, SUBJECT_SMART_MONEY_EVENTS, smartMoneyEvent); + emitCounters.smartMoneyEvents += 1; } catch (error) { if (isExpectedShutdownNatsError(error)) { return; @@ -945,8 +1026,9 @@ const emitClassifiers = async ( for (const hit of hitEvents) { try { - await insertClassifierHit(clickhouse, hit); + enqueueClassifierHitInsert(batchWriter, hit); await publishJson(js, SUBJECT_CLASSIFIER_HITS, hit); + emitCounters.classifierHits += 1; } catch (error) { if (isExpectedShutdownNatsError(error)) { continue; @@ -981,8 +1063,9 @@ const emitClassifiers = async ( }); try { - await insertAlert(clickhouse, alert); + enqueueAlertInsert(batchWriter, alert); await publishJson(js, SUBJECT_ALERTS, alert); + emitCounters.alerts += 1; } catch (error) { if (isExpectedShutdownNatsError(error)) { return; @@ -995,17 +1078,21 @@ const emitClassifiers = async ( }; const emitEquityJoin = async ( - clickhouse: ReturnType, js: Awaited>["js"], + batchWriter: ClickHouseBatchWriter, print: EquityPrint ): Promise => { const join = selectEquityQuote(print.underlying_id, print.ts); const payload: EquityPrintJoin = EquityPrintJoinSchema.parse(buildEquityPrintJoin(print, join)); try { - await insertEquityPrintJoin(clickhouse, payload); + enqueueEquityPrintJoinInsert(batchWriter, payload); } catch (error) { - logger.error("failed to emit equity print join", { + if (isExpectedShutdownNatsError(error)) { + return; + } + + logger.error("failed to queue equity print join", { error: error instanceof Error ? error.message : String(error), trace_id: payload.trace_id }); @@ -1014,6 +1101,7 @@ const emitEquityJoin = async ( try { await publishJson(js, SUBJECT_EQUITY_JOINS, payload); + emitCounters.equityJoins += 1; } catch (error) { if (isExpectedShutdownNatsError(error)) { return; @@ -1024,20 +1112,26 @@ const emitEquityJoin = async ( }); } - await emitDarkInferences(clickhouse, js, payload); + await emitDarkInferences(js, batchWriter, payload); }; const emitDarkInferences = async ( - clickhouse: ReturnType, js: Awaited>["js"], + batchWriter: ClickHouseBatchWriter, join: EquityPrintJoin ): Promise => { const events = evaluateDarkInferences(join, darkInferenceConfig, darkInferenceState); for (const event of events) { const validated: InferredDarkEvent = InferredDarkEventSchema.parse(event); try { - await insertInferredDark(clickhouse, validated); + enqueueInferredDarkInsert(batchWriter, validated); await publishJson(js, SUBJECT_INFERRED_DARK, validated); + emitCounters.darkEvents += 1; + const underlyingId = + typeof join.features?.underlying_id === "string" ? join.features.underlying_id : null; + if (underlyingId) { + darkInferenceTouchedAt.set(underlyingId, Date.now()); + } } catch (error) { if (isExpectedShutdownNatsError(error)) { continue; @@ -1051,10 +1145,9 @@ const emitDarkInferences = async ( }; const flushEligibleClusters = async ( - clickhouse: ReturnType, js: Awaited>["js"], - redis: ReturnType, - rollingConfig: RollingStatsConfig, + batchWriter: ClickHouseBatchWriter, + rollingStore: RollingWindowStore, currentTs: number, skipContractId: string ): Promise => { @@ -1065,7 +1158,7 @@ const flushEligibleClusters = async ( if (currentTs - cluster.endTs > env.CLUSTER_WINDOW_MS) { clusters.delete(contractId); - await flushCluster(clickhouse, js, redis, rollingConfig, cluster); + await flushCluster(js, batchWriter, rollingStore, cluster); } } }; @@ -1081,135 +1174,16 @@ const run = async () => { { attempts: 120, delayMs: 500 } ); - await ensureStream(jsm, { - name: STREAM_OPTION_SIGNAL_PRINTS, - subjects: [SUBJECT_OPTION_SIGNAL_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_OPTION_NBBO, - subjects: [SUBJECT_OPTION_NBBO], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_PRINTS, - subjects: [SUBJECT_EQUITY_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_QUOTES, - subjects: [SUBJECT_EQUITY_QUOTES], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_FLOW_PACKETS, - subjects: [SUBJECT_FLOW_PACKETS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_SMART_MONEY_EVENTS, - subjects: [SUBJECT_SMART_MONEY_EVENTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_JOINS, - subjects: [SUBJECT_EQUITY_JOINS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_INFERRED_DARK, - subjects: [SUBJECT_INFERRED_DARK], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_CLASSIFIER_HITS, - subjects: [SUBJECT_CLASSIFIER_HITS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_ALERTS, - subjects: [SUBJECT_ALERTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_FLOW_PACKETS, SUBJECT_FLOW_PACKETS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_SMART_MONEY_EVENTS, SUBJECT_SMART_MONEY_EVENTS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_JOINS, SUBJECT_EQUITY_JOINS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_INFERRED_DARK, SUBJECT_INFERRED_DARK, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_CLASSIFIER_HITS, SUBJECT_CLASSIFIER_HITS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_ALERTS, SUBJECT_ALERTS, "derived")); const clickhouse = createClickHouseClient({ url: env.CLICKHOUSE_URL, @@ -1242,6 +1216,51 @@ const run = async () => { windowSize: env.ROLLING_WINDOW_SIZE, ttlSeconds: env.ROLLING_TTL_SEC }; + const rollingStore = new RollingWindowStore({ + ...rollingConfig, + flushIntervalMs: env.ROLLING_CACHE_FLUSH_INTERVAL_MS, + maxKeys: env.ROLLING_CACHE_MAX_KEYS + } satisfies RollingWindowStoreConfig); + const batchWriter = new ClickHouseBatchWriter(clickhouse, { + flushIntervalMs: 100, + maxRows: 250, + onError: (table, error, rowCount) => { + logger.error("batched clickhouse insert failed", { + table, + row_count: rowCount, + error: error instanceof Error ? error.message : String(error), + action: "dropped" + }); + } + }); + const rollingFlushTimer = setInterval(() => { + void rollingStore.flushToRedis(redis); + }, env.ROLLING_CACHE_FLUSH_INTERVAL_MS); + const pruneTimer = setInterval(() => { + pruneComputeCaches(rollingStore); + }, CACHE_PRUNE_INTERVAL_MS); + const summaryTimer = setInterval(() => { + logger.info("compute minute summary", { + flow_packets_emitted: emitCounters.flowPackets, + structure_packets_emitted: emitCounters.structurePackets, + smart_money_events_emitted: emitCounters.smartMoneyEvents, + classifier_hits_emitted: emitCounters.classifierHits, + alerts_emitted: emitCounters.alerts, + equity_joins_emitted: emitCounters.equityJoins, + dark_events_emitted: emitCounters.darkEvents, + rolling_stats_cache_size: rollingStore.size + }); + emitCounters.flowPackets = 0; + emitCounters.structurePackets = 0; + emitCounters.smartMoneyEvents = 0; + emitCounters.classifierHits = 0; + emitCounters.alerts = 0; + emitCounters.equityJoins = 0; + emitCounters.darkEvents = 0; + }, 60_000); + rollingFlushTimer.unref?.(); + pruneTimer.unref?.(); + summaryTimer.unref?.(); await retry("clickhouse table init", 120, 500, async () => { await ensureFlowPacketsTable(clickhouse); @@ -1578,7 +1597,7 @@ const run = async () => { try { const print = EquityPrintSchema.parse(equitySubscription.decode(msg)); - await emitEquityJoin(clickhouse, js, print); + await emitEquityJoin(js, batchWriter, print); msg.ack(); } catch (error) { logger.error("failed to process equity print", { @@ -1602,11 +1621,16 @@ const run = async () => { runtimeState.shuttingDown = true; runtimeState.shutdownPromise = (async () => { logger.info("service stopping", { signal }); + clearInterval(rollingFlushTimer); + clearInterval(pruneTimer); + clearInterval(summaryTimer); for (const cluster of [...clusters.values()]) { - await flushCluster(clickhouse, js, redis, rollingConfig, cluster); + await flushCluster(js, batchWriter, rollingStore, cluster); } clusters.clear(); + await batchWriter.close(); + await rollingStore.flushToRedis(redis); try { await nc.drain(); @@ -1655,10 +1679,9 @@ const run = async () => { try { const print = OptionPrintSchema.parse(subscription.decode(msg)); await flushEligibleClusters( - clickhouse, js, - redis, - rollingConfig, + batchWriter, + rollingStore, print.ts, print.option_contract_id ); @@ -1674,7 +1697,7 @@ const run = async () => { updateCluster(existing, print); } else { clusters.delete(print.option_contract_id); - await flushCluster(clickhouse, js, redis, rollingConfig, existing); + await flushCluster(js, batchWriter, rollingStore, existing); clusters.set(print.option_contract_id, buildCluster(print)); } diff --git a/services/compute/src/rolling-stats.ts b/services/compute/src/rolling-stats.ts index 63c6caa..d30b930 100644 --- a/services/compute/src/rolling-stats.ts +++ b/services/compute/src/rolling-stats.ts @@ -5,6 +5,11 @@ export type RollingStatsConfig = { ttlSeconds: number; }; +export type RollingWindowStoreConfig = RollingStatsConfig & { + flushIntervalMs: number; + maxKeys: number; +}; + export type RollingSnapshot = { baselineCount: number; mean: number; @@ -12,6 +17,12 @@ export type RollingSnapshot = { zscore: number; }; +type RollingWindowEntry = { + values: number[]; + updatedAt: number; + dirty: boolean; +}; + const toNumbers = (values: string[]): number[] => { return values .map((value) => Number(value)) @@ -49,26 +60,120 @@ export const createRedisClient = (url: string) => { return createClient({ url }); }; -export const updateRollingStats = async ( - client: ReturnType, - key: string, - value: number, - config: RollingStatsConfig -): Promise => { - const limit = Math.max(0, config.windowSize - 1); - const existing = await client.lRange(key, 0, limit); - const baseline = toNumbers(existing); - const snapshot = computeSnapshot(baseline, value); +const getOldestKey = (store: Map): string | null => { + let oldestKey: string | null = null; + let oldestUpdatedAt = Number.POSITIVE_INFINITY; - const multi = client.multi(); - multi.lPush(key, value.toString()); - if (config.windowSize > 0) { - multi.lTrim(key, 0, config.windowSize - 1); + for (const [key, entry] of store) { + if (entry.updatedAt < oldestUpdatedAt) { + oldestUpdatedAt = entry.updatedAt; + oldestKey = key; + } } - if (config.ttlSeconds > 0) { - multi.expire(key, config.ttlSeconds); - } - await multi.exec(); - return snapshot; + return oldestKey; }; + +export class RollingWindowStore { + private readonly store = new Map(); + private readonly ttlMs: number; + private readonly windowSize: number; + private readonly maxKeys: number; + + constructor(private readonly config: RollingWindowStoreConfig) { + this.ttlMs = Math.max(0, config.ttlSeconds * 1000); + this.windowSize = Math.max(1, config.windowSize); + this.maxKeys = Math.max(1, config.maxKeys); + } + + get size(): number { + return this.store.size; + } + + update(key: string, value: number, now = Date.now()): RollingSnapshot { + this.prune(now); + + const existing = this.store.get(key); + const baseline = existing?.values ?? []; + const snapshot = computeSnapshot(baseline, value); + const nextValues = [value, ...baseline].slice(0, this.windowSize); + + this.store.set(key, { + values: nextValues, + updatedAt: now, + dirty: true + }); + + this.enforceMaxKeys(); + return snapshot; + } + + prune(now = Date.now()): number { + if (this.ttlMs <= 0) { + return 0; + } + + let removed = 0; + for (const [key, entry] of this.store) { + if (now - entry.updatedAt > this.ttlMs) { + this.store.delete(key); + removed += 1; + } + } + return removed; + } + + async hydrateFromRedis( + client: ReturnType, + keys: string[], + now = Date.now() + ): Promise { + for (const key of keys) { + const values = toNumbers(await client.lRange(key, 0, this.windowSize - 1)); + if (values.length === 0) { + continue; + } + this.store.set(key, { + values, + updatedAt: now, + dirty: false + }); + } + this.enforceMaxKeys(); + } + + async flushToRedis(client: ReturnType): Promise { + let flushed = 0; + for (const [key, entry] of this.store) { + if (!entry.dirty) { + continue; + } + + const multi = client.multi(); + multi.lTrim(key, 1, 0); + for (let idx = entry.values.length - 1; idx >= 0; idx -= 1) { + const value = entry.values[idx]; + if (typeof value === "number" && Number.isFinite(value)) { + multi.lPush(key, value.toString()); + } + } + if (this.config.ttlSeconds > 0) { + multi.expire(key, this.config.ttlSeconds); + } + await multi.exec(); + entry.dirty = false; + flushed += 1; + } + return flushed; + } + + private enforceMaxKeys(): void { + while (this.store.size > this.maxKeys) { + const oldestKey = getOldestKey(this.store); + if (!oldestKey) { + break; + } + this.store.delete(oldestKey); + } + } +} diff --git a/services/compute/tests/rolling-stats.test.ts b/services/compute/tests/rolling-stats.test.ts index 555d77c..aa9d738 100644 --- a/services/compute/tests/rolling-stats.test.ts +++ b/services/compute/tests/rolling-stats.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { computeSnapshot, computeStats } from "../src/rolling-stats"; +import { computeSnapshot, computeStats, RollingWindowStore } from "../src/rolling-stats"; describe("rolling stats helpers", () => { test("computeStats handles empty baseline", () => { @@ -21,4 +21,18 @@ describe("rolling stats helpers", () => { expect(snapshot.baselineCount).toBe(3); expect(snapshot.zscore).toBeCloseTo(1.84, 2); }); + + test("RollingWindowStore prunes stale keys by ttl", () => { + const store = new RollingWindowStore({ + windowSize: 3, + ttlSeconds: 1, + flushIntervalMs: 30_000, + maxKeys: 10 + }); + + store.update("rolling:premium:ABC", 10, 0); + expect(store.size).toBe(1); + expect(store.prune(1_500)).toBe(1); + expect(store.size).toBe(0); + }); }); diff --git a/services/ingest-equities/src/index.ts b/services/ingest-equities/src/index.ts index 3b77642..15dff9e 100644 --- a/services/ingest-equities/src/index.ts +++ b/services/ingest-equities/src/index.ts @@ -5,6 +5,7 @@ import { SUBJECT_EQUITY_QUOTES, STREAM_EQUITY_PRINTS, STREAM_EQUITY_QUOTES, + buildStreamConfig, connectJetStreamWithRetry, ensureStream, publishJson @@ -194,31 +195,8 @@ const run = async () => { { attempts: 120, delayMs: 500 } ); - await ensureStream(jsm, { - name: STREAM_EQUITY_PRINTS, - subjects: [SUBJECT_EQUITY_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_QUOTES, - subjects: [SUBJECT_EQUITY_QUOTES], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw")); const clickhouse = createClickHouseClient({ url: env.CLICKHOUSE_URL, @@ -251,11 +229,6 @@ const run = async () => { try { await insertEquityPrint(clickhouse, print); await publishJson(js, SUBJECT_EQUITY_PRINTS, print); - logger.info("published equity print", { - trace_id: print.trace_id, - seq: print.seq, - underlying_id: print.underlying_id - }); } catch (error) { if (isExpectedShutdownError(error)) { return; diff --git a/services/ingest-options/src/index.ts b/services/ingest-options/src/index.ts index bf50431..8e2bf41 100644 --- a/services/ingest-options/src/index.ts +++ b/services/ingest-options/src/index.ts @@ -9,6 +9,7 @@ import { STREAM_OPTION_NBBO, STREAM_OPTION_PRINTS, STREAM_OPTION_SIGNAL_PRINTS, + buildStreamConfig, buildDurableConsumer, connectJetStreamWithRetry, ensureStream, @@ -109,7 +110,9 @@ const envSchema = z.object({ return value; }, z.boolean()) .default(false), - TESTING_THROTTLE_MS: z.coerce.number().int().nonnegative().default(200) + TESTING_THROTTLE_MS: z.coerce.number().int().nonnegative().default(200), + OPTION_CONTEXT_MAX_KEYS: z.coerce.number().int().positive().default(20_000), + OPTION_CONTEXT_TTL_MS: z.coerce.number().int().positive().default(900_000) }); const env = readEnv(envSchema); @@ -143,6 +146,44 @@ const state = { const nbboHistoryByContract: ContextHistory = new Map(); const equityQuoteHistoryByUnderlying: ContextHistory = new Map(); +const OPTION_CONTEXT_PRUNE_INTERVAL_MS = 60_000; + +const pruneContextHistory = ( + history: ContextHistory, + maxKeys: number, + ttlMs: number, + now = Date.now() +): number => { + let removed = 0; + for (const [key, items] of history) { + const filtered = items.filter((item) => now - item.ts <= ttlMs); + if (filtered.length === 0) { + history.delete(key); + removed += 1; + continue; + } + if (filtered.length !== items.length) { + history.set(key, filtered); + } + } + + if (history.size <= maxKeys) { + return removed; + } + + const overflow = history.size - maxKeys; + const oldestKeys = [...history.entries()] + .map(([key, items]) => [key, items.at(-1)?.ts ?? Number.NEGATIVE_INFINITY] as const) + .sort((a, b) => a[1] - b[1]) + .slice(0, overflow); + + for (const [key] of oldestKeys) { + history.delete(key); + removed += 1; + } + + return removed; +}; const getErrorMessage = (error: unknown): string => { return error instanceof Error ? error.message : String(error); @@ -305,57 +346,10 @@ const run = async () => { { attempts: 120, delayMs: 500 } ); - await ensureStream(jsm, { - name: STREAM_OPTION_PRINTS, - subjects: [SUBJECT_OPTION_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_OPTION_NBBO, - subjects: [SUBJECT_OPTION_NBBO], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_OPTION_SIGNAL_PRINTS, - subjects: [SUBJECT_OPTION_SIGNAL_PRINTS], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); - - await ensureStream(jsm, { - name: STREAM_EQUITY_QUOTES, - subjects: [SUBJECT_EQUITY_QUOTES], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 - }); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_PRINTS, SUBJECT_OPTION_PRINTS, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw")); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived")); + await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw")); const clickhouse = createClickHouseClient({ url: env.CLICKHOUSE_URL, @@ -400,14 +394,6 @@ const run = async () => { if (print.signal_pass) { await publishJson(js, SUBJECT_OPTION_SIGNAL_PRINTS, print); } - logger.info("published option print", { - trace_id: print.trace_id, - seq: print.seq, - option_contract_id: print.option_contract_id, - signal_pass: print.signal_pass, - nbbo_side: print.nbbo_side, - notional: print.notional - }); } catch (error) { if (isExpectedShutdownError(error)) { return; @@ -475,6 +461,18 @@ const run = async () => { } })(); + const pruneTimer = setInterval(() => { + const removed = + pruneContextHistory(nbboHistoryByContract, env.OPTION_CONTEXT_MAX_KEYS, env.OPTION_CONTEXT_TTL_MS) + + pruneContextHistory(equityQuoteHistoryByUnderlying, env.OPTION_CONTEXT_MAX_KEYS, env.OPTION_CONTEXT_TTL_MS); + logger.info("option context cache summary", { + nbbo_context_keys: nbboHistoryByContract.size, + equity_quote_context_keys: equityQuoteHistoryByUnderlying.size, + removed + }); + }, OPTION_CONTEXT_PRUNE_INTERVAL_MS); + pruneTimer.unref?.(); + const shutdown = async (signal: string) => { if (state.shutdownPromise) { return state.shutdownPromise; @@ -483,6 +481,7 @@ const run = async () => { state.shuttingDown = true; state.shutdownPromise = (async () => { logger.info("service stopping", { signal }); + clearInterval(pruneTimer); await stopAdapter(); try { diff --git a/services/replay/src/index.ts b/services/replay/src/index.ts index 1ba8342..21e4981 100644 --- a/services/replay/src/index.ts +++ b/services/replay/src/index.ts @@ -11,6 +11,7 @@ import { STREAM_OPTION_NBBO, STREAM_OPTION_PRINTS, STREAM_OPTION_SIGNAL_PRINTS, + buildStreamConfig, connectJetStreamWithRetry, ensureStream, publishJson @@ -180,19 +181,6 @@ const parseStreamList = (value: string): ReplayStreamKind[] => { return result; }; -const buildStreamConfig = (name: string, subject: string) => ({ - name, - subjects: [subject], - retention: "limits", - storage: "file", - discard: "old", - max_msgs_per_subject: -1, - max_msgs: -1, - max_bytes: -1, - max_age: 0, - num_replicas: 1 -}); - const buildStartCursor = (startTs: number): ReplayCursor => { if (startTs <= 0) { return { ts: 0, seq: 0 }; @@ -304,10 +292,10 @@ const run = async () => { for (const kind of streamKinds) { const def = STREAM_DEFS[kind]; - await ensureStream(jsm, buildStreamConfig(def.streamName, def.subject)); + await ensureStream(jsm, buildStreamConfig(def.streamName, def.subject, "raw")); } if (streamKinds.includes("options")) { - await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS)); + await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived")); } const clickhouse = createClickHouseClient({