Merge pull request #32 from dirtydishes/load-reduction

Implement first-pass load reduction controls

.beads/issues.jsonl

@@ -1,3 +1,10 @@
+{"_type":"issue","id":"islandflow-dil","title":"Run production baseline and post-rollout verification for load reduction","description":"Run the production verification checklist from the load-reduction plan on the VPS, capture baseline container/resource stats, validate replay remains disabled, and confirm JetStream/Redis behavior after rollout.\n\nThis follow-up is operational rather than code-local and could not be executed from the current workspace. It should compare pre/post CPU, RSS, Redis memory, and retention growth using the documented commands.\n","status":"open","priority":1,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:45:06Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:45:06Z","dependencies":[{"issue_id":"islandflow-dil","depends_on_id":"islandflow-1ln","type":"discovered-from","created_at":"2026-05-08T02:45:06Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"islandflow-1ln","title":"Implement VPS load reduction plan","description":"Implement load-reduction plan across API, compute, logging, retention, and cache pruning.\n\nThis issue tracks the first-pass implementation of VPS load mitigations: lower live cache limits, async Redis write-behind in API live state, scoped cache eviction, reduced hot-path logging, bounded JetStream retention via shared config, in-memory rolling stats with async Redis snapshots, batched ClickHouse inserts for derived tables, and TTL/cardinality pruning for long-lived in-process maps.\n\nAcceptance:\n- Config surface for live limits, logging, rolling cache, and stream retention added\n- API live ingest avoids per-event full resort in monotonic case and avoids synchronous Redis writes per event\n- Compute rolling stats leave Redis hot path and derived ClickHouse writes batch\n- Long-lived caches/maps are pruned by TTL/cardinality\n- Tests cover monotonic/out-of-order live ingest, scoped eviction, rolling stats, and pruning behavior\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:27:41Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:46:23Z","started_at":"2026-05-08T06:27:54Z","closed_at":"2026-05-08T06:46:23Z","close_reason":"Implemented in code; rollout verification follow-up is islandflow-dil and Redis durability decision follow-up is islandflow-ybs","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"islandflow-pre","title":"Fix contract-focused options tape hydration","description":"Implement contract-focused options tape hydration so focused contract views preserve the clicked seed row, stop reapplying broad flow filters in the Options pane, and use raw contract-scoped ClickHouse queries consistently across live snapshots, history, and replay. Includes frontend replay source-grouping changes and regression tests for focus seed durability, focused filtering, and contract-scoped API behavior.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T03:27:31Z","created_by":"dirtydishes","updated_at":"2026-05-08T03:37:18Z","started_at":"2026-05-08T03:27:35Z","closed_at":"2026-05-08T03:37:18Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"islandflow-9xs","title":"Fix terminal hydration and virtual row measurement crash","description":"Fix client crash caused by options-support hydration on non-JSON/404 responses and satisfy tanstack virtual measured-row data-index requirement across virtualized tables.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T06:14:33Z","created_by":"dirtydishes","updated_at":"2026-05-07T06:17:09Z","started_at":"2026-05-07T06:14:43Z","closed_at":"2026-05-07T06:17:09Z","close_reason":"Completed: added data-index attributes on measured virtual rows, hardened options-support hydration error handling/content-type validation, and guarded trace-id hydration loops against malformed payload entries.","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"islandflow-35g","title":"Fix Docker deployment workspace lockfile drift","description":"Refresh deployment/docker workspace lockfile for Docker builds, add a drift guard for Docker-built workspaces, and document the separate deployment snapshot so frozen Bun installs cannot fail when repo dependencies change.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T06:02:06Z","created_by":"dirtydishes","updated_at":"2026-05-07T06:07:50Z","started_at":"2026-05-07T06:02:15Z","closed_at":"2026-05-07T06:07:50Z","close_reason":"Completed: synced deployment Docker workspace snapshot from repo root, refreshed deployment bun.lock, added sync/check scripts, and documented maintenance workflow. Local docker compose build validation is blocked here because Docker daemon is unavailable.","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"islandflow-2ij","title":"Harden tape virtualization, scoped focus, and live feed health","description":"Implement the coordinated tape stability plan across web and API.\n\nScope:\n- replace fixed-height tape virtualization with measured virtualization and virtual-end history loading\n- replace scrollHeight anchoring with key-based anchor restore\n- compose canonical tape lists across seed/live/history sources\n- preserve clicked contract/ticker context during scoped focus transitions\n- separate backend hot-channel health from scoped quiet empty states\n- shrink browser hot windows and modestly reduce server cache limits\n- add regression tests and development instrumentation\n\nAcceptance:\n- no giant blank spacer gaps during tape scrolling\n- scroll remains stable while live data and history mutate the list\n- clicked deep-history option/equity rows remain visible immediately after focus\n- narrow scopes do not surface Feed behind unless backend channel health is stale\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T05:35:18Z","created_by":"dirtydishes","updated_at":"2026-05-07T05:52:14Z","started_at":"2026-05-07T05:35:21Z","closed_at":"2026-05-07T05:52:14Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"islandflow-uj7","title":"Fix home to tape navigation","description":"Home rail Tape navigation was not reliably switching to the tape route. Use browser-native top-level navigation for Home/Tape rail links so /tape remains reachable even if client router handling stalls.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T03:18:14Z","created_by":"dirtydishes","updated_at":"2026-05-07T03:18:21Z","started_at":"2026-05-07T03:18:20Z","closed_at":"2026-05-07T03:18:21Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-84s","title":"Implement seamless /tape live-to-history scroll gate","description":"Implement seamless live-to-ClickHouse scroll-gated history for /tape panes, including split live/history buffers in the web client, snapshot_limit support on live subscriptions, a bundled options support lookup endpoint, ClickHouse helpers for parity hydration, and test coverage for live head retention, background history loading, scoped options deep-hydration, and historical options decor restoration.\n","status":"in_progress","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T02:10:43Z","created_by":"dirtydishes","updated_at":"2026-05-07T02:10:47Z","started_at":"2026-05-07T02:10:47Z","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-sh1","title":"Fix live websocket stale lag and reconnect loop","description":"Investigate and fix API live consumer lag causing stale timestamps, feed-behind status, and reconnect loops. Optimize live cache persistence path, add lag telemetry/alerts, and validate in runtime.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T17:04:34Z","created_by":"dirtydishes","updated_at":"2026-05-04T17:09:44Z","started_at":"2026-05-04T17:04:38Z","closed_at":"2026-05-04T17:09:44Z","close_reason":"Completed: optimized live cache persistence path, added lag telemetry, deployed api via docker compose on di, verified ws freshness and low hotFeedLagMs","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-b3o","title":"Implement options tape table with execution spot","description":"Redesign OptionsPane into a dense classifier-colored table and preserve execution-time underlying spot on option prints from equity quote mid.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:41:59Z","created_by":"dirtydishes","updated_at":"2026-05-04T05:14:26Z","started_at":"2026-05-04T04:42:08Z","closed_at":"2026-05-04T05:14:26Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
@@ -5,6 +12,7 @@
 {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"islandflow-ybs","title":"Decide Redis AOF and cache/durable split after load rollout","description":"Decide whether the deployment Redis should keep AOF enabled or be split into cache vs durable roles after the first rollout data is available.\n\nThe current code changes reduce cache churn, but the operational durability/caching tradeoff still needs a production decision.\n","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:45:05Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:45:05Z","dependencies":[{"issue_id":"islandflow-ybs","depends_on_id":"islandflow-1ln","type":"discovered-from","created_at":"2026-05-08T02:45:04Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-vbk","title":"Remove deprecated Alpaca key-pair auth","description":"Remove legacy Alpaca key-pair authentication support and keep ALPACA_API_KEY as the only supported auth method across options/equities ingest and docs.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:19:51Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:21:10Z","started_at":"2026-05-05T07:19:54Z","closed_at":"2026-05-05T07:21:10Z","close_reason":"Removed key-pair auth and kept ALPACA_API_KEY only","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-h47","title":"Support single-token Alpaca auth","description":"Support single-token Alpaca authentication across ingest adapters using ALPACA_API_KEY with fallback to ALPACA_KEY_ID/ALPACA_SECRET_KEY, and document env usage.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:12:22Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:13:54Z","started_at":"2026-05-05T07:12:25Z","closed_at":"2026-05-05T07:13:54Z","close_reason":"Added ALPACA_API_KEY support with key-pair fallback","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-neu","title":"Add Alpha Vantage event calendar provider","description":"Add an Alpha Vantage earnings-calendar provider to services/refdata that fetches CSV, normalizes entries, writes the JSON cache consumed by compute, and documents the required env variables.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:00:31Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:02:30Z","started_at":"2026-05-05T07:00:37Z","closed_at":"2026-05-05T07:02:30Z","close_reason":"Added Alpha Vantage event-calendar provider","dependency_count":0,"dependent_count":0,"comment_count":0}

.env.example

@@ -58,8 +58,8 @@ API_DELIVER_POLICY=new
 API_CONSUMER_RESET=false
 NBBO_MAX_AGE_MS=1000
 NEXT_PUBLIC_NBBO_MAX_AGE_MS=1000
-NEXT_PUBLIC_LIVE_HOT_WINDOW=2000
+NEXT_PUBLIC_LIVE_HOT_WINDOW=600
-NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS=25000
+NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS=1200
 NEXT_PUBLIC_PINNED_EVIDENCE_TTL_MS=1200000
 NEXT_PUBLIC_PINNED_EVIDENCE_MAX_ITEMS=4000
 ROLLING_WINDOW_SIZE=50
@@ -91,6 +91,7 @@ ALPHA_VANTAGE_EARNINGS_SYMBOL=
 REFDATA_EVENT_CALENDAR_REFRESH_MS=86400000
 
 # Replay service
+LOG_LEVEL=info
 REPLAY_ENABLED=false
 REPLAY_STREAMS=options,nbbo,equities,equity-quotes
 REPLAY_START_TS=0
@@ -100,12 +101,33 @@ REPLAY_BATCH_SIZE=200
 REPLAY_LOG_EVERY=1000
 
 # API live retention (generic channels)
-LIVE_LIMIT_OPTIONS=10000
+LIVE_LIMIT_DEFAULT=1000
-LIVE_LIMIT_NBBO=10000
+LIVE_LIMIT_OPTIONS=1000
-LIVE_LIMIT_EQUITIES=10000
+LIVE_LIMIT_NBBO=1000
-LIVE_LIMIT_EQUITY_QUOTES=10000
+LIVE_LIMIT_EQUITIES=1000
-LIVE_LIMIT_EQUITY_JOINS=10000
+LIVE_LIMIT_EQUITY_QUOTES=500
-LIVE_LIMIT_FLOW=10000
+LIVE_LIMIT_EQUITY_JOINS=500
-LIVE_LIMIT_CLASSIFIER_HITS=10000
+LIVE_LIMIT_FLOW=500
-LIVE_LIMIT_ALERTS=10000
+LIVE_LIMIT_SMART_MONEY=300
-LIVE_LIMIT_INFERRED_DARK=10000
+LIVE_LIMIT_CLASSIFIER_HITS=300
+LIVE_LIMIT_ALERTS=300
+LIVE_LIMIT_INFERRED_DARK=300
+LIVE_SCOPED_CACHE_MAX_KEYS=32
+LIVE_REDIS_FLUSH_INTERVAL_MS=250
+LIVE_REDIS_FLUSH_MAX_ITEMS=100
+
+# Compute rolling/cache retention
+ROLLING_CACHE_FLUSH_INTERVAL_MS=30000
+ROLLING_CACHE_MAX_KEYS=20000
+COMPUTE_NBBO_CACHE_MAX_KEYS=20000
+COMPUTE_NBBO_CACHE_TTL_MS=900000
+
+# Ingest context retention
+OPTION_CONTEXT_MAX_KEYS=20000
+OPTION_CONTEXT_TTL_MS=900000
+
+# JetStream retention
+STREAM_RAW_MAX_AGE_MS=7200000
+STREAM_RAW_MAX_BYTES=1073741824
+STREAM_DERIVED_MAX_AGE_MS=86400000
+STREAM_DERIVED_MAX_BYTES=536870912

apps/web/app/globals.css

@@ -954,19 +954,37 @@ h3 {
 }
 
 .data-table-wrap {
+  display: flex;
   flex: 1 1 auto;
   min-height: 0;
-  overflow: auto;
+  overflow-x: auto;
+  overflow-y: hidden;
   border-top: 1px solid var(--border);
   border-bottom: 1px solid var(--border);
   background: rgba(5, 8, 12, 0.42);
 }
 
 .data-table {
-  display: block;
+  display: flex;
+  flex: 1 1 auto;
+  flex-direction: column;
+  height: 100%;
+  min-height: 0;
   min-width: 980px;
 }
 
+.data-table-scroll {
+  flex: 1 1 auto;
+  min-height: 0;
+  overflow-y: auto;
+  overflow-x: hidden;
+}
+
+.data-table-body {
+  position: relative;
+  min-width: 100%;
+}
+
 .data-table-options {
   min-width: 1280px;
 }
@@ -999,10 +1017,8 @@ h3 {
 }
 
 .data-table-head {
-  position: sticky;
+  flex: 0 0 auto;
-  top: 0;
+  height: 30px;
-  z-index: 2;
-  min-height: 30px;
   padding: 0 10px;
   border-bottom: 1px solid rgba(255, 255, 255, 0.095);
   background: rgba(8, 11, 16, 0.98);
@@ -1014,7 +1030,7 @@ h3 {
 
 .data-table-row {
   width: 100%;
-  min-height: 40px;
+  height: 40px;
   padding: 0 10px;
   border: 0;
   border-bottom: 1px solid rgba(255, 255, 255, 0.055);
@@ -1024,10 +1040,17 @@ h3 {
   text-align: left;
 }
 
-.data-table-row:nth-child(even) {
+.data-table-row.is-even {
   background: rgba(255, 255, 255, 0.022);
 }
 
+.data-table-virtual-row {
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+}
+
 .data-table-row:hover,
 .data-table-row:focus-visible {
   outline: none;
@@ -1039,18 +1062,18 @@ h3 {
 }
 
 .data-table-row-options {
-  min-height: 36px;
+  height: 36px;
 }
 
 .data-table-row-equities {
-  min-height: 34px;
+  height: 36px;
 }
 
 .data-table-row-flow,
 .data-table-row-alerts,
 .data-table-row-classifier,
 .data-table-row-dark {
-  min-height: 44px;
+  height: 44px;
 }
 
 .data-table-row-classified {

apps/web/app/terminal.test.ts

@@ -1,24 +1,40 @@
 import { describe, expect, it } from "bun:test";
+import { getSubscriptionKey as getLiveSubscriptionKey } from "@islandflow/types";
 import {
   NAV_ITEMS,
+  appendHistoryTail,
   buildDefaultFlowFilters,
+  buildOptionTapeQueryParams,
   classifierToneForFamily,
+  composeTapeItems,
   deriveAlertDirection,
   countActiveFlowFilterGroups,
+  filterOptionTapeItems,
+  findAnchorRestoreIndex,
   formatCompactUsd,
   formatOptionContractLabel,
   flushPausableTapeData,
+  getEffectiveOptionPrintFilters,
   getAlertWindowAnchorTs,
+  getHotChannelFeedStatus,
+  getScopedLiveAutoHydrationChannels,
+  getLiveHistoryRetentionCap,
   getOptionTableSnapshot,
+  getOptionScope,
   getLiveFeedStatus,
   getLiveManifest,
+  getRouteFeatures,
+  getTapeVirtualConfig,
+  mergeNewestWithOverflow,
   normalizeAlertSeverity,
   nextFlowFilterPopoverState,
   projectPausableTapeState,
   reducePausableTapeData,
   shouldRetainLiveSnapshotHistory,
+  shouldIncludeEquitiesForDarkUnderlyingFallback,
   shouldShowEquitiesSilentFeedWarning,
   selectPrimaryClassifierHit,
+  shouldClearOptionFocusSeed,
   smartMoneyProfileLabel,
   smartMoneyToneForProfile,
   statusLabel,
@@ -31,6 +47,25 @@ const makeItem = (traceId: string, seq: number, ts: number) => ({
   ts
 });
 
+const makeOptionPrint = (overrides: Record<string, unknown> = {}) =>
+  ({
+    trace_id: "opt-1",
+    seq: 1,
+    ts: 1_000,
+    source_ts: 1_000,
+    ingest_ts: 1_001,
+    option_contract_id: "AAPL-2025-01-17-200-C",
+    underlying_id: "AAPL",
+    option_type: "call",
+    nbbo_side: "A",
+    notional: 250_000,
+    signal_pass: true,
+    price: 1,
+    size: 10,
+    exchange: "X",
+    ...overrides
+  }) as any;
+
 const makeAlert = (overrides: Record<string, unknown> = {}) =>
   ({
     trace_id: "alert-1",
@@ -43,15 +78,13 @@ const makeAlert = (overrides: Record<string, unknown> = {}) =>
   }) as any;
 
 describe("live manifest", () => {
-  it("includes options on home and tape", () => {
+  it("includes only tape channels on /tape", () => {
     const filters = buildDefaultFlowFilters();
-    for (const pathname of ["/", "/tape"]) {
+    const channels = getLiveManifest("/tape", "SPY", 60000, filters).map(
-      expect(
+      (subscription) => subscription.channel
-        getLiveManifest(pathname, "SPY", 60000, filters).some(
+    );
-          (subscription) => subscription.channel === "options"
+
-        )
+    expect(channels).toEqual(["options", "nbbo", "equities", "flow"]);
-      ).toBe(true);
-    }
   });
 
   it("dedupes tape options subscription", () => {
@@ -64,37 +97,29 @@ describe("live manifest", () => {
     expect(tapeOptionsSubscriptions).toHaveLength(1);
   });
 
-  it("keeps option filters on baseline subscription across page changes", () => {
+  it("keeps option filters on /tape options subscriptions", () => {
     const filters = {
       ...buildDefaultFlowFilters(),
       minNotional: 125_000
     };
 
-    const homeOptionsSubscription = getLiveManifest("/", "SPY", 60000, filters).find(
-      (subscription) => subscription.channel === "options"
-    );
     const tapeOptionsSubscription = getLiveManifest("/tape", "SPY", 60000, filters).find(
       (subscription) => subscription.channel === "options"
     );
 
-    expect(homeOptionsSubscription?.filters).toBe(filters);
     expect(tapeOptionsSubscription?.filters).toBe(filters);
   });
 
-  it("applies global flow filters to flow subscriptions on home and tape", () => {
+  it("applies global flow filters to flow subscriptions on /tape", () => {
     const filters = {
       ...buildDefaultFlowFilters(),
       minNotional: 50_000
     };
 
-    const homeFlowSubscription = getLiveManifest("/", "SPY", 60000, filters).find(
-      (subscription) => subscription.channel === "flow"
-    );
     const tapeFlowSubscription = getLiveManifest("/tape", "SPY", 60000, filters).find(
       (subscription) => subscription.channel === "flow"
     );
 
-    expect(homeFlowSubscription?.filters).toBe(filters);
     expect(tapeFlowSubscription?.filters).toBe(filters);
   });
 
@@ -123,6 +148,239 @@ describe("live manifest", () => {
     expect(optionsSubscription?.option_contract_id).toBe("AAPL-2025-01-17-200-C");
     expect(equitiesSubscription?.underlying_ids).toEqual(["AAPL"]);
   });
+
+  it("drops option-print filters for contract-focused options subscriptions but keeps flow filters", () => {
+    const filters = {
+      ...buildDefaultFlowFilters(),
+      minNotional: 500_000,
+      optionTypes: ["put"] as const
+    };
+    const manifest = getLiveManifest(
+      "/tape",
+      "AAPL",
+      60000,
+      filters,
+      {
+        underlying_ids: ["AAPL"],
+        option_contract_id: "AAPL-2025-01-17-200-C"
+      },
+      { underlying_ids: ["AAPL"] },
+      undefined
+    );
+    const optionsSubscription = manifest.find((subscription) => subscription.channel === "options");
+    const flowSubscription = manifest.find((subscription) => subscription.channel === "flow");
+
+    expect(optionsSubscription?.filters).toBeUndefined();
+    expect(flowSubscription?.filters).toBe(filters);
+  });
+
+  it("scopes /signals subscriptions to signals channels only", () => {
+    const channels = getLiveManifest("/signals", "SPY", 60000, buildDefaultFlowFilters()).map(
+      (subscription) => subscription.channel
+    );
+
+    expect(channels).toEqual([
+      "alerts",
+      "smart-money",
+      "classifier-hits",
+      "inferred-dark",
+      "equity-joins"
+    ]);
+  });
+
+  it("scopes /charts subscriptions to chart channels only", () => {
+    const channels = getLiveManifest("/charts", "SPY", 60000, buildDefaultFlowFilters()).map(
+      (subscription) => subscription.channel
+    );
+
+    expect(channels).toEqual([
+      "smart-money",
+      "inferred-dark",
+      "equity-joins",
+      "equity-candles",
+      "equity-overlay"
+    ]);
+  });
+});
+
+describe("contract-focused option helpers", () => {
+  it("uses the focused contract underlying for option scope even when ticker input differs", () => {
+    expect(
+      getOptionScope(["MSFT"], "AAPL", {
+        kind: "option-contract",
+        contractId: "AAPL-2025-01-17-200-C",
+        underlyingId: "AAPL"
+      })
+    ).toEqual({
+      underlying_ids: ["AAPL"],
+      option_contract_id: "AAPL-2025-01-17-200-C"
+    });
+  });
+
+  it("ignores broad flow filters for focused contract options", () => {
+    const filters = {
+      ...buildDefaultFlowFilters(),
+      minNotional: 500_000
+    };
+    const items = [
+      makeOptionPrint({
+        trace_id: "focused-low",
+        option_contract_id: "AAPL-2025-01-17-200-C",
+        notional: 100_000,
+        signal_pass: false
+      }),
+      makeOptionPrint({
+        trace_id: "focused-high",
+        seq: 2,
+        ts: 2_000,
+        option_contract_id: "AAPL-2025-01-17-200-C",
+        notional: 750_000
+      }),
+      makeOptionPrint({
+        trace_id: "other-contract",
+        seq: 3,
+        ts: 3_000,
+        option_contract_id: "MSFT-2025-01-17-300-C",
+        underlying_id: "MSFT",
+        notional: 900_000
+      })
+    ];
+
+    expect(
+      filterOptionTapeItems(
+        items,
+        getEffectiveOptionPrintFilters(filters, true),
+        {
+          kind: "option-contract",
+          contractId: "AAPL-2025-01-17-200-C",
+          underlyingId: "AAPL"
+        },
+        new Set(["MSFT"]),
+        "AAPL"
+      ).map((item) => item.trace_id)
+    ).toEqual(["focused-low", "focused-high"]);
+  });
+
+  it("includes option_contract_id and drops broad filters in focused replay query params", () => {
+    const filters = {
+      ...buildDefaultFlowFilters(),
+      minNotional: 500_000,
+      optionTypes: ["put"] as const
+    };
+
+    expect(
+      buildOptionTapeQueryParams(getEffectiveOptionPrintFilters(filters, true), {
+        underlying_ids: ["AAPL"],
+        option_contract_id: "AAPL-2025-01-17-200-C"
+      })
+    ).toEqual({
+      underlying_ids: "AAPL",
+      option_contract_id: "AAPL-2025-01-17-200-C"
+    });
+  });
+
+  it("keeps the focus seed until the matching scoped subscription has loaded it", () => {
+    const seedItem = makeOptionPrint({
+      trace_id: "focused-seed",
+      option_contract_id: "AAPL-2025-01-17-200-C"
+    });
+    const seed = {
+      scopeKey: "option-contract:AAPL-2025-01-17-200-C",
+      subscriptionKey: getLiveSubscriptionKey({
+        channel: "options",
+        underlying_ids: ["AAPL"],
+        option_contract_id: "AAPL-2025-01-17-200-C"
+      }),
+      items: [seedItem]
+    };
+
+    expect(
+      shouldClearOptionFocusSeed(
+        seed,
+        "option-contract:AAPL-2025-01-17-200-C",
+        getLiveSubscriptionKey({
+          channel: "options",
+          filters: {
+            ...buildDefaultFlowFilters(),
+            minNotional: 500_000
+          },
+          underlying_ids: ["AAPL"]
+        }),
+        [makeOptionPrint({ trace_id: "broad-old" })],
+        []
+      )
+    ).toBe(false);
+
+    expect(
+      shouldClearOptionFocusSeed(
+        seed,
+        "option-contract:AAPL-2025-01-17-200-C",
+        getLiveSubscriptionKey({
+          channel: "options",
+          underlying_ids: ["AAPL"],
+          option_contract_id: "AAPL-2025-01-17-200-C"
+        }),
+        [seedItem],
+        []
+      )
+    ).toBe(true);
+  });
+});
+
+describe("route feature map", () => {
+  it("maps /tape to tape panes and dependencies", () => {
+    const features = getRouteFeatures("/tape");
+    expect(features.showOptionsPane).toBe(true);
+    expect(features.showEquitiesPane).toBe(true);
+    expect(features.showFlowPane).toBe(true);
+    expect(features.needsClassifierDecor).toBe(true);
+    expect(features.alerts).toBe(false);
+  });
+
+  it("maps /signals to signal panes and dependencies", () => {
+    const features = getRouteFeatures("/signals");
+    expect(features.showAlertsPane).toBe(true);
+    expect(features.showClassifierPane).toBe(true);
+    expect(features.showDarkPane).toBe(true);
+    expect(features.options).toBe(false);
+    expect(features.equityJoins).toBe(true);
+  });
+
+  it("maps /charts to chart panes and dependencies", () => {
+    const features = getRouteFeatures("/charts");
+    expect(features.showChartPane).toBe(true);
+    expect(features.showFocusPane).toBe(true);
+    expect(features.equityCandles).toBe(true);
+    expect(features.equityOverlay).toBe(true);
+    expect(features.alerts).toBe(false);
+  });
+});
+
+describe("fixed tape virtualization config", () => {
+  it("uses expected fixed row heights and overscan by table", () => {
+    expect(getTapeVirtualConfig("options")).toEqual({ rowHeight: 36, overscan: 24, debugLabel: "options" });
+    expect(getTapeVirtualConfig("equities")).toEqual({ rowHeight: 36, overscan: 20, debugLabel: "equities" });
+    expect(getTapeVirtualConfig("flow")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "flow" });
+    expect(getTapeVirtualConfig("alerts")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "alerts" });
+    expect(getTapeVirtualConfig("classifier")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "classifier" });
+    expect(getTapeVirtualConfig("dark")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "dark" });
+  });
+});
+
+describe("dark underlying route dependency helper", () => {
+  it("does not keep extra equities subscriptions when joins+trace fallback are sufficient", () => {
+    expect(shouldIncludeEquitiesForDarkUnderlyingFallback()).toBe(false);
+    expect(
+      getLiveManifest("/signals", "SPY", 60000, buildDefaultFlowFilters()).some(
+        (subscription) => subscription.channel === "equities"
+      )
+    ).toBe(false);
+    expect(
+      getLiveManifest("/charts", "SPY", 60000, buildDefaultFlowFilters()).some(
+        (subscription) => subscription.channel === "equities"
+      )
+    ).toBe(false);
+  });
 });
 
 describe("terminal navigation", () => {
@@ -240,6 +498,171 @@ describe("live tape pausable helpers", () => {
   });
 });
 
+describe("live tape history helpers", () => {
+  it("composes tape items across seed, live, and history without seam duplicates", () => {
+    const seed = [makeItem("seed", 1, 100), makeItem("dup", 2, 200)];
+    const live = [makeItem("live", 5, 500), makeItem("dup", 2, 200)];
+    const history = [makeItem("old", 0, 50), makeItem("mid", 3, 300)];
+
+    expect(composeTapeItems(seed, live, history).map((item) => item.trace_id)).toEqual([
+      "live",
+      "mid",
+      "dup",
+      "seed",
+      "old"
+    ]);
+  });
+
+  it("keeps a clicked seed row visible before scoped live and history arrive", () => {
+    const clicked = makeItem("clicked", 3, 300);
+
+    expect(composeTapeItems([clicked], [], []).map((item) => item.trace_id)).toEqual(["clicked"]);
+  });
+
+  it("drops focus seed duplicates once equivalent live or history rows arrive", () => {
+    const clicked = makeItem("clicked", 3, 300);
+    const live = [makeItem("new", 4, 400)];
+    const history = [makeItem("clicked", 3, 300)];
+
+    expect(composeTapeItems([clicked], live, history).map((item) => item.trace_id)).toEqual([
+      "new",
+      "clicked"
+    ]);
+  });
+
+  it("promotes hot-window overflow into the history tail", () => {
+    const currentHot = [makeItem("hot-3", 3, 300), makeItem("hot-2", 2, 200), makeItem("hot-1", 1, 100)];
+    const incoming = [makeItem("hot-4", 4, 400)];
+
+    const { kept, evicted } = mergeNewestWithOverflow(incoming, currentHot, 3);
+    const nextHistory = appendHistoryTail([], evicted, kept, 5000);
+
+    expect(kept.map((item) => item.trace_id)).toEqual(["hot-4", "hot-3", "hot-2"]);
+    expect(nextHistory.map((item) => item.trace_id)).toEqual(["hot-1"]);
+  });
+
+  it("keeps the combined tape continuous beyond the hot live window", () => {
+    let hot: Array<ReturnType<typeof makeItem>> = [];
+    let history: Array<ReturnType<typeof makeItem>> = [];
+
+    for (let seq = 1; seq <= 5; seq += 1) {
+      const { kept, evicted } = mergeNewestWithOverflow([makeItem(`row-${seq}`, seq, seq * 100)], hot, 2);
+      hot = kept;
+      history = appendHistoryTail(history, evicted, hot, 5000);
+    }
+
+    expect([...hot, ...history].map((item) => item.trace_id)).toEqual([
+      "row-5",
+      "row-4",
+      "row-3",
+      "row-2",
+      "row-1"
+    ]);
+  });
+
+  it("appends older scoped rows behind the hot live head", () => {
+    const liveHead = Array.from({ length: 100 }, (_, idx) =>
+      makeItem(`hot-${idx}`, 200 - idx, 2_000 - idx)
+    );
+    const older = [makeItem("older-1", 99, 999), makeItem("older-2", 98, 998)];
+
+    const next = appendHistoryTail([], older, liveHead, 5000);
+
+    expect(next.map((item) => item.trace_id)).toEqual(["older-1", "older-2"]);
+  });
+
+  it("skips duplicates already present in the live head", () => {
+    const liveHead = [makeItem("latest", 3, 300), makeItem("duplicate", 2, 200)];
+    const older = [makeItem("duplicate", 2, 200), makeItem("older", 1, 100)];
+
+    const next = appendHistoryTail([], older, liveHead, 5000);
+
+    expect(next.map((item) => item.trace_id)).toEqual(["older"]);
+  });
+
+  it("dedupes the seam between promoted overflow and fetched history", () => {
+    const currentHot = [makeItem("hot-3", 3, 300), makeItem("hot-2", 2, 200), makeItem("hot-1", 1, 100)];
+    const { kept, evicted } = mergeNewestWithOverflow([makeItem("hot-4", 4, 400)], currentHot, 3);
+    const promoted = appendHistoryTail([], evicted, kept, 5000);
+    const merged = appendHistoryTail(promoted, [makeItem("hot-1", 1, 100), makeItem("older", 0, 50)], kept, 5000);
+
+    expect(merged.map((item) => item.trace_id)).toEqual(["hot-1", "older"]);
+    expect(new Set([...kept, ...merged].map((item) => item.trace_id)).size).toBe(kept.length + merged.length);
+  });
+
+  it("trims the history tail to the soft cap", () => {
+    const current = [makeItem("existing", 4, 400)];
+    const older = [makeItem("older-1", 3, 300), makeItem("older-2", 2, 200)];
+
+    const next = appendHistoryTail(current, older, [], 2);
+
+    expect(next.map((item) => item.trace_id)).toEqual(["existing", "older-1"]);
+  });
+
+  it("keeps scoped option and equity history on the normal retention cap", () => {
+    expect(
+      getLiveHistoryRetentionCap({
+        channel: "options",
+        underlying_ids: ["AAPL"],
+        option_contract_id: "AAPL-2025-01-17-200-C"
+      } as any)
+    ).toBeGreaterThan(0);
+    expect(
+      getLiveHistoryRetentionCap({
+        channel: "equities",
+        underlying_ids: ["AAPL"]
+      } as any)
+    ).toBeGreaterThan(0);
+  });
+
+  it("keeps auto-hydrating scoped live history while next_before exists", () => {
+    const manifest = getLiveManifest(
+      "/tape",
+      "AAPL",
+      60000,
+      buildDefaultFlowFilters(),
+      {
+        underlying_ids: ["AAPL"],
+        option_contract_id: "AAPL-2025-01-17-200-C"
+      },
+      { underlying_ids: ["AAPL"] }
+    );
+    const historyCursors = Object.fromEntries(
+      manifest.map((subscription) => [getLiveSubscriptionKey(subscription), { ts: 1, seq: 1 }])
+    );
+
+    expect(
+      getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {})
+    ).toEqual(["options", "equities"]);
+    expect(
+      getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {
+        [getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "options")!)]: true
+      })
+    ).toEqual(["equities"]);
+    expect(
+      getScopedLiveAutoHydrationChannels(true, "/tape", manifest, {
+        ...historyCursors,
+        [getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "equities")!)]: null
+      }, {})
+    ).toEqual(["options"]);
+  });
+
+  it("restores the same anchor key after live insertions at the top", () => {
+    const nextKeys = ["new-1", "new-2", "anchor", "after-1", "after-2"];
+    expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(2);
+  });
+
+  it("falls forward to the nearest surviving key when the anchor is evicted", () => {
+    const nextKeys = ["new-1", "after-1", "after-2"];
+    expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(1);
+  });
+
+  it("keeps the same anchor when history is appended at the bottom", () => {
+    const nextKeys = ["anchor", "after-1", "after-2", "older-1", "older-2"];
+    expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(0);
+  });
+});
+
 describe("options display formatters", () => {
   it("formats dashed option contracts as ticker strike expiry", () => {
     expect(formatOptionContractLabel("SPY-2025-01-17-450-C")).toEqual({
@@ -409,4 +832,13 @@ describe("signals helpers", () => {
     expect(statusLabel("connected", false, "live")).toBe("Connected");
     expect(statusLabel("stale", false, "live")).toBe("Feed behind");
   });
+
+  it("treats healthy scoped channels as connected even when no matching rows are visible", () => {
+    expect(getHotChannelFeedStatus("connected", { healthy: true })).toBe("connected");
+  });
+
+  it("surfaces feed behind only when the backend channel health is stale", () => {
+    expect(getHotChannelFeedStatus("connected", { healthy: false })).toBe("stale");
+    expect(getHotChannelFeedStatus("disconnected", { healthy: true })).toBe("disconnected");
+  });
 });

apps/web/package.json

@@ -9,6 +9,7 @@
   },
   "dependencies": {
     "@islandflow/types": "workspace:*",
+    "@tanstack/react-virtual": "^3.13.24",
     "lightweight-charts": "^4.2.0",
     "next": "^14.2.4",
     "react": "^18.3.1",

bun.lock

@@ -12,6 +12,7 @@
       "name": "@islandflow/web",
       "dependencies": {
         "@islandflow/types": "workspace:*",
+        "@tanstack/react-virtual": "^3.13.24",
         "lightweight-charts": "^4.2.0",
         "next": "^14.2.4",
         "react": "^18.3.1",
@@ -208,6 +209,10 @@
 
     "@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="],
 
+    "@tanstack/react-virtual": ["@tanstack/react-virtual@3.13.24", "", { "dependencies": { "@tanstack/virtual-core": "3.14.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aIJvz5OSkhNIhZIpYivrxrPTKYsjW9Uzy+sP/mx0S3sev2HyvPb7xmjbYvokzEpfgYHy/HjzJ2zFAETuUfgCpg=="],
+
+    "@tanstack/virtual-core": ["@tanstack/virtual-core@3.14.0", "", {}, "sha512-JLANqGy/D6k4Ujmh8Tr25lGimuOXNiaVyXaCAZS0W+1390sADdGnyUdSWNIfd49gebtIxGMij4IktRVzrdr12Q=="],
+
     "@types/node": ["@types/node@20.19.27", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-N2clP5pJhB2YnZJ3PIHFk5RkygRX5WO/5f0WC08tp0wd+sv0rsJk3MqWn3CbNmT2J505a5336jaQj4ph1AdMug=="],
 
     "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="],

deployment/docker/.env.example

@@ -98,6 +98,7 @@ CLASSIFIER_0DTE_MIN_PREMIUM=20000
 CLASSIFIER_0DTE_MIN_SIZE=400
 
 # Smart money refdata
+LOG_LEVEL=warn
 SMART_MONEY_EVENT_CALENDAR_PATH=data/event-calendar.json
 REFDATA_EVENT_CALENDAR_PATH=
 REFDATA_EVENT_CALENDAR_PROVIDER=
@@ -120,3 +121,33 @@ REPLAY_END_TS=0
 REPLAY_SPEED=1
 REPLAY_BATCH_SIZE=200
 REPLAY_LOG_EVERY=1000
+
+# API live retention
+LIVE_LIMIT_DEFAULT=1000
+LIVE_LIMIT_OPTIONS=1000
+LIVE_LIMIT_NBBO=1000
+LIVE_LIMIT_EQUITIES=1000
+LIVE_LIMIT_EQUITY_QUOTES=500
+LIVE_LIMIT_EQUITY_JOINS=500
+LIVE_LIMIT_FLOW=500
+LIVE_LIMIT_SMART_MONEY=300
+LIVE_LIMIT_CLASSIFIER_HITS=300
+LIVE_LIMIT_ALERTS=300
+LIVE_LIMIT_INFERRED_DARK=300
+LIVE_SCOPED_CACHE_MAX_KEYS=32
+LIVE_REDIS_FLUSH_INTERVAL_MS=250
+LIVE_REDIS_FLUSH_MAX_ITEMS=100
+
+# Compute and ingest cache retention
+ROLLING_CACHE_FLUSH_INTERVAL_MS=30000
+ROLLING_CACHE_MAX_KEYS=20000
+OPTION_CONTEXT_MAX_KEYS=20000
+OPTION_CONTEXT_TTL_MS=900000
+COMPUTE_NBBO_CACHE_MAX_KEYS=20000
+COMPUTE_NBBO_CACHE_TTL_MS=900000
+
+# JetStream retention
+STREAM_RAW_MAX_AGE_MS=7200000
+STREAM_RAW_MAX_BYTES=1073741824
+STREAM_DERIVED_MAX_AGE_MS=86400000
+STREAM_DERIVED_MAX_BYTES=536870912

deployment/docker/README.md

@@ -21,6 +21,7 @@ It is separate from the repo-root `docker-compose.yml`, which is still the light
 - `deployment/docker/Dockerfile.service`: shared Bun runtime image for most services
 - `deployment/docker/Dockerfile.ingest-options`: Bun runtime plus Python dependencies for Databento and IBKR adapters
 - `deployment/docker/Dockerfile.web`: multi-stage build for the Next.js web app
+- `deployment/docker/workspace-root/`: deployment-specific workspace snapshot (`package.json`, `tsconfig.base.json`, `bun.lock`) used by Docker builds
 - `deployment/docker/clickhouse/listen.xml`: forces ClickHouse to listen on IPv4 for other containers on the Docker network
 - `deployment/docker/.env.example`: container-oriented environment template
 
@@ -185,6 +186,22 @@ If NPM is on multiple networks and names collide (for example another stack also
 
 ## Updating the deployment
 
+This deployment installs dependencies from `deployment/docker/workspace-root/bun.lock` (not the repo-root lockfile).
+
+When dependencies change in any workspace used by Docker builds, refresh and validate the deployment snapshot first:
+
+```bash
+bun run sync:docker-workspace
+bun run check:docker-workspace
+```
+
+Then validate the VPS build path:
+
+```bash
+cd deployment/docker
+docker compose build web
+```
+
 When you pull new code:
 
 ```bash

deployment/docker/deploy-branch.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+git fetch
+git pull
+docker compose up -d --build --force-recreate

deployment/docker/deploy.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+git fetch
+git switch deployment
+git pull
+docker compose up -d --build --force-recreate

deployment/docker/docker-compose.yml

@@ -14,6 +14,8 @@ x-service-common: &service-common
     dockerfile: Dockerfile.service
   env_file:
     - ./.env
+  environment:
+    LOG_LEVEL: ${LOG_LEVEL:-warn}
   restart: unless-stopped
   init: true
   extra_hosts:
@@ -94,6 +96,8 @@ services:
       dockerfile: Dockerfile.ingest-options
     env_file:
       - ./.env
+    environment:
+      LOG_LEVEL: ${LOG_LEVEL:-warn}
     restart: unless-stopped
     init: true
     extra_hosts:

deployment/docker/workspace-root/bun.lock

@@ -12,6 +12,7 @@
       "name": "@islandflow/web",
       "dependencies": {
         "@islandflow/types": "workspace:*",
+        "@tanstack/react-virtual": "^3.13.24",
         "lightweight-charts": "^4.2.0",
         "next": "^14.2.4",
         "react": "^18.3.1",
@@ -81,6 +82,7 @@
         "@islandflow/bus": "workspace:*",
         "@islandflow/config": "workspace:*",
         "@islandflow/observability": "workspace:*",
+        "@islandflow/refdata": "workspace:*",
         "@islandflow/storage": "workspace:*",
         "@islandflow/types": "workspace:*",
         "redis": "^5.10.0",
@@ -207,6 +209,10 @@
 
     "@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="],
 
+    "@tanstack/react-virtual": ["@tanstack/react-virtual@3.13.24", "", { "dependencies": { "@tanstack/virtual-core": "3.14.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aIJvz5OSkhNIhZIpYivrxrPTKYsjW9Uzy+sP/mx0S3sev2HyvPb7xmjbYvokzEpfgYHy/HjzJ2zFAETuUfgCpg=="],
+
+    "@tanstack/virtual-core": ["@tanstack/virtual-core@3.14.0", "", {}, "sha512-JLANqGy/D6k4Ujmh8Tr25lGimuOXNiaVyXaCAZS0W+1390sADdGnyUdSWNIfd49gebtIxGMij4IktRVzrdr12Q=="],
+
     "@types/node": ["@types/node@20.19.27", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-N2clP5pJhB2YnZJ3PIHFk5RkygRX5WO/5f0WC08tp0wd+sv0rsJk3MqWn3CbNmT2J505a5336jaQj4ph1AdMug=="],
 
     "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="],

deployment/docker/workspace-root/package.json

@@ -12,7 +12,9 @@
     "dev:infra": "docker compose up",
     "dev:infra:down": "docker compose down",
     "dev:web": "bun --cwd=apps/web run dev",
-    "dev:services": "bun run scripts/dev-services.ts"
+    "dev:services": "bun run scripts/dev-services.ts",
+    "sync:docker-workspace": "bun run scripts/sync-docker-workspace.ts",
+    "check:docker-workspace": "bun run scripts/check-docker-workspace.ts"
   },
   "devDependencies": {
     "typescript-language-server": "^5.1.3"

deployment/docker/workspace-root/tsconfig.base.json

@@ -8,6 +8,6 @@
     "isolatedModules": true,
     "resolveJsonModule": true,
     "skipLibCheck": true,
-    "noEmit": true
+    "noEmit": true,
-  }
+  },
 }

package.json

@@ -12,7 +12,9 @@
     "dev:infra": "docker compose up",
     "dev:infra:down": "docker compose down",
     "dev:web": "bun --cwd=apps/web run dev",
-    "dev:services": "bun run scripts/dev-services.ts"
+    "dev:services": "bun run scripts/dev-services.ts",
+    "sync:docker-workspace": "bun run scripts/sync-docker-workspace.ts",
+    "check:docker-workspace": "bun run scripts/check-docker-workspace.ts"
   },
   "devDependencies": {
     "typescript-language-server": "^5.1.3"

packages/bus/src/jetstream.ts

@@ -84,6 +84,50 @@ export const ensureStream = async (
   }
 };
 
+const parseBoundedNumber = (value: string | undefined, fallback: number): number => {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed < 0) {
+    return fallback;
+  }
+  return Math.floor(parsed);
+};
+
+export type StreamRetentionClass = "raw" | "derived";
+
+export const resolveStreamRetention = (
+  streamClass: StreamRetentionClass,
+  env: Record<string, string | undefined> = process.env
+): Pick<StreamConfig, "max_bytes" | "max_age"> => {
+  if (streamClass === "raw") {
+    return {
+      max_age: parseBoundedNumber(env.STREAM_RAW_MAX_AGE_MS, 7_200_000),
+      max_bytes: parseBoundedNumber(env.STREAM_RAW_MAX_BYTES, 1_073_741_824)
+    };
+  }
+
+  return {
+    max_age: parseBoundedNumber(env.STREAM_DERIVED_MAX_AGE_MS, 86_400_000),
+    max_bytes: parseBoundedNumber(env.STREAM_DERIVED_MAX_BYTES, 536_870_912)
+  };
+};
+
+export const buildStreamConfig = (
+  name: string,
+  subject: string,
+  streamClass: StreamRetentionClass,
+  env: Record<string, string | undefined> = process.env
+): StreamConfig => ({
+  name,
+  subjects: [subject],
+  retention: "limits",
+  storage: "file",
+  discard: "old",
+  max_msgs_per_subject: -1,
+  max_msgs: -1,
+  ...resolveStreamRetention(streamClass, env),
+  num_replicas: 1
+});
+
 export const buildDurableConsumer = (
   durableName: string,
   deliverSubject: string = createInbox()

packages/observability/src/logger.ts

@@ -22,20 +22,46 @@ export type LoggerOptions = {
   service: string;
   now?: () => string;
   sink?: (record: LogRecord) => void;
+  level?: LogLevel;
 };
 
 const defaultSink = (record: LogRecord) => {
   console.log(JSON.stringify(record));
 };
 
+const LOG_LEVEL_ORDER: Record<LogLevel, number> = {
+  debug: 10,
+  info: 20,
+  warn: 30,
+  error: 40
+};
+
+const resolveLogLevel = (value: string | undefined): LogLevel => {
+  switch ((value ?? "").trim().toLowerCase()) {
+    case "debug":
+    case "info":
+    case "warn":
+    case "error":
+      return value!.trim().toLowerCase() as LogLevel;
+    default:
+      return "info";
+  }
+};
+
 export const createLogger = ({
   service,
   now = () => new Date().toISOString(),
-  sink = defaultSink
+  sink = defaultSink,
+  level = resolveLogLevel(process.env.LOG_LEVEL)
 }: LoggerOptions): Logger => {
-  const write = (level: LogLevel, msg: string, context?: LogContext) => {
+  const levelThreshold = resolveLogLevel(level);
+
+  const write = (recordLevel: LogLevel, msg: string, context?: LogContext) => {
+    if (LOG_LEVEL_ORDER[recordLevel] < LOG_LEVEL_ORDER[levelThreshold]) {
+      return;
+    }
     const record: LogRecord = {
-      level,
+      level: recordLevel,
       service,
       msg,
       ts: now(),

packages/storage/src/clickhouse.ts

@@ -449,6 +449,157 @@ export const insertAlert = async (client: ClickHouseClient, alert: AlertEvent):
   });
 };
 
+export type ClickHouseBatchWriterOptions = {
+  flushIntervalMs?: number;
+  maxRows?: number;
+  onError?: (table: string, error: unknown, rowCount: number) => void;
+};
+
+type BatchState = {
+  rows: unknown[];
+  timer: ReturnType<typeof setTimeout> | null;
+  flushing: Promise<void> | null;
+};
+
+const createBatchState = (): BatchState => ({
+  rows: [],
+  timer: null,
+  flushing: null
+});
+
+export class ClickHouseBatchWriter {
+  private readonly flushIntervalMs: number;
+  private readonly maxRows: number;
+  private readonly states = new Map<string, BatchState>();
+
+  constructor(
+    private readonly client: ClickHouseClient,
+    options: ClickHouseBatchWriterOptions = {}
+  ) {
+    this.flushIntervalMs = Math.max(1, Math.floor(options.flushIntervalMs ?? 100));
+    this.maxRows = Math.max(1, Math.floor(options.maxRows ?? 250));
+    this.onError = options.onError;
+  }
+
+  private readonly onError?: (table: string, error: unknown, rowCount: number) => void;
+
+  enqueue(table: string, row: unknown): void {
+    const state = this.states.get(table) ?? createBatchState();
+    if (!this.states.has(table)) {
+      this.states.set(table, state);
+    }
+
+    state.rows.push(row);
+
+    if (state.rows.length >= this.maxRows) {
+      void this.flush(table);
+      return;
+    }
+
+    if (!state.timer) {
+      state.timer = setTimeout(() => {
+        state.timer = null;
+        void this.flush(table);
+      }, this.flushIntervalMs);
+    }
+  }
+
+  async flush(table: string): Promise<void> {
+    const state = this.states.get(table);
+    if (!state) {
+      return;
+    }
+
+    if (state.flushing) {
+      await state.flushing;
+      return;
+    }
+
+    if (state.timer) {
+      clearTimeout(state.timer);
+      state.timer = null;
+    }
+
+    if (state.rows.length === 0) {
+      return;
+    }
+
+    const rows = state.rows.splice(0, state.rows.length);
+    state.flushing = this.client
+      .insert({
+        table,
+        values: rows,
+        format: "JSONEachRow"
+      })
+      .catch((error) => {
+        this.onError?.(table, error, rows.length);
+      })
+      .finally(() => {
+        state.flushing = null;
+      });
+
+    await state.flushing;
+  }
+
+  async flushAll(): Promise<void> {
+    for (const table of this.states.keys()) {
+      await this.flush(table);
+    }
+  }
+
+  async close(): Promise<void> {
+    for (const state of this.states.values()) {
+      if (state.timer) {
+        clearTimeout(state.timer);
+        state.timer = null;
+      }
+    }
+    await this.flushAll();
+  }
+}
+
+export const enqueueEquityPrintJoinInsert = (
+  writer: ClickHouseBatchWriter,
+  join: EquityPrintJoin
+): void => {
+  writer.enqueue(EQUITY_PRINT_JOINS_TABLE, toEquityPrintJoinRecord(join));
+};
+
+export const enqueueInferredDarkInsert = (
+  writer: ClickHouseBatchWriter,
+  event: InferredDarkEvent
+): void => {
+  writer.enqueue(INFERRED_DARK_TABLE, toInferredDarkRecord(event));
+};
+
+export const enqueueFlowPacketInsert = (
+  writer: ClickHouseBatchWriter,
+  packet: FlowPacket
+): void => {
+  writer.enqueue(FLOW_PACKETS_TABLE, toFlowPacketRecord(packet));
+};
+
+export const enqueueSmartMoneyEventInsert = (
+  writer: ClickHouseBatchWriter,
+  event: SmartMoneyEvent
+): void => {
+  writer.enqueue(SMART_MONEY_EVENTS_TABLE, toSmartMoneyEventRecord(event));
+};
+
+export const enqueueClassifierHitInsert = (
+  writer: ClickHouseBatchWriter,
+  hit: ClassifierHitEvent
+): void => {
+  writer.enqueue(CLASSIFIER_HITS_TABLE, toClassifierHitRecord(hit));
+};
+
+export const enqueueAlertInsert = (
+  writer: ClickHouseBatchWriter,
+  alert: AlertEvent
+): void => {
+  writer.enqueue(ALERTS_TABLE, toAlertRecord(alert));
+};
+
 const clampLimit = (limit: number): number => {
   if (!Number.isFinite(limit)) {
     return 100;

packages/types/src/live.ts

@@ -54,6 +54,24 @@ export const LiveChannelSchema = z.enum([
 
 export type LiveChannel = z.infer<typeof LiveChannelSchema>;
 export type LiveGenericChannel = z.infer<typeof LiveGenericChannelSchema>;
+export const LiveHotChannelSchema = z.enum(["options", "nbbo", "equities", "flow"]);
+export type LiveHotChannel = z.infer<typeof LiveHotChannelSchema>;
+
+export const LiveChannelHealthSchema = z.object({
+  freshness_age_ms: z.number().int().nonnegative().nullable(),
+  healthy: z.boolean()
+});
+
+export type LiveChannelHealth = z.infer<typeof LiveChannelHealthSchema>;
+
+export const LiveHotChannelHealthSchema = z.object({
+  options: LiveChannelHealthSchema,
+  nbbo: LiveChannelHealthSchema,
+  equities: LiveChannelHealthSchema,
+  flow: LiveChannelHealthSchema
+});
+
+export type LiveHotChannelHealthMap = z.infer<typeof LiveHotChannelHealthSchema>;
 
 export const LiveSubscriptionSchema = z.discriminatedUnion("channel", [
   z.object({
@@ -152,7 +170,8 @@ export const LiveClientMessageSchema = z.discriminatedUnion("op", [
 export type LiveClientMessage = z.infer<typeof LiveClientMessageSchema>;
 
 export const LiveReadyMessageSchema = z.object({
-  op: z.literal("ready")
+  op: z.literal("ready"),
+  channel_health: LiveHotChannelHealthSchema
 });
 
 export type LiveReadyMessage = z.infer<typeof LiveReadyMessageSchema>;
@@ -175,7 +194,8 @@ export type LiveEventMessage = z.infer<typeof LiveEventMessageSchema>;
 
 export const LiveHeartbeatMessageSchema = z.object({
   op: z.literal("heartbeat"),
-  ts: z.number().int().nonnegative()
+  ts: z.number().int().nonnegative(),
+  channel_health: LiveHotChannelHealthSchema
 });
 
 export type LiveHeartbeatMessage = z.infer<typeof LiveHeartbeatMessageSchema>;

plans/terminal-extraction-refactor.md

@@ -0,0 +1,372 @@
+# Terminal Extraction Plan
+
+## Summary
+
+Refactor [`apps/web/app/terminal.tsx`](/Users/kell/Cloud/dev/islandflow/apps/web/app/terminal.tsx:1) from a single 7,974-line client module into a feature folder at `apps/web/terminal/*`, while keeping `apps/web/app/terminal.tsx` as a temporary compatibility facade in the first pass.
+
+This first extraction is a medium-scope, behavior-preserving refactor:
+- no product behavior changes
+- no route behavior changes
+- no visual redesign
+- no data model changes
+- no immediate deletion of the old import surface
+
+Current baseline is healthy and must remain healthy after the refactor:
+- `bun test apps/web/app/terminal.test.ts apps/web/app/routes.test.ts` passes
+- `bun --cwd=apps/web run build` passes
+
+## Target Structure
+
+Create this feature layout:
+
+```text
+apps/web/terminal/
+  index.ts
+  state.tsx
+  shell.tsx
+  routes.tsx
+  core/
+    format.ts
+    filters.ts
+    route-config.ts
+    tape-data.ts
+    signals.ts
+    live-manifest.ts
+  hooks/
+    use-tape-data.ts
+    use-live-session.ts
+    use-virtual-tape.ts
+  components/
+    chrome.tsx
+    chart.tsx
+    drawers.tsx
+    panes.tsx
+  tests/
+    core.test.ts
+    live-manifest.test.ts
+    signals.test.ts
+    tape-data.test.ts
+```
+
+Keep this file in place for the first pass:
+- `apps/web/app/terminal.tsx`
+
+Its final first-pass role is:
+- `"use client"` entrypoint
+- thin re-export facade only
+- no business logic
+- no React state
+- no websocket/session logic
+- no chart implementation
+- target size: under 120 lines
+
+## Dependency Rules
+
+Use this dependency direction and do not violate it:
+- `core/*` may depend only on shared types and other `core/*`
+- `hooks/*` may depend on `core/*`
+- `components/*` may depend on `core/*` and `hooks/*`
+- `state.tsx` may depend on `core/*`, `hooks/*`, and `components/*` types only as needed
+- `shell.tsx` and `routes.tsx` may depend on `state.tsx` and `components/*`
+- `index.ts` re-exports public feature symbols
+- `app/terminal.tsx` re-exports from `apps/web/terminal/index.ts`
+
+Do not allow circular imports.
+
+## Module Mapping
+
+Move code out of `terminal.tsx` in this order.
+
+### 1. Pure helpers first
+
+Move non-React helpers into `apps/web/terminal/core/*`:
+
+- `core/route-config.ts`
+  - `getRouteFeatures`
+  - `getTapeVirtualConfig`
+  - `shouldIncludeEquitiesForDarkUnderlyingFallback`
+
+- `core/live-manifest.ts`
+  - `getLiveManifest`
+  - `getLiveHistoryRetentionCap`
+  - `getScopedLiveAutoHydrationChannels`
+  - `getLiveFeedStatus`
+  - `getHotChannelFeedStatus`
+
+- `core/tape-data.ts`
+  - `mergeNewestWithOverflow`
+  - `composeTapeItems`
+  - `reducePausableTapeData`
+  - `flushPausableTapeData`
+  - `appendHistoryTail`
+  - `projectPausableTapeState`
+  - `findAnchorRestoreIndex`
+  - `shouldRetainLiveSnapshotHistory`
+  - `shouldShowEquitiesSilentFeedWarning`
+  - tape/history support types used only by these helpers
+
+- `core/format.ts`
+  - `formatCompactUsd`
+  - `formatOptionContractLabel`
+  - `getOptionTableSnapshot`
+  - price/size/time/date/contract formatting helpers that support UI rendering
+
+- `core/signals.ts`
+  - `normalizeAlertSeverity`
+  - `deriveAlertDirection`
+  - `getAlertWindowAnchorTs`
+  - `selectPrimaryClassifierHit`
+  - `classifierToneForFamily`
+  - `smartMoneyToneForProfile`
+  - `smartMoneyProfileLabel`
+
+- `core/filters.ts`
+  - `buildDefaultFlowFilters`
+  - `countActiveFlowFilterGroups`
+  - `toggleFilterValue`
+  - `nextFlowFilterPopoverState`
+
+These files must not include `"use client"`.
+
+### 2. Extract hooks and session logic
+
+Move React hooks into `apps/web/terminal/hooks/*`:
+
+- `hooks/use-virtual-tape.ts`
+  - `useListScroll`
+  - `useScrollAnchor`
+  - `useVirtualHistoryGate`
+  - `useTapeVirtualList`
+
+- `hooks/use-tape-data.ts`
+  - `useTape`
+  - `usePausableTapeView`
+  - `useLiveStream`
+  - `useFlowStream`
+  - `statusLabel`
+  - internal tape state types
+
+- `hooks/use-live-session.ts`
+  - `useLiveSession`
+  - live history endpoint constants
+  - live history query builders
+  - subscription dedupe helpers
+  - session-local types
+
+Keep signatures stable unless a change is required to break a circular dependency. If a signature changes, update all callers in the same PR.
+
+### 3. Extract UI components
+
+Move rendering code into `apps/web/terminal/components/*`:
+
+- `components/chrome.tsx`
+  - `TapeStatus`
+  - `TapeControls`
+  - `PageFrame`
+  - `Pane`
+  - `ShellMetricStrip`
+  - `FlowFilterPopover`
+  - local filter UI helpers
+
+- `components/chart.tsx`
+  - `CandleChart`
+  - chart-only local types and overlay helpers
+  - isolate `lightweight-charts` usage here
+
+- `components/drawers.tsx`
+  - `AlertSeverityStrip`
+  - `AlertDrawer`
+  - `ClassifierHitDrawer`
+  - `SmartMoneyDrawer`
+  - `DarkDrawer`
+
+- `components/panes.tsx`
+  - `OptionsPane`
+  - `EquitiesPane`
+  - `FlowPane`
+  - `AlertsPane`
+  - `ClassifierPane`
+  - `DarkPane`
+  - `ChartPane`
+  - `FocusPane`
+  - `ReplayConsole`
+
+### 4. Extract state orchestration
+
+Create `apps/web/terminal/state.tsx` for:
+- `useTerminalState`
+- `TerminalContext`
+- `useTerminal`
+
+This file owns:
+- route-aware feature selection
+- filter input state
+- selected entity/drawer state
+- scroll-anchor wiring
+- assembly of hook outputs into the single terminal state object
+
+Keep `useTerminalState` internal. Do not export it from the feature barrel.
+
+### 5. Extract shell and routes
+
+Create:
+- `apps/web/terminal/shell.tsx`
+  - `TerminalAppShell`
+
+- `apps/web/terminal/routes.tsx`
+  - `NAV_ITEMS`
+  - `OverviewRoute`
+  - `TapeRoute`
+  - `SignalsRoute`
+  - `ChartsRoute`
+  - `ReplayRoute`
+
+Important first-pass rule:
+- keep existing route behavior exactly as-is
+- [app/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/page.tsx:1), [app/layout.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/layout.tsx:4), and [app/tape/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/tape/page.tsx:1) may continue importing from `./terminal` / `../terminal`
+- [app/signals/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/signals/page.tsx:1), [app/charts/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/charts/page.tsx:1), and [app/replay/page.tsx](/Users/kell/Cloud/dev/islandflow/apps/web/app/replay/page.tsx:1) must remain redirect pages in this pass
+
+## Facade Contract
+
+Replace `apps/web/app/terminal.tsx` with a facade that re-exports from `apps/web/terminal/index.ts`.
+
+The facade must continue exporting these symbols in the first pass:
+
+- `getTapeVirtualConfig`
+- `shouldIncludeEquitiesForDarkUnderlyingFallback`
+- `getRouteFeatures`
+- `mergeNewestWithOverflow`
+- `composeTapeItems`
+- `reducePausableTapeData`
+- `flushPausableTapeData`
+- `appendHistoryTail`
+- `getLiveHistoryRetentionCap`
+- `getScopedLiveAutoHydrationChannels`
+- `getLiveFeedStatus`
+- `getHotChannelFeedStatus`
+- `findAnchorRestoreIndex`
+- `formatCompactUsd`
+- `formatOptionContractLabel`
+- `normalizeAlertSeverity`
+- `deriveAlertDirection`
+- `getAlertWindowAnchorTs`
+- `buildDefaultFlowFilters`
+- `countActiveFlowFilterGroups`
+- `toggleFilterValue`
+- `nextFlowFilterPopoverState`
+- `projectPausableTapeState`
+- `shouldShowEquitiesSilentFeedWarning`
+- `shouldRetainLiveSnapshotHistory`
+- `selectPrimaryClassifierHit`
+- `classifierToneForFamily`
+- `smartMoneyToneForProfile`
+- `smartMoneyProfileLabel`
+- `getOptionTableSnapshot`
+- `statusLabel`
+- `getLiveManifest`
+- `NAV_ITEMS`
+- `FlowFilterPopover`
+- `TerminalAppShell`
+- `OverviewRoute`
+- `TapeRoute`
+- `SignalsRoute`
+- `ChartsRoute`
+- `ReplayRoute`
+
+Do not add new facade-only exports.
+
+## Test Plan
+
+Restructure tests so pure logic is tested from its final home instead of through the facade.
+
+### Keep
+- `apps/web/app/routes.test.ts`
+  - still verifies redirect behavior for `/signals`, `/charts`, `/replay`
+
+### Split `app/terminal.test.ts` into feature tests
+- `apps/web/terminal/tests/live-manifest.test.ts`
+  - route feature mapping
+  - manifest composition
+  - nav items if still treated as route metadata
+
+- `apps/web/terminal/tests/tape-data.test.ts`
+  - merge/dedupe logic
+  - pausable tape behavior
+  - history seam behavior
+  - anchor restore behavior
+  - retention cap behavior
+  - scoped history behavior
+
+- `apps/web/terminal/tests/core.test.ts`
+  - option contract formatting
+  - compact USD formatting
+  - option table snapshot formatting
+  - flow filter helpers
+
+- `apps/web/terminal/tests/signals.test.ts`
+  - alert severity normalization
+  - direction derivation
+  - alert window anchor
+  - classifier/smart-money label and tone helpers
+  - live status labeling if kept outside tape-data tests
+
+Optional and recommended:
+- add one tiny `apps/web/app/terminal-facade.test.ts` that imports the facade and asserts a few critical exports exist, so we notice accidental facade breakage during the transition
+
+## Validation Gates
+
+Implementation is not complete unless all of these pass:
+
+1. `bun test apps/web/terminal/tests apps/web/app/routes.test.ts`
+2. `bun --cwd=apps/web run build`
+3. Existing behavior smoke check:
+   - `/` still renders the shell and overview
+   - `/tape` still renders shell and tape panes
+   - `/signals`, `/charts`, `/replay` still redirect to `/`
+4. `apps/web/app/terminal.tsx` is a facade only and contains no moved logic
+5. No extracted pure helper file contains React imports
+6. No new circular imports are introduced
+
+## Non-Goals For This Pass
+
+Do not do these in the first extraction:
+- redesign panes or drawers
+- change websocket or replay behavior
+- change route inventory
+- remove unused legacy route exports
+- change CSS structure beyond import fixes
+- optimize bundle size as a separate objective
+- rewrite tests to different testing tools
+
+## Beads Follow-Up Issues To File
+
+Create these `bd` issues during implementation if they do not already exist:
+
+1. `task`, priority `2`
+   Title: `Remove temporary apps/web/app/terminal.tsx facade after terminal imports are migrated`
+   Description: track deletion of the compatibility facade once route/layout/test imports point at final `apps/web/terminal/*` modules
+
+2. `task`, priority `3`
+   Title: `Audit and remove dead terminal route exports no longer used by app redirects`
+   Description: verify whether `SignalsRoute`, `ChartsRoute`, and `ReplayRoute` should be deleted since App Router pages now redirect to `/`
+
+If additional cleanup is discovered during extraction, create linked `bd` tasks with `discovered-from` dependencies rather than expanding this refactor mid-flight.
+
+## Acceptance Criteria
+
+The first extraction is successful when:
+- terminal logic is split into the target `apps/web/terminal/*` structure
+- `apps/web/app/terminal.tsx` remains only as a thin compatibility layer
+- app entrypoints continue to work without behavior changes
+- tests target the new module homes for pure logic
+- build and tests pass
+- follow-up `bd` issues exist for facade removal and dead-export cleanup
+
+## Assumptions And Defaults
+
+- Chosen scope: medium slice, not full architectural rewrite
+- Chosen transition: keep `apps/web/app/terminal.tsx` as a temporary facade
+- Chosen module home: `apps/web/terminal/*`, not `apps/web/app/terminal/*`
+- Default behavior requirement: strict behavioral parity
+- Default testing approach: split existing monolithic helper tests by concern and colocate them under `apps/web/terminal/tests`
+- Default routing approach: keep redirect pages untouched in the first pass

scripts/check-docker-workspace.ts

@@ -0,0 +1,244 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+
+type DependencyMap = Record<string, string>;
+
+type LockWorkspace = {
+  name?: string;
+  dependencies?: DependencyMap;
+  devDependencies?: DependencyMap;
+  optionalDependencies?: DependencyMap;
+  peerDependencies?: DependencyMap;
+};
+
+type BunLock = {
+  lockfileVersion?: number;
+  configVersion?: number;
+  workspaces?: Record<string, LockWorkspace>;
+  packages?: Record<string, unknown>;
+};
+
+type RootPackageManifest = {
+  workspaces?: string[];
+};
+
+const repoRoot = path.resolve(import.meta.dir, "..");
+const deploymentRoot = path.join(repoRoot, "deployment/docker/workspace-root");
+
+const rootPackagePath = path.join(repoRoot, "package.json");
+const deploymentPackagePath = path.join(deploymentRoot, "package.json");
+const rootTsconfigPath = path.join(repoRoot, "tsconfig.base.json");
+const deploymentTsconfigPath = path.join(deploymentRoot, "tsconfig.base.json");
+const rootLockPath = path.join(repoRoot, "bun.lock");
+const deploymentLockPath = path.join(deploymentRoot, "bun.lock");
+
+const readUtf8 = async (filePath: string): Promise<string> => {
+  return readFile(filePath, "utf8");
+};
+
+const parseObjectLiteral = async <T>(filePath: string): Promise<T> => {
+  const raw = await readUtf8(filePath);
+  try {
+    const parsed = Function(`"use strict"; return (${raw});`)() as T;
+    return parsed;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new Error(`Failed to parse ${filePath}: ${message}`);
+  }
+};
+
+const stableSortObject = (value: unknown): unknown => {
+  if (Array.isArray(value)) {
+    return value.map(stableSortObject);
+  }
+  if (value && typeof value === "object") {
+    const entries = Object.entries(value as Record<string, unknown>)
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([key, nested]) => [key, stableSortObject(nested)] as const);
+    return Object.fromEntries(entries);
+  }
+  return value;
+};
+
+const stableStringify = (value: unknown): string => {
+  return JSON.stringify(stableSortObject(value));
+};
+
+const listWorkspacePaths = async (workspacePatterns: string[]): Promise<string[]> => {
+  const paths = new Set<string>();
+
+  for (const pattern of workspacePatterns) {
+    const globPattern = pattern.endsWith("/") ? `${pattern}package.json` : `${pattern}/package.json`;
+    const glob = new Bun.Glob(globPattern);
+    for await (const match of glob.scan({ cwd: repoRoot })) {
+      const normalized = match.replaceAll("\\", "/");
+      paths.add(path.posix.dirname(normalized));
+    }
+  }
+
+  return Array.from(paths).sort((a, b) => a.localeCompare(b));
+};
+
+const normalizedDependencyMap = (input: DependencyMap | undefined): DependencyMap => {
+  if (!input) {
+    return {};
+  }
+  return Object.fromEntries(
+    Object.entries(input)
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([name, version]) => [name, version])
+  );
+};
+
+const formatDependencyDiff = (
+  workspacePath: string,
+  section: string,
+  expected: DependencyMap,
+  actual: DependencyMap
+): string[] => {
+  const issues: string[] = [];
+  const expectedKeys = new Set(Object.keys(expected));
+  const actualKeys = new Set(Object.keys(actual));
+
+  for (const key of expectedKeys) {
+    if (!actualKeys.has(key)) {
+      issues.push(`${workspacePath} ${section}: missing ${key}@${expected[key]}`);
+      continue;
+    }
+    if (expected[key] !== actual[key]) {
+      issues.push(
+        `${workspacePath} ${section}: ${key} expected ${expected[key]} but found ${actual[key]}`
+      );
+    }
+  }
+
+  for (const key of actualKeys) {
+    if (!expectedKeys.has(key)) {
+      issues.push(`${workspacePath} ${section}: extra ${key}@${actual[key]}`);
+    }
+  }
+
+  return issues;
+};
+
+const check = async (): Promise<number> => {
+  const issues: string[] = [];
+
+  const [rootPackage, deploymentPackage, rootTsconfig, deploymentTsconfig, rootLock, deploymentLock] =
+    await Promise.all([
+      parseObjectLiteral<RootPackageManifest>(rootPackagePath),
+      parseObjectLiteral(deploymentPackagePath),
+      parseObjectLiteral(rootTsconfigPath),
+      parseObjectLiteral(deploymentTsconfigPath),
+      parseObjectLiteral<BunLock>(rootLockPath),
+      parseObjectLiteral<BunLock>(deploymentLockPath)
+    ]);
+
+  const rootPackageSnapshot = stableStringify(rootPackage);
+  const deploymentPackageSnapshot = stableStringify(deploymentPackage);
+  if (rootPackageSnapshot !== deploymentPackageSnapshot) {
+    issues.push(
+      "deployment/docker/workspace-root/package.json does not match repo-root package.json"
+    );
+  }
+
+  const rootTsconfigSnapshot = stableStringify(rootTsconfig);
+  const deploymentTsconfigSnapshot = stableStringify(deploymentTsconfig);
+  if (rootTsconfigSnapshot !== deploymentTsconfigSnapshot) {
+    issues.push(
+      "deployment/docker/workspace-root/tsconfig.base.json does not match repo-root tsconfig.base.json"
+    );
+  }
+
+  const rootWorkspaces = rootLock.workspaces ?? {};
+  const deploymentWorkspaces = deploymentLock.workspaces ?? {};
+
+  const workspacePatterns = rootPackage.workspaces ?? [];
+  const workspacePackagePaths = await listWorkspacePaths(workspacePatterns);
+  for (const workspacePath of workspacePackagePaths) {
+    const packageJsonPath = path.join(repoRoot, workspacePath, "package.json");
+    const workspacePackage = (await parseObjectLiteral(packageJsonPath)) as LockWorkspace;
+    const deploymentWorkspace = deploymentWorkspaces[workspacePath];
+
+    if (!deploymentWorkspace) {
+      issues.push(`deployment lock is missing workspace entry: ${workspacePath}`);
+      continue;
+    }
+
+    const sections: Array<keyof LockWorkspace> = [
+      "dependencies",
+      "devDependencies",
+      "optionalDependencies",
+      "peerDependencies"
+    ];
+    for (const section of sections) {
+      const expectedMap = normalizedDependencyMap(workspacePackage[section] as DependencyMap | undefined);
+      const actualMap = normalizedDependencyMap(
+        deploymentWorkspace[section] as DependencyMap | undefined
+      );
+      issues.push(...formatDependencyDiff(workspacePath, section, expectedMap, actualMap));
+    }
+  }
+
+  const workspacePaths = Array.from(
+    new Set([...Object.keys(rootWorkspaces), ...Object.keys(deploymentWorkspaces)])
+  ).sort((a, b) => a.localeCompare(b));
+
+  for (const workspacePath of workspacePaths) {
+    const rootWorkspace = rootWorkspaces[workspacePath];
+    const deploymentWorkspace = deploymentWorkspaces[workspacePath];
+
+    if (!rootWorkspace) {
+      issues.push(`deployment lock has unexpected workspace entry: ${workspacePath}`);
+      continue;
+    }
+    if (!deploymentWorkspace) {
+      issues.push(`deployment lock is missing workspace entry: ${workspacePath}`);
+      continue;
+    }
+
+    if ((rootWorkspace.name ?? "") !== (deploymentWorkspace.name ?? "")) {
+      issues.push(
+        `${workspacePath} name mismatch: expected ${rootWorkspace.name ?? "(none)"} but found ${
+          deploymentWorkspace.name ?? "(none)"
+        }`
+      );
+    }
+
+    const sections: Array<keyof LockWorkspace> = [
+      "dependencies",
+      "devDependencies",
+      "optionalDependencies",
+      "peerDependencies"
+    ];
+    for (const section of sections) {
+      const expectedMap = normalizedDependencyMap(rootWorkspace[section] as DependencyMap | undefined);
+      const actualMap = normalizedDependencyMap(
+        deploymentWorkspace[section] as DependencyMap | undefined
+      );
+      issues.push(...formatDependencyDiff(workspacePath, section, expectedMap, actualMap));
+    }
+  }
+
+  const rootPackagesSnapshot = stableStringify(rootLock.packages ?? {});
+  const deploymentPackagesSnapshot = stableStringify(deploymentLock.packages ?? {});
+  if (rootPackagesSnapshot !== deploymentPackagesSnapshot) {
+    issues.push(
+      "deployment/docker/workspace-root/bun.lock package resolutions differ from repo-root bun.lock"
+    );
+  }
+
+  if (issues.length > 0) {
+    console.error("Docker workspace snapshot is out of sync:");
+    for (const issue of issues) {
+      console.error(`- ${issue}`);
+    }
+    console.error("Run: bun run sync:docker-workspace");
+    return 1;
+  }
+
+  console.log("Docker workspace snapshot is in sync.");
+  return 0;
+};
+
+process.exitCode = await check();

scripts/sync-docker-workspace.ts

@@ -0,0 +1,19 @@
+import { copyFile } from "node:fs/promises";
+import path from "node:path";
+
+const repoRoot = path.resolve(import.meta.dir, "..");
+const deploymentRoot = path.join(repoRoot, "deployment/docker/workspace-root");
+
+const filesToSync = [
+  "package.json",
+  "bun.lock",
+  "tsconfig.base.json"
+] as const;
+
+for (const fileName of filesToSync) {
+  const source = path.join(repoRoot, fileName);
+  const destination = path.join(deploymentRoot, fileName);
+  await copyFile(source, destination);
+  console.log(`synced ${fileName}`);
+}
+

services/api/src/index.ts

@@ -1,5 +1,5 @@
 import { readEnv } from "@islandflow/config";
-import { createLogger } from "@islandflow/observability";
+import { createLogger, createMetrics } from "@islandflow/observability";
 import {
   SUBJECT_ALERTS,
   SUBJECT_CLASSIFIER_HITS,
@@ -23,6 +23,7 @@ import {
   STREAM_SMART_MONEY_EVENTS,
   STREAM_OPTION_NBBO,
   STREAM_OPTION_SIGNAL_PRINTS,
+  buildStreamConfig,
   buildDurableConsumer,
   connectJetStreamWithRetry,
   ensureStream,
@@ -82,7 +83,7 @@ import {
   fetchClassifierHitsByPacketIds,
   fetchRecentOptionPrints
 } from "@islandflow/storage";
-import type { EquityPrintQueryFilters, OptionPrintQueryFilters } from "@islandflow/storage";
+import type { EquityPrintQueryFilters } from "@islandflow/storage";
 import {
   AlertEventSchema,
   ClassifierHitEventSchema,
@@ -99,11 +100,6 @@ import {
   LiveSubscriptionSchema,
   matchesFlowPacketFilters,
   matchesOptionPrintFilters,
-  OptionFlowFilters,
-  OptionFlowViewSchema,
-  OptionNbboSideSchema,
-  OptionSecurityTypeSchema,
-  OptionTypeSchema,
   FlowPacketSchema,
   SmartMoneyEventSchema,
   OptionNBBOSchema,
@@ -112,10 +108,17 @@ import {
 } from "@islandflow/types";
 import { createClient } from "redis";
 import { z } from "zod";
-import { LIVE_FEED_LOOKBACK_MS, LiveStateManager, shouldFanoutLiveEvent } from "./live";
+import {
+  HOT_LIVE_REDIS_KEYS,
+  LiveStateManager,
+  resolveLiveStateConfig,
+  shouldFanoutLiveEvent
+} from "./live";
+import { parseOptionPrintQuery } from "./option-queries";
 
 const service = "api";
 const logger = createLogger({ service });
+const metrics = createMetrics({ service });
 
 const DeliverPolicySchema = z.enum(["new", "all", "last", "last_per_subject"]);
 
@@ -138,13 +141,6 @@ const state = {
   shutdownPromise: null as Promise<void> | null
 };
 
-const HOT_LIVE_REDIS_KEYS = {
-  options: "live:options",
-  equities: "live:equities",
-  flow: "live:flow",
-  nbbo: "live:nbbo"
-} as const;
-
 const getErrorMessage = (error: unknown): string => {
   return error instanceof Error ? error.message : String(error);
 };
@@ -231,33 +227,6 @@ const equityPrintRangeSchema = z.object({
   end_ts: z.coerce.number().int().nonnegative(),
   limit: limitSchema.optional()
 });
-const optionSideListSchema = z
-  .string()
-  .transform((value) =>
-    value
-      .split(",")
-      .map((entry) => entry.trim())
-      .filter(Boolean)
-  )
-  .pipe(z.array(OptionNbboSideSchema));
-const optionTypeListSchema = z
-  .string()
-  .transform((value) =>
-    value
-      .split(",")
-      .map((entry) => entry.trim())
-      .filter(Boolean)
-  )
-  .pipe(z.array(OptionTypeSchema));
-const optionSecuritySchema = z.enum(["stock", "etf", "all"]);
-const optionFilterQuerySchema = z.object({
-  view: OptionFlowViewSchema.optional(),
-  security: optionSecuritySchema.optional(),
-  side: optionSideListSchema.optional(),
-  type: optionTypeListSchema.optional(),
-  min_notional: z.coerce.number().nonnegative().optional()
-});
-
 type Channel =
   | "options"
   | "options-nbbo"
@@ -358,43 +327,6 @@ const applyDeliverPolicy = (
   }
 };
 
-const parseOptionPrintFilters = (
-  url: URL
-): {
-  view: z.infer<typeof OptionFlowViewSchema>;
-  storageFilters: Parameters<typeof fetchRecentOptionPrints>[3];
-  liveFilters: OptionFlowFilters;
-} => {
-  const parsed = optionFilterQuerySchema.parse({
-    view: url.searchParams.get("view") ?? undefined,
-    security: url.searchParams.get("security") ?? undefined,
-    side: url.searchParams.get("side") ?? undefined,
-    type: url.searchParams.get("type") ?? undefined,
-    min_notional: url.searchParams.get("min_notional") ?? undefined
-  });
-  const view = parsed.view ?? "signal";
-  const security = parsed.security ?? (view === "raw" ? "all" : "stock");
-  const storageFilters = {
-    view,
-    security,
-    minNotional: parsed.min_notional,
-    nbboSides: parsed.side,
-    optionTypes: parsed.type
-  } as const;
-  const liveFilters: OptionFlowFilters = {
-    view,
-    securityTypes:
-      security === "all"
-        ? undefined
-        : ([security] as Array<z.infer<typeof OptionSecurityTypeSchema>>),
-    nbboSides: parsed.side,
-    optionTypes: parsed.type,
-    minNotional: parsed.min_notional
-  };
-
-  return { view, storageFilters, liveFilters };
-};
-
 const parseReplayParams = (url: URL): { afterTs: number; afterSeq: number; limit: number } => {
   const params = replayParamsSchema.parse({
     after_ts: url.searchParams.get("after_ts") ?? undefined,
@@ -612,19 +544,8 @@ const parseScopeList = (url: URL, ...keys: string[]): string[] | undefined => {
   return unique.length > 0 ? unique : undefined;
 };
 
-const parseLiveOptionPrintFilters = (url: URL): OptionPrintQueryFilters => {
-  const { storageFilters } = parseOptionPrintFilters(url);
-  return {
-    ...storageFilters,
-    underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids"),
-    optionContractId: url.searchParams.get("option_contract_id") ?? undefined,
-    sinceTs: Date.now() - LIVE_FEED_LOOKBACK_MS
-  };
-};
-
 const parseLiveEquityPrintFilters = (url: URL): EquityPrintQueryFilters => ({
-  underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids"),
+  underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids")
-  sinceTs: Date.now() - LIVE_FEED_LOOKBACK_MS
 });
 
 const matchesScopedOptionSubscription = (
@@ -703,148 +624,17 @@ const run = async () => {
     { attempts: 120, delayMs: 500 }
   );
 
-  await ensureStream(jsm, {
+  await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived"));
-    name: STREAM_OPTION_SIGNAL_PRINTS,
+  await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw"));
-    subjects: [SUBJECT_OPTION_SIGNAL_PRINTS],
+  await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw"));
-    retention: "limits",
+  await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw"));
-    storage: "file",
+  await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_CANDLES, SUBJECT_EQUITY_CANDLES, "derived"));
-    discard: "old",
+  await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_JOINS, SUBJECT_EQUITY_JOINS, "derived"));
-    max_msgs_per_subject: -1,
+  await ensureStream(jsm, buildStreamConfig(STREAM_INFERRED_DARK, SUBJECT_INFERRED_DARK, "derived"));
-    max_msgs: -1,
+  await ensureStream(jsm, buildStreamConfig(STREAM_FLOW_PACKETS, SUBJECT_FLOW_PACKETS, "derived"));
-    max_bytes: -1,
+  await ensureStream(jsm, buildStreamConfig(STREAM_SMART_MONEY_EVENTS, SUBJECT_SMART_MONEY_EVENTS, "derived"));
-    max_age: 0,
+  await ensureStream(jsm, buildStreamConfig(STREAM_CLASSIFIER_HITS, SUBJECT_CLASSIFIER_HITS, "derived"));
-    num_replicas: 1
+  await ensureStream(jsm, buildStreamConfig(STREAM_ALERTS, SUBJECT_ALERTS, "derived"));
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_OPTION_NBBO,
-    subjects: [SUBJECT_OPTION_NBBO],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_EQUITY_PRINTS,
-    subjects: [SUBJECT_EQUITY_PRINTS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_EQUITY_QUOTES,
-    subjects: [SUBJECT_EQUITY_QUOTES],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_EQUITY_CANDLES,
-    subjects: [SUBJECT_EQUITY_CANDLES],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_EQUITY_JOINS,
-    subjects: [SUBJECT_EQUITY_JOINS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_INFERRED_DARK,
-    subjects: [SUBJECT_INFERRED_DARK],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_FLOW_PACKETS,
-    subjects: [SUBJECT_FLOW_PACKETS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_SMART_MONEY_EVENTS,
-    subjects: [SUBJECT_SMART_MONEY_EVENTS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_CLASSIFIER_HITS,
-    subjects: [SUBJECT_CLASSIFIER_HITS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_ALERTS,
-    subjects: [SUBJECT_ALERTS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
 
   const clickhouse = createClickHouseClient({
     url: env.CLICKHOUSE_URL,
@@ -890,7 +680,7 @@ const run = async () => {
     redis = null;
   }
 
-  const liveState = new LiveStateManager(clickhouse, redis);
+  const liveState = new LiveStateManager(clickhouse, redis, resolveLiveStateConfig());
   await liveState.hydrate();
   const warnLiveLag = (
     channel: keyof typeof HOT_LIVE_REDIS_KEYS,
@@ -910,6 +700,7 @@ const run = async () => {
   };
   const liveStateMetricsTimer = setInterval(() => {
     const snapshot = liveState.getStatsSnapshot();
+    const hotFeedHealth = liveState.getHotChannelHealth();
     const hotFeedLagMs = {
       options: snapshot.freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.options] ?? null,
       equities: snapshot.freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.equities] ?? null,
@@ -918,7 +709,12 @@ const run = async () => {
     };
     logger.info("live cache metrics", {
       ...snapshot,
-      hotFeedLagMs
+      hotFeedLagMs,
+      hotFeedHealth,
+      snapshotSourceCounts: {
+        generic_cache_snapshot: snapshot.genericCacheSnapshots,
+        scoped_clickhouse_snapshot: snapshot.scopedClickHouseSnapshots
+      }
     });
     warnLiveLag("options", hotFeedLagMs.options);
     warnLiveLag("equities", hotFeedLagMs.equities);
@@ -1149,6 +945,11 @@ const run = async () => {
       return;
     }
 
+    const optionItem = ingestChannel === "options" ? (item as Parameters<typeof matchesOptionPrintFilters>[0]) : null;
+    const equityItem = ingestChannel === "equities" ? (item as Parameters<typeof matchesScopedEquitySubscription>[0]) : null;
+    const flowItem = ingestChannel === "flow" ? (item as Parameters<typeof matchesFlowPacketFilters>[0]) : null;
+    let matchedSubscriptions = 0;
+
     for (const [key, candidate] of matchingSubscriptions) {
       const sockets = subscriptionSockets.get(key);
       if (!sockets || sockets.size === 0) {
@@ -1157,26 +958,29 @@ const run = async () => {
 
       if (
         candidate.channel === "options" &&
-        (!matchesOptionPrintFilters(OptionPrintSchema.parse(item), candidate.filters) ||
+        (!optionItem ||
-          !matchesScopedOptionSubscription(OptionPrintSchema.parse(item), candidate))
+          !matchesOptionPrintFilters(optionItem, candidate.filters) ||
+          !matchesScopedOptionSubscription(optionItem, candidate))
       ) {
         continue;
       }
 
       if (
         candidate.channel === "equities" &&
-        !matchesScopedEquitySubscription(EquityPrintSchema.parse(item), candidate)
+        (!equityItem || !matchesScopedEquitySubscription(equityItem, candidate))
       ) {
         continue;
       }
 
       if (
         candidate.channel === "flow" &&
-        !matchesFlowPacketFilters(FlowPacketSchema.parse(item), candidate.filters)
+        (!flowItem || !matchesFlowPacketFilters(flowItem, candidate.filters))
       ) {
         continue;
       }
 
+      matchedSubscriptions += 1;
+
       for (const socket of sockets) {
         sendLiveMessage(socket, {
           op: "event",
@@ -1186,6 +990,10 @@ const run = async () => {
         });
       }
     }
+
+    if (matchedSubscriptions > 0) {
+      metrics.count("api.live.subscription_match_count", matchedSubscriptions);
+    }
   };
 
   const pumpOptions = async () => {
@@ -1402,7 +1210,7 @@ const run = async () => {
         try {
           const limit = parseLimit(url.searchParams.get("limit"));
           const source = parseReplaySource(url) ?? undefined;
-          const { storageFilters } = parseOptionPrintFilters(url);
+          const { storageFilters } = parseOptionPrintQuery(url);
           const data = await fetchRecentOptionPrints(clickhouse, limit, source, storageFilters);
           return jsonResponse({ data });
         } catch (error) {
@@ -1528,7 +1336,7 @@ const run = async () => {
         try {
           const { beforeTs, beforeSeq, limit } = parseBeforeParams(url);
           const source = parseReplaySource(url) ?? undefined;
-          const storageFilters = parseLiveOptionPrintFilters(url);
+          const { storageFilters } = parseOptionPrintQuery(url);
           const data = await fetchOptionPrintsBefore(
             clickhouse,
             beforeTs,
@@ -1671,7 +1479,7 @@ const run = async () => {
         try {
           const { afterTs, afterSeq, limit } = parseReplayParams(url);
           const source = parseReplaySource(url) ?? undefined;
-          const { storageFilters } = parseOptionPrintFilters(url);
+          const { storageFilters } = parseOptionPrintQuery(url);
           const data = await fetchOptionPrintsAfter(
             clickhouse,
             afterTs,
@@ -1894,9 +1702,13 @@ const run = async () => {
     websocket: {
       open: (socket: any) => {
         if (socket.data.channel === "live") {
-          sendLiveMessage(socket, { op: "ready" });
+          sendLiveMessage(socket, { op: "ready", channel_health: liveState.getHotChannelHealth() });
           const heartbeat = setInterval(() => {
-            sendLiveMessage(socket, { op: "heartbeat", ts: Date.now() });
+            sendLiveMessage(socket, {
+              op: "heartbeat",
+              ts: Date.now(),
+              channel_health: liveState.getHotChannelHealth()
+            });
           }, 15000);
           liveHeartbeats.set(socket, heartbeat);
         } else if (socket.data.channel === "options") {
@@ -1937,7 +1749,11 @@ const run = async () => {
               : new TextDecoder().decode(message instanceof Uint8Array ? message : new Uint8Array(message));
           const parsed = LiveClientMessageSchema.parse(JSON.parse(payload));
           if (parsed.op === "ping") {
-            sendLiveMessage(socket, { op: "heartbeat", ts: Date.now() });
+            sendLiveMessage(socket, {
+              op: "heartbeat",
+              ts: Date.now(),
+              channel_health: liveState.getHotChannelHealth()
+            });
             return;
           }
 
@@ -2003,6 +1819,7 @@ const run = async () => {
       logger.info("service stopping", { signal });
       server.stop();
       clearInterval(liveStateMetricsTimer);
+      await liveState.close();
 
       if (redis && redis.isOpen) {
         try {

services/api/src/live.ts

@@ -25,7 +25,10 @@ import {
   FeedSnapshot,
   FlowPacketSchema,
   InferredDarkEventSchema,
+  LiveChannelHealth,
   LiveGenericChannel,
+  LiveHotChannel,
+  LiveHotChannelHealthMap,
   LiveSubscription,
   matchesFlowPacketFilters,
   matchesOptionPrintFilters,
@@ -38,12 +41,15 @@ import {
   type EquityPrint,
   type LiveChannel
 } from "@islandflow/types";
+import { createMetrics } from "@islandflow/observability";
 import type { RedisClientType } from "redis";
 
 const CURSOR_HASH_KEY = "live:cursors";
 export const LIVE_FEED_LOOKBACK_MS = 24 * 60 * 60 * 1000;
 
-const DEFAULT_GENERIC_LIMIT = 10000;
+const metrics = createMetrics({ service: "api" });
+
+const DEFAULT_GENERIC_LIMIT = 1000;
 const MAX_GENERIC_LIMIT = 100000;
 const MIN_GENERIC_LIMIT = 1;
 const GENERIC_LIMIT_ENV_KEYS: Record<LiveGenericChannel, string> = {
@@ -64,6 +70,23 @@ const CHART_LIMITS = {
   overlay: 1500
 } as const;
 
+const DEFAULT_LIVE_LIMITS: GenericLiveLimits = {
+  options: 1000,
+  nbbo: 1000,
+  equities: 1000,
+  "equity-quotes": 500,
+  "equity-joins": 500,
+  flow: 500,
+  "smart-money": 300,
+  "classifier-hits": 300,
+  alerts: 300,
+  "inferred-dark": 300
+};
+
+const DEFAULT_SCOPED_CACHE_MAX_KEYS = 32;
+const DEFAULT_REDIS_FLUSH_INTERVAL_MS = 250;
+const DEFAULT_REDIS_FLUSH_MAX_ITEMS = 100;
+
 type GenericFeedConfig = {
   redisKey: string;
   cursorField: string;
@@ -81,8 +104,22 @@ export const LIVE_FRESHNESS_THRESHOLDS: Partial<Record<LiveGenericChannel, numbe
   flow: 30_000
 };
 
+export const HOT_LIVE_REDIS_KEYS = {
+  options: "live:options",
+  equities: "live:equities",
+  flow: "live:flow",
+  nbbo: "live:nbbo"
+} as const satisfies Record<LiveHotChannel, string>;
+
 export type GenericLiveLimits = Record<LiveGenericChannel, number>;
 
+type LiveStateConfig = {
+  limits: GenericLiveLimits;
+  scopedCacheMaxKeys: number;
+  redisFlushIntervalMs: number;
+  redisFlushMaxItems: number;
+};
+
 const parseGenericLimit = (
   env: NodeJS.ProcessEnv,
   channel: LiveGenericChannel,
@@ -107,17 +144,77 @@ const parseGenericLimit = (
   return bounded;
 };
 
-export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env): GenericLiveLimits => ({
+const parseGenericLimitFallback = (env: NodeJS.ProcessEnv, fallback: number): number => {
-  options: parseGenericLimit(env, "options", DEFAULT_GENERIC_LIMIT),
+  const raw = env.LIVE_LIMIT_DEFAULT;
-  nbbo: parseGenericLimit(env, "nbbo", DEFAULT_GENERIC_LIMIT),
+  if (!raw || raw.trim().length === 0) {
-  equities: parseGenericLimit(env, "equities", DEFAULT_GENERIC_LIMIT),
+    return fallback;
-  "equity-quotes": parseGenericLimit(env, "equity-quotes", DEFAULT_GENERIC_LIMIT),
+  }
-  "equity-joins": parseGenericLimit(env, "equity-joins", DEFAULT_GENERIC_LIMIT),
+
-  flow: parseGenericLimit(env, "flow", DEFAULT_GENERIC_LIMIT),
+  const parsed = Number(raw);
-  "smart-money": parseGenericLimit(env, "smart-money", DEFAULT_GENERIC_LIMIT),
+  if (!Number.isFinite(parsed)) {
-  "classifier-hits": parseGenericLimit(env, "classifier-hits", DEFAULT_GENERIC_LIMIT),
+    console.warn(`Invalid LIVE_LIMIT_DEFAULT="${raw}", using ${fallback}`);
-  alerts: parseGenericLimit(env, "alerts", DEFAULT_GENERIC_LIMIT),
+    return fallback;
-  "inferred-dark": parseGenericLimit(env, "inferred-dark", DEFAULT_GENERIC_LIMIT)
+  }
+
+  return Math.max(MIN_GENERIC_LIMIT, Math.min(MAX_GENERIC_LIMIT, Math.floor(parsed)));
+};
+
+export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env): GenericLiveLimits => {
+  const liveLimitDefault = parseGenericLimitFallback(env, DEFAULT_GENERIC_LIMIT);
+  return {
+    options: parseGenericLimit(env, "options", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.options),
+    nbbo: parseGenericLimit(env, "nbbo", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.nbbo),
+    equities: parseGenericLimit(
+      env,
+      "equities",
+      env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.equities
+    ),
+    "equity-quotes": parseGenericLimit(
+      env,
+      "equity-quotes",
+      env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["equity-quotes"]
+    ),
+    "equity-joins": parseGenericLimit(
+      env,
+      "equity-joins",
+      env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["equity-joins"]
+    ),
+    flow: parseGenericLimit(env, "flow", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.flow),
+    "smart-money": parseGenericLimit(
+      env,
+      "smart-money",
+      env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["smart-money"]
+    ),
+    "classifier-hits": parseGenericLimit(
+      env,
+      "classifier-hits",
+      env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["classifier-hits"]
+    ),
+    alerts: parseGenericLimit(env, "alerts", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.alerts),
+    "inferred-dark": parseGenericLimit(
+      env,
+      "inferred-dark",
+      env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["inferred-dark"]
+    )
+  };
+};
+
+const parsePositiveInt = (value: string | undefined, fallback: number): number => {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) {
+    return fallback;
+  }
+  return Math.max(1, Math.floor(parsed));
+};
+
+export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): LiveStateConfig => ({
+  limits: resolveGenericLiveLimits(env),
+  scopedCacheMaxKeys: parsePositiveInt(env.LIVE_SCOPED_CACHE_MAX_KEYS, DEFAULT_SCOPED_CACHE_MAX_KEYS),
+  redisFlushIntervalMs: parsePositiveInt(
+    env.LIVE_REDIS_FLUSH_INTERVAL_MS,
+    DEFAULT_REDIS_FLUSH_INTERVAL_MS
+  ),
+  redisFlushMaxItems: parsePositiveInt(env.LIVE_REDIS_FLUSH_MAX_ITEMS, DEFAULT_REDIS_FLUSH_MAX_ITEMS)
 });
 
 type RedisLike = Pick<
@@ -335,6 +432,30 @@ const snapshotLimitFor = (subscription: LiveSubscription, configuredLimit: numbe
   return Math.max(1, Math.min(configuredLimit, Math.floor(requested)));
 };
 
+export const buildOptionSnapshotFilters = (
+  subscription: Extract<LiveSubscription, { channel: "options" }>
+): OptionPrintQueryFilters => {
+  if (subscription.option_contract_id) {
+    return {
+      view: "raw",
+      optionContractId: subscription.option_contract_id
+    };
+  }
+
+  return {
+    view: subscription.filters?.view ?? "signal",
+    security:
+      subscription.filters?.securityTypes?.length === 1
+        ? subscription.filters.securityTypes[0]
+        : "all",
+    nbboSides: subscription.filters?.nbboSides,
+    optionTypes: subscription.filters?.optionTypes,
+    minNotional: subscription.filters?.minNotional,
+    underlyingIds: subscription.underlying_ids,
+    optionContractId: subscription.option_contract_id
+  };
+};
+
 const candleRedisKey = (underlyingId: string, intervalMs: number): string =>
   `live:equity-candles:${underlyingId}:${intervalMs}`;
 
@@ -344,7 +465,50 @@ const candleCursorField = (underlyingId: string, intervalMs: number): string =>
 const overlayRedisKey = (underlyingId: string): string => `live:equity-overlay:${underlyingId}`;
 const overlayCursorField = (underlyingId: string): string => `equities:${underlyingId}`;
 
+const dropMatchingCursor = <T>(
+  items: T[],
+  target: Cursor,
+  cursorOf: (item: T) => Cursor
+): T[] => items.filter((item) => compareCursors(cursorOf(item), target) !== 0);
+
+const insertNewestFirst = <T>(
+  items: T[],
+  item: T,
+  cursorOf: (item: T) => Cursor,
+  limit: number
+): { items: T[]; outOfOrder: boolean } => {
+  const cursor = cursorOf(item);
+  const deduped = dropMatchingCursor(items, cursor, cursorOf);
+  const head = deduped[0];
+  const outOfOrder = head ? compareCursors(cursor, cursorOf(head)) > 0 : false;
+
+  if (!outOfOrder) {
+    return {
+      items: [item, ...deduped].slice(0, limit),
+      outOfOrder: false
+    };
+  }
+
+  return {
+    items: sortGenericItems([...deduped, item], cursorOf).slice(0, limit),
+    outOfOrder: true
+  };
+};
+
+type BufferedRedisWrite = {
+  listKey: string;
+  cursorField: string;
+  items: unknown[];
+  limit: number;
+  cursor: Cursor | null;
+  updates: number;
+};
+
+const isLiveStateConfig = (value: GenericLiveLimits | LiveStateConfig): value is LiveStateConfig =>
+  "limits" in value;
+
 export class LiveStateManager {
+  private readonly config: LiveStateConfig;
   private readonly generic: {
     [K in LiveGenericChannel]: GenericFeedConfig;
   };
@@ -352,12 +516,22 @@ export class LiveStateManager {
   private readonly genericCursors = new Map<string, Cursor | null>();
   private readonly candleItems = new Map<string, EquityCandle[]>();
   private readonly candleCursors = new Map<string, Cursor | null>();
+  private readonly candleAccess = new Map<string, number>();
   private readonly overlayItems = new Map<string, EquityPrint[]>();
   private readonly overlayCursors = new Map<string, Cursor | null>();
+  private readonly overlayAccess = new Map<string, number>();
+  private readonly pendingRedisWrites = new Map<string, BufferedRedisWrite>();
+  private readonly redisFlushTimer: ReturnType<typeof setInterval> | null;
   private readonly stats = {
     genericHydrateFromRedis: 0,
     genericHydrateFromClickHouse: 0,
+    genericCacheSnapshots: 0,
+    scopedClickHouseSnapshots: 0,
     trimOperations: 0,
+    redisFlushCount: 0,
+    redisFlushItems: 0,
+    cacheEvictions: 0,
+    outOfOrderEvents: 0,
     cacheDepthByKey: new Map<string, number>(),
     freshnessAgeMsByKey: new Map<string, number>()
   };
@@ -365,27 +539,129 @@ export class LiveStateManager {
   constructor(
     private readonly clickhouse: ClickHouseClient,
     private readonly redis: RedisLike | null,
-    limits: GenericLiveLimits = resolveGenericLiveLimits()
+    config: GenericLiveLimits | LiveStateConfig = resolveLiveStateConfig()
   ) {
-    this.generic = getGenericConfig(limits);
+    this.config = isLiveStateConfig(config)
+      ? config
+      : {
+          limits: config,
+          scopedCacheMaxKeys: DEFAULT_SCOPED_CACHE_MAX_KEYS,
+          redisFlushIntervalMs: DEFAULT_REDIS_FLUSH_INTERVAL_MS,
+          redisFlushMaxItems: DEFAULT_REDIS_FLUSH_MAX_ITEMS
+        };
+    this.generic = getGenericConfig(this.config.limits);
+    this.redisFlushTimer =
+      this.redis && this.redis.isOpen
+        ? setInterval(() => {
+            void this.flushRedisWrites();
+          }, this.config.redisFlushIntervalMs)
+        : null;
+    this.redisFlushTimer?.unref?.();
+  }
+
+  async close(): Promise<void> {
+    if (this.redisFlushTimer) {
+      clearInterval(this.redisFlushTimer);
+    }
+    await this.flushRedisWrites();
   }
 
   getStatsSnapshot(): {
     genericHydrateFromRedis: number;
     genericHydrateFromClickHouse: number;
+    genericCacheSnapshots: number;
+    scopedClickHouseSnapshots: number;
     trimOperations: number;
+    redisFlushCount: number;
+    redisFlushItems: number;
+    cacheEvictions: number;
+    outOfOrderEvents: number;
     cacheDepthByKey: Record<string, number>;
     freshnessAgeMsByKey: Record<string, number>;
   } {
     return {
       genericHydrateFromRedis: this.stats.genericHydrateFromRedis,
       genericHydrateFromClickHouse: this.stats.genericHydrateFromClickHouse,
+      genericCacheSnapshots: this.stats.genericCacheSnapshots,
+      scopedClickHouseSnapshots: this.stats.scopedClickHouseSnapshots,
       trimOperations: this.stats.trimOperations,
+      redisFlushCount: this.stats.redisFlushCount,
+      redisFlushItems: this.stats.redisFlushItems,
+      cacheEvictions: this.stats.cacheEvictions,
+      outOfOrderEvents: this.stats.outOfOrderEvents,
       cacheDepthByKey: Object.fromEntries(this.stats.cacheDepthByKey),
       freshnessAgeMsByKey: Object.fromEntries(this.stats.freshnessAgeMsByKey)
     };
   }
 
+  getHotChannelHealth(): LiveHotChannelHealthMap {
+    return {
+      options: this.getChannelHealth("options"),
+      nbbo: this.getChannelHealth("nbbo"),
+      equities: this.getChannelHealth("equities"),
+      flow: this.getChannelHealth("flow")
+    };
+  }
+
+  async flushRedisWrites(): Promise<void> {
+    if (!this.redis?.isOpen) {
+      return;
+    }
+
+    const writes = Array.from(this.pendingRedisWrites.values());
+    this.pendingRedisWrites.clear();
+
+    for (const write of writes) {
+      await this.persistList(write.listKey, write.cursorField, write.items, write.limit, write.cursor);
+      this.stats.redisFlushCount += 1;
+      this.stats.redisFlushItems += write.items.length;
+      metrics.count("api.live.redis_flush_count", 1);
+      metrics.count("api.live.redis_flush_items", write.items.length);
+    }
+  }
+
+  private getChannelHealth(channel: LiveHotChannel): LiveChannelHealth {
+    const listKey = HOT_LIVE_REDIS_KEYS[channel];
+    const thresholdMs = LIVE_FRESHNESS_THRESHOLDS[channel];
+    const freshnessAgeMs = this.stats.freshnessAgeMsByKey.get(listKey) ?? null;
+    return {
+      freshness_age_ms: freshnessAgeMs,
+      healthy:
+        freshnessAgeMs !== null &&
+        typeof thresholdMs === "number" &&
+        Number.isFinite(freshnessAgeMs) &&
+        freshnessAgeMs <= thresholdMs
+    };
+  }
+
+  private touchAccess(accessMap: Map<string, number>, key: string): void {
+    accessMap.set(key, Date.now());
+  }
+
+  private evictScopedCachesIfNeeded(
+    itemsMap: Map<string, unknown[]>,
+    cursorsMap: Map<string, Cursor | null>,
+    accessMap: Map<string, number>
+  ): void {
+    while (itemsMap.size > this.config.scopedCacheMaxKeys) {
+      const oldest = [...accessMap.entries()].sort((a, b) => a[1] - b[1])[0];
+      if (!oldest) {
+        break;
+      }
+      const [key] = oldest;
+      itemsMap.delete(key);
+      cursorsMap.delete(
+        key.startsWith("live:equity-candles:")
+          ? key.replace("live:", "")
+          : key.replace("live:equity-overlay:", "equities:")
+      );
+      accessMap.delete(key);
+      this.stats.cacheDepthByKey.delete(key);
+      this.stats.cacheEvictions += 1;
+      metrics.count("api.live.cache_evictions", 1);
+    }
+  }
+
   private updateFreshnessMetric(listKey: string, channel: LiveChannel, item: unknown, now = Date.now()): void {
     const ts =
       channel === "equity-candles" || channel === "equity-overlay"
@@ -399,6 +675,32 @@ export class LiveStateManager {
     }
   }
 
+  private queueRedisWrite(
+    listKey: string,
+    cursorField: string,
+    items: unknown[],
+    limit: number,
+    cursor: Cursor | null
+  ): void {
+    if (!this.redis?.isOpen) {
+      return;
+    }
+
+    const existing = this.pendingRedisWrites.get(listKey);
+    const write: BufferedRedisWrite = {
+      listKey,
+      cursorField,
+      items: [...items],
+      limit,
+      cursor,
+      updates: (existing?.updates ?? 0) + 1
+    };
+    this.pendingRedisWrites.set(listKey, write);
+    if (write.updates >= this.config.redisFlushMaxItems) {
+      void this.flushRedisWrites();
+    }
+  }
+
   async hydrate(): Promise<void> {
     const channels = Object.keys(this.generic) as LiveGenericChannel[];
     await Promise.all(channels.map((channel) => this.hydrateGeneric(channel)));
@@ -408,37 +710,22 @@ export class LiveStateManager {
     const config = this.generic[channel];
     if (this.redis?.isOpen) {
       const payloads = await this.redis.lRange(config.redisKey, 0, config.limit - 1);
-      const cached = normalizeGenericItems(
+      const cached = normalizeGenericItems(channel, parseJsonList(payloads, config.parse), config);
-        channel,
-        parseJsonList(payloads, config.parse).filter((item) =>
-          isWithinLiveFeedLookback(channel, item)
-        ),
-        config
-      );
       if (cached.length > 0) {
         this.genericItems.set(channel, cached);
         this.stats.genericHydrateFromRedis += 1;
         this.stats.cacheDepthByKey.set(config.redisKey, cached.length);
         this.updateFreshnessMetric(config.redisKey, channel, cached[0]);
-        this.genericCursors.set(config.cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, config.cursorField)));
+        this.genericCursors.set(
-        await this.persistList(
-          config.redisKey,
           config.cursorField,
-          cached,
+          parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, config.cursorField))
-          config.limit,
-          this.genericCursors.get(config.cursorField) ?? null
         );
+        await this.persistList(config.redisKey, config.cursorField, cached, config.limit, this.genericCursors.get(config.cursorField) ?? null);
         return;
       }
     }
 
-    const fresh = normalizeGenericItems(
+    const fresh = normalizeGenericItems(channel, await config.fetchRecent(this.clickhouse, config.limit), config);
-      channel,
-      (await config.fetchRecent(this.clickhouse, config.limit)).filter((item) =>
-        isWithinLiveFeedLookback(channel, item)
-      ),
-      config
-    );
     this.stats.genericHydrateFromClickHouse += 1;
     this.stats.cacheDepthByKey.set(config.redisKey, fresh.length);
     this.genericItems.set(channel, fresh);
@@ -453,43 +740,26 @@ export class LiveStateManager {
   async getSnapshot(subscription: LiveSubscription): Promise<FeedSnapshot<unknown>> {
     switch (subscription.channel) {
       case "options": {
-        const scoped =
+        const scoped = Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id);
-          Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id);
         if (subscription.filters?.view === "raw" || scoped) {
+          this.stats.scopedClickHouseSnapshots += 1;
           const limit = snapshotLimitFor(subscription, this.generic.options.limit);
-          const storageFilters: OptionPrintQueryFilters = {
+          const storageFilters = buildOptionSnapshotFilters(subscription);
-            view: subscription.filters?.view ?? "signal",
+          const items = await fetchRecentOptionPrints(this.clickhouse, limit, undefined, storageFilters);
-            security:
-              subscription.filters?.securityTypes?.length === 1
-                ? subscription.filters.securityTypes[0]
-                : "all",
-            nbboSides: subscription.filters?.nbboSides,
-            optionTypes: subscription.filters?.optionTypes,
-            minNotional: subscription.filters?.minNotional,
-            underlyingIds: subscription.underlying_ids,
-            optionContractId: subscription.option_contract_id,
-            sinceTs: Date.now() - LIVE_FEED_LOOKBACK_MS
-          };
-          const items = await fetchRecentOptionPrints(
-            this.clickhouse,
-            limit,
-            undefined,
-            storageFilters
-          );
           return {
             subscription,
             items,
             watermark: items[0] ? { ts: items[0].ts, seq: items[0].seq } : null,
-            next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq }))
+            next_before: nextBeforeForItems(items, (entry) => ({ ts: entry.ts, seq: entry.seq }))
           };
         }
 
         const config = this.generic.options;
+        this.stats.genericCacheSnapshots += 1;
         const limit = snapshotLimitFor(subscription, config.limit);
-        const items = (this.genericItems.get("options") ?? []).filter((item) =>
+        const items = (this.genericItems.get("options") ?? [])
-          isWithinLiveFeedLookback("options", item) &&
+          .filter((entry) => matchesOptionPrintFilters(entry, subscription.filters))
-          matchesOptionPrintFilters(item, subscription.filters)
+          .slice(0, limit);
-        ).slice(0, limit);
         return {
           subscription,
           items,
@@ -499,11 +769,11 @@ export class LiveStateManager {
       }
       case "flow": {
         const config = this.generic.flow;
+        this.stats.genericCacheSnapshots += 1;
         const limit = snapshotLimitFor(subscription, config.limit);
-        const items = (this.genericItems.get("flow") ?? []).filter((item) =>
+        const items = (this.genericItems.get("flow") ?? [])
-          isWithinLiveFeedLookback("flow", item) &&
+          .filter((entry) => matchesFlowPacketFilters(entry, subscription.filters))
-          matchesFlowPacketFilters(item, subscription.filters)
+          .slice(0, limit);
-        ).slice(0, limit);
         return {
           subscription,
           items,
@@ -515,10 +785,8 @@ export class LiveStateManager {
         const config = this.generic.equities;
         const limit = snapshotLimitFor(subscription, config.limit);
         if (subscription.underlying_ids?.length) {
-          const filters: EquityPrintQueryFilters = {
+          this.stats.scopedClickHouseSnapshots += 1;
-            underlyingIds: subscription.underlying_ids,
+          const filters: EquityPrintQueryFilters = { underlyingIds: subscription.underlying_ids };
-            sinceTs: Date.now() - LIVE_FEED_LOOKBACK_MS
-          };
           const items = await fetchRecentEquityPrints(this.clickhouse, limit, filters);
           return {
             subscription,
@@ -527,9 +795,8 @@ export class LiveStateManager {
             next_before: nextBeforeForItems(items, config.cursor)
           };
         }
-        const items = (this.genericItems.get("equities") ?? []).filter((item) =>
+        this.stats.genericCacheSnapshots += 1;
-          isWithinLiveFeedLookback("equities", item)
+        const items = (this.genericItems.get("equities") ?? []).slice(0, limit);
-        ).slice(0, limit);
         return {
           subscription,
           items,
@@ -543,12 +810,13 @@ export class LiveStateManager {
         if (!this.candleItems.has(key)) {
           await this.hydrateCandles(subscription.underlying_id, subscription.interval_ms);
         }
+        this.touchAccess(this.candleAccess, key);
         const items = this.candleItems.get(key) ?? [];
         return {
           subscription,
           items,
           watermark: this.candleCursors.get(cursorField) ?? null,
-          next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq }))
+          next_before: nextBeforeForItems(items, (entry) => ({ ts: entry.ts, seq: entry.seq }))
         };
       }
       case "equity-overlay": {
@@ -557,20 +825,20 @@ export class LiveStateManager {
         if (!this.overlayItems.has(key)) {
           await this.hydrateOverlay(subscription.underlying_id);
         }
+        this.touchAccess(this.overlayAccess, key);
         const items = this.overlayItems.get(key) ?? [];
         return {
           subscription,
           items,
           watermark: this.overlayCursors.get(cursorField) ?? null,
-          next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq }))
+          next_before: nextBeforeForItems(items, (entry) => ({ ts: entry.ts, seq: entry.seq }))
         };
       }
       default: {
         const config = this.generic[subscription.channel];
+        this.stats.genericCacheSnapshots += 1;
         const limit = snapshotLimitFor(subscription, config.limit);
-        const items = (this.genericItems.get(subscription.channel) ?? []).filter((item) =>
+        const items = (this.genericItems.get(subscription.channel) ?? []).slice(0, limit);
-          isWithinLiveFeedLookback(subscription.channel, item)
-        ).slice(0, limit);
         return {
           subscription,
           items,
@@ -587,48 +855,52 @@ export class LiveStateManager {
         const candle = EquityCandleSchema.parse(item);
         const key = candleRedisKey(candle.underlying_id, candle.interval_ms);
         const cursorField = candleCursorField(candle.underlying_id, candle.interval_ms);
-        const previousCursor = this.candleCursors.get(cursorField) ?? null;
+        const nextState = insertNewestFirst(
-        const items = this.candleItems.get(key) ?? [];
+          this.candleItems.get(key) ?? [],
-        const next = [candle, ...items]
+          candle,
-          .sort((a, b) => (b.ts - a.ts) || (b.seq - a.seq))
+          (entry) => ({ ts: entry.ts, seq: entry.seq }),
-          .slice(0, CHART_LIMITS.candles);
+          CHART_LIMITS.candles
-        this.candleItems.set(key, next);
+        );
-        this.stats.cacheDepthByKey.set(key, next.length);
         const cursor = { ts: candle.ts, seq: candle.seq };
+        this.candleItems.set(key, nextState.items);
         this.candleCursors.set(cursorField, cursor);
-        if (next.length > 0) {
+        this.touchAccess(this.candleAccess, key);
-          this.updateFreshnessMetric(key, "equity-candles", next[0]);
+        this.evictScopedCachesIfNeeded(this.candleItems as Map<string, unknown[]>, this.candleCursors, this.candleAccess);
+        if (nextState.outOfOrder) {
+          this.stats.outOfOrderEvents += 1;
+          metrics.count("api.live.out_of_order_events", 1);
         }
-        const outOfOrder = previousCursor ? compareCursors(cursor, previousCursor) > 0 : false;
+        this.stats.cacheDepthByKey.set(key, nextState.items.length);
-        if (outOfOrder) {
+        if (nextState.items.length > 0) {
-          await this.persistList(key, cursorField, next, CHART_LIMITS.candles, cursor);
+          this.updateFreshnessMetric(key, "equity-candles", nextState.items[0]);
-        } else {
-          await this.persistItem(key, cursorField, candle, CHART_LIMITS.candles, cursor, next.length);
         }
+        this.queueRedisWrite(key, cursorField, nextState.items, CHART_LIMITS.candles, cursor);
         return cursor;
       }
       case "equity-overlay": {
         const print = EquityPrintSchema.parse(item);
         const key = overlayRedisKey(print.underlying_id);
         const cursorField = overlayCursorField(print.underlying_id);
-        const previousCursor = this.overlayCursors.get(cursorField) ?? null;
+        const nextState = insertNewestFirst(
-        const items = this.overlayItems.get(key) ?? [];
+          this.overlayItems.get(key) ?? [],
-        const next = [print, ...items]
+          print,
-          .sort((a, b) => (b.ts - a.ts) || (b.seq - a.seq))
+          (entry) => ({ ts: entry.ts, seq: entry.seq }),
-          .slice(0, CHART_LIMITS.overlay);
+          CHART_LIMITS.overlay
-        this.overlayItems.set(key, next);
+        );
-        this.stats.cacheDepthByKey.set(key, next.length);
         const cursor = { ts: print.ts, seq: print.seq };
+        this.overlayItems.set(key, nextState.items);
         this.overlayCursors.set(cursorField, cursor);
-        if (next.length > 0) {
+        this.touchAccess(this.overlayAccess, key);
-          this.updateFreshnessMetric(key, "equity-overlay", next[0]);
+        this.evictScopedCachesIfNeeded(this.overlayItems as Map<string, unknown[]>, this.overlayCursors, this.overlayAccess);
+        if (nextState.outOfOrder) {
+          this.stats.outOfOrderEvents += 1;
+          metrics.count("api.live.out_of_order_events", 1);
         }
-        const outOfOrder = previousCursor ? compareCursors(cursor, previousCursor) > 0 : false;
+        this.stats.cacheDepthByKey.set(key, nextState.items.length);
-        if (outOfOrder) {
+        if (nextState.items.length > 0) {
-          await this.persistList(key, cursorField, next, CHART_LIMITS.overlay, cursor);
+          this.updateFreshnessMetric(key, "equity-overlay", nextState.items[0]);
-        } else {
-          await this.persistItem(key, cursorField, print, CHART_LIMITS.overlay, cursor, next.length);
         }
+        this.queueRedisWrite(key, cursorField, nextState.items, CHART_LIMITS.overlay, cursor);
         return cursor;
       }
       default: {
@@ -637,22 +909,28 @@ export class LiveStateManager {
         if (!isWithinLiveFeedLookback(channel, parsed)) {
           return null;
         }
-        const previousCursor = this.genericCursors.get(config.cursorField) ?? null;
+
-        const items = this.genericItems.get(channel) ?? [];
-        const next = normalizeGenericItems(channel, [parsed, ...items], config);
-        this.genericItems.set(channel, next);
-        this.stats.cacheDepthByKey.set(config.redisKey, next.length);
         const cursor = config.cursor(parsed);
+        const nextState =
+          channel === "nbbo"
+            ? {
+                items: normalizeGenericItems(channel, [parsed, ...(this.genericItems.get(channel) ?? [])], config),
+                outOfOrder: false
+              }
+            : insertNewestFirst(this.genericItems.get(channel) ?? [], parsed, config.cursor, config.limit);
+
+        if (nextState.outOfOrder) {
+          this.stats.outOfOrderEvents += 1;
+          metrics.count("api.live.out_of_order_events", 1);
+        }
+
+        this.genericItems.set(channel, nextState.items);
         this.genericCursors.set(config.cursorField, cursor);
-        if (next.length > 0) {
+        this.stats.cacheDepthByKey.set(config.redisKey, nextState.items.length);
-          this.updateFreshnessMetric(config.redisKey, channel, next[0]);
+        if (nextState.items.length > 0) {
-        }
+          this.updateFreshnessMetric(config.redisKey, channel, nextState.items[0]);
-        const outOfOrder = previousCursor ? compareCursors(cursor, previousCursor) > 0 : false;
-        if (channel === "nbbo" || outOfOrder) {
-          await this.persistList(config.redisKey, config.cursorField, next, config.limit, cursor);
-        } else {
-          await this.persistItem(config.redisKey, config.cursorField, parsed, config.limit, cursor, next.length);
         }
+        this.queueRedisWrite(config.redisKey, config.cursorField, nextState.items, config.limit, cursor);
         return cursor;
       }
     }
@@ -666,6 +944,8 @@ export class LiveStateManager {
       const cached = parseJsonList(payloads, (value) => EquityCandleSchema.parse(value));
       if (cached.length > 0) {
         this.candleItems.set(key, cached);
+        this.touchAccess(this.candleAccess, key);
+        this.evictScopedCachesIfNeeded(this.candleItems as Map<string, unknown[]>, this.candleCursors, this.candleAccess);
         this.stats.cacheDepthByKey.set(key, cached.length);
         this.updateFreshnessMetric(key, "equity-candles", cached[0]);
         this.candleCursors.set(cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField)));
@@ -675,6 +955,8 @@ export class LiveStateManager {
 
     const fresh = await fetchRecentEquityCandles(this.clickhouse, underlyingId, intervalMs, CHART_LIMITS.candles);
     this.candleItems.set(key, fresh);
+    this.touchAccess(this.candleAccess, key);
+    this.evictScopedCachesIfNeeded(this.candleItems as Map<string, unknown[]>, this.candleCursors, this.candleAccess);
     this.stats.cacheDepthByKey.set(key, fresh.length);
     if (fresh.length > 0) {
       this.updateFreshnessMetric(key, "equity-candles", fresh[0]);
@@ -692,6 +974,8 @@ export class LiveStateManager {
       const cached = parseJsonList(payloads, (value) => EquityPrintSchema.parse(value));
       if (cached.length > 0) {
         this.overlayItems.set(key, cached);
+        this.touchAccess(this.overlayAccess, key);
+        this.evictScopedCachesIfNeeded(this.overlayItems as Map<string, unknown[]>, this.overlayCursors, this.overlayAccess);
         this.stats.cacheDepthByKey.set(key, cached.length);
         this.updateFreshnessMetric(key, "equity-overlay", cached[0]);
         this.overlayCursors.set(cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField)));
@@ -700,9 +984,11 @@ export class LiveStateManager {
     }
 
     const fresh = (await fetchRecentEquityPrints(this.clickhouse, CHART_LIMITS.overlay)).filter(
-      (item) => item.underlying_id === underlyingId
+      (entry) => entry.underlying_id === underlyingId
     );
     this.overlayItems.set(key, fresh);
+    this.touchAccess(this.overlayAccess, key);
+    this.evictScopedCachesIfNeeded(this.overlayItems as Map<string, unknown[]>, this.overlayCursors, this.overlayAccess);
     this.stats.cacheDepthByKey.set(key, fresh.length);
     if (fresh.length > 0) {
       this.updateFreshnessMetric(key, "equity-overlay", fresh[0]);
@@ -712,25 +998,6 @@ export class LiveStateManager {
     await this.persistList(key, cursorField, fresh, CHART_LIMITS.overlay, watermark);
   }
 
-  private async persistItem<T>(
-    listKey: string,
-    cursorField: string,
-    item: T,
-    limit: number,
-    cursor: Cursor | null,
-    depth: number
-  ): Promise<void> {
-    if (!this.redis?.isOpen) {
-      return;
-    }
-
-    await this.redis.lPush(listKey, JSON.stringify(item));
-    await this.redis.lTrim(listKey, 0, limit - 1);
-    this.stats.trimOperations += 1;
-    this.stats.cacheDepthByKey.set(listKey, Math.min(depth, limit));
-    await this.redis.hSet(CURSOR_HASH_KEY, cursorField, JSON.stringify(cursor));
-  }
-
   private async persistList<T>(
     listKey: string,
     cursorField: string,
@@ -742,7 +1009,7 @@ export class LiveStateManager {
       return;
     }
 
-    const payloads = items.map((item) => JSON.stringify(item));
+    const payloads = items.map((entry) => JSON.stringify(entry));
     await this.redis.lTrim(listKey, 1, 0);
     this.stats.trimOperations += 1;
     if (payloads.length > 0) {

services/api/src/option-queries.ts

@@ -0,0 +1,107 @@
+import type { OptionPrintQueryFilters } from "@islandflow/storage";
+import {
+  OptionFlowViewSchema,
+  OptionNbboSideSchema,
+  OptionSecurityTypeSchema,
+  OptionTypeSchema,
+  type OptionFlowFilters
+} from "@islandflow/types";
+import { z } from "zod";
+
+const optionSideListSchema = z
+  .string()
+  .transform((value) =>
+    value
+      .split(",")
+      .map((entry) => entry.trim())
+      .filter(Boolean)
+  )
+  .pipe(z.array(OptionNbboSideSchema));
+
+const optionTypeListSchema = z
+  .string()
+  .transform((value) =>
+    value
+      .split(",")
+      .map((entry) => entry.trim())
+      .filter(Boolean)
+  )
+  .pipe(z.array(OptionTypeSchema));
+
+const optionSecuritySchema = z.enum(["stock", "etf", "all"]);
+
+const optionFilterQuerySchema = z.object({
+  view: OptionFlowViewSchema.optional(),
+  security: optionSecuritySchema.optional(),
+  side: optionSideListSchema.optional(),
+  type: optionTypeListSchema.optional(),
+  min_notional: z.coerce.number().nonnegative().optional()
+});
+
+export type ParsedOptionPrintQuery = {
+  scope: {
+    underlyingIds?: string[];
+    optionContractId?: string;
+  };
+  flowFilters: OptionFlowFilters;
+  storageFilters: OptionPrintQueryFilters;
+  isContractDrilldown: boolean;
+};
+
+const parseScopeList = (url: URL, ...keys: string[]): string[] | undefined => {
+  const values = keys
+    .flatMap((key) => url.searchParams.getAll(key))
+    .flatMap((value) => value.split(","))
+    .map((value) => value.trim().toUpperCase())
+    .filter(Boolean);
+  const unique = Array.from(new Set(values));
+  return unique.length > 0 ? unique : undefined;
+};
+
+export const parseOptionPrintQuery = (url: URL): ParsedOptionPrintQuery => {
+  const parsed = optionFilterQuerySchema.parse({
+    view: url.searchParams.get("view") ?? undefined,
+    security: url.searchParams.get("security") ?? undefined,
+    side: url.searchParams.get("side") ?? undefined,
+    type: url.searchParams.get("type") ?? undefined,
+    min_notional: url.searchParams.get("min_notional") ?? undefined
+  });
+  const scope = {
+    underlyingIds: parseScopeList(url, "underlying_id", "underlying_ids"),
+    optionContractId: url.searchParams.get("option_contract_id") ?? undefined
+  };
+  const view = parsed.view ?? "signal";
+  const security = parsed.security ?? (view === "raw" ? "all" : "stock");
+  const flowFilters: OptionFlowFilters = {
+    view,
+    securityTypes:
+      security === "all"
+        ? undefined
+        : ([security] as Array<z.infer<typeof OptionSecurityTypeSchema>>),
+    nbboSides: parsed.side,
+    optionTypes: parsed.type,
+    minNotional: parsed.min_notional
+  };
+  const isContractDrilldown = Boolean(scope.optionContractId);
+  const storageFilters: OptionPrintQueryFilters = isContractDrilldown
+    ? {
+        view: "raw",
+        optionContractId: scope.optionContractId
+      }
+    : {
+        view,
+        security,
+        minNotional: parsed.min_notional,
+        nbboSides: parsed.side,
+        optionTypes: parsed.type,
+        underlyingIds: scope.underlyingIds,
+        optionContractId: scope.optionContractId
+      };
+
+  return {
+    scope,
+    flowFilters,
+    storageFilters,
+    isContractDrilldown
+  };
+};

services/api/tests/live.test.ts

@@ -1,21 +1,25 @@
 import { describe, expect, it } from "bun:test";
 import type { ClickHouseClient } from "@islandflow/storage";
 import {
+  buildOptionSnapshotFilters,
+  HOT_LIVE_REDIS_KEYS,
   LiveStateManager,
   isLiveItemFresh,
   resolveGenericLiveLimits,
   shouldFanoutLiveEvent
 } from "../src/live";
 
-const makeClickHouse = (): ClickHouseClient =>
+const makeClickHouse = (
+  queryResolver?: (query: string) => unknown[]
+): ClickHouseClient =>
   ({
     exec: async () => {},
     insert: async () => {},
     ping: async () => ({ success: true }),
     close: async () => {},
-    query: async () => ({
+    query: async ({ query }: { query: string }) => ({
       async json<T>() {
-        return [] as T;
+        return (queryResolver?.(query) ?? []) as T;
       }
     })
   }) as ClickHouseClient;
@@ -62,9 +66,9 @@ describe("LiveStateManager", () => {
 
     expect(limits.options).toBe(777);
     expect(limits.nbbo).toBe(100000);
-    expect(limits.flow).toBe(10000);
+    expect(limits.flow).toBe(500);
-    expect(limits["equity-quotes"]).toBe(10000);
+    expect(limits["equity-quotes"]).toBe(500);
-    expect(limits.alerts).toBe(10000);
+    expect(limits.alerts).toBe(300);
   });
 
   it("hydrates snapshots from redis generic windows", async () => {
@@ -200,13 +204,121 @@ describe("LiveStateManager", () => {
     ]);
 
     const persisted = await redis.lRange("live:flow", 0, 99);
-    expect(persisted).toHaveLength(2);
+    await manager.flushRedisWrites();
+    const flushed = await redis.lRange("live:flow", 0, 99);
+    expect(persisted).toHaveLength(0);
+    expect(flushed).toHaveLength(2);
 
     const stats = manager.getStatsSnapshot();
     expect(stats.trimOperations).toBeGreaterThan(0);
+    expect(stats.redisFlushCount).toBeGreaterThan(0);
     expect(stats.cacheDepthByKey["live:flow"]).toBe(2);
   });
 
+  it("reorders out-of-order live events without dropping newest-first semantics", async () => {
+    const now = Date.now();
+    const manager = new LiveStateManager(makeClickHouse(), null, {
+      limits: {
+        options: 1000,
+        nbbo: 1000,
+        equities: 1000,
+        "equity-quotes": 500,
+        "equity-joins": 500,
+        flow: 3,
+        "smart-money": 300,
+        "classifier-hits": 300,
+        alerts: 300,
+        "inferred-dark": 300
+      },
+      scopedCacheMaxKeys: 32,
+      redisFlushIntervalMs: 250,
+      redisFlushMaxItems: 100
+    });
+
+    await manager.ingest("flow", {
+      source_ts: now,
+      ingest_ts: now + 1,
+      seq: 2,
+      trace_id: "flow-2",
+      id: "flow-2",
+      members: [],
+      features: {},
+      join_quality: {}
+    });
+    await manager.ingest("flow", {
+      source_ts: now - 1_000,
+      ingest_ts: now - 999,
+      seq: 1,
+      trace_id: "flow-1",
+      id: "flow-1",
+      members: [],
+      features: {},
+      join_quality: {}
+    });
+
+    const snapshot = await manager.getSnapshot({ channel: "flow" });
+    expect((snapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([
+      "flow-2",
+      "flow-1"
+    ]);
+    expect(manager.getStatsSnapshot().outOfOrderEvents).toBe(1);
+  });
+
+  it("evicts least-recently-used scoped candle caches past the configured key limit", async () => {
+    const manager = new LiveStateManager(makeClickHouse(), null, {
+      limits: resolveGenericLiveLimits(),
+      scopedCacheMaxKeys: 1,
+      redisFlushIntervalMs: 250,
+      redisFlushMaxItems: 100
+    });
+
+    await manager.ingest("equity-candles", {
+      source_ts: 100,
+      ingest_ts: 101,
+      seq: 1,
+      trace_id: "candle:SPY:60000:100",
+      ts: 100,
+      interval_ms: 60000,
+      underlying_id: "SPY",
+      open: 1,
+      high: 2,
+      low: 1,
+      close: 2,
+      volume: 10,
+      trade_count: 1
+    });
+    await manager.ingest("equity-candles", {
+      source_ts: 200,
+      ingest_ts: 201,
+      seq: 2,
+      trace_id: "candle:QQQ:60000:200",
+      ts: 200,
+      interval_ms: 60000,
+      underlying_id: "QQQ",
+      open: 3,
+      high: 4,
+      low: 3,
+      close: 4,
+      volume: 20,
+      trade_count: 2
+    });
+
+    const qqqSnapshot = await manager.getSnapshot({
+      channel: "equity-candles",
+      underlying_id: "QQQ",
+      interval_ms: 60000
+    });
+    const spySnapshot = await manager.getSnapshot({
+      channel: "equity-candles",
+      underlying_id: "SPY",
+      interval_ms: 60000
+    });
+
+    expect(qqqSnapshot.items).toHaveLength(1);
+    expect(spySnapshot.items).toEqual([]);
+    expect(manager.getStatsSnapshot().cacheEvictions).toBeGreaterThan(0);
+  });
+
   it("filters option and flow snapshots using subscription filters", async () => {
     const manager = new LiveStateManager(makeClickHouse(), null);
     const now = Date.now();
@@ -408,6 +520,228 @@ describe("LiveStateManager", () => {
     ]);
   });
 
+  it("seeds scoped option snapshots from clickhouse rows older than 24h", async () => {
+    const now = Date.now();
+    const staleTs = now - 25 * 60 * 60 * 1000;
+    const manager = new LiveStateManager(
+      makeClickHouse((query) =>
+        query.includes("FROM option_prints")
+          ? [
+              {
+                source_ts: staleTs,
+                ingest_ts: staleTs + 1,
+                seq: 1,
+                trace_id: "opt-ancient",
+                ts: staleTs,
+                option_contract_id: "AAPL-2025-01-17-200-C",
+                underlying_id: "AAPL",
+                price: 1,
+                size: 10,
+                exchange: "X",
+                signal_pass: true
+              }
+            ]
+          : []
+      ),
+      null
+    );
+
+    const snapshot = await manager.getSnapshot({
+      channel: "options",
+      underlying_ids: ["AAPL"],
+      option_contract_id: "AAPL-2025-01-17-200-C"
+    });
+
+    expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
+      "opt-ancient"
+    ]);
+    expect(snapshot.next_before).toEqual({ ts: staleTs, seq: 1 });
+    expect(isLiveItemFresh("options", snapshot.items[0], now)).toBe(false);
+  });
+
+  it("builds raw contract-only snapshot filters for focused option subscriptions", () => {
+    expect(
+      buildOptionSnapshotFilters({
+        channel: "options",
+        filters: {
+          view: "signal",
+          minNotional: 500_000,
+          nbboSides: ["A"],
+          optionTypes: ["call"],
+          securityTypes: ["stock"]
+        },
+        underlying_ids: ["AAPL"],
+        option_contract_id: "AAPL-2025-01-17-200-C"
+      })
+    ).toEqual({
+      view: "raw",
+      optionContractId: "AAPL-2025-01-17-200-C"
+    });
+  });
+
+  it("returns raw contract rows for focused option snapshots even when broad filters would reject them", async () => {
+    const manager = new LiveStateManager(
+      makeClickHouse((query) => {
+        expect(query).toContain("option_contract_id = 'AAPL-2025-01-17-200-C'");
+        expect(query).not.toContain("signal_pass = 1");
+        expect(query).not.toContain("notional >=");
+        expect(query).not.toContain("nbbo_side IN");
+        expect(query).not.toContain("option_type IN");
+        return [
+          {
+            source_ts: 1_000,
+            ingest_ts: 1_001,
+            seq: 1,
+            trace_id: "opt-raw",
+            ts: 1_000,
+            option_contract_id: "AAPL-2025-01-17-200-C",
+            underlying_id: "AAPL",
+            option_type: "put",
+            nbbo_side: "B",
+            notional: 50_000,
+            signal_pass: false,
+            price: 1,
+            size: 5,
+            exchange: "X"
+          }
+        ];
+      }),
+      null
+    );
+
+    const snapshot = await manager.getSnapshot({
+      channel: "options",
+      filters: {
+        view: "signal",
+        minNotional: 500_000,
+        nbboSides: ["A"],
+        optionTypes: ["call"],
+        securityTypes: ["stock"]
+      },
+      underlying_ids: ["AAPL"],
+      option_contract_id: "AAPL-2025-01-17-200-C"
+    });
+
+    expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
+      "opt-raw"
+    ]);
+  });
+
+  it("seeds scoped equity snapshots from clickhouse rows older than 24h", async () => {
+    const now = Date.now();
+    const staleTs = now - 25 * 60 * 60 * 1000;
+    const manager = new LiveStateManager(
+      makeClickHouse((query) =>
+        query.includes("FROM equity_prints")
+          ? [
+              {
+                source_ts: staleTs,
+                ingest_ts: staleTs + 1,
+                seq: 1,
+                trace_id: "eq-ancient",
+                ts: staleTs,
+                underlying_id: "AAPL",
+                price: 100,
+                size: 10,
+                exchange: "X",
+                offExchangeFlag: false
+              }
+            ]
+          : []
+      ),
+      null
+    );
+
+    const snapshot = await manager.getSnapshot({
+      channel: "equities",
+      underlying_ids: ["AAPL"]
+    });
+
+    expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
+      "eq-ancient"
+    ]);
+    expect(snapshot.next_before).toEqual({ ts: staleTs, seq: 1 });
+    expect(isLiveItemFresh("equities", snapshot.items[0], now)).toBe(false);
+  });
+
+  it("hydrates retained rows older than 24h into generic live snapshots and keeps them stale", async () => {
+    const redis = makeRedis();
+    const now = Date.now();
+    const staleTs = now - 25 * 60 * 60 * 1000;
+
+    await redis.lPush(
+      "live:options",
+      JSON.stringify({
+        source_ts: staleTs,
+        ingest_ts: staleTs + 1,
+        seq: 1,
+        trace_id: "opt-retained",
+        ts: staleTs,
+        option_contract_id: "AAPL-2025-01-17-200-C",
+        underlying_id: "AAPL",
+        price: 1,
+        size: 10,
+        exchange: "X",
+        signal_pass: true
+      })
+    );
+    await redis.hSet("live:cursors", "options", JSON.stringify({ ts: staleTs, seq: 1 }));
+
+    await redis.lPush(
+      "live:equities",
+      JSON.stringify({
+        source_ts: staleTs,
+        ingest_ts: staleTs + 1,
+        seq: 2,
+        trace_id: "eq-retained",
+        ts: staleTs,
+        underlying_id: "AAPL",
+        price: 100,
+        size: 10,
+        exchange: "X",
+        offExchangeFlag: false
+      })
+    );
+    await redis.hSet("live:cursors", "equities", JSON.stringify({ ts: staleTs, seq: 2 }));
+
+    await redis.lPush(
+      "live:flow",
+      JSON.stringify({
+        source_ts: staleTs,
+        ingest_ts: staleTs + 1,
+        seq: 3,
+        trace_id: "flow-retained",
+        id: "flow-retained",
+        members: ["opt-retained"],
+        features: {},
+        join_quality: {}
+      })
+    );
+    await redis.hSet("live:cursors", "flow", JSON.stringify({ ts: staleTs, seq: 3 }));
+
+    const manager = new LiveStateManager(makeClickHouse(), redis as never);
+    await manager.hydrate();
+
+    const [optionsSnapshot, equitiesSnapshot, flowSnapshot] = await Promise.all([
+      manager.getSnapshot({ channel: "options" }),
+      manager.getSnapshot({ channel: "equities" }),
+      manager.getSnapshot({ channel: "flow" })
+    ]);
+
+    expect((optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
+      "opt-retained"
+    ]);
+    expect((equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
+      "eq-retained"
+    ]);
+    expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([
+      "flow-retained"
+    ]);
+    expect(isLiveItemFresh("options", optionsSnapshot.items[0], now)).toBe(false);
+    expect(isLiveItemFresh("equities", equitiesSnapshot.items[0], now)).toBe(false);
+    expect(isLiveItemFresh("flow", flowSnapshot.items[0], now)).toBe(false);
+  });
+
   it("keeps only the newest NBBO quote per contract across hydrate and ingest", async () => {
     const redis = makeRedis();
     const now = Date.now();
@@ -573,6 +907,122 @@ describe("LiveStateManager", () => {
     expect(persisted).toHaveLength(1);
   });
 
+  it("includes hot-channel health for options, nbbo, equities, and flow", async () => {
+    const manager = new LiveStateManager(makeClickHouse(), null);
+    const now = Date.now();
+
+    await manager.ingest("options", {
+      source_ts: now,
+      ingest_ts: now + 1,
+      seq: 1,
+      trace_id: "opt-health",
+      ts: now,
+      option_contract_id: "AAPL-2025-01-17-200-C",
+      price: 1,
+      size: 10,
+      exchange: "X"
+    });
+    await manager.ingest("nbbo", {
+      source_ts: now,
+      ingest_ts: now + 1,
+      seq: 1,
+      trace_id: "nbbo-health",
+      ts: now,
+      option_contract_id: "AAPL-2025-01-17-200-C",
+      bid: 1,
+      ask: 1.1,
+      bidSize: 10,
+      askSize: 10
+    });
+    await manager.ingest("equities", {
+      source_ts: now,
+      ingest_ts: now + 1,
+      seq: 1,
+      trace_id: "eq-health",
+      ts: now,
+      underlying_id: "AAPL",
+      price: 100,
+      size: 10,
+      exchange: "X",
+      offExchangeFlag: false
+    });
+    await manager.ingest("flow", {
+      source_ts: now,
+      ingest_ts: now + 1,
+      seq: 1,
+      trace_id: "flow-health",
+      id: "flow-health",
+      members: [],
+      features: {},
+      join_quality: {}
+    });
+
+    const health = manager.getHotChannelHealth();
+    expect(health.options.healthy).toBe(true);
+    expect(health.nbbo.healthy).toBe(true);
+    expect(health.equities.healthy).toBe(true);
+    expect(health.flow.healthy).toBe(true);
+    expect(health.options.freshness_age_ms).not.toBeNull();
+    expect(health.nbbo.freshness_age_ms).not.toBeNull();
+    expect(health.equities.freshness_age_ms).not.toBeNull();
+    expect(health.flow.freshness_age_ms).not.toBeNull();
+  });
+
+  it("tracks generic cache and scoped clickhouse snapshot sources separately", async () => {
+    const manager = new LiveStateManager(makeClickHouse(() => []), null);
+    const now = Date.now();
+
+    await manager.ingest("options", {
+      source_ts: now,
+      ingest_ts: now + 1,
+      seq: 1,
+      trace_id: "opt-snapshot",
+      ts: now,
+      option_contract_id: "SPY-2025-01-17-500-C",
+      price: 1,
+      size: 10,
+      exchange: "X"
+    });
+
+    await manager.getSnapshot({ channel: "options" });
+    await manager.getSnapshot({
+      channel: "options",
+      underlying_ids: ["QQQ"],
+      option_contract_id: "QQQ-2025-01-17-400-C"
+    });
+
+    const stats = manager.getStatsSnapshot();
+    expect(stats.genericCacheSnapshots).toBe(1);
+    expect(stats.scopedClickHouseSnapshots).toBe(1);
+  });
+
+  it("keeps backend channel health healthy when a scoped query is quiet", async () => {
+    const manager = new LiveStateManager(makeClickHouse(() => []), null);
+    const now = Date.now();
+
+    await manager.ingest("options", {
+      source_ts: now,
+      ingest_ts: now + 1,
+      seq: 1,
+      trace_id: "opt-global",
+      ts: now,
+      option_contract_id: "SPY-2025-01-17-500-C",
+      price: 1,
+      size: 10,
+      exchange: "X"
+    });
+
+    const quietSnapshot = await manager.getSnapshot({
+      channel: "options",
+      underlying_ids: ["QQQ"],
+      option_contract_id: "QQQ-2025-01-17-400-C"
+    });
+
+    expect(quietSnapshot.items).toEqual([]);
+    expect(manager.getHotChannelHealth().options.healthy).toBe(true);
+    expect(manager.getStatsSnapshot().freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.options]).toBeLessThanOrEqual(50);
+  });
+
   it("exposes freshness helper for feed status", () => {
     expect(isLiveItemFresh("options", { ts: 1000 }, 1010)).toBe(true);
     expect(isLiveItemFresh("options", { ts: 1000 }, 20_001)).toBe(false);

services/api/tests/option-queries.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, it } from "bun:test";
+import { parseOptionPrintQuery } from "../src/option-queries";
+
+describe("parseOptionPrintQuery", () => {
+  it("keeps broad option flow filters for non-contract requests", () => {
+    const url = new URL(
+      "http://localhost/prints/options?view=signal&security=stock&side=A&type=call&min_notional=500000&underlying_ids=AAPL,MSFT"
+    );
+
+    expect(parseOptionPrintQuery(url)).toEqual({
+      scope: {
+        underlyingIds: ["AAPL", "MSFT"],
+        optionContractId: undefined
+      },
+      flowFilters: {
+        view: "signal",
+        securityTypes: ["stock"],
+        nbboSides: ["A"],
+        optionTypes: ["call"],
+        minNotional: 500000
+      },
+      storageFilters: {
+        view: "signal",
+        security: "stock",
+        nbboSides: ["A"],
+        optionTypes: ["call"],
+        minNotional: 500000,
+        underlyingIds: ["AAPL", "MSFT"],
+        optionContractId: undefined
+      },
+      isContractDrilldown: false
+    });
+  });
+
+  it("switches contract requests to raw contract-only storage filters", () => {
+    const url = new URL(
+      "http://localhost/replay/options?view=signal&security=stock&side=A&type=call&min_notional=500000&underlying_id=AAPL&option_contract_id=AAPL-2025-01-17-200-C"
+    );
+
+    expect(parseOptionPrintQuery(url)).toEqual({
+      scope: {
+        underlyingIds: ["AAPL"],
+        optionContractId: "AAPL-2025-01-17-200-C"
+      },
+      flowFilters: {
+        view: "signal",
+        securityTypes: ["stock"],
+        nbboSides: ["A"],
+        optionTypes: ["call"],
+        minNotional: 500000
+      },
+      storageFilters: {
+        view: "raw",
+        optionContractId: "AAPL-2025-01-17-200-C"
+      },
+      isContractDrilldown: true
+    });
+  });
+});

services/candles/src/index.ts

@@ -5,6 +5,7 @@ import {
   SUBJECT_EQUITY_PRINTS,
   STREAM_EQUITY_CANDLES,
   STREAM_EQUITY_PRINTS,
+  buildStreamConfig,
   buildDurableConsumer,
   connectJetStreamWithRetry,
   ensureStream,
@@ -240,31 +241,8 @@ const run = async () => {
     { attempts: 120, delayMs: 500 }
   );
 
-  await ensureStream(jsm, {
+  await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw"));
-    name: STREAM_EQUITY_PRINTS,
+  await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_CANDLES, SUBJECT_EQUITY_CANDLES, "derived"));
-    subjects: [SUBJECT_EQUITY_PRINTS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_EQUITY_CANDLES,
-    subjects: [SUBJECT_EQUITY_CANDLES],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
 
   const clickhouse = createClickHouseClient({
     url: env.CLICKHOUSE_URL,

services/compute/src/index.ts

@@ -26,6 +26,7 @@ import {
   STREAM_SMART_MONEY_EVENTS,
   STREAM_OPTION_NBBO,
   STREAM_OPTION_SIGNAL_PRINTS,
+  buildStreamConfig,
   buildDurableConsumer,
   connectJetStreamWithRetry,
   ensureStream,
@@ -40,12 +41,13 @@ import {
   ensureInferredDarkTable,
   ensureFlowPacketsTable,
   ensureSmartMoneyEventsTable,
-  insertAlert,
+  ClickHouseBatchWriter,
-  insertClassifierHit,
+  enqueueAlertInsert,
-  insertEquityPrintJoin,
+  enqueueClassifierHitInsert,
-  insertInferredDark,
+  enqueueEquityPrintJoinInsert,
-  insertFlowPacket,
+  enqueueFlowPacketInsert,
-  insertSmartMoneyEvent
+  enqueueInferredDarkInsert,
+  enqueueSmartMoneyEventInsert,
 } from "@islandflow/storage";
 import {
   AlertEventSchema,
@@ -82,7 +84,12 @@ import {
   type DarkInferenceConfig
 } from "./dark-inference";
 import { buildEquityPrintJoin, type EquityQuoteJoin } from "./equity-joins";
-import { createRedisClient, updateRollingStats, type RollingStatsConfig } from "./rolling-stats";
+import {
+  createRedisClient,
+  RollingWindowStore,
+  type RollingStatsConfig,
+  type RollingWindowStoreConfig
+} from "./rolling-stats";
 import { summarizeStructure, type ContractLeg } from "./structures";
 import {
   buildStructureFlowPacket,
@@ -103,6 +110,8 @@ const envSchema = z.object({
   CLUSTER_WINDOW_MS: z.coerce.number().int().positive().default(500),
   ROLLING_WINDOW_SIZE: z.coerce.number().int().positive().default(50),
   ROLLING_TTL_SEC: z.coerce.number().int().nonnegative().default(86400),
+  ROLLING_CACHE_FLUSH_INTERVAL_MS: z.coerce.number().int().positive().default(30_000),
+  ROLLING_CACHE_MAX_KEYS: z.coerce.number().int().positive().default(20_000),
   COMPUTE_DELIVER_POLICY: z.enum(["new", "all", "last", "last_per_subject"]).default("new"),
   COMPUTE_CONSUMER_RESET: z
     .preprocess((value) => {
@@ -119,6 +128,8 @@ const envSchema = z.object({
     }, z.boolean())
     .default(false),
   NBBO_MAX_AGE_MS: z.coerce.number().int().positive().default(1000),
+  COMPUTE_NBBO_CACHE_MAX_KEYS: z.coerce.number().int().positive().default(20_000),
+  COMPUTE_NBBO_CACHE_TTL_MS: z.coerce.number().int().positive().default(900_000),
   EQUITY_QUOTE_MAX_AGE_MS: z.coerce.number().int().positive().default(1000),
   DARK_INFER_WINDOW_MS: z.coerce.number().int().positive().default(60000),
   DARK_INFER_COOLDOWN_MS: z.coerce.number().int().nonnegative().default(30000),
@@ -269,6 +280,9 @@ const clusters = new Map<string, ClusterState>();
 const nbboCache = new Map<string, OptionNBBO>();
 const equityQuoteCache = new Map<string, EquityQuote>();
 const darkInferenceState = createDarkInferenceState();
+const nbboCacheTouchedAt = new Map<string, number>();
+const equityQuoteCacheTouchedAt = new Map<string, number>();
+const darkInferenceTouchedAt = new Map<string, number>();
 const recentLegsByKey = new Map<string, LegEvidence[]>();
 const recentLegsByRoot = new Map<string, LegEvidence[]>();
 const recentStructureEmits = new Map<string, number>();
@@ -278,6 +292,20 @@ const runtimeState = {
 };
 
 const MAX_RECENT_LEGS = 20;
+const EQUITY_QUOTE_CACHE_MAX_KEYS = 2_000;
+const EQUITY_QUOTE_CACHE_TTL_MS = 900_000;
+const DARK_INFERENCE_TTL_MS = 900_000;
+const CACHE_PRUNE_INTERVAL_MS = 60_000;
+
+const emitCounters = {
+  flowPackets: 0,
+  structurePackets: 0,
+  smartMoneyEvents: 0,
+  classifierHits: 0,
+  alerts: 0,
+  equityJoins: 0,
+  darkEvents: 0
+};
 
 const rollingKey = (metric: string, contractId: string): string => {
   return `rolling:${metric}:${contractId}`;
@@ -479,8 +507,8 @@ const pruneRecentStructureEmits = (anchorTs: number): void => {
 };
 
 const emitStructurePacketIfNeeded = async (
-  clickhouse: ReturnType<typeof createClickHouseClient>,
   js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
+  batchWriter: ClickHouseBatchWriter,
   legs: LegEvidence[],
   summary: ReturnType<typeof summarizeStructure>,
   currentContractId: string
@@ -512,16 +540,11 @@ const emitStructurePacketIfNeeded = async (
   const packet = buildStructureFlowPacket(plan, summary);
   const validated = FlowPacketSchema.parse(packet);
 
-  await insertFlowPacket(clickhouse, validated);
+  enqueueFlowPacketInsert(batchWriter, validated);
   await publishJson(js, SUBJECT_FLOW_PACKETS, validated);
-  await emitClassifiers(clickhouse, js, validated);
+  emitCounters.flowPackets += 1;
-
+  emitCounters.structurePackets += 1;
-  logger.info("emitted structure flow packet", {
+  await emitClassifiers(js, batchWriter, validated);
-    id: validated.id,
-    type: summary.type,
-    legs: summary.legs,
-    strikes: summary.strikes
-  });
 };
 
 const applyDeliverPolicy = (
@@ -606,6 +629,7 @@ const updateNbboCache = (nbbo: OptionNBBO): void => {
     (nbbo.ts === existing.ts && nbbo.seq >= existing.seq)
   ) {
     nbboCache.set(nbbo.option_contract_id, nbbo);
+    nbboCacheTouchedAt.set(nbbo.option_contract_id, Date.now());
   }
 };
 
@@ -617,6 +641,7 @@ const updateEquityQuoteCache = (quote: EquityQuote): void => {
     (quote.ts === existing.ts && quote.seq >= existing.seq)
   ) {
     equityQuoteCache.set(quote.underlying_id, quote);
+    equityQuoteCacheTouchedAt.set(quote.underlying_id, Date.now());
   }
 };
 
@@ -626,6 +651,7 @@ const selectNbbo = (contractId: string, ts: number): NbboJoin => {
     return { nbbo: null, ageMs: env.NBBO_MAX_AGE_MS + 1, stale: true };
   }
 
+  nbboCacheTouchedAt.set(contractId, Date.now());
   const ageMs = Math.abs(ts - nbbo.ts);
   const stale = ageMs > env.NBBO_MAX_AGE_MS;
   return { nbbo, ageMs, stale };
@@ -637,11 +663,77 @@ const selectEquityQuote = (underlyingId: string, ts: number): EquityQuoteJoin =>
     return { quote: null, ageMs: env.EQUITY_QUOTE_MAX_AGE_MS + 1, stale: true };
   }
 
+  equityQuoteCacheTouchedAt.set(underlyingId, Date.now());
   const ageMs = Math.abs(ts - quote.ts);
   const stale = ageMs > env.EQUITY_QUOTE_MAX_AGE_MS;
   return { quote, ageMs, stale };
 };
 
+const pruneTimedMap = <T>(
+  values: Map<string, T>,
+  touchedAt: Map<string, number>,
+  maxKeys: number,
+  ttlMs: number,
+  now = Date.now()
+): number => {
+  let removed = 0;
+
+  for (const [key, touched] of touchedAt) {
+    if (now - touched > ttlMs) {
+      touchedAt.delete(key);
+      values.delete(key);
+      removed += 1;
+    }
+  }
+
+  if (values.size <= maxKeys) {
+    return removed;
+  }
+
+  const overflow = values.size - maxKeys;
+  const oldest = [...touchedAt.entries()].sort((a, b) => a[1] - b[1]).slice(0, overflow);
+  for (const [key] of oldest) {
+    touchedAt.delete(key);
+    values.delete(key);
+    removed += 1;
+  }
+
+  return removed;
+};
+
+const pruneComputeCaches = (rollingStore: RollingWindowStore, now = Date.now()) => {
+  const nbboRemoved = pruneTimedMap(
+    nbboCache,
+    nbboCacheTouchedAt,
+    env.COMPUTE_NBBO_CACHE_MAX_KEYS,
+    env.COMPUTE_NBBO_CACHE_TTL_MS,
+    now
+  );
+  const quoteRemoved = pruneTimedMap(
+    equityQuoteCache,
+    equityQuoteCacheTouchedAt,
+    EQUITY_QUOTE_CACHE_MAX_KEYS,
+    EQUITY_QUOTE_CACHE_TTL_MS,
+    now
+  );
+  const darkRemoved = pruneTimedMap(
+    darkInferenceState.lastEmittedByUnderlying,
+    darkInferenceTouchedAt,
+    EQUITY_QUOTE_CACHE_MAX_KEYS,
+    DARK_INFERENCE_TTL_MS,
+    now
+  );
+  const rollingRemoved = rollingStore.prune(now);
+
+  logger.info("compute cache summary", {
+    nbbo_cache_size: nbboCache.size,
+    equity_quote_cache_size: equityQuoteCache.size,
+    dark_inference_cache_size: darkInferenceState.lastEmittedByUnderlying.size,
+    rolling_cache_size: rollingStore.size,
+    removed: nbboRemoved + quoteRemoved + darkRemoved + rollingRemoved
+  });
+};
+
 const classifyPlacement = (price: number, join: NbboJoin): NbboPlacement => {
   if (!Number.isFinite(price)) {
     return "MISSING";
@@ -679,10 +771,9 @@ const classifyPlacement = (price: number, join: NbboJoin): NbboPlacement => {
 };
 
 const flushCluster = async (
-  clickhouse: ReturnType<typeof createClickHouseClient>,
   js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
-  redis: ReturnType<typeof createRedisClient>,
+  batchWriter: ClickHouseBatchWriter,
-  rollingConfig: RollingStatsConfig,
+  rollingStore: RollingWindowStore,
   cluster: ClusterState
 ): Promise<void> => {
   if (cluster.flushed) {
@@ -784,12 +875,7 @@ const flushCluster = async (
     prefix: string
   ): Promise<void> => {
     try {
-      const snapshot = await updateRollingStats(
+      const snapshot = rollingStore.update(rollingKey(metric, cluster.contractId), value);
-        redis,
-        rollingKey(metric, cluster.contractId),
-        value,
-        rollingConfig
-      );
       features[`${prefix}_mean`] = roundTo(snapshot.mean);
       features[`${prefix}_std`] = roundTo(snapshot.stddev);
       features[`${prefix}_z`] = roundTo(snapshot.zscore);
@@ -824,7 +910,7 @@ const flushCluster = async (
       features.structure_rights = summary.rights;
     }
 
-    await emitStructurePacketIfNeeded(clickhouse, js, legs, summary, currentLeg.contractId);
+    await emitStructurePacketIfNeeded(js, batchWriter, legs, summary, currentLeg.contractId);
 
     const rootKey = buildRootKey(currentLeg);
     const rootCandidates = [
@@ -834,7 +920,7 @@ const flushCluster = async (
     const rollLegs = [currentLeg, ...rootCandidates];
     const rollSummary = summarizeStructure(rollLegs);
     if (rollSummary?.type === "roll") {
-      await emitStructurePacketIfNeeded(clickhouse, js, rollLegs, rollSummary, currentLeg.contractId);
+      await emitStructurePacketIfNeeded(js, batchWriter, rollLegs, rollSummary, currentLeg.contractId);
     }
 
     storeRecentLeg(currentLeg, anchorTs);
@@ -873,16 +959,10 @@ const flushCluster = async (
 
   const validated = FlowPacketSchema.parse(packet);
   try {
-    await insertFlowPacket(clickhouse, validated);
+    enqueueFlowPacketInsert(batchWriter, validated);
     await publishJson(js, SUBJECT_FLOW_PACKETS, validated);
-
+    emitCounters.flowPackets += 1;
-    await emitClassifiers(clickhouse, js, validated);
+    await emitClassifiers(js, batchWriter, validated);
-
-    logger.info("emitted flow packet", {
-      id: validated.id,
-      contract: cluster.contractId,
-      count: cluster.members.length
-    });
   } catch (error) {
     if (isExpectedShutdownNatsError(error)) {
       logger.info("skipped flow packet publish during shutdown", {
@@ -899,8 +979,8 @@ const flushCluster = async (
 };
 
 const emitClassifiers = async (
-  clickhouse: ReturnType<typeof createClickHouseClient>,
   js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
+  batchWriter: ClickHouseBatchWriter,
   packet: FlowPacket
 ): Promise<void> => {
   let smartMoneyEvent: SmartMoneyEvent;
@@ -915,8 +995,9 @@ const emitClassifiers = async (
         : packet.source_ts;
     const eventCalendarMatch = underlyingId ? eventCalendarProvider.findNextEvent(underlyingId, referenceTs) : null;
     smartMoneyEvent = SmartMoneyEventSchema.parse(buildSmartMoneyEventFromPacket(packet, { eventCalendarMatch }));
-    await insertSmartMoneyEvent(clickhouse, smartMoneyEvent);
+    enqueueSmartMoneyEventInsert(batchWriter, smartMoneyEvent);
     await publishJson(js, SUBJECT_SMART_MONEY_EVENTS, smartMoneyEvent);
+    emitCounters.smartMoneyEvents += 1;
   } catch (error) {
     if (isExpectedShutdownNatsError(error)) {
       return;
@@ -945,8 +1026,9 @@ const emitClassifiers = async (
 
   for (const hit of hitEvents) {
     try {
-      await insertClassifierHit(clickhouse, hit);
+      enqueueClassifierHitInsert(batchWriter, hit);
       await publishJson(js, SUBJECT_CLASSIFIER_HITS, hit);
+      emitCounters.classifierHits += 1;
     } catch (error) {
       if (isExpectedShutdownNatsError(error)) {
         continue;
@@ -981,8 +1063,9 @@ const emitClassifiers = async (
   });
 
   try {
-    await insertAlert(clickhouse, alert);
+    enqueueAlertInsert(batchWriter, alert);
     await publishJson(js, SUBJECT_ALERTS, alert);
+    emitCounters.alerts += 1;
   } catch (error) {
     if (isExpectedShutdownNatsError(error)) {
       return;
@@ -995,17 +1078,21 @@ const emitClassifiers = async (
 };
 
 const emitEquityJoin = async (
-  clickhouse: ReturnType<typeof createClickHouseClient>,
   js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
+  batchWriter: ClickHouseBatchWriter,
   print: EquityPrint
 ): Promise<void> => {
   const join = selectEquityQuote(print.underlying_id, print.ts);
   const payload: EquityPrintJoin = EquityPrintJoinSchema.parse(buildEquityPrintJoin(print, join));
 
   try {
-    await insertEquityPrintJoin(clickhouse, payload);
+    enqueueEquityPrintJoinInsert(batchWriter, payload);
   } catch (error) {
-    logger.error("failed to emit equity print join", {
+    if (isExpectedShutdownNatsError(error)) {
+      return;
+    }
+
+    logger.error("failed to queue equity print join", {
       error: error instanceof Error ? error.message : String(error),
       trace_id: payload.trace_id
     });
@@ -1014,6 +1101,7 @@ const emitEquityJoin = async (
 
   try {
     await publishJson(js, SUBJECT_EQUITY_JOINS, payload);
+    emitCounters.equityJoins += 1;
   } catch (error) {
     if (isExpectedShutdownNatsError(error)) {
       return;
@@ -1024,20 +1112,26 @@ const emitEquityJoin = async (
     });
   }
 
-  await emitDarkInferences(clickhouse, js, payload);
+  await emitDarkInferences(js, batchWriter, payload);
 };
 
 const emitDarkInferences = async (
-  clickhouse: ReturnType<typeof createClickHouseClient>,
   js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
+  batchWriter: ClickHouseBatchWriter,
   join: EquityPrintJoin
 ): Promise<void> => {
   const events = evaluateDarkInferences(join, darkInferenceConfig, darkInferenceState);
   for (const event of events) {
     const validated: InferredDarkEvent = InferredDarkEventSchema.parse(event);
     try {
-      await insertInferredDark(clickhouse, validated);
+      enqueueInferredDarkInsert(batchWriter, validated);
       await publishJson(js, SUBJECT_INFERRED_DARK, validated);
+      emitCounters.darkEvents += 1;
+      const underlyingId =
+        typeof join.features?.underlying_id === "string" ? join.features.underlying_id : null;
+      if (underlyingId) {
+        darkInferenceTouchedAt.set(underlyingId, Date.now());
+      }
     } catch (error) {
       if (isExpectedShutdownNatsError(error)) {
         continue;
@@ -1051,10 +1145,9 @@ const emitDarkInferences = async (
 };
 
 const flushEligibleClusters = async (
-  clickhouse: ReturnType<typeof createClickHouseClient>,
   js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
-  redis: ReturnType<typeof createRedisClient>,
+  batchWriter: ClickHouseBatchWriter,
-  rollingConfig: RollingStatsConfig,
+  rollingStore: RollingWindowStore,
   currentTs: number,
   skipContractId: string
 ): Promise<void> => {
@@ -1065,7 +1158,7 @@ const flushEligibleClusters = async (
 
     if (currentTs - cluster.endTs > env.CLUSTER_WINDOW_MS) {
       clusters.delete(contractId);
-      await flushCluster(clickhouse, js, redis, rollingConfig, cluster);
+      await flushCluster(js, batchWriter, rollingStore, cluster);
     }
   }
 };
@@ -1081,135 +1174,16 @@ const run = async () => {
     { attempts: 120, delayMs: 500 }
   );
 
-  await ensureStream(jsm, {
+  await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived"));
-    name: STREAM_OPTION_SIGNAL_PRINTS,
+  await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw"));
-    subjects: [SUBJECT_OPTION_SIGNAL_PRINTS],
+  await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw"));
-    retention: "limits",
+  await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw"));
-    storage: "file",
+  await ensureStream(jsm, buildStreamConfig(STREAM_FLOW_PACKETS, SUBJECT_FLOW_PACKETS, "derived"));
-    discard: "old",
+  await ensureStream(jsm, buildStreamConfig(STREAM_SMART_MONEY_EVENTS, SUBJECT_SMART_MONEY_EVENTS, "derived"));
-    max_msgs_per_subject: -1,
+  await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_JOINS, SUBJECT_EQUITY_JOINS, "derived"));
-    max_msgs: -1,
+  await ensureStream(jsm, buildStreamConfig(STREAM_INFERRED_DARK, SUBJECT_INFERRED_DARK, "derived"));
-    max_bytes: -1,
+  await ensureStream(jsm, buildStreamConfig(STREAM_CLASSIFIER_HITS, SUBJECT_CLASSIFIER_HITS, "derived"));
-    max_age: 0,
+  await ensureStream(jsm, buildStreamConfig(STREAM_ALERTS, SUBJECT_ALERTS, "derived"));
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_OPTION_NBBO,
-    subjects: [SUBJECT_OPTION_NBBO],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_EQUITY_PRINTS,
-    subjects: [SUBJECT_EQUITY_PRINTS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_EQUITY_QUOTES,
-    subjects: [SUBJECT_EQUITY_QUOTES],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_FLOW_PACKETS,
-    subjects: [SUBJECT_FLOW_PACKETS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_SMART_MONEY_EVENTS,
-    subjects: [SUBJECT_SMART_MONEY_EVENTS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_EQUITY_JOINS,
-    subjects: [SUBJECT_EQUITY_JOINS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_INFERRED_DARK,
-    subjects: [SUBJECT_INFERRED_DARK],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_CLASSIFIER_HITS,
-    subjects: [SUBJECT_CLASSIFIER_HITS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_ALERTS,
-    subjects: [SUBJECT_ALERTS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
 
   const clickhouse = createClickHouseClient({
     url: env.CLICKHOUSE_URL,
@@ -1242,6 +1216,51 @@ const run = async () => {
     windowSize: env.ROLLING_WINDOW_SIZE,
     ttlSeconds: env.ROLLING_TTL_SEC
   };
+  const rollingStore = new RollingWindowStore({
+    ...rollingConfig,
+    flushIntervalMs: env.ROLLING_CACHE_FLUSH_INTERVAL_MS,
+    maxKeys: env.ROLLING_CACHE_MAX_KEYS
+  } satisfies RollingWindowStoreConfig);
+  const batchWriter = new ClickHouseBatchWriter(clickhouse, {
+    flushIntervalMs: 100,
+    maxRows: 250,
+    onError: (table, error, rowCount) => {
+      logger.error("batched clickhouse insert failed", {
+        table,
+        row_count: rowCount,
+        error: error instanceof Error ? error.message : String(error),
+        action: "dropped"
+      });
+    }
+  });
+  const rollingFlushTimer = setInterval(() => {
+    void rollingStore.flushToRedis(redis);
+  }, env.ROLLING_CACHE_FLUSH_INTERVAL_MS);
+  const pruneTimer = setInterval(() => {
+    pruneComputeCaches(rollingStore);
+  }, CACHE_PRUNE_INTERVAL_MS);
+  const summaryTimer = setInterval(() => {
+    logger.info("compute minute summary", {
+      flow_packets_emitted: emitCounters.flowPackets,
+      structure_packets_emitted: emitCounters.structurePackets,
+      smart_money_events_emitted: emitCounters.smartMoneyEvents,
+      classifier_hits_emitted: emitCounters.classifierHits,
+      alerts_emitted: emitCounters.alerts,
+      equity_joins_emitted: emitCounters.equityJoins,
+      dark_events_emitted: emitCounters.darkEvents,
+      rolling_stats_cache_size: rollingStore.size
+    });
+    emitCounters.flowPackets = 0;
+    emitCounters.structurePackets = 0;
+    emitCounters.smartMoneyEvents = 0;
+    emitCounters.classifierHits = 0;
+    emitCounters.alerts = 0;
+    emitCounters.equityJoins = 0;
+    emitCounters.darkEvents = 0;
+  }, 60_000);
+  rollingFlushTimer.unref?.();
+  pruneTimer.unref?.();
+  summaryTimer.unref?.();
 
   await retry("clickhouse table init", 120, 500, async () => {
     await ensureFlowPacketsTable(clickhouse);
@@ -1578,7 +1597,7 @@ const run = async () => {
 
       try {
         const print = EquityPrintSchema.parse(equitySubscription.decode(msg));
-        await emitEquityJoin(clickhouse, js, print);
+        await emitEquityJoin(js, batchWriter, print);
         msg.ack();
       } catch (error) {
         logger.error("failed to process equity print", {
@@ -1602,11 +1621,16 @@ const run = async () => {
     runtimeState.shuttingDown = true;
     runtimeState.shutdownPromise = (async () => {
       logger.info("service stopping", { signal });
+      clearInterval(rollingFlushTimer);
+      clearInterval(pruneTimer);
+      clearInterval(summaryTimer);
 
       for (const cluster of [...clusters.values()]) {
-        await flushCluster(clickhouse, js, redis, rollingConfig, cluster);
+        await flushCluster(js, batchWriter, rollingStore, cluster);
       }
       clusters.clear();
+      await batchWriter.close();
+      await rollingStore.flushToRedis(redis);
 
       try {
         await nc.drain();
@@ -1655,10 +1679,9 @@ const run = async () => {
     try {
       const print = OptionPrintSchema.parse(subscription.decode(msg));
       await flushEligibleClusters(
-        clickhouse,
         js,
-        redis,
+        batchWriter,
-        rollingConfig,
+        rollingStore,
         print.ts,
         print.option_contract_id
       );
@@ -1674,7 +1697,7 @@ const run = async () => {
         updateCluster(existing, print);
       } else {
         clusters.delete(print.option_contract_id);
-        await flushCluster(clickhouse, js, redis, rollingConfig, existing);
+        await flushCluster(js, batchWriter, rollingStore, existing);
         clusters.set(print.option_contract_id, buildCluster(print));
       }
 

services/compute/src/rolling-stats.ts

@@ -5,6 +5,11 @@ export type RollingStatsConfig = {
   ttlSeconds: number;
 };
 
+export type RollingWindowStoreConfig = RollingStatsConfig & {
+  flushIntervalMs: number;
+  maxKeys: number;
+};
+
 export type RollingSnapshot = {
   baselineCount: number;
   mean: number;
@@ -12,6 +17,12 @@ export type RollingSnapshot = {
   zscore: number;
 };
 
+type RollingWindowEntry = {
+  values: number[];
+  updatedAt: number;
+  dirty: boolean;
+};
+
 const toNumbers = (values: string[]): number[] => {
   return values
     .map((value) => Number(value))
@@ -49,26 +60,120 @@ export const createRedisClient = (url: string) => {
   return createClient({ url });
 };
 
-export const updateRollingStats = async (
+const getOldestKey = (store: Map<string, RollingWindowEntry>): string | null => {
-  client: ReturnType<typeof createClient>,
+  let oldestKey: string | null = null;
-  key: string,
+  let oldestUpdatedAt = Number.POSITIVE_INFINITY;
-  value: number,
+
-  config: RollingStatsConfig
+  for (const [key, entry] of store) {
-): Promise<RollingSnapshot> => {
+    if (entry.updatedAt < oldestUpdatedAt) {
-  const limit = Math.max(0, config.windowSize - 1);
+      oldestUpdatedAt = entry.updatedAt;
-  const existing = await client.lRange(key, 0, limit);
+      oldestKey = key;
-  const baseline = toNumbers(existing);
+    }
+  }
+
+  return oldestKey;
+};
+
+export class RollingWindowStore {
+  private readonly store = new Map<string, RollingWindowEntry>();
+  private readonly ttlMs: number;
+  private readonly windowSize: number;
+  private readonly maxKeys: number;
+
+  constructor(private readonly config: RollingWindowStoreConfig) {
+    this.ttlMs = Math.max(0, config.ttlSeconds * 1000);
+    this.windowSize = Math.max(1, config.windowSize);
+    this.maxKeys = Math.max(1, config.maxKeys);
+  }
+
+  get size(): number {
+    return this.store.size;
+  }
+
+  update(key: string, value: number, now = Date.now()): RollingSnapshot {
+    this.prune(now);
+
+    const existing = this.store.get(key);
+    const baseline = existing?.values ?? [];
     const snapshot = computeSnapshot(baseline, value);
+    const nextValues = [value, ...baseline].slice(0, this.windowSize);
+
+    this.store.set(key, {
+      values: nextValues,
+      updatedAt: now,
+      dirty: true
+    });
+
+    this.enforceMaxKeys();
+    return snapshot;
+  }
+
+  prune(now = Date.now()): number {
+    if (this.ttlMs <= 0) {
+      return 0;
+    }
+
+    let removed = 0;
+    for (const [key, entry] of this.store) {
+      if (now - entry.updatedAt > this.ttlMs) {
+        this.store.delete(key);
+        removed += 1;
+      }
+    }
+    return removed;
+  }
+
+  async hydrateFromRedis(
+    client: ReturnType<typeof createClient>,
+    keys: string[],
+    now = Date.now()
+  ): Promise<void> {
+    for (const key of keys) {
+      const values = toNumbers(await client.lRange(key, 0, this.windowSize - 1));
+      if (values.length === 0) {
+        continue;
+      }
+      this.store.set(key, {
+        values,
+        updatedAt: now,
+        dirty: false
+      });
+    }
+    this.enforceMaxKeys();
+  }
+
+  async flushToRedis(client: ReturnType<typeof createClient>): Promise<number> {
+    let flushed = 0;
+    for (const [key, entry] of this.store) {
+      if (!entry.dirty) {
+        continue;
+      }
 
       const multi = client.multi();
+      multi.lTrim(key, 1, 0);
+      for (let idx = entry.values.length - 1; idx >= 0; idx -= 1) {
+        const value = entry.values[idx];
+        if (typeof value === "number" && Number.isFinite(value)) {
           multi.lPush(key, value.toString());
-  if (config.windowSize > 0) {
-    multi.lTrim(key, 0, config.windowSize - 1);
         }
-  if (config.ttlSeconds > 0) {
+      }
-    multi.expire(key, config.ttlSeconds);
+      if (this.config.ttlSeconds > 0) {
+        multi.expire(key, this.config.ttlSeconds);
       }
       await multi.exec();
+      entry.dirty = false;
+      flushed += 1;
+    }
+    return flushed;
+  }
 
-  return snapshot;
+  private enforceMaxKeys(): void {
-};
+    while (this.store.size > this.maxKeys) {
+      const oldestKey = getOldestKey(this.store);
+      if (!oldestKey) {
+        break;
+      }
+      this.store.delete(oldestKey);
+    }
+  }
+}

services/compute/tests/rolling-stats.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { computeSnapshot, computeStats } from "../src/rolling-stats";
+import { computeSnapshot, computeStats, RollingWindowStore } from "../src/rolling-stats";
 
 describe("rolling stats helpers", () => {
   test("computeStats handles empty baseline", () => {
@@ -21,4 +21,18 @@ describe("rolling stats helpers", () => {
     expect(snapshot.baselineCount).toBe(3);
     expect(snapshot.zscore).toBeCloseTo(1.84, 2);
   });
+
+  test("RollingWindowStore prunes stale keys by ttl", () => {
+    const store = new RollingWindowStore({
+      windowSize: 3,
+      ttlSeconds: 1,
+      flushIntervalMs: 30_000,
+      maxKeys: 10
+    });
+
+    store.update("rolling:premium:ABC", 10, 0);
+    expect(store.size).toBe(1);
+    expect(store.prune(1_500)).toBe(1);
+    expect(store.size).toBe(0);
+  });
 });

services/ingest-equities/src/index.ts

@@ -5,6 +5,7 @@ import {
   SUBJECT_EQUITY_QUOTES,
   STREAM_EQUITY_PRINTS,
   STREAM_EQUITY_QUOTES,
+  buildStreamConfig,
   connectJetStreamWithRetry,
   ensureStream,
   publishJson
@@ -194,31 +195,8 @@ const run = async () => {
     { attempts: 120, delayMs: 500 }
   );
 
-  await ensureStream(jsm, {
+  await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw"));
-    name: STREAM_EQUITY_PRINTS,
+  await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw"));
-    subjects: [SUBJECT_EQUITY_PRINTS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_EQUITY_QUOTES,
-    subjects: [SUBJECT_EQUITY_QUOTES],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
 
   const clickhouse = createClickHouseClient({
     url: env.CLICKHOUSE_URL,
@@ -251,11 +229,6 @@ const run = async () => {
       try {
         await insertEquityPrint(clickhouse, print);
         await publishJson(js, SUBJECT_EQUITY_PRINTS, print);
-        logger.info("published equity print", {
-          trace_id: print.trace_id,
-          seq: print.seq,
-          underlying_id: print.underlying_id
-        });
       } catch (error) {
         if (isExpectedShutdownError(error)) {
           return;

services/ingest-options/src/index.ts

@@ -9,6 +9,7 @@ import {
   STREAM_OPTION_NBBO,
   STREAM_OPTION_PRINTS,
   STREAM_OPTION_SIGNAL_PRINTS,
+  buildStreamConfig,
   buildDurableConsumer,
   connectJetStreamWithRetry,
   ensureStream,
@@ -109,7 +110,9 @@ const envSchema = z.object({
       return value;
     }, z.boolean())
     .default(false),
-  TESTING_THROTTLE_MS: z.coerce.number().int().nonnegative().default(200)
+  TESTING_THROTTLE_MS: z.coerce.number().int().nonnegative().default(200),
+  OPTION_CONTEXT_MAX_KEYS: z.coerce.number().int().positive().default(20_000),
+  OPTION_CONTEXT_TTL_MS: z.coerce.number().int().positive().default(900_000)
 });
 
 const env = readEnv(envSchema);
@@ -143,6 +146,44 @@ const state = {
 
 const nbboHistoryByContract: ContextHistory<OptionNBBO> = new Map();
 const equityQuoteHistoryByUnderlying: ContextHistory<EquityQuote> = new Map();
+const OPTION_CONTEXT_PRUNE_INTERVAL_MS = 60_000;
+
+const pruneContextHistory = <T extends { ts: number }>(
+  history: ContextHistory<T>,
+  maxKeys: number,
+  ttlMs: number,
+  now = Date.now()
+): number => {
+  let removed = 0;
+  for (const [key, items] of history) {
+    const filtered = items.filter((item) => now - item.ts <= ttlMs);
+    if (filtered.length === 0) {
+      history.delete(key);
+      removed += 1;
+      continue;
+    }
+    if (filtered.length !== items.length) {
+      history.set(key, filtered);
+    }
+  }
+
+  if (history.size <= maxKeys) {
+    return removed;
+  }
+
+  const overflow = history.size - maxKeys;
+  const oldestKeys = [...history.entries()]
+    .map(([key, items]) => [key, items.at(-1)?.ts ?? Number.NEGATIVE_INFINITY] as const)
+    .sort((a, b) => a[1] - b[1])
+    .slice(0, overflow);
+
+  for (const [key] of oldestKeys) {
+    history.delete(key);
+    removed += 1;
+  }
+
+  return removed;
+};
 
 const getErrorMessage = (error: unknown): string => {
   return error instanceof Error ? error.message : String(error);
@@ -305,57 +346,10 @@ const run = async () => {
     { attempts: 120, delayMs: 500 }
   );
 
-  await ensureStream(jsm, {
+  await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_PRINTS, SUBJECT_OPTION_PRINTS, "raw"));
-    name: STREAM_OPTION_PRINTS,
+  await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw"));
-    subjects: [SUBJECT_OPTION_PRINTS],
+  await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived"));
-    retention: "limits",
+  await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw"));
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_OPTION_NBBO,
-    subjects: [SUBJECT_OPTION_NBBO],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_OPTION_SIGNAL_PRINTS,
-    subjects: [SUBJECT_OPTION_SIGNAL_PRINTS],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
-
-  await ensureStream(jsm, {
-    name: STREAM_EQUITY_QUOTES,
-    subjects: [SUBJECT_EQUITY_QUOTES],
-    retention: "limits",
-    storage: "file",
-    discard: "old",
-    max_msgs_per_subject: -1,
-    max_msgs: -1,
-    max_bytes: -1,
-    max_age: 0,
-    num_replicas: 1
-  });
 
   const clickhouse = createClickHouseClient({
     url: env.CLICKHOUSE_URL,
@@ -400,14 +394,6 @@ const run = async () => {
         if (print.signal_pass) {
           await publishJson(js, SUBJECT_OPTION_SIGNAL_PRINTS, print);
         }
-        logger.info("published option print", {
-          trace_id: print.trace_id,
-          seq: print.seq,
-          option_contract_id: print.option_contract_id,
-          signal_pass: print.signal_pass,
-          nbbo_side: print.nbbo_side,
-          notional: print.notional
-        });
       } catch (error) {
         if (isExpectedShutdownError(error)) {
           return;
@@ -475,6 +461,18 @@ const run = async () => {
     }
   })();
 
+  const pruneTimer = setInterval(() => {
+    const removed =
+      pruneContextHistory(nbboHistoryByContract, env.OPTION_CONTEXT_MAX_KEYS, env.OPTION_CONTEXT_TTL_MS) +
+      pruneContextHistory(equityQuoteHistoryByUnderlying, env.OPTION_CONTEXT_MAX_KEYS, env.OPTION_CONTEXT_TTL_MS);
+    logger.info("option context cache summary", {
+      nbbo_context_keys: nbboHistoryByContract.size,
+      equity_quote_context_keys: equityQuoteHistoryByUnderlying.size,
+      removed
+    });
+  }, OPTION_CONTEXT_PRUNE_INTERVAL_MS);
+  pruneTimer.unref?.();
+
   const shutdown = async (signal: string) => {
     if (state.shutdownPromise) {
       return state.shutdownPromise;
@@ -483,6 +481,7 @@ const run = async () => {
     state.shuttingDown = true;
     state.shutdownPromise = (async () => {
       logger.info("service stopping", { signal });
+      clearInterval(pruneTimer);
       await stopAdapter();
 
       try {

services/replay/src/index.ts

@@ -11,6 +11,7 @@ import {
   STREAM_OPTION_NBBO,
   STREAM_OPTION_PRINTS,
   STREAM_OPTION_SIGNAL_PRINTS,
+  buildStreamConfig,
   connectJetStreamWithRetry,
   ensureStream,
   publishJson
@@ -180,19 +181,6 @@ const parseStreamList = (value: string): ReplayStreamKind[] => {
   return result;
 };
 
-const buildStreamConfig = (name: string, subject: string) => ({
-  name,
-  subjects: [subject],
-  retention: "limits",
-  storage: "file",
-  discard: "old",
-  max_msgs_per_subject: -1,
-  max_msgs: -1,
-  max_bytes: -1,
-  max_age: 0,
-  num_replicas: 1
-});
-
 const buildStartCursor = (startTs: number): ReplayCursor => {
   if (startTs <= 0) {
     return { ts: 0, seq: 0 };
@@ -304,10 +292,10 @@ const run = async () => {
 
   for (const kind of streamKinds) {
     const def = STREAM_DEFS[kind];
-    await ensureStream(jsm, buildStreamConfig(def.streamName, def.subject));
+    await ensureStream(jsm, buildStreamConfig(def.streamName, def.subject, "raw"));
   }
   if (streamKinds.includes("options")) {
-    await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS));
+    await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived"));
   }
 
   const clickhouse = createClickHouseClient({