From 2c088cb2878dff857356741a37ea19bccba00e63 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 00:21:30 -0400 Subject: [PATCH 01/18] Add frontend cooker preview --- .../frontend-cooker.module.css | 2 + apps/web/app/frontend-cooker/page.tsx | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 apps/web/app/frontend-cooker/frontend-cooker.module.css create mode 100644 apps/web/app/frontend-cooker/page.tsx diff --git a/apps/web/app/frontend-cooker/frontend-cooker.module.css b/apps/web/app/frontend-cooker/frontend-cooker.module.css new file mode 100644 index 0000000..34df997 --- /dev/null +++ b/apps/web/app/frontend-cooker/frontend-cooker.module.css @@ -0,0 +1,2 @@ +.cookerShell{min-height:100vh;display:grid;grid-template-columns:280px 1fr;background:#080806;color:#f4efe3}.chrome{position:sticky;top:0;height:100vh;padding:18px;display:flex;flex-direction:column;gap:22px;background:#111;border-right:1px solid #333;z-index:5}.chrome p{margin:0 0 8px;color:#d6a84f;text-transform:uppercase;letter-spacing:.18em;font-size:12px}.chrome h2{margin:0 0 8px;font-family:Georgia,serif;font-size:28px;line-height:1}.chrome small,.chrome footer{color:#aaa;line-height:1.45}.chrome footer{margin-top:auto;font-size:12px}.switcher{display:grid;gap:9px}.switcher button{display:grid;grid-template-columns:28px 1fr;gap:10px;align-items:center;text-align:left;padding:10px;border:1px solid #333;border-radius:14px;background:#191919;color:#ddd;cursor:pointer;transition:.18s}.switcher button:hover,.switcher .active{transform:translateX(3px);border-color:#d6a84f;background:#272111}.switcher b{display:grid;place-items:center;width:24px;height:24px;border-radius:50%;background:#333;color:#fff}.mock{min-height:100vh;padding:28px;font-family:var(--body,serif);transition:background .25s,color .25s}.productNav{display:flex;align-items:center;gap:18px;margin-bottom:28px}.productNav strong{margin-right:auto;letter-spacing:.16em}.productNav span{opacity:.75}.productNav button,.panelHead button{border:0;border-radius:999px;padding:10px 14px;cursor:pointer;background:var(--accent);color:var(--accentText)}.hero{display:grid;grid-template-columns:minmax(0,1.25fr)360px;gap:24px;align-items:stretch}.kicker{margin:0 0 10px;color:var(--accent);letter-spacing:.18em;text-transform:uppercase;font-size:12px}.hero h1{margin:0;font-family:var(--display,Georgia,serif);font-size:clamp(42px,6vw,92px);line-height:.9;letter-spacing:-.05em;text-transform:none}.copy{max-width:680px;font-size:18px;line-height:1.5;opacity:.78}.statusCard,.metrics article,.primaryPanel,.sidePanel,.tableWrap{border:1px solid var(--line);background:var(--panel);box-shadow:var(--shadow);border-radius:var(--radius)}.statusCard{padding:24px;font-size:15px}.statusCard b{display:block;margin:28px 0 4px;font-size:48px;font-family:var(--display)}.liveDot{display:inline-block;width:10px;height:10px;border-radius:50%;background:#28d77f;box-shadow:0 0 18px #28d77f;margin-right:8px}.metrics{display:grid;grid-template-columns:repeat(4,1fr);gap:14px;margin:20px 0}.metrics article{padding:18px;font-weight:700}.workspace{display:grid;grid-template-columns:1.45fr .75fr;gap:18px}.primaryPanel,.sidePanel{padding:18px}.panelHead{display:flex;justify-content:space-between;align-items:center;margin-bottom:12px}.panelHead h2,.sidePanel h2{margin:0;font-family:var(--display);font-size:24px}.chart{height:330px;position:relative;display:flex;align-items:flex-end;gap:1.8%;padding:24px;overflow:hidden;background:var(--chart);border-radius:calc(var(--radius) - 6px)}.chart i{flex:1;background:var(--bar);border-radius:99px 99px 0 0;animation:rise .7s both}.chart b{position:absolute;left:5%;right:5%;top:45%;height:3px;background:var(--accent);transform:rotate(-8deg);box-shadow:0 0 24px var(--accent)}.alert,.empty,.loading,.error{padding:14px;margin-top:12px;border-radius:14px;border:1px solid var(--line);background:rgba(255,255,255,.06)}.loading{background:repeating-linear-gradient(90deg,rgba(255,255,255,.08),rgba(255,255,255,.08) 12px,transparent 12px,transparent 24px)}.error{color:#ffb1a8}.tableWrap{margin-top:18px;overflow:auto}.tableWrap table{width:100%;border-collapse:collapse}.tableWrap th,.tableWrap td{padding:14px 16px;border-bottom:1px solid var(--line);text-align:left}.tableWrap tr:hover td{background:rgba(255,255,255,.08)}@keyframes rise{from{transform:scaleY(.25);opacity:.2}to{transform:scaleY(1);opacity:1}} +.pit{--display:Impact,Haettenschweiler,'Arial Narrow Bold',sans-serif;--body:'Trebuchet MS',sans-serif;--accent:#ffb000;--accentText:#1a0c00;--line:#3e321b;--panel:#16120b;--chart:#080602;--bar:linear-gradient(#ffcf52,#b35b00);--radius:4px;--shadow:inset 0 0 0 1px #000,0 18px 0 rgba(0,0,0,.25);background:radial-gradient(circle at 70% -10%,#5d2500,transparent 35%),#0b0905;color:#fff0c9}.pit .productNav{border-bottom:6px solid #ffb000;padding-bottom:12px}.atlas{--display:'Didot','Bodoni 72',serif;--body:'Avenir Next',Verdana,sans-serif;--accent:#00b894;--accentText:#001b15;--line:rgba(16,80,70,.28);--panel:rgba(235,255,250,.68);--chart:linear-gradient(135deg,#dff9ef,#b8d6e5);--bar:#0b8874;--radius:28px;--shadow:0 30px 80px rgba(30,90,90,.16);background:linear-gradient(120deg,#eef8f3,#cbdde1);color:#17322f}.ledger{--display:'Iowan Old Style',Georgia,serif;--body:Georgia,serif;--accent:#8b3f1f;--accentText:#fff8ee;--line:#d8c7a9;--panel:#fffaf0;--chart:#f7ecd8;--bar:#1f3f35;--radius:0;--shadow:8px 8px 0 #d8c7a9;background:#f4ead8;color:#24190f}.ledger.mock,.ledger .tableWrap table{font-size:17px}.neon{--display:'Courier New',monospace;--body:'Courier New',monospace;--accent:#39ff14;--accentText:#001400;--line:#263cff;--panel:rgba(4,8,28,.82);--chart:#03040f;--bar:linear-gradient(#ff2bd6,#263cff);--radius:18px;--shadow:0 0 32px rgba(57,255,20,.2),inset 0 0 24px rgba(38,60,255,.18);background:linear-gradient(180deg,#050718,#110014);color:#d6fff4}.neon .hero h1{text-shadow:0 0 20px #ff2bd6}.paper{--display:'Franklin Gothic Medium','Arial Narrow',sans-serif;--body:'Times New Roman',serif;--accent:#c5281c;--accentText:#fff;--line:#111;--panel:#f8f1df;--chart:repeating-linear-gradient(0deg,#efe4cc,#efe4cc 14px,#e2d3b8 15px);--bar:#111;--radius:0;--shadow:none;background:#eee2c8;color:#111}.paper .productNav,.paper .hero,.paper .metrics{border-bottom:3px double #111;padding-bottom:14px}@media(max-width:900px){.cookerShell{grid-template-columns:1fr}.chrome{height:auto;position:relative}.switcher{grid-template-columns:repeat(2,1fr)}.hero,.workspace,.metrics{grid-template-columns:1fr}.productNav{flex-wrap:wrap}.mock{padding:18px}} \ No newline at end of file diff --git a/apps/web/app/frontend-cooker/page.tsx b/apps/web/app/frontend-cooker/page.tsx new file mode 100644 index 0000000..c985524 --- /dev/null +++ b/apps/web/app/frontend-cooker/page.tsx @@ -0,0 +1,55 @@ +"use client"; + +import { useMemo, useState } from "react"; +import styles from "./frontend-cooker.module.css"; + +const variations = [ + { id: "pit", name: "Open-Outcry Pit", rationale: "A loud exchange-floor command center optimized for immediate threat recognition and dense scan paths." }, + { id: "atlas", name: "Glass Atlas", rationale: "A calm geospatial intelligence room that makes flow feel mapped, layered, and explorable." }, + { id: "ledger", name: "Ivory Ledger", rationale: "A refined analyst notebook with editorial hierarchy for slower, higher-confidence review." }, + { id: "neon", name: "Neon Underpass", rationale: "A kinetic cyberpunk tape for traders who want momentum, heat, and speed above all." }, + { id: "paper", name: "Signal Gazette", rationale: "A newspaper-like briefing that turns raw options activity into a morning intelligence digest." } +]; + +const flowRows = [ + ["NVDA", "910C", "05-17", "$4.8M", "AA", "+92%", "Sweep"], + ["TSLA", "175P", "05-10", "$2.1M", "BB", "−68%", "ISO"], + ["AAPL", "205C", "06-21", "$1.4M", "A", "+41%", "Block"], + ["SPY", "520P", "05-03", "$8.7M", "B", "−53%", "Split"], + ["AMD", "162C", "05-24", "$910K", "AA", "+77%", "Sweep"] +]; + +function MiniChart({ variant }: { variant: string }) { + return
+ {Array.from({ length: 22 }).map((_, i) => )} + +
; +} + +function AppMock({ id }: { id: string }) { + return
+ +
+

Live Options Intelligence

Unusual flow surfaced before the crowd.

Representative redesign of the IslandFlow terminal: live status, option sweeps, inferred dark activity, classifier hits, and replay controls.

+
Connected · 1,284 msgs/min
$42.6M premium tracked in active window
+
+
{["Alert score 87", "Bullish 62%", "Dark pool 14", "Stale feeds 0"].map(x =>
{x}
)}
+
+

Flow Radar

+

Classifier Hits

High conviction: NVDA call sweep above ask with confirming equity print.
Empty state: no stale NBBO quotes in the last 15s.
Loading replay baseline…
Error state: dark inference source delayed.
+
+
{["Ticker", "Contract", "Expiry", "Notional", "Side", "Delta", "Condition"].map(h => )}{flowRows.map((r) => {r.map((c, i) => )})}
{h}
{c}
+
; +} + +export default function FrontendCooker() { + const [active, setActive] = useState(0); + const current = variations[active]; + const nav = useMemo(() => variations.slice(0, 5), []); + return
+ + +
; +} From 6abfff30d3ff9a26a40e8b026f1fb28ed6ed78bf Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 00:26:21 -0400 Subject: [PATCH 02/18] Track plan mode extension work --- .beads/issues.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 7a44caa..8d689c2 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -2,4 +2,5 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-igk","title":"Add plan mode","description":"Implement a user-facing plan mode in the application so users can switch into planning before taking action. Scope to be clarified from existing app patterns.","status":"closed","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:22:37Z","created_by":"dirtydishes","updated_at":"2026-05-04T04:26:18Z","started_at":"2026-05-04T04:22:40Z","closed_at":"2026-05-04T04:26:18Z","close_reason":"Implemented as a global pi extension toggled with Shift+P","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-biq","title":"Finish raw live options delivery and filter/backpressure observability","description":"The smart-money signal path and Tape filters are in place, but the next firehose pass should finish server-side selective raw live delivery for options subscriptions and add explicit filtered-out/backpressure observability for API/web counters. This was discovered while landing islandflow-e4r.\n","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:28:58Z","created_by":"dirtydishes","updated_at":"2026-04-29T03:54:12Z","started_at":"2026-04-29T03:54:12Z","dependencies":[{"issue_id":"islandflow-biq","depends_on_id":"islandflow-e4r","type":"discovered-from","created_at":"2026-04-28T16:28:58Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} From e78387130a5082c0fb0a59f342e0aca44e128779 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 01:14:52 -0400 Subject: [PATCH 03/18] Implement options snapshot tape table --- .beads/issues.jsonl | 1 + apps/web/app/globals.css | 83 ++++++ apps/web/app/terminal.test.ts | 51 ++++ apps/web/app/terminal.tsx | 282 ++++++++++++------ options-overhaul-phase1.md | 20 ++ packages/storage/src/clickhouse.ts | 18 +- packages/storage/src/option-prints.ts | 38 +++ packages/storage/tests/option-prints.test.ts | 10 + packages/types/src/events.ts | 25 ++ packages/types/tests/events.test.ts | 41 +++ .../ingest-options/src/adapters/synthetic.ts | 95 +++++- services/ingest-options/src/enrichment.ts | 125 ++++++++ services/ingest-options/src/index.ts | 90 ++++-- .../ingest-options/tests/enrichment.test.ts | 88 ++++++ .../ingest-options/tests/synthetic.test.ts | 65 +++- 15 files changed, 904 insertions(+), 128 deletions(-) create mode 100644 options-overhaul-phase1.md create mode 100644 packages/types/tests/events.test.ts create mode 100644 services/ingest-options/src/enrichment.ts create mode 100644 services/ingest-options/tests/enrichment.test.ts diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 8d689c2..287cf8e 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-b3o","title":"Implement options tape table with execution spot","description":"Redesign OptionsPane into a dense classifier-colored table and preserve execution-time underlying spot on option prints from equity quote mid.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:41:59Z","created_by":"dirtydishes","updated_at":"2026-05-04T05:14:26Z","started_at":"2026-05-04T04:42:08Z","closed_at":"2026-05-04T05:14:26Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-ug1","title":"Fix false NBBO-missing badges in live Options tape","description":"Investigate and fix client-side cases where Options rows show NBBO missing/stale even when a fresh NBBO quote exists in the live nbbo map. Update rendering logic to prefer fresh quote-derived status and add regression tests.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-29T15:58:31Z","created_by":"dirtydishes","updated_at":"2026-04-29T16:01:28Z","started_at":"2026-04-29T15:58:35Z","closed_at":"2026-04-29T16:01:28Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index 3399a97..8e2cfca 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -873,6 +873,89 @@ h3 { background: linear-gradient(180deg, rgba(245, 166, 35, 0.07), rgba(255, 255, 255, 0.018)); } +.options-table-wrap { + min-height: 0; + overflow: auto; +} + +.options-table { + min-width: 1040px; +} + +.options-table-head, +.options-table-row { + display: grid; + grid-template-columns: 88px 72px 76px 72px 44px 76px 130px 70px 82px 64px 56px minmax(150px, 1fr); + align-items: center; + column-gap: 10px; +} + +.options-table-head { + position: sticky; + top: 0; + z-index: 2; + height: 30px; + padding: 0 10px; + border-bottom: 1px solid var(--border); + background: rgba(8, 11, 16, 0.98); + color: var(--muted); + font-size: 0.64rem; + font-weight: 700; + letter-spacing: 0.08em; +} + +.options-table-row { + width: 100%; + min-height: 34px; + padding: 0 10px; + border: 0; + border-bottom: 1px solid rgba(255, 255, 255, 0.055); + background: + linear-gradient(90deg, rgba(var(--classifier-rgb, 192, 200, 210), calc(0.02 + var(--classifier-intensity, 0) * 0.12)), transparent 62%), + rgba(255, 255, 255, 0.012); + color: inherit; + font: inherit; + text-align: left; +} + +.options-table-row:hover, +.options-table-row:focus-visible { + outline: none; + background: + linear-gradient(90deg, rgba(var(--classifier-rgb, 192, 200, 210), calc(0.04 + var(--classifier-intensity, 0) * 0.18)), transparent 68%), + rgba(255, 255, 255, 0.03); +} + +.options-table-row.is-classified { + cursor: pointer; + border-left: 3px solid rgba(var(--classifier-rgb), calc(0.35 + var(--classifier-intensity) * 0.45)); + padding-left: 7px; +} + +.options-table-row > span { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: 0.72rem; +} + +.mono { + font-variant-numeric: tabular-nums; +} + +.classifier-green { --classifier-rgb: 37, 193, 122; } +.classifier-red { --classifier-rgb: 255, 107, 95; } +.classifier-amber { --classifier-rgb: 245, 166, 35; } +.classifier-copper { --classifier-rgb: 198, 122, 75; } +.classifier-blue { --classifier-rgb: 77, 163, 255; } +.classifier-teal { --classifier-rgb: 64, 210, 190; } +.classifier-yellowgreen { --classifier-rgb: 174, 210, 78; } +.classifier-violet { --classifier-rgb: 170, 130, 255; } +.classifier-cyan { --classifier-rgb: 94, 214, 255; } +.classifier-magenta { --classifier-rgb: 255, 92, 205; } +.classifier-neutral { --classifier-rgb: 192, 200, 210; } + .contract, .drawer-row-title { margin-bottom: 6px; diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index f3f10be..883b9cd 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -1,12 +1,14 @@ import { describe, expect, it } from "bun:test"; import { buildDefaultFlowFilters, + classifierToneForFamily, deriveAlertDirection, countActiveFlowFilterGroups, formatCompactUsd, formatOptionContractLabel, flushPausableTapeData, getAlertWindowAnchorTs, + getOptionTableSnapshot, getLiveFeedStatus, normalizeAlertSeverity, nextFlowFilterPopoverState, @@ -14,6 +16,7 @@ import { reducePausableTapeData, shouldRetainLiveSnapshotHistory, shouldShowEquitiesSilentFeedWarning, + selectPrimaryClassifierHit, statusLabel, toggleFilterValue } from "./terminal"; @@ -171,6 +174,54 @@ describe("options display formatters", () => { expect(formatCompactUsd(1_250_000)).toBe("1.3M"); expect(formatCompactUsd(Number.NaN)).toBe("0.00"); }); + + it("renders options table snapshot values from preserved spot and IV", () => { + expect( + getOptionTableSnapshot({ + price: 1.25, + size: 10, + notional: 12_500, + execution_nbbo_side: "A", + execution_underlying_spot: 450.05, + execution_iv: 0.42 + }) + ).toEqual({ + spot: "450.05", + iv: "42%", + side: "A", + details: "10@1.25_A", + value: "12.5K" + }); + }); + + it("renders legacy options table snapshot spot and IV as dashes", () => { + const snapshot = getOptionTableSnapshot({ + price: 1, + size: 2 + }); + + expect(snapshot.spot).toBe("--"); + expect(snapshot.iv).toBe("--"); + }); +}); + +describe("classifier row decoration helpers", () => { + it("maps classifier families to row tones", () => { + expect(classifierToneForFamily("large_bullish_call_sweep")).toBe("green"); + expect(classifierToneForFamily("large_bearish_put_sweep")).toBe("red"); + expect(classifierToneForFamily("straddle")).toBe("blue"); + expect(classifierToneForFamily("unknown_family")).toBe("neutral"); + }); + + it("selects primary hits by confidence, source timestamp, then seq", () => { + const hit = selectPrimaryClassifierHit([ + { ...makeAlert({ classifier_id: "old", confidence: 0.9, source_ts: 1_000, seq: 1 }), direction: "bullish", explanations: [] }, + { ...makeAlert({ classifier_id: "new", confidence: 0.9, source_ts: 2_000, seq: 1 }), direction: "bullish", explanations: [] }, + { ...makeAlert({ classifier_id: "low", confidence: 0.5, source_ts: 3_000, seq: 9 }), direction: "bullish", explanations: [] } + ]); + + expect(hit?.classifier_id).toBe("new"); + }); }); describe("flow filter popup helpers", () => { diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index e2a0d9a..4a29481 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -11,6 +11,7 @@ import { useMemo, useRef, useState, + type CSSProperties, type Dispatch, type ReactNode, type SetStateAction @@ -982,7 +983,8 @@ const LIVE_SNAPSHOT_HISTORY_CHANNELS = new Set([ "options", "nbbo", "equities", - "flow" + "flow", + "classifier-hits" ]); export const shouldRetainLiveSnapshotHistory = ( @@ -1027,6 +1029,80 @@ const classifyNbboSide = (price: number, quote: OptionNBBO | null | undefined): return price >= mid ? "A" : "B"; }; +type ClassifierDecor = { + hit: ClassifierHitEvent; + family: string; + tone: string; + intensity: number; +}; + +const CLASSIFIER_FAMILY_TONES: Record = { + large_bullish_call_sweep: "green", + large_bearish_put_sweep: "red", + unusual_contract_spike: "amber", + large_call_sell_overwrite: "copper", + large_put_sell_write: "copper", + straddle: "blue", + strangle: "blue", + vertical_spread: "teal", + ladder_accumulation: "yellowgreen", + roll_up_down_out: "violet", + far_dated_conviction: "cyan", + zero_dte_gamma_punch: "magenta" +}; + +export const selectPrimaryClassifierHit = ( + hits: readonly ClassifierHitEvent[] +): ClassifierHitEvent | null => { + if (hits.length === 0) { + return null; + } + return [...hits].sort((a, b) => { + const confidenceDelta = b.confidence - a.confidence; + if (confidenceDelta !== 0) { + return confidenceDelta; + } + const tsDelta = b.source_ts - a.source_ts; + if (tsDelta !== 0) { + return tsDelta; + } + return b.seq - a.seq; + })[0]; +}; + +export const classifierToneForFamily = (classifierId: string): string => + CLASSIFIER_FAMILY_TONES[classifierId] ?? "neutral"; + +const buildClassifierDecor = (hit: ClassifierHitEvent): ClassifierDecor => ({ + hit, + family: hit.classifier_id, + tone: classifierToneForFamily(hit.classifier_id), + intensity: clamp(hit.confidence, 0.25, 1) +}); + +export const getOptionTableSnapshot = ( + print: Pick< + OptionPrint, + | "price" + | "size" + | "notional" + | "nbbo_side" + | "execution_nbbo_side" + | "execution_underlying_spot" + | "execution_iv" + >, + fallbackSide: OptionNbboSide | null = null +): { spot: string; iv: string; side: string; details: string; value: string } => { + const side = print.execution_nbbo_side ?? print.nbbo_side ?? fallbackSide ?? "--"; + return { + spot: typeof print.execution_underlying_spot === "number" ? formatPrice(print.execution_underlying_spot) : "--", + iv: typeof print.execution_iv === "number" ? formatPct(print.execution_iv) : "--", + side, + details: `${formatSize(print.size)}@${formatPrice(print.price)}_${side}`, + value: formatCompactUsd(print.notional ?? print.price * print.size * 100) + }; +}; + type ListScrollState = { listRef: React.RefObject; isAtTop: boolean; @@ -2125,7 +2201,8 @@ const getLiveManifest = ( { channel: "options", filters: flowFilters }, { channel: "nbbo" }, { channel: "equities" }, - { channel: "flow", filters: flowFilters } + { channel: "flow", filters: flowFilters }, + { channel: "classifier-hits" } ]; } @@ -4157,6 +4234,39 @@ const useTerminalState = () => { return traceId.slice(idx); }, []); + const classifierHitsByPacketId = useMemo(() => { + const map = new Map(); + for (const hit of classifierHitsFeed.items) { + const packetId = extractPacketIdFromClassifierHitTrace(hit.trace_id); + if (!packetId) { + continue; + } + map.set(packetId, [...(map.get(packetId) ?? []), hit]); + } + return map; + }, [classifierHitsFeed.items, extractPacketIdFromClassifierHitTrace]); + + const packetIdByOptionTraceId = useMemo(() => { + const map = new Map(); + for (const packet of flowFeed.items) { + for (const member of packet.members) { + map.set(member, packet.id); + } + } + return map; + }, [flowFeed.items]); + + const classifierDecorByOptionTraceId = useMemo(() => { + const map = new Map(); + for (const [traceId, packetId] of packetIdByOptionTraceId) { + const primary = selectPrimaryClassifierHit(classifierHitsByPacketId.get(packetId) ?? []); + if (primary) { + map.set(traceId, buildClassifierDecor(primary)); + } + } + return map; + }, [classifierHitsByPacketId, packetIdByOptionTraceId]); + const selectedClassifierPacketId = useMemo(() => { if (!selectedClassifierHit) { return null; @@ -4632,6 +4742,9 @@ const useTerminalState = () => { equityPrintMap, equityJoinMap: resolvedEquityJoinMap, flowPacketMap: resolvedFlowPacketMap, + classifierHitsByPacketId, + packetIdByOptionTraceId, + classifierDecorByOptionTraceId, selectedEvidence, selectedFlowPacket, selectedDarkEvidence, @@ -5002,7 +5115,7 @@ type OptionsPaneProps = { const OptionsPane = ({ limit }: OptionsPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredOptions.slice(0, limit) : state.filteredOptions; - const virtual = useVirtualList(items, state.optionsScroll.listRef, !limit, 96); + const virtual = useVirtualList(items, state.optionsScroll.listRef, !limit, 34); return ( { /> } > -
+
{items.length === 0 ? (
{state.tickerSet.size > 0 @@ -5040,103 +5153,92 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."}
) : ( - <> +
+
+ TIME + SYM + EXP + STRIKE + C/P + SPOT + DETAILS + TYPE + VALUE + SIDE + IV + CLASSIFIER +
{virtual.topSpacerHeight > 0 ? (
) : null} {virtual.visibleItems.map((print) => { - const contractId = normalizeContractId(print.option_contract_id); - const contractDisplay = formatOptionContractLabel(contractId); - const quote = state.nbboMap.get(contractId); - const nbboAge = quote ? Math.abs(print.ts - quote.ts) : null; - const nbboStale = nbboAge !== null && nbboAge > NBBO_MAX_AGE_MS_SAFE; - const nbboMid = quote ? (quote.bid + quote.ask) / 2 : null; - const nbboSide = print.nbbo_side ?? classifyNbboSide(print.price, quote); - const notional = print.notional ?? print.price * print.size * 100; - - return ( -
-
-
- {contractDisplay ? ( - <> - {contractDisplay.ticker} - {contractDisplay.strike} - {contractDisplay.expiration} - + const contractId = normalizeContractId(print.option_contract_id); + const parsed = parseOptionContractId(contractId); + const contractDisplay = formatOptionContractLabel(contractId); + const quote = state.nbboMap.get(contractId); + const hasPreservedNbbo = typeof print.execution_nbbo_side === "string"; + const nbboSide = + print.execution_nbbo_side ?? + print.nbbo_side ?? + (!hasPreservedNbbo ? classifyNbboSide(print.price, quote) : null); + const notional = print.notional ?? print.price * print.size * 100; + const spot = print.execution_underlying_spot; + const iv = print.execution_iv; + const decor = state.classifierDecorByOptionTraceId.get(print.trace_id); + const commonProps = { + className: `options-table-row${decor ? ` is-classified classifier-${decor.tone}` : ""}`, + style: decor ? ({ "--classifier-intensity": decor.intensity } as CSSProperties) : undefined + }; + const cells = ( + <> + {formatTime(print.ts)} + {contractDisplay?.ticker ?? parsed?.root ?? formatContractLabel(contractId)} + {contractDisplay?.expiration ?? parsed?.expiry ?? "--"} + {contractDisplay?.strike.replace(/[CP]$/, "") ?? "--"} + {parsed?.right ?? contractDisplay?.strike.slice(-1) ?? "--"} + {typeof spot === "number" ? formatPrice(spot) : "--"} + + {formatSize(print.size)}@{formatPrice(print.price)}_{nbboSide ?? "--"} + + {print.option_type ?? "--"} + ${formatCompactUsd(notional)} + + {nbboSide ? ( + {nbboSide} ) : ( - formatContractLabel(contractId) + "--" )} -
-
- ${formatPrice(print.price)} - {formatSize(print.size)}x - {print.exchange} - Notional ${formatCompactUsd(notional)} - {print.conditions?.map((condition) => { - const normalized = condition.toUpperCase(); - const tone = - normalized === "SWEEP" - ? "condition-sweep" - : normalized === "ISO" - ? "condition-iso" - : "condition-neutral"; - return ( - - {normalized} - - ); - })} -
- {quote ? ( -
- Bid ${formatPrice(quote.bid)} - Ask ${formatPrice(quote.ask)} - Mid ${formatPrice(nbboMid ?? 0)} - {Math.round(nbboAge ?? 0)}ms - {nbboSide ? ( - - - {nbboSide} - - - - A - Ask - - - AA - Above Ask - - - B - Bid - - - BB - Below Bid - - - - ) : null} - {print.nbbo_side === "STALE" || nbboStale ? Stale : null} -
- ) : ( -
- - {print.nbbo_side === "STALE" ? "NBBO stale" : "NBBO missing"} - -
- )} + + {typeof iv === "number" ? formatPct(iv) : "--"} + {decor ? humanizeClassifierId(decor.family) : "--"} + + ); + + return decor ? ( + + ) : ( +
+ {cells}
-
{formatTime(print.ts)}
-
- ); + ); })} {virtual.bottomSpacerHeight > 0 ? (
) : null} - +
)}
diff --git a/options-overhaul-phase1.md b/options-overhaul-phase1.md new file mode 100644 index 0000000..e8dbdaa --- /dev/null +++ b/options-overhaul-phase1.md @@ -0,0 +1,20 @@ +# Options Overhaul Phase 1: Snapshot Tape Table + +Implemented Phase 1 snapshot semantics for the Options tape. + +## Completed + +- Added flat execution snapshot fields to `OptionPrintSchema` / `OptionPrint`. +- Added ClickHouse columns and migrations for execution NBBO, underlying spot, and IV context. +- Added ingest enrichment that selects option NBBO and equity quote context at or before the option print timestamp. +- New enriched prints mirror `nbbo_side` from `execution_nbbo_side`. +- Added synthetic per-contract IV state with pressure, decay, and clamps. +- Redesigned the Options pane as a dense table using preserved spot/IV/NBBO side first. +- Added classifier-hit row color mapping and click/keyboard drawer interaction for classified rows. +- Updated `/tape` live subscriptions to include `classifier-hits`. +- Added focused tests for schema, storage, enrichment, synthetic IV, and frontend table/classifier helpers. + +## Verification + +- `bun test packages/types/tests/events.test.ts packages/storage/tests/option-prints.test.ts services/ingest-options/tests/enrichment.test.ts services/ingest-options/tests/synthetic.test.ts apps/web/app/terminal.test.ts` +- `bun run build` from `apps/web` diff --git a/packages/storage/src/clickhouse.ts b/packages/storage/src/clickhouse.ts index 5656214..c53caa4 100644 --- a/packages/storage/src/clickhouse.ts +++ b/packages/storage/src/clickhouse.ts @@ -512,7 +512,23 @@ const normalizeOptionRow = (row: unknown): unknown => { "ts", "price", "size", - "notional" + "notional", + "execution_nbbo_bid", + "execution_nbbo_ask", + "execution_nbbo_mid", + "execution_nbbo_spread", + "execution_nbbo_bid_size", + "execution_nbbo_ask_size", + "execution_nbbo_ts", + "execution_nbbo_age_ms", + "execution_underlying_spot", + "execution_underlying_bid", + "execution_underlying_ask", + "execution_underlying_mid", + "execution_underlying_spread", + "execution_underlying_ts", + "execution_underlying_age_ms", + "execution_iv" ]); if ("is_etf" in record) { diff --git a/packages/storage/src/option-prints.ts b/packages/storage/src/option-prints.ts index 7d9c983..8d28472 100644 --- a/packages/storage/src/option-prints.ts +++ b/packages/storage/src/option-prints.ts @@ -19,6 +19,25 @@ CREATE TABLE IF NOT EXISTS ${OPTION_PRINTS_TABLE} ( option_type Nullable(String), notional Nullable(Float64), nbbo_side Nullable(String), + execution_nbbo_bid Nullable(Float64), + execution_nbbo_ask Nullable(Float64), + execution_nbbo_mid Nullable(Float64), + execution_nbbo_spread Nullable(Float64), + execution_nbbo_bid_size Nullable(UInt32), + execution_nbbo_ask_size Nullable(UInt32), + execution_nbbo_ts Nullable(UInt64), + execution_nbbo_age_ms Nullable(Float64), + execution_nbbo_side Nullable(String), + execution_underlying_spot Nullable(Float64), + execution_underlying_bid Nullable(Float64), + execution_underlying_ask Nullable(Float64), + execution_underlying_mid Nullable(Float64), + execution_underlying_spread Nullable(Float64), + execution_underlying_ts Nullable(UInt64), + execution_underlying_age_ms Nullable(Float64), + execution_underlying_source Nullable(String), + execution_iv Nullable(Float64), + execution_iv_source Nullable(String), is_etf Nullable(Bool), signal_pass Nullable(Bool), signal_reasons Array(String) DEFAULT [], @@ -35,6 +54,25 @@ export const optionPrintsTableMigrations = (): string[] => { `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS option_type Nullable(String)`, `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS notional Nullable(Float64)`, `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS nbbo_side Nullable(String)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_nbbo_bid Nullable(Float64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_nbbo_ask Nullable(Float64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_nbbo_mid Nullable(Float64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_nbbo_spread Nullable(Float64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_nbbo_bid_size Nullable(UInt32)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_nbbo_ask_size Nullable(UInt32)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_nbbo_ts Nullable(UInt64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_nbbo_age_ms Nullable(Float64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_nbbo_side Nullable(String)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_underlying_spot Nullable(Float64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_underlying_bid Nullable(Float64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_underlying_ask Nullable(Float64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_underlying_mid Nullable(Float64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_underlying_spread Nullable(Float64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_underlying_ts Nullable(UInt64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_underlying_age_ms Nullable(Float64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_underlying_source Nullable(String)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_iv Nullable(Float64)`, + `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS execution_iv_source Nullable(String)`, `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS is_etf Nullable(Bool)`, `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS signal_pass Nullable(Bool)`, `ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS signal_reasons Array(String) DEFAULT []`, diff --git a/packages/storage/tests/option-prints.test.ts b/packages/storage/tests/option-prints.test.ts index 7643eeb..203ca9f 100644 --- a/packages/storage/tests/option-prints.test.ts +++ b/packages/storage/tests/option-prints.test.ts @@ -25,10 +25,20 @@ describe("option-prints storage helpers", () => { expect(normalized.conditions).toEqual([]); }); + it("normalizes legacy rows with missing execution context", () => { + const normalized = normalizeOptionPrint(basePrint); + expect(normalized.execution_nbbo_bid).toBeUndefined(); + expect(normalized.execution_underlying_spot).toBeUndefined(); + expect(normalized.execution_iv).toBeUndefined(); + }); + it("includes the correct table name in the DDL", () => { const ddl = optionPrintsTableDDL(); expect(ddl).toContain(OPTION_PRINTS_TABLE); expect(ddl).toContain("CREATE TABLE IF NOT EXISTS"); + expect(ddl).toContain("execution_nbbo_bid Nullable(Float64)"); + expect(ddl).toContain("execution_underlying_spot Nullable(Float64)"); + expect(ddl).toContain("execution_iv Nullable(Float64)"); }); it("builds before/history and trace lookup queries", async () => { diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 072e427..0ba5e57 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -22,6 +22,31 @@ export const OptionPrintSchema = EventMetaSchema.merge( option_type: z.preprocess((value) => (value === null ? undefined : value), OptionTypeSchema.optional()), notional: z.preprocess((value) => (value === null ? undefined : value), z.number().nonnegative().optional()), nbbo_side: z.preprocess((value) => (value === null ? undefined : value), OptionNbboSideSchema.optional()), + execution_nbbo_bid: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), + execution_nbbo_ask: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), + execution_nbbo_mid: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), + execution_nbbo_spread: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), + execution_nbbo_bid_size: z.preprocess((value) => (value === null ? undefined : value), z.number().int().nonnegative().optional()), + execution_nbbo_ask_size: z.preprocess((value) => (value === null ? undefined : value), z.number().int().nonnegative().optional()), + execution_nbbo_ts: z.preprocess((value) => (value === null ? undefined : value), z.number().int().nonnegative().optional()), + execution_nbbo_age_ms: z.preprocess((value) => (value === null ? undefined : value), z.number().nonnegative().optional()), + execution_nbbo_side: z.preprocess((value) => (value === null ? undefined : value), OptionNbboSideSchema.optional()), + execution_underlying_spot: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), + execution_underlying_bid: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), + execution_underlying_ask: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), + execution_underlying_mid: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), + execution_underlying_spread: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), + execution_underlying_ts: z.preprocess((value) => (value === null ? undefined : value), z.number().int().nonnegative().optional()), + execution_underlying_age_ms: z.preprocess((value) => (value === null ? undefined : value), z.number().nonnegative().optional()), + execution_underlying_source: z.preprocess( + (value) => (value === null ? undefined : value), + z.literal("equity_quote_mid").optional() + ), + execution_iv: z.preprocess((value) => (value === null ? undefined : value), z.number().nonnegative().optional()), + execution_iv_source: z.preprocess( + (value) => (value === null ? undefined : value), + z.enum(["provider", "synthetic_pressure_model"]).optional() + ), is_etf: z.preprocess((value) => (value === null ? undefined : value), z.boolean().optional()), signal_pass: z.preprocess((value) => (value === null ? undefined : value), z.boolean().optional()), signal_reasons: z.array(z.string().min(1)).optional(), diff --git a/packages/types/tests/events.test.ts b/packages/types/tests/events.test.ts new file mode 100644 index 0000000..c4b6b7e --- /dev/null +++ b/packages/types/tests/events.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "bun:test"; +import { OptionPrintSchema } from "../src/events"; + +describe("event schemas", () => { + it("accepts option print execution context fields", () => { + const parsed = OptionPrintSchema.parse({ + source_ts: 100, + ingest_ts: 101, + seq: 1, + trace_id: "trace-1", + ts: 100, + option_contract_id: "SPY-2025-01-17-450-C", + price: 1.25, + size: 10, + exchange: "TEST", + execution_nbbo_bid: 1.2, + execution_nbbo_ask: 1.3, + execution_nbbo_mid: 1.25, + execution_nbbo_spread: 0.1, + execution_nbbo_bid_size: 20, + execution_nbbo_ask_size: 30, + execution_nbbo_ts: 99, + execution_nbbo_age_ms: 1, + execution_nbbo_side: "MID", + execution_underlying_spot: 450.05, + execution_underlying_bid: 450, + execution_underlying_ask: 450.1, + execution_underlying_mid: 450.05, + execution_underlying_spread: 0.1, + execution_underlying_ts: 98, + execution_underlying_age_ms: 2, + execution_underlying_source: "equity_quote_mid", + execution_iv: 0.42, + execution_iv_source: "synthetic_pressure_model" + }); + + expect(parsed.execution_nbbo_side).toBe("MID"); + expect(parsed.execution_underlying_spot).toBe(450.05); + expect(parsed.execution_iv_source).toBe("synthetic_pressure_model"); + }); +}); diff --git a/services/ingest-options/src/adapters/synthetic.ts b/services/ingest-options/src/adapters/synthetic.ts index 7875f4f..a1d50e1 100644 --- a/services/ingest-options/src/adapters/synthetic.ts +++ b/services/ingest-options/src/adapters/synthetic.ts @@ -13,6 +13,9 @@ type SyntheticOptionsAdapterConfig = { type Burst = { contractId: string; + underlying: number; + expiryOffsetDays: number; + strike: number; basePrice: number; baseSize: number; exchange: string; @@ -23,7 +26,16 @@ type Burst = { seed: number; }; +export type SyntheticContractIvState = { + iv: number; + pressure: number; + lastTs: number; +}; + const OPTION_CONTRACT_MULTIPLIER = 100; +const IV_MIN = 0.05; +const IV_MAX = 2.5; +const IV_DECAY_HALF_LIFE_MS = 60_000; const SYNTHETIC_SYMBOLS = ["SPY", ...(SP500_SYMBOLS as readonly string[])]; const MS_PER_DAY = 24 * 60 * 60 * 1000; @@ -36,7 +48,7 @@ type SyntheticOptionsProfile = { pricePlacements: Record[]>; }; -type PricePlacement = "AA" | "A" | "MID" | "B" | "BB"; +export type PricePlacement = "AA" | "A" | "MID" | "B" | "BB"; type WeightedValue = { value: T; @@ -347,6 +359,55 @@ const formatExpiry = (now: number, offsetDays: number): string => { return expiryDate.toISOString().slice(0, 10); }; +const clampValue = (value: number, min: number, max: number): number => { + if (!Number.isFinite(value)) { + return min; + } + return Math.max(min, Math.min(max, value)); +}; + +const initializeSyntheticIv = (dteDays: number, moneyness: number): number => { + const dteBoost = dteDays <= 0 ? 0.22 : dteDays <= 7 ? 0.14 : dteDays <= 30 ? 0.06 : 0; + const moneynessBoost = clampValue(Math.abs(moneyness - 1) * 0.8, 0, 0.2); + return clampValue(0.24 + dteBoost + moneynessBoost, 0.18, 0.65); +}; + +export const updateSyntheticIvForTest = ( + state: SyntheticContractIvState | undefined, + input: { + ts: number; + placement: PricePlacement; + size: number; + notional: number; + dteDays: number; + moneyness: number; + } +): SyntheticContractIvState => { + const previous = state ?? { + iv: initializeSyntheticIv(input.dteDays, input.moneyness), + pressure: 0, + lastTs: input.ts + }; + const elapsed = Math.max(0, input.ts - previous.lastTs); + const decay = Math.pow(0.5, elapsed / IV_DECAY_HALF_LIFE_MS); + let pressure = previous.pressure * decay; + + if (input.placement === "AA" || input.placement === "A") { + const sizeImpact = Math.log10(Math.max(10, input.size)) * 0.012; + const notionalImpact = Math.log10(Math.max(1_000, input.notional)) * 0.01; + pressure += input.placement === "AA" ? sizeImpact + notionalImpact : (sizeImpact + notionalImpact) * 0.65; + } else if (input.placement === "MID") { + pressure += 0.001; + } else { + pressure -= input.placement === "BB" ? 0.018 : 0.01; + } + + pressure = clampValue(pressure, -0.25, 1.85); + const baseline = initializeSyntheticIv(input.dteDays, input.moneyness); + const iv = clampValue(baseline + pressure * 0.42, IV_MIN, IV_MAX); + return { iv: Number(iv.toFixed(4)), pressure, lastTs: input.ts }; +}; + const buildBurst = (burstIndex: number, now: number, profile: SyntheticOptionsProfile): Burst => { const symbol = SYNTHETIC_SYMBOLS[burstIndex % SYNTHETIC_SYMBOLS.length]; const symbolHash = hashSymbol(symbol); @@ -392,6 +453,9 @@ const buildBurst = (burstIndex: number, now: number, profile: SyntheticOptionsPr return { contractId, + underlying: baseUnderlying, + expiryOffsetDays: expiryOffset, + strike, basePrice: basePricePer, baseSize, exchange, @@ -420,6 +484,7 @@ export const createSyntheticOptionsAdapter = ( let nbboSeq = 0; let burstIndex = 0; let currentBurst: Burst | null = null; + const ivByContract = new Map(); let remainingRuns = 0; let timer: ReturnType | null = null; let stopped = false; @@ -448,12 +513,28 @@ export const createSyntheticOptionsAdapter = ( const priceJitter = ((i % 3) - 1) * 0.004; const sizeJitter = ((i % 3) - 1) * 0.08; const priceMultiplier = 1 + burst.priceStep * i + priceJitter; - const mid = Math.max(0.05, Number((burst.basePrice * priceMultiplier).toFixed(2))); - const spread = Math.max(0.02, Number((mid * 0.02).toFixed(2))); + const placement = pickPlacement(burst, i, profile); + const size = Math.max(1, Math.round(burst.baseSize * (1 + sizeJitter))); + const previousIv = ivByContract.get(burst.contractId); + const provisionalNotional = burst.basePrice * size * OPTION_CONTRACT_MULTIPLIER; + const ivState = updateSyntheticIvForTest(previousIv, { + ts: now + i * 5, + placement, + size, + notional: provisionalNotional, + dteDays: burst.expiryOffsetDays, + moneyness: burst.strike / burst.underlying + }); + ivByContract.set(burst.contractId, ivState); + const ivDrift = Math.max(0, ivState.iv - initializeSyntheticIv(burst.expiryOffsetDays, burst.strike / burst.underlying)); + const mid = Math.max( + 0.05, + Number((burst.basePrice * priceMultiplier * (1 + ivDrift * 1.15)).toFixed(2)) + ); + const spread = Math.max(0.02, Number((mid * (0.02 + Math.min(0.035, ivState.iv * 0.01))).toFixed(2))); const bid = Math.max(0.01, Number((mid - spread / 2).toFixed(2))); const ask = Math.max(bid + 0.01, Number((mid + spread / 2).toFixed(2))); const tick = Math.max(0.01, Number((spread * 0.25).toFixed(2))); - const placement = pickPlacement(burst, i, profile); let tradePrice = mid; if (placement === "AA") { @@ -476,9 +557,11 @@ export const createSyntheticOptionsAdapter = ( ts: now + i * 5, option_contract_id: burst.contractId, price: tradePrice, - size: Math.max(1, Math.round(burst.baseSize * (1 + sizeJitter))), + size, exchange: burst.exchange, - conditions: burst.conditions + conditions: burst.conditions, + execution_iv: ivState.iv, + execution_iv_source: "synthetic_pressure_model" }; if (handlers.onNBBO) { diff --git a/services/ingest-options/src/enrichment.ts b/services/ingest-options/src/enrichment.ts new file mode 100644 index 0000000..2104990 --- /dev/null +++ b/services/ingest-options/src/enrichment.ts @@ -0,0 +1,125 @@ +import { + OptionPrintSchema, + classifyOptionNbboSide, + deriveOptionPrintMetadata, + evaluateOptionSignal, + type EquityQuote, + type OptionNBBO, + type OptionPrint, + type OptionsSignalConfig +} from "@islandflow/types"; + +export const MAX_CONTEXT_HISTORY = 64; + +export type ContextHistory = Map; + +export const rememberContext = ( + history: ContextHistory, + key: string, + value: T +): void => { + const bucket = history.get(key) ?? []; + const existingIndex = bucket.findIndex((item) => item.ts === value.ts && item.seq === value.seq); + if (existingIndex >= 0) { + bucket[existingIndex] = value; + } else { + bucket.push(value); + } + bucket.sort((a, b) => { + const delta = a.ts - b.ts; + return delta !== 0 ? delta : a.seq - b.seq; + }); + if (bucket.length > MAX_CONTEXT_HISTORY) { + bucket.splice(0, bucket.length - MAX_CONTEXT_HISTORY); + } + history.set(key, bucket); +}; + +export const selectAtOrBefore = ( + items: readonly T[] | undefined, + ts: number +): T | null => { + if (!items?.length) { + return null; + } + + let selected: T | null = null; + for (const item of items) { + if (item.ts > ts) { + continue; + } + if (!selected || item.ts > selected.ts || (item.ts === selected.ts && item.seq >= selected.seq)) { + selected = item; + } + } + return selected; +}; + +export const enrichOptionPrint = ( + rawPrint: OptionPrint, + optionQuote: OptionNBBO | null | undefined, + equityQuote: EquityQuote | null | undefined, + config: OptionsSignalConfig +): OptionPrint => { + const derived = deriveOptionPrintMetadata(rawPrint, optionQuote, config); + const executionNbboSide = optionQuote + ? classifyOptionNbboSide(rawPrint.price, optionQuote, rawPrint.ts, config.nbboMaxAgeMs) + : undefined; + const nbboMid = + optionQuote && Number.isFinite(optionQuote.bid) && Number.isFinite(optionQuote.ask) + ? Number(((optionQuote.bid + optionQuote.ask) / 2).toFixed(4)) + : undefined; + const nbboSpread = + optionQuote && Number.isFinite(optionQuote.bid) && Number.isFinite(optionQuote.ask) + ? Number(Math.max(0, optionQuote.ask - optionQuote.bid).toFixed(4)) + : undefined; + const underlyingMid = + equityQuote && Number.isFinite(equityQuote.bid) && Number.isFinite(equityQuote.ask) + ? Number(((equityQuote.bid + equityQuote.ask) / 2).toFixed(4)) + : undefined; + const underlyingSpread = + equityQuote && Number.isFinite(equityQuote.bid) && Number.isFinite(equityQuote.ask) + ? Number(Math.max(0, equityQuote.ask - equityQuote.bid).toFixed(4)) + : undefined; + + const enrichedForSignal: OptionPrint = { + ...rawPrint, + ...derived, + nbbo_side: executionNbboSide ?? derived.nbbo_side, + ...(optionQuote + ? { + execution_nbbo_bid: optionQuote.bid, + execution_nbbo_ask: optionQuote.ask, + execution_nbbo_mid: nbboMid, + execution_nbbo_spread: nbboSpread, + execution_nbbo_bid_size: optionQuote.bidSize, + execution_nbbo_ask_size: optionQuote.askSize, + execution_nbbo_ts: optionQuote.ts, + execution_nbbo_age_ms: rawPrint.ts - optionQuote.ts, + execution_nbbo_side: executionNbboSide, + nbbo_side: executionNbboSide + } + : {}), + ...(equityQuote && underlyingMid !== undefined + ? { + execution_underlying_spot: underlyingMid, + execution_underlying_bid: equityQuote.bid, + execution_underlying_ask: equityQuote.ask, + execution_underlying_mid: underlyingMid, + execution_underlying_spread: underlyingSpread, + execution_underlying_ts: equityQuote.ts, + execution_underlying_age_ms: rawPrint.ts - equityQuote.ts, + execution_underlying_source: "equity_quote_mid" as const + } + : {}), + signal_profile: config.mode + }; + + const signalDecision = evaluateOptionSignal(enrichedForSignal, config); + return OptionPrintSchema.parse({ + ...enrichedForSignal, + signal_pass: signalDecision.signalPass, + signal_reasons: signalDecision.signalReasons, + signal_profile: signalDecision.signalProfile + }); +}; diff --git a/services/ingest-options/src/index.ts b/services/ingest-options/src/index.ts index 4c8010c..a5fe14c 100644 --- a/services/ingest-options/src/index.ts +++ b/services/ingest-options/src/index.ts @@ -4,12 +4,16 @@ import { SUBJECT_OPTION_NBBO, SUBJECT_OPTION_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, + SUBJECT_EQUITY_QUOTES, + STREAM_EQUITY_QUOTES, STREAM_OPTION_NBBO, STREAM_OPTION_PRINTS, STREAM_OPTION_SIGNAL_PRINTS, + buildDurableConsumer, connectJetStreamWithRetry, ensureStream, - publishJson + publishJson, + subscribeJson } from "@islandflow/bus"; import { createClickHouseClient, @@ -21,9 +25,10 @@ import { import { OptionNBBOSchema, OptionPrintSchema, - evaluateOptionSignal, + EquityQuoteSchema, deriveOptionPrintMetadata, resolveSyntheticMarketModes, + type EquityQuote, type OptionNBBO, type OptionPrint, type OptionsSignalConfig @@ -33,6 +38,7 @@ import { createDatabentoOptionsAdapter } from "./adapters/databento"; import { createIbkrOptionsAdapter } from "./adapters/ibkr"; import { createSyntheticOptionsAdapter } from "./adapters/synthetic"; import type { OptionIngestAdapter, StopHandler } from "./adapters/types"; +import { enrichOptionPrint, rememberContext, selectAtOrBefore, type ContextHistory } from "./enrichment"; import { z } from "zod"; const service = "ingest-options"; @@ -135,7 +141,9 @@ const state = { shuttingDown: false, shutdownPromise: null as Promise | null }; -const latestNbboByContract = new Map(); + +const nbboHistoryByContract: ContextHistory = new Map(); +const equityQuoteHistoryByUnderlying: ContextHistory = new Map(); const getErrorMessage = (error: unknown): string => { return error instanceof Error ? error.message : String(error); @@ -338,6 +346,19 @@ const run = async () => { num_replicas: 1 }); + await ensureStream(jsm, { + name: STREAM_EQUITY_QUOTES, + subjects: [SUBJECT_EQUITY_QUOTES], + retention: "limits", + storage: "file", + discard: "old", + max_msgs_per_subject: -1, + max_msgs: -1, + max_bytes: -1, + max_age: 0, + num_replicas: 1 + }); + const clickhouse = createClickHouseClient({ url: env.CLICKHOUSE_URL, database: env.CLICKHOUSE_DATABASE @@ -365,26 +386,15 @@ const run = async () => { } const rawPrint = OptionPrintSchema.parse(candidate); - const derived = deriveOptionPrintMetadata( - rawPrint, - latestNbboByContract.get(rawPrint.option_contract_id), - optionsSignalConfig + const parsedMetadata = deriveOptionPrintMetadata(rawPrint, null, optionsSignalConfig); + const optionQuote = selectAtOrBefore( + nbboHistoryByContract.get(rawPrint.option_contract_id), + rawPrint.ts ); - const signalDecision = evaluateOptionSignal( - { - ...rawPrint, - ...derived, - signal_profile: optionsSignalConfig.mode - }, - optionsSignalConfig - ); - const print = OptionPrintSchema.parse({ - ...rawPrint, - ...derived, - signal_pass: signalDecision.signalPass, - signal_reasons: signalDecision.signalReasons, - signal_profile: signalDecision.signalProfile - }); + const equityQuote = parsedMetadata.underlying_id + ? selectAtOrBefore(equityQuoteHistoryByUnderlying.get(parsedMetadata.underlying_id), rawPrint.ts) + : null; + const print = enrichOptionPrint(rawPrint, optionQuote, equityQuote, optionsSignalConfig); try { await insertOptionPrint(clickhouse, print); @@ -422,14 +432,7 @@ const run = async () => { } const nbbo = OptionNBBOSchema.parse(candidate); - const existing = latestNbboByContract.get(nbbo.option_contract_id); - if ( - !existing || - nbbo.ts > existing.ts || - (nbbo.ts === existing.ts && nbbo.seq >= existing.seq) - ) { - latestNbboByContract.set(nbbo.option_contract_id, nbbo); - } + rememberContext(nbboHistoryByContract, nbbo.option_contract_id, nbbo); try { await insertOptionNBBO(clickhouse, nbbo); @@ -447,6 +450,33 @@ const run = async () => { } }); + const equityQuoteConsumer = buildDurableConsumer("ingest-options-equity-quotes"); + equityQuoteConsumer.deliverAll(); + const equityQuoteSubscription = await subscribeJson( + js, + SUBJECT_EQUITY_QUOTES, + equityQuoteConsumer + ); + + void (async () => { + for await (const msg of equityQuoteSubscription.messages) { + if (state.shuttingDown) { + msg.ack(); + continue; + } + try { + const quote = EquityQuoteSchema.parse(equityQuoteSubscription.decode(msg)); + rememberContext(equityQuoteHistoryByUnderlying, quote.underlying_id.toUpperCase(), quote); + msg.ack(); + } catch (error) { + logger.error("failed to process equity quote context", { + error: getErrorMessage(error) + }); + msg.ack(); + } + } + })(); + const shutdown = async (signal: string) => { if (state.shutdownPromise) { return state.shutdownPromise; diff --git a/services/ingest-options/tests/enrichment.test.ts b/services/ingest-options/tests/enrichment.test.ts new file mode 100644 index 0000000..d5d505a --- /dev/null +++ b/services/ingest-options/tests/enrichment.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "bun:test"; +import type { EquityQuote, OptionNBBO, OptionPrint, OptionsSignalConfig } from "@islandflow/types"; +import { enrichOptionPrint, selectAtOrBefore } from "../src/enrichment"; + +const config: OptionsSignalConfig = { + mode: "all", + minNotional: 0, + etfMinNotional: 0, + bidSideMinNotional: 0, + midMinNotional: 0, + missingNbboMinNotional: 0, + largePrintMinSize: 1, + largePrintMinNotional: 0, + sweepMinNotional: 0, + autoKeepMinNotional: 100_000, + nbboMaxAgeMs: 1_500, + etfUnderlyings: new Set(["SPY"]) +}; + +const print: OptionPrint = { + source_ts: 1_000, + ingest_ts: 1_000, + seq: 1, + trace_id: "print-1", + ts: 1_000, + option_contract_id: "SPY-2025-01-17-450-C", + price: 1.3, + size: 10, + exchange: "TEST" +}; + +const nbbo = (overrides: Partial = {}): OptionNBBO => ({ + source_ts: 990, + ingest_ts: 990, + seq: 1, + trace_id: "nbbo-1", + ts: 990, + option_contract_id: "SPY-2025-01-17-450-C", + bid: 1.2, + ask: 1.3, + bidSize: 20, + askSize: 30, + ...overrides +}); + +const equityQuote = (overrides: Partial = {}): EquityQuote => ({ + source_ts: 980, + ingest_ts: 980, + seq: 1, + trace_id: "eq-1", + ts: 980, + underlying_id: "SPY", + bid: 450, + ask: 450.1, + ...overrides +}); + +describe("option print enrichment", () => { + it("attaches preserved NBBO context and mirrors nbbo_side", () => { + const enriched = enrichOptionPrint(print, nbbo(), null, config); + + expect(enriched.execution_nbbo_bid).toBe(1.2); + expect(enriched.execution_nbbo_ask).toBe(1.3); + expect(enriched.execution_nbbo_mid).toBe(1.25); + expect(enriched.execution_nbbo_age_ms).toBe(10); + expect(enriched.execution_nbbo_side).toBe("A"); + expect(enriched.nbbo_side).toBe(enriched.execution_nbbo_side); + }); + + it("attaches preserved underlying quote mid as spot", () => { + const enriched = enrichOptionPrint(print, null, equityQuote(), config); + + expect(enriched.execution_underlying_spot).toBe(450.05); + expect(enriched.execution_underlying_mid).toBe(450.05); + expect(enriched.execution_underlying_source).toBe("equity_quote_mid"); + expect(enriched.execution_underlying_age_ms).toBe(20); + }); + + it("selects context at or before the print timestamp only", () => { + const selected = selectAtOrBefore( + [nbbo({ ts: 900, seq: 1, bid: 1 }), nbbo({ ts: 1_001, seq: 2, bid: 2 })], + print.ts + ); + + expect(selected?.ts).toBe(900); + expect(selected?.bid).toBe(1); + }); +}); diff --git a/services/ingest-options/tests/synthetic.test.ts b/services/ingest-options/tests/synthetic.test.ts index 95f11e3..e0c8407 100644 --- a/services/ingest-options/tests/synthetic.test.ts +++ b/services/ingest-options/tests/synthetic.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "bun:test"; -import { buildSyntheticBurstForTest } from "../src/adapters/synthetic"; +import { buildSyntheticBurstForTest, updateSyntheticIvForTest } from "../src/adapters/synthetic"; const totalBurstNotional = (burst: { basePrice: number; @@ -24,3 +24,66 @@ describe("synthetic options burst sizing", () => { expect(totalBurstNotional(burst)).toBeLessThanOrEqual(240_000); }); }); + +describe("synthetic options IV model", () => { + it("increases under repeated same-contract ask buying", () => { + let state = updateSyntheticIvForTest(undefined, { + ts: 1_000, + placement: "A", + size: 100, + notional: 20_000, + dteDays: 1, + moneyness: 1.02 + }); + const firstIv = state.iv; + + state = updateSyntheticIvForTest(state, { + ts: 1_100, + placement: "AA", + size: 300, + notional: 80_000, + dteDays: 1, + moneyness: 1.02 + }); + + expect(state.iv).toBeGreaterThan(firstIv); + }); + + it("decays after inactivity", () => { + const active = updateSyntheticIvForTest(undefined, { + ts: 1_000, + placement: "AA", + size: 500, + notional: 120_000, + dteDays: 7, + moneyness: 1.1 + }); + const decayed = updateSyntheticIvForTest(active, { + ts: 181_000, + placement: "MID", + size: 10, + notional: 1_000, + dteDays: 7, + moneyness: 1.1 + }); + + expect(decayed.iv).toBeLessThan(active.iv); + }); + + it("keeps IV within clamps", () => { + let state = undefined; + for (let i = 0; i < 80; i += 1) { + state = updateSyntheticIvForTest(state, { + ts: 1_000 + i * 10, + placement: "AA", + size: 10_000, + notional: 5_000_000, + dteDays: 0, + moneyness: 1.8 + }); + } + + expect(state.iv).toBeGreaterThanOrEqual(0.05); + expect(state.iv).toBeLessThanOrEqual(2.5); + }); +}); From cfc3836815a440e6b2aebb4f0998ba070e07d48a Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 01:33:25 -0400 Subject: [PATCH 04/18] Document phase 1 options tape overhaul - Describe snapshot-based option print storage and ingest enrichment - Cover synthetic IV behavior, table redesign, and classifier coloring --- tape-overhaul-phase1.md | 320 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 tape-overhaul-phase1.md diff --git a/tape-overhaul-phase1.md b/tape-overhaul-phase1.md new file mode 100644 index 0000000..ead0bd6 --- /dev/null +++ b/tape-overhaul-phase1.md @@ -0,0 +1,320 @@ +# Options Overhaul Phase 1: Snapshot Tape Table + +## Summary + +Convert the Options tape into a dense table where every row is an individual option print with preserved execution context. The print itself becomes the authoritative record for what was known around that trade at the moment it printed: option NBBO, underlying spot, IV, notional, side/classification metadata, and classifier-derived row coloring. + +This phase includes backend enrichment, storage/type changes, synthetic IV behavior, and the frontend table redesign together. + +## Core Principle + +Do not treat NBBO, spot, or IV as live lookups in the table once the print has been recorded. + +Each option print should carry a snapshot of its execution context. The UI should prefer those preserved fields and only fall back to current side maps for legacy rows that predate the migration. + +## Public Type Changes + +Extend `OptionPrintSchema` / `OptionPrint` in `packages/types/src/events.ts`. + +Add optional flat fields: + +```ts +execution_nbbo_bid?: number; +execution_nbbo_ask?: number; +execution_nbbo_mid?: number; +execution_nbbo_spread?: number; +execution_nbbo_bid_size?: number; +execution_nbbo_ask_size?: number; +execution_nbbo_ts?: number; +execution_nbbo_age_ms?: number; +execution_nbbo_side?: OptionNbboSide; + +execution_underlying_spot?: number; +execution_underlying_bid?: number; +execution_underlying_ask?: number; +execution_underlying_mid?: number; +execution_underlying_spread?: number; +execution_underlying_ts?: number; +execution_underlying_age_ms?: number; +execution_underlying_source?: "equity_quote_mid"; + +execution_iv?: number; +execution_iv_source?: "provider" | "synthetic_pressure_model"; +``` + +Keep existing fields for compatibility: + +- `nbbo_side` +- `notional` +- `underlying_id` +- `option_type` +- `signal_*` + +Set `nbbo_side` to match `execution_nbbo_side` for new prints so existing filters continue working. + +## Storage Changes + +Update `packages/storage/src/option-prints.ts`. + +Add ClickHouse columns: + +```sql +execution_nbbo_bid Nullable(Float64), +execution_nbbo_ask Nullable(Float64), +execution_nbbo_mid Nullable(Float64), +execution_nbbo_spread Nullable(Float64), +execution_nbbo_bid_size Nullable(UInt32), +execution_nbbo_ask_size Nullable(UInt32), +execution_nbbo_ts Nullable(UInt64), +execution_nbbo_age_ms Nullable(Float64), +execution_nbbo_side Nullable(String), + +execution_underlying_spot Nullable(Float64), +execution_underlying_bid Nullable(Float64), +execution_underlying_ask Nullable(Float64), +execution_underlying_mid Nullable(Float64), +execution_underlying_spread Nullable(Float64), +execution_underlying_ts Nullable(UInt64), +execution_underlying_age_ms Nullable(Float64), +execution_underlying_source Nullable(String), + +execution_iv Nullable(Float64), +execution_iv_source Nullable(String) +``` + +Add `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` migrations for all fields. + +Update row normalization so missing legacy values parse as `undefined`. + +## Ingest Enrichment + +Update `services/ingest-options/src/index.ts`. + +Maintain caches: + +- latest option NBBO by contract +- latest equity quote by underlying +- synthetic/adapter-provided IV by contract when available + +When an option trade arrives: + +1. Parse raw print. +2. Derive underlying, option type, notional, ETF flag as today. +3. Select latest option NBBO for the contract at or before `print.ts`. +4. Attach preserved NBBO fields: + - bid, ask, mid, spread + - bid/ask sizes + - quote timestamp + - quote age + - execution NBBO side +5. Select latest equity quote for the underlying at or before `print.ts`. +6. Attach preserved underlying fields: + - bid, ask, mid + - spread + - quote timestamp + - quote age + - `execution_underlying_spot = mid` + - `execution_underlying_source = "equity_quote_mid"` +7. Attach IV if available. +8. Evaluate signal filters using preserved execution fields. +9. Persist and publish the enriched print. + +Important behavior: + +- Do not mark these preserved fields stale in the UI. +- Age fields are still stored for auditability. +- If no at-or-before quote exists, leave that context unset. +- Never use a quote after the option print timestamp for preserved execution context. + +## Synthetic IV Model + +Update `services/ingest-options/src/adapters/synthetic.ts`. + +Add persistent contract-level IV state: + +```ts +type SyntheticContractIvState = { + iv: number; + pressure: number; + lastTs: number; +}; +``` + +Behavior: + +- Initialize IV from a plausible baseline based on DTE and moneyness. +- Maintain IV per contract across bursts. +- Repeated aggressive buying of the same contract raises pressure and IV. +- Aggressive buying means synthetic placement `A` or `AA`. +- `MID` has small/no pressure. +- `B` or `BB` reduces pressure slightly. +- Pressure decays over time after inactivity. +- IV is clamped to a plausible range. + +Recommended defaults: + +- Baseline IV: `0.18` to `0.65` +- 0DTE contracts start higher than far-dated contracts. +- Out-of-the-money contracts start slightly higher than near-the-money contracts. +- Ask/above-ask print pressure increment: proportional to size and notional. +- Decay half-life: roughly 30-90 seconds in synthetic time. +- Clamp IV to `0.05..2.5`. + +Each synthetic `OptionPrint` should include: + +```ts +execution_iv +execution_iv_source: "synthetic_pressure_model" +``` + +Synthetic NBBO and trade price generation should remain coherent: + +- As IV rises, option mid/ask should drift higher for that contract. +- Rapid same-contract buying should visibly increase both print price and IV over subsequent prints. +- Bid/ask spread may widen mildly with higher IV. + +## Real Adapter IV Behavior + +For Alpaca, Databento, and IBKR in Phase 1: + +- Preserve NBBO and underlying spot context through ingest enrichment. +- Leave `execution_iv` unset unless the adapter already provides a reliable IV value. +- Do not invent IV for real feeds in Phase 1. + +Synthetic is the only source that must generate IV in this phase. + +## Frontend Table Redesign + +Update `apps/web/app/terminal.tsx` and `apps/web/app/globals.css`. + +Each Options row remains an `OptionPrint`. + +Default columns: + +- `TIME` +- `SYM` +- `EXP` +- `STRIKE` +- `C/P` +- `SPOT` +- `DETAILS` +- `TYPE` +- `VALUE` +- `SIDE` +- `IV` +- `CLASSIFIER` + +Column sources: + +- `SPOT`: `execution_underlying_spot`, fallback `--` +- `SIDE`: `execution_nbbo_side ?? nbbo_side` +- `IV`: `execution_iv`, formatted as percent, fallback `--` +- `DETAILS`: `{size}@{price}_{side}` +- `VALUE`: `notional ?? price * size * 100` + +For legacy rows only: + +- If preserved NBBO is missing, fallback to existing frontend NBBO map. +- If preserved spot/IV is missing, render `--`. + +## Classifier Row Coloring + +Add derived indexes in `TerminalProvider`: + +- `classifierHitsByPacketId` +- `packetIdByOptionTraceId` +- `classifierDecorByOptionTraceId` + +A print inherits classifier color if its trace ID belongs to a flow packet that produced classifier hits. + +Primary hit selection: + +1. Highest confidence +2. Newest `source_ts` +3. Highest `seq` + +Classifier families: + +- `large_bullish_call_sweep`: green +- `large_bearish_put_sweep`: red +- `unusual_contract_spike`: amber +- `large_call_sell_overwrite`: copper +- `large_put_sell_write`: copper +- `straddle` / `strangle`: blue +- `vertical_spread`: teal +- `ladder_accumulation`: yellow-green +- `roll_up_down_out`: violet +- `far_dated_conviction`: cyan +- `zero_dte_gamma_punch`: magenta +- unknown: neutral + +Confidence controls row intensity. + +## Interaction + +Classified rows: + +- Click opens existing classifier/alert drawer behavior through `state.openFromClassifierHit(primaryHit)`. +- Keyboard Enter/Space does the same. +- Row remains compact and table-like. + +Unclassified rows: + +- Hover only. +- No drawer action. + +## Live Manifest + +Update `/tape` live subscriptions to include classifier hits: + +```ts +[ + { channel: "options", filters: flowFilters }, + { channel: "nbbo" }, + { channel: "equities" }, + { channel: "flow", filters: flowFilters }, + { channel: "classifier-hits" } +] +``` + +The table uses preserved execution context from options first, not these side feeds. + +## Tests + +Add/update tests for: + +- `OptionPrintSchema` accepts preserved execution context fields. +- ClickHouse option print normalization handles missing legacy context fields. +- Ingest enrichment attaches preserved NBBO context. +- Ingest enrichment attaches preserved underlying quote mid as spot. +- Enrichment never uses quotes after the option print timestamp. +- `nbbo_side` mirrors `execution_nbbo_side` for new enriched prints. +- Synthetic IV increases under repeated same-contract ask/above-ask buying. +- Synthetic IV decays after inactivity. +- Synthetic IV remains within clamps. +- Options table renders SPOT from `execution_underlying_spot`. +- Options table renders IV from `execution_iv`. +- Legacy rows render `--` for missing SPOT/IV. +- Classifier family mapping and primary hit selection work. +- Classified row opens existing classifier/alert drawer path. + +## Acceptance Criteria + +- The Options tape is a dense table, not card rows. +- Every new option print stores preserved execution NBBO context. +- Every new option print stores preserved execution underlying spot when an at-or-before equity quote exists. +- Synthetic option prints store dynamic IV. +- Synthetic repeated buying of the same contract visibly increases IV. +- The table reads NBBO, SPOT, and IV from preserved print fields first. +- Classifier-hit rows are color-coded by classifier family. +- Existing live/replay filters and tape controls still work. +- No context field is visually treated as stale after being attached to the print. +- Legacy data remains readable with graceful fallbacks. + +## Assumptions + +- Phase 1 uses flat fields for queryability and simple table rendering. +- Underlying spot means equity quote mid at or before the option print timestamp. +- NBBO context means option quote at or before the option print timestamp. +- Preserved age fields are audit metadata, not UI freshness warnings. +- Real-feed IV can remain absent until a reliable provider value is available. From b4f87b50d2489c1e4b6e60c178b1529a27a20123 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 02:57:20 -0400 Subject: [PATCH 05/18] Hydrate options feed across live routes --- apps/web/app/terminal.test.ts | 37 +++++++++++++++++++++++++++++++ apps/web/app/terminal.tsx | 41 ++++++++++++++++++++++++++++------- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index 883b9cd..9eb51d0 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -10,6 +10,7 @@ import { getAlertWindowAnchorTs, getOptionTableSnapshot, getLiveFeedStatus, + getLiveManifest, normalizeAlertSeverity, nextFlowFilterPopoverState, projectPausableTapeState, @@ -38,6 +39,42 @@ const makeAlert = (overrides: Record = {}) => ...overrides }) as any; +describe("live manifest", () => { + it("includes options on every live route", () => { + const filters = buildDefaultFlowFilters(); + for (const pathname of ["/", "/tape", "/signals", "/charts", "/replay"]) { + expect( + getLiveManifest(pathname, "SPY", 60000, filters).some( + (subscription) => subscription.channel === "options" + ) + ).toBe(true); + } + }); + + it("dedupes tape options subscription", () => { + const tapeOptionsSubscriptions = getLiveManifest( + "/tape", + "SPY", + 60000, + buildDefaultFlowFilters() + ).filter((subscription) => subscription.channel === "options"); + expect(tapeOptionsSubscriptions).toHaveLength(1); + }); + + it("keeps option filters on baseline subscription", () => { + const filters = { + ...buildDefaultFlowFilters(), + minNotional: 125_000 + }; + + const optionsSubscription = getLiveManifest("/signals", "SPY", 60000, filters).find( + (subscription) => subscription.channel === "options" + ); + + expect(optionsSubscription?.filters).toBe(filters); + }); +}); + describe("live tape pausable helpers", () => { it("queues new items while paused and flushes them on resume", () => { let state = reducePausableTapeData( diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 4a29481..9f56047 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -2185,47 +2185,72 @@ type LiveSessionState = { chartOverlay: EquityPrint[]; }; -const getLiveManifest = ( +const dedupeLiveSubscriptions = (subscriptions: LiveSubscription[]): LiveSubscription[] => { + const seen = new Set(); + return subscriptions.filter((subscription) => { + const key = getLiveSubscriptionKey(subscription); + if (seen.has(key)) { + return false; + } + seen.add(key); + return true; + }); +}; + +export const getLiveManifest = ( pathname: string, chartTicker: string, chartIntervalMs: number, flowFilters: OptionFlowFilters ): LiveSubscription[] => { + const baselineSubs: LiveSubscription[] = [{ channel: "options", filters: flowFilters }]; const chartSubs: LiveSubscription[] = [ { channel: "equity-candles", underlying_id: chartTicker, interval_ms: chartIntervalMs }, { channel: "equity-overlay", underlying_id: chartTicker } ]; if (pathname === "/tape") { - return [ + return dedupeLiveSubscriptions([ + ...baselineSubs, { channel: "options", filters: flowFilters }, { channel: "nbbo" }, { channel: "equities" }, { channel: "flow", filters: flowFilters }, { channel: "classifier-hits" } - ]; + ]); } if (pathname === "/signals") { - return [{ channel: "alerts" }, { channel: "classifier-hits" }, { channel: "inferred-dark" }]; + return dedupeLiveSubscriptions([ + ...baselineSubs, + { channel: "alerts" }, + { channel: "classifier-hits" }, + { channel: "inferred-dark" } + ]); } if (pathname === "/charts") { - return [...chartSubs, { channel: "classifier-hits" }, { channel: "inferred-dark" }]; + return dedupeLiveSubscriptions([ + ...baselineSubs, + ...chartSubs, + { channel: "classifier-hits" }, + { channel: "inferred-dark" } + ]); } if (pathname === "/replay") { - return []; + return baselineSubs; } - return [ + return dedupeLiveSubscriptions([ + ...baselineSubs, { channel: "equities" }, { channel: "flow" }, { channel: "alerts" }, { channel: "classifier-hits" }, { channel: "inferred-dark" }, ...chartSubs - ]; + ]); }; const useLiveSession = ( From ba0daf52084c71661ada724debd74f4c8a3934be Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 03:29:38 -0400 Subject: [PATCH 06/18] Implement server-backed live history --- .env.example | 1 + README.md | 6 +- apps/web/app/globals.css | 24 +++ apps/web/app/terminal.tsx | 209 +++++++++++++++++++ packages/storage/src/clickhouse.ts | 16 ++ packages/storage/tests/equity-quotes.test.ts | 32 +++ packages/types/src/live.ts | 6 +- services/api/src/index.ts | 14 +- services/api/src/live.ts | 54 ++--- services/api/tests/live.test.ts | 84 +++++++- 10 files changed, 402 insertions(+), 44 deletions(-) diff --git a/.env.example b/.env.example index f86691a..8a9ead7 100644 --- a/.env.example +++ b/.env.example @@ -92,6 +92,7 @@ REPLAY_LOG_EVERY=1000 LIVE_LIMIT_OPTIONS=10000 LIVE_LIMIT_NBBO=10000 LIVE_LIMIT_EQUITIES=10000 +LIVE_LIMIT_EQUITY_QUOTES=10000 LIVE_LIMIT_EQUITY_JOINS=10000 LIVE_LIMIT_FLOW=10000 LIVE_LIMIT_CLASSIFIER_HITS=10000 diff --git a/README.md b/README.md index f07916b..b5720fa 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,7 @@ Default `smart-money` policy rejects lower-information prints and keeps high-con | `LIVE_LIMIT_OPTIONS` | `10000` | In-memory/Redis live cache depth for options channel (clamped `1..100000`). | | `LIVE_LIMIT_NBBO` | `10000` | Live cache depth for options NBBO channel (clamped `1..100000`). | | `LIVE_LIMIT_EQUITIES` | `10000` | Live cache depth for equities channel (clamped `1..100000`). | +| `LIVE_LIMIT_EQUITY_QUOTES` | `10000` | Live cache depth for equity quotes channel (clamped `1..100000`). | | `LIVE_LIMIT_EQUITY_JOINS` | `10000` | Live cache depth for equity join channel (clamped `1..100000`). | | `LIVE_LIMIT_FLOW` | `10000` | Live cache depth for flow packet channel (clamped `1..100000`). | | `LIVE_LIMIT_CLASSIFIER_HITS` | `10000` | Live cache depth for classifier hits channel (clamped `1..100000`). | @@ -303,7 +304,10 @@ Default `smart-money` policy rejects lower-information prints and keeps high-con - `view=raw` — audit/debug path that preserves every stored print. - The default Tape page options/packets posture is now stock-only, hides `B` / `BB`, keeps calls and puts visible, and applies in-memory min-notional controls immediately. - Live retention uses a two-tier model: - - API/Redis maintain a bounded hot cache per live generic channel. + - ClickHouse is durable server history; Redis is a bounded hot cache per live generic channel. + - `LIVE_LIMIT_*` controls initial snapshot/hot-cache depth, not total persisted history. + - Browser state is only a rendering window and UI preferences, not a market-data database. + - Devices connected to the same API hydrate from the same server-seen history. - UI keeps a bounded hot window for rendering performance around the signal view rather than raw noise. - Options prints can use a deeper dedicated cap via `NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS` without raising every other feed. - Alert/drawer evidence is pinned and hydrated by id/trace so details remain inspectable after hot-window eviction. diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index 8e2cfca..1dfa32d 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -783,6 +783,30 @@ h3 { white-space: nowrap; } +.load-older { + display: flex; + flex: 0 0 auto; + align-items: center; + justify-content: center; + gap: 10px; + padding: 4px 0 0; + font-size: 0.76rem; + color: var(--muted); +} + +.load-older button { + min-width: 112px; + white-space: nowrap; +} + +.load-older span { + max-width: 260px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--danger); +} + .missed-count { width: 86px; font-size: 0.72rem; diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 9f56047..d720116 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -23,6 +23,7 @@ import type { EquityCandle, EquityPrint, EquityPrintJoin, + EquityQuote, FlowPacket, InferredDarkEvent, LiveServerMessage, @@ -2173,9 +2174,15 @@ type LiveSessionState = { connectedAt: number | null; lastUpdate: number | null; lastEventByChannel: Partial>; + manifest: LiveSubscription[]; + historyCursors: Partial>; + historyLoading: Partial>; + historyErrors: Partial>; + loadOlder: (channel: LiveSubscription["channel"]) => Promise; options: OptionPrint[]; nbbo: OptionNBBO[]; equities: EquityPrint[]; + equityQuotes: EquityQuote[]; equityJoins: EquityPrintJoin[]; flow: FlowPacket[]; classifierHits: ClassifierHitEvent[]; @@ -2185,6 +2192,46 @@ type LiveSessionState = { chartOverlay: EquityPrint[]; }; +type LiveHistoryResponse = { + data: T[]; + next_before: Cursor | null; +}; + +const LIVE_HISTORY_ENDPOINTS: Partial> = { + options: "/history/options", + nbbo: "/history/nbbo", + equities: "/history/equities", + "equity-quotes": "/history/equity-quotes", + "equity-joins": "/history/equity-joins", + flow: "/history/flow", + "classifier-hits": "/history/classifier-hits", + alerts: "/history/alerts", + "inferred-dark": "/history/inferred-dark" +}; + +const appendOptionFlowFilters = (params: URLSearchParams, filters: OptionFlowFilters | undefined): void => { + if (!filters) { + return; + } + if (filters.view) { + params.set("view", filters.view); + } + if (filters.securityTypes?.length === 1) { + params.set("security", filters.securityTypes[0]); + } else if (filters.securityTypes && filters.securityTypes.length > 1) { + params.set("security", "all"); + } + if (filters.nbboSides?.length) { + params.set("side", filters.nbboSides.join(",")); + } + if (filters.optionTypes?.length) { + params.set("type", filters.optionTypes.join(",")); + } + if (typeof filters.minNotional === "number") { + params.set("min_notional", String(filters.minNotional)); + } +}; + const dedupeLiveSubscriptions = (subscriptions: LiveSubscription[]): LiveSubscription[] => { const seen = new Set(); return subscriptions.filter((subscription) => { @@ -2266,9 +2313,13 @@ const useLiveSession = ( const [lastEventByChannel, setLastEventByChannel] = useState< Partial> >({}); + const [historyCursors, setHistoryCursors] = useState>>({}); + const [historyLoading, setHistoryLoading] = useState>>({}); + const [historyErrors, setHistoryErrors] = useState>>({}); const [options, setOptions] = useState([]); const [nbbo, setNbbo] = useState([]); const [equities, setEquities] = useState([]); + const [equityQuotes, setEquityQuotes] = useState([]); const [equityJoins, setEquityJoins] = useState([]); const [flow, setFlow] = useState([]); const [classifierHits, setClassifierHits] = useState([]); @@ -2291,9 +2342,13 @@ const useLiveSession = ( setConnectedAt(null); setLastUpdate(null); setLastEventByChannel({}); + setHistoryCursors({}); + setHistoryLoading({}); + setHistoryErrors({}); setOptions([]); setNbbo([]); setEquities([]); + setEquityQuotes([]); setEquityJoins([]); setFlow([]); setClassifierHits([]); @@ -2347,6 +2402,7 @@ const useLiveSession = ( const subscription = message.op === "snapshot" ? message.snapshot.subscription : message.subscription; const items = message.op === "snapshot" ? message.snapshot.items : [message.item]; + const subscriptionKey = getLiveSubscriptionKey(subscription); const updateAt = Date.now(); const mergeItems = ( @@ -2380,6 +2436,9 @@ const useLiveSession = ( case "equities": mergeItems(setEquities, items as EquityPrint[]); break; + case "equity-quotes": + mergeItems(setEquityQuotes, items as EquityQuote[]); + break; case "equity-joins": mergeItems(setEquityJoins, items as EquityPrintJoin[]); break; @@ -2403,6 +2462,17 @@ const useLiveSession = ( break; } + if (message.op === "snapshot") { + setHistoryCursors((current) => ({ + ...current, + [subscriptionKey]: message.snapshot.next_before + })); + setHistoryErrors((current) => ({ + ...current, + [subscriptionKey]: null + })); + } + if (items.length > 0) { setLastEventByChannel((current) => ({ ...current, @@ -2503,14 +2573,114 @@ const useLiveSession = ( subscribedMapRef.current = nextMap; }, [enabled, manifest]); + const loadOlder = useCallback( + async (channel: LiveSubscription["channel"]) => { + const subscription = manifest.find((candidate) => candidate.channel === channel); + if (!enabled || !subscription) { + return; + } + const endpoint = LIVE_HISTORY_ENDPOINTS[subscription.channel]; + if (!endpoint) { + return; + } + const key = getLiveSubscriptionKey(subscription); + const cursor = historyCursors[key]; + if (!cursor || historyLoading[key]) { + return; + } + + setHistoryLoading((current) => ({ ...current, [key]: true })); + setHistoryErrors((current) => ({ ...current, [key]: null })); + + try { + const params = new URLSearchParams({ + before_ts: String(cursor.ts), + before_seq: String(cursor.seq), + limit: String(subscription.channel === "options" ? 500 : 200) + }); + if (subscription.channel === "options" || subscription.channel === "flow") { + appendOptionFlowFilters(params, subscription.filters); + } + const response = await fetch(buildApiUrl(`${endpoint}?${params.toString()}`)); + if (!response.ok) { + const detail = await readErrorDetail(response); + throw new Error(detail || `HTTP ${response.status}`); + } + const payload = (await response.json()) as LiveHistoryResponse; + const older = payload.data ?? []; + + const mergeOlder = ( + setter: Dispatch>, + limit: number + ) => { + setter((prev) => + mergeNewest(older as T[], prev, limit, (evicted) => + incrementRetentionMetric("hotWindowEvictions", evicted) + ) + ); + }; + + switch (subscription.channel) { + case "options": + mergeOlder(setOptions, LIVE_HOT_WINDOW_OPTIONS); + break; + case "nbbo": + mergeOlder(setNbbo, LIVE_HOT_WINDOW); + break; + case "equities": + mergeOlder(setEquities, LIVE_HOT_WINDOW); + break; + case "equity-quotes": + mergeOlder(setEquityQuotes, LIVE_HOT_WINDOW); + break; + case "equity-joins": + mergeOlder(setEquityJoins, LIVE_HOT_WINDOW); + break; + case "flow": + mergeOlder(setFlow, LIVE_HOT_WINDOW); + break; + case "classifier-hits": + mergeOlder(setClassifierHits, LIVE_HOT_WINDOW); + break; + case "alerts": + mergeOlder(setAlerts, LIVE_HOT_WINDOW); + break; + case "inferred-dark": + mergeOlder(setInferredDark, LIVE_HOT_WINDOW); + break; + } + + setHistoryCursors((current) => ({ + ...current, + [key]: older.length > 0 ? payload.next_before : null + })); + setLastUpdate(Date.now()); + } catch (error) { + setHistoryErrors((current) => ({ + ...current, + [key]: error instanceof Error ? error.message : String(error) + })); + } finally { + setHistoryLoading((current) => ({ ...current, [key]: false })); + } + }, + [enabled, manifest, historyCursors, historyLoading] + ); + return { status, connectedAt, lastUpdate, lastEventByChannel, + manifest, + historyCursors, + historyLoading, + historyErrors, + loadOlder, options, nbbo, equities, + equityQuotes, equityJoins, flow, classifierHits, @@ -2582,6 +2752,39 @@ const TapeControls = ({ paused, onTogglePause, isAtTop, missed, onJump }: TapeCo ); }; +type LoadOlderControlProps = { + channel: LiveSubscription["channel"]; +}; + +const LoadOlderControl = ({ channel }: LoadOlderControlProps) => { + const state = useTerminal(); + const subscription = state.liveSession.manifest.find((candidate) => candidate.channel === channel); + if (state.mode !== "live" || !subscription || !(subscription.channel in LIVE_HISTORY_ENDPOINTS)) { + return null; + } + + const key = getLiveSubscriptionKey(subscription); + const cursor = state.liveSession.historyCursors[key]; + const loading = Boolean(state.liveSession.historyLoading[key]); + const error = state.liveSession.historyErrors[key]; + if (!cursor && !loading && !error) { + return null; + } + + return ( +
+ + {error ? {error} : null} +
+ ); +}; + type CandleChartProps = { ticker: string; intervalMs: number; @@ -5265,6 +5468,7 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { ) : null}
)} + {!limit ? : null}
); @@ -5342,6 +5546,7 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => { {virtual.bottomSpacerHeight > 0 ? (
) : null} + {!limit ? : null} )}
@@ -5481,6 +5686,7 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { {virtual.bottomSpacerHeight > 0 ? (
) : null} + {!limit ? : null} )}
@@ -5576,6 +5782,7 @@ const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => {virtual.bottomSpacerHeight > 0 ? (
) : null} + {!limit ? : null} )}
@@ -5656,6 +5863,7 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { {virtual.bottomSpacerHeight > 0 ? (
) : null} + {!limit ? : null} )}
@@ -5743,6 +5951,7 @@ const DarkPane = ({ limit, className }: DarkPaneProps) => { {virtual.bottomSpacerHeight > 0 ? (
) : null} + {!limit ? : null} )}
diff --git a/packages/storage/src/clickhouse.ts b/packages/storage/src/clickhouse.ts index c53caa4..850b699 100644 --- a/packages/storage/src/clickhouse.ts +++ b/packages/storage/src/clickhouse.ts @@ -1264,6 +1264,22 @@ export const fetchEquityPrintsBefore = async ( return EquityPrintSchema.array().parse(rows.map(normalizeEquityRow)); }; +export const fetchEquityQuotesBefore = async ( + client: ClickHouseClient, + beforeTs: number, + beforeSeq: number, + limit: number +): Promise => { + const safeLimit = clampLimit(limit); + const result = await client.query({ + query: `SELECT * FROM ${EQUITY_QUOTES_TABLE} WHERE ${buildBeforeTupleCondition("ts", "seq", beforeTs, beforeSeq)} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`, + format: "JSONEachRow" + }); + + const rows = await result.json(); + return EquityQuoteSchema.array().parse(rows.map(normalizeEquityQuoteRow)); +}; + export const fetchEquityPrintJoinsBefore = async ( client: ClickHouseClient, beforeTs: number, diff --git a/packages/storage/tests/equity-quotes.test.ts b/packages/storage/tests/equity-quotes.test.ts index bc3917e..23a34f0 100644 --- a/packages/storage/tests/equity-quotes.test.ts +++ b/packages/storage/tests/equity-quotes.test.ts @@ -4,6 +4,7 @@ import { EQUITY_QUOTES_TABLE, normalizeEquityQuote } from "../src/equity-quotes"; +import { fetchEquityQuotesBefore, type ClickHouseClient } from "../src/clickhouse"; const baseQuote = { source_ts: 100, @@ -27,4 +28,35 @@ describe("equity-quotes storage helpers", () => { expect(ddl).toContain(EQUITY_QUOTES_TABLE); expect(ddl).toContain("CREATE TABLE IF NOT EXISTS"); }); + + it("fetches older quotes with tuple cursor ordering", async () => { + let queryText = ""; + const client = { + query: async ({ query }: { query: string }) => { + queryText = query; + return { + async json() { + return [ + { + ...baseQuote, + source_ts: 90, + ingest_ts: 201, + seq: 2, + trace_id: "trace-2", + ts: 90 + } + ] as T; + } + }; + } + } as unknown as ClickHouseClient; + + const rows = await fetchEquityQuotesBefore(client, 100, 3, 25); + + expect(rows).toHaveLength(1); + expect(rows[0]?.trace_id).toBe("trace-2"); + expect(queryText).toContain(EQUITY_QUOTES_TABLE); + expect(queryText).toContain("WHERE (ts, seq) < (100, 3)"); + expect(queryText).toContain("ORDER BY ts DESC, seq DESC LIMIT 25"); + }); }); diff --git a/packages/types/src/live.ts b/packages/types/src/live.ts index 3d86883..da86c86 100644 --- a/packages/types/src/live.ts +++ b/packages/types/src/live.ts @@ -5,6 +5,7 @@ import { EquityCandleSchema, EquityPrintJoinSchema, EquityPrintSchema, + EquityQuoteSchema, FlowPacketSchema, InferredDarkEventSchema, OptionNBBOSchema, @@ -26,6 +27,7 @@ export const LiveGenericChannelSchema = z.enum([ "options", "nbbo", "equities", + "equity-quotes", "equity-joins", "flow", "classifier-hits", @@ -37,6 +39,7 @@ export const LiveChannelSchema = z.enum([ "options", "nbbo", "equities", + "equity-quotes", "equity-joins", "flow", "classifier-hits", @@ -59,7 +62,7 @@ export const LiveSubscriptionSchema = z.discriminatedUnion("channel", [ filters: OptionFlowFiltersSchema.optional() }), z.object({ - channel: z.enum(["nbbo", "equities", "equity-joins", "classifier-hits", "alerts", "inferred-dark"]) + channel: z.enum(["nbbo", "equities", "equity-quotes", "equity-joins", "classifier-hits", "alerts", "inferred-dark"]) }), z.object({ channel: z.literal("equity-candles"), @@ -78,6 +81,7 @@ const livePayloadSchemas = { options: OptionPrintSchema, nbbo: OptionNBBOSchema, equities: EquityPrintSchema, + "equity-quotes": EquityQuoteSchema, "equity-joins": EquityPrintJoinSchema, flow: FlowPacketSchema, "classifier-hits": ClassifierHitEventSchema, diff --git a/services/api/src/index.ts b/services/api/src/index.ts index c8fa667..911c4bf 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -60,6 +60,7 @@ import { fetchEquityPrintsBefore, fetchEquityPrintsRange, fetchEquityPrintJoinsAfter, + fetchEquityQuotesBefore, fetchEquityQuotesAfter, fetchInferredDarkBefore, fetchInferredDarkAfter, @@ -977,19 +978,21 @@ const run = async () => { const fanoutLive = async ( subscription: LiveSubscription, item: unknown, - ingestChannel: "options" | "nbbo" | "equities" | "equity-candles" | "equity-overlay" | "equity-joins" | "flow" | "classifier-hits" | "alerts" | "inferred-dark" + ingestChannel: "options" | "nbbo" | "equities" | "equity-quotes" | "equity-candles" | "equity-overlay" | "equity-joins" | "flow" | "classifier-hits" | "alerts" | "inferred-dark" ) => { + const watermark = await liveState.ingest(ingestChannel, item); + if ( (ingestChannel === "options" || ingestChannel === "nbbo" || ingestChannel === "equities" || + ingestChannel === "equity-quotes" || ingestChannel === "flow") && !isLiveItemFresh(ingestChannel, item) ) { return; } - const watermark = await liveState.ingest(ingestChannel, item); const matchingSubscriptions = subscription.channel === "options" || subscription.channel === "flow" ? [...subscriptionDefinitions.entries()].filter(([, candidate]) => candidate.channel === subscription.channel) @@ -1088,6 +1091,7 @@ const run = async () => { try { const payload = EquityQuoteSchema.parse(equityQuoteSubscription.decode(msg)); broadcast(equityQuoteSockets, { type: "equity-quote", payload }); + await fanoutLive({ channel: "equity-quotes" }, payload, "equity-quotes"); msg.ack(); } catch (error) { logger.error("failed to process equity quote", { @@ -1380,6 +1384,12 @@ const run = async () => { return jsonResponse(buildHistoryResponse(data, (item) => ({ ts: item.ts, seq: item.seq }))); } + if (req.method === "GET" && url.pathname === "/history/equity-quotes") { + const { beforeTs, beforeSeq, limit } = parseBeforeParams(url); + const data = await fetchEquityQuotesBefore(clickhouse, beforeTs, beforeSeq, limit); + return jsonResponse(buildHistoryResponse(data, (item) => ({ ts: item.ts, seq: item.seq }))); + } + if (req.method === "GET" && url.pathname === "/history/equity-joins") { const { beforeTs, beforeSeq, limit } = parseBeforeParams(url); const data = await fetchEquityPrintJoinsBefore(clickhouse, beforeTs, beforeSeq, limit); diff --git a/services/api/src/live.ts b/services/api/src/live.ts index df916fb..f10cb33 100644 --- a/services/api/src/live.ts +++ b/services/api/src/live.ts @@ -5,6 +5,7 @@ import { fetchRecentEquityCandles, fetchRecentEquityPrintJoins, fetchRecentEquityPrints, + fetchRecentEquityQuotes, fetchRecentFlowPackets, fetchRecentInferredDark, fetchRecentOptionNBBO, @@ -18,6 +19,7 @@ import { EquityCandleSchema, EquityPrintJoinSchema, EquityPrintSchema, + EquityQuoteSchema, FeedSnapshot, FlowPacketSchema, InferredDarkEventSchema, @@ -44,6 +46,7 @@ const GENERIC_LIMIT_ENV_KEYS: Record = { options: "LIVE_LIMIT_OPTIONS", nbbo: "LIVE_LIMIT_NBBO", equities: "LIVE_LIMIT_EQUITIES", + "equity-quotes": "LIVE_LIMIT_EQUITY_QUOTES", "equity-joins": "LIVE_LIMIT_EQUITY_JOINS", flow: "LIVE_LIMIT_FLOW", "classifier-hits": "LIVE_LIMIT_CLASSIFIER_HITS", @@ -69,6 +72,7 @@ export const LIVE_FRESHNESS_THRESHOLDS: Partial ({ ts: item.ts, seq: item.seq }), fetchRecent: fetchRecentEquityPrints }, + "equity-quotes": { + redisKey: "live:equity-quotes", + cursorField: "equity-quotes", + limit: limits["equity-quotes"], + parse: (value) => EquityQuoteSchema.parse(value), + cursor: (item) => ({ ts: item.ts, seq: item.seq }), + fetchRecent: fetchRecentEquityQuotes + }, "equity-joins": { redisKey: "live:equity-joins", cursorField: "equity-joins", @@ -251,6 +264,7 @@ const extractFreshnessTs = (channel: LiveGenericChannel, item: any): number | nu case "options": case "nbbo": case "equities": + case "equity-quotes": return typeof item.ts === "number" ? item.ts : null; case "flow": return typeof item.source_ts === "number" ? item.source_ts : null; @@ -275,19 +289,6 @@ export const isLiveItemFresh = ( return now - ts <= thresholdMs; }; -const filterFreshGenericItems = ( - channel: LiveGenericChannel, - items: T[], - now = Date.now() -): T[] => { - const thresholdMs = LIVE_FRESHNESS_THRESHOLDS[channel]; - if (!thresholdMs) { - return items; - } - - return items.filter((item) => isLiveItemFresh(channel, item, now)); -}; - const nextBeforeForItems = (items: T[], cursorOf: (item: T) => Cursor): Cursor | null => { const last = items.at(-1); return last ? cursorOf(last) : null; @@ -396,21 +397,17 @@ export class LiveStateManager { undefined, storageFilters ); - const freshItems = filterFreshGenericItems("options", items); return { subscription, - items: freshItems, + items, watermark: items[0] ? { ts: items[0].ts, seq: items[0].seq } : null, - next_before: nextBeforeForItems(freshItems, (item) => ({ ts: item.ts, seq: item.seq })) + next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq })) }; } const config = this.generic.options; - const items = filterFreshGenericItems( - "options", - (this.genericItems.get("options") ?? []).filter((item) => - matchesOptionPrintFilters(item, subscription.filters) - ) + const items = (this.genericItems.get("options") ?? []).filter((item) => + matchesOptionPrintFilters(item, subscription.filters) ); return { subscription, @@ -421,11 +418,8 @@ export class LiveStateManager { } case "flow": { const config = this.generic.flow; - const items = filterFreshGenericItems( - "flow", - (this.genericItems.get("flow") ?? []).filter((item) => - matchesFlowPacketFilters(item, subscription.filters) - ) + const items = (this.genericItems.get("flow") ?? []).filter((item) => + matchesFlowPacketFilters(item, subscription.filters) ); return { subscription, @@ -464,10 +458,7 @@ export class LiveStateManager { } default: { const config = this.generic[subscription.channel]; - const items = filterFreshGenericItems( - subscription.channel, - this.genericItems.get(subscription.channel) ?? [] - ); + const items = this.genericItems.get(subscription.channel) ?? []; return { subscription, items, @@ -513,9 +504,6 @@ export class LiveStateManager { default: { const config = this.generic[channel]; const parsed = config.parse(item); - if (!isLiveItemFresh(channel, parsed)) { - return this.genericCursors.get(config.cursorField) ?? null; - } const items = this.genericItems.get(channel) ?? []; const next = normalizeGenericItems(channel, [parsed, ...items], config); this.genericItems.set(channel, next); diff --git a/services/api/tests/live.test.ts b/services/api/tests/live.test.ts index 21bcd28..41ad732 100644 --- a/services/api/tests/live.test.ts +++ b/services/api/tests/live.test.ts @@ -58,6 +58,7 @@ describe("LiveStateManager", () => { expect(limits.options).toBe(777); expect(limits.nbbo).toBe(100000); expect(limits.flow).toBe(10000); + expect(limits["equity-quotes"]).toBe(10000); expect(limits.alerts).toBe(10000); }); @@ -145,6 +146,7 @@ describe("LiveStateManager", () => { options: 10000, nbbo: 10000, equities: 10000, + "equity-quotes": 10000, "equity-joins": 10000, flow: 2, "classifier-hits": 10000, @@ -277,7 +279,7 @@ describe("LiveStateManager", () => { expect(flowSnapshot.items).toHaveLength(1); }); - it("suppresses stale items from live snapshots while preserving fresh ones", async () => { + it("keeps stale persisted items in live snapshots", async () => { const manager = new LiveStateManager(makeClickHouse(), null); const now = Date.now(); @@ -383,16 +385,20 @@ describe("LiveStateManager", () => { ]); expect((optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ - "opt-fresh" + "opt-fresh", + "opt-stale" ]); expect((nbboSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ - "nbbo-fresh" + "nbbo-fresh", + "nbbo-stale" ]); expect((equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ - "eq-fresh" + "eq-fresh", + "eq-stale" ]); expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([ - "flow-fresh" + "flow-fresh", + "flow-stale" ]); }); @@ -476,7 +482,7 @@ describe("LiveStateManager", () => { ]); }); - it("rejects stale ingest for freshness-gated channels", async () => { + it("stores older valid ingest for freshness-gated channels", async () => { const manager = new LiveStateManager(makeClickHouse(), null); const now = Date.now(); @@ -494,7 +500,71 @@ describe("LiveStateManager", () => { }); const snapshot = await manager.getSnapshot({ channel: "equities" }); - expect(snapshot.items).toHaveLength(0); + expect(snapshot.items).toHaveLength(1); + expect(snapshot.next_before).toEqual({ ts: now - 60_000, seq: 1 }); + }); + + it("hydrates equity quotes from redis", async () => { + const redis = makeRedis(); + const now = Date.now(); + await redis.lPush( + "live:equity-quotes", + JSON.stringify({ + source_ts: now, + ingest_ts: now + 1, + seq: 1, + trace_id: "quote-1", + ts: now, + underlying_id: "SPY", + bid: 450, + ask: 450.01 + }) + ); + await redis.hSet("live:cursors", "equity-quotes", JSON.stringify({ ts: now, seq: 1 })); + + const manager = new LiveStateManager(makeClickHouse(), redis as never); + await manager.hydrate(); + const snapshot = await manager.getSnapshot({ channel: "equity-quotes" }); + + expect(snapshot.items).toHaveLength(1); + expect(snapshot.watermark).toEqual({ ts: now, seq: 1 }); + expect(snapshot.next_before).toEqual({ ts: now, seq: 1 }); + }); + + it("hydrates equity quotes from clickhouse when redis is empty and persists hot cache", async () => { + const redis = makeRedis(); + const now = Date.now(); + const clickhouse = { + ...makeClickHouse(), + query: async ({ query }: { query: string }) => ({ + async json() { + if (query.includes("equity_quotes")) { + return [ + { + source_ts: now, + ingest_ts: now + 1, + seq: 2, + trace_id: "quote-2", + ts: now, + underlying_id: "SPY", + bid: 451, + ask: 451.01 + } + ] as T; + } + return [] as T; + } + }) + } as ClickHouseClient; + + const manager = new LiveStateManager(clickhouse, redis as never); + await manager.hydrate(); + const snapshot = await manager.getSnapshot({ channel: "equity-quotes" }); + const persisted = await redis.lRange("live:equity-quotes", 0, 10); + + expect(snapshot.items).toHaveLength(1); + expect(snapshot.watermark).toEqual({ ts: now, seq: 2 }); + expect(persisted).toHaveLength(1); }); it("exposes freshness helper for event fanout gating", () => { From c31d59ea79bb3b6b55bede62865e8439e7b95f90 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 04:02:41 -0400 Subject: [PATCH 07/18] Align options table scrolling --- apps/web/app/globals.css | 30 +++++++++++++++++++++--------- apps/web/app/terminal.tsx | 20 +++++++++++--------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index 1dfa32d..64fe95c 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -898,28 +898,33 @@ h3 { } .options-table-wrap { + display: flex; + flex: 1 1 auto; min-height: 0; - overflow: auto; + flex-direction: column; + overflow: hidden; } .options-table { - min-width: 1040px; + display: flex; + min-height: 0; + flex: 1 1 auto; + flex-direction: column; + overflow: hidden; } .options-table-head, .options-table-row { display: grid; - grid-template-columns: 88px 72px 76px 72px 44px 76px 130px 70px 82px 64px 56px minmax(150px, 1fr); + grid-template-columns: minmax(72px, 0.8fr) minmax(50px, 0.55fr) minmax(64px, 0.7fr) minmax(58px, 0.6fr) minmax(34px, 0.35fr) minmax(62px, 0.65fr) minmax(104px, 1fr) minmax(54px, 0.55fr) minmax(66px, 0.7fr) minmax(48px, 0.5fr) minmax(42px, 0.45fr) minmax(92px, 0.9fr); align-items: center; - column-gap: 10px; + column-gap: 8px; } .options-table-head { - position: sticky; - top: 0; - z-index: 2; + flex: 0 0 auto; height: 30px; - padding: 0 10px; + padding: 0 8px; border-bottom: 1px solid var(--border); background: rgba(8, 11, 16, 0.98); color: var(--muted); @@ -928,10 +933,17 @@ h3 { letter-spacing: 0.08em; } +.options-table-body { + flex: 1 1 auto; + min-height: 0; + overflow-y: auto; + overflow-x: hidden; +} + .options-table-row { width: 100%; min-height: 34px; - padding: 0 10px; + padding: 0 8px; border: 0; border-bottom: 1px solid rgba(255, 255, 255, 0.055); background: diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index d720116..1092460 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -5369,7 +5369,7 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { /> } > -
+
{items.length === 0 ? (
{state.tickerSet.size > 0 @@ -5396,10 +5396,11 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { IV CLASSIFIER
- {virtual.topSpacerHeight > 0 ? ( -
- ) : null} - {virtual.visibleItems.map((print) => { +
+ {virtual.topSpacerHeight > 0 ? ( +
+ ) : null} + {virtual.visibleItems.map((print) => { const contractId = normalizeContractId(print.option_contract_id); const parsed = parseOptionContractId(contractId); const contractDisplay = formatOptionContractLabel(contractId); @@ -5462,10 +5463,11 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { {cells}
); - })} - {virtual.bottomSpacerHeight > 0 ? ( -
- ) : null} + })} + {virtual.bottomSpacerHeight > 0 ? ( +
+ ) : null} +
)} {!limit ? : null} From 5fcdb015c0712e849b6a382e7ff52fc37099cc1e Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 04:16:42 -0400 Subject: [PATCH 08/18] Make tape feeds compact tables --- apps/web/app/globals.css | 186 +++++++++++++++++++++ apps/web/app/terminal.tsx | 339 ++++++++++++++++++++------------------ 2 files changed, 363 insertions(+), 162 deletions(-) diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index 64fe95c..9ea6697 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -897,6 +897,7 @@ h3 { background: linear-gradient(180deg, rgba(245, 166, 35, 0.07), rgba(255, 255, 255, 0.018)); } +.data-table-shell, .options-table-wrap { display: flex; flex: 1 1 auto; @@ -905,6 +906,191 @@ h3 { overflow: hidden; } +.data-table-wrap { + flex: 1 1 auto; + min-height: 0; + overflow: auto; + border-top: 1px solid var(--border); + border-bottom: 1px solid var(--border); + background: rgba(5, 8, 12, 0.42); +} + +.data-table { + display: block; + min-width: 980px; +} + +.data-table-options { + min-width: 1280px; +} + +.data-table-equities { + min-width: 660px; +} + +.data-table-flow { + min-width: 1260px; +} + +.data-table-alerts { + min-width: 900px; +} + +.data-table-classifier { + min-width: 760px; +} + +.data-table-dark { + min-width: 820px; +} + +.data-table-head, +.data-table-row { + display: grid; + align-items: center; + column-gap: 8px; +} + +.data-table-head { + position: sticky; + top: 0; + z-index: 2; + min-height: 30px; + padding: 0 10px; + border-bottom: 1px solid rgba(255, 255, 255, 0.095); + background: rgba(8, 11, 16, 0.98); + color: var(--text-faint); + font-size: 0.64rem; + font-weight: 700; + letter-spacing: 0.08em; +} + +.data-table-row { + width: 100%; + min-height: 40px; + padding: 0 10px; + border: 0; + border-bottom: 1px solid rgba(255, 255, 255, 0.055); + background: rgba(255, 255, 255, 0.008); + color: inherit; + font: inherit; + text-align: left; +} + +.data-table-row:nth-child(even) { + background: rgba(255, 255, 255, 0.022); +} + +.data-table-row:hover, +.data-table-row:focus-visible { + outline: none; + background: rgba(245, 166, 35, 0.055); +} + +.data-table-row-button { + cursor: pointer; +} + +.data-table-row-options { + min-height: 36px; +} + +.data-table-row-equities { + min-height: 34px; +} + +.data-table-row-flow, +.data-table-row-alerts, +.data-table-row-classifier, +.data-table-row-dark { + min-height: 44px; +} + +.data-table-row-classified { + background: + linear-gradient(90deg, rgba(var(--classifier-rgb, 192, 200, 210), calc(0.02 + var(--classifier-intensity, 0) * 0.12)), transparent 62%), + rgba(255, 255, 255, 0.008); +} + +.data-table-row-classified:hover, +.data-table-row-classified:focus-visible { + background: + linear-gradient(90deg, rgba(var(--classifier-rgb, 192, 200, 210), calc(0.04 + var(--classifier-intensity, 0) * 0.18)), transparent 68%), + rgba(245, 166, 35, 0.04); +} + +.data-table-row-classified.is-classified { + border-left: 3px solid rgba(var(--classifier-rgb), calc(0.35 + var(--classifier-intensity) * 0.45)); + padding-left: 7px; +} + +.data-table-row-warn, +.data-table-row-severity-high, +.data-table-row-direction-bearish { + border-left: 3px solid rgba(255, 107, 95, 0.58); + padding-left: 7px; +} + +.data-table-row-severity-medium, +.data-table-row-direction-neutral { + border-left: 3px solid rgba(77, 163, 255, 0.46); + padding-left: 7px; +} + +.data-table-row-severity-low, +.data-table-row-direction-bullish { + border-left: 3px solid rgba(37, 193, 122, 0.5); + padding-left: 7px; +} + +.data-table-options .data-table-head, +.data-table-options .data-table-row { + grid-template-columns: minmax(72px, 0.8fr) minmax(50px, 0.55fr) minmax(64px, 0.7fr) minmax(58px, 0.6fr) minmax(34px, 0.35fr) minmax(62px, 0.65fr) minmax(104px, 1fr) minmax(54px, 0.55fr) minmax(66px, 0.7fr) minmax(48px, 0.5fr) minmax(42px, 0.45fr) minmax(92px, 0.9fr); +} + +.data-table-equities .data-table-head, +.data-table-equities .data-table-row { + grid-template-columns: minmax(76px, 0.9fr) minmax(70px, 0.8fr) minmax(76px, 0.8fr) minmax(70px, 0.75fr) minmax(80px, 0.8fr) minmax(76px, 0.75fr); +} + +.data-table-flow .data-table-head, +.data-table-flow .data-table-row { + grid-template-columns: minmax(148px, 1.1fr) minmax(180px, 1.4fr) minmax(62px, 0.45fr) minmax(70px, 0.5fr) minmax(88px, 0.7fr) minmax(74px, 0.55fr) minmax(132px, 1fr) minmax(110px, 0.8fr) minmax(210px, 1.6fr); +} + +.data-table-alerts .data-table-head, +.data-table-alerts .data-table-row { + grid-template-columns: minmax(76px, 0.75fr) minmax(170px, 1.4fr) minmax(52px, 0.45fr) minmax(58px, 0.45fr) minmax(52px, 0.4fr) minmax(66px, 0.55fr) minmax(260px, 2fr); +} + +.data-table-classifier .data-table-head, +.data-table-classifier .data-table-row { + grid-template-columns: minmax(76px, 0.75fr) minmax(180px, 1.45fr) minmax(70px, 0.6fr) minmax(74px, 0.65fr) minmax(300px, 2.2fr); +} + +.data-table-dark .data-table-head, +.data-table-dark .data-table-row { + grid-template-columns: minmax(76px, 0.75fr) minmax(170px, 1.35fr) minmax(76px, 0.65fr) minmax(74px, 0.65fr) minmax(74px, 0.65fr) minmax(260px, 2fr); +} + +.data-table-cell { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: 0.72rem; +} + +.data-table-cell-number { + font-family: var(--font-mono), monospace; + font-variant-numeric: tabular-nums; +} + +.data-table-spacer { + min-width: 100%; + pointer-events: none; +} + .options-table { display: flex; min-height: 0; diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 1092460..09b4d18 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -5343,7 +5343,7 @@ type OptionsPaneProps = { const OptionsPane = ({ limit }: OptionsPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredOptions.slice(0, limit) : state.filteredOptions; - const virtual = useVirtualList(items, state.optionsScroll.listRef, !limit, 34); + const virtual = useVirtualList(items, state.optionsScroll.listRef, !limit, 36); return ( { /> } > -
+
{items.length === 0 ? (
{state.tickerSet.size > 0 @@ -5381,24 +5381,24 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."}
) : ( -
-
- TIME - SYM - EXP - STRIKE - C/P - SPOT - DETAILS - TYPE - VALUE - SIDE - IV - CLASSIFIER -
-
+
+
+
+ TIME + SYM + EXP + STRIKE + C/P + SPOT + DETAILS + TYPE + VALUE + SIDE + IV + CLASSIFIER +
{virtual.topSpacerHeight > 0 ? ( -
+
) : null} {virtual.visibleItems.map((print) => { const contractId = normalizeContractId(print.option_contract_id); @@ -5415,31 +5415,31 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { const iv = print.execution_iv; const decor = state.classifierDecorByOptionTraceId.get(print.trace_id); const commonProps = { - className: `options-table-row${decor ? ` is-classified classifier-${decor.tone}` : ""}`, + className: `data-table-row data-table-row-button data-table-row-classified data-table-row-options${decor ? ` is-classified classifier-${decor.tone}` : ""}`, style: decor ? ({ "--classifier-intensity": decor.intensity } as CSSProperties) : undefined }; const cells = ( <> - {formatTime(print.ts)} - {contractDisplay?.ticker ?? parsed?.root ?? formatContractLabel(contractId)} - {contractDisplay?.expiration ?? parsed?.expiry ?? "--"} - {contractDisplay?.strike.replace(/[CP]$/, "") ?? "--"} - {parsed?.right ?? contractDisplay?.strike.slice(-1) ?? "--"} - {typeof spot === "number" ? formatPrice(spot) : "--"} - + {formatTime(print.ts)} + {contractDisplay?.ticker ?? parsed?.root ?? formatContractLabel(contractId)} + {contractDisplay?.expiration ?? parsed?.expiry ?? "--"} + {contractDisplay?.strike.replace(/[CP]$/, "") ?? "--"} + {parsed?.right ?? contractDisplay?.strike.slice(-1) ?? "--"} + {typeof spot === "number" ? formatPrice(spot) : "--"} + {formatSize(print.size)}@{formatPrice(print.price)}_{nbboSide ?? "--"} - {print.option_type ?? "--"} - ${formatCompactUsd(notional)} - + {print.option_type ?? "--"} + ${formatCompactUsd(notional)} + {nbboSide ? ( {nbboSide} ) : ( "--" )} - {typeof iv === "number" ? formatPct(iv) : "--"} - {decor ? humanizeClassifierId(decor.family) : "--"} + {typeof iv === "number" ? formatPct(iv) : "--"} + {decor ? humanizeClassifierId(decor.family) : "--"} ); @@ -5465,7 +5465,7 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { ); })} {virtual.bottomSpacerHeight > 0 ? ( -
+
) : null}
@@ -5483,7 +5483,7 @@ type EquitiesPaneProps = { const EquitiesPane = ({ limit }: EquitiesPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredEquities.slice(0, limit) : state.filteredEquities; - const virtual = useVirtualList(items, state.equitiesScroll.listRef, !limit, 78); + const virtual = useVirtualList(items, state.equitiesScroll.listRef, !limit, 36); return ( { /> } > -
+
{items.length === 0 ? (
{state.tickerSet.size > 0 @@ -5523,34 +5523,36 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."}
) : ( - <> +
+
+
+ TIME + SYM + PRICE + SIZE + VENUE + TAPE +
{virtual.topSpacerHeight > 0 ? ( -
+
) : null} {virtual.visibleItems.map((print) => ( -
-
-
{print.underlying_id}
-
- ${formatPrice(print.price)} - {formatSize(print.size)}x - {print.exchange} - {print.offExchangeFlag ? ( - Off-Ex - ) : ( - Lit - )} -
-
-
{formatTime(print.ts)}
+
+ {formatTime(print.ts)} + {print.underlying_id} + ${formatPrice(print.price)} + {formatSize(print.size)}x + {print.exchange} + {print.offExchangeFlag ? "Off-Ex" : "Lit"}
))} {virtual.bottomSpacerHeight > 0 ? ( -
+
) : null} - {!limit ? : null} - +
+
)} + {!limit ? : null}
); @@ -5564,7 +5566,7 @@ type FlowPaneProps = { const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredFlow.slice(0, limit) : state.filteredFlow; - const virtual = useVirtualList(items, state.flowScroll.listRef, !limit, 104); + const virtual = useVirtualList(items, state.flowScroll.listRef, !limit, 44); return ( { /> } > -
+
{items.length === 0 ? (
{state.tickerSet.size > 0 @@ -5602,9 +5604,21 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."}
) : ( - <> +
+
+
+ TIME + CONTRACT + PRINTS + SIZE + NOTIONAL + WINDOW + STRUCTURE + NBBO + QUALITY +
{virtual.topSpacerHeight > 0 ? ( -
+
) : null} {virtual.visibleItems.map((packet) => { const features = packet.features ?? {}; @@ -5638,59 +5652,46 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => { const nbboAge = parseNumber(packet.join_quality.nbbo_age_ms, Number.NaN); const nbboStale = parseNumber(packet.join_quality.nbbo_stale, 0) > 0; const nbboMissing = parseNumber(packet.join_quality.nbbo_missing, 0) > 0; + const structureLabel = structureType + ? `${structureType.replace(/_/g, " ")}${structureRights ? ` ${structureRights}` : ""}${structureLegs > 0 ? ` ${structureLegs}L` : ""}${structureStrikes > 0 ? ` ${structureStrikes}K` : ""}` + : "--"; + const nbboLabel = Number.isFinite(nbboBid) && Number.isFinite(nbboAsk) + ? `${formatPrice(nbboBid)} x ${formatPrice(nbboAsk)}` + : Number.isFinite(nbboMid) + ? `Mid ${formatPrice(nbboMid)}` + : "--"; + const qualityLabel = [ + Number.isFinite(aggressiveCoverage) && aggressiveCoverage > 0 + ? `Agg ${formatPct(aggressiveBuyRatio)}/${formatPct(aggressiveSellRatio)} ${formatPct(aggressiveCoverage)} cov` + : null, + Number.isFinite(insideRatio) && insideRatio > 0 ? `In ${formatPct(insideRatio)}` : null, + Number.isFinite(nbboSpread) ? `Spr ${formatPrice(nbboSpread)}` : null, + Number.isFinite(nbboAge) ? `${Math.round(nbboAge)}ms` : null, + nbboStale ? "Stale" : null, + nbboMissing ? "Missing" : null + ].filter(Boolean).join(" | "); return ( -
-
-
{contract}
-
- {formatFlowMetric(count)} prints - {formatFlowMetric(totalSize)} size - Notional ${formatUsd(notional)} - {windowMs > 0 ? {formatFlowMetric(windowMs, "ms")} : null} - {structureType ? ( - - {structureType.replace(/_/g, " ")} - {structureRights ? ` ${structureRights}` : ""} - {structureLegs > 0 ? ` ${structureLegs}L` : ""} - {structureStrikes > 0 ? ` ${structureStrikes}K` : ""} - - ) : null} - {Number.isFinite(aggressiveCoverage) && aggressiveCoverage > 0 ? ( - - Agg {formatPct(aggressiveBuyRatio)} / {formatPct(aggressiveSellRatio)} - {Number.isFinite(insideRatio) && insideRatio > 0 - ? ` · In ${formatPct(insideRatio)}` - : ""} - {` · ${formatPct(aggressiveCoverage)} cov`} - - ) : null} - {Number.isFinite(nbboBid) && Number.isFinite(nbboAsk) ? ( - - NBBO ${formatPrice(nbboBid)} x ${formatPrice(nbboAsk)} - - ) : null} - {Number.isFinite(nbboMid) ? Mid ${formatPrice(nbboMid)} : null} - {Number.isFinite(nbboSpread) ? ( - Spread ${formatPrice(nbboSpread)} - ) : null} - {Number.isFinite(nbboAge) ? {Math.round(nbboAge)}ms : null} - {nbboStale ? NBBO stale : null} - {nbboMissing ? NBBO missing : null} -
-
-
- {formatTime(startTs)} → {formatTime(endTs)} -
+
+ {formatTime(startTs)} → {formatTime(endTs)} + {contract} + {formatFlowMetric(count)} + {formatFlowMetric(totalSize)} + ${formatUsd(notional)} + {windowMs > 0 ? formatFlowMetric(windowMs, "ms") : "--"} + {structureLabel} + {nbboLabel} + {qualityLabel || "--"}
); })} {virtual.bottomSpacerHeight > 0 ? ( -
+
) : null} - {!limit ? : null} - +
+
)} + {!limit ? : null}
); @@ -5705,7 +5706,7 @@ type AlertsPaneProps = { const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredAlerts.slice(0, limit) : state.filteredAlerts; - const virtual = useVirtualList(items, state.alertsScroll.listRef, !limit, 92); + const virtual = useVirtualList(items, state.alertsScroll.listRef, !limit, 46); return ( } > {withStrip ? : null} -
+
{items.length === 0 ? (
{state.tickerSet.size > 0 @@ -5743,9 +5744,19 @@ const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => : "Replay queue empty. Ensure ClickHouse has data."}
) : ( - <> +
+
+
+ TIME + ALERT + SEV + SCORE + HITS + DIR + NOTE +
{virtual.topSpacerHeight > 0 ? ( -
+
) : null} {virtual.visibleItems.map((alert) => { const primary = alert.hits[0]; @@ -5754,7 +5765,7 @@ const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) => return ( ); })} {virtual.bottomSpacerHeight > 0 ? ( -
+
) : null} - {!limit ? : null} - +
+
)} + {!limit ? : null}
); @@ -5800,7 +5804,7 @@ type ClassifierPaneProps = { const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredClassifierHits.slice(0, limit) : state.filteredClassifierHits; - const virtual = useVirtualList(items, state.classifierScroll.listRef, !limit, 88); + const virtual = useVirtualList(items, state.classifierScroll.listRef, !limit, 44); return ( { /> } > -
+
{items.length === 0 ? (
{state.tickerSet.size > 0 @@ -5837,37 +5841,42 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."}
) : ( - <> +
+
+
+ TIME + RULE + DIR + CONF + NOTE +
{virtual.topSpacerHeight > 0 ? ( -
+
) : null} {virtual.visibleItems.map((hit) => { const direction = normalizeDirection(hit.direction); return ( ); })} {virtual.bottomSpacerHeight > 0 ? ( -
+
) : null} - {!limit ? : null} - +
+
)} + {!limit ? : null}
); @@ -5881,7 +5890,7 @@ type DarkPaneProps = { const DarkPane = ({ limit, className }: DarkPaneProps) => { const state = useTerminal(); const items = limit ? state.filteredInferredDark.slice(0, limit) : state.filteredInferredDark; - const virtual = useVirtualList(items, state.darkScroll.listRef, !limit, 88); + const virtual = useVirtualList(items, state.darkScroll.listRef, !limit, 44); return ( { /> } > -
+
{items.length === 0 ? (
{state.tickerSet.size > 0 @@ -5918,9 +5927,18 @@ const DarkPane = ({ limit, className }: DarkPaneProps) => { : "Replay queue empty. Ensure ClickHouse has data."}
) : ( - <> +
+
+
+ TIME + TYPE + SYM + CONF + EVIDENCE + NOTE +
{virtual.topSpacerHeight > 0 ? ( -
+
) : null} {virtual.visibleItems.map((event) => { const underlying = inferDarkUnderlying(event, state.equityPrintMap, state.equityJoinMap); @@ -5928,7 +5946,7 @@ const DarkPane = ({ limit, className }: DarkPaneProps) => { return ( ); })} {virtual.bottomSpacerHeight > 0 ? ( -
+
) : null} - {!limit ? : null} - +
+
)} + {!limit ? : null}
); From 820681f7b68e1abc179c142bf8288a66989656b7 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 04:34:00 -0400 Subject: [PATCH 09/18] Reconnect idle live tape socket --- apps/web/app/terminal.tsx | 55 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 09b4d18..23f2fd6 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -97,6 +97,15 @@ const CANDLE_INTERVALS = [ { label: "1m", ms: 60000 }, { label: "5m", ms: 300000 } ]; +const LIVE_SESSION_IDLE_RECONNECT_MS = 12_000; +const LIVE_SESSION_IDLE_CHECK_MS = 3_000; +const LIVE_SESSION_HOT_CHANNELS = new Set([ + "options", + "nbbo", + "equities", + "flow", + "equity-overlay" +]); type CandlestickSeries = ReturnType; @@ -2329,6 +2338,9 @@ const useLiveSession = ( const [chartOverlay, setChartOverlay] = useState([]); const socketRef = useRef(null); const reconnectRef = useRef(null); + const idleWatchdogRef = useRef(null); + const connectedAtRef = useRef(null); + const lastEventAtRef = useRef(null); const subscribedKeysRef = useRef>(new Set()); const subscribedMapRef = useRef>(new Map()); const manifest = useMemo( @@ -2366,6 +2378,12 @@ const useLiveSession = ( window.clearTimeout(reconnectRef.current); reconnectRef.current = null; } + if (idleWatchdogRef.current !== null) { + window.clearInterval(idleWatchdogRef.current); + idleWatchdogRef.current = null; + } + connectedAtRef.current = null; + lastEventAtRef.current = null; return; } @@ -2474,6 +2492,7 @@ const useLiveSession = ( } if (items.length > 0) { + lastEventAtRef.current = updateAt; setLastEventByChannel((current) => ({ ...current, [subscription.channel]: updateAt @@ -2496,7 +2515,10 @@ const useLiveSession = ( return; } setStatus("connected"); - setConnectedAt(Date.now()); + const now = Date.now(); + setConnectedAt(now); + connectedAtRef.current = now; + lastEventAtRef.current = null; syncSubscriptions(socket); }; @@ -2518,6 +2540,8 @@ const useLiveSession = ( } setStatus("disconnected"); setConnectedAt(null); + connectedAtRef.current = null; + lastEventAtRef.current = null; subscribedKeysRef.current = new Set(); subscribedMapRef.current = new Map(); reconnectRef.current = window.setTimeout(connect, 1000); @@ -2529,14 +2553,43 @@ const useLiveSession = ( } setStatus("disconnected"); setConnectedAt(null); + connectedAtRef.current = null; + lastEventAtRef.current = null; socket.close(); }; }; connect(); + idleWatchdogRef.current = window.setInterval(() => { + if (!active) { + return; + } + const socket = socketRef.current; + if (!socket || socket.readyState !== WebSocket.OPEN) { + return; + } + const hasHotSubscription = Array.from(subscribedMapRef.current.values()).some((sub) => + LIVE_SESSION_HOT_CHANNELS.has(sub.channel) + ); + if (!hasHotSubscription) { + return; + } + const baseline = lastEventAtRef.current ?? connectedAtRef.current; + if (baseline === null) { + return; + } + if (Date.now() - baseline >= LIVE_SESSION_IDLE_RECONNECT_MS) { + console.warn("Live socket idle; reconnecting"); + socket.close(); + } + }, LIVE_SESSION_IDLE_CHECK_MS); return () => { active = false; + if (idleWatchdogRef.current !== null) { + window.clearInterval(idleWatchdogRef.current); + idleWatchdogRef.current = null; + } if (reconnectRef.current !== null) { window.clearTimeout(reconnectRef.current); } From 85dfebb8f00a4144add7ef039981a38662b2df65 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 04:40:35 -0400 Subject: [PATCH 10/18] Disable static caching for live terminal routes --- apps/web/app/charts/page.tsx | 2 ++ apps/web/app/page.tsx | 2 ++ apps/web/app/replay/page.tsx | 2 ++ apps/web/app/signals/page.tsx | 2 ++ apps/web/app/tape/page.tsx | 2 ++ 5 files changed, 10 insertions(+) diff --git a/apps/web/app/charts/page.tsx b/apps/web/app/charts/page.tsx index b1b8870..a2eb858 100644 --- a/apps/web/app/charts/page.tsx +++ b/apps/web/app/charts/page.tsx @@ -1,5 +1,7 @@ import { ChartsRoute } from "../terminal"; +export const dynamic = "force-dynamic"; + export default function Page() { return ; } diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx index a6807b8..326a63d 100644 --- a/apps/web/app/page.tsx +++ b/apps/web/app/page.tsx @@ -1,5 +1,7 @@ import { OverviewRoute } from "./terminal"; +export const dynamic = "force-dynamic"; + export default function Page() { return ; } diff --git a/apps/web/app/replay/page.tsx b/apps/web/app/replay/page.tsx index 2044bee..fbf4635 100644 --- a/apps/web/app/replay/page.tsx +++ b/apps/web/app/replay/page.tsx @@ -1,5 +1,7 @@ import { ReplayRoute } from "../terminal"; +export const dynamic = "force-dynamic"; + export default function Page() { return ; } diff --git a/apps/web/app/signals/page.tsx b/apps/web/app/signals/page.tsx index e510e26..a33ddfa 100644 --- a/apps/web/app/signals/page.tsx +++ b/apps/web/app/signals/page.tsx @@ -1,5 +1,7 @@ import { SignalsRoute } from "../terminal"; +export const dynamic = "force-dynamic"; + export default function Page() { return ; } diff --git a/apps/web/app/tape/page.tsx b/apps/web/app/tape/page.tsx index 344ada0..a692698 100644 --- a/apps/web/app/tape/page.tsx +++ b/apps/web/app/tape/page.tsx @@ -1,5 +1,7 @@ import { TapeRoute } from "../terminal"; +export const dynamic = "force-dynamic"; + export default function Page() { return ; } From b88ef2b371f34e6b595ca10b3b0552c370125468 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 04:59:09 -0400 Subject: [PATCH 11/18] Stream delayed live feed events --- services/api/src/index.ts | 11 ++--------- services/api/src/live.ts | 2 ++ services/api/tests/live.test.ts | 15 +++++++++++++-- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/services/api/src/index.ts b/services/api/src/index.ts index 911c4bf..e898e45 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -100,7 +100,7 @@ import { } from "@islandflow/types"; import { createClient } from "redis"; import { z } from "zod"; -import { LiveStateManager, isLiveItemFresh } from "./live"; +import { LiveStateManager, shouldFanoutLiveEvent } from "./live"; const service = "api"; const logger = createLogger({ service }); @@ -982,14 +982,7 @@ const run = async () => { ) => { const watermark = await liveState.ingest(ingestChannel, item); - if ( - (ingestChannel === "options" || - ingestChannel === "nbbo" || - ingestChannel === "equities" || - ingestChannel === "equity-quotes" || - ingestChannel === "flow") && - !isLiveItemFresh(ingestChannel, item) - ) { + if (!shouldFanoutLiveEvent(ingestChannel, item)) { return; } diff --git a/services/api/src/live.ts b/services/api/src/live.ts index f10cb33..c15774f 100644 --- a/services/api/src/live.ts +++ b/services/api/src/live.ts @@ -289,6 +289,8 @@ export const isLiveItemFresh = ( return now - ts <= thresholdMs; }; +export const shouldFanoutLiveEvent = (_channel: LiveChannel, _item: unknown): boolean => true; + const nextBeforeForItems = (items: T[], cursorOf: (item: T) => Cursor): Cursor | null => { const last = items.at(-1); return last ? cursorOf(last) : null; diff --git a/services/api/tests/live.test.ts b/services/api/tests/live.test.ts index 41ad732..784fafd 100644 --- a/services/api/tests/live.test.ts +++ b/services/api/tests/live.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from "bun:test"; import type { ClickHouseClient } from "@islandflow/storage"; -import { LiveStateManager, isLiveItemFresh, resolveGenericLiveLimits } from "../src/live"; +import { + LiveStateManager, + isLiveItemFresh, + resolveGenericLiveLimits, + shouldFanoutLiveEvent +} from "../src/live"; const makeClickHouse = (): ClickHouseClient => ({ @@ -567,9 +572,15 @@ describe("LiveStateManager", () => { expect(persisted).toHaveLength(1); }); - it("exposes freshness helper for event fanout gating", () => { + it("exposes freshness helper for feed status", () => { expect(isLiveItemFresh("options", { ts: 1000 }, 1010)).toBe(true); expect(isLiveItemFresh("options", { ts: 1000 }, 20_001)).toBe(false); expect(isLiveItemFresh("equity-joins", { source_ts: 1 }, 1_000_000)).toBe(true); }); + + it("fans out stale live events so delayed data remains visible without refresh", () => { + expect(shouldFanoutLiveEvent("options", { ts: 1000 })).toBe(true); + expect(shouldFanoutLiveEvent("equities", { ts: 1000 })).toBe(true); + expect(shouldFanoutLiveEvent("flow", { source_ts: 1000 })).toBe(true); + }); }); From f28c8e641f77574a59c76b8f91dd0209c0361609 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 05:13:07 -0400 Subject: [PATCH 12/18] Fix seen-key handling in pausable tape reduction - Break on the first previously seen item in newest-first merges - Avoid cloning seen key sets unless new items are added - Add tape overhaul phase 1 notes --- apps/web/app/terminal.tsx | 16 ++-- tape-overhaul-phase1-1.md | 170 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 tape-overhaul-phase1-1.md diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 23f2fd6..bf87281 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -409,13 +409,19 @@ export const reducePausableTapeData = ( return current; } - const nextSeenKeys = new Set(current.seenKeys); + const seenKeys = current.seenKeys; + let nextSeenKeys: Set | null = null; const unseen: T[] = []; + // Incoming items are maintained newest-first by mergeNewest. + // Once we hit a previously seen key, the remainder is older history. for (const item of incoming) { const key = getTapeItemKey(item); - if (nextSeenKeys.has(key)) { - continue; + if (seenKeys.has(key)) { + break; + } + if (!nextSeenKeys) { + nextSeenKeys = new Set(seenKeys); } nextSeenKeys.add(key); unseen.push(item); @@ -431,7 +437,7 @@ export const reducePausableTapeData = ( queued: mergeNewest(unseen, current.queued, retentionLimit, (evicted) => incrementRetentionMetric("hotWindowEvictions", evicted) ), - seenKeys: nextSeenKeys, + seenKeys: nextSeenKeys ?? seenKeys, dropped: current.dropped + unseen.length }; } @@ -442,7 +448,7 @@ export const reducePausableTapeData = ( incrementRetentionMetric("hotWindowEvictions", evicted) ), queued: [], - seenKeys: nextSeenKeys, + seenKeys: nextSeenKeys ?? seenKeys, dropped: 0 }; }; diff --git a/tape-overhaul-phase1-1.md b/tape-overhaul-phase1-1.md new file mode 100644 index 0000000..c2a1016 --- /dev/null +++ b/tape-overhaul-phase1-1.md @@ -0,0 +1,170 @@ +# Server-Backed Persistent History + +## Summary + +Make live mode server-authoritative across refreshes, sessions, and devices. The browser will not own data persistence. On load, the app will hydrate from ClickHouse-backed server history, then layer live WebSocket updates on top. Users will immediately see a substantial recent persisted window, with older records available through history pagination. + +## Chosen Defaults + +- Source of truth: ClickHouse on the server. +- Browser persistence: UI preferences only, no market-data cache. +- Initial load: recent persisted window per active channel. +- Older data: fetched on demand using cursor pagination. +- Scope: every channel the server handles, including options, NBBO, equities, equity quotes, equity joins, flow packets, classifier hits, alerts, inferred dark events, candles, and chart overlays. +- Freshness: freshness affects status labels only; it must not hide persisted history from a refreshed browser. + +## Current State To Change + +- `LiveStateManager` hydrates from Redis or ClickHouse, but freshness gates currently suppress stale options, NBBO, equities, and flow snapshots. +- The unified `/ws/live` protocol supports snapshots and `next_before`, but the frontend does not retain/use per-channel history cursors for live-mode pagination. +- Some channels have REST history endpoints, but `equity-quotes` is not fully represented in the unified live protocol/history API. +- Charts already query ClickHouse for candle and overlay ranges, but should be treated as part of the same server-history model. + +## Public Interfaces And Types + +Update `packages/types/src/live.ts`: + +- Add `"equity-quotes"` to: + - `LiveGenericChannelSchema` + - `LiveChannelSchema` + - `LiveSubscriptionSchema` + - `livePayloadSchemas` +- Preserve existing `FeedSnapshot` shape: + - `items` + - `watermark` + - `next_before` + +Update API routes in `services/api/src/index.ts`: + +- Add `GET /history/equity-quotes?before_ts=&before_seq=&limit=`. +- Include `equity-quotes` in `/ws/live` subscriptions and fanout. +- Keep existing recent/replay endpoints compatible. + +Update storage in `packages/storage/src/clickhouse.ts`: + +- Add `fetchEquityQuotesBefore`. +- Reuse existing `(ts, seq)` cursor ordering. +- Keep limits clamped consistently with other history endpoints. + +## Server Implementation + +In `services/api/src/live.ts`: + +1. Add generic config for `equity-quotes`: + - Redis key: `live:equity-quotes` + - cursor field: `equity-quotes` + - parser: `EquityQuoteSchema` + - cursor: `{ ts, seq }` + - fetchRecent: `fetchRecentEquityQuotes` +2. Stop filtering historical snapshots by freshness: + - Remove `filterFreshGenericItems` from snapshot construction. + - Keep `isLiveItemFresh` available for UI status/fanout behavior if needed. + - Do not reject persisted ClickHouse rows just because market timestamps are older than 15s/30s. +3. Stop rejecting stale ingests inside `LiveStateManager.ingest`. + - The manager should store valid events it receives. + - Event fanout can still choose how to label status, but should not silently lose durable cache state. +4. Preserve Redis as a hot cache: + - Redis remains an optimization. + - ClickHouse remains the fallback and source of truth. + - API startup should hydrate from Redis if present, otherwise from ClickHouse. + +In `services/api/src/index.ts`: + +1. Include `equity-quotes` in `consumerBindings`. +2. Pump `EquityQuoteSchema` payloads into: + - legacy `/ws/equity-quotes` + - unified `/ws/live` + - `LiveStateManager` +3. Add `/history/equity-quotes`. +4. Keep durable consumer defaults unchanged unless a test proves old events are skipped in a live-running API scenario. ClickHouse hydration handles restart and refresh persistence. + +## Frontend Implementation + +In `apps/web/app/terminal.tsx`: + +1. Extend `LiveSessionState` with: + - per-subscription `next_before` cursors + - per-subscription loading/error state for older history + - equity quotes if exposed in UI state +2. When handling `snapshot` messages: + - Replace the channel's current items with snapshot items when non-empty. + - Store `snapshot.next_before`. + - Do not discard stale-but-persisted rows. + - Continue deduping by `trace_id/seq` or `id`. +3. Add a generic live-history loader: + - Map subscription channel to history endpoint: + - `options` -> `/history/options` + - `nbbo` -> `/history/nbbo` + - `equities` -> `/history/equities` + - `equity-quotes` -> `/history/equity-quotes` + - `equity-joins` -> `/history/equity-joins` + - `flow` -> `/history/flow` + - `classifier-hits` -> `/history/classifier-hits` + - `alerts` -> `/history/alerts` + - `inferred-dark` -> `/history/inferred-dark` + - Carry option/flow filters into options history queries. + - Merge older results into existing channel state. + - Advance `next_before` from the response. + - Stop when `next_before` is null or the response is empty. +4. UI behavior: + - Add a compact "Load older" control at the bottom of each applicable tape/list. + - Disable it while loading. + - Hide it when no more history exists. + - Keep existing pause/jump controls unchanged. + - Do not add browser market-data storage. +5. Chart behavior: + - Keep candles loading from `/candles/equities`. + - Keep overlay loading from `/prints/equities/range`. + - Ensure refresh and device changes show the same server data for the same ticker/window. + +## Config And Deployment + +Update `.env.example`: + +- Add `LIVE_LIMIT_EQUITY_QUOTES=10000`. +- Document that `LIVE_LIMIT_*` controls initial server snapshot/hot-cache depth, not total persisted history. + +Update README if needed: + +- Clarify persistence model: + - ClickHouse is durable history. + - Redis is hot cache. + - Browser is not a market-data database. + - All devices connected to the same API see the same server-seen data. + +Docker volumes already persist ClickHouse/Redis/NATS data locally and in deployment compose, so no migration is needed for volume-backed persistence. + +## Tests + +API tests in `services/api/tests/live.test.ts`: + +- Snapshot hydration returns stale historical options/NBBO/equities/flow instead of filtering them out. +- `LiveStateManager.ingest` stores older valid events. +- `equity-quotes` hydrates from Redis. +- `equity-quotes` hydrates from ClickHouse when Redis is empty. +- `next_before` is set from the oldest item in returned snapshot. +- Redis hot cache persists hydrated ClickHouse data. + +Storage tests: + +- Add `fetchEquityQuotesBefore` coverage using the existing storage test style. + +Frontend tests in `apps/web/app/terminal.test.ts`: + +- Live snapshot with older persisted rows populates visible rows. +- Empty snapshot does not wipe existing visible rows only when preserving an already visible channel during reconnect. +- Older-history merge dedupes existing items. +- History cursor advances after loading older rows. +- "No more history" state is reached when `next_before` is null. +- Live status can be stale while items remain visible. + +## Acceptance Criteria + +- Refreshing the app shows persisted data immediately, even when no new live events arrive after page load. +- Opening the app on another device connected to the same API shows the same server-backed recent history. +- Stale market timestamps do not cause persisted history to disappear. +- Users can load older data beyond the initial recent window. +- Live WebSocket updates still appear without requiring refresh. +- Redis loss does not lose history; API falls back to ClickHouse. +- Browser cache deletion does not lose market data. +- `bun test services/api/tests/live.test.ts apps/web/app/terminal.test.ts packages/storage/tests/*.test.ts` passes, or any unavailable test target is documented. From 48b0d980a68a24c610f9b43cae42a2a676de3494 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 4 May 2026 05:52:38 -0400 Subject: [PATCH 13/18] Implement scoped live 24h feed visibility --- apps/web/app/globals.css | 53 ++++++ apps/web/app/terminal.test.ts | 26 +++ apps/web/app/terminal.tsx | 190 +++++++++++++++++-- packages/storage/src/clickhouse.ts | 60 +++++- packages/storage/tests/equity-prints.test.ts | 41 ++++ packages/storage/tests/option-prints.test.ts | 8 +- packages/types/src/live.ts | 26 ++- packages/types/tests/live.test.ts | 13 ++ services/api/src/index.ts | 81 +++++++- services/api/src/live.ts | 88 ++++++++- services/api/tests/live.test.ts | 10 +- 11 files changed, 547 insertions(+), 49 deletions(-) diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index 9ea6697..d8a7377 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -384,6 +384,59 @@ input { color: #ffd89a; } +.instrument-focus-chip { + display: inline-flex; + align-items: center; + gap: 8px; + min-height: 34px; + max-width: min(360px, 32vw); + padding: 6px 8px 6px 10px; + border: 1px solid rgba(255, 216, 154, 0.34); + border-radius: 8px; + background: rgba(245, 166, 35, 0.08); + color: #ffe2aa; + font-family: var(--font-mono), monospace; + font-size: 0.72rem; + font-weight: 700; +} + +.instrument-focus-chip span { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.instrument-focus-chip button, +.instrument-cell-button { + border: 0; + background: transparent; + color: inherit; + font: inherit; + cursor: pointer; +} + +.instrument-focus-chip button { + padding: 4px 6px; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.08em; + font-size: 0.62rem; +} + +.instrument-cell-button { + padding: 0; + text-align: inherit; + text-decoration: underline; + text-decoration-color: rgba(255, 216, 154, 0.36); + text-underline-offset: 3px; +} + +.instrument-cell-button:hover, +.instrument-cell-button:focus-visible { + color: #ffd89a; + outline: none; +} + .pause-button { padding: 7px 10px; font-size: 0.66rem; diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index 9eb51d0..36a231e 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -73,6 +73,32 @@ describe("live manifest", () => { expect(optionsSubscription?.filters).toBe(filters); }); + + it("includes scoped option and equity subscriptions", () => { + const manifest = getLiveManifest( + "/tape", + "AAPL", + 60000, + buildDefaultFlowFilters(), + { + underlying_ids: ["AAPL"], + option_contract_id: "AAPL-2025-01-17-200-C" + }, + { underlying_ids: ["AAPL"] } + ); + const optionsSubscription = manifest.find( + (subscription): subscription is Extract<(typeof manifest)[number], { channel: "options" }> => + subscription.channel === "options" + ); + const equitiesSubscription = manifest.find( + (subscription): subscription is Extract<(typeof manifest)[number], { channel: "equities" }> => + subscription.channel === "equities" + ); + + expect(optionsSubscription?.underlying_ids).toEqual(["AAPL"]); + expect(optionsSubscription?.option_contract_id).toBe("AAPL-2025-01-17-200-C"); + expect(equitiesSubscription?.underlying_ids).toEqual(["AAPL"]); + }); }); describe("live tape pausable helpers", () => { diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index bf87281..d3fa9c8 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -13,6 +13,7 @@ import { useState, type CSSProperties, type Dispatch, + type MouseEvent as ReactMouseEvent, type ReactNode, type SetStateAction } from "react"; @@ -124,6 +125,11 @@ type ChartCandle = { close: number; }; +type SelectedInstrument = + | null + | { kind: "equity"; underlyingId: string } + | { kind: "option-contract"; contractId: string; underlyingId: string }; + const formatIntervalLabel = (intervalMs: number): string => { const match = CANDLE_INTERVALS.find((interval) => interval.ms === intervalMs); if (match) { @@ -2247,6 +2253,15 @@ const appendOptionFlowFilters = (params: URLSearchParams, filters: OptionFlowFil } }; +const appendLiveScopeParams = (params: URLSearchParams, subscription: LiveSubscription): void => { + if ((subscription.channel === "options" || subscription.channel === "equities") && subscription.underlying_ids?.length) { + params.set("underlying_ids", subscription.underlying_ids.join(",")); + } + if (subscription.channel === "options" && subscription.option_contract_id) { + params.set("option_contract_id", subscription.option_contract_id); + } +}; + const dedupeLiveSubscriptions = (subscriptions: LiveSubscription[]): LiveSubscription[] => { const seen = new Set(); return subscriptions.filter((subscription) => { @@ -2263,9 +2278,11 @@ export const getLiveManifest = ( pathname: string, chartTicker: string, chartIntervalMs: number, - flowFilters: OptionFlowFilters + flowFilters: OptionFlowFilters, + optionScope?: Pick, "underlying_ids" | "option_contract_id">, + equityScope?: Pick, "underlying_ids"> ): LiveSubscription[] => { - const baselineSubs: LiveSubscription[] = [{ channel: "options", filters: flowFilters }]; + const baselineSubs: LiveSubscription[] = [{ channel: "options", filters: flowFilters, ...optionScope }]; const chartSubs: LiveSubscription[] = [ { channel: "equity-candles", underlying_id: chartTicker, interval_ms: chartIntervalMs }, { channel: "equity-overlay", underlying_id: chartTicker } @@ -2274,9 +2291,9 @@ export const getLiveManifest = ( if (pathname === "/tape") { return dedupeLiveSubscriptions([ ...baselineSubs, - { channel: "options", filters: flowFilters }, + { channel: "options", filters: flowFilters, ...optionScope }, { channel: "nbbo" }, - { channel: "equities" }, + { channel: "equities", ...equityScope }, { channel: "flow", filters: flowFilters }, { channel: "classifier-hits" } ]); @@ -2306,7 +2323,7 @@ export const getLiveManifest = ( return dedupeLiveSubscriptions([ ...baselineSubs, - { channel: "equities" }, + { channel: "equities", ...equityScope }, { channel: "flow" }, { channel: "alerts" }, { channel: "classifier-hits" }, @@ -2320,7 +2337,9 @@ const useLiveSession = ( pathname: string, chartTicker: string, chartIntervalMs: number, - flowFilters: OptionFlowFilters + flowFilters: OptionFlowFilters, + optionScope?: Pick, "underlying_ids" | "option_contract_id">, + equityScope?: Pick, "underlying_ids"> ): LiveSessionState => { const [status, setStatus] = useState(enabled ? "connecting" : "disconnected"); const [connectedAt, setConnectedAt] = useState(null); @@ -2350,8 +2369,8 @@ const useLiveSession = ( const subscribedKeysRef = useRef>(new Set()); const subscribedMapRef = useRef>(new Map()); const manifest = useMemo( - () => getLiveManifest(pathname, chartTicker.toUpperCase(), chartIntervalMs, flowFilters), - [pathname, chartTicker, chartIntervalMs, flowFilters] + () => getLiveManifest(pathname, chartTicker.toUpperCase(), chartIntervalMs, flowFilters, optionScope, equityScope), + [pathname, chartTicker, chartIntervalMs, flowFilters, optionScope, equityScope] ); useEffect(() => { @@ -2616,6 +2635,42 @@ const useLiveSession = ( const currentKeys = subscribedKeysRef.current; const toSubscribe = manifest.filter((sub) => !currentKeys.has(getLiveSubscriptionKey(sub))); const removedKeys = Array.from(currentKeys).filter((key) => !nextKeys.has(key)); + const resetScopedChannels = new Set( + [...removedKeys, ...toSubscribe.map(getLiveSubscriptionKey)] + .map((key) => subscribedMapRef.current.get(key) ?? nextMap.get(key) ?? null) + .filter((sub): sub is LiveSubscription => sub !== null) + .map((sub) => sub.channel) + .filter((channel) => channel === "options" || channel === "equities") + ); + if (resetScopedChannels.has("options")) { + setOptions([]); + } + if (resetScopedChannels.has("equities")) { + setEquities([]); + } + if (resetScopedChannels.size > 0) { + setHistoryCursors((current) => { + const next = { ...current }; + for (const key of [...removedKeys, ...toSubscribe.map(getLiveSubscriptionKey)]) { + delete next[key]; + } + return next; + }); + setHistoryLoading((current) => { + const next = { ...current }; + for (const key of [...removedKeys, ...toSubscribe.map(getLiveSubscriptionKey)]) { + delete next[key]; + } + return next; + }); + setHistoryErrors((current) => { + const next = { ...current }; + for (const key of [...removedKeys, ...toSubscribe.map(getLiveSubscriptionKey)]) { + delete next[key]; + } + return next; + }); + } if (removedKeys.length > 0) { const removedSubs = removedKeys @@ -2660,6 +2715,7 @@ const useLiveSession = ( if (subscription.channel === "options" || subscription.channel === "flow") { appendOptionFlowFilters(params, subscription.filters); } + appendLiveScopeParams(params, subscription); const response = await fetch(buildApiUrl(`${endpoint}?${params.toString()}`)); if (!response.ok) { const detail = await readErrorDetail(response); @@ -3981,6 +4037,7 @@ const useTerminalState = () => { const [selectedAlert, setSelectedAlert] = useState(null); const [selectedDarkEvent, setSelectedDarkEvent] = useState(null); const [selectedClassifierHit, setSelectedClassifierHit] = useState(null); + const [selectedInstrument, setSelectedInstrument] = useState(null); const [filterInput, setFilterInput] = useState(""); const [flowFilters, setFlowFilters] = useState(() => buildDefaultFlowFilters()); const [chartIntervalMs, setChartIntervalMs] = useState(CANDLE_INTERVALS[0].ms); @@ -3992,20 +4049,52 @@ const useTerminalState = () => { return Array.from(new Set(parts)); }, [filterInput]); const tickerSet = useMemo(() => new Set(activeTickers), [activeTickers]); - const chartTicker = useMemo(() => activeTickers[0] ?? "SPY", [activeTickers]); + const instrumentUnderlying = selectedInstrument?.underlyingId.toUpperCase() ?? null; + const optionScope = useMemo( + () => ({ + underlying_ids: activeTickers.length > 0 ? activeTickers : instrumentUnderlying ? [instrumentUnderlying] : undefined, + option_contract_id: + selectedInstrument?.kind === "option-contract" ? selectedInstrument.contractId : undefined + }), + [activeTickers, instrumentUnderlying, selectedInstrument] + ); + const equityScope = useMemo( + () => ({ + underlying_ids: activeTickers.length > 0 ? activeTickers : instrumentUnderlying ? [instrumentUnderlying] : undefined + }), + [activeTickers, instrumentUnderlying] + ); + const chartTicker = useMemo( + () => instrumentUnderlying ?? activeTickers[0] ?? "SPY", + [activeTickers, instrumentUnderlying] + ); + const selectedInstrumentLabel = useMemo(() => { + if (!selectedInstrument) { + return null; + } + if (selectedInstrument.kind === "equity") { + return `Equity: ${selectedInstrument.underlyingId}`; + } + const display = formatOptionContractLabel(selectedInstrument.contractId); + return display + ? `Contract: ${display.ticker} ${display.expiration} ${display.strike}` + : `Contract: ${selectedInstrument.contractId}`; + }, [selectedInstrument]); const liveSession = useLiveSession( mode === "live", pathname, chartTicker, chartIntervalMs, - flowFilters + flowFilters, + optionScope, + equityScope ); const equitiesLiveSubscriptionActive = useMemo( () => - getLiveManifest(pathname, chartTicker.toUpperCase(), chartIntervalMs, flowFilters).some( + getLiveManifest(pathname, chartTicker.toUpperCase(), chartIntervalMs, flowFilters, optionScope, equityScope).some( (sub) => sub.channel === "equities" ), - [pathname, chartTicker, chartIntervalMs, flowFilters] + [pathname, chartTicker, chartIntervalMs, flowFilters, optionScope, equityScope] ); const handleReplaySource = useCallback((value: string | null) => { @@ -4665,19 +4754,31 @@ const useTerminalState = () => { if (!matchesOptionPrintFilters(print, flowFilters)) { return false; } + if ( + selectedInstrument?.kind === "option-contract" && + normalizeContractId(print.option_contract_id) !== selectedInstrument.contractId + ) { + return false; + } if (tickerSet.size === 0) { - return true; + return ( + !instrumentUnderlying || + extractUnderlying(normalizeContractId(print.option_contract_id)) === instrumentUnderlying + ); } return matchesTicker(extractUnderlying(normalizeContractId(print.option_contract_id))); }); - }, [flowFilters, optionsFeed.items, matchesTicker, tickerSet]); + }, [flowFilters, optionsFeed.items, matchesTicker, tickerSet, selectedInstrument, instrumentUnderlying]); const filteredEquities = useMemo(() => { if (tickerSet.size === 0) { + if (instrumentUnderlying) { + return equitiesFeed.items.filter((print) => print.underlying_id.toUpperCase() === instrumentUnderlying); + } return equitiesFeed.items; } return equitiesFeed.items.filter((print) => matchesTicker(print.underlying_id)); - }, [equitiesFeed.items, matchesTicker, tickerSet]); + }, [equitiesFeed.items, matchesTicker, tickerSet, instrumentUnderlying]); const equitiesSilentWarning = shouldShowEquitiesSilentFeedWarning({ wsStatus: liveSession.status, @@ -5000,6 +5101,9 @@ const useTerminalState = () => { setSelectedDarkEvent, selectedClassifierHit, setSelectedClassifierHit, + selectedInstrument, + setSelectedInstrument, + selectedInstrumentLabel, filterInput, setFilterInput, flowFilters, @@ -5473,6 +5577,15 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { const spot = print.execution_underlying_spot; const iv = print.execution_iv; const decor = state.classifierDecorByOptionTraceId.get(print.trace_id); + const underlyingId = (print.underlying_id ?? parsed?.root ?? extractUnderlying(contractId)).toUpperCase(); + const focusContract = (event: ReactMouseEvent) => { + event.stopPropagation(); + state.setSelectedInstrument({ + kind: "option-contract", + contractId, + underlyingId + }); + }; const commonProps = { className: `data-table-row data-table-row-button data-table-row-classified data-table-row-options${decor ? ` is-classified classifier-${decor.tone}` : ""}`, style: decor ? ({ "--classifier-intensity": decor.intensity } as CSSProperties) : undefined @@ -5480,10 +5593,26 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => { const cells = ( <> {formatTime(print.ts)} - {contractDisplay?.ticker ?? parsed?.root ?? formatContractLabel(contractId)} - {contractDisplay?.expiration ?? parsed?.expiry ?? "--"} - {contractDisplay?.strike.replace(/[CP]$/, "") ?? "--"} - {parsed?.right ?? contractDisplay?.strike.slice(-1) ?? "--"} + + + + + + + + + + + + {typeof spot === "number" ? formatPrice(spot) : "--"} {formatSize(print.size)}@{formatPrice(print.price)}_{nbboSide ?? "--"} @@ -5598,7 +5727,20 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => { {virtual.visibleItems.map((print) => (
{formatTime(print.ts)} - {print.underlying_id} + + + ${formatPrice(print.price)} {formatSize(print.size)}x {print.exchange} @@ -6237,6 +6379,14 @@ export function TerminalAppShell({ children }: { children: ReactNode }) { > Clear + {state.selectedInstrumentLabel ? ( + + {state.selectedInstrumentLabel} + + + ) : null}
+ + ) : null} +
- {error ? {error} : null} -
- ); -}; - type CandleChartProps = { ticker: string; intervalMs: number; @@ -5615,7 +5584,6 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => {
)} - {!limit ? : null}
); @@ -5710,7 +5678,6 @@ const EquitiesPane = ({ limit }: EquitiesPaneProps) => {
)} - {!limit ? : null}
); @@ -5849,7 +5816,6 @@ const FlowPane = ({ limit, title = "Flow" }: FlowPaneProps) => {
)} - {!limit ? : null}
); @@ -5948,7 +5914,6 @@ const AlertsPane = ({ limit, withStrip = false, className }: AlertsPaneProps) =>
)} - {!limit ? : null}
); @@ -6034,7 +5999,6 @@ const ClassifierPane = ({ limit, className }: ClassifierPaneProps) => {
)} - {!limit ? : null}
); @@ -6128,7 +6092,6 @@ const DarkPane = ({ limit, className }: DarkPaneProps) => {
)} - {!limit ? : null}
); @@ -6323,7 +6286,6 @@ export function TerminalAppShell({ children }: { children: ReactNode }) { ) : null} -