diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 065a612..605077e 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-k4f","title":"Gate deploy script on docker workspace snapshot sync","description":"Prevent frozen-lockfile build failures during deploy by adding a local preflight in scripts/deploy.ts that runs bun run check:docker-workspace and aborts with a clear sync+commit remediation message when stale.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:01:44Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:04:11Z","started_at":"2026-05-15T23:01:48Z","closed_at":"2026-05-15T23:04:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-xll","title":"Fix bun.lock drift causing frozen-lockfile Docker build failures","description":"Docker image builds fail in multiple targets (candles, web, ingest services) because bun install --frozen-lockfile detects lockfile changes. Update workspace lockfile to match manifests and verify frozen install succeeds.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T22:52:38Z","created_by":"dirtydishes","updated_at":"2026-05-15T22:55:23Z","started_at":"2026-05-15T22:52:40Z","closed_at":"2026-05-15T22:55:23Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-9nd","title":"Hosted synthetic tape redesign with internal control surface","description":"Implement hosted synthetic market redesign with shared deterministic regime engine, internal JetStream KV control plane, ingest coupling across options and equities, and an internal bottom-right synthetic-control drawer with Next proxy routes. Preserve the six public smart-money categories while adding hidden subtype families, soft coverage accounting, and backend-only admin endpoints.\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-14T01:25:02Z","created_by":"dirtydishes","updated_at":"2026-05-14T02:10:03Z","started_at":"2026-05-14T01:25:09Z","closed_at":"2026-05-14T02:10:03Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index 23bdb2e..1b2205c 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -606,6 +606,13 @@ h3 { text-transform: uppercase; } +.flow-filter-section-copy { + margin: -2px 0 0; + color: var(--text-muted); + font-size: 0.78rem; + line-height: 1.35; +} + .flow-filter-checkbox-grid, .flow-filter-chip-grid { display: grid; @@ -617,6 +624,10 @@ h3 { grid-template-columns: repeat(3, minmax(0, 1fr)); } +.flow-filter-chip-grid-two { + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + .flow-filter-check { display: inline-flex; align-items: center; diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index 0362723..03114c4 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -17,7 +17,6 @@ import { getEffectiveOptionPrintFilters, getAlertWindowAnchorTs, getHotChannelFeedStatus, - getScopedLiveAutoHydrationChannels, getLiveHistoryRetentionCap, getOptionTableSnapshot, getOptionScope, @@ -298,6 +297,24 @@ describe("contract-focused option helpers", () => { }); }); + it("includes the selected options view in tape query params", () => { + expect( + buildOptionTapeQueryParams( + { + ...buildDefaultFlowFilters(), + view: "raw", + securityTypes: undefined, + nbboSides: undefined, + optionTypes: undefined + }, + { underlying_ids: ["AAPL"] } + ) + ).toEqual({ + view: "raw", + underlying_ids: "AAPL" + }); + }); + it("keeps the focus seed until the matching scoped subscription has loaded it", () => { const seedItem = makeOptionPrint({ trace_id: "focused-seed", @@ -652,32 +669,6 @@ describe("live tape history helpers", () => { ).toBe(0); }); - it("does not auto-hydrate scoped live history before the scroll gate is reached", () => { - const manifest = getLiveManifest( - "/tape", - "AAPL", - 60000, - buildDefaultFlowFilters(), - { - underlying_ids: ["AAPL"], - option_contract_id: "AAPL-2025-01-17-200-C" - }, - { underlying_ids: ["AAPL"] } - ); - const historyCursors = Object.fromEntries( - manifest.map((subscription) => [getLiveSubscriptionKey(subscription), { ts: 1, seq: 1 }]) - ); - - expect( - getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {}) - ).toEqual([]); - expect( - getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, { - [getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "options")!)]: true - }) - ).toEqual([]); - }); - it("restores the same anchor key after live insertions at the top", () => { const nextKeys = ["new-1", "new-2", "anchor", "after-1", "after-2"]; expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(2); @@ -806,6 +797,7 @@ describe("flow filter popup helpers", () => { expect(countActiveFlowFilterGroups(defaults)).toBe(0); expect(countActiveFlowFilterGroups(next)).toBe(3); + expect(countActiveFlowFilterGroups({ ...defaults, view: "raw" })).toBe(1); expect(buildDefaultFlowFilters()).toEqual(defaults); }); }); diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 33eec33..2135a75 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -34,6 +34,7 @@ import type { LiveHotChannelHealthMap, LiveSubscription, OptionFlowFilters, + OptionFlowView, OptionNbboSide, OptionSecurityType, OptionType, @@ -853,21 +854,6 @@ export const getLiveHistoryRetentionCap = (subscription: LiveSubscription): numb } }; -export const getScopedLiveAutoHydrationChannels = ( - enabled: boolean, - pathname: string, - manifest: LiveSubscription[], - historyCursors: Partial>, - historyLoading: Partial> -): Array> => { - void enabled; - void pathname; - void manifest; - void historyCursors; - void historyLoading; - return []; -}; - export const getLiveFeedStatus = ( sourceStatus: WsStatus, freshestTs: number | null, @@ -1436,6 +1422,9 @@ export const countActiveFlowFilterGroups = (filters: OptionFlowFilters): number if ((filters.minNotional ?? undefined) !== (defaults.minNotional ?? undefined)) { count += 1; } + if ((filters.view ?? defaults.view) !== defaults.view) { + count += 1; + } return count; }; @@ -3684,18 +3673,6 @@ const useLiveSession = ( [enabled, manifest, historyCursors, historyLoading] ); - useEffect(() => { - for (const channel of getScopedLiveAutoHydrationChannels( - enabled, - pathname, - manifest, - historyCursors, - historyLoading - )) { - void loadOlder(channel); - } - }, [enabled, pathname, manifest, historyCursors, historyLoading, loadOlder]); - return { status, connectedAt, @@ -6904,6 +6881,17 @@ export const FlowFilterPopover = ({ filters, onChange }: FlowFilterPopoverProps) })); }; + const applyView = (view: OptionFlowView) => { + onChange((prev) => ({ + ...prev, + view, + securityTypes: view === "raw" ? undefined : prev.securityTypes ?? DEFAULT_FLOW_SECURITY_TYPES, + nbboSides: view === "raw" ? undefined : prev.nbboSides, + optionTypes: view === "raw" ? undefined : prev.optionTypes, + minNotional: view === "raw" ? undefined : prev.minNotional + })); + }; + useEffect(() => { if (!open) { return; @@ -6968,6 +6956,27 @@ export const FlowFilterPopover = ({ filters, onChange }: FlowFilterPopoverProps)
+ +
+ {[ + { label: "Signal", value: "signal" as const }, + { label: "All prints", value: "raw" as const } + ].map((preset) => ( + + ))} +
+

+ Signal keeps classifier-ready prints. All prints includes raw option tape rows. +

+ +
{(["stock", "etf"] as OptionSecurityType[]).map((value) => ( diff --git a/docs/clickhouse-reset-runbook.md b/docs/clickhouse-reset-runbook.md new file mode 100644 index 0000000..dac1775 --- /dev/null +++ b/docs/clickhouse-reset-runbook.md @@ -0,0 +1,57 @@ +# ClickHouse Reset Runbook + +This runbook is for deliberately wiping durable market-data history from ClickHouse in local development or on the VPS. It is destructive. Do not run these commands from application startup, deployment hooks, or unattended scripts. + +## When To Use + +Use this only when an operator has decided that existing option, equity, flow, and derived-event history should be discarded and rebuilt from fresh ingest. + +Before running a reset: + +- Confirm the target environment: local Docker or VPS Docker. +- Confirm there is no active analysis depending on the existing history. +- Take a backup if the data may be needed later. +- Stop ingest and API services so new writes do not race the reset. + +## Local Docker Reset + +From the repository root: + +```bash +bun run dev:infra +docker compose exec clickhouse clickhouse-client --query "SHOW TABLES" +docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS option_prints" +docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS option_nbbo" +docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS equity_prints" +docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS equity_quotes" +docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS equity_print_joins" +docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS flow_packets" +docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS smart_money_events" +docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS classifier_hits" +docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS alerts" +docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS inferred_dark_events" +``` + +If the local compose project uses `deployment/docker/docker-compose.yml`, run the same commands with `docker compose -f deployment/docker/docker-compose.yml exec clickhouse ...`. + +## VPS Docker Reset + +On the VPS, first identify the active compose project and ClickHouse service: + +```bash +docker ps --format "table {{.Names}}\t{{.Image}}\t{{.Status}}" +docker compose -f deployment/docker/docker-compose.yml ps +``` + +Then stop writers and run the same `TRUNCATE TABLE IF EXISTS ...` commands against the active ClickHouse container. Prefer `docker compose exec clickhouse clickhouse-client --query ""` when the compose project is healthy; otherwise use `docker exec clickhouse-client --query ""`. + +## Verification + +After the reset: + +```bash +docker compose exec clickhouse clickhouse-client --query "SELECT count() FROM option_prints" +docker compose exec clickhouse clickhouse-client --query "SELECT count() FROM flow_packets" +``` + +Restart ingest/API services through the normal dev or deployment path. The options tape should repopulate its 100-row hot head from new signal prints, and older rows should appear only after the scroll gate asks `/history/options` for ClickHouse-backed history. diff --git a/docs/turns/2026-05-16-1725-durable-options-tape-history.html b/docs/turns/2026-05-16-1725-durable-options-tape-history.html new file mode 100644 index 0000000..a586496 --- /dev/null +++ b/docs/turns/2026-05-16-1725-durable-options-tape-history.html @@ -0,0 +1,245 @@ + + + + + + Durable Options Tape History + + + +
+
+

Turn Document

+

Durable Options Tape History

+

+ Implemented the durable options tape plan: the live hot head is capped at 100 rows, older rows are preserved behind + the scroll gate, ClickHouse history keeps execution context, and the Filter menu now exposes Signal versus All + prints semantics. +

+
+ 2026-05-16 17:25 + Beads: islandflow-200 + Surface: Options Tape +
+
+ +
+

Summary

+

+ The options tape now behaves as a continuous instrument: the live cache stays lean, historical rows arrive only + when scrolling asks for them, and old valid rows are not treated as degraded just because they came from durable + history. +

+
+ +
+

Changes Made

+
    +
  • Changed the API default options live cache limit to 100.
    • +
  • Removed the unused scoped live auto-hydration path so history is loaded by the scroll gate.
    • +
  • Fixed unbounded options/equities history retention so a cap of 0 means keep the loaded tail.
    • +
  • Added a Filter menu Options View toggle for Signal and All prints.
    • +
  • Ensured All prints clears signal-only side/type/min-notional/security constraints.
    • +
  • Added a destructive ClickHouse reset runbook for local and VPS operators.
    • +
+
+ +
+

Context

+

+ The prior plan called out useful partial work already in the repo: ClickHouse history endpoints, execution-context + columns, scroll-hold behavior, and a shared row renderer. This implementation keeps those pieces and removes the + ambiguous history/autohydration behavior around them. +

+
+ +
+

Important Implementation Details

+
    +
  • /history/options still uses the selected option filters and scope, including raw contract drilldowns.
    • +
  • Storage tests now verify execution NBBO side, underlying spot, IV, and signal reasons survive normalization.
    • +
  • The options row path already preferred execution_nbbo_side, execution_underlying_spot, and execution_iv; tests cover that behavior.
    • +
  • The reset runbook is documented in docs/clickhouse-reset-runbook.md and is explicitly operator-confirmed.
    • +
+
+ +
+

Expected Impact for End-Users

+

+ Traders can stay on a signal-first tape by default, switch to raw prints when investigating, and scroll into older + ClickHouse-backed flow without seeing a separate stale-history treatment. +

+
+ +
+

Validation

+
    +
  • Passed: bun test packages/storage/tests/option-prints.test.ts services/api/tests/live.test.ts apps/web/app/terminal.test.ts
    • +
  • Passed: bun --cwd=apps/web run build
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • The ClickHouse reset remains destructive. Mitigation: documented as a manual runbook only, never automatic startup behavior.
    • +
  • No live browser smoke test was run in this turn. Mitigation: unit coverage and production build exercised the changed web paths.
    • +
+
+ +
+

Follow-up Work

+

No new follow-up issue was needed. The implementation task is tracked and completed in islandflow-200.

+
+
+ + diff --git a/packages/storage/tests/option-prints.test.ts b/packages/storage/tests/option-prints.test.ts index 17b3e29..139b66a 100644 --- a/packages/storage/tests/option-prints.test.ts +++ b/packages/storage/tests/option-prints.test.ts @@ -48,6 +48,25 @@ describe("option-prints storage helpers", () => { queries.push(query); return { async json() { + if (query.includes("trace-ctx")) { + return [ + { + ...basePrint, + trace_id: "trace-ctx", + conditions: [], + execution_nbbo_bid: "1.20", + execution_nbbo_ask: "1.30", + execution_nbbo_mid: "1.25", + execution_nbbo_side: "A", + execution_underlying_spot: "450.05", + execution_underlying_source: "equity_quote_mid", + execution_iv: "0.42", + execution_iv_source: "synthetic_pressure_model", + signal_reasons: ["large_notional"], + signal_pass: 1 + } + ] as T; + } return [] as T; } }; @@ -63,8 +82,9 @@ describe("option-prints storage helpers", () => { optionContractId: "AAPL-2025-01-17-200-C", sinceTs: 123 }); - await fetchOptionPrintsBefore(client, 100, 5, 20, "alpaca"); + await fetchOptionPrintsBefore(client, 100, 5, 20, "alpaca", { view: "raw" }); await fetchOptionPrintsByTraceIds(client, ["trace-1", "trace-2"]); + const rows = await fetchRecentOptionPrints(client, 1, "trace-ctx", { view: "signal" }); expect(queries[0]).toContain("signal_pass = 1"); expect(queries[0]).toContain("(is_etf = 0 OR is_etf IS NULL)"); @@ -76,7 +96,12 @@ describe("option-prints storage helpers", () => { expect(queries[0]).toContain("ts >= 123"); expect(queries[1]).toContain("(ts, seq) < (100, 5)"); expect(queries[1]).toContain("startsWith(trace_id, 'alpaca')"); + expect(queries[1]).not.toContain("signal_pass = 1"); expect(queries[1]).toContain("ORDER BY ts DESC, seq DESC LIMIT 20"); expect(queries[2]).toContain("trace_id IN ('trace-1', 'trace-2')"); + expect(rows[0].execution_nbbo_side).toBe("A"); + expect(rows[0].execution_underlying_spot).toBe(450.05); + expect(rows[0].execution_iv).toBe(0.42); + expect(rows[0].signal_reasons).toEqual(["large_notional"]); }); }); diff --git a/services/api/src/live.ts b/services/api/src/live.ts index ab4ceee..024935e 100644 --- a/services/api/src/live.ts +++ b/services/api/src/live.ts @@ -72,7 +72,7 @@ const CHART_LIMITS = { } as const; const DEFAULT_LIVE_LIMITS: GenericLiveLimits = { - options: 1000, + options: 100, nbbo: 1000, equities: 1000, "equity-quotes": 500, diff --git a/services/api/tests/live.test.ts b/services/api/tests/live.test.ts index fff1d61..78807ca 100644 --- a/services/api/tests/live.test.ts +++ b/services/api/tests/live.test.ts @@ -69,6 +69,7 @@ describe("LiveStateManager", () => { expect(limits.flow).toBe(500); expect(limits["equity-quotes"]).toBe(500); expect(limits.alerts).toBe(300); + expect(resolveGenericLiveLimits({} as NodeJS.ProcessEnv).options).toBe(100); }); it("hydrates snapshots from redis generic windows", async () => { @@ -520,6 +521,32 @@ describe("LiveStateManager", () => { ]); }); + it("caps generic options snapshots at the 100-row hot head by default", async () => { + const manager = new LiveStateManager(makeClickHouse(), null); + const now = Date.now(); + + for (let seq = 1; seq <= 150; seq += 1) { + await manager.ingest("options", { + source_ts: now + seq, + ingest_ts: now + seq, + seq, + trace_id: `opt-${seq}`, + ts: now + seq, + option_contract_id: "AAPL-2025-01-17-200-C", + price: 1, + size: 10, + exchange: "X", + signal_pass: true + }); + } + + const snapshot = await manager.getSnapshot({ channel: "options" }); + + expect(snapshot.items).toHaveLength(100); + expect((snapshot.items as Array<{ trace_id: string }>)[0].trace_id).toBe("opt-150"); + expect(snapshot.next_before).toEqual({ ts: now + 51, seq: 51 }); + }); + it("seeds scoped option snapshots from clickhouse rows older than 24h", async () => { const now = Date.now(); const staleTs = now - 25 * 60 * 60 * 1000;