fix durable options history routing

.beads/issues.jsonl

@@ -1,3 +1,4 @@
+{"_type":"issue","id":"islandflow-qso","title":"Fix durable options tape history routing","description":"Implement the fix-tape plan: make same-origin history routing durable, add deployment/public smoke checks for required API routes, expose tape history loading failures in the UI, document the work, and track api.flow.deltaisland.io migration separately.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T01:53:22Z","created_by":"dirtydishes","updated_at":"2026-05-17T02:00:04Z","started_at":"2026-05-17T01:53:25Z","closed_at":"2026-05-17T02:00:04Z","close_reason":"Implemented durable same-origin history routing, public route smoke checks, tape history diagnostics, docs, validation, and follow-up tracking for api.flow.deltaisland.io.","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-k4f","title":"Gate deploy script on docker workspace snapshot sync","description":"Prevent frozen-lockfile build failures during deploy by adding a local preflight in scripts/deploy.ts that runs bun run check:docker-workspace and aborts with a clear sync+commit remediation message when stale.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:01:44Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:04:11Z","started_at":"2026-05-15T23:01:48Z","closed_at":"2026-05-15T23:04:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-xll","title":"Fix bun.lock drift causing frozen-lockfile Docker build failures","description":"Docker image builds fail in multiple targets (candles, web, ingest services) because bun install --frozen-lockfile detects lockfile changes. Update workspace lockfile to match manifests and verify frozen install succeeds.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T22:52:38Z","created_by":"dirtydishes","updated_at":"2026-05-15T22:55:23Z","started_at":"2026-05-15T22:52:40Z","closed_at":"2026-05-15T22:55:23Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
@@ -11,6 +12,7 @@
 {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"islandflow-qd7","title":"Migrate production web to api.flow.deltaisland.io","description":"Follow-up from the durable options tape history fix. Plan and migrate production from same-origin API path proxying on flow.deltaisland.io to a dedicated api.flow.deltaisland.io origin, including DNS, proxy config, CORS/websocket behavior, deployment docs, and public smoke checks.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-17T01:55:02Z","created_by":"dirtydishes","updated_at":"2026-05-17T01:55:02Z","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-09a","title":"Speed up Docker deployment builds","description":"Implement the Docker deployment optimization plan from /Users/kell/Desktop/speed-up-docker.md: split dependency installation from source copy, add BuildKit caches, make scoped deploys build only their target services, update Docker deployment docs, validate, document the turn, commit, and push.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:50:24Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:53:48Z","started_at":"2026-05-16T21:50:37Z","closed_at":"2026-05-16T21:53:48Z","close_reason":"Implemented Docker dependency-layer caching, scoped deploy build/up flow, Docker docs updates, validation, and turn documentation. Follow-up islandflow-cnk tracks daemon-backed image build verification.","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-0sa","title":"Fix live tape auto-hold, history seam, and remove manual pause control","description":"The live tape should automatically hold when the user scrolls away from the top, resume when they return to the top or use Jump to top, and keep older prints available seamlessly beyond the hot window. Manual Pause/Resume control is now redundant and should be removed from live tape panes. This work should also fix the current regression where paused/held tapes still mutate, and align the options tape with a strict 100-row hot head backed by ClickHouse history.","notes":"Implemented live scroll-hold with no live pause button, demand-loaded ClickHouse history, a 100-row options hot head, and cache-first scoped snapshots. Validated with bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts and bun --cwd=apps/web run build.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T18:12:51Z","created_by":"dirtydishes","updated_at":"2026-05-16T18:23:43Z","started_at":"2026-05-16T18:12:54Z","closed_at":"2026-05-16T18:23:43Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-2db","title":"Manually remove stale islandflow local-infra containers from VPS","description":"The live VPS still has an older compose project named islandflow created from the repo-root docker-compose.yml. Inspection shows it is separate from the supported islandflow-vps deployment stack and exposes NATS, ClickHouse, and Redis on host ports. Container removal commands currently hang when run as the delta user through Docker, so cleanup likely needs a focused maintenance window and possibly host-level intervention or a Docker daemon restart.","notes":"The duplicate islandflow compose project on the VPS was confirmed live during inspection. Nginx Proxy Manager routes public traffic only to islandflow-vps web/api by Docker name, so the stale islandflow project appears to be stray local-infra state rather than part of the supported production path. Attempts to remove the stale containers with docker compose down and docker rm -f as the delta user hung and timed out, so manual cleanup likely needs a maintenance window and possibly Docker daemon intervention.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:27:27Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:28:59Z","dependency_count":0,"dependent_count":0,"comment_count":0}

apps/web/app/globals.css

@@ -1003,6 +1003,17 @@ h3 {
   overflow: hidden;
 }
 
+.history-load-warning {
+  flex: 0 0 auto;
+  padding: 8px 12px;
+  border-top: 1px solid oklch(0.72 0.13 58 / 0.45);
+  border-bottom: 1px solid oklch(0.72 0.13 58 / 0.45);
+  background: oklch(0.24 0.05 58 / 0.72);
+  color: oklch(0.91 0.08 72);
+  font-size: 0.78rem;
+  line-height: 1.35;
+}
+
 .data-table-wrap {
   display: flex;
   flex: 1 1 auto;

apps/web/app/terminal.tsx

@@ -7109,6 +7109,13 @@ type OptionsPaneProps = {
 const OptionsPane = memo(({ state, limit }: OptionsPaneProps) => {
   const items = limit ? state.filteredOptions.slice(0, limit) : state.filteredOptions;
   const virtual = useTapeVirtualList(items, state.optionsScroll.listRef, getTapeVirtualConfig("options"));
+  const optionHistorySubscription = state.liveSession.manifest.find(
+    (subscription) => subscription.channel === "options"
+  );
+  const optionHistoryKey = optionHistorySubscription ? getLiveSubscriptionKey(optionHistorySubscription) : null;
+  const optionHistoryError = optionHistoryKey
+    ? state.liveSession.historyErrors[optionHistoryKey]
+    : null;
   useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () =>
     void state.liveSession.loadOlder("options")
   );
@@ -7139,6 +7146,11 @@ const OptionsPane = memo(({ state, limit }: OptionsPaneProps) => {
       }
     >
       <div className="data-table-shell">
+        {state.mode === "live" && optionHistoryError ? (
+          <div className="history-load-warning" role="status">
+            Older option history failed to load: {optionHistoryError}
+          </div>
+        ) : null}
         {items.length === 0 ? (
           <div className="empty">
             {state.mode === "live"

deployment/docker/README.md

@@ -119,10 +119,16 @@ Supported routing modes:
    - Build web with `NEXT_PUBLIC_API_URL=` (empty).
    - Point `app.<domain>` at the web host port.
    - Proxy these API routes from the app origin to the API host port:
-     - `/ws/*`, `/replay/*`, `/prints/*`, `/joins/*`, `/nbbo/*`, `/dark/*`, `/flow/*`, `/candles/*`
+     - `/ws/*`, `/replay/*`, `/prints/*`, `/joins/*`, `/nbbo/*`, `/dark/*`, `/flow/*`, `/candles/*`, `/history/*`
 
 Enable websocket support on whichever host serves `/ws/*`.
 
+For the current live Nginx Proxy Manager setup behind `flow.deltaisland.io`, keep the API location regex durable in the proxy host advanced config or API, not by hand-editing generated files under `/data/nginx/proxy_host/`. The route matcher should include history:
+
+```nginx
+^/(ws|replay|prints|joins|nbbo|dark|flow|candles|history)/
+```
+
 ## Replay service
 
 Replay is disabled by default in this stack.
@@ -441,3 +447,4 @@ After the stack is up:
 - `curl -I http://127.0.0.1:3000/` should return a successful HTTP status on the server.
 - In two-origin mode, browser requests should target `https://api.<domain>/...` and live feeds should use `wss://api.<domain>/ws/...`.
 - In same-origin mode, browser requests should target `https://app.<domain>/...` for API paths and live feeds should use `wss://app.<domain>/ws/...`.
+- In same-origin mode, `bun run check:public-api-routes` should pass for `/prints/options`, `/history/options`, `/replay/options`, `/nbbo/options`, and `/ws/live`.

docs/turns/2026-05-16-2159-fix-durable-options-history-routing.html

@@ -0,0 +1,195 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Fix Durable Options History Routing</title>
+    <style>
+      :root {
+        color-scheme: dark;
+        --bg: oklch(0.14 0.012 250);
+        --panel: oklch(0.19 0.018 250);
+        --panel-soft: oklch(0.23 0.018 250);
+        --border: oklch(0.44 0.018 250);
+        --text: oklch(0.9 0.018 250);
+        --muted: oklch(0.7 0.028 250);
+        --amber: oklch(0.78 0.14 72);
+        --green: oklch(0.72 0.16 154);
+      }
+
+      body {
+        margin: 0;
+        background: var(--bg);
+        color: var(--text);
+        font-family: IBM Plex Sans, Inter, ui-sans-serif, system-ui, sans-serif;
+        line-height: 1.55;
+      }
+
+      main {
+        width: min(960px, calc(100vw - 40px));
+        margin: 0 auto;
+        padding: 48px 0;
+      }
+
+      header {
+        border-bottom: 1px solid var(--border);
+        padding-bottom: 20px;
+        margin-bottom: 28px;
+      }
+
+      h1,
+      h2 {
+        line-height: 1.1;
+        margin: 0;
+      }
+
+      h1 {
+        font-size: clamp(2rem, 4vw, 3rem);
+        letter-spacing: 0;
+      }
+
+      h2 {
+        margin-top: 30px;
+        font-size: 1rem;
+        color: var(--amber);
+        text-transform: uppercase;
+      }
+
+      p {
+        max-width: 74ch;
+        color: var(--muted);
+      }
+
+      ul {
+        padding-left: 1.2rem;
+      }
+
+      li {
+        margin: 0.45rem 0;
+      }
+
+      code,
+      pre {
+        font-family: IBM Plex Mono, ui-monospace, SFMono-Regular, monospace;
+      }
+
+      pre {
+        overflow-x: auto;
+        padding: 14px;
+        border: 1px solid var(--border);
+        border-radius: 8px;
+        background: var(--panel);
+      }
+
+      .summary {
+        padding: 18px;
+        border: 1px solid oklch(0.58 0.08 72);
+        border-radius: 8px;
+        background: oklch(0.21 0.035 72);
+        color: var(--text);
+      }
+
+      .status {
+        display: inline-flex;
+        gap: 8px;
+        align-items: center;
+        padding: 4px 9px;
+        border: 1px solid oklch(0.55 0.11 154);
+        border-radius: 999px;
+        color: var(--green);
+        font-size: 0.78rem;
+        font-weight: 700;
+        text-transform: uppercase;
+      }
+
+      section {
+        border-top: 1px solid oklch(0.32 0.018 250);
+        padding-top: 4px;
+      }
+    </style>
+  </head>
+  <body>
+    <main>
+      <header>
+        <span class="status">Validated</span>
+        <h1>Fix Durable Options History Routing</h1>
+        <p>Turn completed on 2026-05-16 21:59 America/New_York.</p>
+      </header>
+
+      <section>
+        <h2>Summary</h2>
+        <p class="summary">
+          Options tape history now has a durable public route through same-origin deployments. The live Nginx Proxy Manager route was updated to include <code>/history/*</code>, deployment checks now fail when required API paths reach the web app, and the tape UI surfaces older-history load failures instead of leaving the user to infer that only the hot window exists.
+        </p>
+      </section>
+
+      <section>
+        <h2>Changes Made</h2>
+        <ul>
+          <li>Added <code>scripts/check-public-api-routes.ts</code> and the <code>check:public-api-routes</code> package script.</li>
+          <li>Updated <code>scripts/deploy.ts</code> so same-origin API deploy verification probes required public routes.</li>
+          <li>Updated <code>deployment/docker/README.md</code> to include <code>/history/*</code> in same-origin proxy routing and document the Nginx Proxy Manager regex.</li>
+          <li>Added an options tape warning banner for live <code>/history/options</code> load errors.</li>
+          <li>Updated live Nginx Proxy Manager config for <code>flow.deltaisland.io</code> so the public route regex includes <code>history</code>.</li>
+          <li>Created follow-up Beads issue <code>islandflow-qd7</code> for the later <code>api.flow.deltaisland.io</code> migration.</li>
+        </ul>
+      </section>
+
+      <section>
+        <h2>Context</h2>
+        <p>
+          The API and ClickHouse path already supported older options history, but the public same-origin route sent <code>/history/options</code> to the Next.js app. That made the live tape feel capped at the newest hot-window rows even though durable history existed behind the API.
+        </p>
+      </section>
+
+      <section>
+        <h2>Important Implementation Details</h2>
+        <p>
+          The deploy smoke check performs GET probes and verifies JSON responses for these same-origin routes:
+        </p>
+        <pre>/prints/options
+/history/options
+/replay/options
+/nbbo/options
+/ws/live</pre>
+        <p>
+          The live proxy matcher is now:
+        </p>
+        <pre>^/(ws|replay|prints|joins|nbbo|dark|flow|candles|history)/</pre>
+      </section>
+
+      <section>
+        <h2>Expected Impact for End-Users</h2>
+        <p>
+          Users on <code>/tape</code> can scroll beyond the initial options hot window and receive older ClickHouse-backed rows through the same cursor path for Signal and All prints. If public routing regresses, the tape now shows a visible history loading failure.
+        </p>
+      </section>
+
+      <section>
+        <h2>Validation</h2>
+        <ul>
+          <li>Passed: <code>bun test apps/web/app/terminal.test.ts</code></li>
+          <li>Passed: <code>bun test</code></li>
+          <li>Passed: <code>bun --cwd=apps/web run build</code></li>
+          <li>Passed: <code>bun run check:public-api-routes</code></li>
+          <li>Passed: remote Nginx syntax check after updating the route.</li>
+        </ul>
+      </section>
+
+      <section>
+        <h2>Issues, Limitations, and Mitigations</h2>
+        <ul>
+          <li>The long-term API subdomain migration remains separate work. Mitigation: tracked as <code>islandflow-qd7</code>.</li>
+          <li>The Nginx Proxy Manager database and generated proxy host file were both updated because the existing live file had prior generated-file edits. Mitigation: deployment docs now call out the durable advanced-config/API path.</li>
+        </ul>
+      </section>
+
+      <section>
+        <h2>Follow-up Work</h2>
+        <p>
+          Complete <code>islandflow-qd7</code> to move production API traffic to <code>api.flow.deltaisland.io</code> deliberately, including DNS, proxy behavior, CORS/websocket checks, docs, and deployment verification.
+        </p>
+      </section>
+    </main>
+  </body>
+</html>

package.json

@@ -20,6 +20,7 @@
     "deploy": "bun run scripts/deploy.ts",
     "deploy:main": "./deploy main",
     "deploy:current-branch": "./deploy current-branch",
+    "check:public-api-routes": "bun run scripts/check-public-api-routes.ts",
     "sync:docker-workspace": "bun run scripts/sync-docker-workspace.ts",
     "check:docker-workspace": "bun run scripts/check-docker-workspace.ts"
   },

scripts/check-public-api-routes.ts

@@ -0,0 +1,41 @@
+#!/usr/bin/env bun
+
+type RouteCheck = {
+  path: string;
+  expectJson: boolean;
+};
+
+const routeChecks: RouteCheck[] = [
+  { path: "/prints/options?view=signal&limit=1", expectJson: true },
+  { path: "/history/options?view=signal&before_ts=4102444800000&before_seq=999999999&limit=1", expectJson: true },
+  { path: "/replay/options?view=signal&after_ts=0&after_seq=0&limit=1", expectJson: true },
+  { path: "/nbbo/options?limit=1", expectJson: true },
+  { path: "/ws/live", expectJson: true }
+];
+
+const appUrl = process.env.DEPLOY_PUBLIC_APP_URL?.trim() || process.argv[2]?.trim();
+const baseUrl = appUrl || "https://flow.deltaisland.io";
+
+const isJsonResponse = (response: Response): boolean => {
+  return (response.headers.get("content-type") ?? "").toLowerCase().includes("application/json");
+};
+
+const assertPublicApiRoute = async ({ path, expectJson }: RouteCheck): Promise<void> => {
+  const url = new URL(path, baseUrl);
+  const response = await fetch(url);
+  const responseText = await response.text();
+
+  if (response.status === 404) {
+    throw new Error(`${url.pathname} returned 404; route is likely reaching the web app`);
+  }
+
+  if (expectJson && !isJsonResponse(response)) {
+    const sample = responseText.replace(/\s+/g, " ").slice(0, 120);
+    throw new Error(`${url.pathname} returned non-JSON content (${response.headers.get("content-type") ?? "none"}): ${sample}`);
+  }
+};
+
+for (const check of routeChecks) {
+  await assertPublicApiRoute(check);
+  console.log(`ok ${check.path}`);
+}

scripts/deploy.ts

@@ -732,9 +732,7 @@ function publicVerification(scope: DeployScope): void {
   }
 
   if (scopeIncludesApi(scope)) {
-    console.log(
+    runChecked("bun", ["run", "scripts/check-public-api-routes.ts", PUBLIC_APP_URL]);
-      "Skipping separate public API health check; same-origin mode relies on the public app check plus runtime-local API verification."
-    );
   }
 }