implement durable options tape history
This commit is contained in:
parent
bd60d0d5d5
commit
2abdd24e2c
2 changed files with 389 additions and 0 deletions
26
.codex/hooks.json
Normal file
26
.codex/hooks.json
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
{
|
||||||
|
"hooks": {
|
||||||
|
"PreCompact": [
|
||||||
|
{
|
||||||
|
"matcher": "",
|
||||||
|
"hooks": [
|
||||||
|
{
|
||||||
|
"type": "command",
|
||||||
|
"command": "bd prime"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"SessionStart": [
|
||||||
|
{
|
||||||
|
"matcher": "",
|
||||||
|
"hooks": [
|
||||||
|
{
|
||||||
|
"type": "command",
|
||||||
|
"command": "bd prime"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
363
docs/plans/2026-05-16-1711-durable-options-tape-history.html
Normal file
363
docs/plans/2026-05-16-1711-durable-options-tape-history.html
Normal file
|
|
@ -0,0 +1,363 @@
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<title>Plan: Durable Options Tape History</title>
|
||||||
|
<style>
|
||||||
|
:root {
|
||||||
|
color-scheme: dark;
|
||||||
|
--bg: #06080b;
|
||||||
|
--surface: #0b1016;
|
||||||
|
--panel: #111820;
|
||||||
|
--panel-2: #0d141b;
|
||||||
|
--border: rgba(255, 255, 255, 0.1);
|
||||||
|
--border-strong: rgba(245, 166, 35, 0.32);
|
||||||
|
--text: #e6edf4;
|
||||||
|
--muted: #90a0b2;
|
||||||
|
--faint: #6e7b8c;
|
||||||
|
--accent: #f5a623;
|
||||||
|
--accent-soft: rgba(245, 166, 35, 0.14);
|
||||||
|
--good: #25c17a;
|
||||||
|
}
|
||||||
|
|
||||||
|
* {
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
background:
|
||||||
|
linear-gradient(180deg, rgba(17, 24, 32, 0.86), rgba(6, 8, 11, 0.98) 340px),
|
||||||
|
var(--bg);
|
||||||
|
color: var(--text);
|
||||||
|
font: 15px/1.55 "IBM Plex Sans", -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
|
||||||
|
}
|
||||||
|
|
||||||
|
main {
|
||||||
|
max-width: 1040px;
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 40px 24px 56px;
|
||||||
|
}
|
||||||
|
|
||||||
|
header {
|
||||||
|
padding-bottom: 24px;
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
|
||||||
|
.eyebrow {
|
||||||
|
margin: 0 0 10px;
|
||||||
|
color: var(--accent);
|
||||||
|
font: 700 0.78rem/1.2 "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
letter-spacing: 0.12em;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 {
|
||||||
|
margin: 0;
|
||||||
|
max-width: 760px;
|
||||||
|
font-size: 2.1rem;
|
||||||
|
line-height: 1.1;
|
||||||
|
letter-spacing: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.summary {
|
||||||
|
max-width: 820px;
|
||||||
|
margin: 16px 0 0;
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.meta {
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 8px;
|
||||||
|
margin-top: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chip {
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 999px;
|
||||||
|
padding: 4px 9px;
|
||||||
|
background: rgba(255, 255, 255, 0.04);
|
||||||
|
color: var(--muted);
|
||||||
|
font: 700 0.72rem/1.2 "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
letter-spacing: 0.06em;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
section {
|
||||||
|
margin-top: 28px;
|
||||||
|
padding: 22px;
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 8px;
|
||||||
|
background: linear-gradient(180deg, rgba(17, 24, 32, 0.92), rgba(13, 20, 27, 0.92));
|
||||||
|
}
|
||||||
|
|
||||||
|
section.highlight {
|
||||||
|
border-color: var(--border-strong);
|
||||||
|
background: linear-gradient(180deg, rgba(245, 166, 35, 0.12), rgba(17, 24, 32, 0.9));
|
||||||
|
}
|
||||||
|
|
||||||
|
h2 {
|
||||||
|
margin: 0 0 12px;
|
||||||
|
color: var(--text);
|
||||||
|
font: 700 0.86rem/1.2 "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
letter-spacing: 0.11em;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
p {
|
||||||
|
margin: 0;
|
||||||
|
color: var(--muted);
|
||||||
|
}
|
||||||
|
|
||||||
|
ul {
|
||||||
|
margin: 0;
|
||||||
|
padding-left: 1.15rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
li {
|
||||||
|
margin: 8px 0;
|
||||||
|
color: var(--muted);
|
||||||
|
}
|
||||||
|
|
||||||
|
strong {
|
||||||
|
color: var(--text);
|
||||||
|
font-weight: 700;
|
||||||
|
}
|
||||||
|
|
||||||
|
code {
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||||
|
border-radius: 6px;
|
||||||
|
padding: 0.12rem 0.35rem;
|
||||||
|
background: rgba(255, 255, 255, 0.05);
|
||||||
|
color: var(--text);
|
||||||
|
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
font-size: 0.9em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.callout {
|
||||||
|
margin-top: 14px;
|
||||||
|
border: 1px solid rgba(37, 193, 122, 0.28);
|
||||||
|
border-radius: 8px;
|
||||||
|
padding: 14px 16px;
|
||||||
|
background: rgba(37, 193, 122, 0.08);
|
||||||
|
}
|
||||||
|
|
||||||
|
.callout p {
|
||||||
|
color: var(--text);
|
||||||
|
}
|
||||||
|
|
||||||
|
.note {
|
||||||
|
color: var(--faint);
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 720px) {
|
||||||
|
main {
|
||||||
|
padding: 28px 16px 40px;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 {
|
||||||
|
font-size: 1.6rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
section {
|
||||||
|
padding: 18px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<main>
|
||||||
|
<header>
|
||||||
|
<p class="eyebrow">Plan Document</p>
|
||||||
|
<h1>Durable Options Tape History</h1>
|
||||||
|
<p class="summary">
|
||||||
|
Make the options tape a signal-first live instrument with scroll-gated historical depth: keep the hot cache at
|
||||||
|
100 option prints, load older rows from ClickHouse only at the scroll gate, preserve execution context, and
|
||||||
|
render ClickHouse-backed rows exactly like any other valid flow row.
|
||||||
|
</p>
|
||||||
|
<div class="meta" aria-label="Plan metadata">
|
||||||
|
<span class="chip">Created 2026-05-16 17:11</span>
|
||||||
|
<span class="chip">Mode: Plan</span>
|
||||||
|
<span class="chip">Surface: Options Tape</span>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<section class="highlight">
|
||||||
|
<h2>Plan Summary</h2>
|
||||||
|
<p>
|
||||||
|
Treat <strong>stale</strong> strictly as feed health, not as historical-row quality. The user should be able to
|
||||||
|
analyze current live prints and earlier flow in one continuous tape, with no visual distinction between hot-cache
|
||||||
|
rows and ClickHouse-backed rows.
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Goals</h2>
|
||||||
|
<ul>
|
||||||
|
<li>Keep the options tape scrolling infinitely from the user's perspective.</li>
|
||||||
|
<li>Hold only the 100 newest option prints in the hot live cache.</li>
|
||||||
|
<li>Use ClickHouse as the durable source for older rows once the scroll gate requests history.</li>
|
||||||
|
<li>Store all option-print data, including synthetic prints and execution context such as NBBO, spot, and IV.</li>
|
||||||
|
<li>Surface historical flow as real analyzable flow, not as stale, old, or degraded data.</li>
|
||||||
|
<li>Keep the default tape view signal-first while exposing all/raw prints from the existing Filter menu.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Proposed Changes</h2>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
Keep <code>islandflow-0sa</code>'s useful pieces: scroll-hold behavior, <code>LIVE_OPTIONS_HEAD_LIMIT = 100</code>,
|
||||||
|
lazy <code>/history/options</code> loading, cache-first scoped snapshots, and preserved execution-context columns.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Stop tests and UI copy from asserting that valid rows older than 24 hours are <strong>stale</strong> when shown as
|
||||||
|
history.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Keep freshness gating only for live fanout/cache admission and channel health, not for historical validity.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Remove dead <code>LiveHistoryBuffer</code> and auto-hydration scaffolding if it remains unused after the flow is
|
||||||
|
explicit.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Keep the default options tape view as <code>signal</code>, and add a filter-menu view control with
|
||||||
|
<strong>Signal</strong> and <strong>All prints</strong>.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Ensure hot-cache rows and ClickHouse history rows use the same row component, same styling, same sorting, and
|
||||||
|
same interactions.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Keep cursor/key-based deduping so scroll-gated history does not duplicate the 100-row hot head.
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Relevant Context</h2>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
Prior work in <code>islandflow-0sa</code> already introduced scroll hold, a 100-row options head, lazy history,
|
||||||
|
and cache-first scoped snapshots.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
The current storage/types path already includes execution context fields such as <code>execution_nbbo_*</code>,
|
||||||
|
<code>execution_underlying_*</code>, and <code>execution_iv*</code>.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Synthetic options prints already emit some execution context; the durable fix should verify this data survives
|
||||||
|
ClickHouse writes and reads.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
The UI should prefer preserved execution context in row rendering before falling back to current NBBO lookup.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Beads has related work in <code>islandflow-biq</code> for raw live options delivery and filter/backpressure
|
||||||
|
observability.
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Implementation Steps</h2>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
Audit the existing options tape flow from ingest, ClickHouse write/read, live snapshot, history endpoint, and web
|
||||||
|
composition.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Adjust API/live semantics so valid ClickHouse history can be older than freshness thresholds without being treated
|
||||||
|
as degraded.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Add the Filter-menu view toggle for <code>Signal</code> and <code>All prints</code>, with short copy explaining
|
||||||
|
the difference.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Ensure <code>buildOptionTapeQueryParams</code>, live subscriptions, and <code>/history/options</code> all receive
|
||||||
|
the selected view consistently.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Confirm option row rendering uses preserved <code>execution_nbbo_side</code>, <code>execution_underlying_spot</code>,
|
||||||
|
and <code>execution_iv</code> when present.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Remove deprecated or unused history/autohydration code paths that no longer help the intended scroll-gated flow.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Add a deliberate reset path for local and VPS ClickHouse, documented as destructive and operator-confirmed.
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Risks, Limitations, and Mitigations</h2>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
<strong>Risk:</strong> Resetting VPS data is destructive. <strong>Mitigation:</strong> make it a runbook or explicit
|
||||||
|
command with backup/confirmation, never automatic app startup behavior.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<strong>Risk:</strong> The signal/raw toggle could affect both options and flow filters unexpectedly.
|
||||||
|
<strong>Mitigation:</strong> test option subscriptions, history query params, and flow packet filtering separately.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<strong>Risk:</strong> Older history fetch latency could be visible at the scroll gate. <strong>Mitigation:</strong>
|
||||||
|
keep lazy loading, expose loading/error state if needed, and avoid background auto-hydration.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<strong>Risk:</strong> Prior fixes may have left overlapping history logic. <strong>Mitigation:</strong> remove unused
|
||||||
|
scaffolding only after tests cover the intended hot-cache plus ClickHouse path.
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Validation</h2>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
Storage tests: <code>fetchRecentOptionPrints</code> and <code>fetchOptionPrintsBefore</code> return execution NBBO,
|
||||||
|
spot, IV, signal metadata, and raw/signal filtering correctly.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
API/live tests: generic options snapshots cap at 100, scoped snapshots prefer hot cache, history preserves
|
||||||
|
<code>next_before</code>, and rows older than 24 hours return as valid history.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Web tests: Filter menu toggles <code>Signal</code>/<code>All prints</code>, scroll gate calls
|
||||||
|
<code>loadOlder("options")</code>, ClickHouse rows compose with no duplicate seam and no distinct styling, and
|
||||||
|
preserved execution context drives Spot, Side, Details, and IV display.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Validation commands: <code>bun test packages/storage/tests/option-prints.test.ts services/api/tests/live.test.ts apps/web/app/terminal.test.ts</code>
|
||||||
|
and <code>bun --cwd=apps/web run build</code>.
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Open Questions</h2>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
Exact VPS reset command sequence should be confirmed against the live deployment state before execution.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Decide during implementation whether to track the reset/runbook in a new Beads issue or fold it into
|
||||||
|
<code>islandflow-biq</code>.
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<div class="callout">
|
||||||
|
<p>
|
||||||
|
Fixed assumptions: historical ClickHouse rows should be visually indistinguishable from hot-cache rows, and local
|
||||||
|
plus VPS wipe should be an operator-confirmed reset path rather than a background migration.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Loading…
Add table
Add a link
Reference in a new issue