implement durable options tape history
This commit is contained in:
parent
e3940eb0a6
commit
bd60d0d5d5
9 changed files with 423 additions and 56 deletions
|
|
@ -1,3 +1,4 @@
|
||||||
|
{"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
{"_type":"issue","id":"islandflow-k4f","title":"Gate deploy script on docker workspace snapshot sync","description":"Prevent frozen-lockfile build failures during deploy by adding a local preflight in scripts/deploy.ts that runs bun run check:docker-workspace and aborts with a clear sync+commit remediation message when stale.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:01:44Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:04:11Z","started_at":"2026-05-15T23:01:48Z","closed_at":"2026-05-15T23:04:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-k4f","title":"Gate deploy script on docker workspace snapshot sync","description":"Prevent frozen-lockfile build failures during deploy by adding a local preflight in scripts/deploy.ts that runs bun run check:docker-workspace and aborts with a clear sync+commit remediation message when stale.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:01:44Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:04:11Z","started_at":"2026-05-15T23:01:48Z","closed_at":"2026-05-15T23:04:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
{"_type":"issue","id":"islandflow-xll","title":"Fix bun.lock drift causing frozen-lockfile Docker build failures","description":"Docker image builds fail in multiple targets (candles, web, ingest services) because bun install --frozen-lockfile detects lockfile changes. Update workspace lockfile to match manifests and verify frozen install succeeds.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T22:52:38Z","created_by":"dirtydishes","updated_at":"2026-05-15T22:55:23Z","started_at":"2026-05-15T22:52:40Z","closed_at":"2026-05-15T22:55:23Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-xll","title":"Fix bun.lock drift causing frozen-lockfile Docker build failures","description":"Docker image builds fail in multiple targets (candles, web, ingest services) because bun install --frozen-lockfile detects lockfile changes. Update workspace lockfile to match manifests and verify frozen install succeeds.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T22:52:38Z","created_by":"dirtydishes","updated_at":"2026-05-15T22:55:23Z","started_at":"2026-05-15T22:52:40Z","closed_at":"2026-05-15T22:55:23Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
{"_type":"issue","id":"islandflow-9nd","title":"Hosted synthetic tape redesign with internal control surface","description":"Implement hosted synthetic market redesign with shared deterministic regime engine, internal JetStream KV control plane, ingest coupling across options and equities, and an internal bottom-right synthetic-control drawer with Next proxy routes. Preserve the six public smart-money categories while adding hidden subtype families, soft coverage accounting, and backend-only admin endpoints.\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-14T01:25:02Z","created_by":"dirtydishes","updated_at":"2026-05-14T02:10:03Z","started_at":"2026-05-14T01:25:09Z","closed_at":"2026-05-14T02:10:03Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-9nd","title":"Hosted synthetic tape redesign with internal control surface","description":"Implement hosted synthetic market redesign with shared deterministic regime engine, internal JetStream KV control plane, ingest coupling across options and equities, and an internal bottom-right synthetic-control drawer with Next proxy routes. Preserve the six public smart-money categories while adding hidden subtype families, soft coverage accounting, and backend-only admin endpoints.\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-14T01:25:02Z","created_by":"dirtydishes","updated_at":"2026-05-14T02:10:03Z","started_at":"2026-05-14T01:25:09Z","closed_at":"2026-05-14T02:10:03Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
|
|
|
||||||
|
|
@ -606,6 +606,13 @@ h3 {
|
||||||
text-transform: uppercase;
|
text-transform: uppercase;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.flow-filter-section-copy {
|
||||||
|
margin: -2px 0 0;
|
||||||
|
color: var(--text-muted);
|
||||||
|
font-size: 0.78rem;
|
||||||
|
line-height: 1.35;
|
||||||
|
}
|
||||||
|
|
||||||
.flow-filter-checkbox-grid,
|
.flow-filter-checkbox-grid,
|
||||||
.flow-filter-chip-grid {
|
.flow-filter-chip-grid {
|
||||||
display: grid;
|
display: grid;
|
||||||
|
|
@ -617,6 +624,10 @@ h3 {
|
||||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.flow-filter-chip-grid-two {
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
}
|
||||||
|
|
||||||
.flow-filter-check {
|
.flow-filter-check {
|
||||||
display: inline-flex;
|
display: inline-flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,6 @@ import {
|
||||||
getEffectiveOptionPrintFilters,
|
getEffectiveOptionPrintFilters,
|
||||||
getAlertWindowAnchorTs,
|
getAlertWindowAnchorTs,
|
||||||
getHotChannelFeedStatus,
|
getHotChannelFeedStatus,
|
||||||
getScopedLiveAutoHydrationChannels,
|
|
||||||
getLiveHistoryRetentionCap,
|
getLiveHistoryRetentionCap,
|
||||||
getOptionTableSnapshot,
|
getOptionTableSnapshot,
|
||||||
getOptionScope,
|
getOptionScope,
|
||||||
|
|
@ -298,6 +297,24 @@ describe("contract-focused option helpers", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("includes the selected options view in tape query params", () => {
|
||||||
|
expect(
|
||||||
|
buildOptionTapeQueryParams(
|
||||||
|
{
|
||||||
|
...buildDefaultFlowFilters(),
|
||||||
|
view: "raw",
|
||||||
|
securityTypes: undefined,
|
||||||
|
nbboSides: undefined,
|
||||||
|
optionTypes: undefined
|
||||||
|
},
|
||||||
|
{ underlying_ids: ["AAPL"] }
|
||||||
|
)
|
||||||
|
).toEqual({
|
||||||
|
view: "raw",
|
||||||
|
underlying_ids: "AAPL"
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("keeps the focus seed until the matching scoped subscription has loaded it", () => {
|
it("keeps the focus seed until the matching scoped subscription has loaded it", () => {
|
||||||
const seedItem = makeOptionPrint({
|
const seedItem = makeOptionPrint({
|
||||||
trace_id: "focused-seed",
|
trace_id: "focused-seed",
|
||||||
|
|
@ -652,32 +669,6 @@ describe("live tape history helpers", () => {
|
||||||
).toBe(0);
|
).toBe(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not auto-hydrate scoped live history before the scroll gate is reached", () => {
|
|
||||||
const manifest = getLiveManifest(
|
|
||||||
"/tape",
|
|
||||||
"AAPL",
|
|
||||||
60000,
|
|
||||||
buildDefaultFlowFilters(),
|
|
||||||
{
|
|
||||||
underlying_ids: ["AAPL"],
|
|
||||||
option_contract_id: "AAPL-2025-01-17-200-C"
|
|
||||||
},
|
|
||||||
{ underlying_ids: ["AAPL"] }
|
|
||||||
);
|
|
||||||
const historyCursors = Object.fromEntries(
|
|
||||||
manifest.map((subscription) => [getLiveSubscriptionKey(subscription), { ts: 1, seq: 1 }])
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(
|
|
||||||
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {})
|
|
||||||
).toEqual([]);
|
|
||||||
expect(
|
|
||||||
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {
|
|
||||||
[getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "options")!)]: true
|
|
||||||
})
|
|
||||||
).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("restores the same anchor key after live insertions at the top", () => {
|
it("restores the same anchor key after live insertions at the top", () => {
|
||||||
const nextKeys = ["new-1", "new-2", "anchor", "after-1", "after-2"];
|
const nextKeys = ["new-1", "new-2", "anchor", "after-1", "after-2"];
|
||||||
expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(2);
|
expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(2);
|
||||||
|
|
@ -806,6 +797,7 @@ describe("flow filter popup helpers", () => {
|
||||||
|
|
||||||
expect(countActiveFlowFilterGroups(defaults)).toBe(0);
|
expect(countActiveFlowFilterGroups(defaults)).toBe(0);
|
||||||
expect(countActiveFlowFilterGroups(next)).toBe(3);
|
expect(countActiveFlowFilterGroups(next)).toBe(3);
|
||||||
|
expect(countActiveFlowFilterGroups({ ...defaults, view: "raw" })).toBe(1);
|
||||||
expect(buildDefaultFlowFilters()).toEqual(defaults);
|
expect(buildDefaultFlowFilters()).toEqual(defaults);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,7 @@ import type {
|
||||||
LiveHotChannelHealthMap,
|
LiveHotChannelHealthMap,
|
||||||
LiveSubscription,
|
LiveSubscription,
|
||||||
OptionFlowFilters,
|
OptionFlowFilters,
|
||||||
|
OptionFlowView,
|
||||||
OptionNbboSide,
|
OptionNbboSide,
|
||||||
OptionSecurityType,
|
OptionSecurityType,
|
||||||
OptionType,
|
OptionType,
|
||||||
|
|
@ -853,21 +854,6 @@ export const getLiveHistoryRetentionCap = (subscription: LiveSubscription): numb
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
export const getScopedLiveAutoHydrationChannels = (
|
|
||||||
enabled: boolean,
|
|
||||||
pathname: string,
|
|
||||||
manifest: LiveSubscription[],
|
|
||||||
historyCursors: Partial<Record<string, Cursor | null>>,
|
|
||||||
historyLoading: Partial<Record<string, boolean>>
|
|
||||||
): Array<Extract<LiveSubscription["channel"], "options" | "equities">> => {
|
|
||||||
void enabled;
|
|
||||||
void pathname;
|
|
||||||
void manifest;
|
|
||||||
void historyCursors;
|
|
||||||
void historyLoading;
|
|
||||||
return [];
|
|
||||||
};
|
|
||||||
|
|
||||||
export const getLiveFeedStatus = (
|
export const getLiveFeedStatus = (
|
||||||
sourceStatus: WsStatus,
|
sourceStatus: WsStatus,
|
||||||
freshestTs: number | null,
|
freshestTs: number | null,
|
||||||
|
|
@ -1436,6 +1422,9 @@ export const countActiveFlowFilterGroups = (filters: OptionFlowFilters): number
|
||||||
if ((filters.minNotional ?? undefined) !== (defaults.minNotional ?? undefined)) {
|
if ((filters.minNotional ?? undefined) !== (defaults.minNotional ?? undefined)) {
|
||||||
count += 1;
|
count += 1;
|
||||||
}
|
}
|
||||||
|
if ((filters.view ?? defaults.view) !== defaults.view) {
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
|
||||||
return count;
|
return count;
|
||||||
};
|
};
|
||||||
|
|
@ -3684,18 +3673,6 @@ const useLiveSession = (
|
||||||
[enabled, manifest, historyCursors, historyLoading]
|
[enabled, manifest, historyCursors, historyLoading]
|
||||||
);
|
);
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
for (const channel of getScopedLiveAutoHydrationChannels(
|
|
||||||
enabled,
|
|
||||||
pathname,
|
|
||||||
manifest,
|
|
||||||
historyCursors,
|
|
||||||
historyLoading
|
|
||||||
)) {
|
|
||||||
void loadOlder(channel);
|
|
||||||
}
|
|
||||||
}, [enabled, pathname, manifest, historyCursors, historyLoading, loadOlder]);
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
status,
|
status,
|
||||||
connectedAt,
|
connectedAt,
|
||||||
|
|
@ -6904,6 +6881,17 @@ export const FlowFilterPopover = ({ filters, onChange }: FlowFilterPopoverProps)
|
||||||
}));
|
}));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const applyView = (view: OptionFlowView) => {
|
||||||
|
onChange((prev) => ({
|
||||||
|
...prev,
|
||||||
|
view,
|
||||||
|
securityTypes: view === "raw" ? undefined : prev.securityTypes ?? DEFAULT_FLOW_SECURITY_TYPES,
|
||||||
|
nbboSides: view === "raw" ? undefined : prev.nbboSides,
|
||||||
|
optionTypes: view === "raw" ? undefined : prev.optionTypes,
|
||||||
|
minNotional: view === "raw" ? undefined : prev.minNotional
|
||||||
|
}));
|
||||||
|
};
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!open) {
|
if (!open) {
|
||||||
return;
|
return;
|
||||||
|
|
@ -6968,6 +6956,27 @@ export const FlowFilterPopover = ({ filters, onChange }: FlowFilterPopoverProps)
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="flow-filter-popover-body">
|
<div className="flow-filter-popover-body">
|
||||||
|
<FlowFilterSection title="Options View">
|
||||||
|
<div className="flow-filter-chip-grid flow-filter-chip-grid-two">
|
||||||
|
{[
|
||||||
|
{ label: "Signal", value: "signal" as const },
|
||||||
|
{ label: "All prints", value: "raw" as const }
|
||||||
|
].map((preset) => (
|
||||||
|
<button
|
||||||
|
className={`filter-chip ${filters.view === preset.value ? "is-active" : ""}`}
|
||||||
|
key={preset.value}
|
||||||
|
type="button"
|
||||||
|
onClick={() => applyView(preset.value)}
|
||||||
|
>
|
||||||
|
{preset.label}
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
<p className="flow-filter-section-copy">
|
||||||
|
Signal keeps classifier-ready prints. All prints includes raw option tape rows.
|
||||||
|
</p>
|
||||||
|
</FlowFilterSection>
|
||||||
|
|
||||||
<FlowFilterSection title="Security">
|
<FlowFilterSection title="Security">
|
||||||
<div className="flow-filter-checkbox-grid">
|
<div className="flow-filter-checkbox-grid">
|
||||||
{(["stock", "etf"] as OptionSecurityType[]).map((value) => (
|
{(["stock", "etf"] as OptionSecurityType[]).map((value) => (
|
||||||
|
|
|
||||||
57
docs/clickhouse-reset-runbook.md
Normal file
57
docs/clickhouse-reset-runbook.md
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
# ClickHouse Reset Runbook
|
||||||
|
|
||||||
|
This runbook is for deliberately wiping durable market-data history from ClickHouse in local development or on the VPS. It is destructive. Do not run these commands from application startup, deployment hooks, or unattended scripts.
|
||||||
|
|
||||||
|
## When To Use
|
||||||
|
|
||||||
|
Use this only when an operator has decided that existing option, equity, flow, and derived-event history should be discarded and rebuilt from fresh ingest.
|
||||||
|
|
||||||
|
Before running a reset:
|
||||||
|
|
||||||
|
- Confirm the target environment: local Docker or VPS Docker.
|
||||||
|
- Confirm there is no active analysis depending on the existing history.
|
||||||
|
- Take a backup if the data may be needed later.
|
||||||
|
- Stop ingest and API services so new writes do not race the reset.
|
||||||
|
|
||||||
|
## Local Docker Reset
|
||||||
|
|
||||||
|
From the repository root:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bun run dev:infra
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "SHOW TABLES"
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS option_prints"
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS option_nbbo"
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS equity_prints"
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS equity_quotes"
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS equity_print_joins"
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS flow_packets"
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS smart_money_events"
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS classifier_hits"
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS alerts"
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS inferred_dark_events"
|
||||||
|
```
|
||||||
|
|
||||||
|
If the local compose project uses `deployment/docker/docker-compose.yml`, run the same commands with `docker compose -f deployment/docker/docker-compose.yml exec clickhouse ...`.
|
||||||
|
|
||||||
|
## VPS Docker Reset
|
||||||
|
|
||||||
|
On the VPS, first identify the active compose project and ClickHouse service:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker ps --format "table {{.Names}}\t{{.Image}}\t{{.Status}}"
|
||||||
|
docker compose -f deployment/docker/docker-compose.yml ps
|
||||||
|
```
|
||||||
|
|
||||||
|
Then stop writers and run the same `TRUNCATE TABLE IF EXISTS ...` commands against the active ClickHouse container. Prefer `docker compose exec clickhouse clickhouse-client --query "<query>"` when the compose project is healthy; otherwise use `docker exec <clickhouse-container> clickhouse-client --query "<query>"`.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
After the reset:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "SELECT count() FROM option_prints"
|
||||||
|
docker compose exec clickhouse clickhouse-client --query "SELECT count() FROM flow_packets"
|
||||||
|
```
|
||||||
|
|
||||||
|
Restart ingest/API services through the normal dev or deployment path. The options tape should repopulate its 100-row hot head from new signal prints, and older rows should appear only after the scroll gate asks `/history/options` for ClickHouse-backed history.
|
||||||
245
docs/turns/2026-05-16-1725-durable-options-tape-history.html
Normal file
245
docs/turns/2026-05-16-1725-durable-options-tape-history.html
Normal file
|
|
@ -0,0 +1,245 @@
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<title>Durable Options Tape History</title>
|
||||||
|
<style>
|
||||||
|
:root {
|
||||||
|
color-scheme: dark;
|
||||||
|
--bg: #06080b;
|
||||||
|
--panel: #111820;
|
||||||
|
--panel-2: #0d141b;
|
||||||
|
--border: rgba(255, 255, 255, 0.1);
|
||||||
|
--border-strong: rgba(245, 166, 35, 0.34);
|
||||||
|
--text: #e6edf4;
|
||||||
|
--muted: #90a0b2;
|
||||||
|
--faint: #6e7b8c;
|
||||||
|
--accent: #f5a623;
|
||||||
|
--accent-soft: rgba(245, 166, 35, 0.14);
|
||||||
|
--good: #25c17a;
|
||||||
|
}
|
||||||
|
|
||||||
|
* {
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
background: linear-gradient(180deg, rgba(17, 24, 32, 0.92), rgba(6, 8, 11, 0.98) 320px), var(--bg);
|
||||||
|
color: var(--text);
|
||||||
|
font: 15px/1.55 "IBM Plex Sans", -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
|
||||||
|
}
|
||||||
|
|
||||||
|
main {
|
||||||
|
max-width: 1040px;
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 40px 24px 56px;
|
||||||
|
}
|
||||||
|
|
||||||
|
header {
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
padding-bottom: 24px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.eyebrow,
|
||||||
|
h2,
|
||||||
|
.chip {
|
||||||
|
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
letter-spacing: 0.1em;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
.eyebrow {
|
||||||
|
margin: 0 0 10px;
|
||||||
|
color: var(--accent);
|
||||||
|
font-size: 0.76rem;
|
||||||
|
font-weight: 700;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 {
|
||||||
|
margin: 0;
|
||||||
|
max-width: 760px;
|
||||||
|
font-size: 2rem;
|
||||||
|
line-height: 1.1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.summary {
|
||||||
|
max-width: 820px;
|
||||||
|
margin: 16px 0 0;
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.meta {
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 8px;
|
||||||
|
margin-top: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chip {
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 999px;
|
||||||
|
padding: 4px 9px;
|
||||||
|
background: rgba(255, 255, 255, 0.04);
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 0.7rem;
|
||||||
|
font-weight: 700;
|
||||||
|
}
|
||||||
|
|
||||||
|
section {
|
||||||
|
margin-top: 26px;
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 8px;
|
||||||
|
padding: 20px 22px;
|
||||||
|
background: linear-gradient(180deg, rgba(17, 24, 32, 0.94), rgba(13, 20, 27, 0.94));
|
||||||
|
}
|
||||||
|
|
||||||
|
section.summary-band {
|
||||||
|
border-color: var(--border-strong);
|
||||||
|
background: linear-gradient(180deg, rgba(245, 166, 35, 0.12), rgba(17, 24, 32, 0.92));
|
||||||
|
}
|
||||||
|
|
||||||
|
h2 {
|
||||||
|
margin: 0 0 12px;
|
||||||
|
color: var(--text);
|
||||||
|
font-size: 0.82rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
p,
|
||||||
|
li {
|
||||||
|
color: var(--muted);
|
||||||
|
}
|
||||||
|
|
||||||
|
p {
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
ul {
|
||||||
|
margin: 0;
|
||||||
|
padding-left: 1.1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
li {
|
||||||
|
margin: 8px 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
strong {
|
||||||
|
color: var(--text);
|
||||||
|
}
|
||||||
|
|
||||||
|
code {
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||||
|
border-radius: 6px;
|
||||||
|
padding: 0.12rem 0.35rem;
|
||||||
|
background: rgba(255, 255, 255, 0.05);
|
||||||
|
color: var(--text);
|
||||||
|
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
font-size: 0.9em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.ok {
|
||||||
|
color: var(--good);
|
||||||
|
font-weight: 700;
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 720px) {
|
||||||
|
main {
|
||||||
|
padding: 28px 16px 40px;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 {
|
||||||
|
font-size: 1.55rem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<main>
|
||||||
|
<header>
|
||||||
|
<p class="eyebrow">Turn Document</p>
|
||||||
|
<h1>Durable Options Tape History</h1>
|
||||||
|
<p class="summary">
|
||||||
|
Implemented the durable options tape plan: the live hot head is capped at 100 rows, older rows are preserved behind
|
||||||
|
the scroll gate, ClickHouse history keeps execution context, and the Filter menu now exposes Signal versus All
|
||||||
|
prints semantics.
|
||||||
|
</p>
|
||||||
|
<div class="meta" aria-label="Turn metadata">
|
||||||
|
<span class="chip">2026-05-16 17:25</span>
|
||||||
|
<span class="chip">Beads: islandflow-200</span>
|
||||||
|
<span class="chip">Surface: Options Tape</span>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<section class="summary-band">
|
||||||
|
<h2>Summary</h2>
|
||||||
|
<p>
|
||||||
|
The options tape now behaves as a continuous instrument: the live cache stays lean, historical rows arrive only
|
||||||
|
when scrolling asks for them, and old valid rows are not treated as degraded just because they came from durable
|
||||||
|
history.
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Changes Made</h2>
|
||||||
|
<ul>
|
||||||
|
<li>Changed the API default options live cache limit to <code>100</code>.</li>
|
||||||
|
<li>Removed the unused scoped live auto-hydration path so history is loaded by the scroll gate.</li>
|
||||||
|
<li>Fixed unbounded options/equities history retention so a cap of <code>0</code> means keep the loaded tail.</li>
|
||||||
|
<li>Added a Filter menu <code>Options View</code> toggle for <code>Signal</code> and <code>All prints</code>.</li>
|
||||||
|
<li>Ensured All prints clears signal-only side/type/min-notional/security constraints.</li>
|
||||||
|
<li>Added a destructive ClickHouse reset runbook for local and VPS operators.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Context</h2>
|
||||||
|
<p>
|
||||||
|
The prior plan called out useful partial work already in the repo: ClickHouse history endpoints, execution-context
|
||||||
|
columns, scroll-hold behavior, and a shared row renderer. This implementation keeps those pieces and removes the
|
||||||
|
ambiguous history/autohydration behavior around them.
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Important Implementation Details</h2>
|
||||||
|
<ul>
|
||||||
|
<li><code>/history/options</code> still uses the selected option filters and scope, including raw contract drilldowns.</li>
|
||||||
|
<li>Storage tests now verify execution NBBO side, underlying spot, IV, and signal reasons survive normalization.</li>
|
||||||
|
<li>The options row path already preferred <code>execution_nbbo_side</code>, <code>execution_underlying_spot</code>, and <code>execution_iv</code>; tests cover that behavior.</li>
|
||||||
|
<li>The reset runbook is documented in <code>docs/clickhouse-reset-runbook.md</code> and is explicitly operator-confirmed.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Expected Impact for End-Users</h2>
|
||||||
|
<p>
|
||||||
|
Traders can stay on a signal-first tape by default, switch to raw prints when investigating, and scroll into older
|
||||||
|
ClickHouse-backed flow without seeing a separate stale-history treatment.
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Validation</h2>
|
||||||
|
<ul>
|
||||||
|
<li><span class="ok">Passed:</span> <code>bun test packages/storage/tests/option-prints.test.ts services/api/tests/live.test.ts apps/web/app/terminal.test.ts</code></li>
|
||||||
|
<li><span class="ok">Passed:</span> <code>bun --cwd=apps/web run build</code></li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Issues, Limitations, and Mitigations</h2>
|
||||||
|
<ul>
|
||||||
|
<li>The ClickHouse reset remains destructive. Mitigation: documented as a manual runbook only, never automatic startup behavior.</li>
|
||||||
|
<li>No live browser smoke test was run in this turn. Mitigation: unit coverage and production build exercised the changed web paths.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Follow-up Work</h2>
|
||||||
|
<p>No new follow-up issue was needed. The implementation task is tracked and completed in <code>islandflow-200</code>.</p>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -48,6 +48,25 @@ describe("option-prints storage helpers", () => {
|
||||||
queries.push(query);
|
queries.push(query);
|
||||||
return {
|
return {
|
||||||
async json<T>() {
|
async json<T>() {
|
||||||
|
if (query.includes("trace-ctx")) {
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
...basePrint,
|
||||||
|
trace_id: "trace-ctx",
|
||||||
|
conditions: [],
|
||||||
|
execution_nbbo_bid: "1.20",
|
||||||
|
execution_nbbo_ask: "1.30",
|
||||||
|
execution_nbbo_mid: "1.25",
|
||||||
|
execution_nbbo_side: "A",
|
||||||
|
execution_underlying_spot: "450.05",
|
||||||
|
execution_underlying_source: "equity_quote_mid",
|
||||||
|
execution_iv: "0.42",
|
||||||
|
execution_iv_source: "synthetic_pressure_model",
|
||||||
|
signal_reasons: ["large_notional"],
|
||||||
|
signal_pass: 1
|
||||||
|
}
|
||||||
|
] as T;
|
||||||
|
}
|
||||||
return [] as T;
|
return [] as T;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
@ -63,8 +82,9 @@ describe("option-prints storage helpers", () => {
|
||||||
optionContractId: "AAPL-2025-01-17-200-C",
|
optionContractId: "AAPL-2025-01-17-200-C",
|
||||||
sinceTs: 123
|
sinceTs: 123
|
||||||
});
|
});
|
||||||
await fetchOptionPrintsBefore(client, 100, 5, 20, "alpaca");
|
await fetchOptionPrintsBefore(client, 100, 5, 20, "alpaca", { view: "raw" });
|
||||||
await fetchOptionPrintsByTraceIds(client, ["trace-1", "trace-2"]);
|
await fetchOptionPrintsByTraceIds(client, ["trace-1", "trace-2"]);
|
||||||
|
const rows = await fetchRecentOptionPrints(client, 1, "trace-ctx", { view: "signal" });
|
||||||
|
|
||||||
expect(queries[0]).toContain("signal_pass = 1");
|
expect(queries[0]).toContain("signal_pass = 1");
|
||||||
expect(queries[0]).toContain("(is_etf = 0 OR is_etf IS NULL)");
|
expect(queries[0]).toContain("(is_etf = 0 OR is_etf IS NULL)");
|
||||||
|
|
@ -76,7 +96,12 @@ describe("option-prints storage helpers", () => {
|
||||||
expect(queries[0]).toContain("ts >= 123");
|
expect(queries[0]).toContain("ts >= 123");
|
||||||
expect(queries[1]).toContain("(ts, seq) < (100, 5)");
|
expect(queries[1]).toContain("(ts, seq) < (100, 5)");
|
||||||
expect(queries[1]).toContain("startsWith(trace_id, 'alpaca')");
|
expect(queries[1]).toContain("startsWith(trace_id, 'alpaca')");
|
||||||
|
expect(queries[1]).not.toContain("signal_pass = 1");
|
||||||
expect(queries[1]).toContain("ORDER BY ts DESC, seq DESC LIMIT 20");
|
expect(queries[1]).toContain("ORDER BY ts DESC, seq DESC LIMIT 20");
|
||||||
expect(queries[2]).toContain("trace_id IN ('trace-1', 'trace-2')");
|
expect(queries[2]).toContain("trace_id IN ('trace-1', 'trace-2')");
|
||||||
|
expect(rows[0].execution_nbbo_side).toBe("A");
|
||||||
|
expect(rows[0].execution_underlying_spot).toBe(450.05);
|
||||||
|
expect(rows[0].execution_iv).toBe(0.42);
|
||||||
|
expect(rows[0].signal_reasons).toEqual(["large_notional"]);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -72,7 +72,7 @@ const CHART_LIMITS = {
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
const DEFAULT_LIVE_LIMITS: GenericLiveLimits = {
|
const DEFAULT_LIVE_LIMITS: GenericLiveLimits = {
|
||||||
options: 1000,
|
options: 100,
|
||||||
nbbo: 1000,
|
nbbo: 1000,
|
||||||
equities: 1000,
|
equities: 1000,
|
||||||
"equity-quotes": 500,
|
"equity-quotes": 500,
|
||||||
|
|
|
||||||
|
|
@ -69,6 +69,7 @@ describe("LiveStateManager", () => {
|
||||||
expect(limits.flow).toBe(500);
|
expect(limits.flow).toBe(500);
|
||||||
expect(limits["equity-quotes"]).toBe(500);
|
expect(limits["equity-quotes"]).toBe(500);
|
||||||
expect(limits.alerts).toBe(300);
|
expect(limits.alerts).toBe(300);
|
||||||
|
expect(resolveGenericLiveLimits({} as NodeJS.ProcessEnv).options).toBe(100);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("hydrates snapshots from redis generic windows", async () => {
|
it("hydrates snapshots from redis generic windows", async () => {
|
||||||
|
|
@ -520,6 +521,32 @@ describe("LiveStateManager", () => {
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("caps generic options snapshots at the 100-row hot head by default", async () => {
|
||||||
|
const manager = new LiveStateManager(makeClickHouse(), null);
|
||||||
|
const now = Date.now();
|
||||||
|
|
||||||
|
for (let seq = 1; seq <= 150; seq += 1) {
|
||||||
|
await manager.ingest("options", {
|
||||||
|
source_ts: now + seq,
|
||||||
|
ingest_ts: now + seq,
|
||||||
|
seq,
|
||||||
|
trace_id: `opt-${seq}`,
|
||||||
|
ts: now + seq,
|
||||||
|
option_contract_id: "AAPL-2025-01-17-200-C",
|
||||||
|
price: 1,
|
||||||
|
size: 10,
|
||||||
|
exchange: "X",
|
||||||
|
signal_pass: true
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const snapshot = await manager.getSnapshot({ channel: "options" });
|
||||||
|
|
||||||
|
expect(snapshot.items).toHaveLength(100);
|
||||||
|
expect((snapshot.items as Array<{ trace_id: string }>)[0].trace_id).toBe("opt-150");
|
||||||
|
expect(snapshot.next_before).toEqual({ ts: now + 51, seq: 51 });
|
||||||
|
});
|
||||||
|
|
||||||
it("seeds scoped option snapshots from clickhouse rows older than 24h", async () => {
|
it("seeds scoped option snapshots from clickhouse rows older than 24h", async () => {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const staleTs = now - 25 * 60 * 60 * 1000;
|
const staleTs = now - 25 * 60 * 60 * 1000;
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue