implement durable options tape history

This commit is contained in:
dirtydishes 2026-05-16 17:27:02 -04:00
parent e3940eb0a6
commit bd60d0d5d5
9 changed files with 423 additions and 56 deletions

View file

@ -1,3 +1,4 @@
{"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-k4f","title":"Gate deploy script on docker workspace snapshot sync","description":"Prevent frozen-lockfile build failures during deploy by adding a local preflight in scripts/deploy.ts that runs bun run check:docker-workspace and aborts with a clear sync+commit remediation message when stale.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:01:44Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:04:11Z","started_at":"2026-05-15T23:01:48Z","closed_at":"2026-05-15T23:04:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-k4f","title":"Gate deploy script on docker workspace snapshot sync","description":"Prevent frozen-lockfile build failures during deploy by adding a local preflight in scripts/deploy.ts that runs bun run check:docker-workspace and aborts with a clear sync+commit remediation message when stale.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:01:44Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:04:11Z","started_at":"2026-05-15T23:01:48Z","closed_at":"2026-05-15T23:04:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-xll","title":"Fix bun.lock drift causing frozen-lockfile Docker build failures","description":"Docker image builds fail in multiple targets (candles, web, ingest services) because bun install --frozen-lockfile detects lockfile changes. Update workspace lockfile to match manifests and verify frozen install succeeds.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T22:52:38Z","created_by":"dirtydishes","updated_at":"2026-05-15T22:55:23Z","started_at":"2026-05-15T22:52:40Z","closed_at":"2026-05-15T22:55:23Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-xll","title":"Fix bun.lock drift causing frozen-lockfile Docker build failures","description":"Docker image builds fail in multiple targets (candles, web, ingest services) because bun install --frozen-lockfile detects lockfile changes. Update workspace lockfile to match manifests and verify frozen install succeeds.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T22:52:38Z","created_by":"dirtydishes","updated_at":"2026-05-15T22:55:23Z","started_at":"2026-05-15T22:52:40Z","closed_at":"2026-05-15T22:55:23Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-9nd","title":"Hosted synthetic tape redesign with internal control surface","description":"Implement hosted synthetic market redesign with shared deterministic regime engine, internal JetStream KV control plane, ingest coupling across options and equities, and an internal bottom-right synthetic-control drawer with Next proxy routes. Preserve the six public smart-money categories while adding hidden subtype families, soft coverage accounting, and backend-only admin endpoints.\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-14T01:25:02Z","created_by":"dirtydishes","updated_at":"2026-05-14T02:10:03Z","started_at":"2026-05-14T01:25:09Z","closed_at":"2026-05-14T02:10:03Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-9nd","title":"Hosted synthetic tape redesign with internal control surface","description":"Implement hosted synthetic market redesign with shared deterministic regime engine, internal JetStream KV control plane, ingest coupling across options and equities, and an internal bottom-right synthetic-control drawer with Next proxy routes. Preserve the six public smart-money categories while adding hidden subtype families, soft coverage accounting, and backend-only admin endpoints.\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-14T01:25:02Z","created_by":"dirtydishes","updated_at":"2026-05-14T02:10:03Z","started_at":"2026-05-14T01:25:09Z","closed_at":"2026-05-14T02:10:03Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}

View file

@ -606,6 +606,13 @@ h3 {
text-transform: uppercase; text-transform: uppercase;
} }
.flow-filter-section-copy {
margin: -2px 0 0;
color: var(--text-muted);
font-size: 0.78rem;
line-height: 1.35;
}
.flow-filter-checkbox-grid, .flow-filter-checkbox-grid,
.flow-filter-chip-grid { .flow-filter-chip-grid {
display: grid; display: grid;
@ -617,6 +624,10 @@ h3 {
grid-template-columns: repeat(3, minmax(0, 1fr)); grid-template-columns: repeat(3, minmax(0, 1fr));
} }
.flow-filter-chip-grid-two {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.flow-filter-check { .flow-filter-check {
display: inline-flex; display: inline-flex;
align-items: center; align-items: center;

View file

@ -17,7 +17,6 @@ import {
getEffectiveOptionPrintFilters, getEffectiveOptionPrintFilters,
getAlertWindowAnchorTs, getAlertWindowAnchorTs,
getHotChannelFeedStatus, getHotChannelFeedStatus,
getScopedLiveAutoHydrationChannels,
getLiveHistoryRetentionCap, getLiveHistoryRetentionCap,
getOptionTableSnapshot, getOptionTableSnapshot,
getOptionScope, getOptionScope,
@ -298,6 +297,24 @@ describe("contract-focused option helpers", () => {
}); });
}); });
it("includes the selected options view in tape query params", () => {
expect(
buildOptionTapeQueryParams(
{
...buildDefaultFlowFilters(),
view: "raw",
securityTypes: undefined,
nbboSides: undefined,
optionTypes: undefined
},
{ underlying_ids: ["AAPL"] }
)
).toEqual({
view: "raw",
underlying_ids: "AAPL"
});
});
it("keeps the focus seed until the matching scoped subscription has loaded it", () => { it("keeps the focus seed until the matching scoped subscription has loaded it", () => {
const seedItem = makeOptionPrint({ const seedItem = makeOptionPrint({
trace_id: "focused-seed", trace_id: "focused-seed",
@ -652,32 +669,6 @@ describe("live tape history helpers", () => {
).toBe(0); ).toBe(0);
}); });
it("does not auto-hydrate scoped live history before the scroll gate is reached", () => {
const manifest = getLiveManifest(
"/tape",
"AAPL",
60000,
buildDefaultFlowFilters(),
{
underlying_ids: ["AAPL"],
option_contract_id: "AAPL-2025-01-17-200-C"
},
{ underlying_ids: ["AAPL"] }
);
const historyCursors = Object.fromEntries(
manifest.map((subscription) => [getLiveSubscriptionKey(subscription), { ts: 1, seq: 1 }])
);
expect(
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {})
).toEqual([]);
expect(
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {
[getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "options")!)]: true
})
).toEqual([]);
});
it("restores the same anchor key after live insertions at the top", () => { it("restores the same anchor key after live insertions at the top", () => {
const nextKeys = ["new-1", "new-2", "anchor", "after-1", "after-2"]; const nextKeys = ["new-1", "new-2", "anchor", "after-1", "after-2"];
expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(2); expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(2);
@ -806,6 +797,7 @@ describe("flow filter popup helpers", () => {
expect(countActiveFlowFilterGroups(defaults)).toBe(0); expect(countActiveFlowFilterGroups(defaults)).toBe(0);
expect(countActiveFlowFilterGroups(next)).toBe(3); expect(countActiveFlowFilterGroups(next)).toBe(3);
expect(countActiveFlowFilterGroups({ ...defaults, view: "raw" })).toBe(1);
expect(buildDefaultFlowFilters()).toEqual(defaults); expect(buildDefaultFlowFilters()).toEqual(defaults);
}); });
}); });

View file

@ -34,6 +34,7 @@ import type {
LiveHotChannelHealthMap, LiveHotChannelHealthMap,
LiveSubscription, LiveSubscription,
OptionFlowFilters, OptionFlowFilters,
OptionFlowView,
OptionNbboSide, OptionNbboSide,
OptionSecurityType, OptionSecurityType,
OptionType, OptionType,
@ -853,21 +854,6 @@ export const getLiveHistoryRetentionCap = (subscription: LiveSubscription): numb
} }
}; };
export const getScopedLiveAutoHydrationChannels = (
enabled: boolean,
pathname: string,
manifest: LiveSubscription[],
historyCursors: Partial<Record<string, Cursor | null>>,
historyLoading: Partial<Record<string, boolean>>
): Array<Extract<LiveSubscription["channel"], "options" | "equities">> => {
void enabled;
void pathname;
void manifest;
void historyCursors;
void historyLoading;
return [];
};
export const getLiveFeedStatus = ( export const getLiveFeedStatus = (
sourceStatus: WsStatus, sourceStatus: WsStatus,
freshestTs: number | null, freshestTs: number | null,
@ -1436,6 +1422,9 @@ export const countActiveFlowFilterGroups = (filters: OptionFlowFilters): number
if ((filters.minNotional ?? undefined) !== (defaults.minNotional ?? undefined)) { if ((filters.minNotional ?? undefined) !== (defaults.minNotional ?? undefined)) {
count += 1; count += 1;
} }
if ((filters.view ?? defaults.view) !== defaults.view) {
count += 1;
}
return count; return count;
}; };
@ -3684,18 +3673,6 @@ const useLiveSession = (
[enabled, manifest, historyCursors, historyLoading] [enabled, manifest, historyCursors, historyLoading]
); );
useEffect(() => {
for (const channel of getScopedLiveAutoHydrationChannels(
enabled,
pathname,
manifest,
historyCursors,
historyLoading
)) {
void loadOlder(channel);
}
}, [enabled, pathname, manifest, historyCursors, historyLoading, loadOlder]);
return { return {
status, status,
connectedAt, connectedAt,
@ -6904,6 +6881,17 @@ export const FlowFilterPopover = ({ filters, onChange }: FlowFilterPopoverProps)
})); }));
}; };
const applyView = (view: OptionFlowView) => {
onChange((prev) => ({
...prev,
view,
securityTypes: view === "raw" ? undefined : prev.securityTypes ?? DEFAULT_FLOW_SECURITY_TYPES,
nbboSides: view === "raw" ? undefined : prev.nbboSides,
optionTypes: view === "raw" ? undefined : prev.optionTypes,
minNotional: view === "raw" ? undefined : prev.minNotional
}));
};
useEffect(() => { useEffect(() => {
if (!open) { if (!open) {
return; return;
@ -6968,6 +6956,27 @@ export const FlowFilterPopover = ({ filters, onChange }: FlowFilterPopoverProps)
</div> </div>
<div className="flow-filter-popover-body"> <div className="flow-filter-popover-body">
<FlowFilterSection title="Options View">
<div className="flow-filter-chip-grid flow-filter-chip-grid-two">
{[
{ label: "Signal", value: "signal" as const },
{ label: "All prints", value: "raw" as const }
].map((preset) => (
<button
className={`filter-chip ${filters.view === preset.value ? "is-active" : ""}`}
key={preset.value}
type="button"
onClick={() => applyView(preset.value)}
>
{preset.label}
</button>
))}
</div>
<p className="flow-filter-section-copy">
Signal keeps classifier-ready prints. All prints includes raw option tape rows.
</p>
</FlowFilterSection>
<FlowFilterSection title="Security"> <FlowFilterSection title="Security">
<div className="flow-filter-checkbox-grid"> <div className="flow-filter-checkbox-grid">
{(["stock", "etf"] as OptionSecurityType[]).map((value) => ( {(["stock", "etf"] as OptionSecurityType[]).map((value) => (

View file

@ -0,0 +1,57 @@
# ClickHouse Reset Runbook
This runbook is for deliberately wiping durable market-data history from ClickHouse in local development or on the VPS. It is destructive. Do not run these commands from application startup, deployment hooks, or unattended scripts.
## When To Use
Use this only when an operator has decided that existing option, equity, flow, and derived-event history should be discarded and rebuilt from fresh ingest.
Before running a reset:
- Confirm the target environment: local Docker or VPS Docker.
- Confirm there is no active analysis depending on the existing history.
- Take a backup if the data may be needed later.
- Stop ingest and API services so new writes do not race the reset.
## Local Docker Reset
From the repository root:
```bash
bun run dev:infra
docker compose exec clickhouse clickhouse-client --query "SHOW TABLES"
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS option_prints"
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS option_nbbo"
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS equity_prints"
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS equity_quotes"
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS equity_print_joins"
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS flow_packets"
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS smart_money_events"
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS classifier_hits"
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS alerts"
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS inferred_dark_events"
```
If the local compose project uses `deployment/docker/docker-compose.yml`, run the same commands with `docker compose -f deployment/docker/docker-compose.yml exec clickhouse ...`.
## VPS Docker Reset
On the VPS, first identify the active compose project and ClickHouse service:
```bash
docker ps --format "table {{.Names}}\t{{.Image}}\t{{.Status}}"
docker compose -f deployment/docker/docker-compose.yml ps
```
Then stop writers and run the same `TRUNCATE TABLE IF EXISTS ...` commands against the active ClickHouse container. Prefer `docker compose exec clickhouse clickhouse-client --query "<query>"` when the compose project is healthy; otherwise use `docker exec <clickhouse-container> clickhouse-client --query "<query>"`.
## Verification
After the reset:
```bash
docker compose exec clickhouse clickhouse-client --query "SELECT count() FROM option_prints"
docker compose exec clickhouse clickhouse-client --query "SELECT count() FROM flow_packets"
```
Restart ingest/API services through the normal dev or deployment path. The options tape should repopulate its 100-row hot head from new signal prints, and older rows should appear only after the scroll gate asks `/history/options` for ClickHouse-backed history.

View file

@ -0,0 +1,245 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Durable Options Tape History</title>
<style>
:root {
color-scheme: dark;
--bg: #06080b;
--panel: #111820;
--panel-2: #0d141b;
--border: rgba(255, 255, 255, 0.1);
--border-strong: rgba(245, 166, 35, 0.34);
--text: #e6edf4;
--muted: #90a0b2;
--faint: #6e7b8c;
--accent: #f5a623;
--accent-soft: rgba(245, 166, 35, 0.14);
--good: #25c17a;
}
* {
box-sizing: border-box;
}
body {
margin: 0;
background: linear-gradient(180deg, rgba(17, 24, 32, 0.92), rgba(6, 8, 11, 0.98) 320px), var(--bg);
color: var(--text);
font: 15px/1.55 "IBM Plex Sans", -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
}
main {
max-width: 1040px;
margin: 0 auto;
padding: 40px 24px 56px;
}
header {
border-bottom: 1px solid var(--border);
padding-bottom: 24px;
}
.eyebrow,
h2,
.chip {
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
letter-spacing: 0.1em;
text-transform: uppercase;
}
.eyebrow {
margin: 0 0 10px;
color: var(--accent);
font-size: 0.76rem;
font-weight: 700;
}
h1 {
margin: 0;
max-width: 760px;
font-size: 2rem;
line-height: 1.1;
}
.summary {
max-width: 820px;
margin: 16px 0 0;
color: var(--muted);
font-size: 1rem;
}
.meta {
display: flex;
flex-wrap: wrap;
gap: 8px;
margin-top: 18px;
}
.chip {
border: 1px solid var(--border);
border-radius: 999px;
padding: 4px 9px;
background: rgba(255, 255, 255, 0.04);
color: var(--muted);
font-size: 0.7rem;
font-weight: 700;
}
section {
margin-top: 26px;
border: 1px solid var(--border);
border-radius: 8px;
padding: 20px 22px;
background: linear-gradient(180deg, rgba(17, 24, 32, 0.94), rgba(13, 20, 27, 0.94));
}
section.summary-band {
border-color: var(--border-strong);
background: linear-gradient(180deg, rgba(245, 166, 35, 0.12), rgba(17, 24, 32, 0.92));
}
h2 {
margin: 0 0 12px;
color: var(--text);
font-size: 0.82rem;
}
p,
li {
color: var(--muted);
}
p {
margin: 0;
}
ul {
margin: 0;
padding-left: 1.1rem;
}
li {
margin: 8px 0;
}
strong {
color: var(--text);
}
code {
border: 1px solid rgba(255, 255, 255, 0.08);
border-radius: 6px;
padding: 0.12rem 0.35rem;
background: rgba(255, 255, 255, 0.05);
color: var(--text);
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 0.9em;
}
.ok {
color: var(--good);
font-weight: 700;
}
@media (max-width: 720px) {
main {
padding: 28px 16px 40px;
}
h1 {
font-size: 1.55rem;
}
}
</style>
</head>
<body>
<main>
<header>
<p class="eyebrow">Turn Document</p>
<h1>Durable Options Tape History</h1>
<p class="summary">
Implemented the durable options tape plan: the live hot head is capped at 100 rows, older rows are preserved behind
the scroll gate, ClickHouse history keeps execution context, and the Filter menu now exposes Signal versus All
prints semantics.
</p>
<div class="meta" aria-label="Turn metadata">
<span class="chip">2026-05-16 17:25</span>
<span class="chip">Beads: islandflow-200</span>
<span class="chip">Surface: Options Tape</span>
</div>
</header>
<section class="summary-band">
<h2>Summary</h2>
<p>
The options tape now behaves as a continuous instrument: the live cache stays lean, historical rows arrive only
when scrolling asks for them, and old valid rows are not treated as degraded just because they came from durable
history.
</p>
</section>
<section>
<h2>Changes Made</h2>
<ul>
<li>Changed the API default options live cache limit to <code>100</code>.</li>
<li>Removed the unused scoped live auto-hydration path so history is loaded by the scroll gate.</li>
<li>Fixed unbounded options/equities history retention so a cap of <code>0</code> means keep the loaded tail.</li>
<li>Added a Filter menu <code>Options View</code> toggle for <code>Signal</code> and <code>All prints</code>.</li>
<li>Ensured All prints clears signal-only side/type/min-notional/security constraints.</li>
<li>Added a destructive ClickHouse reset runbook for local and VPS operators.</li>
</ul>
</section>
<section>
<h2>Context</h2>
<p>
The prior plan called out useful partial work already in the repo: ClickHouse history endpoints, execution-context
columns, scroll-hold behavior, and a shared row renderer. This implementation keeps those pieces and removes the
ambiguous history/autohydration behavior around them.
</p>
</section>
<section>
<h2>Important Implementation Details</h2>
<ul>
<li><code>/history/options</code> still uses the selected option filters and scope, including raw contract drilldowns.</li>
<li>Storage tests now verify execution NBBO side, underlying spot, IV, and signal reasons survive normalization.</li>
<li>The options row path already preferred <code>execution_nbbo_side</code>, <code>execution_underlying_spot</code>, and <code>execution_iv</code>; tests cover that behavior.</li>
<li>The reset runbook is documented in <code>docs/clickhouse-reset-runbook.md</code> and is explicitly operator-confirmed.</li>
</ul>
</section>
<section>
<h2>Expected Impact for End-Users</h2>
<p>
Traders can stay on a signal-first tape by default, switch to raw prints when investigating, and scroll into older
ClickHouse-backed flow without seeing a separate stale-history treatment.
</p>
</section>
<section>
<h2>Validation</h2>
<ul>
<li><span class="ok">Passed:</span> <code>bun test packages/storage/tests/option-prints.test.ts services/api/tests/live.test.ts apps/web/app/terminal.test.ts</code></li>
<li><span class="ok">Passed:</span> <code>bun --cwd=apps/web run build</code></li>
</ul>
</section>
<section>
<h2>Issues, Limitations, and Mitigations</h2>
<ul>
<li>The ClickHouse reset remains destructive. Mitigation: documented as a manual runbook only, never automatic startup behavior.</li>
<li>No live browser smoke test was run in this turn. Mitigation: unit coverage and production build exercised the changed web paths.</li>
</ul>
</section>
<section>
<h2>Follow-up Work</h2>
<p>No new follow-up issue was needed. The implementation task is tracked and completed in <code>islandflow-200</code>.</p>
</section>
</main>
</body>
</html>

View file

@ -48,6 +48,25 @@ describe("option-prints storage helpers", () => {
queries.push(query); queries.push(query);
return { return {
async json<T>() { async json<T>() {
if (query.includes("trace-ctx")) {
return [
{
...basePrint,
trace_id: "trace-ctx",
conditions: [],
execution_nbbo_bid: "1.20",
execution_nbbo_ask: "1.30",
execution_nbbo_mid: "1.25",
execution_nbbo_side: "A",
execution_underlying_spot: "450.05",
execution_underlying_source: "equity_quote_mid",
execution_iv: "0.42",
execution_iv_source: "synthetic_pressure_model",
signal_reasons: ["large_notional"],
signal_pass: 1
}
] as T;
}
return [] as T; return [] as T;
} }
}; };
@ -63,8 +82,9 @@ describe("option-prints storage helpers", () => {
optionContractId: "AAPL-2025-01-17-200-C", optionContractId: "AAPL-2025-01-17-200-C",
sinceTs: 123 sinceTs: 123
}); });
await fetchOptionPrintsBefore(client, 100, 5, 20, "alpaca"); await fetchOptionPrintsBefore(client, 100, 5, 20, "alpaca", { view: "raw" });
await fetchOptionPrintsByTraceIds(client, ["trace-1", "trace-2"]); await fetchOptionPrintsByTraceIds(client, ["trace-1", "trace-2"]);
const rows = await fetchRecentOptionPrints(client, 1, "trace-ctx", { view: "signal" });
expect(queries[0]).toContain("signal_pass = 1"); expect(queries[0]).toContain("signal_pass = 1");
expect(queries[0]).toContain("(is_etf = 0 OR is_etf IS NULL)"); expect(queries[0]).toContain("(is_etf = 0 OR is_etf IS NULL)");
@ -76,7 +96,12 @@ describe("option-prints storage helpers", () => {
expect(queries[0]).toContain("ts >= 123"); expect(queries[0]).toContain("ts >= 123");
expect(queries[1]).toContain("(ts, seq) < (100, 5)"); expect(queries[1]).toContain("(ts, seq) < (100, 5)");
expect(queries[1]).toContain("startsWith(trace_id, 'alpaca')"); expect(queries[1]).toContain("startsWith(trace_id, 'alpaca')");
expect(queries[1]).not.toContain("signal_pass = 1");
expect(queries[1]).toContain("ORDER BY ts DESC, seq DESC LIMIT 20"); expect(queries[1]).toContain("ORDER BY ts DESC, seq DESC LIMIT 20");
expect(queries[2]).toContain("trace_id IN ('trace-1', 'trace-2')"); expect(queries[2]).toContain("trace_id IN ('trace-1', 'trace-2')");
expect(rows[0].execution_nbbo_side).toBe("A");
expect(rows[0].execution_underlying_spot).toBe(450.05);
expect(rows[0].execution_iv).toBe(0.42);
expect(rows[0].signal_reasons).toEqual(["large_notional"]);
}); });
}); });

View file

@ -72,7 +72,7 @@ const CHART_LIMITS = {
} as const; } as const;
const DEFAULT_LIVE_LIMITS: GenericLiveLimits = { const DEFAULT_LIVE_LIMITS: GenericLiveLimits = {
options: 1000, options: 100,
nbbo: 1000, nbbo: 1000,
equities: 1000, equities: 1000,
"equity-quotes": 500, "equity-quotes": 500,

View file

@ -69,6 +69,7 @@ describe("LiveStateManager", () => {
expect(limits.flow).toBe(500); expect(limits.flow).toBe(500);
expect(limits["equity-quotes"]).toBe(500); expect(limits["equity-quotes"]).toBe(500);
expect(limits.alerts).toBe(300); expect(limits.alerts).toBe(300);
expect(resolveGenericLiveLimits({} as NodeJS.ProcessEnv).options).toBe(100);
}); });
it("hydrates snapshots from redis generic windows", async () => { it("hydrates snapshots from redis generic windows", async () => {
@ -520,6 +521,32 @@ describe("LiveStateManager", () => {
]); ]);
}); });
it("caps generic options snapshots at the 100-row hot head by default", async () => {
const manager = new LiveStateManager(makeClickHouse(), null);
const now = Date.now();
for (let seq = 1; seq <= 150; seq += 1) {
await manager.ingest("options", {
source_ts: now + seq,
ingest_ts: now + seq,
seq,
trace_id: `opt-${seq}`,
ts: now + seq,
option_contract_id: "AAPL-2025-01-17-200-C",
price: 1,
size: 10,
exchange: "X",
signal_pass: true
});
}
const snapshot = await manager.getSnapshot({ channel: "options" });
expect(snapshot.items).toHaveLength(100);
expect((snapshot.items as Array<{ trace_id: string }>)[0].trace_id).toBe("opt-150");
expect(snapshot.next_before).toEqual({ ts: now + 51, seq: 51 });
});
it("seeds scoped option snapshots from clickhouse rows older than 24h", async () => { it("seeds scoped option snapshots from clickhouse rows older than 24h", async () => {
const now = Date.now(); const now = Date.now();
const staleTs = now - 25 * 60 * 60 * 1000; const staleTs = now - 25 * 60 * 60 * 1000;