Compare commits
10 commits
7caef80718
...
88b2c33ab3
| Author | SHA1 | Date | |
|---|---|---|---|
|
88b2c33ab3 | ||
| d334e16874 | |||
| 1424a2716f | |||
| 23ed3809cc | |||
|
|
f4108b9fe2 | ||
| 2abdd24e2c | |||
| bd60d0d5d5 | |||
| e3940eb0a6 | |||
| 39fb5ce9f1 | |||
| eaddf4b7a0 |
23 changed files with 2054 additions and 161 deletions
|
|
@ -1,3 +1,6 @@
|
|||
{"_type":"issue","id":"islandflow-9dg","title":"Fix live tape scroll stability","description":"Live tape rows can shift while a user is scrolled away from the hot head because newer live prints and ClickHouse history are merged into the displayed segment. Implement held-history freezing so only truly older rows append below the current tail, resync on jump-to-top, and tune virtualization/background rendering to reduce fast-scroll blank gaps.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T07:28:52Z","created_by":"dirtydishes","updated_at":"2026-05-17T07:32:53Z","started_at":"2026-05-17T07:29:00Z","closed_at":"2026-05-17T07:32:53Z","close_reason":"Implemented held live tape history freezing, older-only held history append, jump-to-top resync behavior, virtualizer overscan tuning, and stable row-lane table background. Validated with scoped Bun tests, web production build, and local /tape HTTP smoke check.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-qso","title":"Fix durable options tape history routing","description":"Implement the fix-tape plan: make same-origin history routing durable, add deployment/public smoke checks for required API routes, expose tape history loading failures in the UI, document the work, and track api.flow.deltaisland.io migration separately.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T01:53:22Z","created_by":"dirtydishes","updated_at":"2026-05-17T02:00:04Z","started_at":"2026-05-17T01:53:25Z","closed_at":"2026-05-17T02:00:04Z","close_reason":"Implemented durable same-origin history routing, public route smoke checks, tape history diagnostics, docs, validation, and follow-up tracking for api.flow.deltaisland.io.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-k4f","title":"Gate deploy script on docker workspace snapshot sync","description":"Prevent frozen-lockfile build failures during deploy by adding a local preflight in scripts/deploy.ts that runs bun run check:docker-workspace and aborts with a clear sync+commit remediation message when stale.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:01:44Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:04:11Z","started_at":"2026-05-15T23:01:48Z","closed_at":"2026-05-15T23:04:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-xll","title":"Fix bun.lock drift causing frozen-lockfile Docker build failures","description":"Docker image builds fail in multiple targets (candles, web, ingest services) because bun install --frozen-lockfile detects lockfile changes. Update workspace lockfile to match manifests and verify frozen install succeeds.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T22:52:38Z","created_by":"dirtydishes","updated_at":"2026-05-15T22:55:23Z","started_at":"2026-05-15T22:52:40Z","closed_at":"2026-05-15T22:55:23Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-9nd","title":"Hosted synthetic tape redesign with internal control surface","description":"Implement hosted synthetic market redesign with shared deterministic regime engine, internal JetStream KV control plane, ingest coupling across options and equities, and an internal bottom-right synthetic-control drawer with Next proxy routes. Preserve the six public smart-money categories while adding hidden subtype families, soft coverage accounting, and backend-only admin endpoints.\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-14T01:25:02Z","created_by":"dirtydishes","updated_at":"2026-05-14T02:10:03Z","started_at":"2026-05-14T01:25:09Z","closed_at":"2026-05-14T02:10:03Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
|
|
@ -10,6 +13,9 @@
|
|||
{"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-qd7","title":"Migrate production web to api.flow.deltaisland.io","description":"Follow-up from the durable options tape history fix. Plan and migrate production from same-origin API path proxying on flow.deltaisland.io to a dedicated api.flow.deltaisland.io origin, including DNS, proxy config, CORS/websocket behavior, deployment docs, and public smoke checks.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-17T01:55:02Z","created_by":"dirtydishes","updated_at":"2026-05-17T01:55:02Z","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-09a","title":"Speed up Docker deployment builds","description":"Implement the Docker deployment optimization plan from /Users/kell/Desktop/speed-up-docker.md: split dependency installation from source copy, add BuildKit caches, make scoped deploys build only their target services, update Docker deployment docs, validate, document the turn, commit, and push.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:50:24Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:53:48Z","started_at":"2026-05-16T21:50:37Z","closed_at":"2026-05-16T21:53:48Z","close_reason":"Implemented Docker dependency-layer caching, scoped deploy build/up flow, Docker docs updates, validation, and turn documentation. Follow-up islandflow-cnk tracks daemon-backed image build verification.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-0sa","title":"Fix live tape auto-hold, history seam, and remove manual pause control","description":"The live tape should automatically hold when the user scrolls away from the top, resume when they return to the top or use Jump to top, and keep older prints available seamlessly beyond the hot window. Manual Pause/Resume control is now redundant and should be removed from live tape panes. This work should also fix the current regression where paused/held tapes still mutate, and align the options tape with a strict 100-row hot head backed by ClickHouse history.","notes":"Implemented live scroll-hold with no live pause button, demand-loaded ClickHouse history, a 100-row options hot head, and cache-first scoped snapshots. Validated with bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts and bun --cwd=apps/web run build.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T18:12:51Z","created_by":"dirtydishes","updated_at":"2026-05-16T18:23:43Z","started_at":"2026-05-16T18:12:54Z","closed_at":"2026-05-16T18:23:43Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-2db","title":"Manually remove stale islandflow local-infra containers from VPS","description":"The live VPS still has an older compose project named islandflow created from the repo-root docker-compose.yml. Inspection shows it is separate from the supported islandflow-vps deployment stack and exposes NATS, ClickHouse, and Redis on host ports. Container removal commands currently hang when run as the delta user through Docker, so cleanup likely needs a focused maintenance window and possibly host-level intervention or a Docker daemon restart.","notes":"The duplicate islandflow compose project on the VPS was confirmed live during inspection. Nginx Proxy Manager routes public traffic only to islandflow-vps web/api by Docker name, so the stale islandflow project appears to be stray local-infra state rather than part of the supported production path. Attempts to remove the stale containers with docker compose down and docker rm -f as the delta user hung and timed out, so manual cleanup likely needs a maintenance window and possibly Docker daemon intervention.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:27:27Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:28:59Z","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-c87","title":"Clean up duplicate Islandflow Docker infra on VPS","description":"The live VPS is currently running both the production-style islandflow-vps Docker stack and an older root-level islandflow infra stack that publishes NATS, ClickHouse, and Redis on host ports. Investigate whether the older stack is unused, remove it safely if so, and update docs/deploy guidance so the server topology is clearer.","notes":"Inspected the live VPS and confirmed the duplicate compose project: islandflow-vps is the supported deployment stack, while a separate islandflow project from the repo-root docker-compose.yml still runs exposed NATS/ClickHouse/Redis containers. Verified Nginx Proxy Manager routes only to islandflow-vps web/api by Docker name. Attempted cleanup via docker compose down and docker rm -f on the stale islandflow containers, but those commands hung for the delta user and timed out. Added repo guardrails and docs so deploy warns when the duplicate project exists, and opened islandflow-2db for manual host-level cleanup during a maintenance window.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:16:05Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:28:07Z","started_at":"2026-05-16T01:16:09Z","closed_at":"2026-05-16T01:28:07Z","close_reason":"Completed the repo-side investigation and guardrails. Actual server-side container removal is blocked by hanging Docker operations and is tracked separately in islandflow-2db for a maintenance window.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-4gj","title":"Clarify Docker-first deploy workflow and mark native runtime experimental","description":"After inspecting the live VPS, native deployment is not ready for routine use: Nginx Proxy Manager routes to Docker container names, Bun is not installed on the host, sudo systemctl is not passwordless, and no Islandflow units exist. Update deploy messaging and docs so Docker remains the clearly recommended deployment path and native runtime is labeled experimental/future-facing with server prerequisites called out.","notes":"Updated deploy messaging and docs after live VPS inspection. scripts/deploy.ts now marks Docker as the default and recommended runtime, labels native as experimental, switches native systemctl default to sudo -n systemctl, and prints explicit native precheck failures for missing Bun/systemctl access/units. Updated README.md, deployment/docker/README.md, and deployment/native/README.md to reflect the current Docker + Nginx Proxy Manager topology. Validation: ./deploy --help, ./deploy main --runtime native --no-build (fails fast with Bun-missing message), bun run check:docker-workspace.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:10:11Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:12:39Z","started_at":"2026-05-16T01:10:14Z","closed_at":"2026-05-16T01:12:39Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
|
|
@ -36,5 +42,6 @@
|
|||
{"_type":"issue","id":"islandflow-zs0","title":"Migrate terminal UI to smart-money profiles","description":"Migrate apps/web terminal rendering to consume SmartMoneyEvent directly: primary profile, probability ladder, reason codes, and suppression/abstention state, while preserving legacy alert/classifier displays during the bridge.","status":"closed","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:23Z","created_by":"dirtydishes","updated_at":"2026-05-05T05:39:58Z","closed_at":"2026-05-05T05:39:58Z","close_reason":"Completed terminal smart-money profile migration","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-igk","title":"Add plan mode","description":"Implement a user-facing plan mode in the application so users can switch into planning before taking action. Scope to be clarified from existing app patterns.","status":"closed","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:22:37Z","created_by":"dirtydishes","updated_at":"2026-05-04T04:26:18Z","started_at":"2026-05-04T04:22:40Z","closed_at":"2026-05-04T04:26:18Z","close_reason":"Implemented as a global pi extension toggled with Shift+P","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-biq","title":"Finish raw live options delivery and filter/backpressure observability","description":"The smart-money signal path and Tape filters are in place, but the next firehose pass should finish server-side selective raw live delivery for options subscriptions and add explicit filtered-out/backpressure observability for API/web counters. This was discovered while landing islandflow-e4r.\n","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:28:58Z","created_by":"dirtydishes","updated_at":"2026-04-29T03:54:12Z","started_at":"2026-04-29T03:54:12Z","dependencies":[{"issue_id":"islandflow-biq","depends_on_id":"islandflow-e4r","type":"discovered-from","created_at":"2026-04-28T16:28:58Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-cnk","title":"Run Docker image build verification with active Docker daemon","description":"Targeted image builds could not run in the implementation session because the local Docker daemon was unavailable at unix:///Users/kell/.orbstack/run/docker.sock. When Docker or OrbStack is running, validate the refactored deployment Dockerfiles with: docker compose -f deployment/docker/docker-compose.yml build api; docker compose -f deployment/docker/docker-compose.yml build web; docker compose -f deployment/docker/docker-compose.yml build ingest-options.","status":"open","priority":3,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:53:41Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:53:41Z","dependencies":[{"issue_id":"islandflow-cnk","depends_on_id":"islandflow-09a","type":"discovered-from","created_at":"2026-05-16T17:53:40Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-38p","title":"Add native deployment unit templates and rollback helpers","description":"The deploy helper now supports --runtime native, but the repo still relies on operator-managed systemd units and manual rollback. Add checked-in native deployment templates or provisioning guidance for the expected units, and consider lightweight rollback/smoke-test helpers once the host-native path is exercised on the real VPS.","status":"open","priority":3,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:46:42Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:46:42Z","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-575","title":"Document smart-money event calendar env","description":"Document smart-money event-calendar environment configuration in env examples and README.\n","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T06:57:14Z","created_by":"dirtydishes","updated_at":"2026-05-05T06:57:57Z","started_at":"2026-05-05T06:57:17Z","closed_at":"2026-05-05T06:57:57Z","close_reason":"Documented event-calendar env variables","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
|
|
|
|||
26
.codex/hooks.json
Normal file
26
.codex/hooks.json
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
{
|
||||
"hooks": {
|
||||
"PreCompact": [
|
||||
{
|
||||
"matcher": "",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "bd prime"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"SessionStart": [
|
||||
{
|
||||
"matcher": "",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "bd prime"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -117,6 +117,7 @@ Each turn document must include these sections:
|
|||
2. **Changes Made**
|
||||
3. **Context**
|
||||
4. **Important Implementation Details**
|
||||
5. **Expected Impact for End-Users**
|
||||
5. **Validation**
|
||||
6. **Issues, Limitations, and Mitigations**
|
||||
7. **Follow-up Work**
|
||||
|
|
|
|||
|
|
@ -606,6 +606,13 @@ h3 {
|
|||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.flow-filter-section-copy {
|
||||
margin: -2px 0 0;
|
||||
color: var(--text-muted);
|
||||
font-size: 0.78rem;
|
||||
line-height: 1.35;
|
||||
}
|
||||
|
||||
.flow-filter-checkbox-grid,
|
||||
.flow-filter-chip-grid {
|
||||
display: grid;
|
||||
|
|
@ -617,6 +624,10 @@ h3 {
|
|||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.flow-filter-chip-grid-two {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.flow-filter-check {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
|
|
@ -992,6 +1003,17 @@ h3 {
|
|||
overflow: hidden;
|
||||
}
|
||||
|
||||
.history-load-warning {
|
||||
flex: 0 0 auto;
|
||||
padding: 8px 12px;
|
||||
border-top: 1px solid oklch(0.72 0.13 58 / 0.45);
|
||||
border-bottom: 1px solid oklch(0.72 0.13 58 / 0.45);
|
||||
background: oklch(0.24 0.05 58 / 0.72);
|
||||
color: oklch(0.91 0.08 72);
|
||||
font-size: 0.78rem;
|
||||
line-height: 1.35;
|
||||
}
|
||||
|
||||
.data-table-wrap {
|
||||
display: flex;
|
||||
flex: 1 1 auto;
|
||||
|
|
@ -1017,11 +1039,27 @@ h3 {
|
|||
min-height: 0;
|
||||
overflow-y: auto;
|
||||
overflow-x: hidden;
|
||||
background-color: oklch(0.12 0.01 250);
|
||||
}
|
||||
|
||||
.data-table-body {
|
||||
position: relative;
|
||||
min-width: 100%;
|
||||
--tape-row-height: 36px;
|
||||
--tape-row-double-height: 72px;
|
||||
background:
|
||||
repeating-linear-gradient(
|
||||
to bottom,
|
||||
oklch(0.98 0.008 250 / 0.01) 0,
|
||||
oklch(0.98 0.008 250 / 0.01) calc(var(--tape-row-height) - 1px),
|
||||
oklch(0.72 0.012 250 / 0.08) calc(var(--tape-row-height) - 1px),
|
||||
oklch(0.72 0.012 250 / 0.08) var(--tape-row-height),
|
||||
oklch(0.98 0.008 250 / 0.018) var(--tape-row-height),
|
||||
oklch(0.98 0.008 250 / 0.018) calc(var(--tape-row-double-height) - 1px),
|
||||
oklch(0.72 0.012 250 / 0.08) calc(var(--tape-row-double-height) - 1px),
|
||||
oklch(0.72 0.012 250 / 0.08) var(--tape-row-double-height)
|
||||
),
|
||||
oklch(0.12 0.01 250);
|
||||
}
|
||||
|
||||
.data-table-options {
|
||||
|
|
@ -1115,6 +1153,14 @@ h3 {
|
|||
height: 44px;
|
||||
}
|
||||
|
||||
.data-table-flow .data-table-body,
|
||||
.data-table-alerts .data-table-body,
|
||||
.data-table-classifier .data-table-body,
|
||||
.data-table-dark .data-table-body {
|
||||
--tape-row-height: 44px;
|
||||
--tape-row-double-height: 88px;
|
||||
}
|
||||
|
||||
.data-table-row-classified {
|
||||
background:
|
||||
linear-gradient(90deg, rgba(var(--classifier-rgb, 192, 200, 210), calc(0.012 + var(--classifier-intensity, 0) * 0.06)), transparent 62%),
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ import {
|
|||
getEffectiveOptionPrintFilters,
|
||||
getAlertWindowAnchorTs,
|
||||
getHotChannelFeedStatus,
|
||||
getScopedLiveAutoHydrationChannels,
|
||||
getLiveHistoryRetentionCap,
|
||||
getOptionTableSnapshot,
|
||||
getOptionScope,
|
||||
|
|
@ -25,6 +24,7 @@ import {
|
|||
getLiveManifest,
|
||||
getRouteFeatures,
|
||||
getTapeVirtualConfig,
|
||||
mergeHeldTapeHistory,
|
||||
mergeNewestWithOverflow,
|
||||
normalizeAlertSeverity,
|
||||
normalizeTickerFilterInput,
|
||||
|
|
@ -164,6 +164,7 @@ describe("live manifest", () => {
|
|||
|
||||
expect(optionsSubscription?.underlying_ids).toEqual(["AAPL"]);
|
||||
expect(optionsSubscription?.option_contract_id).toBe("AAPL-2025-01-17-200-C");
|
||||
expect(optionsSubscription?.snapshot_limit).toBe(100);
|
||||
expect(equitiesSubscription?.underlying_ids).toEqual(["AAPL"]);
|
||||
});
|
||||
|
||||
|
|
@ -297,6 +298,24 @@ describe("contract-focused option helpers", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("includes the selected options view in tape query params", () => {
|
||||
expect(
|
||||
buildOptionTapeQueryParams(
|
||||
{
|
||||
...buildDefaultFlowFilters(),
|
||||
view: "raw",
|
||||
securityTypes: undefined,
|
||||
nbboSides: undefined,
|
||||
optionTypes: undefined
|
||||
},
|
||||
{ underlying_ids: ["AAPL"] }
|
||||
)
|
||||
).toEqual({
|
||||
view: "raw",
|
||||
underlying_ids: "AAPL"
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the focus seed until the matching scoped subscription has loaded it", () => {
|
||||
const seedItem = makeOptionPrint({
|
||||
trace_id: "focused-seed",
|
||||
|
|
@ -376,12 +395,12 @@ describe("route feature map", () => {
|
|||
|
||||
describe("fixed tape virtualization config", () => {
|
||||
it("uses expected fixed row heights and overscan by table", () => {
|
||||
expect(getTapeVirtualConfig("options")).toEqual({ rowHeight: 36, overscan: 24, debugLabel: "options" });
|
||||
expect(getTapeVirtualConfig("equities")).toEqual({ rowHeight: 36, overscan: 20, debugLabel: "equities" });
|
||||
expect(getTapeVirtualConfig("flow")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "flow" });
|
||||
expect(getTapeVirtualConfig("alerts")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "alerts" });
|
||||
expect(getTapeVirtualConfig("classifier")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "classifier" });
|
||||
expect(getTapeVirtualConfig("dark")).toEqual({ rowHeight: 44, overscan: 16, debugLabel: "dark" });
|
||||
expect(getTapeVirtualConfig("options")).toEqual({ rowHeight: 36, overscan: 44, debugLabel: "options" });
|
||||
expect(getTapeVirtualConfig("equities")).toEqual({ rowHeight: 36, overscan: 36, debugLabel: "equities" });
|
||||
expect(getTapeVirtualConfig("flow")).toEqual({ rowHeight: 44, overscan: 24, debugLabel: "flow" });
|
||||
expect(getTapeVirtualConfig("alerts")).toEqual({ rowHeight: 44, overscan: 24, debugLabel: "alerts" });
|
||||
expect(getTapeVirtualConfig("classifier")).toEqual({ rowHeight: 44, overscan: 24, debugLabel: "classifier" });
|
||||
expect(getTapeVirtualConfig("dark")).toEqual({ rowHeight: 44, overscan: 24, debugLabel: "dark" });
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -635,52 +654,20 @@ describe("live tape history helpers", () => {
|
|||
expect(next.map((item) => item.trace_id)).toEqual(["existing", "older-1"]);
|
||||
});
|
||||
|
||||
it("keeps scoped option and equity history on the normal retention cap", () => {
|
||||
it("keeps option and equity history effectively unbounded while scrolling", () => {
|
||||
expect(
|
||||
getLiveHistoryRetentionCap({
|
||||
channel: "options",
|
||||
underlying_ids: ["AAPL"],
|
||||
option_contract_id: "AAPL-2025-01-17-200-C"
|
||||
} as any)
|
||||
).toBeGreaterThan(0);
|
||||
).toBe(0);
|
||||
expect(
|
||||
getLiveHistoryRetentionCap({
|
||||
channel: "equities",
|
||||
underlying_ids: ["AAPL"]
|
||||
} as any)
|
||||
).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("keeps auto-hydrating scoped live history while next_before exists", () => {
|
||||
const manifest = getLiveManifest(
|
||||
"/tape",
|
||||
"AAPL",
|
||||
60000,
|
||||
buildDefaultFlowFilters(),
|
||||
{
|
||||
underlying_ids: ["AAPL"],
|
||||
option_contract_id: "AAPL-2025-01-17-200-C"
|
||||
},
|
||||
{ underlying_ids: ["AAPL"] }
|
||||
);
|
||||
const historyCursors = Object.fromEntries(
|
||||
manifest.map((subscription) => [getLiveSubscriptionKey(subscription), { ts: 1, seq: 1 }])
|
||||
);
|
||||
|
||||
expect(
|
||||
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {})
|
||||
).toEqual(["options", "equities"]);
|
||||
expect(
|
||||
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {
|
||||
[getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "options")!)]: true
|
||||
})
|
||||
).toEqual(["equities"]);
|
||||
expect(
|
||||
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, {
|
||||
...historyCursors,
|
||||
[getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "equities")!)]: null
|
||||
}, {})
|
||||
).toEqual(["options"]);
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it("restores the same anchor key after live insertions at the top", () => {
|
||||
|
|
@ -697,6 +684,53 @@ describe("live tape history helpers", () => {
|
|||
const nextKeys = ["anchor", "after-1", "after-2", "older-1", "older-2"];
|
||||
expect(findAnchorRestoreIndex(nextKeys, "anchor", ["anchor", "after-1", "after-2"])).toBe(0);
|
||||
});
|
||||
|
||||
it("keeps held ClickHouse history stable when newer live overflow arrives", () => {
|
||||
const frozenLive = [makeItem("hot-5", 5, 500), makeItem("hot-4", 4, 400)];
|
||||
const displayed = [makeItem("hist-3", 3, 300), makeItem("hist-2", 2, 200)];
|
||||
const incoming = [
|
||||
makeItem("overflow-newer", 6, 600),
|
||||
makeItem("hot-4", 4, 400),
|
||||
makeItem("hist-3", 3, 300),
|
||||
makeItem("hist-2", 2, 200)
|
||||
];
|
||||
|
||||
expect(mergeHeldTapeHistory(displayed, incoming, frozenLive).map((item) => item.trace_id)).toEqual([
|
||||
"hist-3",
|
||||
"hist-2"
|
||||
]);
|
||||
});
|
||||
|
||||
it("appends truly older lazy-loaded rows to the held history tail", () => {
|
||||
const frozenLive = [makeItem("hot-5", 5, 500), makeItem("hot-4", 4, 400)];
|
||||
const displayed = [makeItem("hist-3", 3, 300), makeItem("hist-2", 2, 200)];
|
||||
const incoming = [
|
||||
makeItem("hist-3", 3, 300),
|
||||
makeItem("hist-2", 2, 200),
|
||||
makeItem("older-1", 1, 100),
|
||||
makeItem("older-0", 0, 50)
|
||||
];
|
||||
|
||||
expect(mergeHeldTapeHistory(displayed, incoming, frozenLive).map((item) => item.trace_id)).toEqual([
|
||||
"hist-3",
|
||||
"hist-2",
|
||||
"older-1",
|
||||
"older-0"
|
||||
]);
|
||||
});
|
||||
|
||||
it("resyncs buffered live history by replacing the held segment after resume", () => {
|
||||
const frozenLive = [makeItem("hot-5", 5, 500), makeItem("hot-4", 4, 400)];
|
||||
const held = mergeHeldTapeHistory(
|
||||
[makeItem("hist-3", 3, 300), makeItem("hist-2", 2, 200)],
|
||||
[makeItem("overflow-newer", 6, 600), makeItem("hist-3", 3, 300), makeItem("older-1", 1, 100)],
|
||||
frozenLive
|
||||
);
|
||||
const resynced = appendHistoryTail([], [makeItem("overflow-newer", 6, 600), ...held], [], 0);
|
||||
|
||||
expect(held.map((item) => item.trace_id)).toEqual(["hist-3", "hist-2", "older-1"]);
|
||||
expect(resynced.map((item) => item.trace_id)).toEqual(["overflow-newer", "hist-3", "hist-2", "older-1"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("options display formatters", () => {
|
||||
|
|
@ -811,6 +845,7 @@ describe("flow filter popup helpers", () => {
|
|||
|
||||
expect(countActiveFlowFilterGroups(defaults)).toBe(0);
|
||||
expect(countActiveFlowFilterGroups(next)).toBe(3);
|
||||
expect(countActiveFlowFilterGroups({ ...defaults, view: "raw" })).toBe(1);
|
||||
expect(buildDefaultFlowFilters()).toEqual(defaults);
|
||||
});
|
||||
});
|
||||
|
|
@ -864,9 +899,15 @@ describe("signals helpers", () => {
|
|||
expect(getAlertWindowAnchorTs([], 42)).toBe(42);
|
||||
});
|
||||
|
||||
it("returns connected/stale live status labels without live wording", () => {
|
||||
it("returns connected/held/stale live status labels without live wording", () => {
|
||||
expect(statusLabel("connected", false, "live")).toBe("Connected");
|
||||
expect(statusLabel("connected", true, "live")).toBe("Held");
|
||||
expect(statusLabel("stale", false, "live")).toBe("Feed behind");
|
||||
expect(statusLabel("stale", true, "live")).toBe("Feed behind");
|
||||
});
|
||||
|
||||
it("keeps replay pause wording on replay tapes", () => {
|
||||
expect(statusLabel("connected", true, "replay")).toBe("Paused");
|
||||
});
|
||||
|
||||
it("treats healthy scoped channels as connected even when no matching rows are visible", () => {
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import type {
|
|||
LiveHotChannelHealthMap,
|
||||
LiveSubscription,
|
||||
OptionFlowFilters,
|
||||
OptionFlowView,
|
||||
OptionNbboSide,
|
||||
OptionSecurityType,
|
||||
OptionType,
|
||||
|
|
@ -77,6 +78,7 @@ const LIVE_HOT_WINDOW_OPTIONS = parseBoundedInt(
|
|||
1,
|
||||
100000
|
||||
);
|
||||
const LIVE_OPTIONS_HEAD_LIMIT = 100;
|
||||
const LIVE_HISTORY_SOFT_CAP = parseBoundedInt(
|
||||
process.env.NEXT_PUBLIC_LIVE_HISTORY_SOFT_CAP,
|
||||
5000,
|
||||
|
|
@ -140,12 +142,12 @@ type TapeVirtualListConfig = {
|
|||
};
|
||||
|
||||
const TAPE_VIRTUAL_CONFIG: Record<TapeVirtualPane, TapeVirtualListConfig> = {
|
||||
options: { rowHeight: 36, overscan: 24, debugLabel: "options" },
|
||||
equities: { rowHeight: 36, overscan: 20, debugLabel: "equities" },
|
||||
flow: { rowHeight: 44, overscan: 16, debugLabel: "flow" },
|
||||
alerts: { rowHeight: 44, overscan: 16, debugLabel: "alerts" },
|
||||
classifier: { rowHeight: 44, overscan: 16, debugLabel: "classifier" },
|
||||
dark: { rowHeight: 44, overscan: 16, debugLabel: "dark" }
|
||||
options: { rowHeight: 36, overscan: 44, debugLabel: "options" },
|
||||
equities: { rowHeight: 36, overscan: 36, debugLabel: "equities" },
|
||||
flow: { rowHeight: 44, overscan: 24, debugLabel: "flow" },
|
||||
alerts: { rowHeight: 44, overscan: 24, debugLabel: "alerts" },
|
||||
classifier: { rowHeight: 44, overscan: 24, debugLabel: "classifier" },
|
||||
dark: { rowHeight: 44, overscan: 24, debugLabel: "dark" }
|
||||
};
|
||||
|
||||
export const getTapeVirtualConfig = (pane: TapeVirtualPane): TapeVirtualListConfig =>
|
||||
|
|
@ -842,46 +844,40 @@ export const appendHistoryTail = <T extends SortableItem>(
|
|||
return cap > 0 ? combined.slice(0, cap) : combined;
|
||||
};
|
||||
|
||||
export const mergeHeldTapeHistory = <T extends SortableItem>(
|
||||
displayedHistory: T[],
|
||||
incomingHistory: T[],
|
||||
frozenLiveHead: T[]
|
||||
): T[] => {
|
||||
if (displayedHistory.length === 0) {
|
||||
return appendHistoryTail([], incomingHistory, frozenLiveHead, 0);
|
||||
}
|
||||
|
||||
const sortedDisplayed = appendHistoryTail([], displayedHistory, frozenLiveHead, 0);
|
||||
const tail = sortedDisplayed.at(-1);
|
||||
const tailTs = tail ? extractSortTs(tail) : Number.POSITIVE_INFINITY;
|
||||
const tailSeq = tail ? extractSortSeq(tail) : Number.POSITIVE_INFINITY;
|
||||
const olderIncoming = incomingHistory.filter((item) => {
|
||||
const itemTs = extractSortTs(item);
|
||||
if (itemTs < tailTs) {
|
||||
return true;
|
||||
}
|
||||
return itemTs === tailTs && extractSortSeq(item) < tailSeq;
|
||||
});
|
||||
|
||||
return appendHistoryTail(sortedDisplayed, olderIncoming, frozenLiveHead, 0);
|
||||
};
|
||||
|
||||
export const getLiveHistoryRetentionCap = (subscription: LiveSubscription): number => {
|
||||
switch (subscription.channel) {
|
||||
case "options":
|
||||
case "equities":
|
||||
return LIVE_HISTORY_SOFT_CAP;
|
||||
return 0;
|
||||
default:
|
||||
return LIVE_HISTORY_SOFT_CAP;
|
||||
}
|
||||
};
|
||||
|
||||
export const getScopedLiveAutoHydrationChannels = (
|
||||
enabled: boolean,
|
||||
pathname: string,
|
||||
manifest: LiveSubscription[],
|
||||
historyCursors: Partial<Record<string, Cursor | null>>,
|
||||
historyLoading: Partial<Record<string, boolean>>
|
||||
): Array<Extract<LiveSubscription["channel"], "options" | "equities">> => {
|
||||
if (!enabled || pathname !== "/tape") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const channels: Array<Extract<LiveSubscription["channel"], "options" | "equities">> = [];
|
||||
for (const subscription of manifest) {
|
||||
const scoped =
|
||||
(subscription.channel === "options" &&
|
||||
(subscription.underlying_ids?.length || subscription.option_contract_id)) ||
|
||||
(subscription.channel === "equities" && subscription.underlying_ids?.length);
|
||||
if (!scoped) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const key = getLiveSubscriptionKey(subscription);
|
||||
if (historyCursors[key] && !historyLoading[key]) {
|
||||
channels.push(subscription.channel);
|
||||
}
|
||||
}
|
||||
|
||||
return channels;
|
||||
};
|
||||
|
||||
export const getLiveFeedStatus = (
|
||||
sourceStatus: WsStatus,
|
||||
freshestTs: number | null,
|
||||
|
|
@ -1450,6 +1446,9 @@ export const countActiveFlowFilterGroups = (filters: OptionFlowFilters): number
|
|||
if ((filters.minNotional ?? undefined) !== (defaults.minNotional ?? undefined)) {
|
||||
count += 1;
|
||||
}
|
||||
if ((filters.view ?? defaults.view) !== defaults.view) {
|
||||
count += 1;
|
||||
}
|
||||
|
||||
return count;
|
||||
};
|
||||
|
|
@ -2027,7 +2026,10 @@ export const prunePinnedEntries = <T,>(
|
|||
|
||||
export const statusLabel = (status: WsStatus, paused: boolean, mode: TapeMode): string => {
|
||||
if (paused) {
|
||||
return "Paused";
|
||||
if (mode === "replay") {
|
||||
return "Paused";
|
||||
}
|
||||
return status === "connected" ? "Held" : statusLabel(status, false, mode);
|
||||
}
|
||||
|
||||
if (mode === "replay") {
|
||||
|
|
@ -2512,22 +2514,21 @@ type PausableTapeViewConfig<T extends SortableItem & { seq: number }> = {
|
|||
const usePausableTapeView = <T extends SortableItem & { seq: number }>(
|
||||
config: PausableTapeViewConfig<T>
|
||||
): TapeState<T> => {
|
||||
const [paused, setPaused] = useState(false);
|
||||
const [data, setData] = useState<PausableTapeData<T>>(EMPTY_PAUSABLE_TAPE);
|
||||
const displayedHistoryRef = useRef<T[]>([]);
|
||||
const holdForScroll = config.enabled ? (config.shouldHold ? config.shouldHold() : false) : false;
|
||||
|
||||
useEffect(() => {
|
||||
if (!config.enabled) {
|
||||
setPaused(false);
|
||||
setData(EMPTY_PAUSABLE_TAPE);
|
||||
return;
|
||||
}
|
||||
|
||||
const holdForScroll = config.shouldHold ? config.shouldHold() : false;
|
||||
setData((current) => {
|
||||
const next = reducePausableTapeData(
|
||||
current,
|
||||
config.sourceItems,
|
||||
paused || holdForScroll,
|
||||
holdForScroll,
|
||||
config.retentionLimit ?? LIVE_HOT_WINDOW
|
||||
);
|
||||
if (next === current) {
|
||||
|
|
@ -2535,7 +2536,7 @@ const usePausableTapeView = <T extends SortableItem & { seq: number }>(
|
|||
}
|
||||
|
||||
const unseenCount = next.seenKeys.size - current.seenKeys.size;
|
||||
if (!paused && unseenCount > 0) {
|
||||
if (unseenCount > 0) {
|
||||
config.onNewItems?.(unseenCount);
|
||||
config.captureScroll?.();
|
||||
}
|
||||
|
|
@ -2548,17 +2549,11 @@ const usePausableTapeView = <T extends SortableItem & { seq: number }>(
|
|||
config.onNewItems,
|
||||
config.captureScroll,
|
||||
config.retentionLimit,
|
||||
config.shouldHold,
|
||||
paused
|
||||
holdForScroll
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!config.enabled || paused) {
|
||||
return;
|
||||
}
|
||||
|
||||
const holdForScroll = config.shouldHold ? config.shouldHold() : false;
|
||||
if (holdForScroll) {
|
||||
if (!config.enabled || holdForScroll) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -2581,30 +2576,43 @@ const usePausableTapeView = <T extends SortableItem & { seq: number }>(
|
|||
config.onNewItems,
|
||||
config.retentionLimit,
|
||||
config.resumeSignal,
|
||||
config.shouldHold,
|
||||
paused
|
||||
holdForScroll
|
||||
]);
|
||||
|
||||
const togglePause = useCallback(() => {
|
||||
setPaused((current) => !current);
|
||||
}, []);
|
||||
|
||||
const status = config.enabled ? config.sourceStatus : "disconnected";
|
||||
const projected = projectPausableTapeState(data.visible, status, config.lastUpdate);
|
||||
const historyItems = config.historyTail ?? [];
|
||||
const items = useMemo(() => composeTapeItems([], projected.items, historyItems), [projected.items, historyItems]);
|
||||
const displayedHistoryItems = useMemo(() => {
|
||||
if (!config.enabled) {
|
||||
displayedHistoryRef.current = [];
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!holdForScroll) {
|
||||
displayedHistoryRef.current = historyItems;
|
||||
return historyItems;
|
||||
}
|
||||
|
||||
const next = mergeHeldTapeHistory(displayedHistoryRef.current, historyItems, projected.items);
|
||||
displayedHistoryRef.current = next;
|
||||
return next;
|
||||
}, [config.enabled, historyItems, holdForScroll, projected.items]);
|
||||
const items = useMemo(
|
||||
() => composeTapeItems([], projected.items, displayedHistoryItems),
|
||||
[projected.items, displayedHistoryItems]
|
||||
);
|
||||
|
||||
return {
|
||||
status,
|
||||
items,
|
||||
liveItems: projected.items,
|
||||
historyItems,
|
||||
historyItems: displayedHistoryItems,
|
||||
lastUpdate: projected.lastUpdate,
|
||||
replayTime: null,
|
||||
replayComplete: false,
|
||||
paused,
|
||||
paused: holdForScroll,
|
||||
dropped: data.dropped,
|
||||
togglePause
|
||||
togglePause: () => {}
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -3052,7 +3060,7 @@ export const getLiveManifest = (
|
|||
? undefined
|
||||
: optionPrintFilters ?? flowFilters,
|
||||
...optionScope,
|
||||
snapshot_limit: LIVE_HOT_WINDOW_OPTIONS
|
||||
snapshot_limit: LIVE_OPTIONS_HEAD_LIMIT
|
||||
});
|
||||
}
|
||||
if (features.nbbo) {
|
||||
|
|
@ -3337,7 +3345,7 @@ const useLiveSession = (
|
|||
|
||||
switch (subscription.channel) {
|
||||
case "options":
|
||||
mergeItems(setOptions, optionsRef, items as OptionPrint[], LIVE_HOT_WINDOW_OPTIONS, {
|
||||
mergeItems(setOptions, optionsRef, items as OptionPrint[], LIVE_OPTIONS_HEAD_LIMIT, {
|
||||
setter: setOptionsHistory,
|
||||
ref: optionsHistoryRef,
|
||||
cap: getLiveHistoryRetentionCap(subscription)
|
||||
|
|
@ -3708,18 +3716,6 @@ const useLiveSession = (
|
|||
[enabled, manifest, historyCursors, historyLoading]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
for (const channel of getScopedLiveAutoHydrationChannels(
|
||||
enabled,
|
||||
pathname,
|
||||
manifest,
|
||||
historyCursors,
|
||||
historyLoading
|
||||
)) {
|
||||
void loadOlder(channel);
|
||||
}
|
||||
}, [enabled, pathname, manifest, historyCursors, historyLoading, loadOlder]);
|
||||
|
||||
return {
|
||||
status,
|
||||
connectedAt,
|
||||
|
|
@ -3794,6 +3790,7 @@ const TapeStatus = ({
|
|||
};
|
||||
|
||||
type TapeControlsProps = {
|
||||
mode: TapeMode;
|
||||
paused: boolean;
|
||||
onTogglePause: () => void;
|
||||
isAtTop: boolean;
|
||||
|
|
@ -3801,13 +3798,15 @@ type TapeControlsProps = {
|
|||
onJump: () => void;
|
||||
};
|
||||
|
||||
const TapeControls = ({ paused, onTogglePause, isAtTop, missed, onJump }: TapeControlsProps) => {
|
||||
const TapeControls = ({ mode, paused, onTogglePause, isAtTop, missed, onJump }: TapeControlsProps) => {
|
||||
const active = !isAtTop && missed > 0;
|
||||
return (
|
||||
<div className={`tape-controls${active ? " tape-controls-active" : ""}`}>
|
||||
<button className="pause-button" type="button" onClick={onTogglePause}>
|
||||
{paused ? "Resume" : "Pause"}
|
||||
</button>
|
||||
{mode === "replay" ? (
|
||||
<button className="pause-button" type="button" onClick={onTogglePause}>
|
||||
{paused ? "Resume" : "Pause"}
|
||||
</button>
|
||||
) : null}
|
||||
<button className="jump-button" type="button" onClick={onJump} disabled={isAtTop}>
|
||||
Jump to top
|
||||
</button>
|
||||
|
|
@ -5373,7 +5372,7 @@ const useTerminalState = () => {
|
|||
sourceItems: liveSession.options,
|
||||
historyTail: liveSession.optionsHistory,
|
||||
lastUpdate: liveSession.lastUpdate,
|
||||
retentionLimit: LIVE_HOT_WINDOW_OPTIONS,
|
||||
retentionLimit: LIVE_OPTIONS_HEAD_LIMIT,
|
||||
captureScroll: optionsAnchor.capture,
|
||||
onNewItems: optionsScroll.onNewItems,
|
||||
shouldHold: () => !optionsScroll.isAtTopRef.current,
|
||||
|
|
@ -6925,6 +6924,17 @@ export const FlowFilterPopover = ({ filters, onChange }: FlowFilterPopoverProps)
|
|||
}));
|
||||
};
|
||||
|
||||
const applyView = (view: OptionFlowView) => {
|
||||
onChange((prev) => ({
|
||||
...prev,
|
||||
view,
|
||||
securityTypes: view === "raw" ? undefined : prev.securityTypes ?? DEFAULT_FLOW_SECURITY_TYPES,
|
||||
nbboSides: view === "raw" ? undefined : prev.nbboSides,
|
||||
optionTypes: view === "raw" ? undefined : prev.optionTypes,
|
||||
minNotional: view === "raw" ? undefined : prev.minNotional
|
||||
}));
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) {
|
||||
return;
|
||||
|
|
@ -6989,6 +6999,27 @@ export const FlowFilterPopover = ({ filters, onChange }: FlowFilterPopoverProps)
|
|||
</div>
|
||||
|
||||
<div className="flow-filter-popover-body">
|
||||
<FlowFilterSection title="Options View">
|
||||
<div className="flow-filter-chip-grid flow-filter-chip-grid-two">
|
||||
{[
|
||||
{ label: "Signal", value: "signal" as const },
|
||||
{ label: "All prints", value: "raw" as const }
|
||||
].map((preset) => (
|
||||
<button
|
||||
className={`filter-chip ${filters.view === preset.value ? "is-active" : ""}`}
|
||||
key={preset.value}
|
||||
type="button"
|
||||
onClick={() => applyView(preset.value)}
|
||||
>
|
||||
{preset.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<p className="flow-filter-section-copy">
|
||||
Signal keeps classifier-ready prints. All prints includes raw option tape rows.
|
||||
</p>
|
||||
</FlowFilterSection>
|
||||
|
||||
<FlowFilterSection title="Security">
|
||||
<div className="flow-filter-checkbox-grid">
|
||||
{(["stock", "etf"] as OptionSecurityType[]).map((value) => (
|
||||
|
|
@ -7121,6 +7152,13 @@ type OptionsPaneProps = {
|
|||
const OptionsPane = memo(({ state, limit }: OptionsPaneProps) => {
|
||||
const items = limit ? state.filteredOptions.slice(0, limit) : state.filteredOptions;
|
||||
const virtual = useTapeVirtualList(items, state.optionsScroll.listRef, getTapeVirtualConfig("options"));
|
||||
const optionHistorySubscription = state.liveSession.manifest.find(
|
||||
(subscription) => subscription.channel === "options"
|
||||
);
|
||||
const optionHistoryKey = optionHistorySubscription ? getLiveSubscriptionKey(optionHistorySubscription) : null;
|
||||
const optionHistoryError = optionHistoryKey
|
||||
? state.liveSession.historyErrors[optionHistoryKey]
|
||||
: null;
|
||||
useVirtualHistoryGate(state.mode === "live" && !limit, items.length, virtual.virtualItems.at(-1)?.index ?? -1, () =>
|
||||
void state.liveSession.loadOlder("options")
|
||||
);
|
||||
|
|
@ -7141,6 +7179,7 @@ const OptionsPane = memo(({ state, limit }: OptionsPaneProps) => {
|
|||
}
|
||||
actions={
|
||||
<TapeControls
|
||||
mode={state.mode}
|
||||
paused={state.options.paused}
|
||||
onTogglePause={state.options.togglePause}
|
||||
isAtTop={state.optionsScroll.isAtTop}
|
||||
|
|
@ -7150,6 +7189,11 @@ const OptionsPane = memo(({ state, limit }: OptionsPaneProps) => {
|
|||
}
|
||||
>
|
||||
<div className="data-table-shell">
|
||||
{state.mode === "live" && optionHistoryError ? (
|
||||
<div className="history-load-warning" role="status">
|
||||
Older option history failed to load: {optionHistoryError}
|
||||
</div>
|
||||
) : null}
|
||||
{items.length === 0 ? (
|
||||
<div className="empty">
|
||||
{state.mode === "live"
|
||||
|
|
@ -7329,6 +7373,7 @@ const EquitiesPane = memo(({ state, limit }: EquitiesPaneProps) => {
|
|||
}
|
||||
actions={
|
||||
<TapeControls
|
||||
mode={state.mode}
|
||||
paused={state.equities.paused}
|
||||
onTogglePause={state.equities.togglePause}
|
||||
isAtTop={state.equitiesScroll.isAtTop}
|
||||
|
|
@ -7432,6 +7477,7 @@ const FlowPane = memo(({ state, limit, title = "Flow" }: FlowPaneProps) => {
|
|||
}
|
||||
actions={
|
||||
<TapeControls
|
||||
mode={state.mode}
|
||||
paused={state.flow.paused}
|
||||
onTogglePause={state.flow.togglePause}
|
||||
isAtTop={state.flowScroll.isAtTop}
|
||||
|
|
@ -7581,6 +7627,7 @@ const AlertsPane = memo(({ state, limit, withStrip = false, className }: AlertsP
|
|||
}
|
||||
actions={
|
||||
<TapeControls
|
||||
mode={state.mode}
|
||||
paused={state.alerts.paused}
|
||||
onTogglePause={state.alerts.togglePause}
|
||||
isAtTop={state.alertsScroll.isAtTop}
|
||||
|
|
@ -7695,6 +7742,7 @@ const ClassifierPane = memo(({ state, limit, className }: ClassifierPaneProps) =
|
|||
}
|
||||
actions={
|
||||
<TapeControls
|
||||
mode={state.mode}
|
||||
paused={state.smartMoney.paused}
|
||||
onTogglePause={state.smartMoney.togglePause}
|
||||
isAtTop={state.classifierScroll.isAtTop}
|
||||
|
|
@ -7818,6 +7866,7 @@ const DarkPane = memo(({ state, limit, className }: DarkPaneProps) => {
|
|||
}
|
||||
actions={
|
||||
<TapeControls
|
||||
mode={state.mode}
|
||||
paused={state.inferredDark.paused}
|
||||
onTogglePause={state.inferredDark.togglePause}
|
||||
isAtTop={state.darkScroll.isAtTop}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
FROM oven/bun:1.3.11
|
||||
|
||||
WORKDIR /app
|
||||
|
|
@ -9,15 +11,39 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
|
|||
COPY --from=workspace package.json ./package.json
|
||||
COPY --from=workspace bun.lock ./bun.lock
|
||||
COPY --from=workspace tsconfig.base.json ./tsconfig.base.json
|
||||
COPY --from=services . ./services
|
||||
COPY --from=packages . ./packages
|
||||
COPY --from=apps . ./apps
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends python3 python3-pip python3-venv \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& python3 -m venv "${VIRTUAL_ENV}" \
|
||||
&& "${VIRTUAL_ENV}/bin/pip" install --no-cache-dir -r services/ingest-options/py/requirements.txt \
|
||||
&& bun install --frozen-lockfile
|
||||
&& python3 -m venv "${VIRTUAL_ENV}"
|
||||
|
||||
COPY --from=apps desktop/package.json ./apps/desktop/package.json
|
||||
COPY --from=apps web/package.json ./apps/web/package.json
|
||||
|
||||
COPY --from=packages bus/package.json ./packages/bus/package.json
|
||||
COPY --from=packages config/package.json ./packages/config/package.json
|
||||
COPY --from=packages observability/package.json ./packages/observability/package.json
|
||||
COPY --from=packages storage/package.json ./packages/storage/package.json
|
||||
COPY --from=packages types/package.json ./packages/types/package.json
|
||||
|
||||
COPY --from=services api/package.json ./services/api/package.json
|
||||
COPY --from=services candles/package.json ./services/candles/package.json
|
||||
COPY --from=services compute/package.json ./services/compute/package.json
|
||||
COPY --from=services eod-enricher/package.json ./services/eod-enricher/package.json
|
||||
COPY --from=services ingest-equities/package.json ./services/ingest-equities/package.json
|
||||
COPY --from=services ingest-options/package.json ./services/ingest-options/package.json
|
||||
COPY --from=services ingest-options/py/requirements.txt ./services/ingest-options/py/requirements.txt
|
||||
COPY --from=services refdata/package.json ./services/refdata/package.json
|
||||
COPY --from=services replay/package.json ./services/replay/package.json
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
"${VIRTUAL_ENV}/bin/pip" install -r services/ingest-options/py/requirements.txt
|
||||
|
||||
RUN --mount=type=cache,target=/root/.bun/install/cache \
|
||||
bun install --frozen-lockfile
|
||||
|
||||
COPY --from=services . ./services
|
||||
COPY --from=packages . ./packages
|
||||
COPY --from=apps . ./apps
|
||||
|
||||
ENTRYPOINT ["bun"]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
FROM oven/bun:1.3.11
|
||||
|
||||
WORKDIR /app
|
||||
|
|
@ -7,10 +9,30 @@ ENV NODE_ENV=production
|
|||
COPY --from=workspace package.json ./package.json
|
||||
COPY --from=workspace bun.lock ./bun.lock
|
||||
COPY --from=workspace tsconfig.base.json ./tsconfig.base.json
|
||||
|
||||
COPY --from=apps desktop/package.json ./apps/desktop/package.json
|
||||
COPY --from=apps web/package.json ./apps/web/package.json
|
||||
|
||||
COPY --from=packages bus/package.json ./packages/bus/package.json
|
||||
COPY --from=packages config/package.json ./packages/config/package.json
|
||||
COPY --from=packages observability/package.json ./packages/observability/package.json
|
||||
COPY --from=packages storage/package.json ./packages/storage/package.json
|
||||
COPY --from=packages types/package.json ./packages/types/package.json
|
||||
|
||||
COPY --from=services api/package.json ./services/api/package.json
|
||||
COPY --from=services candles/package.json ./services/candles/package.json
|
||||
COPY --from=services compute/package.json ./services/compute/package.json
|
||||
COPY --from=services eod-enricher/package.json ./services/eod-enricher/package.json
|
||||
COPY --from=services ingest-equities/package.json ./services/ingest-equities/package.json
|
||||
COPY --from=services ingest-options/package.json ./services/ingest-options/package.json
|
||||
COPY --from=services refdata/package.json ./services/refdata/package.json
|
||||
COPY --from=services replay/package.json ./services/replay/package.json
|
||||
|
||||
RUN --mount=type=cache,target=/root/.bun/install/cache \
|
||||
bun install --frozen-lockfile
|
||||
|
||||
COPY --from=services . ./services
|
||||
COPY --from=packages . ./packages
|
||||
COPY --from=apps . ./apps
|
||||
|
||||
RUN bun install --frozen-lockfile
|
||||
|
||||
ENTRYPOINT ["bun"]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
FROM oven/bun:1.3.11 AS build
|
||||
|
||||
WORKDIR /app
|
||||
|
|
@ -13,11 +15,32 @@ ENV NEXT_PUBLIC_NBBO_MAX_AGE_MS=${NEXT_PUBLIC_NBBO_MAX_AGE_MS}
|
|||
COPY --from=workspace package.json ./package.json
|
||||
COPY --from=workspace bun.lock ./bun.lock
|
||||
COPY --from=workspace tsconfig.base.json ./tsconfig.base.json
|
||||
|
||||
COPY --from=apps desktop/package.json ./apps/desktop/package.json
|
||||
COPY --from=apps web/package.json ./apps/web/package.json
|
||||
|
||||
COPY --from=packages bus/package.json ./packages/bus/package.json
|
||||
COPY --from=packages config/package.json ./packages/config/package.json
|
||||
COPY --from=packages observability/package.json ./packages/observability/package.json
|
||||
COPY --from=packages storage/package.json ./packages/storage/package.json
|
||||
COPY --from=packages types/package.json ./packages/types/package.json
|
||||
|
||||
COPY --from=services api/package.json ./services/api/package.json
|
||||
COPY --from=services candles/package.json ./services/candles/package.json
|
||||
COPY --from=services compute/package.json ./services/compute/package.json
|
||||
COPY --from=services eod-enricher/package.json ./services/eod-enricher/package.json
|
||||
COPY --from=services ingest-equities/package.json ./services/ingest-equities/package.json
|
||||
COPY --from=services ingest-options/package.json ./services/ingest-options/package.json
|
||||
COPY --from=services refdata/package.json ./services/refdata/package.json
|
||||
COPY --from=services replay/package.json ./services/replay/package.json
|
||||
|
||||
RUN --mount=type=cache,target=/root/.bun/install/cache \
|
||||
bun install --frozen-lockfile
|
||||
|
||||
COPY --from=services . ./services
|
||||
COPY --from=packages . ./packages
|
||||
COPY --from=apps . ./apps
|
||||
|
||||
RUN bun install --frozen-lockfile
|
||||
RUN bun run --cwd apps/web build
|
||||
|
||||
FROM oven/bun:1.3.11 AS runtime
|
||||
|
|
|
|||
|
|
@ -65,14 +65,16 @@ Important defaults:
|
|||
3. Build and start the stack:
|
||||
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
docker compose build api web compute candles ingest-options ingest-equities
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
If you are updating an existing deployment that already has failing `api` restart loops, do a full recreate so the ClickHouse config mount and dependency changes are applied cleanly:
|
||||
|
||||
```bash
|
||||
docker compose down
|
||||
docker compose up -d --build --force-recreate
|
||||
docker compose build api web compute candles ingest-options ingest-equities
|
||||
docker compose up -d --force-recreate
|
||||
```
|
||||
|
||||
4. Confirm the containers are healthy:
|
||||
|
|
@ -117,10 +119,16 @@ Supported routing modes:
|
|||
- Build web with `NEXT_PUBLIC_API_URL=` (empty).
|
||||
- Point `app.<domain>` at the web host port.
|
||||
- Proxy these API routes from the app origin to the API host port:
|
||||
- `/ws/*`, `/replay/*`, `/prints/*`, `/joins/*`, `/nbbo/*`, `/dark/*`, `/flow/*`, `/candles/*`
|
||||
- `/ws/*`, `/replay/*`, `/prints/*`, `/joins/*`, `/nbbo/*`, `/dark/*`, `/flow/*`, `/candles/*`, `/history/*`
|
||||
|
||||
Enable websocket support on whichever host serves `/ws/*`.
|
||||
|
||||
For the current live Nginx Proxy Manager setup behind `flow.deltaisland.io`, keep the API location regex durable in the proxy host advanced config or API, not by hand-editing generated files under `/data/nginx/proxy_host/`. The route matcher should include history:
|
||||
|
||||
```nginx
|
||||
^/(ws|replay|prints|joins|nbbo|dark|flow|candles|history)/
|
||||
```
|
||||
|
||||
## Replay service
|
||||
|
||||
Replay is disabled by default in this stack.
|
||||
|
|
@ -190,6 +198,19 @@ cd deployment/docker
|
|||
docker compose build web
|
||||
```
|
||||
|
||||
### Faster Docker builds
|
||||
|
||||
The app images are structured so dependency installation is isolated from source code changes:
|
||||
|
||||
- Docker first copies `package.json`, `bun.lock`, `tsconfig.base.json`, and workspace `package.json` files.
|
||||
- `bun install --frozen-lockfile` runs in a cacheable layer with a BuildKit Bun cache mount.
|
||||
- Source from `apps`, `services`, and `packages` is copied only after dependencies are installed.
|
||||
- `ingest-options` also installs its Python sidecar dependencies from `services/ingest-options/py/requirements.txt` before source copy, using a BuildKit pip cache mount.
|
||||
|
||||
That means normal TypeScript edits should reuse dependency layers. The first build after a fresh server checkout, Docker cache cleanup, dependency change, or Python requirement change can still be slow; later deploys should spend their time on changed source and the specific service images being rolled out.
|
||||
|
||||
BuildKit cache mounts require a modern Docker Engine with Dockerfile frontend support. Docker Compose v2 on the VPS path enables this by default.
|
||||
|
||||
## Safe rollouts on `152.53.80.229`
|
||||
|
||||
The current live VPS uses Nginx Proxy Manager on the shared Docker network and routes public traffic to the Docker `web` and `api` containers by container name. Because of that, this Docker path remains the operationally correct default for the live server today.
|
||||
|
|
@ -218,7 +239,7 @@ This flow:
|
|||
- checks the server checkout before switching anything
|
||||
- stops if the server has tracked local modifications
|
||||
- allows the known untracked tarball at `deployment/docker/signal-cli-0.14.3-Linux-native.tar.gz`
|
||||
- runs `git switch main`, `git pull --ff-only origin main`, and `docker compose up -d --build`
|
||||
- runs `git switch main`, `git pull --ff-only origin main`, `docker compose build api web compute candles ingest-options ingest-equities`, and `docker compose up -d`
|
||||
- verifies the stack with `docker compose ps`, recent service logs, container-local health checks, and public HTTPS checks
|
||||
|
||||
### Deploy the current local branch
|
||||
|
|
@ -253,6 +274,14 @@ Examples:
|
|||
./deploy main --runtime docker --web-only --no-build
|
||||
```
|
||||
|
||||
Scoped Docker deploys now build only the selected image set and then restart only those services:
|
||||
|
||||
- `--web-only`: `docker compose build web`, then `docker compose up -d web`
|
||||
- `--api-only`: `docker compose build api`, then `docker compose up -d api`
|
||||
- `--services-only`: builds and restarts `api`, `compute`, `candles`, `ingest-options`, and `ingest-equities`
|
||||
|
||||
Use `--no-build` only when the image is already correct and you need Compose to recreate or restart containers, such as after changing server-side environment values that do not affect a Next.js build-time variable. Do not use `--no-build` for dependency changes, application source changes, or `NEXT_PUBLIC_*` changes.
|
||||
|
||||
### Escalation path
|
||||
|
||||
Use force recreate only when a normal refresh does not update the services cleanly:
|
||||
|
|
@ -299,7 +328,8 @@ git switch main
|
|||
git pull --ff-only origin main
|
||||
|
||||
cd /home/delta/islandflow/deployment/docker
|
||||
docker compose up -d --build
|
||||
docker compose build api web compute candles ingest-options ingest-equities
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
Deploy the current branch manually:
|
||||
|
|
@ -314,7 +344,8 @@ git switch <current-branch> || git switch -c <current-branch> --track origin/<cu
|
|||
git pull --ff-only origin <current-branch>
|
||||
|
||||
cd /home/delta/islandflow/deployment/docker
|
||||
docker compose up -d --build
|
||||
docker compose build api web compute candles ingest-options ingest-equities
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
If you changed only env values for the Bun services on the server:
|
||||
|
|
@ -416,3 +447,4 @@ After the stack is up:
|
|||
- `curl -I http://127.0.0.1:3000/` should return a successful HTTP status on the server.
|
||||
- In two-origin mode, browser requests should target `https://api.<domain>/...` and live feeds should use `wss://api.<domain>/ws/...`.
|
||||
- In same-origin mode, browser requests should target `https://app.<domain>/...` for API paths and live feeds should use `wss://app.<domain>/ws/...`.
|
||||
- In same-origin mode, `bun run check:public-api-routes` should pass for `/prints/options`, `/history/options`, `/replay/options`, `/nbbo/options`, and `/ws/live`.
|
||||
|
|
|
|||
57
docs/clickhouse-reset-runbook.md
Normal file
57
docs/clickhouse-reset-runbook.md
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
# ClickHouse Reset Runbook
|
||||
|
||||
This runbook is for deliberately wiping durable market-data history from ClickHouse in local development or on the VPS. It is destructive. Do not run these commands from application startup, deployment hooks, or unattended scripts.
|
||||
|
||||
## When To Use
|
||||
|
||||
Use this only when an operator has decided that existing option, equity, flow, and derived-event history should be discarded and rebuilt from fresh ingest.
|
||||
|
||||
Before running a reset:
|
||||
|
||||
- Confirm the target environment: local Docker or VPS Docker.
|
||||
- Confirm there is no active analysis depending on the existing history.
|
||||
- Take a backup if the data may be needed later.
|
||||
- Stop ingest and API services so new writes do not race the reset.
|
||||
|
||||
## Local Docker Reset
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
bun run dev:infra
|
||||
docker compose exec clickhouse clickhouse-client --query "SHOW TABLES"
|
||||
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS option_prints"
|
||||
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS option_nbbo"
|
||||
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS equity_prints"
|
||||
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS equity_quotes"
|
||||
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS equity_print_joins"
|
||||
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS flow_packets"
|
||||
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS smart_money_events"
|
||||
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS classifier_hits"
|
||||
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS alerts"
|
||||
docker compose exec clickhouse clickhouse-client --query "TRUNCATE TABLE IF EXISTS inferred_dark_events"
|
||||
```
|
||||
|
||||
If the local compose project uses `deployment/docker/docker-compose.yml`, run the same commands with `docker compose -f deployment/docker/docker-compose.yml exec clickhouse ...`.
|
||||
|
||||
## VPS Docker Reset
|
||||
|
||||
On the VPS, first identify the active compose project and ClickHouse service:
|
||||
|
||||
```bash
|
||||
docker ps --format "table {{.Names}}\t{{.Image}}\t{{.Status}}"
|
||||
docker compose -f deployment/docker/docker-compose.yml ps
|
||||
```
|
||||
|
||||
Then stop writers and run the same `TRUNCATE TABLE IF EXISTS ...` commands against the active ClickHouse container. Prefer `docker compose exec clickhouse clickhouse-client --query "<query>"` when the compose project is healthy; otherwise use `docker exec <clickhouse-container> clickhouse-client --query "<query>"`.
|
||||
|
||||
## Verification
|
||||
|
||||
After the reset:
|
||||
|
||||
```bash
|
||||
docker compose exec clickhouse clickhouse-client --query "SELECT count() FROM option_prints"
|
||||
docker compose exec clickhouse clickhouse-client --query "SELECT count() FROM flow_packets"
|
||||
```
|
||||
|
||||
Restart ingest/API services through the normal dev or deployment path. The options tape should repopulate its 100-row hot head from new signal prints, and older rows should appear only after the scroll gate asks `/history/options` for ClickHouse-backed history.
|
||||
363
docs/plans/2026-05-16-1711-durable-options-tape-history.html
Normal file
363
docs/plans/2026-05-16-1711-durable-options-tape-history.html
Normal file
|
|
@ -0,0 +1,363 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Plan: Durable Options Tape History</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: #06080b;
|
||||
--surface: #0b1016;
|
||||
--panel: #111820;
|
||||
--panel-2: #0d141b;
|
||||
--border: rgba(255, 255, 255, 0.1);
|
||||
--border-strong: rgba(245, 166, 35, 0.32);
|
||||
--text: #e6edf4;
|
||||
--muted: #90a0b2;
|
||||
--faint: #6e7b8c;
|
||||
--accent: #f5a623;
|
||||
--accent-soft: rgba(245, 166, 35, 0.14);
|
||||
--good: #25c17a;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
background:
|
||||
linear-gradient(180deg, rgba(17, 24, 32, 0.86), rgba(6, 8, 11, 0.98) 340px),
|
||||
var(--bg);
|
||||
color: var(--text);
|
||||
font: 15px/1.55 "IBM Plex Sans", -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
|
||||
}
|
||||
|
||||
main {
|
||||
max-width: 1040px;
|
||||
margin: 0 auto;
|
||||
padding: 40px 24px 56px;
|
||||
}
|
||||
|
||||
header {
|
||||
padding-bottom: 24px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.eyebrow {
|
||||
margin: 0 0 10px;
|
||||
color: var(--accent);
|
||||
font: 700 0.78rem/1.2 "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
letter-spacing: 0.12em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0;
|
||||
max-width: 760px;
|
||||
font-size: 2.1rem;
|
||||
line-height: 1.1;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
.summary {
|
||||
max-width: 820px;
|
||||
margin: 16px 0 0;
|
||||
color: var(--muted);
|
||||
font-size: 1rem;
|
||||
}
|
||||
|
||||
.meta {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-top: 18px;
|
||||
}
|
||||
|
||||
.chip {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 999px;
|
||||
padding: 4px 9px;
|
||||
background: rgba(255, 255, 255, 0.04);
|
||||
color: var(--muted);
|
||||
font: 700 0.72rem/1.2 "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
letter-spacing: 0.06em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
section {
|
||||
margin-top: 28px;
|
||||
padding: 22px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
background: linear-gradient(180deg, rgba(17, 24, 32, 0.92), rgba(13, 20, 27, 0.92));
|
||||
}
|
||||
|
||||
section.highlight {
|
||||
border-color: var(--border-strong);
|
||||
background: linear-gradient(180deg, rgba(245, 166, 35, 0.12), rgba(17, 24, 32, 0.9));
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin: 0 0 12px;
|
||||
color: var(--text);
|
||||
font: 700 0.86rem/1.2 "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
letter-spacing: 0.11em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
ul {
|
||||
margin: 0;
|
||||
padding-left: 1.15rem;
|
||||
}
|
||||
|
||||
li {
|
||||
margin: 8px 0;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
strong {
|
||||
color: var(--text);
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
code {
|
||||
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||
border-radius: 6px;
|
||||
padding: 0.12rem 0.35rem;
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
color: var(--text);
|
||||
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
.callout {
|
||||
margin-top: 14px;
|
||||
border: 1px solid rgba(37, 193, 122, 0.28);
|
||||
border-radius: 8px;
|
||||
padding: 14px 16px;
|
||||
background: rgba(37, 193, 122, 0.08);
|
||||
}
|
||||
|
||||
.callout p {
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.note {
|
||||
color: var(--faint);
|
||||
}
|
||||
|
||||
@media (max-width: 720px) {
|
||||
main {
|
||||
padding: 28px 16px 40px;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 1.6rem;
|
||||
}
|
||||
|
||||
section {
|
||||
padding: 18px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<header>
|
||||
<p class="eyebrow">Plan Document</p>
|
||||
<h1>Durable Options Tape History</h1>
|
||||
<p class="summary">
|
||||
Make the options tape a signal-first live instrument with scroll-gated historical depth: keep the hot cache at
|
||||
100 option prints, load older rows from ClickHouse only at the scroll gate, preserve execution context, and
|
||||
render ClickHouse-backed rows exactly like any other valid flow row.
|
||||
</p>
|
||||
<div class="meta" aria-label="Plan metadata">
|
||||
<span class="chip">Created 2026-05-16 17:11</span>
|
||||
<span class="chip">Mode: Plan</span>
|
||||
<span class="chip">Surface: Options Tape</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section class="highlight">
|
||||
<h2>Plan Summary</h2>
|
||||
<p>
|
||||
Treat <strong>stale</strong> strictly as feed health, not as historical-row quality. The user should be able to
|
||||
analyze current live prints and earlier flow in one continuous tape, with no visual distinction between hot-cache
|
||||
rows and ClickHouse-backed rows.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Goals</h2>
|
||||
<ul>
|
||||
<li>Keep the options tape scrolling infinitely from the user's perspective.</li>
|
||||
<li>Hold only the 100 newest option prints in the hot live cache.</li>
|
||||
<li>Use ClickHouse as the durable source for older rows once the scroll gate requests history.</li>
|
||||
<li>Store all option-print data, including synthetic prints and execution context such as NBBO, spot, and IV.</li>
|
||||
<li>Surface historical flow as real analyzable flow, not as stale, old, or degraded data.</li>
|
||||
<li>Keep the default tape view signal-first while exposing all/raw prints from the existing Filter menu.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Proposed Changes</h2>
|
||||
<ul>
|
||||
<li>
|
||||
Keep <code>islandflow-0sa</code>'s useful pieces: scroll-hold behavior, <code>LIVE_OPTIONS_HEAD_LIMIT = 100</code>,
|
||||
lazy <code>/history/options</code> loading, cache-first scoped snapshots, and preserved execution-context columns.
|
||||
</li>
|
||||
<li>
|
||||
Stop tests and UI copy from asserting that valid rows older than 24 hours are <strong>stale</strong> when shown as
|
||||
history.
|
||||
</li>
|
||||
<li>
|
||||
Keep freshness gating only for live fanout/cache admission and channel health, not for historical validity.
|
||||
</li>
|
||||
<li>
|
||||
Remove dead <code>LiveHistoryBuffer</code> and auto-hydration scaffolding if it remains unused after the flow is
|
||||
explicit.
|
||||
</li>
|
||||
<li>
|
||||
Keep the default options tape view as <code>signal</code>, and add a filter-menu view control with
|
||||
<strong>Signal</strong> and <strong>All prints</strong>.
|
||||
</li>
|
||||
<li>
|
||||
Ensure hot-cache rows and ClickHouse history rows use the same row component, same styling, same sorting, and
|
||||
same interactions.
|
||||
</li>
|
||||
<li>
|
||||
Keep cursor/key-based deduping so scroll-gated history does not duplicate the 100-row hot head.
|
||||
</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Relevant Context</h2>
|
||||
<ul>
|
||||
<li>
|
||||
Prior work in <code>islandflow-0sa</code> already introduced scroll hold, a 100-row options head, lazy history,
|
||||
and cache-first scoped snapshots.
|
||||
</li>
|
||||
<li>
|
||||
The current storage/types path already includes execution context fields such as <code>execution_nbbo_*</code>,
|
||||
<code>execution_underlying_*</code>, and <code>execution_iv*</code>.
|
||||
</li>
|
||||
<li>
|
||||
Synthetic options prints already emit some execution context; the durable fix should verify this data survives
|
||||
ClickHouse writes and reads.
|
||||
</li>
|
||||
<li>
|
||||
The UI should prefer preserved execution context in row rendering before falling back to current NBBO lookup.
|
||||
</li>
|
||||
<li>
|
||||
Beads has related work in <code>islandflow-biq</code> for raw live options delivery and filter/backpressure
|
||||
observability.
|
||||
</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Implementation Steps</h2>
|
||||
<ul>
|
||||
<li>
|
||||
Audit the existing options tape flow from ingest, ClickHouse write/read, live snapshot, history endpoint, and web
|
||||
composition.
|
||||
</li>
|
||||
<li>
|
||||
Adjust API/live semantics so valid ClickHouse history can be older than freshness thresholds without being treated
|
||||
as degraded.
|
||||
</li>
|
||||
<li>
|
||||
Add the Filter-menu view toggle for <code>Signal</code> and <code>All prints</code>, with short copy explaining
|
||||
the difference.
|
||||
</li>
|
||||
<li>
|
||||
Ensure <code>buildOptionTapeQueryParams</code>, live subscriptions, and <code>/history/options</code> all receive
|
||||
the selected view consistently.
|
||||
</li>
|
||||
<li>
|
||||
Confirm option row rendering uses preserved <code>execution_nbbo_side</code>, <code>execution_underlying_spot</code>,
|
||||
and <code>execution_iv</code> when present.
|
||||
</li>
|
||||
<li>
|
||||
Remove deprecated or unused history/autohydration code paths that no longer help the intended scroll-gated flow.
|
||||
</li>
|
||||
<li>
|
||||
Add a deliberate reset path for local and VPS ClickHouse, documented as destructive and operator-confirmed.
|
||||
</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Risks, Limitations, and Mitigations</h2>
|
||||
<ul>
|
||||
<li>
|
||||
<strong>Risk:</strong> Resetting VPS data is destructive. <strong>Mitigation:</strong> make it a runbook or explicit
|
||||
command with backup/confirmation, never automatic app startup behavior.
|
||||
</li>
|
||||
<li>
|
||||
<strong>Risk:</strong> The signal/raw toggle could affect both options and flow filters unexpectedly.
|
||||
<strong>Mitigation:</strong> test option subscriptions, history query params, and flow packet filtering separately.
|
||||
</li>
|
||||
<li>
|
||||
<strong>Risk:</strong> Older history fetch latency could be visible at the scroll gate. <strong>Mitigation:</strong>
|
||||
keep lazy loading, expose loading/error state if needed, and avoid background auto-hydration.
|
||||
</li>
|
||||
<li>
|
||||
<strong>Risk:</strong> Prior fixes may have left overlapping history logic. <strong>Mitigation:</strong> remove unused
|
||||
scaffolding only after tests cover the intended hot-cache plus ClickHouse path.
|
||||
</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Validation</h2>
|
||||
<ul>
|
||||
<li>
|
||||
Storage tests: <code>fetchRecentOptionPrints</code> and <code>fetchOptionPrintsBefore</code> return execution NBBO,
|
||||
spot, IV, signal metadata, and raw/signal filtering correctly.
|
||||
</li>
|
||||
<li>
|
||||
API/live tests: generic options snapshots cap at 100, scoped snapshots prefer hot cache, history preserves
|
||||
<code>next_before</code>, and rows older than 24 hours return as valid history.
|
||||
</li>
|
||||
<li>
|
||||
Web tests: Filter menu toggles <code>Signal</code>/<code>All prints</code>, scroll gate calls
|
||||
<code>loadOlder("options")</code>, ClickHouse rows compose with no duplicate seam and no distinct styling, and
|
||||
preserved execution context drives Spot, Side, Details, and IV display.
|
||||
</li>
|
||||
<li>
|
||||
Validation commands: <code>bun test packages/storage/tests/option-prints.test.ts services/api/tests/live.test.ts apps/web/app/terminal.test.ts</code>
|
||||
and <code>bun --cwd=apps/web run build</code>.
|
||||
</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Open Questions</h2>
|
||||
<ul>
|
||||
<li>
|
||||
Exact VPS reset command sequence should be confirmed against the live deployment state before execution.
|
||||
</li>
|
||||
<li>
|
||||
Decide during implementation whether to track the reset/runbook in a new Beads issue or fold it into
|
||||
<code>islandflow-biq</code>.
|
||||
</li>
|
||||
</ul>
|
||||
<div class="callout">
|
||||
<p>
|
||||
Fixed assumptions: historical ClickHouse rows should be visually indistinguishable from hot-cache rows, and local
|
||||
plus VPS wipe should be an operator-confirmed reset path rather than a background migration.
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
245
docs/turns/2026-05-16-1725-durable-options-tape-history.html
Normal file
245
docs/turns/2026-05-16-1725-durable-options-tape-history.html
Normal file
|
|
@ -0,0 +1,245 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Durable Options Tape History</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: #06080b;
|
||||
--panel: #111820;
|
||||
--panel-2: #0d141b;
|
||||
--border: rgba(255, 255, 255, 0.1);
|
||||
--border-strong: rgba(245, 166, 35, 0.34);
|
||||
--text: #e6edf4;
|
||||
--muted: #90a0b2;
|
||||
--faint: #6e7b8c;
|
||||
--accent: #f5a623;
|
||||
--accent-soft: rgba(245, 166, 35, 0.14);
|
||||
--good: #25c17a;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
background: linear-gradient(180deg, rgba(17, 24, 32, 0.92), rgba(6, 8, 11, 0.98) 320px), var(--bg);
|
||||
color: var(--text);
|
||||
font: 15px/1.55 "IBM Plex Sans", -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
|
||||
}
|
||||
|
||||
main {
|
||||
max-width: 1040px;
|
||||
margin: 0 auto;
|
||||
padding: 40px 24px 56px;
|
||||
}
|
||||
|
||||
header {
|
||||
border-bottom: 1px solid var(--border);
|
||||
padding-bottom: 24px;
|
||||
}
|
||||
|
||||
.eyebrow,
|
||||
h2,
|
||||
.chip {
|
||||
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
letter-spacing: 0.1em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.eyebrow {
|
||||
margin: 0 0 10px;
|
||||
color: var(--accent);
|
||||
font-size: 0.76rem;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0;
|
||||
max-width: 760px;
|
||||
font-size: 2rem;
|
||||
line-height: 1.1;
|
||||
}
|
||||
|
||||
.summary {
|
||||
max-width: 820px;
|
||||
margin: 16px 0 0;
|
||||
color: var(--muted);
|
||||
font-size: 1rem;
|
||||
}
|
||||
|
||||
.meta {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-top: 18px;
|
||||
}
|
||||
|
||||
.chip {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 999px;
|
||||
padding: 4px 9px;
|
||||
background: rgba(255, 255, 255, 0.04);
|
||||
color: var(--muted);
|
||||
font-size: 0.7rem;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
section {
|
||||
margin-top: 26px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
padding: 20px 22px;
|
||||
background: linear-gradient(180deg, rgba(17, 24, 32, 0.94), rgba(13, 20, 27, 0.94));
|
||||
}
|
||||
|
||||
section.summary-band {
|
||||
border-color: var(--border-strong);
|
||||
background: linear-gradient(180deg, rgba(245, 166, 35, 0.12), rgba(17, 24, 32, 0.92));
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin: 0 0 12px;
|
||||
color: var(--text);
|
||||
font-size: 0.82rem;
|
||||
}
|
||||
|
||||
p,
|
||||
li {
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
ul {
|
||||
margin: 0;
|
||||
padding-left: 1.1rem;
|
||||
}
|
||||
|
||||
li {
|
||||
margin: 8px 0;
|
||||
}
|
||||
|
||||
strong {
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
code {
|
||||
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||
border-radius: 6px;
|
||||
padding: 0.12rem 0.35rem;
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
color: var(--text);
|
||||
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
.ok {
|
||||
color: var(--good);
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
@media (max-width: 720px) {
|
||||
main {
|
||||
padding: 28px 16px 40px;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 1.55rem;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<header>
|
||||
<p class="eyebrow">Turn Document</p>
|
||||
<h1>Durable Options Tape History</h1>
|
||||
<p class="summary">
|
||||
Implemented the durable options tape plan: the live hot head is capped at 100 rows, older rows are preserved behind
|
||||
the scroll gate, ClickHouse history keeps execution context, and the Filter menu now exposes Signal versus All
|
||||
prints semantics.
|
||||
</p>
|
||||
<div class="meta" aria-label="Turn metadata">
|
||||
<span class="chip">2026-05-16 17:25</span>
|
||||
<span class="chip">Beads: islandflow-200</span>
|
||||
<span class="chip">Surface: Options Tape</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section class="summary-band">
|
||||
<h2>Summary</h2>
|
||||
<p>
|
||||
The options tape now behaves as a continuous instrument: the live cache stays lean, historical rows arrive only
|
||||
when scrolling asks for them, and old valid rows are not treated as degraded just because they came from durable
|
||||
history.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Changes Made</h2>
|
||||
<ul>
|
||||
<li>Changed the API default options live cache limit to <code>100</code>.</li>
|
||||
<li>Removed the unused scoped live auto-hydration path so history is loaded by the scroll gate.</li>
|
||||
<li>Fixed unbounded options/equities history retention so a cap of <code>0</code> means keep the loaded tail.</li>
|
||||
<li>Added a Filter menu <code>Options View</code> toggle for <code>Signal</code> and <code>All prints</code>.</li>
|
||||
<li>Ensured All prints clears signal-only side/type/min-notional/security constraints.</li>
|
||||
<li>Added a destructive ClickHouse reset runbook for local and VPS operators.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Context</h2>
|
||||
<p>
|
||||
The prior plan called out useful partial work already in the repo: ClickHouse history endpoints, execution-context
|
||||
columns, scroll-hold behavior, and a shared row renderer. This implementation keeps those pieces and removes the
|
||||
ambiguous history/autohydration behavior around them.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Important Implementation Details</h2>
|
||||
<ul>
|
||||
<li><code>/history/options</code> still uses the selected option filters and scope, including raw contract drilldowns.</li>
|
||||
<li>Storage tests now verify execution NBBO side, underlying spot, IV, and signal reasons survive normalization.</li>
|
||||
<li>The options row path already preferred <code>execution_nbbo_side</code>, <code>execution_underlying_spot</code>, and <code>execution_iv</code>; tests cover that behavior.</li>
|
||||
<li>The reset runbook is documented in <code>docs/clickhouse-reset-runbook.md</code> and is explicitly operator-confirmed.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Expected Impact for End-Users</h2>
|
||||
<p>
|
||||
Traders can stay on a signal-first tape by default, switch to raw prints when investigating, and scroll into older
|
||||
ClickHouse-backed flow without seeing a separate stale-history treatment.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Validation</h2>
|
||||
<ul>
|
||||
<li><span class="ok">Passed:</span> <code>bun test packages/storage/tests/option-prints.test.ts services/api/tests/live.test.ts apps/web/app/terminal.test.ts</code></li>
|
||||
<li><span class="ok">Passed:</span> <code>bun --cwd=apps/web run build</code></li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Issues, Limitations, and Mitigations</h2>
|
||||
<ul>
|
||||
<li>The ClickHouse reset remains destructive. Mitigation: documented as a manual runbook only, never automatic startup behavior.</li>
|
||||
<li>No live browser smoke test was run in this turn. Mitigation: unit coverage and production build exercised the changed web paths.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Follow-up Work</h2>
|
||||
<p>No new follow-up issue was needed. The implementation task is tracked and completed in <code>islandflow-200</code>.</p>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
219
docs/turns/2026-05-16-1752-speed-up-docker-deploys.html
Normal file
219
docs/turns/2026-05-16-1752-speed-up-docker-deploys.html
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Speed Up Docker Deploys</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: #06080b;
|
||||
--panel: #111820;
|
||||
--panel-2: #0d141b;
|
||||
--text: #e6edf4;
|
||||
--muted: #90a0b2;
|
||||
--faint: #6e7b8c;
|
||||
--line: #ffffff14;
|
||||
--accent: #f5a623;
|
||||
--accent-soft: #f5a6231f;
|
||||
--ok: #25c17a;
|
||||
--warn: #ffb130;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font: 15px/1.55 "IBM Plex Sans", system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
}
|
||||
|
||||
main {
|
||||
max-width: 920px;
|
||||
margin: 0 auto;
|
||||
padding: 48px 24px 64px;
|
||||
}
|
||||
|
||||
header {
|
||||
border-bottom: 1px solid var(--line);
|
||||
margin-bottom: 28px;
|
||||
padding-bottom: 24px;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0 0 10px;
|
||||
font-size: 2rem;
|
||||
line-height: 1.1;
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin: 30px 0 10px;
|
||||
font-size: 1rem;
|
||||
letter-spacing: 0.08em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
p {
|
||||
max-width: 74ch;
|
||||
margin: 0 0 12px;
|
||||
}
|
||||
|
||||
ul {
|
||||
margin: 0;
|
||||
padding-left: 20px;
|
||||
}
|
||||
|
||||
li {
|
||||
margin: 7px 0;
|
||||
}
|
||||
|
||||
code {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 6px;
|
||||
background: var(--panel-2);
|
||||
color: var(--text);
|
||||
padding: 1px 5px;
|
||||
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
font-size: 0.92em;
|
||||
}
|
||||
|
||||
pre {
|
||||
overflow-x: auto;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
background: var(--panel-2);
|
||||
padding: 14px 16px;
|
||||
}
|
||||
|
||||
pre code {
|
||||
border: 0;
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
.summary {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
background: var(--panel);
|
||||
padding: 16px 18px;
|
||||
}
|
||||
|
||||
.meta {
|
||||
color: var(--muted);
|
||||
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
font-size: 0.78rem;
|
||||
letter-spacing: 0.04em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.badge {
|
||||
display: inline-block;
|
||||
margin-right: 8px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 999px;
|
||||
background: var(--accent-soft);
|
||||
color: var(--accent);
|
||||
padding: 2px 8px;
|
||||
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
font-size: 0.72rem;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0.08em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.ok {
|
||||
color: var(--ok);
|
||||
}
|
||||
|
||||
.warn {
|
||||
color: var(--warn);
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<header>
|
||||
<div class="meta">2026-05-16 17:52 America/New_York</div>
|
||||
<h1>Speed Up Docker Deploys</h1>
|
||||
<p class="summary">
|
||||
<span class="badge">Summary</span>
|
||||
Docker app images now cache dependency installation separately from source changes, and Docker rollouts now build only the images required by the selected deploy scope before restarting containers.
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<section>
|
||||
<h2>Summary</h2>
|
||||
<p>
|
||||
Implemented the Docker deployment speed-up plan from <code>/Users/kell/Desktop/speed-up-docker.md</code>. The first build after this change may still be slow, but source-only changes should no longer invalidate the expensive Bun and Python dependency layers.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Changes Made</h2>
|
||||
<ul>
|
||||
<li>Refactored <code>deployment/docker/Dockerfile.service</code> to copy workspace manifests, run cached <code>bun install --frozen-lockfile</code>, then copy source.</li>
|
||||
<li>Applied the same dependency-first build model to <code>deployment/docker/Dockerfile.web</code>, keeping the Next.js build after source copy.</li>
|
||||
<li>Updated <code>deployment/docker/Dockerfile.ingest-options</code> with separate cached pip and Bun install layers before copying source.</li>
|
||||
<li>Changed <code>scripts/deploy.ts</code> so Docker rollouts run explicit <code>docker compose build <services></code> followed by <code>docker compose up -d <services></code>.</li>
|
||||
<li>Documented the faster-build model, scoped rollouts, and appropriate <code>--no-build</code> usage in <code>deployment/docker/README.md</code>.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Context</h2>
|
||||
<p>
|
||||
The previous Dockerfiles copied all app, service, and package source before dependency installation. That made nearly every code change invalidate <code>bun install</code>, increasing VPS deploy time. The deployment helper also used broad <code>up -d --build</code> behavior rather than a clean build phase scoped to the selected service set.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Important Implementation Details</h2>
|
||||
<p>
|
||||
Each app image now copies root deployment manifests plus every workspace <code>package.json</code> before installing dependencies. The source tree is copied only after the install layer is complete.
|
||||
</p>
|
||||
<pre><code>RUN --mount=type=cache,target=/root/.bun/install/cache \
|
||||
bun install --frozen-lockfile</code></pre>
|
||||
<p>
|
||||
The <code>ingest-options</code> image also copies <code>services/ingest-options/py/requirements.txt</code> before source and uses a pip cache mount:
|
||||
</p>
|
||||
<pre><code>RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
"${VIRTUAL_ENV}/bin/pip" install -r services/ingest-options/py/requirements.txt</code></pre>
|
||||
<p>
|
||||
For full Docker deploys, the helper builds the six core app services explicitly. For scoped deploys, it builds and restarts only the requested services.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Expected Impact for End-Users</h2>
|
||||
<p>
|
||||
Users should see faster deployment turnaround after ordinary source edits because dependency installation is reused when manifests and locks have not changed. Scoped deploys should also disturb fewer containers, reducing restart surface for web-only, API-only, and backend-only updates.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Validation</h2>
|
||||
<ul>
|
||||
<li><span class="ok">Passed:</span> <code>bun run check:docker-workspace</code></li>
|
||||
<li><span class="ok">Passed:</span> <code>./deploy --help</code></li>
|
||||
<li><span class="ok">Passed:</span> <code>docker compose -f deployment/docker/docker-compose.yml config --quiet</code> with a temporary copy of <code>.env.example</code></li>
|
||||
<li><span class="ok">Passed:</span> <code>bun --cwd=apps/web run build</code></li>
|
||||
<li><span class="ok">Passed:</span> <code>bun test</code> with 222 passing tests</li>
|
||||
<li><span class="warn">Not run:</span> targeted Docker image builds because this session could not connect to the Docker daemon at <code>unix:///Users/kell/.orbstack/run/docker.sock</code>.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Issues, Limitations, and Mitigations</h2>
|
||||
<p>
|
||||
Docker daemon access was unavailable locally, so image builds still need to be exercised on a machine with a running Docker daemon or during the next VPS rollout. Static Compose validation and repo test coverage passed, and the Dockerfiles use standard BuildKit cache mounts supported by modern Docker Compose v2.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Follow-up Work</h2>
|
||||
<p>
|
||||
No separate follow-up issue was created. The remaining verification is operational: run the targeted image builds once Docker or OrbStack is available.
|
||||
</p>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Fix Durable Options History Routing</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: oklch(0.14 0.012 250);
|
||||
--panel: oklch(0.19 0.018 250);
|
||||
--panel-soft: oklch(0.23 0.018 250);
|
||||
--border: oklch(0.44 0.018 250);
|
||||
--text: oklch(0.9 0.018 250);
|
||||
--muted: oklch(0.7 0.028 250);
|
||||
--amber: oklch(0.78 0.14 72);
|
||||
--green: oklch(0.72 0.16 154);
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font-family: IBM Plex Sans, Inter, ui-sans-serif, system-ui, sans-serif;
|
||||
line-height: 1.55;
|
||||
}
|
||||
|
||||
main {
|
||||
width: min(960px, calc(100vw - 40px));
|
||||
margin: 0 auto;
|
||||
padding: 48px 0;
|
||||
}
|
||||
|
||||
header {
|
||||
border-bottom: 1px solid var(--border);
|
||||
padding-bottom: 20px;
|
||||
margin-bottom: 28px;
|
||||
}
|
||||
|
||||
h1,
|
||||
h2 {
|
||||
line-height: 1.1;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: clamp(2rem, 4vw, 3rem);
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin-top: 30px;
|
||||
font-size: 1rem;
|
||||
color: var(--amber);
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
p {
|
||||
max-width: 74ch;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
ul {
|
||||
padding-left: 1.2rem;
|
||||
}
|
||||
|
||||
li {
|
||||
margin: 0.45rem 0;
|
||||
}
|
||||
|
||||
code,
|
||||
pre {
|
||||
font-family: IBM Plex Mono, ui-monospace, SFMono-Regular, monospace;
|
||||
}
|
||||
|
||||
pre {
|
||||
overflow-x: auto;
|
||||
padding: 14px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
background: var(--panel);
|
||||
}
|
||||
|
||||
.summary {
|
||||
padding: 18px;
|
||||
border: 1px solid oklch(0.58 0.08 72);
|
||||
border-radius: 8px;
|
||||
background: oklch(0.21 0.035 72);
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.status {
|
||||
display: inline-flex;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
padding: 4px 9px;
|
||||
border: 1px solid oklch(0.55 0.11 154);
|
||||
border-radius: 999px;
|
||||
color: var(--green);
|
||||
font-size: 0.78rem;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
section {
|
||||
border-top: 1px solid oklch(0.32 0.018 250);
|
||||
padding-top: 4px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<header>
|
||||
<span class="status">Validated</span>
|
||||
<h1>Fix Durable Options History Routing</h1>
|
||||
<p>Turn completed on 2026-05-16 21:59 America/New_York.</p>
|
||||
</header>
|
||||
|
||||
<section>
|
||||
<h2>Summary</h2>
|
||||
<p class="summary">
|
||||
Options tape history now has a durable public route through same-origin deployments. The live Nginx Proxy Manager route was updated to include <code>/history/*</code>, deployment checks now fail when required API paths reach the web app, and the tape UI surfaces older-history load failures instead of leaving the user to infer that only the hot window exists.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Changes Made</h2>
|
||||
<ul>
|
||||
<li>Added <code>scripts/check-public-api-routes.ts</code> and the <code>check:public-api-routes</code> package script.</li>
|
||||
<li>Updated <code>scripts/deploy.ts</code> so same-origin API deploy verification probes required public routes.</li>
|
||||
<li>Updated <code>deployment/docker/README.md</code> to include <code>/history/*</code> in same-origin proxy routing and document the Nginx Proxy Manager regex.</li>
|
||||
<li>Added an options tape warning banner for live <code>/history/options</code> load errors.</li>
|
||||
<li>Updated live Nginx Proxy Manager config for <code>flow.deltaisland.io</code> so the public route regex includes <code>history</code>.</li>
|
||||
<li>Created follow-up Beads issue <code>islandflow-qd7</code> for the later <code>api.flow.deltaisland.io</code> migration.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Context</h2>
|
||||
<p>
|
||||
The API and ClickHouse path already supported older options history, but the public same-origin route sent <code>/history/options</code> to the Next.js app. That made the live tape feel capped at the newest hot-window rows even though durable history existed behind the API.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Important Implementation Details</h2>
|
||||
<p>
|
||||
The deploy smoke check performs GET probes and verifies JSON responses for these same-origin routes:
|
||||
</p>
|
||||
<pre>/prints/options
|
||||
/history/options
|
||||
/replay/options
|
||||
/nbbo/options
|
||||
/ws/live</pre>
|
||||
<p>
|
||||
The live proxy matcher is now:
|
||||
</p>
|
||||
<pre>^/(ws|replay|prints|joins|nbbo|dark|flow|candles|history)/</pre>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Expected Impact for End-Users</h2>
|
||||
<p>
|
||||
Users on <code>/tape</code> can scroll beyond the initial options hot window and receive older ClickHouse-backed rows through the same cursor path for Signal and All prints. If public routing regresses, the tape now shows a visible history loading failure.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Validation</h2>
|
||||
<ul>
|
||||
<li>Passed: <code>bun test apps/web/app/terminal.test.ts</code></li>
|
||||
<li>Passed: <code>bun test</code></li>
|
||||
<li>Passed: <code>bun --cwd=apps/web run build</code></li>
|
||||
<li>Passed: <code>bun run check:public-api-routes</code></li>
|
||||
<li>Passed: remote Nginx syntax check after updating the route.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Issues, Limitations, and Mitigations</h2>
|
||||
<ul>
|
||||
<li>The long-term API subdomain migration remains separate work. Mitigation: tracked as <code>islandflow-qd7</code>.</li>
|
||||
<li>The Nginx Proxy Manager database and generated proxy host file were both updated because the existing live file had prior generated-file edits. Mitigation: deployment docs now call out the durable advanced-config/API path.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Follow-up Work</h2>
|
||||
<p>
|
||||
Complete <code>islandflow-qd7</code> to move production API traffic to <code>api.flow.deltaisland.io</code> deliberately, including DNS, proxy behavior, CORS/websocket checks, docs, and deployment verification.
|
||||
</p>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
158
docs/turns/2026-05-16-live-tape-scroll-hold-history.html
Normal file
158
docs/turns/2026-05-16-live-tape-scroll-hold-history.html
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Turn Summary: Live tape scroll hold and lazy history</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: #0b1016;
|
||||
--panel: #111820;
|
||||
--border: rgba(255,255,255,0.1);
|
||||
--text: #e6edf4;
|
||||
--muted: #90a0b2;
|
||||
--accent: #f5a623;
|
||||
--accent-soft: rgba(245,166,35,0.16);
|
||||
--ok: #25c17a;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 32px;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font: 16px/1.55 "IBM Plex Sans", system-ui, sans-serif;
|
||||
}
|
||||
main { max-width: 980px; margin: 0 auto; }
|
||||
h1, h2 { line-height: 1.15; }
|
||||
h1 { margin: 0 0 8px; font-size: 2rem; }
|
||||
h2 { margin: 32px 0 12px; font-size: 1.1rem; letter-spacing: 0.04em; text-transform: uppercase; }
|
||||
p, li { color: var(--muted); }
|
||||
.summary {
|
||||
background: linear-gradient(180deg, rgba(245,166,35,0.14), rgba(245,166,35,0.05));
|
||||
border: 1px solid rgba(245,166,35,0.24);
|
||||
border-radius: 14px;
|
||||
padding: 20px 22px;
|
||||
margin-bottom: 28px;
|
||||
}
|
||||
.meta { color: var(--muted); font-size: 0.92rem; margin-bottom: 18px; }
|
||||
.panel {
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 14px;
|
||||
padding: 18px 20px;
|
||||
margin-top: 14px;
|
||||
}
|
||||
code {
|
||||
font-family: "IBM Plex Mono", ui-monospace, monospace;
|
||||
font-size: 0.92em;
|
||||
color: var(--text);
|
||||
background: rgba(255,255,255,0.06);
|
||||
padding: 0.14rem 0.35rem;
|
||||
border-radius: 6px;
|
||||
}
|
||||
pre {
|
||||
margin: 0;
|
||||
padding: 14px;
|
||||
overflow: auto;
|
||||
border-radius: 12px;
|
||||
border: 1px solid var(--border);
|
||||
background: rgba(0,0,0,0.22);
|
||||
}
|
||||
.ok { color: var(--ok); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<div class="summary">
|
||||
<h1>Live tape now holds on scroll, resumes at top, and lazy-loads deep history</h1>
|
||||
<p>
|
||||
The live tape no longer depends on a manual pause button in live mode. Scrolling away from the top now holds the
|
||||
tape automatically, <code>Jump to top</code> resumes it, the options hot head is capped at 100 rows, and older
|
||||
history is fetched from ClickHouse only when the scroll gate requests it.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="meta">Created 2026-05-16 during issue <code>islandflow-0sa</code>.</div>
|
||||
|
||||
<section>
|
||||
<h2>Summary</h2>
|
||||
<div class="panel">
|
||||
<p>
|
||||
This change aligns the tape with the intended operator workflow: hold the live head while investigating older
|
||||
rows, keep historical prints valid even when old, and avoid preloading a large ClickHouse backlog until the
|
||||
user actually scrolls into it.
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Changes Made</h2>
|
||||
<div class="panel">
|
||||
<ul>
|
||||
<li>Removed the live-mode Pause/Resume control from tape pane actions while keeping replay pause behavior intact.</li>
|
||||
<li>Changed live tape status copy from manual <code>Paused</code> semantics to scroll-held <code>Held</code>.</li>
|
||||
<li>Capped the live options head at <code>100</code> rows.</li>
|
||||
<li>Stopped scoped live history from auto-hydrating in the background.</li>
|
||||
<li>Made scoped options and equities snapshots prefer hot cached rows first, then backfill from ClickHouse when needed.</li>
|
||||
<li>Made options and equities history retention effectively unbounded on the client so deep scrolling does not get trimmed away prematurely.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Context</h2>
|
||||
<div class="panel">
|
||||
<p>
|
||||
The tape previously mixed several behaviors: a manual pause button, automatic scroll holding, scoped background
|
||||
auto-hydration, and a much deeper options hot head. That created two user-visible problems: the live control model
|
||||
felt redundant, and older prints could disappear or feel inconsistent when switching views or waiting for newer
|
||||
rows to arrive.
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Important Implementation Details</h2>
|
||||
<div class="panel">
|
||||
<ul>
|
||||
<li><code>apps/web/app/terminal.tsx</code>: live <code>usePausableTapeView</code> now treats scroll position as the hold source of truth.</li>
|
||||
<li><code>apps/web/app/terminal.tsx</code>: options live snapshot and retention now use a strict <code>LIVE_OPTIONS_HEAD_LIMIT = 100</code>.</li>
|
||||
<li><code>apps/web/app/terminal.tsx</code>: scoped history auto-hydration helper now returns no channels, so ClickHouse history stays lazy.</li>
|
||||
<li><code>services/api/src/live.ts</code>: scoped option/equity snapshots now filter the hot cache first, then merge ClickHouse backfill without seam duplicates.</li>
|
||||
</ul>
|
||||
<pre><code>statusLabel("connected", true, "live") === "Held"
|
||||
statusLabel("connected", true, "replay") === "Paused"</code></pre>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Validation</h2>
|
||||
<div class="panel">
|
||||
<ul>
|
||||
<li class="ok">Passed: <code>bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts</code></li>
|
||||
<li class="ok">Passed: <code>bun --cwd=apps/web run build</code></li>
|
||||
</ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Issues, Limitations, and Mitigations</h2>
|
||||
<div class="panel">
|
||||
<ul>
|
||||
<li>Scoped snapshots can still backfill from ClickHouse when the hot cache does not have enough matching rows. This is intentional so focused views do not start empty.</li>
|
||||
<li>Deep history is now lazy rather than eager, which reduces surprise and load, but the first deep-scroll request still depends on ClickHouse latency.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Follow-up Work</h2>
|
||||
<div class="panel">
|
||||
<p>No additional follow-up issues were created in this turn.</p>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
168
docs/turns/2026-05-17-0331-fix-live-tape-scroll-stability.html
Normal file
168
docs/turns/2026-05-17-0331-fix-live-tape-scroll-stability.html
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Fix Live Tape Scroll Stability</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: oklch(0.14 0.012 250);
|
||||
--panel: oklch(0.18 0.014 250);
|
||||
--text: oklch(0.92 0.012 250);
|
||||
--muted: oklch(0.72 0.018 250);
|
||||
--accent: oklch(0.76 0.12 74);
|
||||
--border: oklch(0.72 0.012 250 / 0.18);
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font: 15px/1.6 ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
}
|
||||
|
||||
main {
|
||||
max-width: 920px;
|
||||
margin: 0 auto;
|
||||
padding: 48px 24px 64px;
|
||||
}
|
||||
|
||||
header {
|
||||
margin-bottom: 28px;
|
||||
padding-bottom: 18px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0 0 8px;
|
||||
font-size: 2rem;
|
||||
line-height: 1.15;
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin: 28px 0 10px;
|
||||
color: var(--accent);
|
||||
font-size: 1rem;
|
||||
letter-spacing: 0.04em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
p,
|
||||
li {
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
code {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 5px;
|
||||
padding: 1px 5px;
|
||||
background: var(--panel);
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
pre {
|
||||
overflow: auto;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
padding: 14px;
|
||||
background: var(--panel);
|
||||
}
|
||||
|
||||
pre code {
|
||||
border: 0;
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<header>
|
||||
<h1>Fix Live Tape Scroll Stability</h1>
|
||||
<p>
|
||||
Completed on 2026-05-17 at 03:31 America/New_York for Beads issue
|
||||
<code>islandflow-9dg</code>.
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<section>
|
||||
<h2>Summary</h2>
|
||||
<p>
|
||||
The live tape now keeps the visible scrolled segment stable while new prints arrive. When
|
||||
the user is away from the top, the view freezes both the hot live head and the displayed
|
||||
history segment, only allowing genuinely older history to append below the current tail.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Changes Made</h2>
|
||||
<ul>
|
||||
<li>Added <code>mergeHeldTapeHistory</code> to filter held history updates by the visible tail.</li>
|
||||
<li>Updated <code>usePausableTapeView</code> to keep a displayed history ref while scroll-held.</li>
|
||||
<li>Resynced displayed history automatically when the user jumps back to the top or otherwise resumes.</li>
|
||||
<li>Increased tape virtualizer overscan for options, equities, flow, alerts, classifier, and dark panes.</li>
|
||||
<li>Added a fixed row-lane table background so fast scrolling shows a stable substrate instead of blank holes.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Context</h2>
|
||||
<p>
|
||||
Live session history receives both ClickHouse history and hot-window overflow from new live
|
||||
prints. Before this change, the pausable view froze live rows during scroll hold but still
|
||||
composed against the mutating history array, so newer overflow rows could insert above the
|
||||
user's current viewport.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Important Implementation Details</h2>
|
||||
<p>
|
||||
The stable merge compares incoming history with the current displayed history tail. Rows
|
||||
newer than that tail are withheld during hold, duplicates from the frozen live head are
|
||||
removed, and older lazy-loaded rows remain eligible to append.
|
||||
</p>
|
||||
<pre><code>const next = mergeHeldTapeHistory(displayedHistoryRef.current, historyItems, projected.items);</code></pre>
|
||||
<p>
|
||||
When hold ends, <code>displayedHistoryRef</code> is replaced with the latest live session
|
||||
history, so buffered overflow catches up cleanly on jump-to-top.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Expected Impact for End-Users</h2>
|
||||
<p>
|
||||
Users can scroll into older options or equities prints without the rows shifting under them
|
||||
as new live prints arrive. The <code>+N new</code> counter can continue accumulating until
|
||||
they jump back to the top, where the tape catches up.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Validation</h2>
|
||||
<ul>
|
||||
<li><code>bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts</code>: passed, 90 tests.</li>
|
||||
<li><code>bun --cwd=apps/web run build</code>: passed.</li>
|
||||
<li><code>curl -I http://localhost:3000/tape</code> against the local dev server: returned 200 OK.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Issues, Limitations, and Mitigations</h2>
|
||||
<p>
|
||||
This change preserves row stability in the frontend view model. It does not alter backend
|
||||
history pagination or wire protocols. The fixed table substrate mitigates visual blanking
|
||||
during fast scrolls, while actual row rendering remains virtualized. Browser automation was
|
||||
attempted, but the local Node automation runtime did not have Playwright installed, so the
|
||||
handoff relies on unit tests, production build, and the local HTTP smoke check.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Follow-up Work</h2>
|
||||
<p>No follow-up Beads issues were needed for this turn.</p>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -20,6 +20,7 @@
|
|||
"deploy": "bun run scripts/deploy.ts",
|
||||
"deploy:main": "./deploy main",
|
||||
"deploy:current-branch": "./deploy current-branch",
|
||||
"check:public-api-routes": "bun run scripts/check-public-api-routes.ts",
|
||||
"sync:docker-workspace": "bun run scripts/sync-docker-workspace.ts",
|
||||
"check:docker-workspace": "bun run scripts/check-docker-workspace.ts"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -48,6 +48,25 @@ describe("option-prints storage helpers", () => {
|
|||
queries.push(query);
|
||||
return {
|
||||
async json<T>() {
|
||||
if (query.includes("trace-ctx")) {
|
||||
return [
|
||||
{
|
||||
...basePrint,
|
||||
trace_id: "trace-ctx",
|
||||
conditions: [],
|
||||
execution_nbbo_bid: "1.20",
|
||||
execution_nbbo_ask: "1.30",
|
||||
execution_nbbo_mid: "1.25",
|
||||
execution_nbbo_side: "A",
|
||||
execution_underlying_spot: "450.05",
|
||||
execution_underlying_source: "equity_quote_mid",
|
||||
execution_iv: "0.42",
|
||||
execution_iv_source: "synthetic_pressure_model",
|
||||
signal_reasons: ["large_notional"],
|
||||
signal_pass: 1
|
||||
}
|
||||
] as T;
|
||||
}
|
||||
return [] as T;
|
||||
}
|
||||
};
|
||||
|
|
@ -63,8 +82,9 @@ describe("option-prints storage helpers", () => {
|
|||
optionContractId: "AAPL-2025-01-17-200-C",
|
||||
sinceTs: 123
|
||||
});
|
||||
await fetchOptionPrintsBefore(client, 100, 5, 20, "alpaca");
|
||||
await fetchOptionPrintsBefore(client, 100, 5, 20, "alpaca", { view: "raw" });
|
||||
await fetchOptionPrintsByTraceIds(client, ["trace-1", "trace-2"]);
|
||||
const rows = await fetchRecentOptionPrints(client, 1, "trace-ctx", { view: "signal" });
|
||||
|
||||
expect(queries[0]).toContain("signal_pass = 1");
|
||||
expect(queries[0]).toContain("(is_etf = 0 OR is_etf IS NULL)");
|
||||
|
|
@ -76,7 +96,12 @@ describe("option-prints storage helpers", () => {
|
|||
expect(queries[0]).toContain("ts >= 123");
|
||||
expect(queries[1]).toContain("(ts, seq) < (100, 5)");
|
||||
expect(queries[1]).toContain("startsWith(trace_id, 'alpaca')");
|
||||
expect(queries[1]).not.toContain("signal_pass = 1");
|
||||
expect(queries[1]).toContain("ORDER BY ts DESC, seq DESC LIMIT 20");
|
||||
expect(queries[2]).toContain("trace_id IN ('trace-1', 'trace-2')");
|
||||
expect(rows[0].execution_nbbo_side).toBe("A");
|
||||
expect(rows[0].execution_underlying_spot).toBe(450.05);
|
||||
expect(rows[0].execution_iv).toBe(0.42);
|
||||
expect(rows[0].signal_reasons).toEqual(["large_notional"]);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
41
scripts/check-public-api-routes.ts
Normal file
41
scripts/check-public-api-routes.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
#!/usr/bin/env bun
|
||||
|
||||
type RouteCheck = {
|
||||
path: string;
|
||||
expectJson: boolean;
|
||||
};
|
||||
|
||||
const routeChecks: RouteCheck[] = [
|
||||
{ path: "/prints/options?view=signal&limit=1", expectJson: true },
|
||||
{ path: "/history/options?view=signal&before_ts=4102444800000&before_seq=999999999&limit=1", expectJson: true },
|
||||
{ path: "/replay/options?view=signal&after_ts=0&after_seq=0&limit=1", expectJson: true },
|
||||
{ path: "/nbbo/options?limit=1", expectJson: true },
|
||||
{ path: "/ws/live", expectJson: true }
|
||||
];
|
||||
|
||||
const appUrl = process.env.DEPLOY_PUBLIC_APP_URL?.trim() || process.argv[2]?.trim();
|
||||
const baseUrl = appUrl || "https://flow.deltaisland.io";
|
||||
|
||||
const isJsonResponse = (response: Response): boolean => {
|
||||
return (response.headers.get("content-type") ?? "").toLowerCase().includes("application/json");
|
||||
};
|
||||
|
||||
const assertPublicApiRoute = async ({ path, expectJson }: RouteCheck): Promise<void> => {
|
||||
const url = new URL(path, baseUrl);
|
||||
const response = await fetch(url);
|
||||
const responseText = await response.text();
|
||||
|
||||
if (response.status === 404) {
|
||||
throw new Error(`${url.pathname} returned 404; route is likely reaching the web app`);
|
||||
}
|
||||
|
||||
if (expectJson && !isJsonResponse(response)) {
|
||||
const sample = responseText.replace(/\s+/g, " ").slice(0, 120);
|
||||
throw new Error(`${url.pathname} returned non-JSON content (${response.headers.get("content-type") ?? "none"}): ${sample}`);
|
||||
}
|
||||
};
|
||||
|
||||
for (const check of routeChecks) {
|
||||
await assertPublicApiRoute(check);
|
||||
console.log(`ok ${check.path}`);
|
||||
}
|
||||
|
|
@ -324,6 +324,15 @@ function dockerServicesForScope(scope: DeployScope): string[] {
|
|||
}
|
||||
}
|
||||
|
||||
function dockerBuildServicesForScope(scope: DeployScope): string[] {
|
||||
switch (scope) {
|
||||
case "full":
|
||||
return [...DOCKER_CORE_SERVICES];
|
||||
default:
|
||||
return dockerServicesForScope(scope);
|
||||
}
|
||||
}
|
||||
|
||||
function dockerLogServicesForScope(scope: DeployScope): string[] {
|
||||
switch (scope) {
|
||||
case "web":
|
||||
|
|
@ -565,15 +574,16 @@ function remoteDockerRollout(
|
|||
forceRecreate: boolean,
|
||||
noBuild: boolean
|
||||
): void {
|
||||
const services = dockerServicesForScope(scope);
|
||||
const args = ["up", "-d"];
|
||||
if (!noBuild) {
|
||||
args.push("--build");
|
||||
}
|
||||
const rolloutServices = dockerServicesForScope(scope);
|
||||
const upArgs = ["up", "-d"];
|
||||
if (forceRecreate) {
|
||||
args.push("--force-recreate");
|
||||
upArgs.push("--force-recreate");
|
||||
}
|
||||
const command = `docker compose ${[...args, ...services].join(" ")}`;
|
||||
const buildServices = dockerBuildServicesForScope(scope);
|
||||
const buildCommand = noBuild
|
||||
? null
|
||||
: `docker compose build ${buildServices.join(" ")}`;
|
||||
const upCommand = `docker compose ${[...upArgs, ...rolloutServices].join(" ")}`;
|
||||
|
||||
runRemoteScript(
|
||||
"Remote Rollout",
|
||||
|
|
@ -583,7 +593,7 @@ set -euo pipefail
|
|||
${remoteGitUpdateScript(mode, branch)}
|
||||
|
||||
cd ${shellEscape(REMOTE_DOCKER_DEPLOYMENT)}
|
||||
${command}
|
||||
${buildCommand ? `${buildCommand}\n` : ""}${upCommand}
|
||||
`
|
||||
);
|
||||
}
|
||||
|
|
@ -722,9 +732,7 @@ function publicVerification(scope: DeployScope): void {
|
|||
}
|
||||
|
||||
if (scopeIncludesApi(scope)) {
|
||||
console.log(
|
||||
"Skipping separate public API health check; same-origin mode relies on the public app check plus runtime-local API verification."
|
||||
);
|
||||
runChecked("bun", ["run", "scripts/check-public-api-routes.ts", PUBLIC_APP_URL]);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,8 @@ import {
|
|||
type Cursor,
|
||||
type EquityCandle,
|
||||
type EquityPrint,
|
||||
type LiveChannel
|
||||
type LiveChannel,
|
||||
type OptionPrint
|
||||
} from "@islandflow/types";
|
||||
import { createMetrics } from "@islandflow/observability";
|
||||
import type { RedisClientType } from "redis";
|
||||
|
|
@ -71,7 +72,7 @@ const CHART_LIMITS = {
|
|||
} as const;
|
||||
|
||||
const DEFAULT_LIVE_LIMITS: GenericLiveLimits = {
|
||||
options: 1000,
|
||||
options: 100,
|
||||
nbbo: 1000,
|
||||
equities: 1000,
|
||||
"equity-quotes": 500,
|
||||
|
|
@ -456,6 +457,54 @@ export const buildOptionSnapshotFilters = (
|
|||
};
|
||||
};
|
||||
|
||||
const matchesScopedOptionSnapshot = (
|
||||
item: OptionPrint,
|
||||
subscription: Extract<LiveSubscription, { channel: "options" }>
|
||||
): boolean => {
|
||||
if (!matchesOptionPrintFilters(item, subscription.filters)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (subscription.option_contract_id && item.option_contract_id !== subscription.option_contract_id) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!subscription.underlying_ids?.length) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const allowed = new Set(subscription.underlying_ids.map((value) => value.toUpperCase()));
|
||||
return allowed.has(item.underlying_id.toUpperCase());
|
||||
};
|
||||
|
||||
const matchesScopedEquitySnapshot = (
|
||||
item: EquityPrint,
|
||||
subscription: Extract<LiveSubscription, { channel: "equities" }>
|
||||
): boolean => {
|
||||
if (!subscription.underlying_ids?.length) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const allowed = new Set(subscription.underlying_ids.map((value) => value.toUpperCase()));
|
||||
return allowed.has(item.underlying_id.toUpperCase());
|
||||
};
|
||||
|
||||
const mergeSnapshotBackfill = <T>(
|
||||
cached: T[],
|
||||
backfill: T[],
|
||||
limit: number,
|
||||
cursorOf: (item: T) => Cursor
|
||||
): T[] => {
|
||||
const deduped = new Map<string, T>();
|
||||
|
||||
for (const item of [...cached, ...backfill]) {
|
||||
const cursor = cursorOf(item);
|
||||
deduped.set(`${cursor.ts}:${cursor.seq}`, item);
|
||||
}
|
||||
|
||||
return sortGenericItems(Array.from(deduped.values()), cursorOf).slice(0, limit);
|
||||
};
|
||||
|
||||
const candleRedisKey = (underlyingId: string, intervalMs: number): string =>
|
||||
`live:equity-candles:${underlyingId}:${intervalMs}`;
|
||||
|
||||
|
|
@ -740,12 +789,20 @@ export class LiveStateManager {
|
|||
async getSnapshot(subscription: LiveSubscription): Promise<FeedSnapshot<unknown>> {
|
||||
switch (subscription.channel) {
|
||||
case "options": {
|
||||
const config = this.generic.options;
|
||||
const limit = snapshotLimitFor(subscription, config.limit);
|
||||
const scoped = Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id);
|
||||
if (subscription.filters?.view === "raw" || scoped) {
|
||||
this.stats.scopedClickHouseSnapshots += 1;
|
||||
const limit = snapshotLimitFor(subscription, this.generic.options.limit);
|
||||
const storageFilters = buildOptionSnapshotFilters(subscription);
|
||||
const items = await fetchRecentOptionPrints(this.clickhouse, limit, undefined, storageFilters);
|
||||
const cached = (this.genericItems.get("options") ?? [])
|
||||
.filter((entry) => matchesScopedOptionSnapshot(entry, subscription))
|
||||
.slice(0, limit);
|
||||
let items = cached;
|
||||
if (cached.length < limit) {
|
||||
this.stats.scopedClickHouseSnapshots += 1;
|
||||
const storageFilters = buildOptionSnapshotFilters(subscription);
|
||||
const backfill = await fetchRecentOptionPrints(this.clickhouse, limit, undefined, storageFilters);
|
||||
items = mergeSnapshotBackfill(cached, backfill, limit, (entry) => ({ ts: entry.ts, seq: entry.seq }));
|
||||
}
|
||||
return {
|
||||
subscription,
|
||||
items,
|
||||
|
|
@ -754,9 +811,7 @@ export class LiveStateManager {
|
|||
};
|
||||
}
|
||||
|
||||
const config = this.generic.options;
|
||||
this.stats.genericCacheSnapshots += 1;
|
||||
const limit = snapshotLimitFor(subscription, config.limit);
|
||||
const items = (this.genericItems.get("options") ?? [])
|
||||
.filter((entry) => matchesOptionPrintFilters(entry, subscription.filters))
|
||||
.slice(0, limit);
|
||||
|
|
@ -785,9 +840,16 @@ export class LiveStateManager {
|
|||
const config = this.generic.equities;
|
||||
const limit = snapshotLimitFor(subscription, config.limit);
|
||||
if (subscription.underlying_ids?.length) {
|
||||
this.stats.scopedClickHouseSnapshots += 1;
|
||||
const filters: EquityPrintQueryFilters = { underlyingIds: subscription.underlying_ids };
|
||||
const items = await fetchRecentEquityPrints(this.clickhouse, limit, filters);
|
||||
const cached = (this.genericItems.get("equities") ?? [])
|
||||
.filter((entry) => matchesScopedEquitySnapshot(entry, subscription))
|
||||
.slice(0, limit);
|
||||
let items = cached;
|
||||
if (cached.length < limit) {
|
||||
this.stats.scopedClickHouseSnapshots += 1;
|
||||
const filters: EquityPrintQueryFilters = { underlyingIds: subscription.underlying_ids };
|
||||
const backfill = await fetchRecentEquityPrints(this.clickhouse, limit, filters);
|
||||
items = mergeSnapshotBackfill(cached, backfill, limit, config.cursor);
|
||||
}
|
||||
return {
|
||||
subscription,
|
||||
items,
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ describe("LiveStateManager", () => {
|
|||
expect(limits.flow).toBe(500);
|
||||
expect(limits["equity-quotes"]).toBe(500);
|
||||
expect(limits.alerts).toBe(300);
|
||||
expect(resolveGenericLiveLimits({} as NodeJS.ProcessEnv).options).toBe(100);
|
||||
});
|
||||
|
||||
it("hydrates snapshots from redis generic windows", async () => {
|
||||
|
|
@ -520,6 +521,32 @@ describe("LiveStateManager", () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it("caps generic options snapshots at the 100-row hot head by default", async () => {
|
||||
const manager = new LiveStateManager(makeClickHouse(), null);
|
||||
const now = Date.now();
|
||||
|
||||
for (let seq = 1; seq <= 150; seq += 1) {
|
||||
await manager.ingest("options", {
|
||||
source_ts: now + seq,
|
||||
ingest_ts: now + seq,
|
||||
seq,
|
||||
trace_id: `opt-${seq}`,
|
||||
ts: now + seq,
|
||||
option_contract_id: "AAPL-2025-01-17-200-C",
|
||||
price: 1,
|
||||
size: 10,
|
||||
exchange: "X",
|
||||
signal_pass: true
|
||||
});
|
||||
}
|
||||
|
||||
const snapshot = await manager.getSnapshot({ channel: "options" });
|
||||
|
||||
expect(snapshot.items).toHaveLength(100);
|
||||
expect((snapshot.items as Array<{ trace_id: string }>)[0].trace_id).toBe("opt-150");
|
||||
expect(snapshot.next_before).toEqual({ ts: now + 51, seq: 51 });
|
||||
});
|
||||
|
||||
it("seeds scoped option snapshots from clickhouse rows older than 24h", async () => {
|
||||
const now = Date.now();
|
||||
const staleTs = now - 25 * 60 * 60 * 1000;
|
||||
|
|
@ -627,6 +654,57 @@ describe("LiveStateManager", () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it("prefers cached scoped option rows before clickhouse backfill", async () => {
|
||||
const now = Date.now();
|
||||
const manager = new LiveStateManager(
|
||||
makeClickHouse((query) =>
|
||||
query.includes("FROM option_prints")
|
||||
? [
|
||||
{
|
||||
source_ts: now - 1_000,
|
||||
ingest_ts: now - 999,
|
||||
seq: 1,
|
||||
trace_id: "opt-backfill",
|
||||
ts: now - 1_000,
|
||||
option_contract_id: "AAPL-2025-01-17-200-C",
|
||||
underlying_id: "AAPL",
|
||||
price: 1,
|
||||
size: 10,
|
||||
exchange: "X",
|
||||
signal_pass: false
|
||||
}
|
||||
]
|
||||
: []
|
||||
),
|
||||
null
|
||||
);
|
||||
|
||||
await manager.ingest("options", {
|
||||
source_ts: now,
|
||||
ingest_ts: now + 1,
|
||||
seq: 2,
|
||||
trace_id: "opt-hot",
|
||||
ts: now,
|
||||
option_contract_id: "AAPL-2025-01-17-200-C",
|
||||
underlying_id: "AAPL",
|
||||
price: 2,
|
||||
size: 10,
|
||||
exchange: "X",
|
||||
signal_pass: true
|
||||
});
|
||||
|
||||
const snapshot = await manager.getSnapshot({
|
||||
channel: "options",
|
||||
underlying_ids: ["AAPL"],
|
||||
option_contract_id: "AAPL-2025-01-17-200-C"
|
||||
});
|
||||
|
||||
expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id).slice(0, 2)).toEqual([
|
||||
"opt-hot",
|
||||
"opt-backfill"
|
||||
]);
|
||||
});
|
||||
|
||||
it("seeds scoped equity snapshots from clickhouse rows older than 24h", async () => {
|
||||
const now = Date.now();
|
||||
const staleTs = now - 25 * 60 * 60 * 1000;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue