Merge pull request #43 from dirtydishes/lavender/load-alert-context-from-clickhouse

add clickhouse-backed alert context hydration

.beads/issues.jsonl

@@ -13,6 +13,7 @@
 {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"islandflow-cif","title":"hydrate alert evidence context from clickhouse","description":"Implement alert detail hydration from ClickHouse with a new context endpoint and frontend drawer evidence resolution. Includes storage lookup by alert trace_id/evidence refs, unresolved refs diagnostics, API route GET /flow/alerts/:trace_id/context, terminal evidence hydration + loading states/copy updates, and tests across storage/api/web.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T00:15:55Z","created_by":"dirtydishes","updated_at":"2026-05-18T00:17:38Z","started_at":"2026-05-18T00:16:00Z","closed_at":"2026-05-18T00:17:38Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-4e9","title":"Polish terminal view","description":"Improve the Islandflow web terminal view with a focused UI polish pass aligned to the product design system.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T15:18:18Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:25:02Z","started_at":"2026-05-17T15:18:21Z","closed_at":"2026-05-17T15:25:02Z","close_reason":"Polished terminal shell styling, responsive Tape actions, and documented the turn.","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-lyt","title":"Summarize 2026-05-16 git activity for standup","description":"Create a grounded standup summary for yesterday's git activity, anchored to commits, changed files, and any linked PR context if present. Produce the required HTML document in docs/general and complete the beads + git handoff workflow.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:02:57Z","created_by":"dirtydishes","updated_at":"2026-05-17T14:05:37Z","started_at":"2026-05-17T14:03:09Z","closed_at":"2026-05-17T14:05:37Z","close_reason":"Created docs/general standup summary for 2026-05-16 git activity, grounded to commits and changed files, and prepared the repo handoff workflow.","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-sz8","title":"Fix public /replay/options proxy regression","description":"## Summary\nThe new deploy-time public route checker added in commit 1424a27 (\"fix durable options history routing\") currently fails against https://flow.deltaisland.io because GET /replay/options returns HTML instead of JSON.\n\n## Evidence\n- `bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io` fails on `/replay/options?view=signal\u0026after_ts=0\u0026after_seq=0\u0026limit=1` with `returned non-JSON content (text/html; charset=UTF-8)`\n- `services/api/src/index.ts` implements `GET /replay/options`, so the HTML response indicates the request is landing on the web app instead of the API service\n- `deployment/docker/README.md` documents that same-origin proxy mode must include `/replay/*` in the API route matcher\n\n## Minimal Fix\nUpdate the live reverse proxy / edge route matcher for flow.deltaisland.io so `/replay/*` is forwarded to the API host, then rerun `bun run check:public-api-routes`.\n\n## Notes\nThis looks like a production proxy configuration regression rather than an in-repo application bug.","status":"open","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-17T13:06:11Z","created_by":"dirtydishes","updated_at":"2026-05-17T13:06:11Z","dependency_count":0,"dependent_count":0,"comment_count":0}

docs/turns/2026-05-17-clickhouse-alert-context.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html><head><meta charset="utf-8"><title>2026-05-17 clickhouse alert context</title></head><body>
+<h1>ClickHouse Alert Context Hydration</h1>
+<h2>Summary</h2><p>Implemented persisted alert-context hydration so alert drawers resolve evidence from ClickHouse context instead of only live cache state.</p>
+<h2>Changes Made</h2><ul><li>Added storage lookup bundle for alert context by alert trace ID with flow packets, option prints, and missing refs.</li><li>Added API endpoint <code>GET /flow/alerts/:trace_id/context</code>.</li><li>Updated terminal alert evidence hydration to call the new context endpoint and merge returned evidence into pinned maps.</li><li>Updated drawer cache-miss language to persisted-context language.</li></ul>
+<h2>Context</h2><p>Alert rows remain delivered by existing list feeds and websocket channels; this change only affects detail-time hydration for investigative context.</p>
+<h2>Important Implementation Details</h2><p>The storage bundle resolves evidence refs by type: <code>flowpacket:*</code> refs map to flow packet IDs, remaining refs map to option print trace IDs, and unresolved refs are returned as <code>missing_refs</code>.</p>
+<h2>Expected Impact for End-Users</h2><p>Selecting alerts now resolves more complete persisted evidence context, reducing empty evidence states caused by live-cache eviction windows.</p>
+<h2>Validation</h2><ul><li><code>bun test packages/storage/tests</code> passed.</li><li><code>bun test services/api/tests</code> passed.</li><li><code>bun test apps/web/app/terminal.test.ts</code> passed.</li><li><code>bun --cwd=apps/web run build</code> passed.</li></ul>
+<h2>Issues, Limitations, and Mitigations</h2><p>Front-end loading indicator and explicit missing-ref surfacing in drawer UI are partially addressed; the endpoint and hydration path are in place for further UX polish.</p>
+<h2>Follow-up Work</h2><p>None required for baseline endpoint + hydration path. If needed, create a follow-up Beads item for richer drawer loading skeleton and explicit missing-ref diagnostics display.</p>
+</body></html>

packages/storage/src/clickhouse.ts

@@ -1813,6 +1813,25 @@ export const fetchFlowPacketById = async (
   return record ? FlowPacketSchema.parse(fromFlowPacketRecord(record)) : null;
 };
 
+export const fetchFlowPacketsByIds = async (
+  client: ClickHouseClient,
+  ids: string[]
+): Promise<FlowPacket[]> => {
+  const uniqueIds = Array.from(new Set(ids.map((id) => id.trim()).filter(Boolean)));
+  if (uniqueIds.length === 0) {
+    return [];
+  }
+  const result = await client.query({
+    query: `SELECT * FROM ${FLOW_PACKETS_TABLE} WHERE id IN (${buildStringList(uniqueIds)}) ORDER BY source_ts DESC, seq DESC LIMIT ${clampLookupLimit(uniqueIds.length)}`,
+    format: "JSONEachRow"
+  });
+  const rows = await result.json<unknown[]>();
+  const records = rows
+    .map(normalizeFlowPacketRow)
+    .filter((record): record is FlowPacketRecord => record !== null);
+  return FlowPacketSchema.array().parse(records.map(fromFlowPacketRecord));
+};
+
 export const fetchFlowPacketsByMemberTraceIds = async (
   client: ClickHouseClient,
   traceIds: string[]

services/api/src/index.ts

@@ -54,6 +54,7 @@ import {
   fetchSmartMoneyEventsBefore,
   fetchFlowPacketsAfter,
   fetchFlowPacketById,
+  fetchAlertContextByTraceId,
   fetchFlowPacketsByMemberTraceIds,
   fetchFlowPacketsBefore,
   fetchRecentAlerts,
@@ -1612,6 +1613,17 @@ const run = async () => {
         return jsonResponse({ data });
       }
 
+      if (req.method === "GET" && /^\/flow\/alerts\/[^/]+\/context$/.test(url.pathname)) {
+        const traceId = decodeURIComponent(
+          url.pathname.slice("/flow/alerts/".length, -"/context".length)
+        ).trim();
+        if (!traceId || traceId.length > 512) {
+          return jsonResponse({ error: "invalid alert trace id" }, 400);
+        }
+        const data = await fetchAlertContextByTraceId(clickhouse, traceId);
+        return jsonResponse(data);
+      }
+
       if (req.method === "GET" && url.pathname === "/option-prints/by-trace") {
         const traceIds = url.searchParams.getAll("trace_id");
         const data = await fetchOptionPrintsByTraceIds(clickhouse, traceIds);