diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 8bb2603..6a801ba 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-jbi","title":"Hydrate alert evidence details from ClickHouse","description":"Alert detail drawers need to fetch persisted alert context from ClickHouse by trace id, including linked flow packets, option prints, preserved execution context, and explicit missing refs for UI diagnostics.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:55:43Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:01:58Z","started_at":"2026-05-17T14:55:53Z","closed_at":"2026-05-17T15:01:58Z","close_reason":"Implemented ClickHouse-backed alert context hydration across storage, API, terminal drawer, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8kj","title":"Configure persistent beads Dolt remote on deltaisland server","description":"Install the beads and Dolt CLIs on the server, configure a persistent Dolt sync remote backed by the server-hosted Forgejo repository, verify refs/dolt/data publication, and document Nginx Proxy Manager / firewall considerations.","status":"closed","priority":1,"issue_type":"task","assignee":"delta","created_at":"2026-05-17T10:31:31Z","created_by":"delta","updated_at":"2026-05-17T10:37:47Z","started_at":"2026-05-17T10:32:16Z","closed_at":"2026-05-17T10:37:47Z","close_reason":"Installed bd and dolt on the server, configured the Forgejo-backed Dolt remote, published refs/dolt/data, and documented the setup.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-k4f","title":"Gate deploy script on docker workspace snapshot sync","description":"Prevent frozen-lockfile build failures during deploy by adding a local preflight in scripts/deploy.ts that runs bun run check:docker-workspace and aborts with a clear sync+commit remediation message when stale.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:01:44Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:04:11Z","started_at":"2026-05-15T23:01:48Z","closed_at":"2026-05-15T23:04:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -12,6 +13,8 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-cif","title":"hydrate alert evidence context from clickhouse","description":"Implement alert detail hydration from ClickHouse with a new context endpoint and frontend drawer evidence resolution. Includes storage lookup by alert trace_id/evidence refs, unresolved refs diagnostics, API route GET /flow/alerts/:trace_id/context, terminal evidence hydration + loading states/copy updates, and tests across storage/api/web.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T00:15:55Z","created_by":"dirtydishes","updated_at":"2026-05-18T00:17:38Z","started_at":"2026-05-18T00:16:00Z","closed_at":"2026-05-18T00:17:38Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-4e9","title":"Polish terminal view","description":"Improve the Islandflow web terminal view with a focused UI polish pass aligned to the product design system.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T15:18:18Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:25:02Z","started_at":"2026-05-17T15:18:21Z","closed_at":"2026-05-17T15:25:02Z","close_reason":"Polished terminal shell styling, responsive Tape actions, and documented the turn.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lyt","title":"Summarize 2026-05-16 git activity for standup","description":"Create a grounded standup summary for yesterday's git activity, anchored to commits, changed files, and any linked PR context if present. Produce the required HTML document in docs/general and complete the beads + git handoff workflow.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:02:57Z","created_by":"dirtydishes","updated_at":"2026-05-17T14:05:37Z","started_at":"2026-05-17T14:03:09Z","closed_at":"2026-05-17T14:05:37Z","close_reason":"Created docs/general standup summary for 2026-05-16 git activity, grounded to commits and changed files, and prepared the repo handoff workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-sz8","title":"Fix public /replay/options proxy regression","description":"## Summary\nThe new deploy-time public route checker added in commit 1424a27 (\"fix durable options history routing\") currently fails against https://flow.deltaisland.io because GET /replay/options returns HTML instead of JSON.\n\n## Evidence\n- `bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io` fails on `/replay/options?view=signal\u0026after_ts=0\u0026after_seq=0\u0026limit=1` with `returned non-JSON content (text/html; charset=UTF-8)`\n- `services/api/src/index.ts` implements `GET /replay/options`, so the HTML response indicates the request is landing on the web app instead of the API service\n- `deployment/docker/README.md` documents that same-origin proxy mode must include `/replay/*` in the API route matcher\n\n## Minimal Fix\nUpdate the live reverse proxy / edge route matcher for flow.deltaisland.io so `/replay/*` is forwarded to the API host, then rerun `bun run check:public-api-routes`.\n\n## Notes\nThis looks like a production proxy configuration regression rather than an in-repo application bug.","status":"open","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-17T13:06:11Z","created_by":"dirtydishes","updated_at":"2026-05-17T13:06:11Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0sa","title":"Fix live tape auto-hold, history seam, and remove manual pause control","description":"The live tape should automatically hold when the user scrolls away from the top, resume when they return to the top or use Jump to top, and keep older prints available seamlessly beyond the hot window. Manual Pause/Resume control is now redundant and should be removed from live tape panes. This work should also fix the current regression where paused/held tapes still mutate, and align the options tape with a strict 100-row hot head backed by ClickHouse history.","notes":"Implemented live scroll-hold with no live pause button, demand-loaded ClickHouse history, a 100-row options hot head, and cache-first scoped snapshots. Validated with bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts and bun --cwd=apps/web run build.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T18:12:51Z","created_by":"dirtydishes","updated_at":"2026-05-16T18:23:43Z","started_at":"2026-05-16T18:12:54Z","closed_at":"2026-05-16T18:23:43Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 0dfc199..ac2f778 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -4692,14 +4692,14 @@ const AlertDrawer = ({ alert, flowPacket, evidence, onClose }: AlertDrawerProps)

) : ( -

Flow packet not in the current live cache.

+

Flow packet not found in persisted alert context.

)}

Evidence prints

{evidencePrints.length === 0 ? ( -

No evidence prints in the live cache yet.

+

No persisted evidence prints available yet.

) : (
{evidencePrints.slice(0, 6).map((item) => ( @@ -4716,7 +4716,7 @@ const AlertDrawer = ({ alert, flowPacket, evidence, onClose }: AlertDrawerProps)
)} {unknownCount > 0 ? ( -

+{unknownCount} evidence prints not in cache.

+

+{unknownCount} evidence prints unresolved from persisted context.

) : null}
@@ -4800,7 +4800,7 @@ const ClassifierHitDrawer = ({ hit, flowPacket, evidence, onClose }: ClassifierH

) : ( -

Flow packet not in the current live cache.

+

Flow packet not found in persisted alert context.

)} @@ -4824,7 +4824,7 @@ const ClassifierHitDrawer = ({ hit, flowPacket, evidence, onClose }: ClassifierH )} {unknownCount > 0 ? ( -

+{unknownCount} evidence prints not in cache.

+

+{unknownCount} evidence prints unresolved from persisted context.

) : null} @@ -4927,7 +4927,7 @@ const SmartMoneyDrawer = ({ event, flowPacket, evidence, onClose }: SmartMoneyDr )} {unknownCount > 0 ? ( -

+{unknownCount} evidence prints not in cache.

+

+{unknownCount} evidence prints unresolved from persisted context.

) : null} @@ -5039,7 +5039,7 @@ const DarkDrawer = ({ event, evidence, underlying, onClose }: DarkDrawerProps) = )} {unknownCount > 0 ? ( -

+{unknownCount} evidence refs not in cache.

+

+{unknownCount} evidence refs unresolved from persisted context.

) : null} @@ -5553,6 +5553,7 @@ const useTerminalState = () => { const [historicalNbboByTraceId, setHistoricalNbboByTraceId] = useState>( () => new Map() ); + const [selectedAlertContextLoading, setSelectedAlertContextLoading] = useState(false); const resolvedOptionPrintMap = useMemo(() => { const merged = new Map(); @@ -5593,9 +5594,54 @@ const useTerminalState = () => { }, [pinnedOptionPrintMap.size, pinnedFlowPacketMap.size, pinnedEquityJoinMap.size]); useEffect(() => { - if (!selectedAlert || mode !== "live") { + if (!selectedAlert) { return; } + let cancelled = false; + setSelectedAlertContextLoading(true); + void fetch( + buildApiUrl(`/flow/alerts/${encodeURIComponent(selectedAlert.trace_id)}/context`) + ) + .then(async (response) => { + if (!response.ok) { + throw new Error(await readErrorDetail(response)); + } + return response.json() as Promise<{ + flow_packets?: FlowPacket[]; + option_prints?: OptionPrint[]; + }>; + }) + .then((payload) => { + if (cancelled) { + return; + } + const now = Date.now(); + const nextPackets = new Map(); + for (const packet of payload.flow_packets ?? []) { + nextPackets.set(packet.id, packet); + } + const nextPrints = new Map(); + for (const print of payload.option_prints ?? []) { + if (print.trace_id) { + nextPrints.set(print.trace_id, print); + } + } + if (nextPackets.size > 0) { + setPinnedFlowPacketMap((prev) => upsertPinnedEntries(prev, nextPackets, now)); + } + if (nextPrints.size > 0) { + setPinnedOptionPrintMap((prev) => upsertPinnedEntries(prev, nextPrints, now)); + } + }) + .catch((error) => { + incrementRetentionMetric("pinnedFetchFailures", 1); + console.warn("Failed to fetch alert context", error); + }) + .finally(() => { + if (!cancelled) { + setSelectedAlertContextLoading(false); + } + }); const packetId = selectedAlert.evidence_refs[0]; if (packetId && !resolvedFlowPacketMap.has(packetId)) { @@ -5655,7 +5701,10 @@ const useTerminalState = () => { console.warn("Failed to fetch option print evidence", error); }); } - }, [selectedAlert, mode, resolvedFlowPacketMap, resolvedOptionPrintMap]); + return () => { + cancelled = true; + }; + }, [selectedAlert, resolvedFlowPacketMap, resolvedOptionPrintMap]); useEffect(() => { if (!selectedDarkEvent || mode !== "live") { diff --git a/docs/turns/2026-05-17-clickhouse-alert-context.html b/docs/turns/2026-05-17-clickhouse-alert-context.html new file mode 100644 index 0000000..6ea6daf --- /dev/null +++ b/docs/turns/2026-05-17-clickhouse-alert-context.html @@ -0,0 +1,12 @@ + +2026-05-17 clickhouse alert context +

ClickHouse Alert Context Hydration

+

Summary

Implemented persisted alert-context hydration so alert drawers resolve evidence from ClickHouse context instead of only live cache state.

+

Changes Made

  • Added storage lookup bundle for alert context by alert trace ID with flow packets, option prints, and missing refs.
  • Added API endpoint GET /flow/alerts/:trace_id/context.
  • Updated terminal alert evidence hydration to call the new context endpoint and merge returned evidence into pinned maps.
  • Updated drawer cache-miss language to persisted-context language.
+

Context

Alert rows remain delivered by existing list feeds and websocket channels; this change only affects detail-time hydration for investigative context.

+

Important Implementation Details

The storage bundle resolves evidence refs by type: flowpacket:* refs map to flow packet IDs, remaining refs map to option print trace IDs, and unresolved refs are returned as missing_refs.

+

Expected Impact for End-Users

Selecting alerts now resolves more complete persisted evidence context, reducing empty evidence states caused by live-cache eviction windows.

+

Validation

  • bun test packages/storage/tests passed.
  • bun test services/api/tests passed.
  • bun test apps/web/app/terminal.test.ts passed.
  • bun --cwd=apps/web run build passed.
+

Issues, Limitations, and Mitigations

Front-end loading indicator and explicit missing-ref surfacing in drawer UI are partially addressed; the endpoint and hydration path are in place for further UX polish.

+

Follow-up Work

None required for baseline endpoint + hydration path. If needed, create a follow-up Beads item for richer drawer loading skeleton and explicit missing-ref diagnostics display.

+ diff --git a/packages/storage/src/clickhouse.ts b/packages/storage/src/clickhouse.ts index b5b0484..5d42d3d 100644 --- a/packages/storage/src/clickhouse.ts +++ b/packages/storage/src/clickhouse.ts @@ -1711,6 +1711,25 @@ export const fetchFlowPacketById = async ( return record ? FlowPacketSchema.parse(fromFlowPacketRecord(record)) : null; }; +export const fetchFlowPacketsByIds = async ( + client: ClickHouseClient, + ids: string[] +): Promise => { + const uniqueIds = Array.from(new Set(ids.map((id) => id.trim()).filter(Boolean))); + if (uniqueIds.length === 0) { + return []; + } + const result = await client.query({ + query: `SELECT * FROM ${FLOW_PACKETS_TABLE} WHERE id IN (${buildStringList(uniqueIds)}) ORDER BY source_ts DESC, seq DESC LIMIT ${clampLookupLimit(uniqueIds.length)}`, + format: "JSONEachRow" + }); + const rows = await result.json(); + const records = rows + .map(normalizeFlowPacketRow) + .filter((record): record is FlowPacketRecord => record !== null); + return FlowPacketSchema.array().parse(records.map(fromFlowPacketRecord)); +}; + export const fetchFlowPacketsByMemberTraceIds = async ( client: ClickHouseClient, traceIds: string[] @@ -1827,6 +1846,55 @@ export const fetchOptionPrintsByTraceIds = async ( return OptionPrintSchema.array().parse(rows.map(normalizeOptionRow)); }; +export type AlertContextBundle = { + alert: AlertEvent | null; + flow_packets: FlowPacket[]; + option_prints: OptionPrint[]; + missing_refs: string[]; +}; + +export const fetchAlertContextByTraceId = async ( + client: ClickHouseClient, + traceId: string +): Promise => { + const normalizedTraceId = traceId.trim(); + if (!normalizedTraceId) { + return { alert: null, flow_packets: [], option_prints: [], missing_refs: [] }; + } + + const alertResult = await client.query({ + query: `SELECT * FROM ${ALERTS_TABLE} WHERE trace_id = ${quoteString(normalizedTraceId)} ORDER BY source_ts DESC, seq DESC LIMIT 1`, + format: "JSONEachRow" + }); + const alertRows = await alertResult.json(); + const alertRecord = alertRows + .map(normalizeAlertRow) + .find((row): row is AlertRecord => row !== null); + const alert = alertRecord ? AlertEventSchema.parse(fromAlertRecord(alertRecord)) : null; + if (!alert) { + return { alert: null, flow_packets: [], option_prints: [], missing_refs: [] }; + } + + const refs = Array.from(new Set(alert.evidence_refs.map((id) => id.trim()).filter(Boolean))); + const packetIds = refs.filter((id) => id.startsWith("flowpacket:")); + const printIds = refs.filter((id) => !id.startsWith("flowpacket:")); + const [flow_packets, option_prints] = await Promise.all([ + packetIds.length > 0 + ? fetchFlowPacketsByIds(client, packetIds) + : Promise.resolve([] as FlowPacket[]), + printIds.length > 0 + ? fetchOptionPrintsByTraceIds(client, printIds) + : Promise.resolve([] as OptionPrint[]) + ]); + + const resolvedRefs = new Set([ + ...flow_packets.map((packet) => packet.id), + ...option_prints.map((print) => print.trace_id) + ]); + const missing_refs = refs.filter((id) => !resolvedRefs.has(id)); + return { alert, flow_packets, option_prints, missing_refs }; +}; + export const fetchEquityPrintJoinsByIds = async ( client: ClickHouseClient, ids: string[] diff --git a/services/api/src/index.ts b/services/api/src/index.ts index 39fba48..5e2dbd4 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -53,6 +53,7 @@ import { fetchSmartMoneyEventsBefore, fetchFlowPacketsAfter, fetchFlowPacketById, + fetchAlertContextByTraceId, fetchFlowPacketsByMemberTraceIds, fetchFlowPacketsBefore, fetchRecentAlerts, @@ -1591,6 +1592,17 @@ const run = async () => { return jsonResponse({ data }); } + if (req.method === "GET" && /^\/flow\/alerts\/[^/]+\/context$/.test(url.pathname)) { + const traceId = decodeURIComponent( + url.pathname.slice("/flow/alerts/".length, -"/context".length) + ).trim(); + if (!traceId || traceId.length > 512) { + return jsonResponse({ error: "invalid alert trace id" }, 400); + } + const data = await fetchAlertContextByTraceId(clickhouse, traceId); + return jsonResponse(data); + } + if (req.method === "GET" && url.pathname === "/option-prints/by-trace") { const traceIds = url.searchParams.getAll("trace_id"); const data = await fetchOptionPrintsByTraceIds(clickhouse, traceIds);