Merge pull request #41 from dirtydishes/codex/load-alert-context-from-clickhouse

hydrate alert evidence details from clickhouse
This commit is contained in:
dirtydishes 2026-05-17 11:38:01 -04:00 committed by GitHub
commit 3e089554f8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 702 additions and 62 deletions

View file

@ -1,3 +1,4 @@
{"_type":"issue","id":"islandflow-jbi","title":"Hydrate alert evidence details from ClickHouse","description":"Alert detail drawers need to fetch persisted alert context from ClickHouse by trace id, including linked flow packets, option prints, preserved execution context, and explicit missing refs for UI diagnostics.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:55:43Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:01:58Z","started_at":"2026-05-17T14:55:53Z","closed_at":"2026-05-17T15:01:58Z","close_reason":"Implemented ClickHouse-backed alert context hydration across storage, API, terminal drawer, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-8kj","title":"Configure persistent beads Dolt remote on deltaisland server","description":"Install the beads and Dolt CLIs on the server, configure a persistent Dolt sync remote backed by the server-hosted Forgejo repository, verify refs/dolt/data publication, and document Nginx Proxy Manager / firewall considerations.","status":"closed","priority":1,"issue_type":"task","assignee":"delta","created_at":"2026-05-17T10:31:31Z","created_by":"delta","updated_at":"2026-05-17T10:37:47Z","started_at":"2026-05-17T10:32:16Z","closed_at":"2026-05-17T10:37:47Z","close_reason":"Installed bd and dolt on the server, configured the Forgejo-backed Dolt remote, published refs/dolt/data, and documented the setup.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-k4f","title":"Gate deploy script on docker workspace snapshot sync","description":"Prevent frozen-lockfile build failures during deploy by adding a local preflight in scripts/deploy.ts that runs bun run check:docker-workspace and aborts with a clear sync+commit remediation message when stale.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:01:44Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:04:11Z","started_at":"2026-05-15T23:01:48Z","closed_at":"2026-05-15T23:04:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
@ -12,6 +13,7 @@
{"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-4e9","title":"Polish terminal view","description":"Improve the Islandflow web terminal view with a focused UI polish pass aligned to the product design system.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T15:18:18Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:25:02Z","started_at":"2026-05-17T15:18:21Z","closed_at":"2026-05-17T15:25:02Z","close_reason":"Polished terminal shell styling, responsive Tape actions, and documented the turn.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-lyt","title":"Summarize 2026-05-16 git activity for standup","description":"Create a grounded standup summary for yesterday's git activity, anchored to commits, changed files, and any linked PR context if present. Produce the required HTML document in docs/general and complete the beads + git handoff workflow.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:02:57Z","created_by":"dirtydishes","updated_at":"2026-05-17T14:05:37Z","started_at":"2026-05-17T14:03:09Z","closed_at":"2026-05-17T14:05:37Z","close_reason":"Created docs/general standup summary for 2026-05-16 git activity, grounded to commits and changed files, and prepared the repo handoff workflow.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-sz8","title":"Fix public /replay/options proxy regression","description":"## Summary\nThe new deploy-time public route checker added in commit 1424a27 (\"fix durable options history routing\") currently fails against https://flow.deltaisland.io because GET /replay/options returns HTML instead of JSON.\n\n## Evidence\n- `bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io` fails on `/replay/options?view=signal\u0026after_ts=0\u0026after_seq=0\u0026limit=1` with `returned non-JSON content (text/html; charset=UTF-8)`\n- `services/api/src/index.ts` implements `GET /replay/options`, so the HTML response indicates the request is landing on the web app instead of the API service\n- `deployment/docker/README.md` documents that same-origin proxy mode must include `/replay/*` in the API route matcher\n\n## Minimal Fix\nUpdate the live reverse proxy / edge route matcher for flow.deltaisland.io so `/replay/*` is forwarded to the API host, then rerun `bun run check:public-api-routes`.\n\n## Notes\nThis looks like a production proxy configuration regression rather than an in-repo application bug.","status":"open","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-17T13:06:11Z","created_by":"dirtydishes","updated_at":"2026-05-17T13:06:11Z","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-0sa","title":"Fix live tape auto-hold, history seam, and remove manual pause control","description":"The live tape should automatically hold when the user scrolls away from the top, resume when they return to the top or use Jump to top, and keep older prints available seamlessly beyond the hot window. Manual Pause/Resume control is now redundant and should be removed from live tape panes. This work should also fix the current regression where paused/held tapes still mutate, and align the options tape with a strict 100-row hot head backed by ClickHouse history.","notes":"Implemented live scroll-hold with no live pause button, demand-loaded ClickHouse history, a 100-row options hot head, and cache-first scoped snapshots. Validated with bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts and bun --cwd=apps/web run build.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T18:12:51Z","created_by":"dirtydishes","updated_at":"2026-05-16T18:23:43Z","started_at":"2026-05-16T18:12:54Z","closed_at":"2026-05-16T18:23:43Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}

View file

@ -1818,6 +1818,28 @@ h3 {
gap: 10px;
}
.drawer-context-loading {
padding: 12px 0 2px;
}
.drawer-skeleton {
width: 64%;
height: 12px;
border-radius: 999px;
background: linear-gradient(90deg, var(--bg-soft), rgba(245, 166, 35, 0.14), var(--bg-soft));
background-size: 180% 100%;
animation: drawer-skeleton 1.2s ease-out infinite;
}
.drawer-skeleton-wide {
width: 100%;
}
.drawer-evidence-context {
margin-top: 8px;
color: var(--text-faint);
}
.drawer-row {
padding: 12px 14px;
border-radius: 12px;
@ -1825,6 +1847,15 @@ h3 {
background: var(--bg-soft);
}
@keyframes drawer-skeleton {
0% {
background-position: 100% 0;
}
100% {
background-position: -100% 0;
}
}
@keyframes pulse {
0% {
transform: scale(1);

View file

@ -3,9 +3,11 @@ import { getSubscriptionKey as getLiveSubscriptionKey } from "@islandflow/types"
import {
NAV_ITEMS,
appendHistoryTail,
buildAlertContextPath,
buildDefaultFlowFilters,
buildOptionTapeQueryParams,
classifierToneForFamily,
collectAlertContextEvidence,
composeTapeItems,
deriveAlertDirection,
countActiveFlowFilterGroups,
@ -95,6 +97,44 @@ describe("pinned evidence pruning", () => {
});
});
describe("alert context hydration helpers", () => {
it("builds the persisted ClickHouse context endpoint path", () => {
expect(buildAlertContextPath("alert:large_call/one")).toBe(
"/flow/alerts/alert%3Alarge_call%2Fone/context"
);
});
it("merges hydrated packets and prints into pinned evidence maps", () => {
const packet = {
trace_id: "flowpacket:1",
id: "flowpacket:1",
members: ["print:1"],
source_ts: 1,
ingest_ts: 2,
seq: 1,
features: {},
join_quality: {}
} as any;
const print = makeOptionPrint({
trace_id: "print:1",
execution_nbbo_bid: 1.2,
execution_nbbo_ask: 1.3,
execution_underlying_spot: 450.05
});
const evidence = collectAlertContextEvidence({
alert: makeAlert({ evidence_refs: ["flowpacket:1", "print:1"] }),
flow_packets: [packet],
option_prints: [print],
missing_refs: []
});
expect(evidence.packets.get("flowpacket:1")).toBe(packet);
expect(evidence.prints.get("print:1")?.execution_nbbo_bid).toBe(1.2);
expect(evidence.prints.get("print:1")?.execution_underlying_spot).toBe(450.05);
});
});
describe("live manifest", () => {
it("includes only tape channels on /tape", () => {
const filters = buildDefaultFlowFilters();

View file

@ -4604,6 +4604,49 @@ type EvidenceItem =
| { kind: "print"; id: string; print: OptionPrint }
| { kind: "unknown"; id: string };
type AlertContextBundle = {
alert: AlertEvent | null;
flow_packets: FlowPacket[];
option_prints: OptionPrint[];
missing_refs: string[];
};
type AlertContextStatus = {
traceId: string | null;
loading: boolean;
missingRefs: string[];
error: string | null;
};
export const buildAlertContextPath = (traceId: string): string =>
`/flow/alerts/${encodeURIComponent(traceId)}/context`;
export const collectAlertContextEvidence = (
bundle: AlertContextBundle
): {
packets: Map<string, FlowPacket>;
prints: Map<string, OptionPrint>;
} => {
const packets = new Map<string, FlowPacket>();
const prints = new Map<string, OptionPrint>();
for (const packet of bundle.flow_packets) {
if (packet.id) {
packets.set(packet.id, packet);
}
if (packet.trace_id) {
packets.set(packet.trace_id, packet);
}
}
for (const print of bundle.option_prints) {
if (print.trace_id) {
prints.set(print.trace_id, print);
}
}
return { packets, prints };
};
type DarkEvidenceItem =
| { kind: "join"; id: string; join: EquityPrintJoin }
| { kind: "unknown"; id: string };
@ -4612,15 +4655,28 @@ type AlertDrawerProps = {
alert: AlertEvent;
flowPacket: FlowPacket | null;
evidence: EvidenceItem[];
contextStatus: AlertContextStatus;
onClose: () => void;
};
const AlertDrawer = ({ alert, flowPacket, evidence, onClose }: AlertDrawerProps) => {
const formatOptionalMoney = (value: unknown): string | null => {
const parsed = parseNumber(value, Number.NaN);
return Number.isFinite(parsed) ? `$${formatPrice(parsed)}` : null;
};
const formatOptionalMs = (value: unknown): string | null => {
const parsed = parseNumber(value, Number.NaN);
return Number.isFinite(parsed) ? `${Math.round(parsed)}ms` : null;
};
const AlertDrawer = ({ alert, flowPacket, evidence, contextStatus, onClose }: AlertDrawerProps) => {
const primary = alert.hits[0];
const direction = deriveAlertDirection(alert);
const severity = normalizeAlertSeverity(alert);
const evidencePrints = evidence.filter((item) => item.kind === "print");
const unknownCount = evidence.filter((item) => item.kind === "unknown").length;
const isContextLoading = contextStatus.traceId === alert.trace_id && contextStatus.loading;
const missingRefs = contextStatus.traceId === alert.trace_id ? contextStatus.missingRefs : [];
return (
<aside className="drawer">
@ -4639,7 +4695,17 @@ const AlertDrawer = ({ alert, flowPacket, evidence, onClose }: AlertDrawerProps)
<span className={`pill severity-${severity}`}>{severity}</span>
<span className="drawer-chip">Score {Math.round(alert.score)}</span>
<span className={`pill direction-${direction}`}>{direction}</span>
{isContextLoading ? <span className="drawer-chip">Loading context</span> : null}
</div>
{isContextLoading ? (
<div className="drawer-section drawer-context-loading" aria-label="Loading persisted evidence">
<div className="drawer-skeleton drawer-skeleton-wide" />
<div className="drawer-skeleton" />
</div>
) : null}
{contextStatus.traceId === alert.trace_id && contextStatus.error ? (
<p className="drawer-empty">Persisted context could not be loaded: {contextStatus.error}</p>
) : null}
<div className="drawer-section">
<h4>Classifier hits</h4>
@ -4692,14 +4758,14 @@ const AlertDrawer = ({ alert, flowPacket, evidence, onClose }: AlertDrawerProps)
</p>
</div>
) : (
<p className="drawer-empty">Flow packet not in the current live cache.</p>
<p className="drawer-empty">Persisted flow packet is not available for this alert.</p>
)}
</div>
<div className="drawer-section">
<h4>Evidence prints</h4>
{evidencePrints.length === 0 ? (
<p className="drawer-empty">No evidence prints in the live cache yet.</p>
<p className="drawer-empty">Persisted evidence prints are not available for this alert.</p>
) : (
<div className="drawer-list">
{evidencePrints.slice(0, 6).map((item) => (
@ -4709,6 +4775,36 @@ const AlertDrawer = ({ alert, flowPacket, evidence, onClose }: AlertDrawerProps)
<span>${formatPrice(item.print.price)}</span>
<span>{formatSize(item.print.size)}x</span>
<span>{item.print.exchange}</span>
{item.print.execution_nbbo_side ? <span>Side {item.print.execution_nbbo_side}</span> : null}
{formatOptionalMs(item.print.execution_nbbo_age_ms) ? (
<span>Quote {formatOptionalMs(item.print.execution_nbbo_age_ms)}</span>
) : null}
</div>
<div className="drawer-row-meta drawer-evidence-context">
{formatOptionalMoney(item.print.execution_nbbo_bid) ? (
<span>Bid {formatOptionalMoney(item.print.execution_nbbo_bid)}</span>
) : null}
{formatOptionalMoney(item.print.execution_nbbo_ask) ? (
<span>Ask {formatOptionalMoney(item.print.execution_nbbo_ask)}</span>
) : null}
{formatOptionalMoney(item.print.execution_nbbo_mid) ? (
<span>Mid {formatOptionalMoney(item.print.execution_nbbo_mid)}</span>
) : null}
{formatOptionalMoney(item.print.execution_nbbo_spread) ? (
<span>Spr {formatOptionalMoney(item.print.execution_nbbo_spread)}</span>
) : null}
{formatOptionalMoney(item.print.execution_underlying_spot) ? (
<span>Spot {formatOptionalMoney(item.print.execution_underlying_spot)}</span>
) : null}
{formatOptionalMoney(item.print.execution_underlying_bid) ? (
<span>U Bid {formatOptionalMoney(item.print.execution_underlying_bid)}</span>
) : null}
{formatOptionalMoney(item.print.execution_underlying_ask) ? (
<span>U Ask {formatOptionalMoney(item.print.execution_underlying_ask)}</span>
) : null}
{formatOptionalMoney(item.print.execution_underlying_mid) ? (
<span>U Mid {formatOptionalMoney(item.print.execution_underlying_mid)}</span>
) : null}
</div>
<p className="drawer-note">{formatTime(item.print.ts)}</p>
</div>
@ -4716,7 +4812,10 @@ const AlertDrawer = ({ alert, flowPacket, evidence, onClose }: AlertDrawerProps)
</div>
)}
{unknownCount > 0 ? (
<p className="drawer-empty">+{unknownCount} evidence prints not in cache.</p>
<p className="drawer-empty">+{unknownCount} evidence refs unresolved in persisted context.</p>
) : null}
{missingRefs.length > 0 ? (
<p className="drawer-empty">Missing refs: {missingRefs.slice(0, 4).join(", ")}</p>
) : null}
</div>
</aside>
@ -5548,6 +5647,12 @@ const useTerminalState = () => {
const [pinnedEquityJoinMap, setPinnedEquityJoinMap] = useState<
Map<string, PinnedEntry<EquityPrintJoin>>
>(() => new Map());
const [selectedAlertContextStatus, setSelectedAlertContextStatus] = useState<AlertContextStatus>({
traceId: null,
loading: false,
missingRefs: [],
error: null
});
const [optionSupportSmartMoney, setOptionSupportSmartMoney] = useState<SmartMoneyEvent[]>([]);
const [optionSupportClassifierHits, setOptionSupportClassifierHits] = useState<ClassifierHitEvent[]>([]);
const [historicalNbboByTraceId, setHistoricalNbboByTraceId] = useState<Map<string, OptionNBBO | null>>(
@ -5593,69 +5698,67 @@ const useTerminalState = () => {
}, [pinnedOptionPrintMap.size, pinnedFlowPacketMap.size, pinnedEquityJoinMap.size]);
useEffect(() => {
if (!selectedAlert || mode !== "live") {
if (!selectedAlert) {
setSelectedAlertContextStatus({
traceId: null,
loading: false,
missingRefs: [],
error: null
});
return;
}
const packetId = selectedAlert.evidence_refs[0];
if (packetId && !resolvedFlowPacketMap.has(packetId)) {
incrementRetentionMetric("pinnedFetchMisses", 1);
void fetch(buildApiUrl(`/flow/packets/${encodeURIComponent(packetId)}`))
const abort = new AbortController();
setSelectedAlertContextStatus({
traceId: selectedAlert.trace_id,
loading: true,
missingRefs: [],
error: null
});
incrementRetentionMetric("pinnedFetchMisses", selectedAlert.evidence_refs.length);
void fetch(buildApiUrl(buildAlertContextPath(selectedAlert.trace_id)), { signal: abort.signal })
.then(async (response) => {
if (!response.ok) {
throw new Error(await readErrorDetail(response));
}
return response.json();
})
.then((payload: { data?: FlowPacket | null }) => {
if (!payload.data) {
.then((payload: AlertContextBundle) => {
if (abort.signal.aborted) {
return;
}
const { packets, prints } = collectAlertContextEvidence(payload);
const now = Date.now();
const next = new Map<string, FlowPacket>([[payload.data.id, payload.data]]);
setPinnedFlowPacketMap((prev) => upsertPinnedEntries(prev, next, now));
if (packets.size > 0) {
setPinnedFlowPacketMap((prev) => upsertPinnedEntries(prev, packets, now));
}
if (prints.size > 0) {
setPinnedOptionPrintMap((prev) => upsertPinnedEntries(prev, prints, now));
}
setSelectedAlertContextStatus({
traceId: selectedAlert.trace_id,
loading: false,
missingRefs: payload.missing_refs ?? [],
error: null
});
})
.catch((error) => {
incrementRetentionMetric("pinnedFetchFailures", 1);
console.warn("Failed to fetch flow packet evidence", error);
});
if (abort.signal.aborted) {
return;
}
incrementRetentionMetric("pinnedFetchFailures", 1);
console.warn("Failed to fetch persisted alert context", error);
setSelectedAlertContextStatus({
traceId: selectedAlert.trace_id,
loading: false,
missingRefs: [],
error: error instanceof Error ? error.message : String(error)
});
});
const missingPrintIds = selectedAlert.evidence_refs.filter(
(id) => !resolvedFlowPacketMap.has(id) && !resolvedOptionPrintMap.has(id)
);
if (missingPrintIds.length > 0) {
incrementRetentionMetric("pinnedFetchMisses", missingPrintIds.length);
const url = new URL(buildApiUrl("/option-prints/by-trace"));
for (const traceId of missingPrintIds) {
url.searchParams.append("trace_id", traceId);
}
void fetch(url.toString())
.then(async (response) => {
if (!response.ok) {
throw new Error(await readErrorDetail(response));
}
return response.json();
})
.then((payload: { data?: OptionPrint[] }) => {
const next = new Map<string, OptionPrint>();
for (const item of payload.data ?? []) {
if (!item || !item.trace_id) {
continue;
}
next.set(item.trace_id, item);
}
if (next.size > 0) {
const now = Date.now();
setPinnedOptionPrintMap((prev) => upsertPinnedEntries(prev, next, now));
}
})
.catch((error) => {
incrementRetentionMetric("pinnedFetchFailures", 1);
console.warn("Failed to fetch option print evidence", error);
});
}
}, [selectedAlert, mode, resolvedFlowPacketMap, resolvedOptionPrintMap]);
return () => abort.abort();
}, [selectedAlert]);
useEffect(() => {
if (!selectedDarkEvent || mode !== "live") {
@ -6802,6 +6905,7 @@ const useTerminalState = () => {
packetIdByOptionTraceId,
classifierDecorByOptionTraceId,
selectedEvidence,
selectedAlertContextStatus,
selectedFlowPacket,
selectedDarkEvidence,
selectedDarkUnderlying,
@ -8515,6 +8619,7 @@ export function TerminalAppShell({ children }: { children: ReactNode }) {
alert={state.selectedAlert}
flowPacket={state.selectedFlowPacket}
evidence={state.selectedEvidence}
contextStatus={state.selectedAlertContextStatus}
onClose={() => state.setSelectedAlert(null)}
/>
) : null}

View file

@ -0,0 +1,194 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>ClickHouse Alert Context Hydration</title>
<style>
:root {
color-scheme: dark;
--bg: #06080b;
--panel: #111820;
--panel-soft: #0d141b;
--border: rgba(255, 255, 255, 0.12);
--text: #e6edf4;
--dim: #90a0b2;
--faint: #6e7b8c;
--accent: #f5a623;
--accent-soft: rgba(245, 166, 35, 0.14);
--ok: #25c17a;
}
body {
margin: 0;
background: var(--bg);
color: var(--text);
font: 15px/1.55 -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
}
main {
max-width: 980px;
margin: 0 auto;
padding: 42px 22px 64px;
}
header {
padding-bottom: 22px;
border-bottom: 1px solid var(--border);
}
h1,
h2 {
margin: 0;
line-height: 1.15;
}
h1 {
font-size: 1.8rem;
letter-spacing: 0.01em;
}
h2 {
margin-top: 30px;
font-size: 0.9rem;
color: var(--accent);
text-transform: uppercase;
letter-spacing: 0.12em;
}
p {
max-width: 74ch;
color: var(--dim);
}
ul {
display: grid;
gap: 8px;
padding-left: 20px;
color: var(--dim);
}
code {
padding: 2px 5px;
border: 1px solid var(--border);
border-radius: 6px;
background: var(--panel-soft);
color: var(--text);
font-family: "SFMono-Regular", Consolas, monospace;
font-size: 0.9em;
}
pre {
overflow: auto;
padding: 14px;
border: 1px solid var(--border);
border-radius: 10px;
background: var(--panel);
}
.summary {
margin-top: 16px;
padding: 16px 18px;
border: 1px solid rgba(245, 166, 35, 0.28);
border-radius: 12px;
background: var(--accent-soft);
}
.status {
display: inline-flex;
margin-top: 12px;
padding: 4px 9px;
border: 1px solid rgba(37, 193, 122, 0.34);
border-radius: 999px;
color: var(--ok);
font-size: 0.72rem;
font-weight: 700;
letter-spacing: 0.12em;
text-transform: uppercase;
}
</style>
</head>
<body>
<main>
<header>
<h1>ClickHouse Alert Context Hydration</h1>
<p class="summary">
Alert detail drawers now fetch persisted investigative context from ClickHouse by alert trace id, then merge linked flow packets and option prints into the existing pinned evidence maps.
</p>
<span class="status">Validated</span>
</header>
<section>
<h2>Summary</h2>
<p>
This change makes alert details durable. Selecting an alert no longer depends only on the live cache to resolve evidence; the terminal asks the API for a ClickHouse-backed alert context bundle and uses that bundle to populate the existing drawer, classifier support, smart-money support, and prefetch evidence stores.
</p>
</section>
<section>
<h2>Changes Made</h2>
<ul>
<li>Added <code>fetchAlertContextByTraceId</code> in storage to load an alert, linked flow packets, linked option prints, and unresolved evidence refs.</li>
<li>Added <code>GET /flow/alerts/:trace_id/context</code> to the API without changing existing alert list, history, replay, or websocket feeds.</li>
<li>Updated the terminal alert selection effect to fetch persisted context in live, replay, and history modes.</li>
<li>Merged hydrated packets and prints into pinned maps so existing evidence consumers share the resolved context.</li>
<li>Adjusted alert drawer copy and loading state to reference persisted context rather than live cache misses.</li>
<li>Expanded alert evidence print rows with execution NBBO side, bid, ask, mid, spread, quote age, underlying spot, bid, ask, and mid where available.</li>
</ul>
</section>
<section>
<h2>Context</h2>
<p>
Alert rows intentionally remain lightweight for live bursts. The detail drawer is the right place to hydrate heavier investigative context because it runs only when a user asks for a specific alert. The authoritative linkage remains <code>AlertEvent.evidence_refs</code>.
</p>
</section>
<section>
<h2>Important Implementation Details</h2>
<p>The new API response shape is:</p>
<pre><code>{
alert: AlertEvent | null,
flow_packets: FlowPacket[],
option_prints: OptionPrint[],
missing_refs: string[]
}</code></pre>
<p>
Flow packet refs are resolved with both prefixed and unprefixed candidates. Option print refs are resolved by <code>trace_id</code>. Missing refs are returned explicitly instead of failing the whole response.
</p>
</section>
<section>
<h2>Expected Impact for End-Users</h2>
<p>
Alert details should feel more trustworthy after cache churn or replay navigation. Users can select an older or non-hot alert and still see the preserved evidence context needed to evaluate the signal.
</p>
</section>
<section>
<h2>Validation</h2>
<ul>
<li><code>bun test packages/storage/tests</code></li>
<li><code>bun test services/api/tests</code></li>
<li><code>bun test apps/web/app/terminal.test.ts</code></li>
<li><code>bun --cwd=apps/web run build</code></li>
</ul>
</section>
<section>
<h2>Issues, Limitations, and Mitigations</h2>
<ul>
<li>The endpoint is detail-time only, which avoids making alert list payloads heavier during bursts.</li>
<li>Malformed trace ids are rejected by route-level validation.</li>
<li>Missing evidence refs remain visible to the drawer as diagnostics rather than hiding partial context.</li>
<li>No schema migration was needed because option prints already persist execution context fields.</li>
</ul>
</section>
<section>
<h2>Follow-up Work</h2>
<p>No follow-up beads issue was filed. The requested storage, API, frontend, tests, build, and documentation work is complete.</p>
</section>
</main>
</body>
</html>

View file

@ -746,6 +746,13 @@ export type EquityPrintQueryFilters = {
sinceTs?: number;
};
export type AlertContextBundle = {
alert: AlertEvent | null;
flow_packets: FlowPacket[];
option_prints: OptionPrint[];
missing_refs: string[];
};
const buildOptionPrintFilterConditions = (
filters: OptionPrintQueryFilters | undefined,
tracePrefix: string | undefined
@ -1200,6 +1207,101 @@ export const fetchRecentAlerts = async (
return AlertEventSchema.array().parse(alerts);
};
const normalizeAlertEvidenceRefs = (refs: string[]): string[] => {
return Array.from(new Set(refs.map((ref) => ref.trim()).filter(Boolean)));
};
const flowPacketCandidatesFromRef = (ref: string): string[] => {
if (!ref) {
return [];
}
if (ref.startsWith("flowpacket:")) {
const raw = ref.slice("flowpacket:".length);
return raw ? [ref, raw] : [ref];
}
return [ref, `flowpacket:${ref}`];
};
const optionPrintCandidatesFromRef = (ref: string): string[] => {
if (!ref || ref.startsWith("flowpacket:")) {
return [];
}
return [ref];
};
export const fetchAlertContextByTraceId = async (
client: ClickHouseClient,
traceId: string
): Promise<AlertContextBundle> => {
const normalizedTraceId = traceId.trim();
if (!normalizedTraceId) {
return {
alert: null,
flow_packets: [],
option_prints: [],
missing_refs: []
};
}
const alertResult = await client.query({
query: `SELECT * FROM ${ALERTS_TABLE} WHERE trace_id = ${quoteString(normalizedTraceId)} ORDER BY source_ts DESC, seq DESC LIMIT 1`,
format: "JSONEachRow"
});
const alertRows = await alertResult.json<unknown[]>();
const alertRecord = alertRows
.map(normalizeAlertRow)
.find((record): record is AlertRecord => record !== null);
const alert = alertRecord ? AlertEventSchema.parse(fromAlertRecord(alertRecord)) : null;
if (!alert) {
return {
alert: null,
flow_packets: [],
option_prints: [],
missing_refs: []
};
}
const refs = normalizeAlertEvidenceRefs(alert.evidence_refs);
const packetLookupIds = Array.from(new Set(refs.flatMap(flowPacketCandidatesFromRef)));
const printLookupIds = Array.from(new Set(refs.flatMap(optionPrintCandidatesFromRef)));
const [flowPackets, optionPrints] = await Promise.all([
packetLookupIds.length > 0
? client
.query({
query: `SELECT * FROM ${FLOW_PACKETS_TABLE} WHERE id IN (${buildStringList(packetLookupIds)}) ORDER BY source_ts DESC, seq DESC LIMIT ${clampLookupLimit(packetLookupIds.length)}`,
format: "JSONEachRow"
})
.then(async (result) => {
const rows = await result.json<unknown[]>();
const records = rows
.map(normalizeFlowPacketRow)
.filter((record): record is FlowPacketRecord => record !== null);
return FlowPacketSchema.array().parse(records.map(fromFlowPacketRecord));
})
: Promise.resolve([]),
printLookupIds.length > 0
? fetchOptionPrintsByTraceIds(client, printLookupIds)
: Promise.resolve([])
]);
const packetIds = new Set(flowPackets.flatMap((packet) => [packet.id, packet.trace_id]));
const printIds = new Set(optionPrints.map((print) => print.trace_id));
const missingRefs = refs.filter((ref) => {
const packetResolved = flowPacketCandidatesFromRef(ref).some((candidate) => packetIds.has(candidate));
const printResolved = optionPrintCandidatesFromRef(ref).some((candidate) => printIds.has(candidate));
return !packetResolved && !printResolved;
});
return {
alert,
flow_packets: flowPackets,
option_prints: optionPrints,
missing_refs: missingRefs
};
};
export const fetchOptionPrintsAfter = async (
client: ClickHouseClient,
afterTs: number,

View file

@ -1,5 +1,8 @@
import { describe, expect, it } from "bun:test";
import type { ClickHouseClient } from "../src/clickhouse";
import { alertsTableDDL, ALERTS_TABLE, fromAlertRecord, toAlertRecord } from "../src/alerts";
import { fetchAlertContextByTraceId } from "../src/clickhouse";
import { toFlowPacketRecord } from "../src/flow-packets";
const alert = {
source_ts: 10,
@ -19,6 +22,62 @@ const alert = {
evidence_refs: ["flowpacket:1", "print:1"]
};
const packet = {
source_ts: 11,
ingest_ts: 21,
seq: 2,
trace_id: "flowpacket:1",
id: "flowpacket:1",
members: ["print:1"],
features: {
option_contract_id: "SPY-2026-06-19-500-C",
count: 1,
total_size: 50
},
join_quality: {}
};
const print = {
source_ts: 12,
ingest_ts: 22,
seq: 3,
trace_id: "print:1",
ts: 12,
option_contract_id: "SPY-2026-06-19-500-C",
price: 1.45,
size: 50,
exchange: "XTEST",
conditions: [],
nbbo_side: "A",
execution_nbbo_bid: 1.4,
execution_nbbo_ask: 1.5,
execution_nbbo_mid: 1.45,
execution_nbbo_spread: 0.1,
execution_nbbo_age_ms: 14,
execution_nbbo_side: "A",
execution_underlying_spot: 500.25,
execution_underlying_bid: 500.2,
execution_underlying_ask: 500.3,
execution_underlying_mid: 500.25,
execution_underlying_age_ms: 9,
execution_iv: 0.31,
signal_reasons: [],
signal_pass: true
};
const makeClient = (resolver: (query: string) => unknown[]): ClickHouseClient =>
({
exec: async () => {},
insert: async () => {},
ping: async () => ({ success: true }),
close: async () => {},
query: async ({ query }: { query: string }) => ({
async json<T>() {
return resolver(query) as T;
}
})
}) as ClickHouseClient;
describe("alerts storage helpers", () => {
it("includes the correct table name in the DDL", () => {
const ddl = alertsTableDDL();
@ -33,4 +92,51 @@ describe("alerts storage helpers", () => {
expect(restored.evidence_refs).toEqual(alert.evidence_refs);
expect(restored.severity).toBe(alert.severity);
});
it("fetches persisted alert context and reports unresolved refs", async () => {
const contextAlert = {
...alert,
trace_id: "alert:ctx",
evidence_refs: ["flowpacket:1", "print:1", "print:missing"]
};
const queries: string[] = [];
const client = makeClient((query) => {
queries.push(query);
if (query.includes(ALERTS_TABLE)) {
return [toAlertRecord(contextAlert)];
}
if (query.includes("flow_packets")) {
return [toFlowPacketRecord(packet)];
}
if (query.includes("option_prints")) {
return [print];
}
return [];
});
const bundle = await fetchAlertContextByTraceId(client, "alert:ctx");
expect(bundle.alert?.trace_id).toBe("alert:ctx");
expect(bundle.flow_packets.map((item) => item.id)).toEqual(["flowpacket:1"]);
expect(bundle.option_prints.map((item) => item.trace_id)).toEqual(["print:1"]);
expect(bundle.option_prints[0]?.execution_nbbo_side).toBe("A");
expect(bundle.option_prints[0]?.execution_nbbo_bid).toBe(1.4);
expect(bundle.option_prints[0]?.execution_underlying_spot).toBe(500.25);
expect(bundle.option_prints[0]?.execution_iv).toBe(0.31);
expect(bundle.missing_refs).toEqual(["print:missing"]);
expect(queries[0]).toContain("trace_id = 'alert:ctx'");
expect(queries[1]).toContain("id IN");
expect(queries[2]).toContain("trace_id IN ('print:1', 'print:missing')");
});
it("returns an empty context when the alert is missing", async () => {
const bundle = await fetchAlertContextByTraceId(makeClient(() => []), "alert:missing");
expect(bundle).toEqual({
alert: null,
flow_packets: [],
option_prints: [],
missing_refs: []
});
});
});

View file

@ -0,0 +1,21 @@
import { z } from "zod";
export const alertContextTraceIdSchema = z
.string()
.trim()
.min(1)
.max(256)
.regex(/^[A-Za-z0-9][A-Za-z0-9:_./-]*$/);
export const isAlertContextPath = (pathname: string): boolean => {
return /^\/flow\/alerts\/[^/]+\/context$/.test(pathname);
};
export const parseAlertContextTraceIdPath = (pathname: string): string | null => {
if (!isAlertContextPath(pathname)) {
return null;
}
const encodedTraceId = pathname.slice("/flow/alerts/".length, -"/context".length);
return alertContextTraceIdSchema.parse(decodeURIComponent(encodedTraceId));
};

View file

@ -47,6 +47,7 @@ import {
ensureOptionPrintsTable,
fetchAlertsAfter,
fetchAlertsBefore,
fetchAlertContextByTraceId,
fetchClassifierHitsAfter,
fetchClassifierHitsBefore,
fetchSmartMoneyEventsAfter,
@ -118,6 +119,7 @@ import {
resolveLiveStateConfig,
shouldFanoutLiveEvent
} from "./live";
import { isAlertContextPath, parseAlertContextTraceIdPath } from "./alert-context";
import { parseOptionPrintQuery } from "./option-queries";
import {
buildSyntheticDerivedStatus,
@ -1487,6 +1489,25 @@ const run = async () => {
return jsonResponse({ data });
}
if (req.method === "GET" && isAlertContextPath(url.pathname)) {
try {
const traceId = parseAlertContextTraceIdPath(url.pathname);
if (traceId === null) {
return jsonResponse({ error: "not found" }, 404);
}
const data = await fetchAlertContextByTraceId(clickhouse, traceId);
return jsonResponse(data);
} catch (error) {
return jsonResponse(
{
error: "invalid alert context query",
detail: error instanceof Error ? error.message : String(error)
},
400
);
}
}
if (req.method === "GET" && url.pathname === "/history/options") {
try {
const { beforeTs, beforeSeq, limit } = parseBeforeParams(url);

View file

@ -0,0 +1,18 @@
import { describe, expect, it } from "bun:test";
import { isAlertContextPath, parseAlertContextTraceIdPath } from "../src/alert-context";
describe("alert context route helpers", () => {
it("extracts a valid alert trace id from the context endpoint path", () => {
expect(parseAlertContextTraceIdPath("/flow/alerts/alert%3Actx%2Fone/context")).toBe("alert:ctx/one");
});
it("returns null for unrelated alert paths", () => {
expect(isAlertContextPath("/flow/alerts")).toBe(false);
expect(parseAlertContextTraceIdPath("/flow/alerts/alert:ctx")).toBeNull();
});
it("rejects malformed trace ids safely", () => {
expect(() => parseAlertContextTraceIdPath("/flow/alerts/%20/context")).toThrow();
expect(() => parseAlertContextTraceIdPath("/flow/alerts/%24bad/context")).toThrow();
});
});