diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index b09ce01..471b404 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -186,7 +186,8 @@ h1 { .card-flow, .card-alerts, -.card-classifiers { +.card-classifiers, +.card-dark { grid-column: span 2; } @@ -857,7 +858,8 @@ h1 { .card-equities, .card-flow, .card-alerts, - .card-classifiers { + .card-classifiers, + .card-dark { grid-column: span 2; } } @@ -871,31 +873,36 @@ h1 { .card-equities, .card-flow, .card-alerts, - .card-classifiers { + .card-classifiers, + .card-dark { grid-column: span 1; } } .card-flow .row, .card-alerts .row, -.card-classifiers .row { +.card-classifiers .row, +.card-dark .row { padding: 12px 14px; } .card-flow .meta, .card-alerts .meta, -.card-classifiers .meta { +.card-classifiers .meta, +.card-dark .meta { font-size: 0.78rem; } .card-flow .note, .card-alerts .note, -.card-classifiers .note { +.card-classifiers .note, +.card-dark .note { font-size: 0.72rem; } .card-flow, .card-alerts, -.card-classifiers { +.card-classifiers, +.card-dark { display: flex; flex-direction: column; height: 960px; @@ -919,7 +926,8 @@ h1 { @media (max-width: 720px) { .card-flow, .card-alerts, - .card-classifiers { + .card-classifiers, + .card-dark { height: 780px; } } diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx index 470c0a3..9da86ff 100644 --- a/apps/web/app/page.tsx +++ b/apps/web/app/page.tsx @@ -5,7 +5,9 @@ import type { AlertEvent, ClassifierHitEvent, EquityPrint, + EquityPrintJoin, FlowPacket, + InferredDarkEvent, OptionNBBO, OptionPrint } from "@islandflow/types"; @@ -24,7 +26,9 @@ type MessageType = | "option-print" | "option-nbbo" | "equity-print" + | "equity-join" | "flow-packet" + | "inferred-dark" | "classifier-hit" | "alert"; @@ -223,6 +227,46 @@ const extractUnderlying = (contractId: string): string => { return contractId.split("-")[0]?.toUpperCase() ?? contractId.toUpperCase(); }; +const extractEquityTraceFromJoin = (joinId: string): string | null => { + const match = joinId.match(/^equityjoin:(.+)$/); + return match?.[1] ?? null; +}; + +const inferDarkUnderlying = ( + event: InferredDarkEvent, + equityPrints: Map, + equityJoins: Map +): string | null => { + for (const ref of event.evidence_refs) { + const join = equityJoins.get(ref); + if (!join) { + continue; + } + const underlying = join.features.underlying_id; + if (typeof underlying === "string" && underlying.length > 0) { + return underlying.toUpperCase(); + } + } + + const match = event.trace_id.match(/^dark:(?:stealth_accumulation|distribution):([^:]+):/); + if (match?.[1]) { + return match[1].toUpperCase(); + } + + for (const ref of event.evidence_refs) { + const traceId = extractEquityTraceFromJoin(ref); + if (!traceId) { + continue; + } + const print = equityPrints.get(traceId); + if (print) { + return print.underlying_id.toUpperCase(); + } + } + + return null; +}; + const parseNumber = (value: unknown, fallback: number): number => { if (typeof value === "number" && Number.isFinite(value)) { return value; @@ -238,6 +282,38 @@ const parseNumber = (value: unknown, fallback: number): number => { return fallback; }; +const parseBoolean = (value: unknown, fallback = false): boolean => { + if (typeof value === "boolean") { + return value; + } + if (typeof value === "number") { + return value !== 0; + } + if (typeof value === "string") { + const normalized = value.trim().toLowerCase(); + if (["true", "1", "yes", "on"].includes(normalized)) { + return true; + } + if (["false", "0", "no", "off"].includes(normalized)) { + return false; + } + } + return fallback; +}; + +const getJoinString = (join: EquityPrintJoin, key: string): string | null => { + const value = join.features[key]; + return typeof value === "string" ? value : null; +}; + +const getJoinNumber = (join: EquityPrintJoin, key: string, fallback = Number.NaN): number => { + return parseNumber(join.features[key], fallback); +}; + +const getJoinBoolean = (join: EquityPrintJoin, key: string): boolean => { + return parseBoolean(join.features[key], false); +}; + type NbboSide = "AA" | "A" | "B" | "BB"; const classifyNbboSide = (price: number, quote: OptionNBBO | null | undefined): NbboSide | null => { @@ -445,14 +521,18 @@ type TapeConfig = { pollMs?: number; captureScroll?: () => void; onNewItems?: (count: number) => void; + getItemTs?: (item: T) => number; + getReplayKey?: (item: T) => string | null; }; -const useTape = ( +const useTape = ( config: TapeConfig ): TapeState => { const { mode, wsPath, replayPath, expectedType, latestPath, onNewItems, captureScroll } = config; const batchSize = config.batchSize ?? 40; const pollMs = config.pollMs ?? 1000; + const getItemTs = config.getItemTs ?? extractSortTs; + const getReplayKey = config.getReplayKey ?? extractTracePrefix; const [status, setStatus] = useState("connecting"); const [items, setItems] = useState([]); const [lastUpdate, setLastUpdate] = useState(null); @@ -561,7 +641,7 @@ const useTape = ( const payload = (await response.json()) as { data?: T[] }; const latest = payload.data?.[0]; if (active && latest) { - replayEndRef.current = latest.ts; + replayEndRef.current = getItemTs(latest); } } catch (error) { console.warn("Failed to load replay end cursor", error); @@ -573,7 +653,7 @@ const useTape = ( return () => { active = false; }; - }, [mode, latestPath]); + }, [mode, latestPath, getItemTs]); useEffect(() => { if (mode !== "live") { @@ -703,21 +783,21 @@ const useTape = ( let sourcePrefix = replaySourceRef.current; if (!sourcePrefix) { - const firstWithTrace = payload.data.find((item) => extractTracePrefix(item)); + const firstWithTrace = payload.data.find((item) => getReplayKey(item)); if (firstWithTrace) { - sourcePrefix = extractTracePrefix(firstWithTrace); + sourcePrefix = getReplayKey(firstWithTrace); replaySourceRef.current = sourcePrefix ?? null; } } const filtered = sourcePrefix - ? payload.data.filter((item) => extractTracePrefix(item) === sourcePrefix) + ? payload.data.filter((item) => getReplayKey(item) === sourcePrefix) : payload.data; const hasForeign = sourcePrefix && payload.data.some((item) => { - const prefix = extractTracePrefix(item); + const prefix = getReplayKey(item); return prefix !== null && prefix !== sourcePrefix; }); @@ -728,9 +808,10 @@ const useTape = ( scheduleFlush(); const last = filtered.at(-1); if (last) { - setReplayTime(last.ts); - if (replayEnd !== null && last.ts >= replayEnd) { - cursorRef.current = { ts: last.ts, seq: last.seq }; + const lastTs = getItemTs(last); + setReplayTime(lastTs); + if (replayEnd !== null && lastTs >= replayEnd) { + cursorRef.current = { ts: lastTs, seq: last.seq }; replayCompleteRef.current = true; setReplayComplete(true); setStatus("disconnected"); @@ -781,7 +862,7 @@ const useTape = ( window.clearInterval(interval); cancelFlush(); }; - }, [mode, replayPath, batchSize, pollMs, scheduleFlush, cancelFlush]); + }, [mode, replayPath, batchSize, pollMs, scheduleFlush, cancelFlush, getItemTs, getReplayKey]); return { status, @@ -1179,6 +1260,10 @@ type EvidenceItem = | { kind: "print"; id: string; print: OptionPrint } | { kind: "unknown"; id: string }; +type DarkEvidenceItem = + | { kind: "join"; id: string; join: EquityPrintJoin } + | { kind: "unknown"; id: string }; + type AlertDrawerProps = { alert: AlertEvent; flowPacket: FlowPacket | null; @@ -1293,6 +1378,118 @@ const AlertDrawer = ({ alert, flowPacket, evidence, onClose }: AlertDrawerProps) ); }; +type DarkDrawerProps = { + event: InferredDarkEvent; + evidence: DarkEvidenceItem[]; + underlying: string | null; + onClose: () => void; +}; + +const DarkDrawer = ({ event, evidence, underlying, onClose }: DarkDrawerProps) => { + const joinEvidence = evidence.filter( + (item): item is { kind: "join"; id: string; join: EquityPrintJoin } => item.kind === "join" + ); + const unknownCount = evidence.filter((item) => item.kind === "unknown").length; + const traceRefs = event.evidence_refs.slice(0, 6); + const extraRefs = Math.max(0, event.evidence_refs.length - traceRefs.length); + + return ( + + ); +}; + const formatFlowMetric = (value: number, suffix?: string): string => { if (suffix) { return `${value}${suffix}`; @@ -1304,21 +1501,25 @@ const formatFlowMetric = (value: number, suffix?: string): string => { export default function HomePage() { const [mode, setMode] = useState("live"); const [selectedAlert, setSelectedAlert] = useState(null); + const [selectedDarkEvent, setSelectedDarkEvent] = useState(null); const [filterInput, setFilterInput] = useState(""); const optionsScroll = useListScroll(); const equitiesScroll = useListScroll(); const flowScroll = useListScroll(); + const darkScroll = useListScroll(); const alertsScroll = useListScroll(); const classifierScroll = useListScroll(); const optionsAnchor = useScrollAnchor(optionsScroll.listRef, optionsScroll.isAtTopRef); const equitiesAnchor = useScrollAnchor(equitiesScroll.listRef, equitiesScroll.isAtTopRef); const flowAnchor = useScrollAnchor(flowScroll.listRef, flowScroll.isAtTopRef); + const darkAnchor = useScrollAnchor(darkScroll.listRef, darkScroll.isAtTopRef); const alertsAnchor = useScrollAnchor(alertsScroll.listRef, alertsScroll.isAtTopRef); const classifierAnchor = useScrollAnchor( classifierScroll.listRef, classifierScroll.isAtTopRef ); + const disableReplayGrouping = useCallback(() => null, []); const options = useTape({ mode, @@ -1344,6 +1545,17 @@ export default function HomePage() { onNewItems: equitiesScroll.onNewItems }); + const equityJoins = useTape({ + mode, + wsPath: "/ws/equity-joins", + replayPath: "/replay/equity-joins", + latestPath: "/joins/equities", + expectedType: "equity-join", + batchSize: mode === "replay" ? 120 : undefined, + pollMs: mode === "replay" ? 200 : undefined, + getReplayKey: disableReplayGrouping + }); + const nbbo = useTape({ mode, wsPath: "/ws/options-nbbo", @@ -1354,6 +1566,19 @@ export default function HomePage() { pollMs: mode === "replay" ? 200 : undefined }); + const inferredDark = useTape({ + mode, + wsPath: "/ws/inferred-dark", + replayPath: "/replay/inferred-dark", + latestPath: "/dark/inferred", + expectedType: "inferred-dark", + batchSize: mode === "replay" ? 120 : undefined, + pollMs: mode === "replay" ? 200 : undefined, + captureScroll: darkAnchor.capture, + onNewItems: darkScroll.onNewItems, + getReplayKey: disableReplayGrouping + }); + const flowHold = useCallback(() => !flowScroll.isAtTopRef.current, [flowScroll.isAtTopRef]); const flow = useFlowStream( mode === "live", @@ -1389,6 +1614,10 @@ export default function HomePage() { flowAnchor.apply(); }, [flow.items, flowAnchor.apply]); + useLayoutEffect(() => { + darkAnchor.apply(); + }, [inferredDark.items, darkAnchor.apply]); + useLayoutEffect(() => { alertsAnchor.apply(); }, [alerts.items, alertsAnchor.apply]); @@ -1432,6 +1661,24 @@ export default function HomePage() { return map; }, [options.items]); + const equityPrintMap = useMemo(() => { + const map = new Map(); + for (const print of equities.items) { + if (print.trace_id) { + map.set(print.trace_id, print); + } + } + return map; + }, [equities.items]); + + const equityJoinMap = useMemo(() => { + const map = new Map(); + for (const join of equityJoins.items) { + map.set(join.id, join); + } + return map; + }, [equityJoins.items]); + const flowPacketMap = useMemo(() => { const map = new Map(); for (const packet of flow.items) { @@ -1466,10 +1713,32 @@ export default function HomePage() { return packetId ? flowPacketMap.get(packetId) ?? null : null; }, [selectedAlert, flowPacketMap]); + const selectedDarkEvidence = useMemo((): DarkEvidenceItem[] => { + if (!selectedDarkEvent) { + return []; + } + + return selectedDarkEvent.evidence_refs.map((id) => { + const join = equityJoinMap.get(id); + if (join) { + return { kind: "join", id, join }; + } + return { kind: "unknown", id }; + }); + }, [selectedDarkEvent, equityJoinMap]); + + const selectedDarkUnderlying = useMemo(() => { + if (!selectedDarkEvent) { + return null; + } + return inferDarkUnderlying(selectedDarkEvent, equityPrintMap, equityJoinMap); + }, [selectedDarkEvent, equityJoinMap, equityPrintMap]); + useEffect(() => { if (mode !== "live") { setSelectedAlert(null); } + setSelectedDarkEvent(null); }, [mode]); const extractPacketContract = useCallback((packet: FlowPacket): string => { @@ -1545,6 +1814,16 @@ export default function HomePage() { return equities.items.filter((print) => matchesTicker(print.underlying_id)); }, [equities.items, matchesTicker, tickerSet]); + const filteredInferredDark = useMemo(() => { + if (tickerSet.size === 0) { + return inferredDark.items; + } + return inferredDark.items.filter((event) => { + const underlying = inferDarkUnderlying(event, equityPrintMap, equityJoinMap); + return matchesTicker(underlying); + }); + }, [equityJoinMap, equityPrintMap, inferredDark.items, matchesTicker, tickerSet]); + const filteredFlow = useMemo(() => { if (tickerSet.size === 0) { return flow.items; @@ -1575,6 +1854,7 @@ export default function HomePage() { return [ options.lastUpdate, equities.lastUpdate, + inferredDark.lastUpdate, flow.lastUpdate, alerts.lastUpdate, classifierHits.lastUpdate @@ -1584,6 +1864,7 @@ export default function HomePage() { }, [ options.lastUpdate, equities.lastUpdate, + inferredDark.lastUpdate, flow.lastUpdate, alerts.lastUpdate, classifierHits.lastUpdate @@ -1975,7 +2256,10 @@ export default function HomePage() { className="row row-button" key={`${alert.trace_id}-${alert.seq}`} type="button" - onClick={() => setSelectedAlert(alert)} + onClick={() => { + setSelectedDarkEvent(null); + setSelectedAlert(alert); + }} >
@@ -2060,6 +2344,75 @@ export default function HomePage() {
+ +
+
+
+

Inferred Dark

+

Off-exchange patterns inferred from equity joins.

+
+
+
+ + +
+ +
+
+ {filteredInferredDark.length === 0 ? ( +
+ {tickerSet.size > 0 + ? "No inferred dark events match the current filter." + : mode === "live" + ? "No inferred dark events yet. Start compute." + : "Replay queue empty. Ensure ClickHouse has data."} +
+ ) : ( + filteredInferredDark.map((event) => { + const underlying = inferDarkUnderlying(event, equityPrintMap, equityJoinMap); + const evidenceCount = event.evidence_refs.length; + return ( + + ); + }) + )} +
+
+
{selectedAlert ? ( @@ -2070,6 +2423,15 @@ export default function HomePage() { onClose={() => setSelectedAlert(null)} /> ) : null} + + {selectedDarkEvent ? ( + setSelectedDarkEvent(null)} + /> + ) : null} ); } diff --git a/scripts/dev.ts b/scripts/dev.ts index 6da5b89..84fbc51 100644 --- a/scripts/dev.ts +++ b/scripts/dev.ts @@ -30,6 +30,11 @@ const spawnChild = ({ name, cmd, cwd }: ChildSpec): void => { const exitCode = code ?? 0; const statusLabel = exitCode === 0 ? "exited" : "failed"; console.error(`[dev] ${name} ${statusLabel} (${exitCode})`); + if (name === "infra" && exitCode !== 0) { + console.error( + "[dev] Infra failed. Ensure Docker is installed and the daemon is running (OrbStack or Docker Desktop), then retry." + ); + } shutdown(exitCode); }); }; diff --git a/services/ingest-equities/src/adapters/synthetic.ts b/services/ingest-equities/src/adapters/synthetic.ts index c1d21fa..861ffa7 100644 --- a/services/ingest-equities/src/adapters/synthetic.ts +++ b/services/ingest-equities/src/adapters/synthetic.ts @@ -6,6 +6,22 @@ type SyntheticEquitiesAdapterConfig = { }; const EXCHANGES = ["NYSE", "NASDAQ", "ARCA", "BATS", "IEX", "TEST"]; +const DARK_EXCHANGE = "OTC"; + +type PricePlacement = "MID" | "A" | "AA" | "B" | "BB"; +type DarkScenario = "block" | "buy" | "sell"; + +const DARK_SEQUENCE: DarkScenario[] = [ + "block", + "buy", + "buy", + "buy", + "buy", + "sell", + "sell", + "sell", + "sell" +]; const hashSymbol = (value: string): number => { let hash = 0; @@ -57,6 +73,50 @@ const buildSyntheticQuote = ( }; }; +const formatPrice = (value: number): number => { + return Number(value.toFixed(2)); +}; + +const buildQuoteFromMid = (mid: number) => { + const spread = Math.max(0.05, Number((mid * 0.002).toFixed(2))); + const half = spread / 2; + const bid = formatPrice(Math.max(0.01, mid - half)); + const ask = formatPrice(Math.max(bid + 0.01, mid + half)); + const epsilon = Math.max(0.01, spread * 0.05); + + return { bid, ask, spread, epsilon }; +}; + +const priceForPlacement = ( + mid: number, + quote: { bid: number; ask: number; epsilon: number }, + placement: PricePlacement +): number => { + const { bid, ask, epsilon } = quote; + + let price = mid; + switch (placement) { + case "AA": + price = ask + epsilon * 1.5; + break; + case "A": + price = ask; + break; + case "BB": + price = bid - epsilon * 1.5; + break; + case "B": + price = bid; + break; + case "MID": + default: + price = mid; + break; + } + + return formatPrice(Math.max(0.01, price)); +}; + export const createSyntheticEquitiesAdapter = ( config: SyntheticEquitiesAdapterConfig ): EquityIngestAdapter => { @@ -65,6 +125,8 @@ export const createSyntheticEquitiesAdapter = ( start: (handlers: EquityIngestHandlers) => { let seq = 0; let quoteSeq = 0; + let darkStep = 0; + let darkSymbolIndex = 0; let timer: ReturnType | null = null; let stopped = false; @@ -76,27 +138,79 @@ export const createSyntheticEquitiesAdapter = ( const now = Date.now(); const batchSize = 3; + const darkSymbol = SP500_SYMBOLS[darkSymbolIndex % SP500_SYMBOLS.length]; + const darkHash = hashSymbol(darkSymbol); + const darkBase = 25 + (darkHash % 475); + const darkDrift = ((darkStep % 24) - 12) * 0.08; + const darkMid = formatPrice(darkBase + darkDrift); + const darkQuote = buildQuoteFromMid(darkMid); + const scenario = DARK_SEQUENCE[darkStep % DARK_SEQUENCE.length]; + const darkTs = now; + + if (handlers.onQuote) { + quoteSeq += 1; + const quoteEvent = buildSyntheticQuote( + quoteSeq, + darkTs - 2, + darkSymbol, + darkQuote.bid, + darkQuote.ask + ); + void handlers.onQuote(quoteEvent); + } + + seq += 1; + let darkPlacement: PricePlacement = "MID"; + let darkSize = 2600; + if (scenario === "buy") { + darkPlacement = darkStep % 2 === 0 ? "A" : "AA"; + darkSize = 800; + } else if (scenario === "sell") { + darkPlacement = darkStep % 2 === 0 ? "B" : "BB"; + darkSize = 800; + } + const darkPrice = priceForPlacement(darkMid, darkQuote, darkPlacement); + const darkPrint = buildSyntheticPrint( + seq, + darkTs, + darkSymbol, + darkPrice, + darkSize, + DARK_EXCHANGE, + true + ); + void handlers.onTrade(darkPrint); + + darkStep += 1; + if (darkStep >= DARK_SEQUENCE.length) { + darkStep = 0; + darkSymbolIndex += 1; + } + for (let i = 0; i < batchSize; i += 1) { seq += 1; const symbol = SP500_SYMBOLS[(seq + i) % SP500_SYMBOLS.length]; const symbolHash = hashSymbol(symbol); const basePrice = 25 + (symbolHash % 475); - const price = Number((basePrice + ((seq % 40) - 20) * 0.05).toFixed(2)); + const mid = formatPrice(basePrice + ((seq % 40) - 20) * 0.05); + const quote = buildQuoteFromMid(mid); + const placement: PricePlacement = + seq % 11 === 0 ? "A" : seq % 13 === 0 ? "B" : "MID"; + const price = priceForPlacement(mid, quote, placement); const size = 10 + (seq % 600); const exchange = EXCHANGES[(seq + symbolHash) % EXCHANGES.length]; const offExchangeFlag = (seq + i) % 6 === 0; const eventTs = now + i * 4; - const print = buildSyntheticPrint(seq, eventTs, symbol, price, size, exchange, offExchangeFlag); - void handlers.onTrade(print); if (handlers.onQuote) { quoteSeq += 1; - const spread = Math.max(0.02, Number((price * 0.002).toFixed(2))); - const bid = Math.max(0.01, Number((price - spread / 2).toFixed(2))); - const ask = Math.max(bid + 0.01, Number((price + spread / 2).toFixed(2))); - const quote = buildSyntheticQuote(quoteSeq, eventTs, symbol, bid, ask); - void handlers.onQuote(quote); + const quoteEventTs = eventTs - 2; + const quoteEvent = buildSyntheticQuote(quoteSeq, quoteEventTs, symbol, quote.bid, quote.ask); + void handlers.onQuote(quoteEvent); } + + const print = buildSyntheticPrint(seq, eventTs, symbol, price, size, exchange, offExchangeFlag); + void handlers.onTrade(print); } };