Add testing-mode throttles and UI batching
Throttle ingest pipelines in TESTING_MODE, document settings in README, and batch live UI updates per frame to reduce scroll lag.
This commit is contained in:
parent
82861408e4
commit
bd1a67a7fc
5 changed files with 341 additions and 58 deletions
|
|
@ -45,6 +45,10 @@ IBKR_PYTHON_BIN=python3
|
||||||
EQUITIES_INGEST_ADAPTER=synthetic
|
EQUITIES_INGEST_ADAPTER=synthetic
|
||||||
EMIT_INTERVAL_MS=1000
|
EMIT_INTERVAL_MS=1000
|
||||||
|
|
||||||
|
# Testing mode
|
||||||
|
TESTING_MODE=false
|
||||||
|
TESTING_THROTTLE_MS=200
|
||||||
|
|
||||||
# Compute consumer behavior
|
# Compute consumer behavior
|
||||||
COMPUTE_DELIVER_POLICY=new
|
COMPUTE_DELIVER_POLICY=new
|
||||||
COMPUTE_CONSUMER_RESET=false
|
COMPUTE_CONSUMER_RESET=false
|
||||||
|
|
|
||||||
24
README.md
24
README.md
|
|
@ -6,24 +6,26 @@ The system ingests real-time options trades/quotes and equity prints, clusters r
|
||||||
|
|
||||||
## CURRENT STATE (Plan Progress)
|
## CURRENT STATE (Plan Progress)
|
||||||
|
|
||||||
Plan progress (rough): [####------]
|
Plan progress (rough): [#####-----]
|
||||||
|
|
||||||
Done now (in repo):
|
Done now (in repo):
|
||||||
- Bun monorepo + infra docker compose (ClickHouse, Redis, NATS JetStream)
|
- Bun monorepo + infra docker compose (ClickHouse, Redis, NATS JetStream)
|
||||||
- Shared event schemas + logging + config helpers
|
- Shared event schemas + logging + config helpers
|
||||||
- Synthetic options/equity prints published to NATS and persisted to ClickHouse
|
- Synthetic options/equity prints (full S&P 500) published to NATS and persisted to ClickHouse
|
||||||
- Deterministic option FlowPacket clustering (time window) + persistence
|
- Deterministic option FlowPacket clustering (time window) + persistence
|
||||||
- API: REST for prints/flow packets, WS for live options/equities/flow, replay endpoints
|
- Rule-first classifiers + alert scoring with ClickHouse persistence + WS/REST endpoints
|
||||||
|
- API: REST for prints/flow packets/classifier hits/alerts, WS for live options/equities/flow/alerts/hits, replay endpoints
|
||||||
- UI: live tapes for options/equities/flow + replay toggle + pause controls + replay time/completion
|
- UI: live tapes for options/equities/flow + replay toggle + pause controls + replay time/completion
|
||||||
|
- UI: alerts + classifier hits panels, ticker filter, evidence drawer, severity strip
|
||||||
- Databento historical replay adapter (options) with symbol mapping
|
- Databento historical replay adapter (options) with symbol mapping
|
||||||
- Alpaca options adapter (dev-only, bounded contract list)
|
- Alpaca options adapter (dev-only, bounded contract list)
|
||||||
|
- Testing-mode throttling for ingest to reduce CPU during local dev
|
||||||
|
|
||||||
In progress / blocked:
|
In progress / blocked:
|
||||||
- Live data adapters beyond dev-only feeds (requires licensed data source)
|
- Live data adapters beyond dev-only feeds (requires licensed data source)
|
||||||
- Rolling stats and advanced clustering
|
- Rolling stats and advanced clustering
|
||||||
|
|
||||||
Not started:
|
Not started:
|
||||||
- Classifiers + alert scoring
|
|
||||||
- Dark pool inference
|
- Dark pool inference
|
||||||
- Candle service and chart overlays
|
- Candle service and chart overlays
|
||||||
- Auth / secure deployment
|
- Auth / secure deployment
|
||||||
|
|
@ -37,19 +39,19 @@ Not started:
|
||||||
|
|
||||||
## Current Capabilities
|
## Current Capabilities
|
||||||
|
|
||||||
- Synthetic options/equity prints with deterministic sequencing
|
- Synthetic options/equity prints with deterministic sequencing across the S&P 500
|
||||||
- Ingest adapter seam (env-selected; options default `alpaca`, equities default `synthetic`)
|
- Ingest adapter seam (env-selected; options default `alpaca`, equities default `synthetic`)
|
||||||
- Raw event persistence in ClickHouse + streaming via NATS JetStream
|
- Raw event persistence in ClickHouse + streaming via NATS JetStream
|
||||||
- Deterministic option FlowPacket clustering (time-window)
|
- Deterministic option FlowPacket clustering (time-window)
|
||||||
|
- Classifiers + alert scoring (rule-first) with WS/REST endpoints
|
||||||
- API gateway with REST, WS, and replay endpoints
|
- API gateway with REST, WS, and replay endpoints
|
||||||
- UI tapes for options/equities/flow packets with live/replay toggle and pause controls
|
- UI tapes for options/equities/flow packets + alerts/hits with live/replay toggle and pause controls
|
||||||
- Alpaca options adapter (dev-only) with bounded contract selection
|
- Alpaca options adapter (dev-only) with bounded contract selection
|
||||||
- Databento historical replay adapter (options, Python sidecar)
|
- Databento historical replay adapter (options, Python sidecar)
|
||||||
|
|
||||||
## Planned Capabilities (from PLAN.md)
|
## Planned Capabilities (from PLAN.md)
|
||||||
|
|
||||||
- Real-time licensed market data ingestors (options + equities)
|
- Real-time licensed market data ingestors (options + equities)
|
||||||
- Rule-first classifiers and alert scoring
|
|
||||||
- Dark pool inference and evidence linking
|
- Dark pool inference and evidence linking
|
||||||
- Candle aggregation + chart overlays
|
- Candle aggregation + chart overlays
|
||||||
- Replay/backtesting metrics and calibration
|
- Replay/backtesting metrics and calibration
|
||||||
|
|
@ -92,8 +94,8 @@ Create env file:
|
||||||
Start everything (infra + services + web):
|
Start everything (infra + services + web):
|
||||||
- `bun run dev`
|
- `bun run dev`
|
||||||
|
|
||||||
Run just the web app (auto-picks a free port in 3001-3005):
|
Run just the web app (fixed to port 3000):
|
||||||
- `bun --cwd apps/web run dev`
|
- `bun run dev:web`
|
||||||
|
|
||||||
Run just the API:
|
Run just the API:
|
||||||
- `bun --cwd services/api run dev`
|
- `bun --cwd services/api run dev`
|
||||||
|
|
@ -103,6 +105,10 @@ Adapter selection (env):
|
||||||
- Equities: `EQUITIES_INGEST_ADAPTER` (defaults to `synthetic`)
|
- Equities: `EQUITIES_INGEST_ADAPTER` (defaults to `synthetic`)
|
||||||
- Compute: `COMPUTE_DELIVER_POLICY` (`new` default), `COMPUTE_CONSUMER_RESET` (force skip backlog)
|
- Compute: `COMPUTE_DELIVER_POLICY` (`new` default), `COMPUTE_CONSUMER_RESET` (force skip backlog)
|
||||||
|
|
||||||
|
Testing mode (throttles ingest to reduce CPU):
|
||||||
|
- `TESTING_MODE=true` enables throttling
|
||||||
|
- `TESTING_THROTTLE_MS=200` minimum spacing between emitted prints (per ingest service)
|
||||||
|
|
||||||
IBKR adapter (options, via Python `ib_insync`):
|
IBKR adapter (options, via Python `ib_insync`):
|
||||||
- Install Python deps: `python3 -m pip install -r services/ingest-options/py/requirements.txt`
|
- Install Python deps: `python3 -m pip install -r services/ingest-options/py/requirements.txt`
|
||||||
- Set `OPTIONS_INGEST_ADAPTER=ibkr` and configure:
|
- Set `OPTIONS_INGEST_ADAPTER=ibkr` and configure:
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
|
||||||
import type { AlertEvent, ClassifierHitEvent, EquityPrint, FlowPacket, OptionPrint } from "@islandflow/types";
|
import type { AlertEvent, ClassifierHitEvent, EquityPrint, FlowPacket, OptionPrint } from "@islandflow/types";
|
||||||
|
|
||||||
const MAX_ITEMS = 500;
|
const MAX_ITEMS = 500;
|
||||||
|
|
@ -213,6 +213,7 @@ const parseNumber = (value: unknown, fallback: number): number => {
|
||||||
type ListScrollState = {
|
type ListScrollState = {
|
||||||
listRef: React.RefObject<HTMLDivElement>;
|
listRef: React.RefObject<HTMLDivElement>;
|
||||||
isAtTop: boolean;
|
isAtTop: boolean;
|
||||||
|
isAtTopRef: React.MutableRefObject<boolean>;
|
||||||
missed: number;
|
missed: number;
|
||||||
onNewItems: (count: number) => void;
|
onNewItems: (count: number) => void;
|
||||||
jumpToTop: () => void;
|
jumpToTop: () => void;
|
||||||
|
|
@ -236,6 +237,7 @@ const useListScroll = (): ListScrollState => {
|
||||||
|
|
||||||
const onScroll = () => {
|
const onScroll = () => {
|
||||||
const atTop = el.scrollTop <= 2;
|
const atTop = el.scrollTop <= 2;
|
||||||
|
isAtTopRef.current = atTop;
|
||||||
setIsAtTop(atTop);
|
setIsAtTop(atTop);
|
||||||
if (atTop) {
|
if (atTop) {
|
||||||
setMissed(0);
|
setMissed(0);
|
||||||
|
|
@ -273,7 +275,55 @@ const useListScroll = (): ListScrollState => {
|
||||||
setMissed(0);
|
setMissed(0);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
return { listRef, isAtTop, missed, onNewItems, jumpToTop };
|
return {
|
||||||
|
listRef,
|
||||||
|
isAtTop,
|
||||||
|
isAtTopRef,
|
||||||
|
missed,
|
||||||
|
onNewItems,
|
||||||
|
jumpToTop
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const useScrollAnchor = (
|
||||||
|
listRef: React.RefObject<HTMLDivElement>,
|
||||||
|
isAtTopRef: React.MutableRefObject<boolean>
|
||||||
|
) => {
|
||||||
|
const pendingRef = useRef<{ top: number; height: number } | null>(null);
|
||||||
|
|
||||||
|
const capture = useCallback(() => {
|
||||||
|
if (isAtTopRef.current) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const el = listRef.current;
|
||||||
|
if (!el) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
pendingRef.current = {
|
||||||
|
top: el.scrollTop,
|
||||||
|
height: el.scrollHeight
|
||||||
|
};
|
||||||
|
}, [isAtTopRef, listRef]);
|
||||||
|
|
||||||
|
const apply = useCallback(() => {
|
||||||
|
const pending = pendingRef.current;
|
||||||
|
if (!pending) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const el = listRef.current;
|
||||||
|
if (!el) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const delta = el.scrollHeight - pending.height;
|
||||||
|
el.scrollTop = pending.top + delta;
|
||||||
|
pendingRef.current = null;
|
||||||
|
}, [listRef]);
|
||||||
|
|
||||||
|
return { capture, apply };
|
||||||
};
|
};
|
||||||
|
|
||||||
const statusLabel = (status: WsStatus, paused: boolean, mode: TapeMode): string => {
|
const statusLabel = (status: WsStatus, paused: boolean, mode: TapeMode): string => {
|
||||||
|
|
@ -304,13 +354,14 @@ type TapeConfig<T> = {
|
||||||
expectedType: MessageType;
|
expectedType: MessageType;
|
||||||
batchSize?: number;
|
batchSize?: number;
|
||||||
pollMs?: number;
|
pollMs?: number;
|
||||||
|
captureScroll?: () => void;
|
||||||
onNewItems?: (count: number) => void;
|
onNewItems?: (count: number) => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
const useTape = <T extends { ts: number; seq: number }>(
|
const useTape = <T extends { ts: number; seq: number }>(
|
||||||
config: TapeConfig<T>
|
config: TapeConfig<T>
|
||||||
): TapeState<T> => {
|
): TapeState<T> => {
|
||||||
const { mode, wsPath, replayPath, expectedType, latestPath, onNewItems } = config;
|
const { mode, wsPath, replayPath, expectedType, latestPath, onNewItems, captureScroll } = config;
|
||||||
const batchSize = config.batchSize ?? 40;
|
const batchSize = config.batchSize ?? 40;
|
||||||
const pollMs = config.pollMs ?? 1000;
|
const pollMs = config.pollMs ?? 1000;
|
||||||
const [status, setStatus] = useState<WsStatus>("connecting");
|
const [status, setStatus] = useState<WsStatus>("connecting");
|
||||||
|
|
@ -328,11 +379,50 @@ const useTape = <T extends { ts: number; seq: number }>(
|
||||||
const replaySourceRef = useRef<string | null>(null);
|
const replaySourceRef = useRef<string | null>(null);
|
||||||
const emptyPollsRef = useRef<number>(0);
|
const emptyPollsRef = useRef<number>(0);
|
||||||
const pausedRef = useRef(paused);
|
const pausedRef = useRef(paused);
|
||||||
|
const pendingRef = useRef<T[]>([]);
|
||||||
|
const pendingCountRef = useRef(0);
|
||||||
|
const flushHandleRef = useRef<number | null>(null);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
pausedRef.current = paused;
|
pausedRef.current = paused;
|
||||||
}, [paused]);
|
}, [paused]);
|
||||||
|
|
||||||
|
const cancelFlush = useCallback(() => {
|
||||||
|
if (flushHandleRef.current !== null) {
|
||||||
|
cancelAnimationFrame(flushHandleRef.current);
|
||||||
|
flushHandleRef.current = null;
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const scheduleFlush = useCallback(() => {
|
||||||
|
if (flushHandleRef.current !== null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
flushHandleRef.current = requestAnimationFrame(() => {
|
||||||
|
flushHandleRef.current = null;
|
||||||
|
const buffered = pendingRef.current;
|
||||||
|
if (buffered.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pendingRef.current = [];
|
||||||
|
|
||||||
|
const pendingCount = pendingCountRef.current;
|
||||||
|
pendingCountRef.current = 0;
|
||||||
|
|
||||||
|
if (onNewItems && pendingCount > 0) {
|
||||||
|
onNewItems(pendingCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (captureScroll) {
|
||||||
|
captureScroll();
|
||||||
|
}
|
||||||
|
|
||||||
|
setItems((prev) => mergeNewest(buffered, prev));
|
||||||
|
setLastUpdate(Date.now());
|
||||||
|
});
|
||||||
|
}, [captureScroll, onNewItems]);
|
||||||
|
|
||||||
const togglePause = useCallback(() => {
|
const togglePause = useCallback(() => {
|
||||||
setPaused((prev) => {
|
setPaused((prev) => {
|
||||||
const next = !prev;
|
const next = !prev;
|
||||||
|
|
@ -354,7 +444,10 @@ const useTape = <T extends { ts: number; seq: number }>(
|
||||||
setDropped(0);
|
setDropped(0);
|
||||||
setStatus("connecting");
|
setStatus("connecting");
|
||||||
cursorRef.current = { ts: 0, seq: 0 };
|
cursorRef.current = { ts: 0, seq: 0 };
|
||||||
}, [mode]);
|
pendingRef.current = [];
|
||||||
|
pendingCountRef.current = 0;
|
||||||
|
cancelFlush();
|
||||||
|
}, [mode, cancelFlush]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (mode !== "replay" || !latestPath) {
|
if (mode !== "replay" || !latestPath) {
|
||||||
|
|
@ -434,12 +527,9 @@ const useTape = <T extends { ts: number; seq: number }>(
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (onNewItems) {
|
pendingRef.current.push(message.payload);
|
||||||
onNewItems(1);
|
pendingCountRef.current += 1;
|
||||||
}
|
scheduleFlush();
|
||||||
|
|
||||||
setItems((prev) => mergeNewest([message.payload], prev));
|
|
||||||
setLastUpdate(Date.now());
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn("Failed to parse websocket payload", error);
|
console.warn("Failed to parse websocket payload", error);
|
||||||
}
|
}
|
||||||
|
|
@ -470,6 +560,7 @@ const useTape = <T extends { ts: number; seq: number }>(
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
active = false;
|
active = false;
|
||||||
|
cancelFlush();
|
||||||
if (reconnectRef.current !== null) {
|
if (reconnectRef.current !== null) {
|
||||||
window.clearTimeout(reconnectRef.current);
|
window.clearTimeout(reconnectRef.current);
|
||||||
}
|
}
|
||||||
|
|
@ -477,7 +568,7 @@ const useTape = <T extends { ts: number; seq: number }>(
|
||||||
socketRef.current.close();
|
socketRef.current.close();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}, [mode, wsPath, expectedType]);
|
}, [mode, wsPath, expectedType, scheduleFlush, cancelFlush]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (mode !== "replay") {
|
if (mode !== "replay") {
|
||||||
|
|
@ -543,11 +634,9 @@ const useTape = <T extends { ts: number; seq: number }>(
|
||||||
|
|
||||||
if (filtered.length > 0) {
|
if (filtered.length > 0) {
|
||||||
const nextItems = [...filtered].reverse();
|
const nextItems = [...filtered].reverse();
|
||||||
if (onNewItems) {
|
pendingRef.current.push(...nextItems);
|
||||||
onNewItems(nextItems.length);
|
pendingCountRef.current += nextItems.length;
|
||||||
}
|
scheduleFlush();
|
||||||
setItems((prev) => mergeNewest(nextItems, prev));
|
|
||||||
setLastUpdate(Date.now());
|
|
||||||
const last = filtered.at(-1);
|
const last = filtered.at(-1);
|
||||||
if (last) {
|
if (last) {
|
||||||
setReplayTime(last.ts);
|
setReplayTime(last.ts);
|
||||||
|
|
@ -601,8 +690,9 @@ const useTape = <T extends { ts: number; seq: number }>(
|
||||||
return () => {
|
return () => {
|
||||||
active = false;
|
active = false;
|
||||||
window.clearInterval(interval);
|
window.clearInterval(interval);
|
||||||
|
cancelFlush();
|
||||||
};
|
};
|
||||||
}, [mode, replayPath, batchSize, pollMs]);
|
}, [mode, replayPath, batchSize, pollMs, scheduleFlush, cancelFlush]);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
status,
|
status,
|
||||||
|
|
@ -617,12 +707,17 @@ const useTape = <T extends { ts: number; seq: number }>(
|
||||||
};
|
};
|
||||||
|
|
||||||
const useLiveStream = <T extends SortableItem>(
|
const useLiveStream = <T extends SortableItem>(
|
||||||
enabled: boolean,
|
config: {
|
||||||
wsPath: string,
|
enabled: boolean;
|
||||||
expectedType: MessageType,
|
wsPath: string;
|
||||||
onNewItems?: (count: number) => void
|
expectedType: MessageType;
|
||||||
|
onNewItems?: (count: number) => void;
|
||||||
|
captureScroll?: () => void;
|
||||||
|
}
|
||||||
): TapeState<T> => {
|
): TapeState<T> => {
|
||||||
const [status, setStatus] = useState<WsStatus>(enabled ? "connecting" : "disconnected");
|
const [status, setStatus] = useState<WsStatus>(
|
||||||
|
config.enabled ? "connecting" : "disconnected"
|
||||||
|
);
|
||||||
const [items, setItems] = useState<T[]>([]);
|
const [items, setItems] = useState<T[]>([]);
|
||||||
const [lastUpdate, setLastUpdate] = useState<number | null>(null);
|
const [lastUpdate, setLastUpdate] = useState<number | null>(null);
|
||||||
const [replayTime] = useState<number | null>(null);
|
const [replayTime] = useState<number | null>(null);
|
||||||
|
|
@ -632,11 +727,50 @@ const useLiveStream = <T extends SortableItem>(
|
||||||
const reconnectRef = useRef<number | null>(null);
|
const reconnectRef = useRef<number | null>(null);
|
||||||
const socketRef = useRef<WebSocket | null>(null);
|
const socketRef = useRef<WebSocket | null>(null);
|
||||||
const pausedRef = useRef(paused);
|
const pausedRef = useRef(paused);
|
||||||
|
const pendingRef = useRef<T[]>([]);
|
||||||
|
const pendingCountRef = useRef(0);
|
||||||
|
const flushHandleRef = useRef<number | null>(null);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
pausedRef.current = paused;
|
pausedRef.current = paused;
|
||||||
}, [paused]);
|
}, [paused]);
|
||||||
|
|
||||||
|
const cancelFlush = useCallback(() => {
|
||||||
|
if (flushHandleRef.current !== null) {
|
||||||
|
cancelAnimationFrame(flushHandleRef.current);
|
||||||
|
flushHandleRef.current = null;
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const scheduleFlush = useCallback(() => {
|
||||||
|
if (flushHandleRef.current !== null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
flushHandleRef.current = requestAnimationFrame(() => {
|
||||||
|
flushHandleRef.current = null;
|
||||||
|
const buffered = pendingRef.current;
|
||||||
|
if (buffered.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pendingRef.current = [];
|
||||||
|
|
||||||
|
const pendingCount = pendingCountRef.current;
|
||||||
|
pendingCountRef.current = 0;
|
||||||
|
|
||||||
|
if (config.onNewItems && pendingCount > 0) {
|
||||||
|
config.onNewItems(pendingCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.captureScroll) {
|
||||||
|
config.captureScroll();
|
||||||
|
}
|
||||||
|
|
||||||
|
setItems((prev) => mergeNewest(buffered, prev));
|
||||||
|
setLastUpdate(Date.now());
|
||||||
|
});
|
||||||
|
}, [config.captureScroll, config.onNewItems]);
|
||||||
|
|
||||||
const togglePause = useCallback(() => {
|
const togglePause = useCallback(() => {
|
||||||
setPaused((prev) => {
|
setPaused((prev) => {
|
||||||
const next = !prev;
|
const next = !prev;
|
||||||
|
|
@ -648,10 +782,13 @@ const useLiveStream = <T extends SortableItem>(
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!enabled) {
|
if (!config.enabled) {
|
||||||
setStatus("disconnected");
|
setStatus("disconnected");
|
||||||
setItems([]);
|
setItems([]);
|
||||||
setLastUpdate(null);
|
setLastUpdate(null);
|
||||||
|
pendingRef.current = [];
|
||||||
|
pendingCountRef.current = 0;
|
||||||
|
cancelFlush();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -664,7 +801,7 @@ const useLiveStream = <T extends SortableItem>(
|
||||||
|
|
||||||
setStatus("connecting");
|
setStatus("connecting");
|
||||||
|
|
||||||
const socket = new WebSocket(buildWsUrl(wsPath));
|
const socket = new WebSocket(buildWsUrl(config.wsPath));
|
||||||
socketRef.current = socket;
|
socketRef.current = socket;
|
||||||
|
|
||||||
socket.onopen = () => {
|
socket.onopen = () => {
|
||||||
|
|
@ -681,7 +818,7 @@ const useLiveStream = <T extends SortableItem>(
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const message = JSON.parse(event.data) as StreamMessage<T>;
|
const message = JSON.parse(event.data) as StreamMessage<T>;
|
||||||
if (!message || message.type !== expectedType) {
|
if (!message || message.type !== config.expectedType) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -691,12 +828,9 @@ const useLiveStream = <T extends SortableItem>(
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (onNewItems) {
|
pendingRef.current.push(message.payload);
|
||||||
onNewItems(1);
|
pendingCountRef.current += 1;
|
||||||
}
|
scheduleFlush();
|
||||||
|
|
||||||
setItems((prev) => mergeNewest([message.payload], prev));
|
|
||||||
setLastUpdate(Date.now());
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn("Failed to parse live stream payload", error);
|
console.warn("Failed to parse live stream payload", error);
|
||||||
}
|
}
|
||||||
|
|
@ -727,6 +861,7 @@ const useLiveStream = <T extends SortableItem>(
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
active = false;
|
active = false;
|
||||||
|
cancelFlush();
|
||||||
if (reconnectRef.current !== null) {
|
if (reconnectRef.current !== null) {
|
||||||
window.clearTimeout(reconnectRef.current);
|
window.clearTimeout(reconnectRef.current);
|
||||||
}
|
}
|
||||||
|
|
@ -734,7 +869,7 @@ const useLiveStream = <T extends SortableItem>(
|
||||||
socketRef.current.close();
|
socketRef.current.close();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}, [enabled, expectedType, wsPath, onNewItems]);
|
}, [config.enabled, config.expectedType, config.wsPath, scheduleFlush, cancelFlush]);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
status,
|
status,
|
||||||
|
|
@ -750,9 +885,16 @@ const useLiveStream = <T extends SortableItem>(
|
||||||
|
|
||||||
const useFlowStream = (
|
const useFlowStream = (
|
||||||
enabled: boolean,
|
enabled: boolean,
|
||||||
onNewItems?: (count: number) => void
|
onNewItems?: (count: number) => void,
|
||||||
|
captureScroll?: () => void
|
||||||
): TapeState<FlowPacket> => {
|
): TapeState<FlowPacket> => {
|
||||||
return useLiveStream<FlowPacket>(enabled, "/ws/flow", "flow-packet", onNewItems);
|
return useLiveStream<FlowPacket>({
|
||||||
|
enabled,
|
||||||
|
wsPath: "/ws/flow",
|
||||||
|
expectedType: "flow-packet",
|
||||||
|
onNewItems,
|
||||||
|
captureScroll
|
||||||
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
type TapeStatusProps = {
|
type TapeStatusProps = {
|
||||||
|
|
@ -1001,6 +1143,15 @@ export default function HomePage() {
|
||||||
const alertsScroll = useListScroll();
|
const alertsScroll = useListScroll();
|
||||||
const classifierScroll = useListScroll();
|
const classifierScroll = useListScroll();
|
||||||
|
|
||||||
|
const optionsAnchor = useScrollAnchor(optionsScroll.listRef, optionsScroll.isAtTopRef);
|
||||||
|
const equitiesAnchor = useScrollAnchor(equitiesScroll.listRef, equitiesScroll.isAtTopRef);
|
||||||
|
const flowAnchor = useScrollAnchor(flowScroll.listRef, flowScroll.isAtTopRef);
|
||||||
|
const alertsAnchor = useScrollAnchor(alertsScroll.listRef, alertsScroll.isAtTopRef);
|
||||||
|
const classifierAnchor = useScrollAnchor(
|
||||||
|
classifierScroll.listRef,
|
||||||
|
classifierScroll.isAtTopRef
|
||||||
|
);
|
||||||
|
|
||||||
const options = useTape<OptionPrint>({
|
const options = useTape<OptionPrint>({
|
||||||
mode,
|
mode,
|
||||||
wsPath: "/ws/options",
|
wsPath: "/ws/options",
|
||||||
|
|
@ -1009,6 +1160,7 @@ export default function HomePage() {
|
||||||
expectedType: "option-print",
|
expectedType: "option-print",
|
||||||
batchSize: mode === "replay" ? 120 : undefined,
|
batchSize: mode === "replay" ? 120 : undefined,
|
||||||
pollMs: mode === "replay" ? 200 : undefined,
|
pollMs: mode === "replay" ? 200 : undefined,
|
||||||
|
captureScroll: optionsAnchor.capture,
|
||||||
onNewItems: optionsScroll.onNewItems
|
onNewItems: optionsScroll.onNewItems
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -1020,22 +1172,45 @@ export default function HomePage() {
|
||||||
expectedType: "equity-print",
|
expectedType: "equity-print",
|
||||||
batchSize: mode === "replay" ? 120 : undefined,
|
batchSize: mode === "replay" ? 120 : undefined,
|
||||||
pollMs: mode === "replay" ? 200 : undefined,
|
pollMs: mode === "replay" ? 200 : undefined,
|
||||||
|
captureScroll: equitiesAnchor.capture,
|
||||||
onNewItems: equitiesScroll.onNewItems
|
onNewItems: equitiesScroll.onNewItems
|
||||||
});
|
});
|
||||||
|
|
||||||
const flow = useFlowStream(mode === "live", flowScroll.onNewItems);
|
const flow = useFlowStream(mode === "live", flowScroll.onNewItems, flowAnchor.capture);
|
||||||
const alerts = useLiveStream<AlertEvent>(
|
const alerts = useLiveStream<AlertEvent>({
|
||||||
mode === "live",
|
enabled: mode === "live",
|
||||||
"/ws/alerts",
|
wsPath: "/ws/alerts",
|
||||||
"alert",
|
expectedType: "alert",
|
||||||
alertsScroll.onNewItems
|
onNewItems: alertsScroll.onNewItems,
|
||||||
);
|
captureScroll: alertsAnchor.capture
|
||||||
const classifierHits = useLiveStream<ClassifierHitEvent>(
|
});
|
||||||
mode === "live",
|
const classifierHits = useLiveStream<ClassifierHitEvent>({
|
||||||
"/ws/classifier-hits",
|
enabled: mode === "live",
|
||||||
"classifier-hit",
|
wsPath: "/ws/classifier-hits",
|
||||||
classifierScroll.onNewItems
|
expectedType: "classifier-hit",
|
||||||
);
|
onNewItems: classifierScroll.onNewItems,
|
||||||
|
captureScroll: classifierAnchor.capture
|
||||||
|
});
|
||||||
|
|
||||||
|
useLayoutEffect(() => {
|
||||||
|
optionsAnchor.apply();
|
||||||
|
}, [options.items, optionsAnchor.apply]);
|
||||||
|
|
||||||
|
useLayoutEffect(() => {
|
||||||
|
equitiesAnchor.apply();
|
||||||
|
}, [equities.items, equitiesAnchor.apply]);
|
||||||
|
|
||||||
|
useLayoutEffect(() => {
|
||||||
|
flowAnchor.apply();
|
||||||
|
}, [flow.items, flowAnchor.apply]);
|
||||||
|
|
||||||
|
useLayoutEffect(() => {
|
||||||
|
alertsAnchor.apply();
|
||||||
|
}, [alerts.items, alertsAnchor.apply]);
|
||||||
|
|
||||||
|
useLayoutEffect(() => {
|
||||||
|
classifierAnchor.apply();
|
||||||
|
}, [classifierHits.items, classifierAnchor.apply]);
|
||||||
|
|
||||||
const activeTickers = useMemo(() => {
|
const activeTickers = useMemo(() => {
|
||||||
const parts = filterInput
|
const parts = filterInput
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,22 @@ const envSchema = z.object({
|
||||||
CLICKHOUSE_URL: z.string().default("http://localhost:8123"),
|
CLICKHOUSE_URL: z.string().default("http://localhost:8123"),
|
||||||
CLICKHOUSE_DATABASE: z.string().default("default"),
|
CLICKHOUSE_DATABASE: z.string().default("default"),
|
||||||
EQUITIES_INGEST_ADAPTER: z.string().min(1).default("synthetic"),
|
EQUITIES_INGEST_ADAPTER: z.string().min(1).default("synthetic"),
|
||||||
EMIT_INTERVAL_MS: z.coerce.number().int().positive().default(1000)
|
EMIT_INTERVAL_MS: z.coerce.number().int().positive().default(1000),
|
||||||
|
TESTING_MODE: z
|
||||||
|
.preprocess((value) => {
|
||||||
|
if (typeof value === "string") {
|
||||||
|
const normalized = value.trim().toLowerCase();
|
||||||
|
if (["1", "true", "yes", "on"].includes(normalized)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (["0", "false", "no", "off"].includes(normalized)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}, z.boolean())
|
||||||
|
.default(false),
|
||||||
|
TESTING_THROTTLE_MS: z.coerce.number().int().nonnegative().default(200)
|
||||||
});
|
});
|
||||||
|
|
||||||
const env = readEnv(envSchema);
|
const env = readEnv(envSchema);
|
||||||
|
|
@ -34,6 +49,34 @@ const state = {
|
||||||
shuttingDown: false
|
shuttingDown: false
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const buildThrottle = (enabled: boolean, throttleMs: number) => {
|
||||||
|
if (!enabled || throttleMs <= 0) {
|
||||||
|
return () => true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let lastEmit = 0;
|
||||||
|
let dropped = 0;
|
||||||
|
let lastLog = Date.now();
|
||||||
|
|
||||||
|
return (now: number) => {
|
||||||
|
if (now - lastEmit < throttleMs) {
|
||||||
|
dropped += 1;
|
||||||
|
if (now - lastLog > 5000) {
|
||||||
|
logger.warn("testing mode throttling equity prints", {
|
||||||
|
dropped,
|
||||||
|
throttle_ms: throttleMs
|
||||||
|
});
|
||||||
|
dropped = 0;
|
||||||
|
lastLog = now;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
lastEmit = now;
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
const retry = async <T>(
|
const retry = async <T>(
|
||||||
label: string,
|
label: string,
|
||||||
attempts: number,
|
attempts: number,
|
||||||
|
|
@ -104,6 +147,7 @@ const run = async () => {
|
||||||
|
|
||||||
const adapter = selectAdapter(env.EQUITIES_INGEST_ADAPTER);
|
const adapter = selectAdapter(env.EQUITIES_INGEST_ADAPTER);
|
||||||
logger.info("ingest adapter selected", { adapter: adapter.name });
|
logger.info("ingest adapter selected", { adapter: adapter.name });
|
||||||
|
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
|
||||||
|
|
||||||
const stopAdapter: StopHandler = await adapter.start({
|
const stopAdapter: StopHandler = await adapter.start({
|
||||||
onTrade: async (candidate: EquityPrint) => {
|
onTrade: async (candidate: EquityPrint) => {
|
||||||
|
|
@ -111,6 +155,11 @@ const run = async () => {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
if (!allowPublish(now)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const print = EquityPrintSchema.parse(candidate);
|
const print = EquityPrintSchema.parse(candidate);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
|
||||||
|
|
@ -62,7 +62,22 @@ const envSchema = z.object({
|
||||||
IBKR_EXCHANGE: z.string().min(1).default("SMART"),
|
IBKR_EXCHANGE: z.string().min(1).default("SMART"),
|
||||||
IBKR_CURRENCY: z.string().min(1).default("USD"),
|
IBKR_CURRENCY: z.string().min(1).default("USD"),
|
||||||
IBKR_PYTHON_BIN: z.string().min(1).default("python3"),
|
IBKR_PYTHON_BIN: z.string().min(1).default("python3"),
|
||||||
EMIT_INTERVAL_MS: z.coerce.number().int().positive().default(1000)
|
EMIT_INTERVAL_MS: z.coerce.number().int().positive().default(1000),
|
||||||
|
TESTING_MODE: z
|
||||||
|
.preprocess((value) => {
|
||||||
|
if (typeof value === "string") {
|
||||||
|
const normalized = value.trim().toLowerCase();
|
||||||
|
if (["1", "true", "yes", "on"].includes(normalized)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (["0", "false", "no", "off"].includes(normalized)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}, z.boolean())
|
||||||
|
.default(false),
|
||||||
|
TESTING_THROTTLE_MS: z.coerce.number().int().nonnegative().default(200)
|
||||||
});
|
});
|
||||||
|
|
||||||
const env = readEnv(envSchema);
|
const env = readEnv(envSchema);
|
||||||
|
|
@ -71,6 +86,34 @@ const state = {
|
||||||
shuttingDown: false
|
shuttingDown: false
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const buildThrottle = (enabled: boolean, throttleMs: number) => {
|
||||||
|
if (!enabled || throttleMs <= 0) {
|
||||||
|
return () => true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let lastEmit = 0;
|
||||||
|
let dropped = 0;
|
||||||
|
let lastLog = Date.now();
|
||||||
|
|
||||||
|
return (now: number) => {
|
||||||
|
if (now - lastEmit < throttleMs) {
|
||||||
|
dropped += 1;
|
||||||
|
if (now - lastLog > 5000) {
|
||||||
|
logger.warn("testing mode throttling option prints", {
|
||||||
|
dropped,
|
||||||
|
throttle_ms: throttleMs
|
||||||
|
});
|
||||||
|
dropped = 0;
|
||||||
|
lastLog = now;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
lastEmit = now;
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
const retry = async <T>(
|
const retry = async <T>(
|
||||||
label: string,
|
label: string,
|
||||||
attempts: number,
|
attempts: number,
|
||||||
|
|
@ -205,6 +248,7 @@ const run = async () => {
|
||||||
|
|
||||||
const adapter = selectAdapter(env.OPTIONS_INGEST_ADAPTER);
|
const adapter = selectAdapter(env.OPTIONS_INGEST_ADAPTER);
|
||||||
logger.info("ingest adapter selected", { adapter: adapter.name });
|
logger.info("ingest adapter selected", { adapter: adapter.name });
|
||||||
|
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
|
||||||
|
|
||||||
const stopAdapter: StopHandler = await adapter.start({
|
const stopAdapter: StopHandler = await adapter.start({
|
||||||
onTrade: async (candidate: OptionPrint) => {
|
onTrade: async (candidate: OptionPrint) => {
|
||||||
|
|
@ -212,6 +256,11 @@ const run = async () => {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
if (!allowPublish(now)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const print = OptionPrintSchema.parse(candidate);
|
const print = OptionPrintSchema.parse(candidate);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue