Restore continuous live tape history

This commit is contained in:
dirtydishes 2026-05-07 00:39:26 -04:00
parent d81b4c0cfb
commit 034d24f8ac
5 changed files with 483 additions and 117 deletions

View file

@ -1,4 +1,5 @@
import { describe, expect, it } from "bun:test";
import { getSubscriptionKey as getLiveSubscriptionKey } from "@islandflow/types";
import {
NAV_ITEMS,
appendHistoryTail,
@ -10,10 +11,12 @@ import {
formatOptionContractLabel,
flushPausableTapeData,
getAlertWindowAnchorTs,
getScopedLiveAutoHydrationChannels,
getLiveHistoryRetentionCap,
getOptionTableSnapshot,
getLiveFeedStatus,
getLiveManifest,
mergeNewestWithOverflow,
normalizeAlertSeverity,
nextFlowFilterPopoverState,
projectPausableTapeState,
@ -243,6 +246,36 @@ describe("live tape pausable helpers", () => {
});
describe("live tape history helpers", () => {
it("promotes hot-window overflow into the history tail", () => {
const currentHot = [makeItem("hot-3", 3, 300), makeItem("hot-2", 2, 200), makeItem("hot-1", 1, 100)];
const incoming = [makeItem("hot-4", 4, 400)];
const { kept, evicted } = mergeNewestWithOverflow(incoming, currentHot, 3);
const nextHistory = appendHistoryTail([], evicted, kept, 5000);
expect(kept.map((item) => item.trace_id)).toEqual(["hot-4", "hot-3", "hot-2"]);
expect(nextHistory.map((item) => item.trace_id)).toEqual(["hot-1"]);
});
it("keeps the combined tape continuous beyond the hot live window", () => {
let hot: Array<ReturnType<typeof makeItem>> = [];
let history: Array<ReturnType<typeof makeItem>> = [];
for (let seq = 1; seq <= 5; seq += 1) {
const { kept, evicted } = mergeNewestWithOverflow([makeItem(`row-${seq}`, seq, seq * 100)], hot, 2);
hot = kept;
history = appendHistoryTail(history, evicted, hot, 5000);
}
expect([...hot, ...history].map((item) => item.trace_id)).toEqual([
"row-5",
"row-4",
"row-3",
"row-2",
"row-1"
]);
});
it("appends older scoped rows behind the hot live head", () => {
const liveHead = Array.from({ length: 100 }, (_, idx) =>
makeItem(`hot-${idx}`, 200 - idx, 2_000 - idx)
@ -263,6 +296,16 @@ describe("live tape history helpers", () => {
expect(next.map((item) => item.trace_id)).toEqual(["older"]);
});
it("dedupes the seam between promoted overflow and fetched history", () => {
const currentHot = [makeItem("hot-3", 3, 300), makeItem("hot-2", 2, 200), makeItem("hot-1", 1, 100)];
const { kept, evicted } = mergeNewestWithOverflow([makeItem("hot-4", 4, 400)], currentHot, 3);
const promoted = appendHistoryTail([], evicted, kept, 5000);
const merged = appendHistoryTail(promoted, [makeItem("hot-1", 1, 100), makeItem("older", 0, 50)], kept, 5000);
expect(merged.map((item) => item.trace_id)).toEqual(["hot-1", "older"]);
expect(new Set([...kept, ...merged].map((item) => item.trace_id)).size).toBe(kept.length + merged.length);
});
it("trims the history tail to the soft cap", () => {
const current = [makeItem("existing", 4, 400)];
const older = [makeItem("older-1", 3, 300), makeItem("older-2", 2, 200)];
@ -287,6 +330,38 @@ describe("live tape history helpers", () => {
} as any)
).toBeGreaterThan(0);
});
it("keeps auto-hydrating scoped live history while next_before exists", () => {
const manifest = getLiveManifest(
"/tape",
"AAPL",
60000,
buildDefaultFlowFilters(),
{
underlying_ids: ["AAPL"],
option_contract_id: "AAPL-2025-01-17-200-C"
},
{ underlying_ids: ["AAPL"] }
);
const historyCursors = Object.fromEntries(
manifest.map((subscription) => [getLiveSubscriptionKey(subscription), { ts: 1, seq: 1 }])
);
expect(
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {})
).toEqual(["options", "equities"]);
expect(
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {
[getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "options")!)]: true
})
).toEqual(["equities"]);
expect(
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, {
...historyCursors,
[getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "equities")!)]: null
}, {})
).toEqual(["options"]);
});
});
describe("options display formatters", () => {

View file

@ -368,15 +368,15 @@ const buildItemKey = (item: SortableItem): string | null => {
return null;
};
const mergeNewest = <T extends SortableItem>(
export const mergeNewestWithOverflow = <T extends SortableItem>(
incoming: T[],
existing: T[],
limit = LIVE_HOT_WINDOW,
onTrim?: (evicted: number) => void
): T[] => {
): { kept: T[]; evicted: T[] } => {
const combined = [...incoming, ...existing];
if (combined.length === 0) {
return combined;
return { kept: combined, evicted: [] };
}
const seen = new Set<string>();
@ -402,12 +402,24 @@ const mergeNewest = <T extends SortableItem>(
});
const safeLimit = Math.max(1, Math.floor(limit));
const evicted = Math.max(0, deduped.length - safeLimit);
if (evicted > 0) {
onTrim?.(evicted);
const evicted = deduped.slice(safeLimit);
if (evicted.length > 0) {
onTrim?.(evicted.length);
}
return deduped.slice(0, safeLimit);
return {
kept: deduped.slice(0, safeLimit),
evicted
};
};
const mergeNewest = <T extends SortableItem>(
incoming: T[],
existing: T[],
limit = LIVE_HOT_WINDOW,
onTrim?: (evicted: number) => void
): T[] => {
return mergeNewestWithOverflow(incoming, existing, limit, onTrim).kept;
};
const getTapeItemKey = (item: SortableItem): string => {
@ -520,25 +532,27 @@ export const appendHistoryTail = <T extends SortableItem>(
return current;
}
const seen = new Set<string>();
for (const item of liveHead) {
seen.add(getTapeItemKey(item));
}
for (const item of current) {
seen.add(getTapeItemKey(item));
}
const seen = new Set<string>(liveHead.map((item) => getTapeItemKey(item)));
const combined: T[] = [];
const appended = [...current];
for (const item of incoming) {
for (const item of [...current, ...incoming]) {
const key = getTapeItemKey(item);
if (seen.has(key)) {
continue;
}
seen.add(key);
appended.push(item);
combined.push(item);
}
return cap > 0 ? appended.slice(0, cap) : appended;
combined.sort((a, b) => {
const delta = extractSortTs(b) - extractSortTs(a);
if (delta !== 0) {
return delta;
}
return extractSortSeq(b) - extractSortSeq(a);
});
return cap > 0 ? combined.slice(0, cap) : combined;
};
export const getLiveHistoryRetentionCap = (subscription: LiveSubscription): number => {
@ -551,6 +565,36 @@ export const getLiveHistoryRetentionCap = (subscription: LiveSubscription): numb
}
};
export const getScopedLiveAutoHydrationChannels = (
enabled: boolean,
pathname: string,
manifest: LiveSubscription[],
historyCursors: Partial<Record<string, Cursor | null>>,
historyLoading: Partial<Record<string, boolean>>
): Array<Extract<LiveSubscription["channel"], "options" | "equities">> => {
if (!enabled || pathname !== "/tape") {
return [];
}
const channels: Array<Extract<LiveSubscription["channel"], "options" | "equities">> = [];
for (const subscription of manifest) {
const scoped =
(subscription.channel === "options" &&
(subscription.underlying_ids?.length || subscription.option_contract_id)) ||
(subscription.channel === "equities" && subscription.underlying_ids?.length);
if (!scoped) {
continue;
}
const key = getLiveSubscriptionKey(subscription);
if (historyCursors[key] && !historyLoading[key]) {
channels.push(subscription.channel);
}
}
return channels;
};
export const getLiveFeedStatus = (
sourceStatus: WsStatus,
freshestTs: number | null,
@ -2544,6 +2588,27 @@ const useLiveSession = (
const [inferredDarkHistory, setInferredDarkHistory] = useState<InferredDarkEvent[]>([]);
const [chartCandles, setChartCandles] = useState<EquityCandle[]>([]);
const [chartOverlay, setChartOverlay] = useState<EquityPrint[]>([]);
const optionsRef = useRef<OptionPrint[]>([]);
const nbboRef = useRef<OptionNBBO[]>([]);
const equitiesRef = useRef<EquityPrint[]>([]);
const equityQuotesRef = useRef<EquityQuote[]>([]);
const equityJoinsRef = useRef<EquityPrintJoin[]>([]);
const flowRef = useRef<FlowPacket[]>([]);
const smartMoneyRef = useRef<SmartMoneyEvent[]>([]);
const classifierHitsRef = useRef<ClassifierHitEvent[]>([]);
const alertsRef = useRef<AlertEvent[]>([]);
const inferredDarkRef = useRef<InferredDarkEvent[]>([]);
const chartCandlesRef = useRef<EquityCandle[]>([]);
const chartOverlayRef = useRef<EquityPrint[]>([]);
const optionsHistoryRef = useRef<OptionPrint[]>([]);
const nbboHistoryRef = useRef<OptionNBBO[]>([]);
const equitiesHistoryRef = useRef<EquityPrint[]>([]);
const equityJoinsHistoryRef = useRef<EquityPrintJoin[]>([]);
const flowHistoryRef = useRef<FlowPacket[]>([]);
const smartMoneyHistoryRef = useRef<SmartMoneyEvent[]>([]);
const classifierHitsHistoryRef = useRef<ClassifierHitEvent[]>([]);
const alertsHistoryRef = useRef<AlertEvent[]>([]);
const inferredDarkHistoryRef = useRef<InferredDarkEvent[]>([]);
const socketRef = useRef<WebSocket | null>(null);
const reconnectRef = useRef<number | null>(null);
const idleWatchdogRef = useRef<number | null>(null);
@ -2556,6 +2621,27 @@ const useLiveSession = (
[pathname, chartTicker, chartIntervalMs, flowFilters, optionScope, equityScope]
);
const replaceArrayState = <T,>(
setter: Dispatch<SetStateAction<T[]>>,
ref: { current: T[] },
next: T[]
): void => {
ref.current = next;
setter(next);
};
const mergeHistoryState = <T extends SortableItem>(
setter: Dispatch<SetStateAction<T[]>>,
ref: { current: T[] },
incoming: T[],
liveHead: T[],
cap = LIVE_HISTORY_SOFT_CAP
): void => {
const next = appendHistoryTail(ref.current, incoming, liveHead, cap);
ref.current = next;
setter(next);
};
useEffect(() => {
if (!enabled) {
setStatus("disconnected");
@ -2586,6 +2672,27 @@ const useLiveSession = (
setInferredDarkHistory([]);
setChartCandles([]);
setChartOverlay([]);
optionsRef.current = [];
nbboRef.current = [];
equitiesRef.current = [];
equityQuotesRef.current = [];
equityJoinsRef.current = [];
flowRef.current = [];
smartMoneyRef.current = [];
classifierHitsRef.current = [];
alertsRef.current = [];
inferredDarkRef.current = [];
chartCandlesRef.current = [];
chartOverlayRef.current = [];
optionsHistoryRef.current = [];
nbboHistoryRef.current = [];
equitiesHistoryRef.current = [];
equityJoinsHistoryRef.current = [];
flowHistoryRef.current = [];
smartMoneyHistoryRef.current = [];
classifierHitsHistoryRef.current = [];
alertsHistoryRef.current = [];
inferredDarkHistoryRef.current = [];
subscribedKeysRef.current = new Set();
subscribedMapRef.current = new Map();
if (socketRef.current) {
@ -2642,62 +2749,112 @@ const useLiveSession = (
const updateAt = Date.now();
const mergeItems = <T extends SortableItem>(
setter: React.Dispatch<React.SetStateAction<T[]>>,
setter: Dispatch<SetStateAction<T[]>>,
ref: { current: T[] },
nextItems: T[],
retentionLimit = LIVE_HOT_WINDOW
retentionLimit = LIVE_HOT_WINDOW,
history?: {
setter: Dispatch<SetStateAction<T[]>>;
ref: { current: T[] };
cap?: number;
}
) => {
setter((prev) =>
message.op === "snapshot"
? shouldRetainLiveSnapshotHistory(
subscription.channel,
true,
nextItems.length,
prev.length
)
? prev
: (nextItems as T[])
: mergeNewest(nextItems as T[], prev, retentionLimit, (evicted) =>
incrementRetentionMetric("hotWindowEvictions", evicted)
)
if (message.op === "snapshot") {
const next = shouldRetainLiveSnapshotHistory(
subscription.channel,
true,
nextItems.length,
ref.current.length
)
? ref.current
: nextItems;
replaceArrayState(setter, ref, next);
return;
}
const { kept, evicted } = mergeNewestWithOverflow(
nextItems,
ref.current,
retentionLimit,
(evictedCount) => incrementRetentionMetric("hotWindowEvictions", evictedCount)
);
replaceArrayState(setter, ref, kept);
if (history && evicted.length > 0) {
mergeHistoryState(history.setter, history.ref, evicted, kept, history.cap);
}
};
switch (subscription.channel) {
case "options":
mergeItems(setOptions, items as OptionPrint[], LIVE_HOT_WINDOW_OPTIONS);
mergeItems(setOptions, optionsRef, items as OptionPrint[], LIVE_HOT_WINDOW_OPTIONS, {
setter: setOptionsHistory,
ref: optionsHistoryRef,
cap: getLiveHistoryRetentionCap(subscription)
});
break;
case "nbbo":
mergeItems(setNbbo, items as OptionNBBO[]);
mergeItems(setNbbo, nbboRef, items as OptionNBBO[], LIVE_HOT_WINDOW, {
setter: setNbboHistory,
ref: nbboHistoryRef
});
break;
case "equities":
mergeItems(setEquities, items as EquityPrint[]);
mergeItems(setEquities, equitiesRef, items as EquityPrint[], LIVE_HOT_WINDOW, {
setter: setEquitiesHistory,
ref: equitiesHistoryRef,
cap: getLiveHistoryRetentionCap(subscription)
});
break;
case "equity-quotes":
mergeItems(setEquityQuotes, items as EquityQuote[]);
mergeItems(setEquityQuotes, equityQuotesRef, items as EquityQuote[]);
break;
case "equity-joins":
mergeItems(setEquityJoins, items as EquityPrintJoin[]);
mergeItems(setEquityJoins, equityJoinsRef, items as EquityPrintJoin[], LIVE_HOT_WINDOW, {
setter: setEquityJoinsHistory,
ref: equityJoinsHistoryRef
});
break;
case "flow":
mergeItems(setFlow, items as FlowPacket[]);
mergeItems(setFlow, flowRef, items as FlowPacket[], LIVE_HOT_WINDOW, {
setter: setFlowHistory,
ref: flowHistoryRef
});
break;
case "smart-money":
mergeItems(setSmartMoney, items as SmartMoneyEvent[]);
mergeItems(setSmartMoney, smartMoneyRef, items as SmartMoneyEvent[], LIVE_HOT_WINDOW, {
setter: setSmartMoneyHistory,
ref: smartMoneyHistoryRef
});
break;
case "classifier-hits":
mergeItems(setClassifierHits, items as ClassifierHitEvent[]);
mergeItems(
setClassifierHits,
classifierHitsRef,
items as ClassifierHitEvent[],
LIVE_HOT_WINDOW,
{
setter: setClassifierHitsHistory,
ref: classifierHitsHistoryRef
}
);
break;
case "alerts":
mergeItems(setAlerts, items as AlertEvent[]);
mergeItems(setAlerts, alertsRef, items as AlertEvent[], LIVE_HOT_WINDOW, {
setter: setAlertsHistory,
ref: alertsHistoryRef
});
break;
case "inferred-dark":
mergeItems(setInferredDark, items as InferredDarkEvent[]);
mergeItems(setInferredDark, inferredDarkRef, items as InferredDarkEvent[], LIVE_HOT_WINDOW, {
setter: setInferredDarkHistory,
ref: inferredDarkHistoryRef
});
break;
case "equity-candles":
mergeItems(setChartCandles, items as EquityCandle[]);
mergeItems(setChartCandles, chartCandlesRef, items as EquityCandle[]);
break;
case "equity-overlay":
mergeItems(setChartOverlay, items as EquityPrint[]);
mergeItems(setChartOverlay, chartOverlayRef, items as EquityPrint[]);
break;
}
@ -2839,10 +2996,14 @@ const useLiveSession = (
.filter((channel) => channel === "options" || channel === "equities")
);
if (resetScopedChannels.has("options")) {
optionsRef.current = [];
optionsHistoryRef.current = [];
setOptions([]);
setOptionsHistory([]);
}
if (resetScopedChannels.has("equities")) {
equitiesRef.current = [];
equitiesHistoryRef.current = [];
setEquities([]);
setEquitiesHistory([]);
}
@ -2926,41 +3087,56 @@ const useLiveSession = (
const mergeOlder = <T extends SortableItem>(
setter: Dispatch<SetStateAction<T[]>>,
ref: { current: T[] },
liveHead: T[],
cap = LIVE_HISTORY_SOFT_CAP
) => {
setter((prev) => appendHistoryTail(prev, older as T[], liveHead, cap));
mergeHistoryState(setter, ref, older as T[], liveHead, cap);
};
switch (subscription.channel) {
case "options":
mergeOlder(setOptionsHistory, options, getLiveHistoryRetentionCap(subscription));
mergeOlder(
setOptionsHistory,
optionsHistoryRef,
optionsRef.current,
getLiveHistoryRetentionCap(subscription)
);
break;
case "nbbo":
mergeOlder(setNbboHistory, nbbo);
mergeOlder(setNbboHistory, nbboHistoryRef, nbboRef.current);
break;
case "equities":
mergeOlder(setEquitiesHistory, equities, getLiveHistoryRetentionCap(subscription));
mergeOlder(
setEquitiesHistory,
equitiesHistoryRef,
equitiesRef.current,
getLiveHistoryRetentionCap(subscription)
);
break;
case "equity-quotes":
break;
case "equity-joins":
mergeOlder(setEquityJoinsHistory, equityJoins);
mergeOlder(setEquityJoinsHistory, equityJoinsHistoryRef, equityJoinsRef.current);
break;
case "flow":
mergeOlder(setFlowHistory, flow);
mergeOlder(setFlowHistory, flowHistoryRef, flowRef.current);
break;
case "smart-money":
mergeOlder(setSmartMoneyHistory, smartMoney);
mergeOlder(setSmartMoneyHistory, smartMoneyHistoryRef, smartMoneyRef.current);
break;
case "classifier-hits":
mergeOlder(setClassifierHitsHistory, classifierHits);
mergeOlder(
setClassifierHitsHistory,
classifierHitsHistoryRef,
classifierHitsRef.current
);
break;
case "alerts":
mergeOlder(setAlertsHistory, alerts);
mergeOlder(setAlertsHistory, alertsHistoryRef, alertsRef.current);
break;
case "inferred-dark":
mergeOlder(setInferredDarkHistory, inferredDark);
mergeOlder(setInferredDarkHistory, inferredDarkHistoryRef, inferredDarkRef.current);
break;
}
@ -2978,41 +3154,18 @@ const useLiveSession = (
setHistoryLoading((current) => ({ ...current, [key]: false }));
}
},
[
enabled,
manifest,
historyCursors,
historyLoading,
options,
nbbo,
equities,
equityJoins,
flow,
smartMoney,
classifierHits,
alerts,
inferredDark
]
[enabled, manifest, historyCursors, historyLoading]
);
useEffect(() => {
if (!enabled || pathname !== "/tape") {
return;
}
const scoped = manifest.filter(
(subscription) =>
(subscription.channel === "options" &&
(subscription.underlying_ids?.length || subscription.option_contract_id)) ||
(subscription.channel === "equities" && subscription.underlying_ids?.length)
);
if (scoped.length === 0) {
return;
}
for (const subscription of scoped) {
const key = getLiveSubscriptionKey(subscription);
if (historyCursors[key] && !historyLoading[key]) {
void loadOlder(subscription.channel);
}
for (const channel of getScopedLiveAutoHydrationChannels(
enabled,
pathname,
manifest,
historyCursors,
historyLoading
)) {
void loadOlder(channel);
}
}, [enabled, pathname, manifest, historyCursors, historyLoading, loadOlder]);