Fix live tape scroll hold and lazy history
This commit is contained in:
parent
eaddf4b7a0
commit
39fb5ce9f1
6 changed files with 332 additions and 75 deletions
|
|
@ -10,7 +10,7 @@
|
||||||
{"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
{"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
{"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
{"_type":"issue","id":"islandflow-0sa","title":"Fix live tape auto-hold, history seam, and remove manual pause control","description":"The live tape should automatically hold when the user scrolls away from the top, resume when they return to the top or use Jump to top, and keep older prints available seamlessly beyond the hot window. Manual Pause/Resume control is now redundant and should be removed from live tape panes. This work should also fix the current regression where paused/held tapes still mutate, and align the options tape with a strict 100-row hot head backed by ClickHouse history.","status":"in_progress","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T18:12:51Z","created_by":"dirtydishes","updated_at":"2026-05-16T18:12:54Z","started_at":"2026-05-16T18:12:54Z","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-0sa","title":"Fix live tape auto-hold, history seam, and remove manual pause control","description":"The live tape should automatically hold when the user scrolls away from the top, resume when they return to the top or use Jump to top, and keep older prints available seamlessly beyond the hot window. Manual Pause/Resume control is now redundant and should be removed from live tape panes. This work should also fix the current regression where paused/held tapes still mutate, and align the options tape with a strict 100-row hot head backed by ClickHouse history.","notes":"Implemented live scroll-hold with no live pause button, demand-loaded ClickHouse history, a 100-row options hot head, and cache-first scoped snapshots. Validated with bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts and bun --cwd=apps/web run build.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T18:12:51Z","created_by":"dirtydishes","updated_at":"2026-05-16T18:23:43Z","started_at":"2026-05-16T18:12:54Z","closed_at":"2026-05-16T18:23:43Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
{"_type":"issue","id":"islandflow-2db","title":"Manually remove stale islandflow local-infra containers from VPS","description":"The live VPS still has an older compose project named islandflow created from the repo-root docker-compose.yml. Inspection shows it is separate from the supported islandflow-vps deployment stack and exposes NATS, ClickHouse, and Redis on host ports. Container removal commands currently hang when run as the delta user through Docker, so cleanup likely needs a focused maintenance window and possibly host-level intervention or a Docker daemon restart.","notes":"The duplicate islandflow compose project on the VPS was confirmed live during inspection. Nginx Proxy Manager routes public traffic only to islandflow-vps web/api by Docker name, so the stale islandflow project appears to be stray local-infra state rather than part of the supported production path. Attempts to remove the stale containers with docker compose down and docker rm -f as the delta user hung and timed out, so manual cleanup likely needs a maintenance window and possibly Docker daemon intervention.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:27:27Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:28:59Z","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-2db","title":"Manually remove stale islandflow local-infra containers from VPS","description":"The live VPS still has an older compose project named islandflow created from the repo-root docker-compose.yml. Inspection shows it is separate from the supported islandflow-vps deployment stack and exposes NATS, ClickHouse, and Redis on host ports. Container removal commands currently hang when run as the delta user through Docker, so cleanup likely needs a focused maintenance window and possibly host-level intervention or a Docker daemon restart.","notes":"The duplicate islandflow compose project on the VPS was confirmed live during inspection. Nginx Proxy Manager routes public traffic only to islandflow-vps web/api by Docker name, so the stale islandflow project appears to be stray local-infra state rather than part of the supported production path. Attempts to remove the stale containers with docker compose down and docker rm -f as the delta user hung and timed out, so manual cleanup likely needs a maintenance window and possibly Docker daemon intervention.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:27:27Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:28:59Z","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
{"_type":"issue","id":"islandflow-c87","title":"Clean up duplicate Islandflow Docker infra on VPS","description":"The live VPS is currently running both the production-style islandflow-vps Docker stack and an older root-level islandflow infra stack that publishes NATS, ClickHouse, and Redis on host ports. Investigate whether the older stack is unused, remove it safely if so, and update docs/deploy guidance so the server topology is clearer.","notes":"Inspected the live VPS and confirmed the duplicate compose project: islandflow-vps is the supported deployment stack, while a separate islandflow project from the repo-root docker-compose.yml still runs exposed NATS/ClickHouse/Redis containers. Verified Nginx Proxy Manager routes only to islandflow-vps web/api by Docker name. Attempted cleanup via docker compose down and docker rm -f on the stale islandflow containers, but those commands hung for the delta user and timed out. Added repo guardrails and docs so deploy warns when the duplicate project exists, and opened islandflow-2db for manual host-level cleanup during a maintenance window.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:16:05Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:28:07Z","started_at":"2026-05-16T01:16:09Z","closed_at":"2026-05-16T01:28:07Z","close_reason":"Completed the repo-side investigation and guardrails. Actual server-side container removal is blocked by hanging Docker operations and is tracked separately in islandflow-2db for a maintenance window.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-c87","title":"Clean up duplicate Islandflow Docker infra on VPS","description":"The live VPS is currently running both the production-style islandflow-vps Docker stack and an older root-level islandflow infra stack that publishes NATS, ClickHouse, and Redis on host ports. Investigate whether the older stack is unused, remove it safely if so, and update docs/deploy guidance so the server topology is clearer.","notes":"Inspected the live VPS and confirmed the duplicate compose project: islandflow-vps is the supported deployment stack, while a separate islandflow project from the repo-root docker-compose.yml still runs exposed NATS/ClickHouse/Redis containers. Verified Nginx Proxy Manager routes only to islandflow-vps web/api by Docker name. Attempted cleanup via docker compose down and docker rm -f on the stale islandflow containers, but those commands hung for the delta user and timed out. Added repo guardrails and docs so deploy warns when the duplicate project exists, and opened islandflow-2db for manual host-level cleanup during a maintenance window.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:16:05Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:28:07Z","started_at":"2026-05-16T01:16:09Z","closed_at":"2026-05-16T01:28:07Z","close_reason":"Completed the repo-side investigation and guardrails. Actual server-side container removal is blocked by hanging Docker operations and is tracked separately in islandflow-2db for a maintenance window.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
{"_type":"issue","id":"islandflow-4gj","title":"Clarify Docker-first deploy workflow and mark native runtime experimental","description":"After inspecting the live VPS, native deployment is not ready for routine use: Nginx Proxy Manager routes to Docker container names, Bun is not installed on the host, sudo systemctl is not passwordless, and no Islandflow units exist. Update deploy messaging and docs so Docker remains the clearly recommended deployment path and native runtime is labeled experimental/future-facing with server prerequisites called out.","notes":"Updated deploy messaging and docs after live VPS inspection. scripts/deploy.ts now marks Docker as the default and recommended runtime, labels native as experimental, switches native systemctl default to sudo -n systemctl, and prints explicit native precheck failures for missing Bun/systemctl access/units. Updated README.md, deployment/docker/README.md, and deployment/native/README.md to reflect the current Docker + Nginx Proxy Manager topology. Validation: ./deploy --help, ./deploy main --runtime native --no-build (fails fast with Bun-missing message), bun run check:docker-workspace.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:10:11Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:12:39Z","started_at":"2026-05-16T01:10:14Z","closed_at":"2026-05-16T01:12:39Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-4gj","title":"Clarify Docker-first deploy workflow and mark native runtime experimental","description":"After inspecting the live VPS, native deployment is not ready for routine use: Nginx Proxy Manager routes to Docker container names, Bun is not installed on the host, sudo systemctl is not passwordless, and no Islandflow units exist. Update deploy messaging and docs so Docker remains the clearly recommended deployment path and native runtime is labeled experimental/future-facing with server prerequisites called out.","notes":"Updated deploy messaging and docs after live VPS inspection. scripts/deploy.ts now marks Docker as the default and recommended runtime, labels native as experimental, switches native systemctl default to sudo -n systemctl, and prints explicit native precheck failures for missing Bun/systemctl access/units. Updated README.md, deployment/docker/README.md, and deployment/native/README.md to reflect the current Docker + Nginx Proxy Manager topology. Validation: ./deploy --help, ./deploy main --runtime native --no-build (fails fast with Bun-missing message), bun run check:docker-workspace.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:10:11Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:12:39Z","started_at":"2026-05-16T01:10:14Z","closed_at":"2026-05-16T01:12:39Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
|
|
|
||||||
|
|
@ -164,6 +164,7 @@ describe("live manifest", () => {
|
||||||
|
|
||||||
expect(optionsSubscription?.underlying_ids).toEqual(["AAPL"]);
|
expect(optionsSubscription?.underlying_ids).toEqual(["AAPL"]);
|
||||||
expect(optionsSubscription?.option_contract_id).toBe("AAPL-2025-01-17-200-C");
|
expect(optionsSubscription?.option_contract_id).toBe("AAPL-2025-01-17-200-C");
|
||||||
|
expect(optionsSubscription?.snapshot_limit).toBe(100);
|
||||||
expect(equitiesSubscription?.underlying_ids).toEqual(["AAPL"]);
|
expect(equitiesSubscription?.underlying_ids).toEqual(["AAPL"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -635,23 +636,23 @@ describe("live tape history helpers", () => {
|
||||||
expect(next.map((item) => item.trace_id)).toEqual(["existing", "older-1"]);
|
expect(next.map((item) => item.trace_id)).toEqual(["existing", "older-1"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps scoped option and equity history on the normal retention cap", () => {
|
it("keeps option and equity history effectively unbounded while scrolling", () => {
|
||||||
expect(
|
expect(
|
||||||
getLiveHistoryRetentionCap({
|
getLiveHistoryRetentionCap({
|
||||||
channel: "options",
|
channel: "options",
|
||||||
underlying_ids: ["AAPL"],
|
underlying_ids: ["AAPL"],
|
||||||
option_contract_id: "AAPL-2025-01-17-200-C"
|
option_contract_id: "AAPL-2025-01-17-200-C"
|
||||||
} as any)
|
} as any)
|
||||||
).toBeGreaterThan(0);
|
).toBe(0);
|
||||||
expect(
|
expect(
|
||||||
getLiveHistoryRetentionCap({
|
getLiveHistoryRetentionCap({
|
||||||
channel: "equities",
|
channel: "equities",
|
||||||
underlying_ids: ["AAPL"]
|
underlying_ids: ["AAPL"]
|
||||||
} as any)
|
} as any)
|
||||||
).toBeGreaterThan(0);
|
).toBe(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps auto-hydrating scoped live history while next_before exists", () => {
|
it("does not auto-hydrate scoped live history before the scroll gate is reached", () => {
|
||||||
const manifest = getLiveManifest(
|
const manifest = getLiveManifest(
|
||||||
"/tape",
|
"/tape",
|
||||||
"AAPL",
|
"AAPL",
|
||||||
|
|
@ -669,18 +670,12 @@ describe("live tape history helpers", () => {
|
||||||
|
|
||||||
expect(
|
expect(
|
||||||
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {})
|
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {})
|
||||||
).toEqual(["options", "equities"]);
|
).toEqual([]);
|
||||||
expect(
|
expect(
|
||||||
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {
|
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, historyCursors, {
|
||||||
[getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "options")!)]: true
|
[getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "options")!)]: true
|
||||||
})
|
})
|
||||||
).toEqual(["equities"]);
|
).toEqual([]);
|
||||||
expect(
|
|
||||||
getScopedLiveAutoHydrationChannels(true, "/tape", manifest, {
|
|
||||||
...historyCursors,
|
|
||||||
[getLiveSubscriptionKey(manifest.find((subscription) => subscription.channel === "equities")!)]: null
|
|
||||||
}, {})
|
|
||||||
).toEqual(["options"]);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("restores the same anchor key after live insertions at the top", () => {
|
it("restores the same anchor key after live insertions at the top", () => {
|
||||||
|
|
@ -864,9 +859,15 @@ describe("signals helpers", () => {
|
||||||
expect(getAlertWindowAnchorTs([], 42)).toBe(42);
|
expect(getAlertWindowAnchorTs([], 42)).toBe(42);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns connected/stale live status labels without live wording", () => {
|
it("returns connected/held/stale live status labels without live wording", () => {
|
||||||
expect(statusLabel("connected", false, "live")).toBe("Connected");
|
expect(statusLabel("connected", false, "live")).toBe("Connected");
|
||||||
|
expect(statusLabel("connected", true, "live")).toBe("Held");
|
||||||
expect(statusLabel("stale", false, "live")).toBe("Feed behind");
|
expect(statusLabel("stale", false, "live")).toBe("Feed behind");
|
||||||
|
expect(statusLabel("stale", true, "live")).toBe("Feed behind");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps replay pause wording on replay tapes", () => {
|
||||||
|
expect(statusLabel("connected", true, "replay")).toBe("Paused");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("treats healthy scoped channels as connected even when no matching rows are visible", () => {
|
it("treats healthy scoped channels as connected even when no matching rows are visible", () => {
|
||||||
|
|
|
||||||
|
|
@ -77,6 +77,7 @@ const LIVE_HOT_WINDOW_OPTIONS = parseBoundedInt(
|
||||||
1,
|
1,
|
||||||
100000
|
100000
|
||||||
);
|
);
|
||||||
|
const LIVE_OPTIONS_HEAD_LIMIT = 100;
|
||||||
const LIVE_HISTORY_SOFT_CAP = parseBoundedInt(
|
const LIVE_HISTORY_SOFT_CAP = parseBoundedInt(
|
||||||
process.env.NEXT_PUBLIC_LIVE_HISTORY_SOFT_CAP,
|
process.env.NEXT_PUBLIC_LIVE_HISTORY_SOFT_CAP,
|
||||||
5000,
|
5000,
|
||||||
|
|
@ -846,7 +847,7 @@ export const getLiveHistoryRetentionCap = (subscription: LiveSubscription): numb
|
||||||
switch (subscription.channel) {
|
switch (subscription.channel) {
|
||||||
case "options":
|
case "options":
|
||||||
case "equities":
|
case "equities":
|
||||||
return LIVE_HISTORY_SOFT_CAP;
|
return 0;
|
||||||
default:
|
default:
|
||||||
return LIVE_HISTORY_SOFT_CAP;
|
return LIVE_HISTORY_SOFT_CAP;
|
||||||
}
|
}
|
||||||
|
|
@ -859,27 +860,12 @@ export const getScopedLiveAutoHydrationChannels = (
|
||||||
historyCursors: Partial<Record<string, Cursor | null>>,
|
historyCursors: Partial<Record<string, Cursor | null>>,
|
||||||
historyLoading: Partial<Record<string, boolean>>
|
historyLoading: Partial<Record<string, boolean>>
|
||||||
): Array<Extract<LiveSubscription["channel"], "options" | "equities">> => {
|
): Array<Extract<LiveSubscription["channel"], "options" | "equities">> => {
|
||||||
if (!enabled || pathname !== "/tape") {
|
void enabled;
|
||||||
return [];
|
void pathname;
|
||||||
}
|
void manifest;
|
||||||
|
void historyCursors;
|
||||||
const channels: Array<Extract<LiveSubscription["channel"], "options" | "equities">> = [];
|
void historyLoading;
|
||||||
for (const subscription of manifest) {
|
return [];
|
||||||
const scoped =
|
|
||||||
(subscription.channel === "options" &&
|
|
||||||
(subscription.underlying_ids?.length || subscription.option_contract_id)) ||
|
|
||||||
(subscription.channel === "equities" && subscription.underlying_ids?.length);
|
|
||||||
if (!scoped) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const key = getLiveSubscriptionKey(subscription);
|
|
||||||
if (historyCursors[key] && !historyLoading[key]) {
|
|
||||||
channels.push(subscription.channel);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return channels;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export const getLiveFeedStatus = (
|
export const getLiveFeedStatus = (
|
||||||
|
|
@ -2027,7 +2013,10 @@ export const prunePinnedEntries = <T,>(
|
||||||
|
|
||||||
export const statusLabel = (status: WsStatus, paused: boolean, mode: TapeMode): string => {
|
export const statusLabel = (status: WsStatus, paused: boolean, mode: TapeMode): string => {
|
||||||
if (paused) {
|
if (paused) {
|
||||||
return "Paused";
|
if (mode === "replay") {
|
||||||
|
return "Paused";
|
||||||
|
}
|
||||||
|
return status === "connected" ? "Held" : statusLabel(status, false, mode);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (mode === "replay") {
|
if (mode === "replay") {
|
||||||
|
|
@ -2512,22 +2501,20 @@ type PausableTapeViewConfig<T extends SortableItem & { seq: number }> = {
|
||||||
const usePausableTapeView = <T extends SortableItem & { seq: number }>(
|
const usePausableTapeView = <T extends SortableItem & { seq: number }>(
|
||||||
config: PausableTapeViewConfig<T>
|
config: PausableTapeViewConfig<T>
|
||||||
): TapeState<T> => {
|
): TapeState<T> => {
|
||||||
const [paused, setPaused] = useState(false);
|
|
||||||
const [data, setData] = useState<PausableTapeData<T>>(EMPTY_PAUSABLE_TAPE);
|
const [data, setData] = useState<PausableTapeData<T>>(EMPTY_PAUSABLE_TAPE);
|
||||||
|
const holdForScroll = config.enabled ? (config.shouldHold ? config.shouldHold() : false) : false;
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!config.enabled) {
|
if (!config.enabled) {
|
||||||
setPaused(false);
|
|
||||||
setData(EMPTY_PAUSABLE_TAPE);
|
setData(EMPTY_PAUSABLE_TAPE);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const holdForScroll = config.shouldHold ? config.shouldHold() : false;
|
|
||||||
setData((current) => {
|
setData((current) => {
|
||||||
const next = reducePausableTapeData(
|
const next = reducePausableTapeData(
|
||||||
current,
|
current,
|
||||||
config.sourceItems,
|
config.sourceItems,
|
||||||
paused || holdForScroll,
|
holdForScroll,
|
||||||
config.retentionLimit ?? LIVE_HOT_WINDOW
|
config.retentionLimit ?? LIVE_HOT_WINDOW
|
||||||
);
|
);
|
||||||
if (next === current) {
|
if (next === current) {
|
||||||
|
|
@ -2535,7 +2522,7 @@ const usePausableTapeView = <T extends SortableItem & { seq: number }>(
|
||||||
}
|
}
|
||||||
|
|
||||||
const unseenCount = next.seenKeys.size - current.seenKeys.size;
|
const unseenCount = next.seenKeys.size - current.seenKeys.size;
|
||||||
if (!paused && unseenCount > 0) {
|
if (unseenCount > 0) {
|
||||||
config.onNewItems?.(unseenCount);
|
config.onNewItems?.(unseenCount);
|
||||||
config.captureScroll?.();
|
config.captureScroll?.();
|
||||||
}
|
}
|
||||||
|
|
@ -2548,17 +2535,11 @@ const usePausableTapeView = <T extends SortableItem & { seq: number }>(
|
||||||
config.onNewItems,
|
config.onNewItems,
|
||||||
config.captureScroll,
|
config.captureScroll,
|
||||||
config.retentionLimit,
|
config.retentionLimit,
|
||||||
config.shouldHold,
|
holdForScroll
|
||||||
paused
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!config.enabled || paused) {
|
if (!config.enabled || holdForScroll) {
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const holdForScroll = config.shouldHold ? config.shouldHold() : false;
|
|
||||||
if (holdForScroll) {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2581,14 +2562,9 @@ const usePausableTapeView = <T extends SortableItem & { seq: number }>(
|
||||||
config.onNewItems,
|
config.onNewItems,
|
||||||
config.retentionLimit,
|
config.retentionLimit,
|
||||||
config.resumeSignal,
|
config.resumeSignal,
|
||||||
config.shouldHold,
|
holdForScroll
|
||||||
paused
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const togglePause = useCallback(() => {
|
|
||||||
setPaused((current) => !current);
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
const status = config.enabled ? config.sourceStatus : "disconnected";
|
const status = config.enabled ? config.sourceStatus : "disconnected";
|
||||||
const projected = projectPausableTapeState(data.visible, status, config.lastUpdate);
|
const projected = projectPausableTapeState(data.visible, status, config.lastUpdate);
|
||||||
const historyItems = config.historyTail ?? [];
|
const historyItems = config.historyTail ?? [];
|
||||||
|
|
@ -2602,9 +2578,9 @@ const usePausableTapeView = <T extends SortableItem & { seq: number }>(
|
||||||
lastUpdate: projected.lastUpdate,
|
lastUpdate: projected.lastUpdate,
|
||||||
replayTime: null,
|
replayTime: null,
|
||||||
replayComplete: false,
|
replayComplete: false,
|
||||||
paused,
|
paused: holdForScroll,
|
||||||
dropped: data.dropped,
|
dropped: data.dropped,
|
||||||
togglePause
|
togglePause: () => {}
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -3052,7 +3028,7 @@ export const getLiveManifest = (
|
||||||
? undefined
|
? undefined
|
||||||
: optionPrintFilters ?? flowFilters,
|
: optionPrintFilters ?? flowFilters,
|
||||||
...optionScope,
|
...optionScope,
|
||||||
snapshot_limit: LIVE_HOT_WINDOW_OPTIONS
|
snapshot_limit: LIVE_OPTIONS_HEAD_LIMIT
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (features.nbbo) {
|
if (features.nbbo) {
|
||||||
|
|
@ -3337,7 +3313,7 @@ const useLiveSession = (
|
||||||
|
|
||||||
switch (subscription.channel) {
|
switch (subscription.channel) {
|
||||||
case "options":
|
case "options":
|
||||||
mergeItems(setOptions, optionsRef, items as OptionPrint[], LIVE_HOT_WINDOW_OPTIONS, {
|
mergeItems(setOptions, optionsRef, items as OptionPrint[], LIVE_OPTIONS_HEAD_LIMIT, {
|
||||||
setter: setOptionsHistory,
|
setter: setOptionsHistory,
|
||||||
ref: optionsHistoryRef,
|
ref: optionsHistoryRef,
|
||||||
cap: getLiveHistoryRetentionCap(subscription)
|
cap: getLiveHistoryRetentionCap(subscription)
|
||||||
|
|
@ -3794,6 +3770,7 @@ const TapeStatus = ({
|
||||||
};
|
};
|
||||||
|
|
||||||
type TapeControlsProps = {
|
type TapeControlsProps = {
|
||||||
|
mode: TapeMode;
|
||||||
paused: boolean;
|
paused: boolean;
|
||||||
onTogglePause: () => void;
|
onTogglePause: () => void;
|
||||||
isAtTop: boolean;
|
isAtTop: boolean;
|
||||||
|
|
@ -3801,13 +3778,15 @@ type TapeControlsProps = {
|
||||||
onJump: () => void;
|
onJump: () => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
const TapeControls = ({ paused, onTogglePause, isAtTop, missed, onJump }: TapeControlsProps) => {
|
const TapeControls = ({ mode, paused, onTogglePause, isAtTop, missed, onJump }: TapeControlsProps) => {
|
||||||
const active = !isAtTop && missed > 0;
|
const active = !isAtTop && missed > 0;
|
||||||
return (
|
return (
|
||||||
<div className={`tape-controls${active ? " tape-controls-active" : ""}`}>
|
<div className={`tape-controls${active ? " tape-controls-active" : ""}`}>
|
||||||
<button className="pause-button" type="button" onClick={onTogglePause}>
|
{mode === "replay" ? (
|
||||||
{paused ? "Resume" : "Pause"}
|
<button className="pause-button" type="button" onClick={onTogglePause}>
|
||||||
</button>
|
{paused ? "Resume" : "Pause"}
|
||||||
|
</button>
|
||||||
|
) : null}
|
||||||
<button className="jump-button" type="button" onClick={onJump} disabled={isAtTop}>
|
<button className="jump-button" type="button" onClick={onJump} disabled={isAtTop}>
|
||||||
Jump to top
|
Jump to top
|
||||||
</button>
|
</button>
|
||||||
|
|
@ -5373,7 +5352,7 @@ const useTerminalState = () => {
|
||||||
sourceItems: liveSession.options,
|
sourceItems: liveSession.options,
|
||||||
historyTail: liveSession.optionsHistory,
|
historyTail: liveSession.optionsHistory,
|
||||||
lastUpdate: liveSession.lastUpdate,
|
lastUpdate: liveSession.lastUpdate,
|
||||||
retentionLimit: LIVE_HOT_WINDOW_OPTIONS,
|
retentionLimit: LIVE_OPTIONS_HEAD_LIMIT,
|
||||||
captureScroll: optionsAnchor.capture,
|
captureScroll: optionsAnchor.capture,
|
||||||
onNewItems: optionsScroll.onNewItems,
|
onNewItems: optionsScroll.onNewItems,
|
||||||
shouldHold: () => !optionsScroll.isAtTopRef.current,
|
shouldHold: () => !optionsScroll.isAtTopRef.current,
|
||||||
|
|
@ -7141,6 +7120,7 @@ const OptionsPane = memo(({ state, limit }: OptionsPaneProps) => {
|
||||||
}
|
}
|
||||||
actions={
|
actions={
|
||||||
<TapeControls
|
<TapeControls
|
||||||
|
mode={state.mode}
|
||||||
paused={state.options.paused}
|
paused={state.options.paused}
|
||||||
onTogglePause={state.options.togglePause}
|
onTogglePause={state.options.togglePause}
|
||||||
isAtTop={state.optionsScroll.isAtTop}
|
isAtTop={state.optionsScroll.isAtTop}
|
||||||
|
|
@ -7329,6 +7309,7 @@ const EquitiesPane = memo(({ state, limit }: EquitiesPaneProps) => {
|
||||||
}
|
}
|
||||||
actions={
|
actions={
|
||||||
<TapeControls
|
<TapeControls
|
||||||
|
mode={state.mode}
|
||||||
paused={state.equities.paused}
|
paused={state.equities.paused}
|
||||||
onTogglePause={state.equities.togglePause}
|
onTogglePause={state.equities.togglePause}
|
||||||
isAtTop={state.equitiesScroll.isAtTop}
|
isAtTop={state.equitiesScroll.isAtTop}
|
||||||
|
|
@ -7432,6 +7413,7 @@ const FlowPane = memo(({ state, limit, title = "Flow" }: FlowPaneProps) => {
|
||||||
}
|
}
|
||||||
actions={
|
actions={
|
||||||
<TapeControls
|
<TapeControls
|
||||||
|
mode={state.mode}
|
||||||
paused={state.flow.paused}
|
paused={state.flow.paused}
|
||||||
onTogglePause={state.flow.togglePause}
|
onTogglePause={state.flow.togglePause}
|
||||||
isAtTop={state.flowScroll.isAtTop}
|
isAtTop={state.flowScroll.isAtTop}
|
||||||
|
|
@ -7581,6 +7563,7 @@ const AlertsPane = memo(({ state, limit, withStrip = false, className }: AlertsP
|
||||||
}
|
}
|
||||||
actions={
|
actions={
|
||||||
<TapeControls
|
<TapeControls
|
||||||
|
mode={state.mode}
|
||||||
paused={state.alerts.paused}
|
paused={state.alerts.paused}
|
||||||
onTogglePause={state.alerts.togglePause}
|
onTogglePause={state.alerts.togglePause}
|
||||||
isAtTop={state.alertsScroll.isAtTop}
|
isAtTop={state.alertsScroll.isAtTop}
|
||||||
|
|
@ -7695,6 +7678,7 @@ const ClassifierPane = memo(({ state, limit, className }: ClassifierPaneProps) =
|
||||||
}
|
}
|
||||||
actions={
|
actions={
|
||||||
<TapeControls
|
<TapeControls
|
||||||
|
mode={state.mode}
|
||||||
paused={state.smartMoney.paused}
|
paused={state.smartMoney.paused}
|
||||||
onTogglePause={state.smartMoney.togglePause}
|
onTogglePause={state.smartMoney.togglePause}
|
||||||
isAtTop={state.classifierScroll.isAtTop}
|
isAtTop={state.classifierScroll.isAtTop}
|
||||||
|
|
@ -7818,6 +7802,7 @@ const DarkPane = memo(({ state, limit, className }: DarkPaneProps) => {
|
||||||
}
|
}
|
||||||
actions={
|
actions={
|
||||||
<TapeControls
|
<TapeControls
|
||||||
|
mode={state.mode}
|
||||||
paused={state.inferredDark.paused}
|
paused={state.inferredDark.paused}
|
||||||
onTogglePause={state.inferredDark.togglePause}
|
onTogglePause={state.inferredDark.togglePause}
|
||||||
isAtTop={state.darkScroll.isAtTop}
|
isAtTop={state.darkScroll.isAtTop}
|
||||||
|
|
|
||||||
158
docs/turns/2026-05-16-live-tape-scroll-hold-history.html
Normal file
158
docs/turns/2026-05-16-live-tape-scroll-hold-history.html
Normal file
|
|
@ -0,0 +1,158 @@
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<title>Turn Summary: Live tape scroll hold and lazy history</title>
|
||||||
|
<style>
|
||||||
|
:root {
|
||||||
|
color-scheme: dark;
|
||||||
|
--bg: #0b1016;
|
||||||
|
--panel: #111820;
|
||||||
|
--border: rgba(255,255,255,0.1);
|
||||||
|
--text: #e6edf4;
|
||||||
|
--muted: #90a0b2;
|
||||||
|
--accent: #f5a623;
|
||||||
|
--accent-soft: rgba(245,166,35,0.16);
|
||||||
|
--ok: #25c17a;
|
||||||
|
}
|
||||||
|
* { box-sizing: border-box; }
|
||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
padding: 32px;
|
||||||
|
background: var(--bg);
|
||||||
|
color: var(--text);
|
||||||
|
font: 16px/1.55 "IBM Plex Sans", system-ui, sans-serif;
|
||||||
|
}
|
||||||
|
main { max-width: 980px; margin: 0 auto; }
|
||||||
|
h1, h2 { line-height: 1.15; }
|
||||||
|
h1 { margin: 0 0 8px; font-size: 2rem; }
|
||||||
|
h2 { margin: 32px 0 12px; font-size: 1.1rem; letter-spacing: 0.04em; text-transform: uppercase; }
|
||||||
|
p, li { color: var(--muted); }
|
||||||
|
.summary {
|
||||||
|
background: linear-gradient(180deg, rgba(245,166,35,0.14), rgba(245,166,35,0.05));
|
||||||
|
border: 1px solid rgba(245,166,35,0.24);
|
||||||
|
border-radius: 14px;
|
||||||
|
padding: 20px 22px;
|
||||||
|
margin-bottom: 28px;
|
||||||
|
}
|
||||||
|
.meta { color: var(--muted); font-size: 0.92rem; margin-bottom: 18px; }
|
||||||
|
.panel {
|
||||||
|
background: var(--panel);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 14px;
|
||||||
|
padding: 18px 20px;
|
||||||
|
margin-top: 14px;
|
||||||
|
}
|
||||||
|
code {
|
||||||
|
font-family: "IBM Plex Mono", ui-monospace, monospace;
|
||||||
|
font-size: 0.92em;
|
||||||
|
color: var(--text);
|
||||||
|
background: rgba(255,255,255,0.06);
|
||||||
|
padding: 0.14rem 0.35rem;
|
||||||
|
border-radius: 6px;
|
||||||
|
}
|
||||||
|
pre {
|
||||||
|
margin: 0;
|
||||||
|
padding: 14px;
|
||||||
|
overflow: auto;
|
||||||
|
border-radius: 12px;
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
background: rgba(0,0,0,0.22);
|
||||||
|
}
|
||||||
|
.ok { color: var(--ok); }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<main>
|
||||||
|
<div class="summary">
|
||||||
|
<h1>Live tape now holds on scroll, resumes at top, and lazy-loads deep history</h1>
|
||||||
|
<p>
|
||||||
|
The live tape no longer depends on a manual pause button in live mode. Scrolling away from the top now holds the
|
||||||
|
tape automatically, <code>Jump to top</code> resumes it, the options hot head is capped at 100 rows, and older
|
||||||
|
history is fetched from ClickHouse only when the scroll gate requests it.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="meta">Created 2026-05-16 during issue <code>islandflow-0sa</code>.</div>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Summary</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<p>
|
||||||
|
This change aligns the tape with the intended operator workflow: hold the live head while investigating older
|
||||||
|
rows, keep historical prints valid even when old, and avoid preloading a large ClickHouse backlog until the
|
||||||
|
user actually scrolls into it.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Changes Made</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<ul>
|
||||||
|
<li>Removed the live-mode Pause/Resume control from tape pane actions while keeping replay pause behavior intact.</li>
|
||||||
|
<li>Changed live tape status copy from manual <code>Paused</code> semantics to scroll-held <code>Held</code>.</li>
|
||||||
|
<li>Capped the live options head at <code>100</code> rows.</li>
|
||||||
|
<li>Stopped scoped live history from auto-hydrating in the background.</li>
|
||||||
|
<li>Made scoped options and equities snapshots prefer hot cached rows first, then backfill from ClickHouse when needed.</li>
|
||||||
|
<li>Made options and equities history retention effectively unbounded on the client so deep scrolling does not get trimmed away prematurely.</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Context</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<p>
|
||||||
|
The tape previously mixed several behaviors: a manual pause button, automatic scroll holding, scoped background
|
||||||
|
auto-hydration, and a much deeper options hot head. That created two user-visible problems: the live control model
|
||||||
|
felt redundant, and older prints could disappear or feel inconsistent when switching views or waiting for newer
|
||||||
|
rows to arrive.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Important Implementation Details</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<ul>
|
||||||
|
<li><code>apps/web/app/terminal.tsx</code>: live <code>usePausableTapeView</code> now treats scroll position as the hold source of truth.</li>
|
||||||
|
<li><code>apps/web/app/terminal.tsx</code>: options live snapshot and retention now use a strict <code>LIVE_OPTIONS_HEAD_LIMIT = 100</code>.</li>
|
||||||
|
<li><code>apps/web/app/terminal.tsx</code>: scoped history auto-hydration helper now returns no channels, so ClickHouse history stays lazy.</li>
|
||||||
|
<li><code>services/api/src/live.ts</code>: scoped option/equity snapshots now filter the hot cache first, then merge ClickHouse backfill without seam duplicates.</li>
|
||||||
|
</ul>
|
||||||
|
<pre><code>statusLabel("connected", true, "live") === "Held"
|
||||||
|
statusLabel("connected", true, "replay") === "Paused"</code></pre>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Validation</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<ul>
|
||||||
|
<li class="ok">Passed: <code>bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts</code></li>
|
||||||
|
<li class="ok">Passed: <code>bun --cwd=apps/web run build</code></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Issues, Limitations, and Mitigations</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<ul>
|
||||||
|
<li>Scoped snapshots can still backfill from ClickHouse when the hot cache does not have enough matching rows. This is intentional so focused views do not start empty.</li>
|
||||||
|
<li>Deep history is now lazy rather than eager, which reduces surprise and load, but the first deep-scroll request still depends on ClickHouse latency.</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Follow-up Work</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<p>No additional follow-up issues were created in this turn.</p>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -39,7 +39,8 @@ import {
|
||||||
type Cursor,
|
type Cursor,
|
||||||
type EquityCandle,
|
type EquityCandle,
|
||||||
type EquityPrint,
|
type EquityPrint,
|
||||||
type LiveChannel
|
type LiveChannel,
|
||||||
|
type OptionPrint
|
||||||
} from "@islandflow/types";
|
} from "@islandflow/types";
|
||||||
import { createMetrics } from "@islandflow/observability";
|
import { createMetrics } from "@islandflow/observability";
|
||||||
import type { RedisClientType } from "redis";
|
import type { RedisClientType } from "redis";
|
||||||
|
|
@ -456,6 +457,54 @@ export const buildOptionSnapshotFilters = (
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const matchesScopedOptionSnapshot = (
|
||||||
|
item: OptionPrint,
|
||||||
|
subscription: Extract<LiveSubscription, { channel: "options" }>
|
||||||
|
): boolean => {
|
||||||
|
if (!matchesOptionPrintFilters(item, subscription.filters)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (subscription.option_contract_id && item.option_contract_id !== subscription.option_contract_id) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!subscription.underlying_ids?.length) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const allowed = new Set(subscription.underlying_ids.map((value) => value.toUpperCase()));
|
||||||
|
return allowed.has(item.underlying_id.toUpperCase());
|
||||||
|
};
|
||||||
|
|
||||||
|
const matchesScopedEquitySnapshot = (
|
||||||
|
item: EquityPrint,
|
||||||
|
subscription: Extract<LiveSubscription, { channel: "equities" }>
|
||||||
|
): boolean => {
|
||||||
|
if (!subscription.underlying_ids?.length) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const allowed = new Set(subscription.underlying_ids.map((value) => value.toUpperCase()));
|
||||||
|
return allowed.has(item.underlying_id.toUpperCase());
|
||||||
|
};
|
||||||
|
|
||||||
|
const mergeSnapshotBackfill = <T>(
|
||||||
|
cached: T[],
|
||||||
|
backfill: T[],
|
||||||
|
limit: number,
|
||||||
|
cursorOf: (item: T) => Cursor
|
||||||
|
): T[] => {
|
||||||
|
const deduped = new Map<string, T>();
|
||||||
|
|
||||||
|
for (const item of [...cached, ...backfill]) {
|
||||||
|
const cursor = cursorOf(item);
|
||||||
|
deduped.set(`${cursor.ts}:${cursor.seq}`, item);
|
||||||
|
}
|
||||||
|
|
||||||
|
return sortGenericItems(Array.from(deduped.values()), cursorOf).slice(0, limit);
|
||||||
|
};
|
||||||
|
|
||||||
const candleRedisKey = (underlyingId: string, intervalMs: number): string =>
|
const candleRedisKey = (underlyingId: string, intervalMs: number): string =>
|
||||||
`live:equity-candles:${underlyingId}:${intervalMs}`;
|
`live:equity-candles:${underlyingId}:${intervalMs}`;
|
||||||
|
|
||||||
|
|
@ -740,12 +789,20 @@ export class LiveStateManager {
|
||||||
async getSnapshot(subscription: LiveSubscription): Promise<FeedSnapshot<unknown>> {
|
async getSnapshot(subscription: LiveSubscription): Promise<FeedSnapshot<unknown>> {
|
||||||
switch (subscription.channel) {
|
switch (subscription.channel) {
|
||||||
case "options": {
|
case "options": {
|
||||||
|
const config = this.generic.options;
|
||||||
|
const limit = snapshotLimitFor(subscription, config.limit);
|
||||||
const scoped = Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id);
|
const scoped = Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id);
|
||||||
if (subscription.filters?.view === "raw" || scoped) {
|
if (subscription.filters?.view === "raw" || scoped) {
|
||||||
this.stats.scopedClickHouseSnapshots += 1;
|
const cached = (this.genericItems.get("options") ?? [])
|
||||||
const limit = snapshotLimitFor(subscription, this.generic.options.limit);
|
.filter((entry) => matchesScopedOptionSnapshot(entry, subscription))
|
||||||
const storageFilters = buildOptionSnapshotFilters(subscription);
|
.slice(0, limit);
|
||||||
const items = await fetchRecentOptionPrints(this.clickhouse, limit, undefined, storageFilters);
|
let items = cached;
|
||||||
|
if (cached.length < limit) {
|
||||||
|
this.stats.scopedClickHouseSnapshots += 1;
|
||||||
|
const storageFilters = buildOptionSnapshotFilters(subscription);
|
||||||
|
const backfill = await fetchRecentOptionPrints(this.clickhouse, limit, undefined, storageFilters);
|
||||||
|
items = mergeSnapshotBackfill(cached, backfill, limit, (entry) => ({ ts: entry.ts, seq: entry.seq }));
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
subscription,
|
subscription,
|
||||||
items,
|
items,
|
||||||
|
|
@ -754,9 +811,7 @@ export class LiveStateManager {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const config = this.generic.options;
|
|
||||||
this.stats.genericCacheSnapshots += 1;
|
this.stats.genericCacheSnapshots += 1;
|
||||||
const limit = snapshotLimitFor(subscription, config.limit);
|
|
||||||
const items = (this.genericItems.get("options") ?? [])
|
const items = (this.genericItems.get("options") ?? [])
|
||||||
.filter((entry) => matchesOptionPrintFilters(entry, subscription.filters))
|
.filter((entry) => matchesOptionPrintFilters(entry, subscription.filters))
|
||||||
.slice(0, limit);
|
.slice(0, limit);
|
||||||
|
|
@ -785,9 +840,16 @@ export class LiveStateManager {
|
||||||
const config = this.generic.equities;
|
const config = this.generic.equities;
|
||||||
const limit = snapshotLimitFor(subscription, config.limit);
|
const limit = snapshotLimitFor(subscription, config.limit);
|
||||||
if (subscription.underlying_ids?.length) {
|
if (subscription.underlying_ids?.length) {
|
||||||
this.stats.scopedClickHouseSnapshots += 1;
|
const cached = (this.genericItems.get("equities") ?? [])
|
||||||
const filters: EquityPrintQueryFilters = { underlyingIds: subscription.underlying_ids };
|
.filter((entry) => matchesScopedEquitySnapshot(entry, subscription))
|
||||||
const items = await fetchRecentEquityPrints(this.clickhouse, limit, filters);
|
.slice(0, limit);
|
||||||
|
let items = cached;
|
||||||
|
if (cached.length < limit) {
|
||||||
|
this.stats.scopedClickHouseSnapshots += 1;
|
||||||
|
const filters: EquityPrintQueryFilters = { underlyingIds: subscription.underlying_ids };
|
||||||
|
const backfill = await fetchRecentEquityPrints(this.clickhouse, limit, filters);
|
||||||
|
items = mergeSnapshotBackfill(cached, backfill, limit, config.cursor);
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
subscription,
|
subscription,
|
||||||
items,
|
items,
|
||||||
|
|
|
||||||
|
|
@ -627,6 +627,57 @@ describe("LiveStateManager", () => {
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("prefers cached scoped option rows before clickhouse backfill", async () => {
|
||||||
|
const now = Date.now();
|
||||||
|
const manager = new LiveStateManager(
|
||||||
|
makeClickHouse((query) =>
|
||||||
|
query.includes("FROM option_prints")
|
||||||
|
? [
|
||||||
|
{
|
||||||
|
source_ts: now - 1_000,
|
||||||
|
ingest_ts: now - 999,
|
||||||
|
seq: 1,
|
||||||
|
trace_id: "opt-backfill",
|
||||||
|
ts: now - 1_000,
|
||||||
|
option_contract_id: "AAPL-2025-01-17-200-C",
|
||||||
|
underlying_id: "AAPL",
|
||||||
|
price: 1,
|
||||||
|
size: 10,
|
||||||
|
exchange: "X",
|
||||||
|
signal_pass: false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
: []
|
||||||
|
),
|
||||||
|
null
|
||||||
|
);
|
||||||
|
|
||||||
|
await manager.ingest("options", {
|
||||||
|
source_ts: now,
|
||||||
|
ingest_ts: now + 1,
|
||||||
|
seq: 2,
|
||||||
|
trace_id: "opt-hot",
|
||||||
|
ts: now,
|
||||||
|
option_contract_id: "AAPL-2025-01-17-200-C",
|
||||||
|
underlying_id: "AAPL",
|
||||||
|
price: 2,
|
||||||
|
size: 10,
|
||||||
|
exchange: "X",
|
||||||
|
signal_pass: true
|
||||||
|
});
|
||||||
|
|
||||||
|
const snapshot = await manager.getSnapshot({
|
||||||
|
channel: "options",
|
||||||
|
underlying_ids: ["AAPL"],
|
||||||
|
option_contract_id: "AAPL-2025-01-17-200-C"
|
||||||
|
});
|
||||||
|
|
||||||
|
expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id).slice(0, 2)).toEqual([
|
||||||
|
"opt-hot",
|
||||||
|
"opt-backfill"
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
it("seeds scoped equity snapshots from clickhouse rows older than 24h", async () => {
|
it("seeds scoped equity snapshots from clickhouse rows older than 24h", async () => {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const staleTs = now - 25 * 60 * 60 * 1000;
|
const staleTs = now - 25 * 60 * 60 * 1000;
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue