dirtydishes/islandflow
1
0
Fork 0
Code Issues 2 Pull requests 4 Projects Releases Packages Wiki Activity Actions 1

Fix live tape freshness and filter UX

Browse source
This commit is contained in:
dirtydishes 2026-04-28 17:13:46 -04:00
parent 27b0a399e6
commit 75fc6f9373
8 changed files with 1087 additions and 159 deletions
Download patch file Download diff file Expand all files Collapse all files

120
services/api/src/live.ts
View file

@ -65,6 +65,13 @@ type GenericFeedConfig = {
fetchRecent: (clickhouse: ClickHouseClient, limit: number) => Promise<any[]>;
};
const LIVE_FRESHNESS_THRESHOLDS: Partial<Record<LiveGenericChannel, number>> = {
options: 15_000,
nbbo: 15_000,
equities: 15_000,
flow: 30_000
};
export type GenericLiveLimits = Record<LiveGenericChannel, number>;
const parseGenericLimit = (
@ -201,6 +208,76 @@ const parseJsonList = <T>(payloads: string[], parse: (value: unknown) => T): T[]
return items;
};
const compareCursors = (a: Cursor, b: Cursor): number => (b.ts - a.ts) || (b.seq - a.seq);
const sortGenericItems = <T>(items: T[], cursorOf: (item: T) => Cursor): T[] =>
[...items].sort((a, b) => compareCursors(cursorOf(a), cursorOf(b)));
const keepNewestNbboByContract = <T extends { option_contract_id: string }>(
items: T[],
cursorOf: (item: T) => Cursor,
limit: number
): T[] => {
const latestByContract = new Map<string, T>();
for (const item of items) {
const existing = latestByContract.get(item.option_contract_id);
if (!existing || compareCursors(cursorOf(item), cursorOf(existing)) < 0) {
latestByContract.set(item.option_contract_id, item);
}
}
return sortGenericItems(Array.from(latestByContract.values()), cursorOf).slice(0, limit);
};
const normalizeGenericItems = <T>(
channel: LiveGenericChannel,
items: T[],
config: GenericFeedConfig
): T[] => {
if (channel === "nbbo") {
return keepNewestNbboByContract(
items as Array<T & { option_contract_id: string }>,
config.cursor,
config.limit
);
}
return sortGenericItems(items, config.cursor).slice(0, config.limit);
};
const extractFreshnessTs = (channel: LiveGenericChannel, item: any): number | null => {
switch (channel) {
case "options":
case "nbbo":
case "equities":
return typeof item.ts === "number" ? item.ts : null;
case "flow":
return typeof item.source_ts === "number" ? item.source_ts : null;
default:
return null;
}
};
const filterFreshGenericItems = <T>(
channel: LiveGenericChannel,
items: T[],
now = Date.now()
): T[] => {
const thresholdMs = LIVE_FRESHNESS_THRESHOLDS[channel];
if (!thresholdMs) {
return items;
}
return items.filter((item) => {
const ts = extractFreshnessTs(channel, item);
if (ts === null) {
return false;
}
return now - ts <= thresholdMs;
});
};
const nextBeforeForItems = <T>(items: T[], cursorOf: (item: T) => Cursor): Cursor | null => {
const last = items.at(-1);
return last ? cursorOf(last) : null;
@ -263,17 +340,24 @@ export class LiveStateManager {
const config = this.generic[channel];
if (this.redis?.isOpen) {
const payloads = await this.redis.lRange(config.redisKey, 0, config.limit - 1);
const cached = parseJsonList(payloads, config.parse);
const cached = normalizeGenericItems(channel, parseJsonList(payloads, config.parse), config);
if (cached.length > 0) {
this.genericItems.set(channel, cached);
this.stats.genericHydrateFromRedis += 1;
this.stats.cacheDepthByKey.set(config.redisKey, cached.length);
this.genericCursors.set(config.cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, config.cursorField)));
await this.persistList(
config.redisKey,
config.cursorField,
cached,
config.limit,
this.genericCursors.get(config.cursorField) ?? null
);
return;
}
}
const fresh = await config.fetchRecent(this.clickhouse, config.limit);
const fresh = normalizeGenericItems(channel, await config.fetchRecent(this.clickhouse, config.limit), config);
this.stats.genericHydrateFromClickHouse += 1;
this.stats.cacheDepthByKey.set(config.redisKey, fresh.length);
this.genericItems.set(channel, fresh);
@ -302,17 +386,21 @@ export class LiveStateManager {
undefined,
storageFilters
);
const freshItems = filterFreshGenericItems("options", items);
return {
subscription,
items,
items: freshItems,
watermark: items[0] ? { ts: items[0].ts, seq: items[0].seq } : null,
next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq }))
next_before: nextBeforeForItems(freshItems, (item) => ({ ts: item.ts, seq: item.seq }))
};
}
const config = this.generic.options;
const items = (this.genericItems.get("options") ?? []).filter((item) =>
matchesOptionPrintFilters(item, subscription.filters)
const items = filterFreshGenericItems(
"options",
(this.genericItems.get("options") ?? []).filter((item) =>
matchesOptionPrintFilters(item, subscription.filters)
)
);
return {
subscription,
@ -323,8 +411,11 @@ export class LiveStateManager {
}
case "flow": {
const config = this.generic.flow;
const items = (this.genericItems.get("flow") ?? []).filter((item) =>
matchesFlowPacketFilters(item, subscription.filters)
const items = filterFreshGenericItems(
"flow",
(this.genericItems.get("flow") ?? []).filter((item) =>
matchesFlowPacketFilters(item, subscription.filters)
)
);
return {
subscription,
@ -363,7 +454,10 @@ export class LiveStateManager {
}
default: {
const config = this.generic[subscription.channel];
const items = this.genericItems.get(subscription.channel) ?? [];
const items = filterFreshGenericItems(
subscription.channel,
this.genericItems.get(subscription.channel) ?? []
);
return {
subscription,
items,
@ -410,13 +504,7 @@ export class LiveStateManager {
const config = this.generic[channel];
const parsed = config.parse(item);
const items = this.genericItems.get(channel) ?? [];
const next = [parsed, ...items]
.sort((a, b) => {
const aCursor = config.cursor(a);
const bCursor = config.cursor(b);
return (bCursor.ts - aCursor.ts) || (bCursor.seq - aCursor.seq);
})
.slice(0, config.limit);
const next = normalizeGenericItems(channel, [parsed, ...items], config);
this.genericItems.set(channel, next);
this.stats.cacheDepthByKey.set(config.redisKey, next.length);
const cursor = config.cursor(parsed);

240
services/api/tests/live.test.ts
View file

@ -63,11 +63,12 @@ describe("LiveStateManager", () => {
it("hydrates snapshots from redis generic windows", async () => {
const redis = makeRedis();
const now = Date.now();
await redis.lPush(
"live:flow",
JSON.stringify({
source_ts: 100,
ingest_ts: 101,
source_ts: now,
ingest_ts: now + 1,
seq: 1,
trace_id: "flow-1",
id: "flow-1",
@ -76,15 +77,15 @@ describe("LiveStateManager", () => {
join_quality: {}
})
);
await redis.hSet("live:cursors", "flow", JSON.stringify({ ts: 100, seq: 1 }));
await redis.hSet("live:cursors", "flow", JSON.stringify({ ts: now, seq: 1 }));
const manager = new LiveStateManager(makeClickHouse(), redis as never);
await manager.hydrate();
const snapshot = await manager.getSnapshot({ channel: "flow" });
expect(snapshot.items).toHaveLength(1);
expect(snapshot.watermark).toEqual({ ts: 100, seq: 1 });
expect(snapshot.next_before).toEqual({ ts: 100, seq: 1 });
expect(snapshot.watermark).toEqual({ ts: now, seq: 1 });
expect(snapshot.next_before).toEqual({ ts: now, seq: 1 });
});
it("persists parameterized candle and overlay caches on ingest", async () => {
@ -136,6 +137,7 @@ describe("LiveStateManager", () => {
it("trims generic windows to configured per-channel limits", async () => {
const redis = makeRedis();
const now = Date.now();
const manager = new LiveStateManager(
makeClickHouse(),
redis as never,
@ -152,8 +154,8 @@ describe("LiveStateManager", () => {
);
await manager.ingest("flow", {
source_ts: 100,
ingest_ts: 101,
source_ts: now,
ingest_ts: now + 1,
seq: 1,
trace_id: "flow-1",
id: "flow-1",
@ -162,8 +164,8 @@ describe("LiveStateManager", () => {
join_quality: {}
});
await manager.ingest("flow", {
source_ts: 110,
ingest_ts: 111,
source_ts: now + 10,
ingest_ts: now + 11,
seq: 2,
trace_id: "flow-2",
id: "flow-2",
@ -172,8 +174,8 @@ describe("LiveStateManager", () => {
join_quality: {}
});
await manager.ingest("flow", {
source_ts: 120,
ingest_ts: 121,
source_ts: now + 20,
ingest_ts: now + 21,
seq: 3,
trace_id: "flow-3",
id: "flow-3",
@ -199,13 +201,14 @@ describe("LiveStateManager", () => {
it("filters option and flow snapshots using subscription filters", async () => {
const manager = new LiveStateManager(makeClickHouse(), null);
const now = Date.now();
await manager.ingest("options", {
source_ts: 100,
ingest_ts: 101,
source_ts: now,
ingest_ts: now + 1,
seq: 1,
trace_id: "opt-1",
ts: 100,
ts: now,
option_contract_id: "AAPL-2025-01-17-200-C",
price: 1,
size: 100,
@ -220,11 +223,11 @@ describe("LiveStateManager", () => {
signal_profile: "smart-money"
});
await manager.ingest("options", {
source_ts: 110,
ingest_ts: 111,
source_ts: now + 10,
ingest_ts: now + 11,
seq: 2,
trace_id: "opt-2",
ts: 110,
ts: now + 10,
option_contract_id: "SPY-2025-01-17-500-P",
price: 1,
size: 100,
@ -239,8 +242,8 @@ describe("LiveStateManager", () => {
signal_profile: "smart-money"
});
await manager.ingest("flow", {
source_ts: 120,
ingest_ts: 121,
source_ts: now + 20,
ingest_ts: now + 21,
seq: 3,
trace_id: "flow-1",
id: "flow-1",
@ -273,4 +276,203 @@ describe("LiveStateManager", () => {
expect(optionSnapshot.items).toHaveLength(1);
expect(flowSnapshot.items).toHaveLength(1);
});
it("suppresses stale items from live snapshots while preserving fresh ones", async () => {
const manager = new LiveStateManager(makeClickHouse(), null);
const now = Date.now();
await manager.ingest("options", {
source_ts: now - 20_000,
ingest_ts: now - 19_999,
seq: 1,
trace_id: "opt-stale",
ts: now - 20_000,
option_contract_id: "AAPL-2025-01-17-200-C",
price: 1,
size: 10,
exchange: "X"
});
await manager.ingest("options", {
source_ts: now - 5_000,
ingest_ts: now - 4_999,
seq: 2,
trace_id: "opt-fresh",
ts: now - 5_000,
option_contract_id: "AAPL-2025-01-17-205-C",
price: 1,
size: 10,
exchange: "X"
});
await manager.ingest("nbbo", {
source_ts: now - 20_000,
ingest_ts: now - 19_999,
seq: 1,
trace_id: "nbbo-stale",
ts: now - 20_000,
option_contract_id: "AAPL-2025-01-17-200-C",
bid: 1,
ask: 1.1,
bidSize: 10,
askSize: 10
});
await manager.ingest("nbbo", {
source_ts: now - 5_000,
ingest_ts: now - 4_999,
seq: 2,
trace_id: "nbbo-fresh",
ts: now - 5_000,
option_contract_id: "AAPL-2025-01-17-205-C",
bid: 1,
ask: 1.1,
bidSize: 10,
askSize: 10
});
await manager.ingest("equities", {
source_ts: now - 20_000,
ingest_ts: now - 19_999,
seq: 1,
trace_id: "eq-stale",
ts: now - 20_000,
underlying_id: "AAPL",
price: 100,
size: 10,
exchange: "X",
offExchangeFlag: false
});
await manager.ingest("equities", {
source_ts: now - 5_000,
ingest_ts: now - 4_999,
seq: 2,
trace_id: "eq-fresh",
ts: now - 5_000,
underlying_id: "AAPL",
price: 101,
size: 10,
exchange: "X",
offExchangeFlag: false
});
await manager.ingest("flow", {
source_ts: now - 40_000,
ingest_ts: now - 39_999,
seq: 1,
trace_id: "flow-stale",
id: "flow-stale",
members: ["opt-stale"],
features: {},
join_quality: {}
});
await manager.ingest("flow", {
source_ts: now - 5_000,
ingest_ts: now - 4_999,
seq: 2,
trace_id: "flow-fresh",
id: "flow-fresh",
members: ["opt-fresh"],
features: {},
join_quality: {}
});
const [optionsSnapshot, nbboSnapshot, equitiesSnapshot, flowSnapshot] = await Promise.all([
manager.getSnapshot({ channel: "options" }),
manager.getSnapshot({ channel: "nbbo" }),
manager.getSnapshot({ channel: "equities" }),
manager.getSnapshot({ channel: "flow" })
]);
expect((optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
"opt-fresh"
]);
expect((nbboSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
"nbbo-fresh"
]);
expect((equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
"eq-fresh"
]);
expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([
"flow-fresh"
]);
});
it("keeps only the newest NBBO quote per contract across hydrate and ingest", async () => {
const redis = makeRedis();
const now = Date.now();
await redis.lPush(
"live:nbbo",
JSON.stringify({
source_ts: now - 2_000,
ingest_ts: now - 1_999,
seq: 1,
trace_id: "nbbo-old",
ts: now - 2_000,
option_contract_id: "AAPL-2025-01-17-200-C",
bid: 1,
ask: 1.1,
bidSize: 10,
askSize: 10
})
);
await redis.lPush(
"live:nbbo",
JSON.stringify({
source_ts: now - 1_000,
ingest_ts: now - 999,
seq: 2,
trace_id: "nbbo-new",
ts: now - 1_000,
option_contract_id: "AAPL-2025-01-17-200-C",
bid: 1.2,
ask: 1.3,
bidSize: 12,
askSize: 12
})
);
await redis.lPush(
"live:nbbo",
JSON.stringify({
source_ts: now - 500,
ingest_ts: now - 499,
seq: 3,
trace_id: "nbbo-other",
ts: now - 500,
option_contract_id: "MSFT-2025-01-17-300-C",
bid: 2,
ask: 2.1,
bidSize: 15,
askSize: 15
})
);
await redis.hSet("live:cursors", "nbbo", JSON.stringify({ ts: now - 500, seq: 3 }));
const manager = new LiveStateManager(makeClickHouse(), redis as never);
await manager.hydrate();
await manager.ingest("nbbo", {
source_ts: now - 250,
ingest_ts: now - 249,
seq: 4,
trace_id: "nbbo-latest",
ts: now - 250,
option_contract_id: "AAPL-2025-01-17-200-C",
bid: 1.4,
ask: 1.5,
bidSize: 14,
askSize: 14
});
const snapshot = await manager.getSnapshot({ channel: "nbbo" });
expect(snapshot.items).toHaveLength(2);
expect(
(snapshot.items as Array<{ option_contract_id: string; trace_id: string }>).map((item) => [
item.option_contract_id,
item.trace_id
])
).toEqual([
["AAPL-2025-01-17-200-C", "nbbo-latest"],
["MSFT-2025-01-17-300-C", "nbbo-other"]
]);
});
});

53
services/ingest-options/src/adapters/synthetic.ts
View file

@ -23,6 +23,8 @@ type Burst = {
seed: number;
};
const OPTION_CONTRACT_MULTIPLIER = 100;
const SYNTHETIC_SYMBOLS = ["SPY", ...(SP500_SYMBOLS as readonly string[])];
const MS_PER_DAY = 24 * 60 * 60 * 1000;
const EXPIRY_OFFSETS = [0, 1, 7, 14, 28, 45, 60, 90];
@ -47,7 +49,7 @@ type Scenario = {
right: "C" | "P" | "either";
countRange: [number, number];
sizeRange: [number, number];
premiumRange: [number, number];
targetNotionalRange: [number, number];
priceTrend: "up" | "down" | "flat";
conditions?: string[];
};
@ -59,7 +61,7 @@ const REALISTIC_SCENARIOS: Scenario[] = [
right: "either",
countRange: [1, 2],
sizeRange: [30, 180],
premiumRange: [9_000, 35_000],
targetNotionalRange: [9_000, 35_000],
priceTrend: "flat",
conditions: ["FILL"]
},
@ -69,7 +71,7 @@ const REALISTIC_SCENARIOS: Scenario[] = [
right: "either",
countRange: [1, 2],
sizeRange: [120, 480],
premiumRange: [12_000, 45_000],
targetNotionalRange: [12_000, 45_000],
priceTrend: "flat",
conditions: ["FILL"]
},
@ -79,7 +81,7 @@ const REALISTIC_SCENARIOS: Scenario[] = [
right: "C",
countRange: [2, 3],
sizeRange: [180, 520],
premiumRange: [25_000, 90_000],
targetNotionalRange: [25_000, 90_000],
priceTrend: "up",
conditions: ["SWEEP"]
},
@ -89,7 +91,7 @@ const REALISTIC_SCENARIOS: Scenario[] = [
right: "P",
countRange: [2, 3],
sizeRange: [180, 520],
premiumRange: [25_000, 90_000],
targetNotionalRange: [25_000, 90_000],
priceTrend: "up",
conditions: ["SWEEP"]
},
@ -99,7 +101,7 @@ const REALISTIC_SCENARIOS: Scenario[] = [
right: "either",
countRange: [2, 3],
sizeRange: [500, 900],
premiumRange: [18_000, 70_000],
targetNotionalRange: [18_000, 70_000],
priceTrend: "flat",
conditions: ["ISO"]
},
@ -109,7 +111,7 @@ const REALISTIC_SCENARIOS: Scenario[] = [
right: "either",
countRange: [1, 2],
sizeRange: [5, 60],
premiumRange: [500, 6_000],
targetNotionalRange: [500, 6_000],
priceTrend: "flat",
conditions: ["FILL"]
}
@ -122,7 +124,7 @@ const ACTIVE_SCENARIOS: Scenario[] = [
right: "C",
countRange: [7, 10],
sizeRange: [600, 1800],
premiumRange: [120_000, 240_000],
targetNotionalRange: [120_000, 240_000],
priceTrend: "up",
conditions: ["SWEEP"]
},
@ -132,7 +134,7 @@ const ACTIVE_SCENARIOS: Scenario[] = [
right: "P",
countRange: [7, 10],
sizeRange: [600, 1800],
premiumRange: [120_000, 240_000],
targetNotionalRange: [120_000, 240_000],
priceTrend: "up",
conditions: ["SWEEP"]
},
@ -142,7 +144,7 @@ const ACTIVE_SCENARIOS: Scenario[] = [
right: "either",
countRange: [5, 8],
sizeRange: [1200, 3200],
premiumRange: [60_000, 140_000],
targetNotionalRange: [60_000, 140_000],
priceTrend: "flat",
conditions: ["ISO"]
},
@ -152,7 +154,7 @@ const ACTIVE_SCENARIOS: Scenario[] = [
right: "either",
countRange: [2, 4],
sizeRange: [10, 200],
premiumRange: [500, 5000],
targetNotionalRange: [500, 5000],
priceTrend: "flat",
conditions: ["FILL"]
}
@ -261,14 +263,17 @@ const SYNTHETIC_PROFILES: Record<SyntheticMarketMode, SyntheticOptionsProfile> =
weight: 20,
countRange: [5, 8],
sizeRange: [20, 300],
premiumRange: [800, 12_000]
targetNotionalRange: [800, 12_000]
}
: {
...scenario,
weight: scenario.weight + 10,
countRange: [scenario.countRange[0] + 2, scenario.countRange[1] + 3],
sizeRange: [scenario.sizeRange[0], scenario.sizeRange[1] * 2],
premiumRange: [scenario.premiumRange[0], scenario.premiumRange[1] * 1.5]
targetNotionalRange: [
scenario.targetNotionalRange[0],
scenario.targetNotionalRange[1] * 1.5
]
}
),
pricePlacements: FIREHOSE_PRICE_PLACEMENTS
@ -367,12 +372,20 @@ const buildBurst = (burstIndex: number, now: number, profile: SyntheticOptionsPr
const exchange = pick(EXCHANGES, burstIndex + symbolHash);
const printCount = pickInt(scenario.countRange[0], scenario.countRange[1], symbolHash + burstIndex * 13);
const baseSize = pickInt(scenario.sizeRange[0], scenario.sizeRange[1], symbolHash + burstIndex * 17);
const premiumTarget = pickFloat(
scenario.premiumRange[0],
scenario.premiumRange[1],
const targetNotional = pickFloat(
scenario.targetNotionalRange[0],
scenario.targetNotionalRange[1],
symbolHash + burstIndex * 19
);
const basePricePer = Math.max(0.05, Number((premiumTarget / (baseSize * printCount)).toFixed(2)));
const basePricePer = Math.max(
0.05,
Number(
(
targetNotional /
(baseSize * printCount * OPTION_CONTRACT_MULTIPLIER)
).toFixed(2)
)
);
const conditions = scenario.conditions?.length ? scenario.conditions : [pick(CONDITIONS, burstIndex)];
const priceStep =
scenario.priceTrend === "up" ? 0.01 : scenario.priceTrend === "down" ? -0.01 : 0;
@ -390,6 +403,12 @@ const buildBurst = (burstIndex: number, now: number, profile: SyntheticOptionsPr
};
};
export const buildSyntheticBurstForTest = (
burstIndex: number,
now: number,
mode: SyntheticMarketMode
): Burst => buildBurst(burstIndex, now, SYNTHETIC_PROFILES[mode]);
export const createSyntheticOptionsAdapter = (
config: SyntheticOptionsAdapterConfig
): OptionIngestAdapter => {

26
services/ingest-options/tests/synthetic.test.ts Normal file
View file

@ -0,0 +1,26 @@
import { describe, expect, it } from "bun:test";
import { buildSyntheticBurstForTest } from "../src/adapters/synthetic";
const totalBurstNotional = (burst: {
basePrice: number;
baseSize: number;
printCount: number;
}): number => burst.basePrice * burst.baseSize * burst.printCount * 100;
describe("synthetic options burst sizing", () => {
it("keeps realistic-mode ask lifts inside the configured notional band", () => {
const burst = buildSyntheticBurstForTest(2, Date.UTC(2026, 0, 2), "realistic");
expect(burst.scenarioId).toBe("ask_lift");
expect(totalBurstNotional(burst)).toBeGreaterThanOrEqual(9_000);
expect(totalBurstNotional(burst)).toBeLessThanOrEqual(35_000);
});
it("keeps active-mode sweeps inside the configured notional band", () => {
const burst = buildSyntheticBurstForTest(1, Date.UTC(2026, 0, 2), "active");
expect(burst.scenarioId).toBe("bearish_sweep");
expect(totalBurstNotional(burst)).toBeGreaterThanOrEqual(120_000);
expect(totalBurstNotional(burst)).toBeLessThanOrEqual(240_000);
});
});