dirtydishes/islandflow
1
0
Fork 0
Code Issues 2 Pull requests 4 Projects Releases Packages Wiki Activity Actions 1

Implement first-pass load reduction controls

Browse source
This commit is contained in:
dirtydishes 2026-05-08 02:46:41 -04:00
parent 5d488fd7f5
commit e7f4805ccc
17 changed files with 1191 additions and 608 deletions
Download patch file Download diff file Expand all files Collapse all files

395
services/compute/src/index.ts
View file

@ -26,6 +26,7 @@ import {
STREAM_SMART_MONEY_EVENTS,
STREAM_OPTION_NBBO,
STREAM_OPTION_SIGNAL_PRINTS,
buildStreamConfig,
buildDurableConsumer,
connectJetStreamWithRetry,
ensureStream,
@ -40,12 +41,13 @@ import {
ensureInferredDarkTable,
ensureFlowPacketsTable,
ensureSmartMoneyEventsTable,
insertAlert,
insertClassifierHit,
insertEquityPrintJoin,
insertInferredDark,
insertFlowPacket,
insertSmartMoneyEvent
ClickHouseBatchWriter,
enqueueAlertInsert,
enqueueClassifierHitInsert,
enqueueEquityPrintJoinInsert,
enqueueFlowPacketInsert,
enqueueInferredDarkInsert,
enqueueSmartMoneyEventInsert,
} from "@islandflow/storage";
import {
AlertEventSchema,
@ -82,7 +84,12 @@ import {
type DarkInferenceConfig
} from "./dark-inference";
import { buildEquityPrintJoin, type EquityQuoteJoin } from "./equity-joins";
import { createRedisClient, updateRollingStats, type RollingStatsConfig } from "./rolling-stats";
import {
createRedisClient,
RollingWindowStore,
type RollingStatsConfig,
type RollingWindowStoreConfig
} from "./rolling-stats";
import { summarizeStructure, type ContractLeg } from "./structures";
import {
buildStructureFlowPacket,
@ -103,6 +110,8 @@ const envSchema = z.object({
CLUSTER_WINDOW_MS: z.coerce.number().int().positive().default(500),
ROLLING_WINDOW_SIZE: z.coerce.number().int().positive().default(50),
ROLLING_TTL_SEC: z.coerce.number().int().nonnegative().default(86400),
ROLLING_CACHE_FLUSH_INTERVAL_MS: z.coerce.number().int().positive().default(30_000),
ROLLING_CACHE_MAX_KEYS: z.coerce.number().int().positive().default(20_000),
COMPUTE_DELIVER_POLICY: z.enum(["new", "all", "last", "last_per_subject"]).default("new"),
COMPUTE_CONSUMER_RESET: z
.preprocess((value) => {
@ -119,6 +128,8 @@ const envSchema = z.object({
}, z.boolean())
.default(false),
NBBO_MAX_AGE_MS: z.coerce.number().int().positive().default(1000),
COMPUTE_NBBO_CACHE_MAX_KEYS: z.coerce.number().int().positive().default(20_000),
COMPUTE_NBBO_CACHE_TTL_MS: z.coerce.number().int().positive().default(900_000),
EQUITY_QUOTE_MAX_AGE_MS: z.coerce.number().int().positive().default(1000),
DARK_INFER_WINDOW_MS: z.coerce.number().int().positive().default(60000),
DARK_INFER_COOLDOWN_MS: z.coerce.number().int().nonnegative().default(30000),
@ -269,6 +280,9 @@ const clusters = new Map<string, ClusterState>();
const nbboCache = new Map<string, OptionNBBO>();
const equityQuoteCache = new Map<string, EquityQuote>();
const darkInferenceState = createDarkInferenceState();
const nbboCacheTouchedAt = new Map<string, number>();
const equityQuoteCacheTouchedAt = new Map<string, number>();
const darkInferenceTouchedAt = new Map<string, number>();
const recentLegsByKey = new Map<string, LegEvidence[]>();
const recentLegsByRoot = new Map<string, LegEvidence[]>();
const recentStructureEmits = new Map<string, number>();
@ -278,6 +292,20 @@ const runtimeState = {
};
const MAX_RECENT_LEGS = 20;
const EQUITY_QUOTE_CACHE_MAX_KEYS = 2_000;
const EQUITY_QUOTE_CACHE_TTL_MS = 900_000;
const DARK_INFERENCE_TTL_MS = 900_000;
const CACHE_PRUNE_INTERVAL_MS = 60_000;
const emitCounters = {
flowPackets: 0,
structurePackets: 0,
smartMoneyEvents: 0,
classifierHits: 0,
alerts: 0,
equityJoins: 0,
darkEvents: 0
};
const rollingKey = (metric: string, contractId: string): string => {
return `rolling:${metric}:${contractId}`;
@ -479,8 +507,8 @@ const pruneRecentStructureEmits = (anchorTs: number): void => {
};
const emitStructurePacketIfNeeded = async (
clickhouse: ReturnType<typeof createClickHouseClient>,
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
batchWriter: ClickHouseBatchWriter,
legs: LegEvidence[],
summary: ReturnType<typeof summarizeStructure>,
currentContractId: string
@ -512,16 +540,11 @@ const emitStructurePacketIfNeeded = async (
const packet = buildStructureFlowPacket(plan, summary);
const validated = FlowPacketSchema.parse(packet);
await insertFlowPacket(clickhouse, validated);
enqueueFlowPacketInsert(batchWriter, validated);
await publishJson(js, SUBJECT_FLOW_PACKETS, validated);
await emitClassifiers(clickhouse, js, validated);
logger.info("emitted structure flow packet", {
id: validated.id,
type: summary.type,
legs: summary.legs,
strikes: summary.strikes
});
emitCounters.flowPackets += 1;
emitCounters.structurePackets += 1;
await emitClassifiers(js, batchWriter, validated);
};
const applyDeliverPolicy = (
@ -606,6 +629,7 @@ const updateNbboCache = (nbbo: OptionNBBO): void => {
(nbbo.ts === existing.ts && nbbo.seq >= existing.seq)
) {
nbboCache.set(nbbo.option_contract_id, nbbo);
nbboCacheTouchedAt.set(nbbo.option_contract_id, Date.now());
}
};
@ -617,6 +641,7 @@ const updateEquityQuoteCache = (quote: EquityQuote): void => {
(quote.ts === existing.ts && quote.seq >= existing.seq)
) {
equityQuoteCache.set(quote.underlying_id, quote);
equityQuoteCacheTouchedAt.set(quote.underlying_id, Date.now());
}
};
@ -626,6 +651,7 @@ const selectNbbo = (contractId: string, ts: number): NbboJoin => {
return { nbbo: null, ageMs: env.NBBO_MAX_AGE_MS + 1, stale: true };
}
nbboCacheTouchedAt.set(contractId, Date.now());
const ageMs = Math.abs(ts - nbbo.ts);
const stale = ageMs > env.NBBO_MAX_AGE_MS;
return { nbbo, ageMs, stale };
@ -637,11 +663,77 @@ const selectEquityQuote = (underlyingId: string, ts: number): EquityQuoteJoin =>
return { quote: null, ageMs: env.EQUITY_QUOTE_MAX_AGE_MS + 1, stale: true };
}
equityQuoteCacheTouchedAt.set(underlyingId, Date.now());
const ageMs = Math.abs(ts - quote.ts);
const stale = ageMs > env.EQUITY_QUOTE_MAX_AGE_MS;
return { quote, ageMs, stale };
};
const pruneTimedMap = <T>(
values: Map<string, T>,
touchedAt: Map<string, number>,
maxKeys: number,
ttlMs: number,
now = Date.now()
): number => {
let removed = 0;
for (const [key, touched] of touchedAt) {
if (now - touched > ttlMs) {
touchedAt.delete(key);
values.delete(key);
removed += 1;
}
}
if (values.size <= maxKeys) {
return removed;
}
const overflow = values.size - maxKeys;
const oldest = [...touchedAt.entries()].sort((a, b) => a[1] - b[1]).slice(0, overflow);
for (const [key] of oldest) {
touchedAt.delete(key);
values.delete(key);
removed += 1;
}
return removed;
};
const pruneComputeCaches = (rollingStore: RollingWindowStore, now = Date.now()) => {
const nbboRemoved = pruneTimedMap(
nbboCache,
nbboCacheTouchedAt,
env.COMPUTE_NBBO_CACHE_MAX_KEYS,
env.COMPUTE_NBBO_CACHE_TTL_MS,
now
);
const quoteRemoved = pruneTimedMap(
equityQuoteCache,
equityQuoteCacheTouchedAt,
EQUITY_QUOTE_CACHE_MAX_KEYS,
EQUITY_QUOTE_CACHE_TTL_MS,
now
);
const darkRemoved = pruneTimedMap(
darkInferenceState.lastEmittedByUnderlying,
darkInferenceTouchedAt,
EQUITY_QUOTE_CACHE_MAX_KEYS,
DARK_INFERENCE_TTL_MS,
now
);
const rollingRemoved = rollingStore.prune(now);
logger.info("compute cache summary", {
nbbo_cache_size: nbboCache.size,
equity_quote_cache_size: equityQuoteCache.size,
dark_inference_cache_size: darkInferenceState.lastEmittedByUnderlying.size,
rolling_cache_size: rollingStore.size,
removed: nbboRemoved + quoteRemoved + darkRemoved + rollingRemoved
});
};
const classifyPlacement = (price: number, join: NbboJoin): NbboPlacement => {
if (!Number.isFinite(price)) {
return "MISSING";
@ -679,10 +771,9 @@ const classifyPlacement = (price: number, join: NbboJoin): NbboPlacement => {
};
const flushCluster = async (
clickhouse: ReturnType<typeof createClickHouseClient>,
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
redis: ReturnType<typeof createRedisClient>,
rollingConfig: RollingStatsConfig,
batchWriter: ClickHouseBatchWriter,
rollingStore: RollingWindowStore,
cluster: ClusterState
): Promise<void> => {
if (cluster.flushed) {
@ -784,12 +875,7 @@ const flushCluster = async (
prefix: string
): Promise<void> => {
try {
const snapshot = await updateRollingStats(
redis,
rollingKey(metric, cluster.contractId),
value,
rollingConfig
);
const snapshot = rollingStore.update(rollingKey(metric, cluster.contractId), value);
features[`${prefix}_mean`] = roundTo(snapshot.mean);
features[`${prefix}_std`] = roundTo(snapshot.stddev);
features[`${prefix}_z`] = roundTo(snapshot.zscore);
@ -824,7 +910,7 @@ const flushCluster = async (
features.structure_rights = summary.rights;
}
await emitStructurePacketIfNeeded(clickhouse, js, legs, summary, currentLeg.contractId);
await emitStructurePacketIfNeeded(js, batchWriter, legs, summary, currentLeg.contractId);
const rootKey = buildRootKey(currentLeg);
const rootCandidates = [
@ -834,7 +920,7 @@ const flushCluster = async (
const rollLegs = [currentLeg, ...rootCandidates];
const rollSummary = summarizeStructure(rollLegs);
if (rollSummary?.type === "roll") {
await emitStructurePacketIfNeeded(clickhouse, js, rollLegs, rollSummary, currentLeg.contractId);
await emitStructurePacketIfNeeded(js, batchWriter, rollLegs, rollSummary, currentLeg.contractId);
}
storeRecentLeg(currentLeg, anchorTs);
@ -873,16 +959,10 @@ const flushCluster = async (
const validated = FlowPacketSchema.parse(packet);
try {
await insertFlowPacket(clickhouse, validated);
enqueueFlowPacketInsert(batchWriter, validated);
await publishJson(js, SUBJECT_FLOW_PACKETS, validated);
await emitClassifiers(clickhouse, js, validated);
logger.info("emitted flow packet", {
id: validated.id,
contract: cluster.contractId,
count: cluster.members.length
});
emitCounters.flowPackets += 1;
await emitClassifiers(js, batchWriter, validated);
} catch (error) {
if (isExpectedShutdownNatsError(error)) {
logger.info("skipped flow packet publish during shutdown", {
@ -899,8 +979,8 @@ const flushCluster = async (
};
const emitClassifiers = async (
clickhouse: ReturnType<typeof createClickHouseClient>,
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
batchWriter: ClickHouseBatchWriter,
packet: FlowPacket
): Promise<void> => {
let smartMoneyEvent: SmartMoneyEvent;
@ -915,8 +995,9 @@ const emitClassifiers = async (
: packet.source_ts;
const eventCalendarMatch = underlyingId ? eventCalendarProvider.findNextEvent(underlyingId, referenceTs) : null;
smartMoneyEvent = SmartMoneyEventSchema.parse(buildSmartMoneyEventFromPacket(packet, { eventCalendarMatch }));
await insertSmartMoneyEvent(clickhouse, smartMoneyEvent);
enqueueSmartMoneyEventInsert(batchWriter, smartMoneyEvent);
await publishJson(js, SUBJECT_SMART_MONEY_EVENTS, smartMoneyEvent);
emitCounters.smartMoneyEvents += 1;
} catch (error) {
if (isExpectedShutdownNatsError(error)) {
return;
@ -945,8 +1026,9 @@ const emitClassifiers = async (
for (const hit of hitEvents) {
try {
await insertClassifierHit(clickhouse, hit);
enqueueClassifierHitInsert(batchWriter, hit);
await publishJson(js, SUBJECT_CLASSIFIER_HITS, hit);
emitCounters.classifierHits += 1;
} catch (error) {
if (isExpectedShutdownNatsError(error)) {
continue;
@ -981,8 +1063,9 @@ const emitClassifiers = async (
});
try {
await insertAlert(clickhouse, alert);
enqueueAlertInsert(batchWriter, alert);
await publishJson(js, SUBJECT_ALERTS, alert);
emitCounters.alerts += 1;
} catch (error) {
if (isExpectedShutdownNatsError(error)) {
return;
@ -995,17 +1078,21 @@ const emitClassifiers = async (
};
const emitEquityJoin = async (
clickhouse: ReturnType<typeof createClickHouseClient>,
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
batchWriter: ClickHouseBatchWriter,
print: EquityPrint
): Promise<void> => {
const join = selectEquityQuote(print.underlying_id, print.ts);
const payload: EquityPrintJoin = EquityPrintJoinSchema.parse(buildEquityPrintJoin(print, join));
try {
await insertEquityPrintJoin(clickhouse, payload);
enqueueEquityPrintJoinInsert(batchWriter, payload);
} catch (error) {
logger.error("failed to emit equity print join", {
if (isExpectedShutdownNatsError(error)) {
return;
}
logger.error("failed to queue equity print join", {
error: error instanceof Error ? error.message : String(error),
trace_id: payload.trace_id
});
@ -1014,6 +1101,7 @@ const emitEquityJoin = async (
try {
await publishJson(js, SUBJECT_EQUITY_JOINS, payload);
emitCounters.equityJoins += 1;
} catch (error) {
if (isExpectedShutdownNatsError(error)) {
return;
@ -1024,20 +1112,26 @@ const emitEquityJoin = async (
});
}
await emitDarkInferences(clickhouse, js, payload);
await emitDarkInferences(js, batchWriter, payload);
};
const emitDarkInferences = async (
clickhouse: ReturnType<typeof createClickHouseClient>,
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
batchWriter: ClickHouseBatchWriter,
join: EquityPrintJoin
): Promise<void> => {
const events = evaluateDarkInferences(join, darkInferenceConfig, darkInferenceState);
for (const event of events) {
const validated: InferredDarkEvent = InferredDarkEventSchema.parse(event);
try {
await insertInferredDark(clickhouse, validated);
enqueueInferredDarkInsert(batchWriter, validated);
await publishJson(js, SUBJECT_INFERRED_DARK, validated);
emitCounters.darkEvents += 1;
const underlyingId =
typeof join.features?.underlying_id === "string" ? join.features.underlying_id : null;
if (underlyingId) {
darkInferenceTouchedAt.set(underlyingId, Date.now());
}
} catch (error) {
if (isExpectedShutdownNatsError(error)) {
continue;
@ -1051,10 +1145,9 @@ const emitDarkInferences = async (
};
const flushEligibleClusters = async (
clickhouse: ReturnType<typeof createClickHouseClient>,
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
redis: ReturnType<typeof createRedisClient>,
rollingConfig: RollingStatsConfig,
batchWriter: ClickHouseBatchWriter,
rollingStore: RollingWindowStore,
currentTs: number,
skipContractId: string
): Promise<void> => {
@ -1065,7 +1158,7 @@ const flushEligibleClusters = async (
if (currentTs - cluster.endTs > env.CLUSTER_WINDOW_MS) {
clusters.delete(contractId);
await flushCluster(clickhouse, js, redis, rollingConfig, cluster);
await flushCluster(js, batchWriter, rollingStore, cluster);
}
}
};
@ -1081,135 +1174,16 @@ const run = async () => {
{ attempts: 120, delayMs: 500 }
);
await ensureStream(jsm, {
name: STREAM_OPTION_SIGNAL_PRINTS,
subjects: [SUBJECT_OPTION_SIGNAL_PRINTS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_OPTION_NBBO,
subjects: [SUBJECT_OPTION_NBBO],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_EQUITY_PRINTS,
subjects: [SUBJECT_EQUITY_PRINTS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_EQUITY_QUOTES,
subjects: [SUBJECT_EQUITY_QUOTES],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_FLOW_PACKETS,
subjects: [SUBJECT_FLOW_PACKETS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_SMART_MONEY_EVENTS,
subjects: [SUBJECT_SMART_MONEY_EVENTS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_EQUITY_JOINS,
subjects: [SUBJECT_EQUITY_JOINS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_INFERRED_DARK,
subjects: [SUBJECT_INFERRED_DARK],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_CLASSIFIER_HITS,
subjects: [SUBJECT_CLASSIFIER_HITS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_ALERTS,
subjects: [SUBJECT_ALERTS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_FLOW_PACKETS, SUBJECT_FLOW_PACKETS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_SMART_MONEY_EVENTS, SUBJECT_SMART_MONEY_EVENTS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_JOINS, SUBJECT_EQUITY_JOINS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_INFERRED_DARK, SUBJECT_INFERRED_DARK, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_CLASSIFIER_HITS, SUBJECT_CLASSIFIER_HITS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_ALERTS, SUBJECT_ALERTS, "derived"));
const clickhouse = createClickHouseClient({
url: env.CLICKHOUSE_URL,
@ -1242,6 +1216,51 @@ const run = async () => {
windowSize: env.ROLLING_WINDOW_SIZE,
ttlSeconds: env.ROLLING_TTL_SEC
};
const rollingStore = new RollingWindowStore({
...rollingConfig,
flushIntervalMs: env.ROLLING_CACHE_FLUSH_INTERVAL_MS,
maxKeys: env.ROLLING_CACHE_MAX_KEYS
} satisfies RollingWindowStoreConfig);
const batchWriter = new ClickHouseBatchWriter(clickhouse, {
flushIntervalMs: 100,
maxRows: 250,
onError: (table, error, rowCount) => {
logger.error("batched clickhouse insert failed", {
table,
row_count: rowCount,
error: error instanceof Error ? error.message : String(error),
action: "dropped"
});
}
});
const rollingFlushTimer = setInterval(() => {
void rollingStore.flushToRedis(redis);
}, env.ROLLING_CACHE_FLUSH_INTERVAL_MS);
const pruneTimer = setInterval(() => {
pruneComputeCaches(rollingStore);
}, CACHE_PRUNE_INTERVAL_MS);
const summaryTimer = setInterval(() => {
logger.info("compute minute summary", {
flow_packets_emitted: emitCounters.flowPackets,
structure_packets_emitted: emitCounters.structurePackets,
smart_money_events_emitted: emitCounters.smartMoneyEvents,
classifier_hits_emitted: emitCounters.classifierHits,
alerts_emitted: emitCounters.alerts,
equity_joins_emitted: emitCounters.equityJoins,
dark_events_emitted: emitCounters.darkEvents,
rolling_stats_cache_size: rollingStore.size
});
emitCounters.flowPackets = 0;
emitCounters.structurePackets = 0;
emitCounters.smartMoneyEvents = 0;
emitCounters.classifierHits = 0;
emitCounters.alerts = 0;
emitCounters.equityJoins = 0;
emitCounters.darkEvents = 0;
}, 60_000);
rollingFlushTimer.unref?.();
pruneTimer.unref?.();
summaryTimer.unref?.();
await retry("clickhouse table init", 120, 500, async () => {
await ensureFlowPacketsTable(clickhouse);
@ -1578,7 +1597,7 @@ const run = async () => {
try {
const print = EquityPrintSchema.parse(equitySubscription.decode(msg));
await emitEquityJoin(clickhouse, js, print);
await emitEquityJoin(js, batchWriter, print);
msg.ack();
} catch (error) {
logger.error("failed to process equity print", {
@ -1602,11 +1621,16 @@ const run = async () => {
runtimeState.shuttingDown = true;
runtimeState.shutdownPromise = (async () => {
logger.info("service stopping", { signal });
clearInterval(rollingFlushTimer);
clearInterval(pruneTimer);
clearInterval(summaryTimer);
for (const cluster of [...clusters.values()]) {
await flushCluster(clickhouse, js, redis, rollingConfig, cluster);
await flushCluster(js, batchWriter, rollingStore, cluster);
}
clusters.clear();
await batchWriter.close();
await rollingStore.flushToRedis(redis);
try {
await nc.drain();
@ -1655,10 +1679,9 @@ const run = async () => {
try {
const print = OptionPrintSchema.parse(subscription.decode(msg));
await flushEligibleClusters(
clickhouse,
js,
redis,
rollingConfig,
batchWriter,
rollingStore,
print.ts,
print.option_contract_id
);
@ -1674,7 +1697,7 @@ const run = async () => {
updateCluster(existing, print);
} else {
clusters.delete(print.option_contract_id);
await flushCluster(clickhouse, js, redis, rollingConfig, existing);
await flushCluster(js, batchWriter, rollingStore, existing);
clusters.set(print.option_contract_id, buildCluster(print));
}

143
services/compute/src/rolling-stats.ts
View file

@ -5,6 +5,11 @@ export type RollingStatsConfig = {
ttlSeconds: number;
};
export type RollingWindowStoreConfig = RollingStatsConfig & {
flushIntervalMs: number;
maxKeys: number;
};
export type RollingSnapshot = {
baselineCount: number;
mean: number;
@ -12,6 +17,12 @@ export type RollingSnapshot = {
zscore: number;
};
type RollingWindowEntry = {
values: number[];
updatedAt: number;
dirty: boolean;
};
const toNumbers = (values: string[]): number[] => {
return values
.map((value) => Number(value))
@ -49,26 +60,120 @@ export const createRedisClient = (url: string) => {
return createClient({ url });
};
export const updateRollingStats = async (
client: ReturnType<typeof createClient>,
key: string,
value: number,
config: RollingStatsConfig
): Promise<RollingSnapshot> => {
const limit = Math.max(0, config.windowSize - 1);
const existing = await client.lRange(key, 0, limit);
const baseline = toNumbers(existing);
const snapshot = computeSnapshot(baseline, value);
const getOldestKey = (store: Map<string, RollingWindowEntry>): string | null => {
let oldestKey: string | null = null;
let oldestUpdatedAt = Number.POSITIVE_INFINITY;
const multi = client.multi();
multi.lPush(key, value.toString());
if (config.windowSize > 0) {
multi.lTrim(key, 0, config.windowSize - 1);
for (const [key, entry] of store) {
if (entry.updatedAt < oldestUpdatedAt) {
oldestUpdatedAt = entry.updatedAt;
oldestKey = key;
}
}
if (config.ttlSeconds > 0) {
multi.expire(key, config.ttlSeconds);
}
await multi.exec();
return snapshot;
return oldestKey;
};
export class RollingWindowStore {
private readonly store = new Map<string, RollingWindowEntry>();
private readonly ttlMs: number;
private readonly windowSize: number;
private readonly maxKeys: number;
constructor(private readonly config: RollingWindowStoreConfig) {
this.ttlMs = Math.max(0, config.ttlSeconds * 1000);
this.windowSize = Math.max(1, config.windowSize);
this.maxKeys = Math.max(1, config.maxKeys);
}
get size(): number {
return this.store.size;
}
update(key: string, value: number, now = Date.now()): RollingSnapshot {
this.prune(now);
const existing = this.store.get(key);
const baseline = existing?.values ?? [];
const snapshot = computeSnapshot(baseline, value);
const nextValues = [value, ...baseline].slice(0, this.windowSize);
this.store.set(key, {
values: nextValues,
updatedAt: now,
dirty: true
});
this.enforceMaxKeys();
return snapshot;
}
prune(now = Date.now()): number {
if (this.ttlMs <= 0) {
return 0;
}
let removed = 0;
for (const [key, entry] of this.store) {
if (now - entry.updatedAt > this.ttlMs) {
this.store.delete(key);
removed += 1;
}
}
return removed;
}
async hydrateFromRedis(
client: ReturnType<typeof createClient>,
keys: string[],
now = Date.now()
): Promise<void> {
for (const key of keys) {
const values = toNumbers(await client.lRange(key, 0, this.windowSize - 1));
if (values.length === 0) {
continue;
}
this.store.set(key, {
values,
updatedAt: now,
dirty: false
});
}
this.enforceMaxKeys();
}
async flushToRedis(client: ReturnType<typeof createClient>): Promise<number> {
let flushed = 0;
for (const [key, entry] of this.store) {
if (!entry.dirty) {
continue;
}
const multi = client.multi();
multi.lTrim(key, 1, 0);
for (let idx = entry.values.length - 1; idx >= 0; idx -= 1) {
const value = entry.values[idx];
if (typeof value === "number" && Number.isFinite(value)) {
multi.lPush(key, value.toString());
}
}
if (this.config.ttlSeconds > 0) {
multi.expire(key, this.config.ttlSeconds);
}
await multi.exec();
entry.dirty = false;
flushed += 1;
}
return flushed;
}
private enforceMaxKeys(): void {
while (this.store.size > this.maxKeys) {
const oldestKey = getOldestKey(this.store);
if (!oldestKey) {
break;
}
this.store.delete(oldestKey);
}
}
}

16
services/compute/tests/rolling-stats.test.ts
View file

@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test";
import { computeSnapshot, computeStats } from "../src/rolling-stats";
import { computeSnapshot, computeStats, RollingWindowStore } from "../src/rolling-stats";
describe("rolling stats helpers", () => {
test("computeStats handles empty baseline", () => {
@ -21,4 +21,18 @@ describe("rolling stats helpers", () => {
expect(snapshot.baselineCount).toBe(3);
expect(snapshot.zscore).toBeCloseTo(1.84, 2);
});
test("RollingWindowStore prunes stale keys by ttl", () => {
const store = new RollingWindowStore({
windowSize: 3,
ttlSeconds: 1,
flushIntervalMs: 30_000,
maxKeys: 10
});
store.update("rolling:premium:ABC", 10, 0);
expect(store.size).toBe(1);
expect(store.prune(1_500)).toBe(1);
expect(store.size).toBe(0);
});
});