expand ci quality gates
All checks were successful
CI / Validate (push) Successful in 1m13s

This commit is contained in:
dirtydishes 2026-05-30 02:34:28 -04:00
parent 65139bf8d0
commit 44431c4e66
71 changed files with 2262 additions and 1173 deletions

View file

@ -465,8 +465,7 @@ const parseCandleParams = (
const endTs = params.end_ts ?? Date.now();
const limit = params.limit ?? env.REST_DEFAULT_LIMIT;
const startTs =
params.start_ts ?? Math.max(0, Math.floor(endTs - params.interval_ms * limit));
const startTs = params.start_ts ?? Math.max(0, Math.floor(endTs - params.interval_ms * limit));
const rangeStart = Math.min(startTs, endTs);
const rangeEnd = Math.max(startTs, endTs);
@ -482,7 +481,13 @@ const parseCandleParams = (
const parseCandleReplayParams = (
url: URL
): { underlyingId: string; intervalMs: number; afterTs: number; afterSeq: number; limit: number } => {
): {
underlyingId: string;
intervalMs: number;
afterTs: number;
afterSeq: number;
limit: number;
} => {
const params = candleReplaySchema.parse({
underlying_id: url.searchParams.get("underlying_id") ?? undefined,
interval_ms: url.searchParams.get("interval_ms") ?? undefined,
@ -601,7 +606,10 @@ const matchesScopedOptionSubscription = (
print: { underlying_id?: string; option_contract_id: string },
subscription: Extract<LiveSubscription, { channel: "options" }>
): boolean => {
if (subscription.option_contract_id && subscription.option_contract_id !== print.option_contract_id) {
if (
subscription.option_contract_id &&
subscription.option_contract_id !== print.option_contract_id
) {
return false;
}
if (subscription.underlying_ids?.length) {
@ -693,8 +701,7 @@ const run = async () => {
env.OPTIONS_INGEST_ADAPTER,
env.EQUITIES_INGEST_ADAPTER
);
const syntheticBackendDisabledReason =
getSyntheticBackendDisabledReason(syntheticBackendMode);
const syntheticBackendDisabledReason = getSyntheticBackendDisabledReason(syntheticBackendMode);
const syntheticControlKv = await openSyntheticControlKv(js);
let syntheticControl = await ensureSyntheticControlState(syntheticControlKv);
const syntheticProfileHits = createRollingSyntheticProfileHits();
@ -899,11 +906,7 @@ const run = async () => {
}
}
const subscribeWithReset = async <T>(
subject: string,
stream: string,
durableName: string
) => {
const subscribeWithReset = async <T>(subject: string, stream: string, durableName: string) => {
const opts = buildDurableConsumer(durableName);
applyDeliverPolicy(opts, env.API_DELIVER_POLICY);
try {
@ -924,7 +927,8 @@ const run = async () => {
try {
await jsm.consumers.delete(stream, durableName);
} catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
const deleteMessage =
deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", {
durable: durableName,
@ -1023,8 +1027,12 @@ const run = async () => {
}
const matchingSubscriptions =
subscription.channel === "options" || subscription.channel === "flow" || subscription.channel === "equities"
? [...subscriptionDefinitions.entries()].filter(([, candidate]) => candidate.channel === subscription.channel)
subscription.channel === "options" ||
subscription.channel === "flow" ||
subscription.channel === "equities"
? [...subscriptionDefinitions.entries()].filter(
([, candidate]) => candidate.channel === subscription.channel
)
: [[getSubscriptionKey(subscription), subscription] as const];
if (matchingSubscriptions.length === 0) {
@ -1032,8 +1040,12 @@ const run = async () => {
}
const optionItem = ingestChannel === "options" ? (item as OptionPrint) : null;
const equityItem = ingestChannel === "equities" ? (item as Parameters<typeof matchesScopedEquitySubscription>[0]) : null;
const flowItem = ingestChannel === "flow" ? (item as Parameters<typeof matchesFlowPacketFilters>[0]) : null;
const equityItem =
ingestChannel === "equities"
? (item as Parameters<typeof matchesScopedEquitySubscription>[0])
: null;
const flowItem =
ingestChannel === "flow" ? (item as Parameters<typeof matchesFlowPacketFilters>[0]) : null;
let matchedSubscriptions = 0;
for (const [key, candidate] of matchingSubscriptions) {
@ -1315,9 +1327,7 @@ const run = async () => {
},
control: syntheticBackendMode === "synthetic" ? syntheticControl : null,
derived,
...(syntheticBackendDisabledReason
? { disabled_reason: syntheticBackendDisabledReason }
: {})
...(syntheticBackendDisabledReason ? { disabled_reason: syntheticBackendDisabledReason } : {})
};
};
@ -1385,11 +1395,7 @@ const run = async () => {
syntheticControl = await writeSyntheticControlState(syntheticControlKv, payload);
return jsonResponse({
control: syntheticControl,
derived: buildSyntheticDerivedStatus(
Date.now(),
syntheticControl,
syntheticProfileHits
)
derived: buildSyntheticDerivedStatus(Date.now(), syntheticControl, syntheticProfileHits)
});
} catch (error) {
return jsonResponse(
@ -1436,7 +1442,13 @@ const run = async () => {
if (req.method === "GET" && url.pathname === "/prints/equities/range") {
try {
const { underlyingId, startTs, endTs, limit } = parseEquityPrintRangeParams(url);
const data = await fetchEquityPrintsRange(clickhouse, underlyingId, startTs, endTs, limit);
const data = await fetchEquityPrintsRange(
clickhouse,
underlyingId,
startTs,
endTs,
limit
);
return jsonResponse({ data });
} catch (error) {
return jsonResponse(
@ -1566,7 +1578,9 @@ const run = async () => {
source,
storageFilters
);
return jsonResponse(buildHistoryResponse(data, (item) => ({ ts: item.ts, seq: item.seq })));
return jsonResponse(
buildHistoryResponse(data, (item) => ({ ts: item.ts, seq: item.seq }))
);
} catch (error) {
return jsonResponse(
{
@ -1986,7 +2000,9 @@ const run = async () => {
const payload =
typeof message === "string"
? message
: new TextDecoder().decode(message instanceof Uint8Array ? message : new Uint8Array(message));
: new TextDecoder().decode(
message instanceof Uint8Array ? message : new Uint8Array(message)
);
const parsed = LiveClientMessageSchema.parse(JSON.parse(payload));
if (parsed.op === "ping") {
sendLiveMessage(socket, {

View file

@ -165,11 +165,21 @@ const parseGenericLimitFallback = (env: NodeJS.ProcessEnv, fallback: number): nu
return Math.max(MIN_GENERIC_LIMIT, Math.min(MAX_GENERIC_LIMIT, Math.floor(parsed)));
};
export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env): GenericLiveLimits => {
export const resolveGenericLiveLimits = (
env: NodeJS.ProcessEnv = process.env
): GenericLiveLimits => {
const liveLimitDefault = parseGenericLimitFallback(env, DEFAULT_GENERIC_LIMIT);
return {
options: parseGenericLimit(env, "options", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.options),
nbbo: parseGenericLimit(env, "nbbo", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.nbbo),
options: parseGenericLimit(
env,
"options",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.options
),
nbbo: parseGenericLimit(
env,
"nbbo",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.nbbo
),
equities: parseGenericLimit(
env,
"equities",
@ -185,7 +195,11 @@ export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env):
"equity-joins",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["equity-joins"]
),
flow: parseGenericLimit(env, "flow", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.flow),
flow: parseGenericLimit(
env,
"flow",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.flow
),
"smart-money": parseGenericLimit(
env,
"smart-money",
@ -196,13 +210,21 @@ export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env):
"classifier-hits",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["classifier-hits"]
),
alerts: parseGenericLimit(env, "alerts", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.alerts),
alerts: parseGenericLimit(
env,
"alerts",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.alerts
),
"inferred-dark": parseGenericLimit(
env,
"inferred-dark",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["inferred-dark"]
),
news: parseGenericLimit(env, "news", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.news)
news: parseGenericLimit(
env,
"news",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.news
)
};
};
@ -227,12 +249,18 @@ const extractFreshnessTs = (channel: LiveGenericChannel, item: any): number | nu
export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): LiveStateConfig => ({
limits: resolveGenericLiveLimits(env),
scopedCacheMaxKeys: parsePositiveInt(env.LIVE_SCOPED_CACHE_MAX_KEYS, DEFAULT_SCOPED_CACHE_MAX_KEYS),
scopedCacheMaxKeys: parsePositiveInt(
env.LIVE_SCOPED_CACHE_MAX_KEYS,
DEFAULT_SCOPED_CACHE_MAX_KEYS
),
redisFlushIntervalMs: parsePositiveInt(
env.LIVE_REDIS_FLUSH_INTERVAL_MS,
DEFAULT_REDIS_FLUSH_INTERVAL_MS
),
redisFlushMaxItems: parsePositiveInt(env.LIVE_REDIS_FLUSH_MAX_ITEMS, DEFAULT_REDIS_FLUSH_MAX_ITEMS)
redisFlushMaxItems: parsePositiveInt(
env.LIVE_REDIS_FLUSH_MAX_ITEMS,
DEFAULT_REDIS_FLUSH_MAX_ITEMS
)
});
const parsePositiveInt = (value: string | undefined, fallback: number): number => {
const parsed = Number(value);
@ -242,10 +270,7 @@ const parsePositiveInt = (value: string | undefined, fallback: number): number =
return Math.max(1, Math.floor(parsed));
};
type RedisLike = Pick<
RedisClientType,
"isOpen" | "lRange" | "lPush" | "lTrim" | "hGet" | "hSet"
>;
type RedisLike = Pick<RedisClientType, "isOpen" | "lRange" | "lPush" | "lTrim" | "hGet" | "hSet">;
const parseCursor = (value: string | null): Cursor | null => {
if (!value) {
@ -259,7 +284,9 @@ const parseCursor = (value: string | null): Cursor | null => {
}
};
const getGenericConfig = (limits: GenericLiveLimits): {
const getGenericConfig = (
limits: GenericLiveLimits
): {
[K in LiveGenericChannel]: GenericFeedConfig;
} => ({
options: {
@ -365,7 +392,7 @@ const parseJsonList = <T>(payloads: string[], parse: (value: unknown) => T): T[]
return items;
};
const compareCursors = (a: Cursor, b: Cursor): number => (b.ts - a.ts) || (b.seq - a.seq);
const compareCursors = (a: Cursor, b: Cursor): number => b.ts - a.ts || b.seq - a.seq;
const sortGenericItems = <T>(items: T[], cursorOf: (item: T) => Cursor): T[] =>
[...items].sort((a, b) => compareCursors(cursorOf(a), cursorOf(b)));
@ -480,7 +507,10 @@ const matchesScopedOptionSnapshot = (
return false;
}
if (subscription.option_contract_id && item.option_contract_id !== subscription.option_contract_id) {
if (
subscription.option_contract_id &&
item.option_contract_id !== subscription.option_contract_id
) {
return false;
}
@ -529,11 +559,8 @@ const candleCursorField = (underlyingId: string, intervalMs: number): string =>
const overlayRedisKey = (underlyingId: string): string => `live:equity-overlay:${underlyingId}`;
const overlayCursorField = (underlyingId: string): string => `equities:${underlyingId}`;
const dropMatchingCursor = <T>(
items: T[],
target: Cursor,
cursorOf: (item: T) => Cursor
): T[] => items.filter((item) => compareCursors(cursorOf(item), target) !== 0);
const dropMatchingCursor = <T>(items: T[], target: Cursor, cursorOf: (item: T) => Cursor): T[] =>
items.filter((item) => compareCursors(cursorOf(item), target) !== 0);
const insertNewestFirst = <T>(
items: T[],
@ -676,7 +703,13 @@ export class LiveStateManager {
this.pendingRedisWrites.clear();
for (const write of writes) {
await this.persistList(write.listKey, write.cursorField, write.items, write.limit, write.cursor);
await this.persistList(
write.listKey,
write.cursorField,
write.items,
write.limit,
write.cursor
);
this.stats.redisFlushCount += 1;
this.stats.redisFlushItems += write.items.length;
metrics.count("api.live.redis_flush_count", 1);
@ -726,7 +759,12 @@ export class LiveStateManager {
}
}
private updateFreshnessMetric(listKey: string, channel: LiveChannel, item: unknown, now = Date.now()): void {
private updateFreshnessMetric(
listKey: string,
channel: LiveChannel,
item: unknown,
now = Date.now()
): void {
const ts =
channel === "equity-candles" || channel === "equity-overlay"
? typeof (item as { ts?: unknown })?.ts === "number"
@ -784,12 +822,22 @@ export class LiveStateManager {
config.cursorField,
parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, config.cursorField))
);
await this.persistList(config.redisKey, config.cursorField, cached, config.limit, this.genericCursors.get(config.cursorField) ?? null);
await this.persistList(
config.redisKey,
config.cursorField,
cached,
config.limit,
this.genericCursors.get(config.cursorField) ?? null
);
return;
}
}
const fresh = normalizeGenericItems(channel, await config.fetchRecent(this.clickhouse, config.limit), config);
const fresh = normalizeGenericItems(
channel,
await config.fetchRecent(this.clickhouse, config.limit),
config
);
this.stats.genericHydrateFromClickHouse += 1;
this.stats.cacheDepthByKey.set(config.redisKey, fresh.length);
this.genericItems.set(channel, fresh);
@ -806,7 +854,8 @@ export class LiveStateManager {
case "options": {
const config = this.generic.options;
const limit = snapshotLimitFor(subscription, config.limit);
const scoped = Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id);
const scoped =
Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id);
if (subscription.filters?.view === "raw" || scoped) {
const cached = (this.genericItems.get("options") ?? [])
.filter((entry) => matchesScopedOptionSnapshot(entry, subscription))
@ -815,8 +864,16 @@ export class LiveStateManager {
if (cached.length < limit) {
this.stats.scopedClickHouseSnapshots += 1;
const storageFilters = buildOptionSnapshotFilters(subscription);
const backfill = await fetchRecentOptionPrints(this.clickhouse, limit, undefined, storageFilters);
items = mergeSnapshotBackfill(cached, backfill, limit, (entry) => ({ ts: entry.ts, seq: entry.seq }));
const backfill = await fetchRecentOptionPrints(
this.clickhouse,
limit,
undefined,
storageFilters
);
items = mergeSnapshotBackfill(cached, backfill, limit, (entry) => ({
ts: entry.ts,
seq: entry.seq
}));
}
return {
subscription,
@ -942,7 +999,11 @@ export class LiveStateManager {
this.candleItems.set(key, nextState.items);
this.candleCursors.set(cursorField, cursor);
this.touchAccess(this.candleAccess, key);
this.evictScopedCachesIfNeeded(this.candleItems as Map<string, unknown[]>, this.candleCursors, this.candleAccess);
this.evictScopedCachesIfNeeded(
this.candleItems as Map<string, unknown[]>,
this.candleCursors,
this.candleAccess
);
if (nextState.outOfOrder) {
this.stats.outOfOrderEvents += 1;
metrics.count("api.live.out_of_order_events", 1);
@ -968,7 +1029,11 @@ export class LiveStateManager {
this.overlayItems.set(key, nextState.items);
this.overlayCursors.set(cursorField, cursor);
this.touchAccess(this.overlayAccess, key);
this.evictScopedCachesIfNeeded(this.overlayItems as Map<string, unknown[]>, this.overlayCursors, this.overlayAccess);
this.evictScopedCachesIfNeeded(
this.overlayItems as Map<string, unknown[]>,
this.overlayCursors,
this.overlayAccess
);
if (nextState.outOfOrder) {
this.stats.outOfOrderEvents += 1;
metrics.count("api.live.out_of_order_events", 1);
@ -991,10 +1056,19 @@ export class LiveStateManager {
const nextState =
channel === "nbbo"
? {
items: normalizeGenericItems(channel, [parsed, ...(this.genericItems.get(channel) ?? [])], config),
items: normalizeGenericItems(
channel,
[parsed, ...(this.genericItems.get(channel) ?? [])],
config
),
outOfOrder: false
}
: insertNewestFirst(this.genericItems.get(channel) ?? [], parsed, config.cursor, config.limit);
: insertNewestFirst(
this.genericItems.get(channel) ?? [],
parsed,
config.cursor,
config.limit
);
if (nextState.outOfOrder) {
this.stats.outOfOrderEvents += 1;
@ -1007,7 +1081,13 @@ export class LiveStateManager {
if (nextState.items.length > 0) {
this.updateFreshnessMetric(config.redisKey, channel, nextState.items[0]);
}
this.queueRedisWrite(config.redisKey, config.cursorField, nextState.items, config.limit, cursor);
this.queueRedisWrite(
config.redisKey,
config.cursorField,
nextState.items,
config.limit,
cursor
);
return cursor;
}
}
@ -1022,18 +1102,34 @@ export class LiveStateManager {
if (cached.length > 0) {
this.candleItems.set(key, cached);
this.touchAccess(this.candleAccess, key);
this.evictScopedCachesIfNeeded(this.candleItems as Map<string, unknown[]>, this.candleCursors, this.candleAccess);
this.evictScopedCachesIfNeeded(
this.candleItems as Map<string, unknown[]>,
this.candleCursors,
this.candleAccess
);
this.stats.cacheDepthByKey.set(key, cached.length);
this.updateFreshnessMetric(key, "equity-candles", cached[0]);
this.candleCursors.set(cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField)));
this.candleCursors.set(
cursorField,
parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField))
);
return;
}
}
const fresh = await fetchRecentEquityCandles(this.clickhouse, underlyingId, intervalMs, CHART_LIMITS.candles);
const fresh = await fetchRecentEquityCandles(
this.clickhouse,
underlyingId,
intervalMs,
CHART_LIMITS.candles
);
this.candleItems.set(key, fresh);
this.touchAccess(this.candleAccess, key);
this.evictScopedCachesIfNeeded(this.candleItems as Map<string, unknown[]>, this.candleCursors, this.candleAccess);
this.evictScopedCachesIfNeeded(
this.candleItems as Map<string, unknown[]>,
this.candleCursors,
this.candleAccess
);
this.stats.cacheDepthByKey.set(key, fresh.length);
if (fresh.length > 0) {
this.updateFreshnessMetric(key, "equity-candles", fresh[0]);
@ -1052,10 +1148,17 @@ export class LiveStateManager {
if (cached.length > 0) {
this.overlayItems.set(key, cached);
this.touchAccess(this.overlayAccess, key);
this.evictScopedCachesIfNeeded(this.overlayItems as Map<string, unknown[]>, this.overlayCursors, this.overlayAccess);
this.evictScopedCachesIfNeeded(
this.overlayItems as Map<string, unknown[]>,
this.overlayCursors,
this.overlayAccess
);
this.stats.cacheDepthByKey.set(key, cached.length);
this.updateFreshnessMetric(key, "equity-overlay", cached[0]);
this.overlayCursors.set(cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField)));
this.overlayCursors.set(
cursorField,
parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField))
);
return;
}
}
@ -1065,7 +1168,11 @@ export class LiveStateManager {
);
this.overlayItems.set(key, fresh);
this.touchAccess(this.overlayAccess, key);
this.evictScopedCachesIfNeeded(this.overlayItems as Map<string, unknown[]>, this.overlayCursors, this.overlayAccess);
this.evictScopedCachesIfNeeded(
this.overlayItems as Map<string, unknown[]>,
this.overlayCursors,
this.overlayAccess
);
this.stats.cacheDepthByKey.set(key, fresh.length);
if (fresh.length > 0) {
this.updateFreshnessMetric(key, "equity-overlay", fresh[0]);

View file

@ -83,11 +83,7 @@ export const buildSyntheticDerivedStatus = (
session_phase: session.session_phase,
regime: session.regime,
focus_symbols: session.focus_symbols,
profile_hit_counts: getSyntheticProfileHitCounts(
state,
now,
control.coverage_window_minutes
),
profile_hit_counts: getSyntheticProfileHitCounts(state, now, control.coverage_window_minutes),
coverage_window_minutes: control.coverage_window_minutes
});
};

View file

@ -3,7 +3,9 @@ import { isAlertContextPath, parseAlertContextTraceIdPath } from "../src/alert-c
describe("alert context route helpers", () => {
it("extracts a valid alert trace id from the context endpoint path", () => {
expect(parseAlertContextTraceIdPath("/flow/alerts/alert%3Actx%2Fone/context")).toBe("alert:ctx/one");
expect(parseAlertContextTraceIdPath("/flow/alerts/alert%3Actx%2Fone/context")).toBe(
"alert:ctx/one"
);
});
it("returns null for unrelated alert paths", () => {

View file

@ -9,9 +9,7 @@ import {
shouldFanoutLiveEvent
} from "../src/live";
const makeClickHouse = (
queryResolver?: (query: string) => unknown[]
): ClickHouseClient =>
const makeClickHouse = (queryResolver?: (query: string) => unknown[]): ClickHouseClient =>
({
exec: async () => {},
insert: async () => {},
@ -149,22 +147,18 @@ describe("LiveStateManager", () => {
it("trims generic windows to configured per-channel limits", async () => {
const redis = makeRedis();
const now = Date.now();
const manager = new LiveStateManager(
makeClickHouse(),
redis as never,
{
options: 10000,
nbbo: 10000,
equities: 10000,
"equity-quotes": 10000,
"equity-joins": 10000,
flow: 2,
"smart-money": 10000,
"classifier-hits": 10000,
alerts: 10000,
"inferred-dark": 10000
}
);
const manager = new LiveStateManager(makeClickHouse(), redis as never, {
options: 10000,
nbbo: 10000,
equities: 10000,
"equity-quotes": 10000,
"equity-joins": 10000,
flow: 2,
"smart-money": 10000,
"classifier-hits": 10000,
alerts: 10000,
"inferred-dark": 10000
});
await manager.ingest("flow", {
source_ts: now,
@ -503,18 +497,15 @@ describe("LiveStateManager", () => {
manager.getSnapshot({ channel: "flow" })
]);
expect((optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
"opt-fresh",
"opt-stale"
]);
expect((nbboSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
"nbbo-fresh",
"nbbo-stale"
]);
expect((equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
"eq-fresh",
"eq-stale"
]);
expect(
(optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
).toEqual(["opt-fresh", "opt-stale"]);
expect(
(nbboSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
).toEqual(["nbbo-fresh", "nbbo-stale"]);
expect(
(equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
).toEqual(["eq-fresh", "eq-stale"]);
expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([
"flow-fresh",
"flow-stale"
@ -699,10 +690,9 @@ describe("LiveStateManager", () => {
option_contract_id: "AAPL-2025-01-17-200-C"
});
expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id).slice(0, 2)).toEqual([
"opt-hot",
"opt-backfill"
]);
expect(
(snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id).slice(0, 2)
).toEqual(["opt-hot", "opt-backfill"]);
});
it("seeds scoped equity snapshots from clickhouse rows older than 24h", async () => {
@ -806,12 +796,12 @@ describe("LiveStateManager", () => {
manager.getSnapshot({ channel: "flow" })
]);
expect((optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
"opt-retained"
]);
expect((equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
"eq-retained"
]);
expect(
(optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
).toEqual(["opt-retained"]);
expect(
(equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
).toEqual(["eq-retained"]);
expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([
"flow-retained"
]);
@ -1047,7 +1037,10 @@ describe("LiveStateManager", () => {
});
it("tracks generic cache and scoped clickhouse snapshot sources separately", async () => {
const manager = new LiveStateManager(makeClickHouse(() => []), null);
const manager = new LiveStateManager(
makeClickHouse(() => []),
null
);
const now = Date.now();
await manager.ingest("options", {
@ -1075,7 +1068,10 @@ describe("LiveStateManager", () => {
});
it("keeps backend channel health healthy when a scoped query is quiet", async () => {
const manager = new LiveStateManager(makeClickHouse(() => []), null);
const manager = new LiveStateManager(
makeClickHouse(() => []),
null
);
const now = Date.now();
await manager.ingest("options", {
@ -1098,7 +1094,9 @@ describe("LiveStateManager", () => {
expect(quietSnapshot.items).toEqual([]);
expect(manager.getHotChannelHealth().options.healthy).toBe(true);
expect(manager.getStatsSnapshot().freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.options]).toBeLessThanOrEqual(50);
expect(
manager.getStatsSnapshot().freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.options]
).toBeLessThanOrEqual(50);
});
it("exposes freshness helper for feed status", () => {

View file

@ -33,9 +33,7 @@ const envSchema = z.object({
CANDLE_INTERVALS_MS: z.string().default("60000,300000"),
CANDLE_MAX_LATE_MS: z.coerce.number().int().nonnegative().default(0),
CANDLE_CACHE_LIMIT: z.coerce.number().int().nonnegative().default(2000),
CANDLE_DELIVER_POLICY: z
.enum(["new", "all", "last", "last_per_subject"])
.default("new"),
CANDLE_DELIVER_POLICY: z.enum(["new", "all", "last", "last_per_subject"]).default("new"),
CANDLE_CONSUMER_RESET: z
.preprocess((value) => {
if (typeof value === "string") {
@ -290,7 +288,10 @@ const run = async () => {
} else {
try {
const info = await jsm.consumers.info(STREAM_EQUITY_PRINTS, durableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.CANDLE_DELIVER_POLICY) {
if (
info?.config?.deliver_policy &&
info.config.deliver_policy !== env.CANDLE_DELIVER_POLICY
) {
logger.warn("resetting consumer due to deliver policy change", {
durable: durableName,
current: info.config.deliver_policy,
@ -301,7 +302,10 @@ const run = async () => {
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) {
logger.warn("failed to inspect jetstream consumer", { durable: durableName, error: message });
logger.warn("failed to inspect jetstream consumer", {
durable: durableName,
error: message
});
}
}
}
@ -327,7 +331,8 @@ const run = async () => {
try {
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, durableName);
} catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
const deleteMessage =
deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", {
durable: durableName,

View file

@ -14,4 +14,3 @@ export const scoreAlert = (
const severity = score >= 80 ? "high" : score >= 45 ? "medium" : "low";
return { score, severity };
};

View file

@ -573,10 +573,7 @@ const buildVerticalSpreadHit = (
};
};
const buildLadderHit = (
packet: FlowPacket,
config: ClassifierConfig
): ClassifierHit | null => {
const buildLadderHit = (packet: FlowPacket, config: ClassifierConfig): ClassifierHit | null => {
const structureType = getStringFeature(packet, "structure_type");
if (structureType !== "ladder") {
return null;
@ -648,7 +645,8 @@ const buildRollHit = (packet: FlowPacket, config: ClassifierConfig): ClassifierH
}
const activity = getLargeActivity(packet, config);
const qualifies = activity.totalPremium >= config.spikeMinPremium || activity.totalSize >= config.spikeMinSize;
const qualifies =
activity.totalPremium >= config.spikeMinPremium || activity.totalSize >= config.spikeMinSize;
if (!qualifies) {
return null;
}
@ -708,7 +706,9 @@ const buildRollHit = (packet: FlowPacket, config: ClassifierConfig): ClassifierH
const expiryNote = hasExpiryPair
? `Expiries: ${fromExpiry} -> ${toExpiry}${
expiryDaysDelta !== null && expiryDaysDelta !== 0 ? ` (${Math.round(expiryDaysDelta)}d)` : ""
expiryDaysDelta !== null && expiryDaysDelta !== 0
? ` (${Math.round(expiryDaysDelta)}d)`
: ""
}.`
: "Expiry pairing unavailable.";
const strikeNote = hasStrikePair
@ -850,9 +850,10 @@ export const evaluateClassifiers = (
const packetKind = getStringFeature(packet, "packet_kind");
const structureOnly = packetKind === "structure";
const contractId = typeof packet.features.option_contract_id === "string"
? packet.features.option_contract_id
: "";
const contractId =
typeof packet.features.option_contract_id === "string"
? packet.features.option_contract_id
: "";
const contract = structureOnly ? null : parseContractId(contractId);
const hits: ClassifierHit[] = [];

View file

@ -15,10 +15,7 @@ const roundTo = (value: number, digits = 4): number => {
return Number(value.toFixed(digits));
};
export const classifyQuotePlacement = (
price: number,
join: EquityQuoteJoin
): QuotePlacement => {
export const classifyQuotePlacement = (price: number, join: EquityQuoteJoin): QuotePlacement => {
if (!Number.isFinite(price)) {
return "MISSING";
}

View file

@ -46,7 +46,7 @@ import {
enqueueEquityPrintJoinInsert,
enqueueFlowPacketInsert,
enqueueInferredDarkInsert,
enqueueSmartMoneyEventInsert,
enqueueSmartMoneyEventInsert
} from "@islandflow/storage";
import {
AlertEventSchema,
@ -324,7 +324,9 @@ const buildPacketId = (cluster: ClusterState): string => {
const isExpectedShutdownNatsError = (error: unknown): boolean => {
const code = getErrorCode(error);
return runtimeState.shuttingDown && (code === "CONNECTION_DRAINING" || code === "CONNECTION_CLOSED");
return (
runtimeState.shuttingDown && (code === "CONNECTION_DRAINING" || code === "CONNECTION_CLOSED")
);
};
const createPlacementCounts = (): NbboPlacementCounts => ({
@ -337,7 +339,14 @@ const createPlacementCounts = (): NbboPlacementCounts => ({
stale: 0
});
const SPECIAL_PRINT_CONDITIONS = new Set(["AUCTION", "CROSS", "OPENING", "CLOSING", "COMPLEX", "SPREAD"]);
const SPECIAL_PRINT_CONDITIONS = new Set([
"AUCTION",
"CROSS",
"OPENING",
"CLOSING",
"COMPLEX",
"SPREAD"
]);
const SYNTHETIC_EVENT_CONDITION_RE = /^EVENT_(\d+)D$/i;
const normalizeConditions = (conditions: readonly string[] | undefined): string[] =>
@ -460,11 +469,7 @@ const storeRecentRootLeg = (leg: LegEvidence, anchorTs: number): void => {
recentLegsByRoot.set(key, next);
};
const collectActiveLegs = (
key: string,
anchorTs: number,
excludeId: string
): LegEvidence[] => {
const collectActiveLegs = (key: string, anchorTs: number, excludeId: string): LegEvidence[] => {
const legs: LegEvidence[] = [];
for (const [contractId, cluster] of clusters) {
if (contractId === excludeId) {
@ -485,11 +490,7 @@ const collectActiveLegs = (
return legs;
};
const collectActiveRootLegs = (
key: string,
anchorTs: number,
excludeId: string
): LegEvidence[] => {
const collectActiveRootLegs = (key: string, anchorTs: number, excludeId: string): LegEvidence[] => {
const legs: LegEvidence[] = [];
for (const [contractId, cluster] of clusters) {
if (contractId === excludeId) {
@ -601,12 +602,19 @@ const applyDeliverPolicy = (
const buildCluster = (print: OptionPrint): ClusterState => {
const placements = createPlacementCounts();
const normalizedConditions = normalizeConditions(print.conditions);
const executionIv = typeof print.execution_iv === "number" && Number.isFinite(print.execution_iv) ? print.execution_iv : null;
const executionIv =
typeof print.execution_iv === "number" && Number.isFinite(print.execution_iv)
? print.execution_iv
: null;
const executionUnderlyingMid =
typeof print.execution_underlying_mid === "number" && Number.isFinite(print.execution_underlying_mid)
typeof print.execution_underlying_mid === "number" &&
Number.isFinite(print.execution_underlying_mid)
? print.execution_underlying_mid
: null;
recordPlacement(placements, classifyPlacement(print.price, selectNbbo(print.option_contract_id, print.ts)));
recordPlacement(
placements,
classifyPlacement(print.price, selectNbbo(print.option_contract_id, print.ts))
);
return {
contractId: print.option_contract_id,
underlyingId: print.underlying_id ?? null,
@ -661,11 +669,18 @@ const updateCluster = (cluster: ClusterState, print: OptionPrint): ClusterState
if (typeof print.execution_iv === "number" && Number.isFinite(print.execution_iv)) {
cluster.lastExecutionIv = print.execution_iv;
cluster.minExecutionIv =
cluster.minExecutionIv === null ? print.execution_iv : Math.min(cluster.minExecutionIv, print.execution_iv);
cluster.minExecutionIv === null
? print.execution_iv
: Math.min(cluster.minExecutionIv, print.execution_iv);
cluster.maxExecutionIv =
cluster.maxExecutionIv === null ? print.execution_iv : Math.max(cluster.maxExecutionIv, print.execution_iv);
cluster.maxExecutionIv === null
? print.execution_iv
: Math.max(cluster.maxExecutionIv, print.execution_iv);
}
if (typeof print.execution_underlying_mid === "number" && Number.isFinite(print.execution_underlying_mid)) {
if (
typeof print.execution_underlying_mid === "number" &&
Number.isFinite(print.execution_underlying_mid)
) {
if (cluster.firstUnderlyingMid === null) {
cluster.firstUnderlyingMid = print.execution_underlying_mid;
}
@ -686,11 +701,7 @@ type NbboJoin = {
const updateNbboCache = (nbbo: OptionNBBO): void => {
const existing = nbboCache.get(nbbo.option_contract_id);
if (
!existing ||
nbbo.ts > existing.ts ||
(nbbo.ts === existing.ts && nbbo.seq >= existing.seq)
) {
if (!existing || nbbo.ts > existing.ts || (nbbo.ts === existing.ts && nbbo.seq >= existing.seq)) {
nbboCache.set(nbbo.option_contract_id, nbbo);
nbboCacheTouchedAt.set(nbbo.option_contract_id, Date.now());
}
@ -907,14 +918,18 @@ const flushCluster = async (
features.special_print_count = cluster.specialPrintCount;
}
if (cluster.minExecutionIv !== null && cluster.maxExecutionIv !== null) {
features.execution_iv_shock = roundTo(Math.max(0, cluster.maxExecutionIv - cluster.minExecutionIv));
features.execution_iv_shock = roundTo(
Math.max(0, cluster.maxExecutionIv - cluster.minExecutionIv)
);
}
if (
cluster.firstUnderlyingMid !== null &&
cluster.lastUnderlyingMid !== null &&
cluster.firstUnderlyingMid > 0
) {
const moveBps = ((cluster.lastUnderlyingMid - cluster.firstUnderlyingMid) / cluster.firstUnderlyingMid) * 10_000;
const moveBps =
((cluster.lastUnderlyingMid - cluster.firstUnderlyingMid) / cluster.firstUnderlyingMid) *
10_000;
features.underlying_move_bps = roundTo(moveBps);
}
const syntheticEventOffsetDays = parseSyntheticEventOffsetDays(cluster.conditions);
@ -1004,7 +1019,13 @@ const flushCluster = async (
const rollLegs = [currentLeg, ...rootCandidates];
const rollSummary = summarizeStructure(rollLegs);
if (rollSummary?.type === "roll") {
await emitStructurePacketIfNeeded(js, batchWriter, rollLegs, rollSummary, currentLeg.contractId);
await emitStructurePacketIfNeeded(
js,
batchWriter,
rollLegs,
rollSummary,
currentLeg.contractId
);
}
storeRecentLeg(currentLeg, anchorTs);
@ -1072,13 +1093,21 @@ const emitClassifiers = async (
const underlyingId =
typeof packet.features.underlying_id === "string"
? packet.features.underlying_id
: parseContractId(typeof packet.features.option_contract_id === "string" ? packet.features.option_contract_id : "")?.root;
: parseContractId(
typeof packet.features.option_contract_id === "string"
? packet.features.option_contract_id
: ""
)?.root;
const referenceTs =
typeof packet.features.end_ts === "number" && Number.isFinite(packet.features.end_ts)
? packet.features.end_ts
: packet.source_ts;
const eventCalendarMatch = underlyingId ? eventCalendarProvider.findNextEvent(underlyingId, referenceTs) : null;
smartMoneyEvent = SmartMoneyEventSchema.parse(buildSmartMoneyEventFromPacket(packet, { eventCalendarMatch }));
const eventCalendarMatch = underlyingId
? eventCalendarProvider.findNextEvent(underlyingId, referenceTs)
: null;
smartMoneyEvent = SmartMoneyEventSchema.parse(
buildSmartMoneyEventFromPacket(packet, { eventCalendarMatch })
);
enqueueSmartMoneyEventInsert(batchWriter, smartMoneyEvent);
await publishJson(js, SUBJECT_SMART_MONEY_EVENTS, smartMoneyEvent);
emitCounters.smartMoneyEvents += 1;
@ -1282,20 +1311,29 @@ const run = async () => {
if (env.SMART_MONEY_EVENT_CALENDAR_PATH) {
try {
eventCalendarProvider = await loadEventCalendarProviderFromFile(env.SMART_MONEY_EVENT_CALENDAR_PATH);
logger.info("smart money event calendar loaded", { path: env.SMART_MONEY_EVENT_CALENDAR_PATH });
eventCalendarProvider = await loadEventCalendarProviderFromFile(
env.SMART_MONEY_EVENT_CALENDAR_PATH
);
logger.info("smart money event calendar loaded", {
path: env.SMART_MONEY_EVENT_CALENDAR_PATH
});
} catch (error) {
eventCalendarProvider = createEmptyEventCalendarProvider();
logger.warn("smart money event calendar unavailable; scoring will use neutral event features", {
path: env.SMART_MONEY_EVENT_CALENDAR_PATH,
error: error instanceof Error ? error.message : String(error)
});
logger.warn(
"smart money event calendar unavailable; scoring will use neutral event features",
{
path: env.SMART_MONEY_EVENT_CALENDAR_PATH,
error: error instanceof Error ? error.message : String(error)
}
);
}
}
const redis = createRedisClient(env.REDIS_URL);
redis.on("error", (error) => {
logger.warn("redis client error", { error: error instanceof Error ? error.message : String(error) });
logger.warn("redis client error", {
error: error instanceof Error ? error.message : String(error)
});
});
await retry("redis connect", 120, 500, async () => {
@ -1379,7 +1417,10 @@ const run = async () => {
} else {
try {
const info = await jsm.consumers.info(STREAM_OPTION_SIGNAL_PRINTS, durableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
if (
info?.config?.deliver_policy &&
info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY
) {
logger.warn("resetting consumer due to deliver policy change", {
durable: durableName,
current: info.config.deliver_policy,
@ -1390,7 +1431,10 @@ const run = async () => {
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) {
logger.warn("failed to inspect jetstream consumer", { durable: durableName, error: message });
logger.warn("failed to inspect jetstream consumer", {
durable: durableName,
error: message
});
}
}
}
@ -1402,13 +1446,19 @@ const run = async () => {
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) {
logger.warn("failed to reset jetstream consumer", { durable: nbboDurableName, error: message });
logger.warn("failed to reset jetstream consumer", {
durable: nbboDurableName,
error: message
});
}
}
} else {
try {
const info = await jsm.consumers.info(STREAM_OPTION_NBBO, nbboDurableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
if (
info?.config?.deliver_policy &&
info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY
) {
logger.warn("resetting consumer due to deliver policy change", {
durable: nbboDurableName,
current: info.config.deliver_policy,
@ -1419,7 +1469,10 @@ const run = async () => {
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) {
logger.warn("failed to inspect jetstream consumer", { durable: nbboDurableName, error: message });
logger.warn("failed to inspect jetstream consumer", {
durable: nbboDurableName,
error: message
});
}
}
}
@ -1440,7 +1493,10 @@ const run = async () => {
} else {
try {
const info = await jsm.consumers.info(STREAM_EQUITY_PRINTS, equityPrintDurableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
if (
info?.config?.deliver_policy &&
info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY
) {
logger.warn("resetting consumer due to deliver policy change", {
durable: equityPrintDurableName,
current: info.config.deliver_policy,
@ -1475,7 +1531,10 @@ const run = async () => {
} else {
try {
const info = await jsm.consumers.info(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
if (
info?.config?.deliver_policy &&
info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY
) {
logger.warn("resetting consumer due to deliver policy change", {
durable: equityQuoteDurableName,
current: info.config.deliver_policy,
@ -1515,7 +1574,8 @@ const run = async () => {
try {
await jsm.consumers.delete(STREAM_OPTION_SIGNAL_PRINTS, durableName);
} catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
const deleteMessage =
deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", {
durable: durableName,
@ -1551,7 +1611,8 @@ const run = async () => {
try {
await jsm.consumers.delete(STREAM_OPTION_NBBO, nbboDurableName);
} catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
const deleteMessage =
deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", {
durable: nbboDurableName,
@ -1582,12 +1643,16 @@ const run = async () => {
throw error;
}
logger.warn("resetting jetstream consumer", { durable: equityPrintDurableName, error: message });
logger.warn("resetting jetstream consumer", {
durable: equityPrintDurableName,
error: message
});
try {
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, equityPrintDurableName);
} catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
const deleteMessage =
deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", {
durable: equityPrintDurableName,
@ -1626,7 +1691,8 @@ const run = async () => {
try {
await jsm.consumers.delete(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
} catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
const deleteMessage =
deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", {
durable: equityQuoteDurableName,

View file

@ -78,7 +78,9 @@ const getDteDays = (packet: FlowPacket): number | null => {
const inferDirection = (packet: FlowPacket): SmartMoneyDirection => {
const structureRights = stringFeature(packet, "structure_rights");
const optionType = stringFeature(packet, "option_type") || parseContractId(stringFeature(packet, "option_contract_id"))?.right;
const optionType =
stringFeature(packet, "option_type") ||
parseContractId(stringFeature(packet, "option_contract_id"))?.right;
const buy = numberFeature(packet, "nbbo_aggressive_buy_ratio");
const sell = numberFeature(packet, "nbbo_aggressive_sell_ratio");
const sellDominant = sell >= buy + 0.12;
@ -102,16 +104,26 @@ export type SmartMoneyParentEventOptions = {
eventCalendarMatch?: EventCalendarMatch | null;
};
const buildFeatures = (packet: FlowPacket, options: SmartMoneyParentEventOptions = {}): SmartMoneyFeatures => {
const buildFeatures = (
packet: FlowPacket,
options: SmartMoneyParentEventOptions = {}
): SmartMoneyFeatures => {
const contractId = stringFeature(packet, "option_contract_id");
const contract = parseContractId(contractId);
const underlyingMid = numberFeature(packet, "underlying_mid");
const quoteAge = numberFeature(packet, "nbbo_age_ms") || numberFeature(packet, "underlying_quote_age_ms");
const printCount = Math.max(0, Math.round(numberFeature(packet, "count") || packet.members.length));
const quoteAge =
numberFeature(packet, "nbbo_age_ms") || numberFeature(packet, "underlying_quote_age_ms");
const printCount = Math.max(
0,
Math.round(numberFeature(packet, "count") || packet.members.length)
);
const staleCount = numberFeature(packet, "nbbo_stale_count");
const missingCount = numberFeature(packet, "nbbo_missing_count");
const structureLegs = Math.max(0, Math.round(numberFeature(packet, "structure_legs")));
const strikeCount = Math.max(1, Math.round(numberFeature(packet, "structure_strikes") || (contract ? 1 : 0)));
const strikeCount = Math.max(
1,
Math.round(numberFeature(packet, "structure_strikes") || (contract ? 1 : 0))
);
const specialCount = numberFeature(packet, "special_print_count");
const calendarEventTs = options.eventCalendarMatch?.event_ts ?? null;
const eventTs = calendarEventTs ?? numberFeature(packet, "corporate_event_ts");
@ -119,7 +131,9 @@ const buildFeatures = (packet: FlowPacket, options: SmartMoneyParentEventOptions
const expiryTs = contract ? Date.parse(`${contract.expiry}T00:00:00Z`) : Number.NaN;
const atmProximity =
contract && underlyingMid > 0 ? Math.abs(contract.strike - underlyingMid) / underlyingMid : null;
contract && underlyingMid > 0
? Math.abs(contract.strike - underlyingMid) / underlyingMid
: null;
return {
contract_count: Math.max(1, structureLegs || 1),
@ -143,14 +157,18 @@ const buildFeatures = (packet: FlowPacket, options: SmartMoneyParentEventOptions
nbbo_stale_ratio: printCount > 0 ? clamp((staleCount + missingCount) / printCount) : 0,
quote_age_ms: quoteAge > 0 ? quoteAge : null,
venue_count: Math.max(1, Math.round(numberFeature(packet, "venue_count") || 1)),
inter_fill_ms_mean: printCount > 1 ? numberFeature(packet, "window_ms") / Math.max(1, printCount - 1) : null,
inter_fill_ms_mean:
printCount > 1 ? numberFeature(packet, "window_ms") / Math.max(1, printCount - 1) : null,
strike_count: strikeCount,
strike_concentration: strikeCount > 0 ? clamp(1 / strikeCount) : 0,
...(stringFeature(packet, "structure_type") ? { structure_type: stringFeature(packet, "structure_type") } : {}),
...(stringFeature(packet, "structure_type")
? { structure_type: stringFeature(packet, "structure_type") }
: {}),
structure_legs: structureLegs,
same_size_leg_symmetry: clamp(numberFeature(packet, "same_size_leg_symmetry")),
net_directional_bias: clamp(
numberFeature(packet, "nbbo_aggressive_buy_ratio") - numberFeature(packet, "nbbo_aggressive_sell_ratio"),
numberFeature(packet, "nbbo_aggressive_buy_ratio") -
numberFeature(packet, "nbbo_aggressive_sell_ratio"),
-1,
1
),
@ -159,7 +177,10 @@ const buildFeatures = (packet: FlowPacket, options: SmartMoneyParentEventOptions
underlying_move_bps: numberFeature(packet, "underlying_move_bps") || null,
days_to_event: eventTs > 0 ? (eventTs - referenceTs) / MS_PER_DAY : null,
expiry_after_event: eventTs > 0 && Number.isFinite(expiryTs) ? expiryTs >= eventTs : null,
pre_event_concentration: eventTs > 0 && eventTs >= referenceTs ? clamp(1 - (eventTs - referenceTs) / (21 * MS_PER_DAY)) : null,
pre_event_concentration:
eventTs > 0 && eventTs >= referenceTs
? clamp(1 - (eventTs - referenceTs) / (21 * MS_PER_DAY))
: null,
special_print_ratio: printCount > 0 ? clamp(specialCount / printCount) : 0
};
};
@ -170,7 +191,10 @@ const detectSuppression = (packet: FlowPacket, features: SmartMoneyFeatures): st
.split(",")
.map((item) => item.trim().toUpperCase())
.filter(Boolean);
if (conditions.some((condition) => SPECIAL_CONDITIONS.has(condition)) || features.special_print_ratio >= 0.34) {
if (
conditions.some((condition) => SPECIAL_CONDITIONS.has(condition)) ||
features.special_print_ratio >= 0.34
) {
reasons.push("special_print_or_complex_context");
}
if (features.nbbo_coverage_ratio < 0.35 || features.nbbo_stale_ratio >= 0.5) {
@ -198,7 +222,10 @@ const evaluateProfiles = (
const burstFactor = clamp(features.print_count / 8);
const quality = clamp(features.nbbo_coverage_ratio - features.nbbo_stale_ratio);
const shortDatedOtm =
dte <= 7 && features.atm_proximity !== null && features.atm_proximity >= 0.05 && features.option_type === "C";
dte <= 7 &&
features.atm_proximity !== null &&
features.atm_proximity >= 0.05 &&
features.option_type === "C";
const nearAtm = features.atm_proximity !== null && features.atm_proximity <= 0.015;
const preEvent =
features.days_to_event !== null &&
@ -211,7 +238,11 @@ const evaluateProfiles = (
"institutional_directional",
suppressed.length > 0 || shortDatedOtm
? 0.18
: 0.2 + premiumFactor * 0.25 + burstFactor * 0.18 + quality * 0.16 + (buy >= 0.58 || sell >= 0.58 ? 0.12 : 0),
: 0.2 +
premiumFactor * 0.25 +
burstFactor * 0.18 +
quality * 0.16 +
(buy >= 0.58 || sell >= 0.58 ? 0.12 : 0),
direction,
[
"large_parent_event",
@ -232,13 +263,19 @@ const evaluateProfiles = (
),
score(
"event_driven",
0.12 + (preEvent ? 0.32 : 0) + premiumFactor * 0.14 + clamp(features.spread_widening ?? 0, 0, 0.16),
0.12 +
(preEvent ? 0.32 : 0) +
premiumFactor * 0.14 +
clamp(features.spread_widening ?? 0, 0, 0.16),
direction === "unknown" ? "neutral" : direction,
["event_calendar_alignment", "expiry_after_event", "pre_event_concentration"]
),
score(
"vol_seller",
0.12 + (sell >= 0.58 ? 0.24 : 0) + (structure === "straddle" || structure === "strangle" ? 0.2 : 0) + premiumFactor * 0.14,
0.12 +
(sell >= 0.58 ? 0.24 : 0) +
(structure === "straddle" || structure === "strangle" ? 0.2 : 0) +
premiumFactor * 0.14,
"neutral",
["sell_side_premium", "short_vol_structure_evidence"]
),
@ -273,11 +310,16 @@ export const buildSmartMoneyEventFromPacket = (
const suppressed = detectSuppression(packet, features);
const profileScores = evaluateProfiles(packet, features, suppressed);
const primary = profileScores[0] ?? null;
const abstained = !primary || primary.probability < 0.42 || suppressed.includes("stale_or_missing_quote_context");
const underlying = stringFeature(packet, "underlying_id") || parseContractId(features.option_contract_id ?? "")?.root || "UNKNOWN";
const eventKind = features.structure_legs >= 2 || stringFeature(packet, "packet_kind") === "structure"
? "multi_leg_event"
: "single_leg_event";
const abstained =
!primary || primary.probability < 0.42 || suppressed.includes("stale_or_missing_quote_context");
const underlying =
stringFeature(packet, "underlying_id") ||
parseContractId(features.option_contract_id ?? "")?.root ||
"UNKNOWN";
const eventKind =
features.structure_legs >= 2 || stringFeature(packet, "packet_kind") === "structure"
? "multi_leg_event"
: "single_leg_event";
return SmartMoneyEventSchema.parse({
source_ts: packet.source_ts,
@ -292,8 +334,8 @@ export const buildSmartMoneyEventFromPacket = (
event_window_ms: features.window_ms,
features,
profile_scores: profileScores,
primary_profile_id: abstained ? null : primary?.profile_id ?? null,
primary_direction: abstained ? "unknown" : primary?.direction ?? "unknown",
primary_profile_id: abstained ? null : (primary?.profile_id ?? null),
primary_direction: abstained ? "unknown" : (primary?.direction ?? "unknown"),
abstained,
suppressed_reasons: suppressed
});
@ -308,7 +350,9 @@ const LEGACY_PROFILE_MAP: Record<SmartMoneyProfileId, string> = {
hedge_reactive: "smart_money_hedge_reactive"
};
export const deriveClassifierHitsFromSmartMoneyEvent = (event: SmartMoneyEvent): ClassifierHit[] => {
export const deriveClassifierHitsFromSmartMoneyEvent = (
event: SmartMoneyEvent
): ClassifierHit[] => {
if (event.abstained || !event.primary_profile_id) {
return [];
}

View file

@ -24,9 +24,7 @@ type RollingWindowEntry = {
};
const toNumbers = (values: string[]): number[] => {
return values
.map((value) => Number(value))
.filter((value) => Number.isFinite(value));
return values.map((value) => Number(value)).filter((value) => Number.isFinite(value));
};
export const computeStats = (values: number[]): { mean: number; stddev: number; count: number } => {

View file

@ -1,4 +1,9 @@
import type { FlowPacket, SmartMoneyDirection, SmartMoneyEvent, SmartMoneyProfileId } from "@islandflow/types";
import type {
FlowPacket,
SmartMoneyDirection,
SmartMoneyEvent,
SmartMoneyProfileId
} from "@islandflow/types";
import { buildSmartMoneyEventFromPacket, type SmartMoneyParentEventOptions } from "./parent-events";
export type SmartMoneyLabel = {
@ -115,8 +120,12 @@ export const compareSmartMoneyReplayOutputs = (
liveEvents: SmartMoneyEvent[],
batchEvents: SmartMoneyEvent[]
): ReplayConsistencyReport => {
const liveById = new Map(liveEvents.map((event) => [event.event_id, smartMoneyEventSignature(event)]));
const batchById = new Map(batchEvents.map((event) => [event.event_id, smartMoneyEventSignature(event)]));
const liveById = new Map(
liveEvents.map((event) => [event.event_id, smartMoneyEventSignature(event)])
);
const batchById = new Map(
batchEvents.map((event) => [event.event_id, smartMoneyEventSignature(event)])
);
const ids = [...new Set([...liveById.keys(), ...batchById.keys()])].sort();
const mismatches: ReplayConsistencyMismatch[] = [];
@ -153,7 +162,9 @@ export const evaluateSmartMoneyEvents = (
const labelsById = new Map(labels.map((label) => [label.event_id, label]));
const labeledEvents = events
.map((event) => ({ event, label: labelsById.get(event.event_id) }))
.filter((entry): entry is { event: SmartMoneyEvent; label: SmartMoneyLabel } => Boolean(entry.label));
.filter((entry): entry is { event: SmartMoneyEvent; label: SmartMoneyLabel } =>
Boolean(entry.label)
);
const emitted = events.filter((event) => !event.abstained && event.primary_profile_id);
const profilePrecision: SmartMoneyEvaluationReport["profile_precision"] = {};
@ -163,7 +174,8 @@ export const evaluateSmartMoneyEvents = (
const predicted = labeledEvents.filter((entry) => entry.event.primary_profile_id === profile);
const actual = labeledEvents.filter((entry) => entry.label.profile_id === profile);
const truePositive = predicted.filter((entry) => entry.label.profile_id === profile).length;
profilePrecision[profile] = predicted.length > 0 ? round(truePositive / predicted.length) : null;
profilePrecision[profile] =
predicted.length > 0 ? round(truePositive / predicted.length) : null;
profileRecall[profile] = actual.length > 0 ? round(truePositive / actual.length) : null;
}
@ -175,7 +187,10 @@ export const evaluateSmartMoneyEvents = (
labeled_count: labeledEvents.length,
emitted_count: emitted.length,
abstained_count: events.filter((event) => event.abstained).length,
abstention_rate: events.length > 0 ? round(events.filter((event) => event.abstained).length / events.length) : 0,
abstention_rate:
events.length > 0
? round(events.filter((event) => event.abstained).length / events.length)
: 0,
profile_precision: profilePrecision,
profile_recall: profileRecall,
calibration,
@ -195,7 +210,9 @@ const buildCalibration = (
}));
for (const { event, label } of entries) {
const probability = event.profile_scores.find((entry) => entry.profile_id === event.primary_profile_id)?.probability ?? 0;
const probability =
event.profile_scores.find((entry) => entry.profile_id === event.primary_profile_id)
?.probability ?? 0;
const index = Math.min(bucketCount - 1, Math.floor(probability * bucketCount));
buckets[index].probabilities.push(probability);
if (!event.abstained && event.primary_profile_id === label.profile_id) {
@ -209,9 +226,13 @@ const buildCalibration = (
count: bucket.probabilities.length,
average_probability:
bucket.probabilities.length > 0
? round(bucket.probabilities.reduce((sum, value) => sum + value, 0) / bucket.probabilities.length)
? round(
bucket.probabilities.reduce((sum, value) => sum + value, 0) /
bucket.probabilities.length
)
: 0,
accuracy: bucket.probabilities.length > 0 ? round(bucket.correct / bucket.probabilities.length) : null
accuracy:
bucket.probabilities.length > 0 ? round(bucket.correct / bucket.probabilities.length) : null
}));
};
@ -223,7 +244,10 @@ const buildEconomicSanity = (
sign: directionalSign(event.primary_direction),
realized: label.realized_return_bps
}))
.filter((entry): entry is { sign: number; realized: number } => entry.sign !== 0 && Number.isFinite(entry.realized));
.filter(
(entry): entry is { sign: number; realized: number } =>
entry.sign !== 0 && Number.isFinite(entry.realized)
);
if (directional.length === 0) {
return {
@ -236,7 +260,12 @@ const buildEconomicSanity = (
const signedReturns = directional.map((entry) => entry.sign * entry.realized);
return {
directional_count: directional.length,
direction_hit_rate: round(signedReturns.filter((value) => value > 0).length / directional.length),
average_signed_return_bps: round(signedReturns.reduce((sum, value) => sum + value, 0) / signedReturns.length, 2)
direction_hit_rate: round(
signedReturns.filter((value) => value > 0).length / directional.length
),
average_signed_return_bps: round(
signedReturns.reduce((sum, value) => sum + value, 0) / signedReturns.length,
2
)
};
};

View file

@ -134,7 +134,9 @@ const dayDiff = (from: string | null, to: string | null): number | null => {
};
const sameSizeLegSymmetry = (legs: LegEvidence[]): number => {
const sizes = legs.map((leg) => leg.totalSize).filter((value) => Number.isFinite(value) && value > 0);
const sizes = legs
.map((leg) => leg.totalSize)
.filter((value) => Number.isFinite(value) && value > 0);
if (sizes.length < 2) {
return 0;
}
@ -146,7 +148,10 @@ const sameSizeLegSymmetry = (legs: LegEvidence[]): number => {
return min / max;
};
export const shouldEmitStructurePacket = (legs: LegEvidence[], currentLegContractId: string): boolean => {
export const shouldEmitStructurePacket = (
legs: LegEvidence[],
currentLegContractId: string
): boolean => {
if (legs.length < 2) {
return false;
}
@ -226,7 +231,8 @@ export const planStructurePacket = (
const totalSize = legs.reduce((sum, leg) => sum + leg.totalSize, 0);
const count = legs.reduce((sum, leg) => sum + leg.members.length, 0);
const placements = mergePlacements(legs);
const placementTotal = placements.aa + placements.a + placements.b + placements.bb + placements.mid;
const placementTotal =
placements.aa + placements.a + placements.b + placements.bb + placements.mid;
const aggressiveTotal = placements.aa + placements.a + placements.b + placements.bb;
const aggressiveBuy = placements.aa + placements.a;
const aggressiveSell = placements.bb + placements.b;
@ -235,7 +241,10 @@ export const planStructurePacket = (
const nbboAggressiveSellRatio = aggressiveTotal > 0 ? aggressiveSell / aggressiveTotal : 0;
const nbboAggressiveRatio = placementTotal > 0 ? aggressiveTotal / placementTotal : 0;
const source_ts = legs.reduce((min, leg) => Math.min(min, leg.source_ts), Number.POSITIVE_INFINITY);
const source_ts = legs.reduce(
(min, leg) => Math.min(min, leg.source_ts),
Number.POSITIVE_INFINITY
);
const ingest_ts = legs.reduce((max, leg) => Math.max(max, leg.ingest_ts), 0);
const seq = legs.reduce((max, leg) => Math.max(max, leg.seq), 0);

View file

@ -47,7 +47,10 @@ export const summarizeStructure = (legs: ContractLeg[]): StructureSummary | null
legs: legs.length,
strikes: strikes.length,
strikeSpan,
rights: rights.size === 2 ? "C/P" : Array.from(rights)[0] ?? "",
contractIds: legs.map((leg) => leg.contractId).slice().sort()
rights: rights.size === 2 ? "C/P" : (Array.from(rights)[0] ?? ""),
contractIds: legs
.map((leg) => leg.contractId)
.slice()
.sort()
};
};

View file

@ -293,4 +293,3 @@ describe("compute classifiers", () => {
expect(hit!.explanations[0]).toMatch(/Consistent with/i);
});
});

View file

@ -17,16 +17,18 @@ export const TEST_CLASSIFIER_CONFIG: ClassifierConfig = {
zeroDteMinSize: 400
};
export const buildFlowPacket = (opts: {
id?: string;
source_ts?: number;
ingest_ts?: number;
seq?: number;
trace_id?: string;
members?: string[];
features?: FlowPacket["features"];
join_quality?: FlowPacket["join_quality"];
} = {}): FlowPacket => {
export const buildFlowPacket = (
opts: {
id?: string;
source_ts?: number;
ingest_ts?: number;
seq?: number;
trace_id?: string;
members?: string[];
features?: FlowPacket["features"];
join_quality?: FlowPacket["join_quality"];
} = {}
): FlowPacket => {
const id = opts.id ?? "flowpacket:test";
const source_ts = opts.source_ts ?? Date.parse("2025-01-01T14:30:00Z");
const ingest_ts = opts.ingest_ts ?? source_ts;
@ -66,4 +68,3 @@ export const buildFlowPacket = (opts: {
export const getHit = (hits: ClassifierHit[], id: string): ClassifierHit | null => {
return hits.find((hit) => hit.classifier_id === id) ?? null;
};

View file

@ -18,7 +18,9 @@ const placements = (overrides?: Partial<LegEvidence["placements"]>): LegEvidence
...overrides
});
const leg = (input: Partial<LegEvidence> & Pick<LegEvidence, "contractId" | "right" | "strike">): LegEvidence => {
const leg = (
input: Partial<LegEvidence> & Pick<LegEvidence, "contractId" | "right" | "strike">
): LegEvidence => {
return {
contractId: input.contractId,
root: "SPY",

View file

@ -85,10 +85,14 @@ const decodePayload = (data: WebSocket.RawData): unknown => {
}
if (ArrayBuffer.isView(data)) {
return JSON.parse(new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))) as unknown;
return JSON.parse(
new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))
) as unknown;
}
return JSON.parse(new TextDecoder().decode(new Uint8Array(data as unknown as ArrayBuffer))) as unknown;
return JSON.parse(
new TextDecoder().decode(new Uint8Array(data as unknown as ArrayBuffer))
) as unknown;
};
const extractExchangeMeta = (payload: unknown): AlpacaExchangeMetaEntry[] => {
@ -103,8 +107,18 @@ const extractExchangeMeta = (payload: unknown): AlpacaExchangeMetaEntry[] => {
continue;
}
const candidate = entry as Record<string, unknown>;
const code = typeof candidate.code === "string" ? candidate.code : typeof candidate.exchange === "string" ? candidate.exchange : null;
const name = typeof candidate.name === "string" ? candidate.name : typeof candidate.description === "string" ? candidate.description : null;
const code =
typeof candidate.code === "string"
? candidate.code
: typeof candidate.exchange === "string"
? candidate.exchange
: null;
const name =
typeof candidate.name === "string"
? candidate.name
: typeof candidate.description === "string"
? candidate.description
: null;
if (!code || !name) {
continue;
}
@ -128,9 +142,19 @@ const buildExchangeNameMap = (entries: AlpacaExchangeMetaEntry[]): Map<string, s
return map;
};
const OFF_EXCHANGE_HINTS = ["FINRA", "TRF", "ADF", "OTC", "Trade Reporting Facility", "Alternative Display Facility"];
const OFF_EXCHANGE_HINTS = [
"FINRA",
"TRF",
"ADF",
"OTC",
"Trade Reporting Facility",
"Alternative Display Facility"
];
export const inferOffExchangeFlag = (exchangeCode: string | undefined, exchangeNameMap: Map<string, string>): boolean => {
export const inferOffExchangeFlag = (
exchangeCode: string | undefined,
exchangeNameMap: Map<string, string>
): boolean => {
if (!exchangeCode) {
return false;
}
@ -151,7 +175,9 @@ const buildWsUrl = (wsBaseUrl: string, feed: AlpacaEquitiesFeed): string => {
return `${parsed.origin}/v2/${feed}`;
};
const fetchExchangeMeta = async (config: AlpacaEquitiesAdapterConfig): Promise<Map<string, string>> => {
const fetchExchangeMeta = async (
config: AlpacaEquitiesAdapterConfig
): Promise<Map<string, string>> => {
const url = new URL("/v2/stocks/meta/exchanges", config.restUrl);
try {
@ -243,7 +269,10 @@ export const createAlpacaEquitiesAdapter = (
continue;
}
const message = entry as (AlpacaTradeMessage | AlpacaQuoteMessage | { T?: string; msg?: string });
const message = entry as
| AlpacaTradeMessage
| AlpacaQuoteMessage
| { T?: string; msg?: string };
const type = message.T;
if (type === "success") {

View file

@ -89,11 +89,7 @@ const priceForPlacement = (
return formatPrice(Math.max(0.01, price));
};
const buildQuoteContext = (
symbol: string,
now: number,
control: SyntheticControlState
) => {
const buildQuoteContext = (symbol: string, now: number, control: SyntheticControlState) => {
const session = getSyntheticSessionState(now, control);
const state = getSyntheticUnderlyingState(symbol, now, control, session);
return {
@ -184,7 +180,9 @@ export const createSyntheticEquitiesAdapter = (
session.regime === "retail_chase";
if (allowDark) {
const darkSymbol = focusSymbols[seq % focusSymbols.length] ?? SYNTHETIC_SYMBOLS[symbolCursor % SYNTHETIC_SYMBOLS.length]!;
const darkSymbol =
focusSymbols[seq % focusSymbols.length] ??
SYNTHETIC_SYMBOLS[symbolCursor % SYNTHETIC_SYMBOLS.length]!;
const darkQuote = buildQuoteContext(darkSymbol, now, control);
const darkPlacement = pickDarkPlacement(
darkQuote.state.driftBps,
@ -203,13 +201,7 @@ export const createSyntheticEquitiesAdapter = (
if (handlers.onQuote) {
quoteSeq += 1;
void handlers.onQuote(
buildSyntheticQuote(
quoteSeq,
now - 2,
darkSymbol,
darkQuote.bid,
darkQuote.ask
)
buildSyntheticQuote(quoteSeq, now - 2, darkSymbol, darkQuote.bid, darkQuote.ask)
);
}
@ -236,11 +228,7 @@ export const createSyntheticEquitiesAdapter = (
const eventTs = now + i * 4;
const quote = buildQuoteContext(symbol, eventTs, control);
const clustered = focusSet.has(symbol);
const placement = pickPrimaryPlacement(
quote.state.driftBps,
session.regime,
seq + i
);
const placement = pickPrimaryPlacement(quote.state.driftBps, session.regime, seq + i);
const exchange = EXCHANGES[(seq + symbol.charCodeAt(0) + i) % EXCHANGES.length]!;
const baseSize =
throughput.litSizeBase +
@ -255,13 +243,7 @@ export const createSyntheticEquitiesAdapter = (
if (handlers.onQuote) {
quoteSeq += 1;
void handlers.onQuote(
buildSyntheticQuote(
quoteSeq,
eventTs - 2,
symbol,
quote.bid,
quote.ask
)
buildSyntheticQuote(quoteSeq, eventTs - 2, symbol, quote.bid, quote.ask)
);
}

View file

@ -240,10 +240,7 @@ const run = async () => {
await ensureEquityQuotesTable(clickhouse);
});
const adapter = selectAdapter(
env.EQUITIES_INGEST_ADAPTER,
() => syntheticControl
);
const adapter = selectAdapter(env.EQUITIES_INGEST_ADAPTER, () => syntheticControl);
logger.info("ingest adapter selected", { adapter: adapter.name });
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
const allowQuotePublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);

View file

@ -126,9 +126,13 @@ const decodePayload = (data: WebSocket.RawData): unknown => {
return JSON.parse(new TextDecoder().decode(new Uint8Array(data))) as unknown;
}
if (ArrayBuffer.isView(data)) {
return JSON.parse(new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))) as unknown;
return JSON.parse(
new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))
) as unknown;
}
return JSON.parse(new TextDecoder().decode(new Uint8Array(data as unknown as ArrayBuffer))) as unknown;
return JSON.parse(
new TextDecoder().decode(new Uint8Array(data as unknown as ArrayBuffer))
) as unknown;
};
const run = async () => {

View file

@ -152,10 +152,7 @@ const normalizeUnderlyings = (value: string[]): string[] => {
return result;
};
const fetchJson = async <T>(
url: URL,
config: AlpacaOptionsAdapterConfig
): Promise<T> => {
const fetchJson = async <T>(url: URL, config: AlpacaOptionsAdapterConfig): Promise<T> => {
const response = await fetch(url.toString(), {
headers: buildAlpacaAuthHeaders(config.credentials)
});
@ -235,10 +232,7 @@ const fetchOptionSnapshots = async (
return contracts;
};
const selectExpiries = (
contracts: OptionContract[],
maxDteDays: number
): ExpiryInfo[] => {
const selectExpiries = (contracts: OptionContract[], maxDteDays: number): ExpiryInfo[] => {
const today = new Date();
const expiryMap = new Map<string, ExpiryInfo>();
@ -332,7 +326,9 @@ const selectContractsForUnderlying = (
const minStrike = price * (1 - config.moneynessPct);
const maxStrike = price * (1 + config.moneynessPct);
const strikePairs = Array.from(strikeMap.entries())
.filter(([strike, pair]) => pair.call && pair.put && strike >= minStrike && strike <= maxStrike)
.filter(
([strike, pair]) => pair.call && pair.put && strike >= minStrike && strike <= maxStrike
)
.map(([strike, pair]) => ({
strike,
call: pair.call as string,
@ -540,7 +536,10 @@ export const createAlpacaOptionsAdapter = (
continue;
}
const message = entry as AlpacaTradeMessage | AlpacaQuoteMessage | { T?: string; msg?: string };
const message = entry as
| AlpacaTradeMessage
| AlpacaQuoteMessage
| { T?: string; msg?: string };
const type = message.T;
if (type === "t") {

View file

@ -235,8 +235,7 @@ export const createDatabentoOptionsAdapter = (
return;
}
const scaledPrice =
config.priceScale === 1 ? price : price / config.priceScale;
const scaledPrice = config.priceScale === 1 ? price : price / config.priceScale;
const conditions = Array.isArray(payload.conditions)
? payload.conditions.map((entry) => String(entry))

View file

@ -59,9 +59,7 @@ const readLines = async (
}
};
export const createIbkrOptionsAdapter = (
config: IbkrOptionsAdapterConfig
): OptionIngestAdapter => {
export const createIbkrOptionsAdapter = (config: IbkrOptionsAdapterConfig): OptionIngestAdapter => {
return {
name: "ibkr",
start: (handlers: OptionIngestHandlers) => {

View file

@ -715,10 +715,7 @@ const SYNTHETIC_PROFILES: Record<SyntheticMarketMode, SyntheticOptionsProfile> =
...scenario,
countRange: [scenario.countRange[0], scenario.countRange[1]],
sizeRange: [scenario.sizeRange[0], scenario.sizeRange[1]],
targetNotionalRange: [
scenario.targetNotionalRange[0],
scenario.targetNotionalRange[1]
]
targetNotionalRange: [scenario.targetNotionalRange[0], scenario.targetNotionalRange[1]]
})),
pricePlacements: PLACEMENTS
},
@ -743,10 +740,7 @@ const SYNTHETIC_PROFILES: Record<SyntheticMarketMode, SyntheticOptionsProfile> =
scenarios: SCENARIO_LIBRARY.map((scenario) => ({
...scenario,
countRange: [scenario.countRange[0] + 2, scenario.countRange[1] + 4],
sizeRange: [
Math.round(scenario.sizeRange[0] * 1.8),
Math.round(scenario.sizeRange[1] * 2.1)
],
sizeRange: [Math.round(scenario.sizeRange[0] * 1.8), Math.round(scenario.sizeRange[1] * 2.1)],
targetNotionalRange: [
Math.round(scenario.targetNotionalRange[0] * 1.7),
Math.round(scenario.targetNotionalRange[1] * 2.0)
@ -768,7 +762,7 @@ const SMART_MONEY_TEMPLATE_SCENARIOS: Record<
hedge_reactive: "reactive_put_wall"
};
const pick = <T,>(items: readonly T[], seed: number): T => {
const pick = <T>(items: readonly T[], seed: number): T => {
return items[Math.abs(seed) % items.length]!;
};
@ -850,9 +844,7 @@ export const updateSyntheticIvForTest = (
const sizeImpact = Math.log10(Math.max(10, input.size)) * 0.012;
const notionalImpact = Math.log10(Math.max(1_000, input.notional)) * 0.01;
pressure +=
input.placement === "AA"
? sizeImpact + notionalImpact
: (sizeImpact + notionalImpact) * 0.65;
input.placement === "AA" ? sizeImpact + notionalImpact : (sizeImpact + notionalImpact) * 0.65;
} else if (input.placement === "MID") {
pressure += 0.001;
} else {
@ -879,8 +871,7 @@ const estimateSyntheticOptionMid = (input: {
: Math.max(0, input.strike - input.underlying);
const timeYears = Math.max(1, input.dteDays + 1) / 365;
const baselineIv = initializeSyntheticIv(input.dteDays, input.moneyness);
const modeBoost =
input.mode === "firehose" ? 1.18 : input.mode === "active" ? 1.08 : 0.96;
const modeBoost = input.mode === "firehose" ? 1.18 : input.mode === "active" ? 1.08 : 0.96;
const distance = Math.abs(input.moneyness - 1);
const extrinsic =
input.underlying *
@ -939,12 +930,7 @@ const chooseScenario = (
): Scenario => {
const session = getSyntheticSessionState(now, control);
const focusSymbol = session.focus_symbols[0] ?? SYNTHETIC_SYMBOLS[0]!;
const familyWeights = getSyntheticScenarioWeights(
focusSymbol,
now,
control,
session
);
const familyWeights = getSyntheticScenarioWeights(focusSymbol, now, control, session);
const coverageCounts = getCoverageCounts(coverageState, now, control);
const weightedScenarios = profile.scenarios.map((scenario, index) => {
const familyWeight = familyWeights[scenario.label];
@ -964,7 +950,10 @@ const chooseScenario = (
: 1;
return {
...scenario,
weight: Math.max(1, Math.round(scenario.weight * familyWeight * coverageBoost * quietBias * 100))
weight: Math.max(
1,
Math.round(scenario.weight * familyWeight * coverageBoost * quietBias * 100)
)
};
});
return pickWeighted(weightedScenarios, now + control.shared_seed * 31);
@ -977,7 +966,8 @@ const pickScenarioSymbol = (
): string => {
const session = getSyntheticSessionState(now, control);
const symbolPool =
scenario.preferredSymbols?.length && (scenario.label === "event_driven" || Math.abs(now) % 4 === 0)
scenario.preferredSymbols?.length &&
(scenario.label === "event_driven" || Math.abs(now) % 4 === 0)
? [...scenario.preferredSymbols]
: session.focus_symbols.length > 0
? [...session.focus_symbols, ...SYNTHETIC_SYMBOLS]
@ -1033,11 +1023,12 @@ const buildDynamicFlowFeatures = (
0,
0.26
),
underlying_move_bps: Math.round(
(Number(scenario.flowFeatures.underlying_move_bps ?? underlying.driftBps) +
underlying.shockBps * 0.35) *
100
) / 100,
underlying_move_bps:
Math.round(
(Number(scenario.flowFeatures.underlying_move_bps ?? underlying.driftBps) +
underlying.shockBps * 0.35) *
100
) / 100,
venue_count: Math.max(
1,
Math.round(
@ -1059,18 +1050,14 @@ const buildBurst = (
coverageState: CoverageWindowState,
scenarioOverride?: Scenario
): Burst => {
const scenario =
scenarioOverride ?? chooseScenario(profile, now, control, coverageState);
const scenario = scenarioOverride ?? chooseScenario(profile, now, control, coverageState);
const symbol = pickScenarioSymbol(scenario, now, control);
const symbolHash = hashSyntheticSymbol(symbol);
const seed = symbolHash + burstIndex * 7;
const session = getSyntheticSessionState(now, control);
const underlyingState = getSyntheticUnderlyingState(symbol, now, control, session);
const baseUnderlying = underlyingState.mid;
const expiryOffset = pick(
scenario.expiryOffsets ?? EXPIRY_OFFSETS,
symbolHash + burstIndex
);
const expiryOffset = pick(scenario.expiryOffsets ?? EXPIRY_OFFSETS, symbolHash + burstIndex);
const strikeStep = baseUnderlying >= 200 ? 10 : baseUnderlying >= 100 ? 5 : 2.5;
const right =
scenario.right === "either"
@ -1099,16 +1086,15 @@ const buildBurst = (
const priceStep =
scenario.priceTrend === "up" ? 0.01 : scenario.priceTrend === "down" ? -0.01 : 0;
const flowFeatures = buildDynamicFlowFeatures(scenario, symbol, now, control);
const legTemplates =
scenario.legs?.length
? scenario.legs
: [
{
right,
strikeMoneyness: scenario.strikeMoneyness,
placementScenarioId: scenario.placementProfile ?? scenario.label
}
];
const legTemplates = scenario.legs?.length
? scenario.legs
: [
{
right,
strikeMoneyness: scenario.strikeMoneyness,
placementScenarioId: scenario.placementProfile ?? scenario.label
}
];
const targetNotionalPerLeg = targetNotional / legTemplates.length;
const legs = legTemplates.map((template, legIndex): BurstLeg => {
@ -1127,8 +1113,7 @@ const buildBurst = (
const strike = Math.max(
1,
templateStrike ??
Math.round(baseUnderlying / strikeStep) * strikeStep +
strikeOffset * strikeStep
Math.round(baseUnderlying / strikeStep) * strikeStep + strikeOffset * strikeStep
);
const legSize = Math.max(1, Math.round(baseSize * (template.sizeMultiplier ?? 1)));
const legMoneyness = strike / baseUnderlying;
@ -1141,13 +1126,13 @@ const buildBurst = (
mode
});
const targetMid =
targetNotionalPerLeg /
Math.max(1, legSize * cycles * OPTION_CONTRACT_MULTIPLIER);
targetNotionalPerLeg / Math.max(1, legSize * cycles * OPTION_CONTRACT_MULTIPLIER);
const cappedTheoreticalMid = Math.min(
theoreticalMid,
Math.max(0.35, targetMid * (scenario.label === "institutional_directional" ? 2.2 : 2.6))
);
const blendedMid = cappedTheoreticalMid * 0.45 + targetMid * 0.55 * (template.priceMultiplier ?? 1);
const blendedMid =
cappedTheoreticalMid * 0.45 + targetMid * 0.55 * (template.priceMultiplier ?? 1);
return {
contractId: `${symbol}-${expiry}-${formatStrike(strike)}-${template.right}`,
right: template.right,
@ -1184,8 +1169,7 @@ const buildBurst = (
scenario.missingQuoteProbability ??
clampValue((1 - session.quote_cleanliness) * 0.16, 0, 0.18),
staleQuoteProbability:
scenario.staleQuoteProbability ??
clampValue((1 - session.quote_cleanliness) * 0.3, 0, 0.42)
scenario.staleQuoteProbability ?? clampValue((1 - session.quote_cleanliness) * 0.3, 0, 0.42)
};
};
@ -1202,7 +1186,9 @@ export const listSyntheticSmartMoneyScenariosForTest = (): SyntheticSmartMoneySc
hiddenLabel:
id === "neutral_noise"
? "single_print_mid"
: SMART_MONEY_TEMPLATE_SCENARIOS[id as Exclude<(typeof SMART_MONEY_SCENARIO_IDS)[number], "neutral_noise">]
: SMART_MONEY_TEMPLATE_SCENARIOS[
id as Exclude<(typeof SMART_MONEY_SCENARIO_IDS)[number], "neutral_noise">
]
}));
export const buildSyntheticSmartMoneyBurstForTest = (
@ -1233,18 +1219,18 @@ export const buildSyntheticSmartMoneyBurstForTest = (
updated_by: "system"
} satisfies SyntheticControlState;
const mode: SyntheticMarketMode =
scenarioId === "retail_whale" || scenarioId === "neutral_noise"
? "realistic"
: "active";
scenarioId === "retail_whale" || scenarioId === "neutral_noise" ? "realistic" : "active";
const profile = SYNTHETIC_PROFILES[mode];
const coverageState = createCoverageWindowState();
const scenario =
scenarioId === "neutral_noise"
? profile.scenarios.find((candidate) => candidate.id === "single_print_mid")!
: profile.scenarios.find(
(candidate) => candidate.id === SMART_MONEY_TEMPLATE_SCENARIOS[
scenarioId as Exclude<(typeof SMART_MONEY_SCENARIO_IDS)[number], "neutral_noise">
]
(candidate) =>
candidate.id ===
SMART_MONEY_TEMPLATE_SCENARIOS[
scenarioId as Exclude<(typeof SMART_MONEY_SCENARIO_IDS)[number], "neutral_noise">
]
)!;
return buildBurst(1, now, mode, profile, control, coverageState, scenario);
};
@ -1255,13 +1241,10 @@ export const buildSyntheticFlowPacketForTest = (
): { packet: FlowPacket; hiddenLabel: string } => {
const burst = buildSyntheticSmartMoneyBurstForTest(scenarioId, now);
const primaryLeg = burst.legs[0]!;
const corporateEventOffset = Number(
burst.flowFeatures.corporate_event_ts_offset_days ?? 0
);
const corporateEventOffset = Number(burst.flowFeatures.corporate_event_ts_offset_days ?? 0);
const totalSize = burst.legs.reduce((sum, leg) => sum + leg.baseSize * burst.cycles, 0);
const totalPremium = burst.legs.reduce(
(sum, leg) =>
sum + leg.basePrice * leg.baseSize * burst.cycles * OPTION_CONTRACT_MULTIPLIER,
(sum, leg) => sum + leg.basePrice * leg.baseSize * burst.cycles * OPTION_CONTRACT_MULTIPLIER,
0
);
const flowFeatures: FlowPacket["features"] = {
@ -1272,15 +1255,10 @@ export const buildSyntheticFlowPacketForTest = (
window_ms: Math.max(0, (burst.printCount - 1) * 45),
total_size: totalSize,
total_premium: Number(totalPremium.toFixed(2)),
total_notional: Number(
(burst.underlying * totalSize * OPTION_CONTRACT_MULTIPLIER).toFixed(2)
),
total_notional: Number((burst.underlying * totalSize * OPTION_CONTRACT_MULTIPLIER).toFixed(2)),
first_price: primaryLeg.basePrice,
last_price: Number(
(
primaryLeg.basePrice *
(1 + burst.priceStep * Math.max(0, burst.cycles - 1))
).toFixed(2)
(primaryLeg.basePrice * (1 + burst.priceStep * Math.max(0, burst.cycles - 1))).toFixed(2)
),
nbbo_missing_count: 0,
nbbo_stale_count: 0,
@ -1300,10 +1278,7 @@ export const buildSyntheticFlowPacketForTest = (
Number(flowFeatures.total_premium ?? totalPremium),
72_000
);
flowFeatures.execution_iv_shock = Math.max(
Number(flowFeatures.execution_iv_shock ?? 0),
0.22
);
flowFeatures.execution_iv_shock = Math.max(Number(flowFeatures.execution_iv_shock ?? 0), 0.22);
}
if (scenarioId === "event_driven") {
flowFeatures.count = 2;
@ -1411,14 +1386,7 @@ export const buildSyntheticBurstForTest = (
return cached[burstIndex - 1]!;
}
for (let index = cached.length + 1; index <= burstIndex; index += 1) {
const current = buildBurst(
index,
now + index * 1_000,
mode,
profile,
control,
coverageState
);
const current = buildBurst(index, now + index * 1_000, mode, profile, control, coverageState);
recordCoverageHit(coverageState, current.label, now + index * 1_000);
cached.push(current);
}
@ -1466,14 +1434,7 @@ export const createSyntheticOptionsAdapter = (
};
if (!currentBurst || remainingRuns <= 0) {
burstIndex += 1;
currentBurst = buildBurst(
burstIndex,
now,
config.mode,
profile,
control,
coverageState
);
currentBurst = buildBurst(burstIndex, now, config.mode, profile, control, coverageState);
recordCoverageHit(coverageState, currentBurst.label, now);
remainingRuns = pickInt(
profile.burstRunRange[0],
@ -1565,8 +1526,7 @@ export const createSyntheticOptionsAdapter = (
const quoteSeed = Math.abs(burst.seed + i * 17) % 1000;
const missingQuote = quoteSeed / 1000 < burst.missingQuoteProbability;
const staleQuote =
!missingQuote &&
((quoteSeed + 233) % 1000) / 1000 < burst.staleQuoteProbability;
!missingQuote && ((quoteSeed + 233) % 1000) / 1000 < burst.staleQuoteProbability;
if (handlers.onNBBO && !missingQuote) {
nbboSeq += 1;

View file

@ -48,7 +48,11 @@ export const selectAtOrBefore = <T extends { ts: number; seq: number }>(
if (item.ts > ts) {
continue;
}
if (!selected || item.ts > selected.ts || (item.ts === selected.ts && item.seq >= selected.seq)) {
if (
!selected ||
item.ts > selected.ts ||
(item.ts === selected.ts && item.seq >= selected.seq)
) {
selected = item;
}
}

View file

@ -43,7 +43,12 @@ import { createDatabentoOptionsAdapter } from "./adapters/databento";
import { createIbkrOptionsAdapter } from "./adapters/ibkr";
import { createSyntheticOptionsAdapter } from "./adapters/synthetic";
import type { OptionIngestAdapter, StopHandler } from "./adapters/types";
import { enrichOptionPrint, rememberContext, selectAtOrBefore, type ContextHistory } from "./enrichment";
import {
enrichOptionPrint,
rememberContext,
selectAtOrBefore,
type ContextHistory
} from "./enrichment";
import { z } from "zod";
const service = "ingest-options";
@ -87,7 +92,10 @@ const envSchema = z.object({
IBKR_EXPIRY: z.string().min(1).default("20250117"),
IBKR_STRIKE: z.coerce.number().positive().default(450),
IBKR_RIGHT: z
.preprocess((value) => (typeof value === "string" ? value.toUpperCase() : value), z.enum(["C", "P"]))
.preprocess(
(value) => (typeof value === "string" ? value.toUpperCase() : value),
z.enum(["C", "P"])
)
.default("C"),
IBKR_EXCHANGE: z.string().min(1).default("SMART"),
IBKR_CURRENCY: z.string().min(1).default("USD"),
@ -395,10 +403,7 @@ const run = async () => {
await ensureOptionNBBOTable(clickhouse);
});
const adapter = selectAdapter(
env.OPTIONS_INGEST_ADAPTER,
() => syntheticControl
);
const adapter = selectAdapter(env.OPTIONS_INGEST_ADAPTER, () => syntheticControl);
logger.info("ingest adapter selected", { adapter: adapter.name });
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
const allowNbboPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
@ -421,7 +426,10 @@ const run = async () => {
rawPrint.ts
);
const equityQuote = parsedMetadata.underlying_id
? selectAtOrBefore(equityQuoteHistoryByUnderlying.get(parsedMetadata.underlying_id), rawPrint.ts)
? selectAtOrBefore(
equityQuoteHistoryByUnderlying.get(parsedMetadata.underlying_id),
rawPrint.ts
)
: null;
const print = enrichOptionPrint(rawPrint, optionQuote, equityQuote, optionsSignalConfig);
@ -500,8 +508,16 @@ const run = async () => {
const pruneTimer = setInterval(() => {
const removed =
pruneContextHistory(nbboHistoryByContract, env.OPTION_CONTEXT_MAX_KEYS, env.OPTION_CONTEXT_TTL_MS) +
pruneContextHistory(equityQuoteHistoryByUnderlying, env.OPTION_CONTEXT_MAX_KEYS, env.OPTION_CONTEXT_TTL_MS);
pruneContextHistory(
nbboHistoryByContract,
env.OPTION_CONTEXT_MAX_KEYS,
env.OPTION_CONTEXT_TTL_MS
) +
pruneContextHistory(
equityQuoteHistoryByUnderlying,
env.OPTION_CONTEXT_MAX_KEYS,
env.OPTION_CONTEXT_TTL_MS
);
logger.info("option context cache summary", {
nbbo_context_keys: nbboHistoryByContract.size,
equity_quote_context_keys: equityQuoteHistoryByUnderlying.size,

View file

@ -1,6 +1,12 @@
import { mkdir } from "node:fs/promises";
export type EventCalendarKind = "earnings" | "dividend" | "corporate_action" | "m_and_a" | "news" | "other";
export type EventCalendarKind =
| "earnings"
| "dividend"
| "corporate_action"
| "m_and_a"
| "news"
| "other";
export type EventCalendarEntry = {
underlying_id: string;
@ -56,7 +62,8 @@ const asNumber = (value: unknown): number | null => {
return null;
};
const asString = (value: unknown): string | null => (typeof value === "string" && value.trim() ? value.trim() : null);
const asString = (value: unknown): string | null =>
typeof value === "string" && value.trim() ? value.trim() : null;
const parseCsvLine = (line: string): string[] => {
const values: string[] = [];
@ -139,9 +146,14 @@ export const parseEventCalendarEntries = (value: unknown): EventCalendarEntry[]
const record = row as Record<string, unknown>;
const underlying = asString(record.underlying_id ?? record.underlying ?? record.symbol);
const eventTs = asNumber(record.event_ts ?? record.event_time ?? record.event_date);
const announcedTs = asNumber(record.announced_ts ?? record.available_ts ?? record.as_of_ts ?? record.created_ts) ?? 0;
const announcedTs =
asNumber(
record.announced_ts ?? record.available_ts ?? record.as_of_ts ?? record.created_ts
) ?? 0;
const rawKind = asString(record.event_kind ?? record.kind ?? record.type) ?? "other";
const eventKind = EVENT_KINDS.has(rawKind as EventCalendarKind) ? (rawKind as EventCalendarKind) : "other";
const eventKind = EVENT_KINDS.has(rawKind as EventCalendarKind)
? (rawKind as EventCalendarKind)
: "other";
if (!underlying || eventTs === null || eventTs < 0 || announcedTs < 0) {
return [];
@ -162,7 +174,9 @@ export const parseEventCalendarEntries = (value: unknown): EventCalendarEntry[]
});
};
export const createStaticEventCalendarProvider = (entries: EventCalendarEntry[]): EventCalendarProvider => {
export const createStaticEventCalendarProvider = (
entries: EventCalendarEntry[]
): EventCalendarProvider => {
const byUnderlying = new Map<string, EventCalendarEntry[]>();
for (const entry of entries) {
const key = normalizeUnderlying(entry.underlying_id);
@ -184,15 +198,20 @@ export const createStaticEventCalendarProvider = (entries: EventCalendarEntry[])
}
const bucket = byUnderlying.get(key) ?? [];
const entry = bucket.find((candidate) => candidate.announced_ts <= asOfTs && candidate.event_ts >= asOfTs);
const entry = bucket.find(
(candidate) => candidate.announced_ts <= asOfTs && candidate.event_ts >= asOfTs
);
return entry ? { ...entry, days_to_event: (entry.event_ts - asOfTs) / MS_PER_DAY } : null;
}
};
};
export const createEmptyEventCalendarProvider = (): EventCalendarProvider => createStaticEventCalendarProvider([]);
export const createEmptyEventCalendarProvider = (): EventCalendarProvider =>
createStaticEventCalendarProvider([]);
export const loadEventCalendarProviderFromFile = async (path: string): Promise<EventCalendarProvider> => {
export const loadEventCalendarProviderFromFile = async (
path: string
): Promise<EventCalendarProvider> => {
const text = await Bun.file(path).text();
return createStaticEventCalendarProvider(parseEventCalendarEntries(JSON.parse(text)));
};
@ -212,7 +231,9 @@ export const fetchAlphaVantageEarningsCalendar = async (
const response = await (options.fetchFn ?? fetch)(url);
const text = await response.text();
if (!response.ok) {
throw new Error(`Alpha Vantage earnings calendar request failed: ${response.status} ${text.slice(0, 160)}`);
throw new Error(
`Alpha Vantage earnings calendar request failed: ${response.status} ${text.slice(0, 160)}`
);
}
if (/^(?:\s*\{|\s*Thank you for using Alpha Vantage)/i.test(text)) {
throw new Error(`Alpha Vantage returned a non-calendar response: ${text.slice(0, 200)}`);
@ -221,7 +242,10 @@ export const fetchAlphaVantageEarningsCalendar = async (
return parseAlphaVantageEarningsCalendar(text, options.nowTs ?? Date.now());
};
export const writeEventCalendarEntries = async (path: string, entries: EventCalendarEntry[]): Promise<void> => {
export const writeEventCalendarEntries = async (
path: string,
entries: EventCalendarEntry[]
): Promise<void> => {
const directory = path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : "";
if (directory) {
await mkdir(directory, { recursive: true });

View file

@ -12,9 +12,14 @@ const logger = createLogger({ service });
logger.info("service starting");
const eventCalendarPath = process.env.REFDATA_EVENT_CALENDAR_PATH ?? process.env.SMART_MONEY_EVENT_CALENDAR_PATH;
const eventCalendarProvider = process.env.REFDATA_EVENT_CALENDAR_PROVIDER ?? process.env.EVENT_CALENDAR_PROVIDER;
const refreshMs = Math.max(0, Number(process.env.REFDATA_EVENT_CALENDAR_REFRESH_MS ?? 86_400_000) || 0);
const eventCalendarPath =
process.env.REFDATA_EVENT_CALENDAR_PATH ?? process.env.SMART_MONEY_EVENT_CALENDAR_PATH;
const eventCalendarProvider =
process.env.REFDATA_EVENT_CALENDAR_PROVIDER ?? process.env.EVENT_CALENDAR_PROVIDER;
const refreshMs = Math.max(
0,
Number(process.env.REFDATA_EVENT_CALENDAR_REFRESH_MS ?? 86_400_000) || 0
);
const getAlphaVantageOptions = (): AlphaVantageEarningsCalendarOptions | null => {
const apiKey = process.env.ALPHA_VANTAGE_API_KEY;
@ -33,7 +38,9 @@ const getAlphaVantageOptions = (): AlphaVantageEarningsCalendarOptions | null =>
const refreshEventCalendar = async (): Promise<void> => {
if (!eventCalendarPath) {
logger.warn("event calendar refresh disabled; missing SMART_MONEY_EVENT_CALENDAR_PATH or REFDATA_EVENT_CALENDAR_PATH");
logger.warn(
"event calendar refresh disabled; missing SMART_MONEY_EVENT_CALENDAR_PATH or REFDATA_EVENT_CALENDAR_PATH"
);
return;
}
if (eventCalendarProvider !== "alpha_vantage") {

View file

@ -52,11 +52,7 @@ type ReplayStreamKind = "options" | "nbbo" | "equities" | "equity-quotes";
type ReplayEvent = OptionPrint | OptionNBBO | EquityPrint | EquityQuote;
type FetchAfter = (
afterTs: number,
afterSeq: number,
limit: number
) => Promise<ReplayEvent[]>;
type FetchAfter = (afterTs: number, afterSeq: number, limit: number) => Promise<ReplayEvent[]>;
type ReplayStream = {
kind: ReplayStreamKind;
@ -79,7 +75,12 @@ const STREAM_DEFS: Record<
subject: string;
streamName: string;
rank: number;
fetchAfter: (client: ReturnType<typeof createClickHouseClient>, afterTs: number, afterSeq: number, limit: number) => Promise<ReplayEvent[]>;
fetchAfter: (
client: ReturnType<typeof createClickHouseClient>,
afterTs: number,
afterSeq: number,
limit: number
) => Promise<ReplayEvent[]>;
}
> = {
options: {
@ -196,7 +197,9 @@ const getEventIngestTs = (event: ReplayEvent): number =>
const getEventSeq = (event: ReplayEvent): number => (Number.isFinite(event.seq) ? event.seq : 0);
const pickNextEvent = (streams: ReplayStream[]): { stream: ReplayStream; event: ReplayEvent } | null => {
const pickNextEvent = (
streams: ReplayStream[]
): { stream: ReplayStream; event: ReplayEvent } | null => {
let choice: { stream: ReplayStream; event: ReplayEvent } | null = null;
for (const stream of streams) {
@ -313,7 +316,8 @@ const run = async () => {
kind,
subject: def.subject,
streamName: def.streamName,
fetchAfter: (afterTs, afterSeq, limit) => def.fetchAfter(clickhouse, afterTs, afterSeq, limit),
fetchAfter: (afterTs, afterSeq, limit) =>
def.fetchAfter(clickhouse, afterTs, afterSeq, limit),
buffer: [],
cursor: { ...startCursor },
done: false,