This commit is contained in:
parent
65139bf8d0
commit
44431c4e66
71 changed files with 2262 additions and 1173 deletions
|
|
@ -465,8 +465,7 @@ const parseCandleParams = (
|
|||
|
||||
const endTs = params.end_ts ?? Date.now();
|
||||
const limit = params.limit ?? env.REST_DEFAULT_LIMIT;
|
||||
const startTs =
|
||||
params.start_ts ?? Math.max(0, Math.floor(endTs - params.interval_ms * limit));
|
||||
const startTs = params.start_ts ?? Math.max(0, Math.floor(endTs - params.interval_ms * limit));
|
||||
const rangeStart = Math.min(startTs, endTs);
|
||||
const rangeEnd = Math.max(startTs, endTs);
|
||||
|
||||
|
|
@ -482,7 +481,13 @@ const parseCandleParams = (
|
|||
|
||||
const parseCandleReplayParams = (
|
||||
url: URL
|
||||
): { underlyingId: string; intervalMs: number; afterTs: number; afterSeq: number; limit: number } => {
|
||||
): {
|
||||
underlyingId: string;
|
||||
intervalMs: number;
|
||||
afterTs: number;
|
||||
afterSeq: number;
|
||||
limit: number;
|
||||
} => {
|
||||
const params = candleReplaySchema.parse({
|
||||
underlying_id: url.searchParams.get("underlying_id") ?? undefined,
|
||||
interval_ms: url.searchParams.get("interval_ms") ?? undefined,
|
||||
|
|
@ -601,7 +606,10 @@ const matchesScopedOptionSubscription = (
|
|||
print: { underlying_id?: string; option_contract_id: string },
|
||||
subscription: Extract<LiveSubscription, { channel: "options" }>
|
||||
): boolean => {
|
||||
if (subscription.option_contract_id && subscription.option_contract_id !== print.option_contract_id) {
|
||||
if (
|
||||
subscription.option_contract_id &&
|
||||
subscription.option_contract_id !== print.option_contract_id
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (subscription.underlying_ids?.length) {
|
||||
|
|
@ -693,8 +701,7 @@ const run = async () => {
|
|||
env.OPTIONS_INGEST_ADAPTER,
|
||||
env.EQUITIES_INGEST_ADAPTER
|
||||
);
|
||||
const syntheticBackendDisabledReason =
|
||||
getSyntheticBackendDisabledReason(syntheticBackendMode);
|
||||
const syntheticBackendDisabledReason = getSyntheticBackendDisabledReason(syntheticBackendMode);
|
||||
const syntheticControlKv = await openSyntheticControlKv(js);
|
||||
let syntheticControl = await ensureSyntheticControlState(syntheticControlKv);
|
||||
const syntheticProfileHits = createRollingSyntheticProfileHits();
|
||||
|
|
@ -899,11 +906,7 @@ const run = async () => {
|
|||
}
|
||||
}
|
||||
|
||||
const subscribeWithReset = async <T>(
|
||||
subject: string,
|
||||
stream: string,
|
||||
durableName: string
|
||||
) => {
|
||||
const subscribeWithReset = async <T>(subject: string, stream: string, durableName: string) => {
|
||||
const opts = buildDurableConsumer(durableName);
|
||||
applyDeliverPolicy(opts, env.API_DELIVER_POLICY);
|
||||
try {
|
||||
|
|
@ -924,7 +927,8 @@ const run = async () => {
|
|||
try {
|
||||
await jsm.consumers.delete(stream, durableName);
|
||||
} catch (deleteError) {
|
||||
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
const deleteMessage =
|
||||
deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
if (!deleteMessage.includes("not found")) {
|
||||
logger.warn("failed to delete jetstream consumer", {
|
||||
durable: durableName,
|
||||
|
|
@ -1023,8 +1027,12 @@ const run = async () => {
|
|||
}
|
||||
|
||||
const matchingSubscriptions =
|
||||
subscription.channel === "options" || subscription.channel === "flow" || subscription.channel === "equities"
|
||||
? [...subscriptionDefinitions.entries()].filter(([, candidate]) => candidate.channel === subscription.channel)
|
||||
subscription.channel === "options" ||
|
||||
subscription.channel === "flow" ||
|
||||
subscription.channel === "equities"
|
||||
? [...subscriptionDefinitions.entries()].filter(
|
||||
([, candidate]) => candidate.channel === subscription.channel
|
||||
)
|
||||
: [[getSubscriptionKey(subscription), subscription] as const];
|
||||
|
||||
if (matchingSubscriptions.length === 0) {
|
||||
|
|
@ -1032,8 +1040,12 @@ const run = async () => {
|
|||
}
|
||||
|
||||
const optionItem = ingestChannel === "options" ? (item as OptionPrint) : null;
|
||||
const equityItem = ingestChannel === "equities" ? (item as Parameters<typeof matchesScopedEquitySubscription>[0]) : null;
|
||||
const flowItem = ingestChannel === "flow" ? (item as Parameters<typeof matchesFlowPacketFilters>[0]) : null;
|
||||
const equityItem =
|
||||
ingestChannel === "equities"
|
||||
? (item as Parameters<typeof matchesScopedEquitySubscription>[0])
|
||||
: null;
|
||||
const flowItem =
|
||||
ingestChannel === "flow" ? (item as Parameters<typeof matchesFlowPacketFilters>[0]) : null;
|
||||
let matchedSubscriptions = 0;
|
||||
|
||||
for (const [key, candidate] of matchingSubscriptions) {
|
||||
|
|
@ -1315,9 +1327,7 @@ const run = async () => {
|
|||
},
|
||||
control: syntheticBackendMode === "synthetic" ? syntheticControl : null,
|
||||
derived,
|
||||
...(syntheticBackendDisabledReason
|
||||
? { disabled_reason: syntheticBackendDisabledReason }
|
||||
: {})
|
||||
...(syntheticBackendDisabledReason ? { disabled_reason: syntheticBackendDisabledReason } : {})
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -1385,11 +1395,7 @@ const run = async () => {
|
|||
syntheticControl = await writeSyntheticControlState(syntheticControlKv, payload);
|
||||
return jsonResponse({
|
||||
control: syntheticControl,
|
||||
derived: buildSyntheticDerivedStatus(
|
||||
Date.now(),
|
||||
syntheticControl,
|
||||
syntheticProfileHits
|
||||
)
|
||||
derived: buildSyntheticDerivedStatus(Date.now(), syntheticControl, syntheticProfileHits)
|
||||
});
|
||||
} catch (error) {
|
||||
return jsonResponse(
|
||||
|
|
@ -1436,7 +1442,13 @@ const run = async () => {
|
|||
if (req.method === "GET" && url.pathname === "/prints/equities/range") {
|
||||
try {
|
||||
const { underlyingId, startTs, endTs, limit } = parseEquityPrintRangeParams(url);
|
||||
const data = await fetchEquityPrintsRange(clickhouse, underlyingId, startTs, endTs, limit);
|
||||
const data = await fetchEquityPrintsRange(
|
||||
clickhouse,
|
||||
underlyingId,
|
||||
startTs,
|
||||
endTs,
|
||||
limit
|
||||
);
|
||||
return jsonResponse({ data });
|
||||
} catch (error) {
|
||||
return jsonResponse(
|
||||
|
|
@ -1566,7 +1578,9 @@ const run = async () => {
|
|||
source,
|
||||
storageFilters
|
||||
);
|
||||
return jsonResponse(buildHistoryResponse(data, (item) => ({ ts: item.ts, seq: item.seq })));
|
||||
return jsonResponse(
|
||||
buildHistoryResponse(data, (item) => ({ ts: item.ts, seq: item.seq }))
|
||||
);
|
||||
} catch (error) {
|
||||
return jsonResponse(
|
||||
{
|
||||
|
|
@ -1986,7 +2000,9 @@ const run = async () => {
|
|||
const payload =
|
||||
typeof message === "string"
|
||||
? message
|
||||
: new TextDecoder().decode(message instanceof Uint8Array ? message : new Uint8Array(message));
|
||||
: new TextDecoder().decode(
|
||||
message instanceof Uint8Array ? message : new Uint8Array(message)
|
||||
);
|
||||
const parsed = LiveClientMessageSchema.parse(JSON.parse(payload));
|
||||
if (parsed.op === "ping") {
|
||||
sendLiveMessage(socket, {
|
||||
|
|
|
|||
|
|
@ -165,11 +165,21 @@ const parseGenericLimitFallback = (env: NodeJS.ProcessEnv, fallback: number): nu
|
|||
return Math.max(MIN_GENERIC_LIMIT, Math.min(MAX_GENERIC_LIMIT, Math.floor(parsed)));
|
||||
};
|
||||
|
||||
export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env): GenericLiveLimits => {
|
||||
export const resolveGenericLiveLimits = (
|
||||
env: NodeJS.ProcessEnv = process.env
|
||||
): GenericLiveLimits => {
|
||||
const liveLimitDefault = parseGenericLimitFallback(env, DEFAULT_GENERIC_LIMIT);
|
||||
return {
|
||||
options: parseGenericLimit(env, "options", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.options),
|
||||
nbbo: parseGenericLimit(env, "nbbo", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.nbbo),
|
||||
options: parseGenericLimit(
|
||||
env,
|
||||
"options",
|
||||
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.options
|
||||
),
|
||||
nbbo: parseGenericLimit(
|
||||
env,
|
||||
"nbbo",
|
||||
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.nbbo
|
||||
),
|
||||
equities: parseGenericLimit(
|
||||
env,
|
||||
"equities",
|
||||
|
|
@ -185,7 +195,11 @@ export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env):
|
|||
"equity-joins",
|
||||
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["equity-joins"]
|
||||
),
|
||||
flow: parseGenericLimit(env, "flow", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.flow),
|
||||
flow: parseGenericLimit(
|
||||
env,
|
||||
"flow",
|
||||
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.flow
|
||||
),
|
||||
"smart-money": parseGenericLimit(
|
||||
env,
|
||||
"smart-money",
|
||||
|
|
@ -196,13 +210,21 @@ export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env):
|
|||
"classifier-hits",
|
||||
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["classifier-hits"]
|
||||
),
|
||||
alerts: parseGenericLimit(env, "alerts", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.alerts),
|
||||
alerts: parseGenericLimit(
|
||||
env,
|
||||
"alerts",
|
||||
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.alerts
|
||||
),
|
||||
"inferred-dark": parseGenericLimit(
|
||||
env,
|
||||
"inferred-dark",
|
||||
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["inferred-dark"]
|
||||
),
|
||||
news: parseGenericLimit(env, "news", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.news)
|
||||
news: parseGenericLimit(
|
||||
env,
|
||||
"news",
|
||||
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.news
|
||||
)
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -227,12 +249,18 @@ const extractFreshnessTs = (channel: LiveGenericChannel, item: any): number | nu
|
|||
|
||||
export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): LiveStateConfig => ({
|
||||
limits: resolveGenericLiveLimits(env),
|
||||
scopedCacheMaxKeys: parsePositiveInt(env.LIVE_SCOPED_CACHE_MAX_KEYS, DEFAULT_SCOPED_CACHE_MAX_KEYS),
|
||||
scopedCacheMaxKeys: parsePositiveInt(
|
||||
env.LIVE_SCOPED_CACHE_MAX_KEYS,
|
||||
DEFAULT_SCOPED_CACHE_MAX_KEYS
|
||||
),
|
||||
redisFlushIntervalMs: parsePositiveInt(
|
||||
env.LIVE_REDIS_FLUSH_INTERVAL_MS,
|
||||
DEFAULT_REDIS_FLUSH_INTERVAL_MS
|
||||
),
|
||||
redisFlushMaxItems: parsePositiveInt(env.LIVE_REDIS_FLUSH_MAX_ITEMS, DEFAULT_REDIS_FLUSH_MAX_ITEMS)
|
||||
redisFlushMaxItems: parsePositiveInt(
|
||||
env.LIVE_REDIS_FLUSH_MAX_ITEMS,
|
||||
DEFAULT_REDIS_FLUSH_MAX_ITEMS
|
||||
)
|
||||
});
|
||||
const parsePositiveInt = (value: string | undefined, fallback: number): number => {
|
||||
const parsed = Number(value);
|
||||
|
|
@ -242,10 +270,7 @@ const parsePositiveInt = (value: string | undefined, fallback: number): number =
|
|||
return Math.max(1, Math.floor(parsed));
|
||||
};
|
||||
|
||||
type RedisLike = Pick<
|
||||
RedisClientType,
|
||||
"isOpen" | "lRange" | "lPush" | "lTrim" | "hGet" | "hSet"
|
||||
>;
|
||||
type RedisLike = Pick<RedisClientType, "isOpen" | "lRange" | "lPush" | "lTrim" | "hGet" | "hSet">;
|
||||
|
||||
const parseCursor = (value: string | null): Cursor | null => {
|
||||
if (!value) {
|
||||
|
|
@ -259,7 +284,9 @@ const parseCursor = (value: string | null): Cursor | null => {
|
|||
}
|
||||
};
|
||||
|
||||
const getGenericConfig = (limits: GenericLiveLimits): {
|
||||
const getGenericConfig = (
|
||||
limits: GenericLiveLimits
|
||||
): {
|
||||
[K in LiveGenericChannel]: GenericFeedConfig;
|
||||
} => ({
|
||||
options: {
|
||||
|
|
@ -365,7 +392,7 @@ const parseJsonList = <T>(payloads: string[], parse: (value: unknown) => T): T[]
|
|||
return items;
|
||||
};
|
||||
|
||||
const compareCursors = (a: Cursor, b: Cursor): number => (b.ts - a.ts) || (b.seq - a.seq);
|
||||
const compareCursors = (a: Cursor, b: Cursor): number => b.ts - a.ts || b.seq - a.seq;
|
||||
|
||||
const sortGenericItems = <T>(items: T[], cursorOf: (item: T) => Cursor): T[] =>
|
||||
[...items].sort((a, b) => compareCursors(cursorOf(a), cursorOf(b)));
|
||||
|
|
@ -480,7 +507,10 @@ const matchesScopedOptionSnapshot = (
|
|||
return false;
|
||||
}
|
||||
|
||||
if (subscription.option_contract_id && item.option_contract_id !== subscription.option_contract_id) {
|
||||
if (
|
||||
subscription.option_contract_id &&
|
||||
item.option_contract_id !== subscription.option_contract_id
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -529,11 +559,8 @@ const candleCursorField = (underlyingId: string, intervalMs: number): string =>
|
|||
const overlayRedisKey = (underlyingId: string): string => `live:equity-overlay:${underlyingId}`;
|
||||
const overlayCursorField = (underlyingId: string): string => `equities:${underlyingId}`;
|
||||
|
||||
const dropMatchingCursor = <T>(
|
||||
items: T[],
|
||||
target: Cursor,
|
||||
cursorOf: (item: T) => Cursor
|
||||
): T[] => items.filter((item) => compareCursors(cursorOf(item), target) !== 0);
|
||||
const dropMatchingCursor = <T>(items: T[], target: Cursor, cursorOf: (item: T) => Cursor): T[] =>
|
||||
items.filter((item) => compareCursors(cursorOf(item), target) !== 0);
|
||||
|
||||
const insertNewestFirst = <T>(
|
||||
items: T[],
|
||||
|
|
@ -676,7 +703,13 @@ export class LiveStateManager {
|
|||
this.pendingRedisWrites.clear();
|
||||
|
||||
for (const write of writes) {
|
||||
await this.persistList(write.listKey, write.cursorField, write.items, write.limit, write.cursor);
|
||||
await this.persistList(
|
||||
write.listKey,
|
||||
write.cursorField,
|
||||
write.items,
|
||||
write.limit,
|
||||
write.cursor
|
||||
);
|
||||
this.stats.redisFlushCount += 1;
|
||||
this.stats.redisFlushItems += write.items.length;
|
||||
metrics.count("api.live.redis_flush_count", 1);
|
||||
|
|
@ -726,7 +759,12 @@ export class LiveStateManager {
|
|||
}
|
||||
}
|
||||
|
||||
private updateFreshnessMetric(listKey: string, channel: LiveChannel, item: unknown, now = Date.now()): void {
|
||||
private updateFreshnessMetric(
|
||||
listKey: string,
|
||||
channel: LiveChannel,
|
||||
item: unknown,
|
||||
now = Date.now()
|
||||
): void {
|
||||
const ts =
|
||||
channel === "equity-candles" || channel === "equity-overlay"
|
||||
? typeof (item as { ts?: unknown })?.ts === "number"
|
||||
|
|
@ -784,12 +822,22 @@ export class LiveStateManager {
|
|||
config.cursorField,
|
||||
parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, config.cursorField))
|
||||
);
|
||||
await this.persistList(config.redisKey, config.cursorField, cached, config.limit, this.genericCursors.get(config.cursorField) ?? null);
|
||||
await this.persistList(
|
||||
config.redisKey,
|
||||
config.cursorField,
|
||||
cached,
|
||||
config.limit,
|
||||
this.genericCursors.get(config.cursorField) ?? null
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const fresh = normalizeGenericItems(channel, await config.fetchRecent(this.clickhouse, config.limit), config);
|
||||
const fresh = normalizeGenericItems(
|
||||
channel,
|
||||
await config.fetchRecent(this.clickhouse, config.limit),
|
||||
config
|
||||
);
|
||||
this.stats.genericHydrateFromClickHouse += 1;
|
||||
this.stats.cacheDepthByKey.set(config.redisKey, fresh.length);
|
||||
this.genericItems.set(channel, fresh);
|
||||
|
|
@ -806,7 +854,8 @@ export class LiveStateManager {
|
|||
case "options": {
|
||||
const config = this.generic.options;
|
||||
const limit = snapshotLimitFor(subscription, config.limit);
|
||||
const scoped = Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id);
|
||||
const scoped =
|
||||
Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id);
|
||||
if (subscription.filters?.view === "raw" || scoped) {
|
||||
const cached = (this.genericItems.get("options") ?? [])
|
||||
.filter((entry) => matchesScopedOptionSnapshot(entry, subscription))
|
||||
|
|
@ -815,8 +864,16 @@ export class LiveStateManager {
|
|||
if (cached.length < limit) {
|
||||
this.stats.scopedClickHouseSnapshots += 1;
|
||||
const storageFilters = buildOptionSnapshotFilters(subscription);
|
||||
const backfill = await fetchRecentOptionPrints(this.clickhouse, limit, undefined, storageFilters);
|
||||
items = mergeSnapshotBackfill(cached, backfill, limit, (entry) => ({ ts: entry.ts, seq: entry.seq }));
|
||||
const backfill = await fetchRecentOptionPrints(
|
||||
this.clickhouse,
|
||||
limit,
|
||||
undefined,
|
||||
storageFilters
|
||||
);
|
||||
items = mergeSnapshotBackfill(cached, backfill, limit, (entry) => ({
|
||||
ts: entry.ts,
|
||||
seq: entry.seq
|
||||
}));
|
||||
}
|
||||
return {
|
||||
subscription,
|
||||
|
|
@ -942,7 +999,11 @@ export class LiveStateManager {
|
|||
this.candleItems.set(key, nextState.items);
|
||||
this.candleCursors.set(cursorField, cursor);
|
||||
this.touchAccess(this.candleAccess, key);
|
||||
this.evictScopedCachesIfNeeded(this.candleItems as Map<string, unknown[]>, this.candleCursors, this.candleAccess);
|
||||
this.evictScopedCachesIfNeeded(
|
||||
this.candleItems as Map<string, unknown[]>,
|
||||
this.candleCursors,
|
||||
this.candleAccess
|
||||
);
|
||||
if (nextState.outOfOrder) {
|
||||
this.stats.outOfOrderEvents += 1;
|
||||
metrics.count("api.live.out_of_order_events", 1);
|
||||
|
|
@ -968,7 +1029,11 @@ export class LiveStateManager {
|
|||
this.overlayItems.set(key, nextState.items);
|
||||
this.overlayCursors.set(cursorField, cursor);
|
||||
this.touchAccess(this.overlayAccess, key);
|
||||
this.evictScopedCachesIfNeeded(this.overlayItems as Map<string, unknown[]>, this.overlayCursors, this.overlayAccess);
|
||||
this.evictScopedCachesIfNeeded(
|
||||
this.overlayItems as Map<string, unknown[]>,
|
||||
this.overlayCursors,
|
||||
this.overlayAccess
|
||||
);
|
||||
if (nextState.outOfOrder) {
|
||||
this.stats.outOfOrderEvents += 1;
|
||||
metrics.count("api.live.out_of_order_events", 1);
|
||||
|
|
@ -991,10 +1056,19 @@ export class LiveStateManager {
|
|||
const nextState =
|
||||
channel === "nbbo"
|
||||
? {
|
||||
items: normalizeGenericItems(channel, [parsed, ...(this.genericItems.get(channel) ?? [])], config),
|
||||
items: normalizeGenericItems(
|
||||
channel,
|
||||
[parsed, ...(this.genericItems.get(channel) ?? [])],
|
||||
config
|
||||
),
|
||||
outOfOrder: false
|
||||
}
|
||||
: insertNewestFirst(this.genericItems.get(channel) ?? [], parsed, config.cursor, config.limit);
|
||||
: insertNewestFirst(
|
||||
this.genericItems.get(channel) ?? [],
|
||||
parsed,
|
||||
config.cursor,
|
||||
config.limit
|
||||
);
|
||||
|
||||
if (nextState.outOfOrder) {
|
||||
this.stats.outOfOrderEvents += 1;
|
||||
|
|
@ -1007,7 +1081,13 @@ export class LiveStateManager {
|
|||
if (nextState.items.length > 0) {
|
||||
this.updateFreshnessMetric(config.redisKey, channel, nextState.items[0]);
|
||||
}
|
||||
this.queueRedisWrite(config.redisKey, config.cursorField, nextState.items, config.limit, cursor);
|
||||
this.queueRedisWrite(
|
||||
config.redisKey,
|
||||
config.cursorField,
|
||||
nextState.items,
|
||||
config.limit,
|
||||
cursor
|
||||
);
|
||||
return cursor;
|
||||
}
|
||||
}
|
||||
|
|
@ -1022,18 +1102,34 @@ export class LiveStateManager {
|
|||
if (cached.length > 0) {
|
||||
this.candleItems.set(key, cached);
|
||||
this.touchAccess(this.candleAccess, key);
|
||||
this.evictScopedCachesIfNeeded(this.candleItems as Map<string, unknown[]>, this.candleCursors, this.candleAccess);
|
||||
this.evictScopedCachesIfNeeded(
|
||||
this.candleItems as Map<string, unknown[]>,
|
||||
this.candleCursors,
|
||||
this.candleAccess
|
||||
);
|
||||
this.stats.cacheDepthByKey.set(key, cached.length);
|
||||
this.updateFreshnessMetric(key, "equity-candles", cached[0]);
|
||||
this.candleCursors.set(cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField)));
|
||||
this.candleCursors.set(
|
||||
cursorField,
|
||||
parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField))
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const fresh = await fetchRecentEquityCandles(this.clickhouse, underlyingId, intervalMs, CHART_LIMITS.candles);
|
||||
const fresh = await fetchRecentEquityCandles(
|
||||
this.clickhouse,
|
||||
underlyingId,
|
||||
intervalMs,
|
||||
CHART_LIMITS.candles
|
||||
);
|
||||
this.candleItems.set(key, fresh);
|
||||
this.touchAccess(this.candleAccess, key);
|
||||
this.evictScopedCachesIfNeeded(this.candleItems as Map<string, unknown[]>, this.candleCursors, this.candleAccess);
|
||||
this.evictScopedCachesIfNeeded(
|
||||
this.candleItems as Map<string, unknown[]>,
|
||||
this.candleCursors,
|
||||
this.candleAccess
|
||||
);
|
||||
this.stats.cacheDepthByKey.set(key, fresh.length);
|
||||
if (fresh.length > 0) {
|
||||
this.updateFreshnessMetric(key, "equity-candles", fresh[0]);
|
||||
|
|
@ -1052,10 +1148,17 @@ export class LiveStateManager {
|
|||
if (cached.length > 0) {
|
||||
this.overlayItems.set(key, cached);
|
||||
this.touchAccess(this.overlayAccess, key);
|
||||
this.evictScopedCachesIfNeeded(this.overlayItems as Map<string, unknown[]>, this.overlayCursors, this.overlayAccess);
|
||||
this.evictScopedCachesIfNeeded(
|
||||
this.overlayItems as Map<string, unknown[]>,
|
||||
this.overlayCursors,
|
||||
this.overlayAccess
|
||||
);
|
||||
this.stats.cacheDepthByKey.set(key, cached.length);
|
||||
this.updateFreshnessMetric(key, "equity-overlay", cached[0]);
|
||||
this.overlayCursors.set(cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField)));
|
||||
this.overlayCursors.set(
|
||||
cursorField,
|
||||
parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField))
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -1065,7 +1168,11 @@ export class LiveStateManager {
|
|||
);
|
||||
this.overlayItems.set(key, fresh);
|
||||
this.touchAccess(this.overlayAccess, key);
|
||||
this.evictScopedCachesIfNeeded(this.overlayItems as Map<string, unknown[]>, this.overlayCursors, this.overlayAccess);
|
||||
this.evictScopedCachesIfNeeded(
|
||||
this.overlayItems as Map<string, unknown[]>,
|
||||
this.overlayCursors,
|
||||
this.overlayAccess
|
||||
);
|
||||
this.stats.cacheDepthByKey.set(key, fresh.length);
|
||||
if (fresh.length > 0) {
|
||||
this.updateFreshnessMetric(key, "equity-overlay", fresh[0]);
|
||||
|
|
|
|||
|
|
@ -83,11 +83,7 @@ export const buildSyntheticDerivedStatus = (
|
|||
session_phase: session.session_phase,
|
||||
regime: session.regime,
|
||||
focus_symbols: session.focus_symbols,
|
||||
profile_hit_counts: getSyntheticProfileHitCounts(
|
||||
state,
|
||||
now,
|
||||
control.coverage_window_minutes
|
||||
),
|
||||
profile_hit_counts: getSyntheticProfileHitCounts(state, now, control.coverage_window_minutes),
|
||||
coverage_window_minutes: control.coverage_window_minutes
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ import { isAlertContextPath, parseAlertContextTraceIdPath } from "../src/alert-c
|
|||
|
||||
describe("alert context route helpers", () => {
|
||||
it("extracts a valid alert trace id from the context endpoint path", () => {
|
||||
expect(parseAlertContextTraceIdPath("/flow/alerts/alert%3Actx%2Fone/context")).toBe("alert:ctx/one");
|
||||
expect(parseAlertContextTraceIdPath("/flow/alerts/alert%3Actx%2Fone/context")).toBe(
|
||||
"alert:ctx/one"
|
||||
);
|
||||
});
|
||||
|
||||
it("returns null for unrelated alert paths", () => {
|
||||
|
|
|
|||
|
|
@ -9,9 +9,7 @@ import {
|
|||
shouldFanoutLiveEvent
|
||||
} from "../src/live";
|
||||
|
||||
const makeClickHouse = (
|
||||
queryResolver?: (query: string) => unknown[]
|
||||
): ClickHouseClient =>
|
||||
const makeClickHouse = (queryResolver?: (query: string) => unknown[]): ClickHouseClient =>
|
||||
({
|
||||
exec: async () => {},
|
||||
insert: async () => {},
|
||||
|
|
@ -149,22 +147,18 @@ describe("LiveStateManager", () => {
|
|||
it("trims generic windows to configured per-channel limits", async () => {
|
||||
const redis = makeRedis();
|
||||
const now = Date.now();
|
||||
const manager = new LiveStateManager(
|
||||
makeClickHouse(),
|
||||
redis as never,
|
||||
{
|
||||
options: 10000,
|
||||
nbbo: 10000,
|
||||
equities: 10000,
|
||||
"equity-quotes": 10000,
|
||||
"equity-joins": 10000,
|
||||
flow: 2,
|
||||
"smart-money": 10000,
|
||||
"classifier-hits": 10000,
|
||||
alerts: 10000,
|
||||
"inferred-dark": 10000
|
||||
}
|
||||
);
|
||||
const manager = new LiveStateManager(makeClickHouse(), redis as never, {
|
||||
options: 10000,
|
||||
nbbo: 10000,
|
||||
equities: 10000,
|
||||
"equity-quotes": 10000,
|
||||
"equity-joins": 10000,
|
||||
flow: 2,
|
||||
"smart-money": 10000,
|
||||
"classifier-hits": 10000,
|
||||
alerts: 10000,
|
||||
"inferred-dark": 10000
|
||||
});
|
||||
|
||||
await manager.ingest("flow", {
|
||||
source_ts: now,
|
||||
|
|
@ -503,18 +497,15 @@ describe("LiveStateManager", () => {
|
|||
manager.getSnapshot({ channel: "flow" })
|
||||
]);
|
||||
|
||||
expect((optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
|
||||
"opt-fresh",
|
||||
"opt-stale"
|
||||
]);
|
||||
expect((nbboSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
|
||||
"nbbo-fresh",
|
||||
"nbbo-stale"
|
||||
]);
|
||||
expect((equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
|
||||
"eq-fresh",
|
||||
"eq-stale"
|
||||
]);
|
||||
expect(
|
||||
(optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
|
||||
).toEqual(["opt-fresh", "opt-stale"]);
|
||||
expect(
|
||||
(nbboSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
|
||||
).toEqual(["nbbo-fresh", "nbbo-stale"]);
|
||||
expect(
|
||||
(equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
|
||||
).toEqual(["eq-fresh", "eq-stale"]);
|
||||
expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([
|
||||
"flow-fresh",
|
||||
"flow-stale"
|
||||
|
|
@ -699,10 +690,9 @@ describe("LiveStateManager", () => {
|
|||
option_contract_id: "AAPL-2025-01-17-200-C"
|
||||
});
|
||||
|
||||
expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id).slice(0, 2)).toEqual([
|
||||
"opt-hot",
|
||||
"opt-backfill"
|
||||
]);
|
||||
expect(
|
||||
(snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id).slice(0, 2)
|
||||
).toEqual(["opt-hot", "opt-backfill"]);
|
||||
});
|
||||
|
||||
it("seeds scoped equity snapshots from clickhouse rows older than 24h", async () => {
|
||||
|
|
@ -806,12 +796,12 @@ describe("LiveStateManager", () => {
|
|||
manager.getSnapshot({ channel: "flow" })
|
||||
]);
|
||||
|
||||
expect((optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
|
||||
"opt-retained"
|
||||
]);
|
||||
expect((equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([
|
||||
"eq-retained"
|
||||
]);
|
||||
expect(
|
||||
(optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
|
||||
).toEqual(["opt-retained"]);
|
||||
expect(
|
||||
(equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
|
||||
).toEqual(["eq-retained"]);
|
||||
expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([
|
||||
"flow-retained"
|
||||
]);
|
||||
|
|
@ -1047,7 +1037,10 @@ describe("LiveStateManager", () => {
|
|||
});
|
||||
|
||||
it("tracks generic cache and scoped clickhouse snapshot sources separately", async () => {
|
||||
const manager = new LiveStateManager(makeClickHouse(() => []), null);
|
||||
const manager = new LiveStateManager(
|
||||
makeClickHouse(() => []),
|
||||
null
|
||||
);
|
||||
const now = Date.now();
|
||||
|
||||
await manager.ingest("options", {
|
||||
|
|
@ -1075,7 +1068,10 @@ describe("LiveStateManager", () => {
|
|||
});
|
||||
|
||||
it("keeps backend channel health healthy when a scoped query is quiet", async () => {
|
||||
const manager = new LiveStateManager(makeClickHouse(() => []), null);
|
||||
const manager = new LiveStateManager(
|
||||
makeClickHouse(() => []),
|
||||
null
|
||||
);
|
||||
const now = Date.now();
|
||||
|
||||
await manager.ingest("options", {
|
||||
|
|
@ -1098,7 +1094,9 @@ describe("LiveStateManager", () => {
|
|||
|
||||
expect(quietSnapshot.items).toEqual([]);
|
||||
expect(manager.getHotChannelHealth().options.healthy).toBe(true);
|
||||
expect(manager.getStatsSnapshot().freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.options]).toBeLessThanOrEqual(50);
|
||||
expect(
|
||||
manager.getStatsSnapshot().freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.options]
|
||||
).toBeLessThanOrEqual(50);
|
||||
});
|
||||
|
||||
it("exposes freshness helper for feed status", () => {
|
||||
|
|
|
|||
|
|
@ -33,9 +33,7 @@ const envSchema = z.object({
|
|||
CANDLE_INTERVALS_MS: z.string().default("60000,300000"),
|
||||
CANDLE_MAX_LATE_MS: z.coerce.number().int().nonnegative().default(0),
|
||||
CANDLE_CACHE_LIMIT: z.coerce.number().int().nonnegative().default(2000),
|
||||
CANDLE_DELIVER_POLICY: z
|
||||
.enum(["new", "all", "last", "last_per_subject"])
|
||||
.default("new"),
|
||||
CANDLE_DELIVER_POLICY: z.enum(["new", "all", "last", "last_per_subject"]).default("new"),
|
||||
CANDLE_CONSUMER_RESET: z
|
||||
.preprocess((value) => {
|
||||
if (typeof value === "string") {
|
||||
|
|
@ -290,7 +288,10 @@ const run = async () => {
|
|||
} else {
|
||||
try {
|
||||
const info = await jsm.consumers.info(STREAM_EQUITY_PRINTS, durableName);
|
||||
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.CANDLE_DELIVER_POLICY) {
|
||||
if (
|
||||
info?.config?.deliver_policy &&
|
||||
info.config.deliver_policy !== env.CANDLE_DELIVER_POLICY
|
||||
) {
|
||||
logger.warn("resetting consumer due to deliver policy change", {
|
||||
durable: durableName,
|
||||
current: info.config.deliver_policy,
|
||||
|
|
@ -301,7 +302,10 @@ const run = async () => {
|
|||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (!message.includes("not found")) {
|
||||
logger.warn("failed to inspect jetstream consumer", { durable: durableName, error: message });
|
||||
logger.warn("failed to inspect jetstream consumer", {
|
||||
durable: durableName,
|
||||
error: message
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -327,7 +331,8 @@ const run = async () => {
|
|||
try {
|
||||
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, durableName);
|
||||
} catch (deleteError) {
|
||||
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
const deleteMessage =
|
||||
deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
if (!deleteMessage.includes("not found")) {
|
||||
logger.warn("failed to delete jetstream consumer", {
|
||||
durable: durableName,
|
||||
|
|
|
|||
|
|
@ -14,4 +14,3 @@ export const scoreAlert = (
|
|||
const severity = score >= 80 ? "high" : score >= 45 ? "medium" : "low";
|
||||
return { score, severity };
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -573,10 +573,7 @@ const buildVerticalSpreadHit = (
|
|||
};
|
||||
};
|
||||
|
||||
const buildLadderHit = (
|
||||
packet: FlowPacket,
|
||||
config: ClassifierConfig
|
||||
): ClassifierHit | null => {
|
||||
const buildLadderHit = (packet: FlowPacket, config: ClassifierConfig): ClassifierHit | null => {
|
||||
const structureType = getStringFeature(packet, "structure_type");
|
||||
if (structureType !== "ladder") {
|
||||
return null;
|
||||
|
|
@ -648,7 +645,8 @@ const buildRollHit = (packet: FlowPacket, config: ClassifierConfig): ClassifierH
|
|||
}
|
||||
|
||||
const activity = getLargeActivity(packet, config);
|
||||
const qualifies = activity.totalPremium >= config.spikeMinPremium || activity.totalSize >= config.spikeMinSize;
|
||||
const qualifies =
|
||||
activity.totalPremium >= config.spikeMinPremium || activity.totalSize >= config.spikeMinSize;
|
||||
if (!qualifies) {
|
||||
return null;
|
||||
}
|
||||
|
|
@ -708,7 +706,9 @@ const buildRollHit = (packet: FlowPacket, config: ClassifierConfig): ClassifierH
|
|||
|
||||
const expiryNote = hasExpiryPair
|
||||
? `Expiries: ${fromExpiry} -> ${toExpiry}${
|
||||
expiryDaysDelta !== null && expiryDaysDelta !== 0 ? ` (${Math.round(expiryDaysDelta)}d)` : ""
|
||||
expiryDaysDelta !== null && expiryDaysDelta !== 0
|
||||
? ` (${Math.round(expiryDaysDelta)}d)`
|
||||
: ""
|
||||
}.`
|
||||
: "Expiry pairing unavailable.";
|
||||
const strikeNote = hasStrikePair
|
||||
|
|
@ -850,9 +850,10 @@ export const evaluateClassifiers = (
|
|||
const packetKind = getStringFeature(packet, "packet_kind");
|
||||
const structureOnly = packetKind === "structure";
|
||||
|
||||
const contractId = typeof packet.features.option_contract_id === "string"
|
||||
? packet.features.option_contract_id
|
||||
: "";
|
||||
const contractId =
|
||||
typeof packet.features.option_contract_id === "string"
|
||||
? packet.features.option_contract_id
|
||||
: "";
|
||||
const contract = structureOnly ? null : parseContractId(contractId);
|
||||
const hits: ClassifierHit[] = [];
|
||||
|
||||
|
|
|
|||
|
|
@ -15,10 +15,7 @@ const roundTo = (value: number, digits = 4): number => {
|
|||
return Number(value.toFixed(digits));
|
||||
};
|
||||
|
||||
export const classifyQuotePlacement = (
|
||||
price: number,
|
||||
join: EquityQuoteJoin
|
||||
): QuotePlacement => {
|
||||
export const classifyQuotePlacement = (price: number, join: EquityQuoteJoin): QuotePlacement => {
|
||||
if (!Number.isFinite(price)) {
|
||||
return "MISSING";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ import {
|
|||
enqueueEquityPrintJoinInsert,
|
||||
enqueueFlowPacketInsert,
|
||||
enqueueInferredDarkInsert,
|
||||
enqueueSmartMoneyEventInsert,
|
||||
enqueueSmartMoneyEventInsert
|
||||
} from "@islandflow/storage";
|
||||
import {
|
||||
AlertEventSchema,
|
||||
|
|
@ -324,7 +324,9 @@ const buildPacketId = (cluster: ClusterState): string => {
|
|||
|
||||
const isExpectedShutdownNatsError = (error: unknown): boolean => {
|
||||
const code = getErrorCode(error);
|
||||
return runtimeState.shuttingDown && (code === "CONNECTION_DRAINING" || code === "CONNECTION_CLOSED");
|
||||
return (
|
||||
runtimeState.shuttingDown && (code === "CONNECTION_DRAINING" || code === "CONNECTION_CLOSED")
|
||||
);
|
||||
};
|
||||
|
||||
const createPlacementCounts = (): NbboPlacementCounts => ({
|
||||
|
|
@ -337,7 +339,14 @@ const createPlacementCounts = (): NbboPlacementCounts => ({
|
|||
stale: 0
|
||||
});
|
||||
|
||||
const SPECIAL_PRINT_CONDITIONS = new Set(["AUCTION", "CROSS", "OPENING", "CLOSING", "COMPLEX", "SPREAD"]);
|
||||
const SPECIAL_PRINT_CONDITIONS = new Set([
|
||||
"AUCTION",
|
||||
"CROSS",
|
||||
"OPENING",
|
||||
"CLOSING",
|
||||
"COMPLEX",
|
||||
"SPREAD"
|
||||
]);
|
||||
const SYNTHETIC_EVENT_CONDITION_RE = /^EVENT_(\d+)D$/i;
|
||||
|
||||
const normalizeConditions = (conditions: readonly string[] | undefined): string[] =>
|
||||
|
|
@ -460,11 +469,7 @@ const storeRecentRootLeg = (leg: LegEvidence, anchorTs: number): void => {
|
|||
recentLegsByRoot.set(key, next);
|
||||
};
|
||||
|
||||
const collectActiveLegs = (
|
||||
key: string,
|
||||
anchorTs: number,
|
||||
excludeId: string
|
||||
): LegEvidence[] => {
|
||||
const collectActiveLegs = (key: string, anchorTs: number, excludeId: string): LegEvidence[] => {
|
||||
const legs: LegEvidence[] = [];
|
||||
for (const [contractId, cluster] of clusters) {
|
||||
if (contractId === excludeId) {
|
||||
|
|
@ -485,11 +490,7 @@ const collectActiveLegs = (
|
|||
return legs;
|
||||
};
|
||||
|
||||
const collectActiveRootLegs = (
|
||||
key: string,
|
||||
anchorTs: number,
|
||||
excludeId: string
|
||||
): LegEvidence[] => {
|
||||
const collectActiveRootLegs = (key: string, anchorTs: number, excludeId: string): LegEvidence[] => {
|
||||
const legs: LegEvidence[] = [];
|
||||
for (const [contractId, cluster] of clusters) {
|
||||
if (contractId === excludeId) {
|
||||
|
|
@ -601,12 +602,19 @@ const applyDeliverPolicy = (
|
|||
const buildCluster = (print: OptionPrint): ClusterState => {
|
||||
const placements = createPlacementCounts();
|
||||
const normalizedConditions = normalizeConditions(print.conditions);
|
||||
const executionIv = typeof print.execution_iv === "number" && Number.isFinite(print.execution_iv) ? print.execution_iv : null;
|
||||
const executionIv =
|
||||
typeof print.execution_iv === "number" && Number.isFinite(print.execution_iv)
|
||||
? print.execution_iv
|
||||
: null;
|
||||
const executionUnderlyingMid =
|
||||
typeof print.execution_underlying_mid === "number" && Number.isFinite(print.execution_underlying_mid)
|
||||
typeof print.execution_underlying_mid === "number" &&
|
||||
Number.isFinite(print.execution_underlying_mid)
|
||||
? print.execution_underlying_mid
|
||||
: null;
|
||||
recordPlacement(placements, classifyPlacement(print.price, selectNbbo(print.option_contract_id, print.ts)));
|
||||
recordPlacement(
|
||||
placements,
|
||||
classifyPlacement(print.price, selectNbbo(print.option_contract_id, print.ts))
|
||||
);
|
||||
return {
|
||||
contractId: print.option_contract_id,
|
||||
underlyingId: print.underlying_id ?? null,
|
||||
|
|
@ -661,11 +669,18 @@ const updateCluster = (cluster: ClusterState, print: OptionPrint): ClusterState
|
|||
if (typeof print.execution_iv === "number" && Number.isFinite(print.execution_iv)) {
|
||||
cluster.lastExecutionIv = print.execution_iv;
|
||||
cluster.minExecutionIv =
|
||||
cluster.minExecutionIv === null ? print.execution_iv : Math.min(cluster.minExecutionIv, print.execution_iv);
|
||||
cluster.minExecutionIv === null
|
||||
? print.execution_iv
|
||||
: Math.min(cluster.minExecutionIv, print.execution_iv);
|
||||
cluster.maxExecutionIv =
|
||||
cluster.maxExecutionIv === null ? print.execution_iv : Math.max(cluster.maxExecutionIv, print.execution_iv);
|
||||
cluster.maxExecutionIv === null
|
||||
? print.execution_iv
|
||||
: Math.max(cluster.maxExecutionIv, print.execution_iv);
|
||||
}
|
||||
if (typeof print.execution_underlying_mid === "number" && Number.isFinite(print.execution_underlying_mid)) {
|
||||
if (
|
||||
typeof print.execution_underlying_mid === "number" &&
|
||||
Number.isFinite(print.execution_underlying_mid)
|
||||
) {
|
||||
if (cluster.firstUnderlyingMid === null) {
|
||||
cluster.firstUnderlyingMid = print.execution_underlying_mid;
|
||||
}
|
||||
|
|
@ -686,11 +701,7 @@ type NbboJoin = {
|
|||
|
||||
const updateNbboCache = (nbbo: OptionNBBO): void => {
|
||||
const existing = nbboCache.get(nbbo.option_contract_id);
|
||||
if (
|
||||
!existing ||
|
||||
nbbo.ts > existing.ts ||
|
||||
(nbbo.ts === existing.ts && nbbo.seq >= existing.seq)
|
||||
) {
|
||||
if (!existing || nbbo.ts > existing.ts || (nbbo.ts === existing.ts && nbbo.seq >= existing.seq)) {
|
||||
nbboCache.set(nbbo.option_contract_id, nbbo);
|
||||
nbboCacheTouchedAt.set(nbbo.option_contract_id, Date.now());
|
||||
}
|
||||
|
|
@ -907,14 +918,18 @@ const flushCluster = async (
|
|||
features.special_print_count = cluster.specialPrintCount;
|
||||
}
|
||||
if (cluster.minExecutionIv !== null && cluster.maxExecutionIv !== null) {
|
||||
features.execution_iv_shock = roundTo(Math.max(0, cluster.maxExecutionIv - cluster.minExecutionIv));
|
||||
features.execution_iv_shock = roundTo(
|
||||
Math.max(0, cluster.maxExecutionIv - cluster.minExecutionIv)
|
||||
);
|
||||
}
|
||||
if (
|
||||
cluster.firstUnderlyingMid !== null &&
|
||||
cluster.lastUnderlyingMid !== null &&
|
||||
cluster.firstUnderlyingMid > 0
|
||||
) {
|
||||
const moveBps = ((cluster.lastUnderlyingMid - cluster.firstUnderlyingMid) / cluster.firstUnderlyingMid) * 10_000;
|
||||
const moveBps =
|
||||
((cluster.lastUnderlyingMid - cluster.firstUnderlyingMid) / cluster.firstUnderlyingMid) *
|
||||
10_000;
|
||||
features.underlying_move_bps = roundTo(moveBps);
|
||||
}
|
||||
const syntheticEventOffsetDays = parseSyntheticEventOffsetDays(cluster.conditions);
|
||||
|
|
@ -1004,7 +1019,13 @@ const flushCluster = async (
|
|||
const rollLegs = [currentLeg, ...rootCandidates];
|
||||
const rollSummary = summarizeStructure(rollLegs);
|
||||
if (rollSummary?.type === "roll") {
|
||||
await emitStructurePacketIfNeeded(js, batchWriter, rollLegs, rollSummary, currentLeg.contractId);
|
||||
await emitStructurePacketIfNeeded(
|
||||
js,
|
||||
batchWriter,
|
||||
rollLegs,
|
||||
rollSummary,
|
||||
currentLeg.contractId
|
||||
);
|
||||
}
|
||||
|
||||
storeRecentLeg(currentLeg, anchorTs);
|
||||
|
|
@ -1072,13 +1093,21 @@ const emitClassifiers = async (
|
|||
const underlyingId =
|
||||
typeof packet.features.underlying_id === "string"
|
||||
? packet.features.underlying_id
|
||||
: parseContractId(typeof packet.features.option_contract_id === "string" ? packet.features.option_contract_id : "")?.root;
|
||||
: parseContractId(
|
||||
typeof packet.features.option_contract_id === "string"
|
||||
? packet.features.option_contract_id
|
||||
: ""
|
||||
)?.root;
|
||||
const referenceTs =
|
||||
typeof packet.features.end_ts === "number" && Number.isFinite(packet.features.end_ts)
|
||||
? packet.features.end_ts
|
||||
: packet.source_ts;
|
||||
const eventCalendarMatch = underlyingId ? eventCalendarProvider.findNextEvent(underlyingId, referenceTs) : null;
|
||||
smartMoneyEvent = SmartMoneyEventSchema.parse(buildSmartMoneyEventFromPacket(packet, { eventCalendarMatch }));
|
||||
const eventCalendarMatch = underlyingId
|
||||
? eventCalendarProvider.findNextEvent(underlyingId, referenceTs)
|
||||
: null;
|
||||
smartMoneyEvent = SmartMoneyEventSchema.parse(
|
||||
buildSmartMoneyEventFromPacket(packet, { eventCalendarMatch })
|
||||
);
|
||||
enqueueSmartMoneyEventInsert(batchWriter, smartMoneyEvent);
|
||||
await publishJson(js, SUBJECT_SMART_MONEY_EVENTS, smartMoneyEvent);
|
||||
emitCounters.smartMoneyEvents += 1;
|
||||
|
|
@ -1282,20 +1311,29 @@ const run = async () => {
|
|||
|
||||
if (env.SMART_MONEY_EVENT_CALENDAR_PATH) {
|
||||
try {
|
||||
eventCalendarProvider = await loadEventCalendarProviderFromFile(env.SMART_MONEY_EVENT_CALENDAR_PATH);
|
||||
logger.info("smart money event calendar loaded", { path: env.SMART_MONEY_EVENT_CALENDAR_PATH });
|
||||
eventCalendarProvider = await loadEventCalendarProviderFromFile(
|
||||
env.SMART_MONEY_EVENT_CALENDAR_PATH
|
||||
);
|
||||
logger.info("smart money event calendar loaded", {
|
||||
path: env.SMART_MONEY_EVENT_CALENDAR_PATH
|
||||
});
|
||||
} catch (error) {
|
||||
eventCalendarProvider = createEmptyEventCalendarProvider();
|
||||
logger.warn("smart money event calendar unavailable; scoring will use neutral event features", {
|
||||
path: env.SMART_MONEY_EVENT_CALENDAR_PATH,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
});
|
||||
logger.warn(
|
||||
"smart money event calendar unavailable; scoring will use neutral event features",
|
||||
{
|
||||
path: env.SMART_MONEY_EVENT_CALENDAR_PATH,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const redis = createRedisClient(env.REDIS_URL);
|
||||
redis.on("error", (error) => {
|
||||
logger.warn("redis client error", { error: error instanceof Error ? error.message : String(error) });
|
||||
logger.warn("redis client error", {
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
});
|
||||
});
|
||||
|
||||
await retry("redis connect", 120, 500, async () => {
|
||||
|
|
@ -1379,7 +1417,10 @@ const run = async () => {
|
|||
} else {
|
||||
try {
|
||||
const info = await jsm.consumers.info(STREAM_OPTION_SIGNAL_PRINTS, durableName);
|
||||
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
|
||||
if (
|
||||
info?.config?.deliver_policy &&
|
||||
info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY
|
||||
) {
|
||||
logger.warn("resetting consumer due to deliver policy change", {
|
||||
durable: durableName,
|
||||
current: info.config.deliver_policy,
|
||||
|
|
@ -1390,7 +1431,10 @@ const run = async () => {
|
|||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (!message.includes("not found")) {
|
||||
logger.warn("failed to inspect jetstream consumer", { durable: durableName, error: message });
|
||||
logger.warn("failed to inspect jetstream consumer", {
|
||||
durable: durableName,
|
||||
error: message
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1402,13 +1446,19 @@ const run = async () => {
|
|||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (!message.includes("not found")) {
|
||||
logger.warn("failed to reset jetstream consumer", { durable: nbboDurableName, error: message });
|
||||
logger.warn("failed to reset jetstream consumer", {
|
||||
durable: nbboDurableName,
|
||||
error: message
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
const info = await jsm.consumers.info(STREAM_OPTION_NBBO, nbboDurableName);
|
||||
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
|
||||
if (
|
||||
info?.config?.deliver_policy &&
|
||||
info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY
|
||||
) {
|
||||
logger.warn("resetting consumer due to deliver policy change", {
|
||||
durable: nbboDurableName,
|
||||
current: info.config.deliver_policy,
|
||||
|
|
@ -1419,7 +1469,10 @@ const run = async () => {
|
|||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (!message.includes("not found")) {
|
||||
logger.warn("failed to inspect jetstream consumer", { durable: nbboDurableName, error: message });
|
||||
logger.warn("failed to inspect jetstream consumer", {
|
||||
durable: nbboDurableName,
|
||||
error: message
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1440,7 +1493,10 @@ const run = async () => {
|
|||
} else {
|
||||
try {
|
||||
const info = await jsm.consumers.info(STREAM_EQUITY_PRINTS, equityPrintDurableName);
|
||||
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
|
||||
if (
|
||||
info?.config?.deliver_policy &&
|
||||
info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY
|
||||
) {
|
||||
logger.warn("resetting consumer due to deliver policy change", {
|
||||
durable: equityPrintDurableName,
|
||||
current: info.config.deliver_policy,
|
||||
|
|
@ -1475,7 +1531,10 @@ const run = async () => {
|
|||
} else {
|
||||
try {
|
||||
const info = await jsm.consumers.info(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
|
||||
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
|
||||
if (
|
||||
info?.config?.deliver_policy &&
|
||||
info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY
|
||||
) {
|
||||
logger.warn("resetting consumer due to deliver policy change", {
|
||||
durable: equityQuoteDurableName,
|
||||
current: info.config.deliver_policy,
|
||||
|
|
@ -1515,7 +1574,8 @@ const run = async () => {
|
|||
try {
|
||||
await jsm.consumers.delete(STREAM_OPTION_SIGNAL_PRINTS, durableName);
|
||||
} catch (deleteError) {
|
||||
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
const deleteMessage =
|
||||
deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
if (!deleteMessage.includes("not found")) {
|
||||
logger.warn("failed to delete jetstream consumer", {
|
||||
durable: durableName,
|
||||
|
|
@ -1551,7 +1611,8 @@ const run = async () => {
|
|||
try {
|
||||
await jsm.consumers.delete(STREAM_OPTION_NBBO, nbboDurableName);
|
||||
} catch (deleteError) {
|
||||
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
const deleteMessage =
|
||||
deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
if (!deleteMessage.includes("not found")) {
|
||||
logger.warn("failed to delete jetstream consumer", {
|
||||
durable: nbboDurableName,
|
||||
|
|
@ -1582,12 +1643,16 @@ const run = async () => {
|
|||
throw error;
|
||||
}
|
||||
|
||||
logger.warn("resetting jetstream consumer", { durable: equityPrintDurableName, error: message });
|
||||
logger.warn("resetting jetstream consumer", {
|
||||
durable: equityPrintDurableName,
|
||||
error: message
|
||||
});
|
||||
|
||||
try {
|
||||
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, equityPrintDurableName);
|
||||
} catch (deleteError) {
|
||||
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
const deleteMessage =
|
||||
deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
if (!deleteMessage.includes("not found")) {
|
||||
logger.warn("failed to delete jetstream consumer", {
|
||||
durable: equityPrintDurableName,
|
||||
|
|
@ -1626,7 +1691,8 @@ const run = async () => {
|
|||
try {
|
||||
await jsm.consumers.delete(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
|
||||
} catch (deleteError) {
|
||||
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
const deleteMessage =
|
||||
deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
if (!deleteMessage.includes("not found")) {
|
||||
logger.warn("failed to delete jetstream consumer", {
|
||||
durable: equityQuoteDurableName,
|
||||
|
|
|
|||
|
|
@ -78,7 +78,9 @@ const getDteDays = (packet: FlowPacket): number | null => {
|
|||
|
||||
const inferDirection = (packet: FlowPacket): SmartMoneyDirection => {
|
||||
const structureRights = stringFeature(packet, "structure_rights");
|
||||
const optionType = stringFeature(packet, "option_type") || parseContractId(stringFeature(packet, "option_contract_id"))?.right;
|
||||
const optionType =
|
||||
stringFeature(packet, "option_type") ||
|
||||
parseContractId(stringFeature(packet, "option_contract_id"))?.right;
|
||||
const buy = numberFeature(packet, "nbbo_aggressive_buy_ratio");
|
||||
const sell = numberFeature(packet, "nbbo_aggressive_sell_ratio");
|
||||
const sellDominant = sell >= buy + 0.12;
|
||||
|
|
@ -102,16 +104,26 @@ export type SmartMoneyParentEventOptions = {
|
|||
eventCalendarMatch?: EventCalendarMatch | null;
|
||||
};
|
||||
|
||||
const buildFeatures = (packet: FlowPacket, options: SmartMoneyParentEventOptions = {}): SmartMoneyFeatures => {
|
||||
const buildFeatures = (
|
||||
packet: FlowPacket,
|
||||
options: SmartMoneyParentEventOptions = {}
|
||||
): SmartMoneyFeatures => {
|
||||
const contractId = stringFeature(packet, "option_contract_id");
|
||||
const contract = parseContractId(contractId);
|
||||
const underlyingMid = numberFeature(packet, "underlying_mid");
|
||||
const quoteAge = numberFeature(packet, "nbbo_age_ms") || numberFeature(packet, "underlying_quote_age_ms");
|
||||
const printCount = Math.max(0, Math.round(numberFeature(packet, "count") || packet.members.length));
|
||||
const quoteAge =
|
||||
numberFeature(packet, "nbbo_age_ms") || numberFeature(packet, "underlying_quote_age_ms");
|
||||
const printCount = Math.max(
|
||||
0,
|
||||
Math.round(numberFeature(packet, "count") || packet.members.length)
|
||||
);
|
||||
const staleCount = numberFeature(packet, "nbbo_stale_count");
|
||||
const missingCount = numberFeature(packet, "nbbo_missing_count");
|
||||
const structureLegs = Math.max(0, Math.round(numberFeature(packet, "structure_legs")));
|
||||
const strikeCount = Math.max(1, Math.round(numberFeature(packet, "structure_strikes") || (contract ? 1 : 0)));
|
||||
const strikeCount = Math.max(
|
||||
1,
|
||||
Math.round(numberFeature(packet, "structure_strikes") || (contract ? 1 : 0))
|
||||
);
|
||||
const specialCount = numberFeature(packet, "special_print_count");
|
||||
const calendarEventTs = options.eventCalendarMatch?.event_ts ?? null;
|
||||
const eventTs = calendarEventTs ?? numberFeature(packet, "corporate_event_ts");
|
||||
|
|
@ -119,7 +131,9 @@ const buildFeatures = (packet: FlowPacket, options: SmartMoneyParentEventOptions
|
|||
const expiryTs = contract ? Date.parse(`${contract.expiry}T00:00:00Z`) : Number.NaN;
|
||||
|
||||
const atmProximity =
|
||||
contract && underlyingMid > 0 ? Math.abs(contract.strike - underlyingMid) / underlyingMid : null;
|
||||
contract && underlyingMid > 0
|
||||
? Math.abs(contract.strike - underlyingMid) / underlyingMid
|
||||
: null;
|
||||
|
||||
return {
|
||||
contract_count: Math.max(1, structureLegs || 1),
|
||||
|
|
@ -143,14 +157,18 @@ const buildFeatures = (packet: FlowPacket, options: SmartMoneyParentEventOptions
|
|||
nbbo_stale_ratio: printCount > 0 ? clamp((staleCount + missingCount) / printCount) : 0,
|
||||
quote_age_ms: quoteAge > 0 ? quoteAge : null,
|
||||
venue_count: Math.max(1, Math.round(numberFeature(packet, "venue_count") || 1)),
|
||||
inter_fill_ms_mean: printCount > 1 ? numberFeature(packet, "window_ms") / Math.max(1, printCount - 1) : null,
|
||||
inter_fill_ms_mean:
|
||||
printCount > 1 ? numberFeature(packet, "window_ms") / Math.max(1, printCount - 1) : null,
|
||||
strike_count: strikeCount,
|
||||
strike_concentration: strikeCount > 0 ? clamp(1 / strikeCount) : 0,
|
||||
...(stringFeature(packet, "structure_type") ? { structure_type: stringFeature(packet, "structure_type") } : {}),
|
||||
...(stringFeature(packet, "structure_type")
|
||||
? { structure_type: stringFeature(packet, "structure_type") }
|
||||
: {}),
|
||||
structure_legs: structureLegs,
|
||||
same_size_leg_symmetry: clamp(numberFeature(packet, "same_size_leg_symmetry")),
|
||||
net_directional_bias: clamp(
|
||||
numberFeature(packet, "nbbo_aggressive_buy_ratio") - numberFeature(packet, "nbbo_aggressive_sell_ratio"),
|
||||
numberFeature(packet, "nbbo_aggressive_buy_ratio") -
|
||||
numberFeature(packet, "nbbo_aggressive_sell_ratio"),
|
||||
-1,
|
||||
1
|
||||
),
|
||||
|
|
@ -159,7 +177,10 @@ const buildFeatures = (packet: FlowPacket, options: SmartMoneyParentEventOptions
|
|||
underlying_move_bps: numberFeature(packet, "underlying_move_bps") || null,
|
||||
days_to_event: eventTs > 0 ? (eventTs - referenceTs) / MS_PER_DAY : null,
|
||||
expiry_after_event: eventTs > 0 && Number.isFinite(expiryTs) ? expiryTs >= eventTs : null,
|
||||
pre_event_concentration: eventTs > 0 && eventTs >= referenceTs ? clamp(1 - (eventTs - referenceTs) / (21 * MS_PER_DAY)) : null,
|
||||
pre_event_concentration:
|
||||
eventTs > 0 && eventTs >= referenceTs
|
||||
? clamp(1 - (eventTs - referenceTs) / (21 * MS_PER_DAY))
|
||||
: null,
|
||||
special_print_ratio: printCount > 0 ? clamp(specialCount / printCount) : 0
|
||||
};
|
||||
};
|
||||
|
|
@ -170,7 +191,10 @@ const detectSuppression = (packet: FlowPacket, features: SmartMoneyFeatures): st
|
|||
.split(",")
|
||||
.map((item) => item.trim().toUpperCase())
|
||||
.filter(Boolean);
|
||||
if (conditions.some((condition) => SPECIAL_CONDITIONS.has(condition)) || features.special_print_ratio >= 0.34) {
|
||||
if (
|
||||
conditions.some((condition) => SPECIAL_CONDITIONS.has(condition)) ||
|
||||
features.special_print_ratio >= 0.34
|
||||
) {
|
||||
reasons.push("special_print_or_complex_context");
|
||||
}
|
||||
if (features.nbbo_coverage_ratio < 0.35 || features.nbbo_stale_ratio >= 0.5) {
|
||||
|
|
@ -198,7 +222,10 @@ const evaluateProfiles = (
|
|||
const burstFactor = clamp(features.print_count / 8);
|
||||
const quality = clamp(features.nbbo_coverage_ratio - features.nbbo_stale_ratio);
|
||||
const shortDatedOtm =
|
||||
dte <= 7 && features.atm_proximity !== null && features.atm_proximity >= 0.05 && features.option_type === "C";
|
||||
dte <= 7 &&
|
||||
features.atm_proximity !== null &&
|
||||
features.atm_proximity >= 0.05 &&
|
||||
features.option_type === "C";
|
||||
const nearAtm = features.atm_proximity !== null && features.atm_proximity <= 0.015;
|
||||
const preEvent =
|
||||
features.days_to_event !== null &&
|
||||
|
|
@ -211,7 +238,11 @@ const evaluateProfiles = (
|
|||
"institutional_directional",
|
||||
suppressed.length > 0 || shortDatedOtm
|
||||
? 0.18
|
||||
: 0.2 + premiumFactor * 0.25 + burstFactor * 0.18 + quality * 0.16 + (buy >= 0.58 || sell >= 0.58 ? 0.12 : 0),
|
||||
: 0.2 +
|
||||
premiumFactor * 0.25 +
|
||||
burstFactor * 0.18 +
|
||||
quality * 0.16 +
|
||||
(buy >= 0.58 || sell >= 0.58 ? 0.12 : 0),
|
||||
direction,
|
||||
[
|
||||
"large_parent_event",
|
||||
|
|
@ -232,13 +263,19 @@ const evaluateProfiles = (
|
|||
),
|
||||
score(
|
||||
"event_driven",
|
||||
0.12 + (preEvent ? 0.32 : 0) + premiumFactor * 0.14 + clamp(features.spread_widening ?? 0, 0, 0.16),
|
||||
0.12 +
|
||||
(preEvent ? 0.32 : 0) +
|
||||
premiumFactor * 0.14 +
|
||||
clamp(features.spread_widening ?? 0, 0, 0.16),
|
||||
direction === "unknown" ? "neutral" : direction,
|
||||
["event_calendar_alignment", "expiry_after_event", "pre_event_concentration"]
|
||||
),
|
||||
score(
|
||||
"vol_seller",
|
||||
0.12 + (sell >= 0.58 ? 0.24 : 0) + (structure === "straddle" || structure === "strangle" ? 0.2 : 0) + premiumFactor * 0.14,
|
||||
0.12 +
|
||||
(sell >= 0.58 ? 0.24 : 0) +
|
||||
(structure === "straddle" || structure === "strangle" ? 0.2 : 0) +
|
||||
premiumFactor * 0.14,
|
||||
"neutral",
|
||||
["sell_side_premium", "short_vol_structure_evidence"]
|
||||
),
|
||||
|
|
@ -273,11 +310,16 @@ export const buildSmartMoneyEventFromPacket = (
|
|||
const suppressed = detectSuppression(packet, features);
|
||||
const profileScores = evaluateProfiles(packet, features, suppressed);
|
||||
const primary = profileScores[0] ?? null;
|
||||
const abstained = !primary || primary.probability < 0.42 || suppressed.includes("stale_or_missing_quote_context");
|
||||
const underlying = stringFeature(packet, "underlying_id") || parseContractId(features.option_contract_id ?? "")?.root || "UNKNOWN";
|
||||
const eventKind = features.structure_legs >= 2 || stringFeature(packet, "packet_kind") === "structure"
|
||||
? "multi_leg_event"
|
||||
: "single_leg_event";
|
||||
const abstained =
|
||||
!primary || primary.probability < 0.42 || suppressed.includes("stale_or_missing_quote_context");
|
||||
const underlying =
|
||||
stringFeature(packet, "underlying_id") ||
|
||||
parseContractId(features.option_contract_id ?? "")?.root ||
|
||||
"UNKNOWN";
|
||||
const eventKind =
|
||||
features.structure_legs >= 2 || stringFeature(packet, "packet_kind") === "structure"
|
||||
? "multi_leg_event"
|
||||
: "single_leg_event";
|
||||
|
||||
return SmartMoneyEventSchema.parse({
|
||||
source_ts: packet.source_ts,
|
||||
|
|
@ -292,8 +334,8 @@ export const buildSmartMoneyEventFromPacket = (
|
|||
event_window_ms: features.window_ms,
|
||||
features,
|
||||
profile_scores: profileScores,
|
||||
primary_profile_id: abstained ? null : primary?.profile_id ?? null,
|
||||
primary_direction: abstained ? "unknown" : primary?.direction ?? "unknown",
|
||||
primary_profile_id: abstained ? null : (primary?.profile_id ?? null),
|
||||
primary_direction: abstained ? "unknown" : (primary?.direction ?? "unknown"),
|
||||
abstained,
|
||||
suppressed_reasons: suppressed
|
||||
});
|
||||
|
|
@ -308,7 +350,9 @@ const LEGACY_PROFILE_MAP: Record<SmartMoneyProfileId, string> = {
|
|||
hedge_reactive: "smart_money_hedge_reactive"
|
||||
};
|
||||
|
||||
export const deriveClassifierHitsFromSmartMoneyEvent = (event: SmartMoneyEvent): ClassifierHit[] => {
|
||||
export const deriveClassifierHitsFromSmartMoneyEvent = (
|
||||
event: SmartMoneyEvent
|
||||
): ClassifierHit[] => {
|
||||
if (event.abstained || !event.primary_profile_id) {
|
||||
return [];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,9 +24,7 @@ type RollingWindowEntry = {
|
|||
};
|
||||
|
||||
const toNumbers = (values: string[]): number[] => {
|
||||
return values
|
||||
.map((value) => Number(value))
|
||||
.filter((value) => Number.isFinite(value));
|
||||
return values.map((value) => Number(value)).filter((value) => Number.isFinite(value));
|
||||
};
|
||||
|
||||
export const computeStats = (values: number[]): { mean: number; stddev: number; count: number } => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
import type { FlowPacket, SmartMoneyDirection, SmartMoneyEvent, SmartMoneyProfileId } from "@islandflow/types";
|
||||
import type {
|
||||
FlowPacket,
|
||||
SmartMoneyDirection,
|
||||
SmartMoneyEvent,
|
||||
SmartMoneyProfileId
|
||||
} from "@islandflow/types";
|
||||
import { buildSmartMoneyEventFromPacket, type SmartMoneyParentEventOptions } from "./parent-events";
|
||||
|
||||
export type SmartMoneyLabel = {
|
||||
|
|
@ -115,8 +120,12 @@ export const compareSmartMoneyReplayOutputs = (
|
|||
liveEvents: SmartMoneyEvent[],
|
||||
batchEvents: SmartMoneyEvent[]
|
||||
): ReplayConsistencyReport => {
|
||||
const liveById = new Map(liveEvents.map((event) => [event.event_id, smartMoneyEventSignature(event)]));
|
||||
const batchById = new Map(batchEvents.map((event) => [event.event_id, smartMoneyEventSignature(event)]));
|
||||
const liveById = new Map(
|
||||
liveEvents.map((event) => [event.event_id, smartMoneyEventSignature(event)])
|
||||
);
|
||||
const batchById = new Map(
|
||||
batchEvents.map((event) => [event.event_id, smartMoneyEventSignature(event)])
|
||||
);
|
||||
const ids = [...new Set([...liveById.keys(), ...batchById.keys()])].sort();
|
||||
const mismatches: ReplayConsistencyMismatch[] = [];
|
||||
|
||||
|
|
@ -153,7 +162,9 @@ export const evaluateSmartMoneyEvents = (
|
|||
const labelsById = new Map(labels.map((label) => [label.event_id, label]));
|
||||
const labeledEvents = events
|
||||
.map((event) => ({ event, label: labelsById.get(event.event_id) }))
|
||||
.filter((entry): entry is { event: SmartMoneyEvent; label: SmartMoneyLabel } => Boolean(entry.label));
|
||||
.filter((entry): entry is { event: SmartMoneyEvent; label: SmartMoneyLabel } =>
|
||||
Boolean(entry.label)
|
||||
);
|
||||
|
||||
const emitted = events.filter((event) => !event.abstained && event.primary_profile_id);
|
||||
const profilePrecision: SmartMoneyEvaluationReport["profile_precision"] = {};
|
||||
|
|
@ -163,7 +174,8 @@ export const evaluateSmartMoneyEvents = (
|
|||
const predicted = labeledEvents.filter((entry) => entry.event.primary_profile_id === profile);
|
||||
const actual = labeledEvents.filter((entry) => entry.label.profile_id === profile);
|
||||
const truePositive = predicted.filter((entry) => entry.label.profile_id === profile).length;
|
||||
profilePrecision[profile] = predicted.length > 0 ? round(truePositive / predicted.length) : null;
|
||||
profilePrecision[profile] =
|
||||
predicted.length > 0 ? round(truePositive / predicted.length) : null;
|
||||
profileRecall[profile] = actual.length > 0 ? round(truePositive / actual.length) : null;
|
||||
}
|
||||
|
||||
|
|
@ -175,7 +187,10 @@ export const evaluateSmartMoneyEvents = (
|
|||
labeled_count: labeledEvents.length,
|
||||
emitted_count: emitted.length,
|
||||
abstained_count: events.filter((event) => event.abstained).length,
|
||||
abstention_rate: events.length > 0 ? round(events.filter((event) => event.abstained).length / events.length) : 0,
|
||||
abstention_rate:
|
||||
events.length > 0
|
||||
? round(events.filter((event) => event.abstained).length / events.length)
|
||||
: 0,
|
||||
profile_precision: profilePrecision,
|
||||
profile_recall: profileRecall,
|
||||
calibration,
|
||||
|
|
@ -195,7 +210,9 @@ const buildCalibration = (
|
|||
}));
|
||||
|
||||
for (const { event, label } of entries) {
|
||||
const probability = event.profile_scores.find((entry) => entry.profile_id === event.primary_profile_id)?.probability ?? 0;
|
||||
const probability =
|
||||
event.profile_scores.find((entry) => entry.profile_id === event.primary_profile_id)
|
||||
?.probability ?? 0;
|
||||
const index = Math.min(bucketCount - 1, Math.floor(probability * bucketCount));
|
||||
buckets[index].probabilities.push(probability);
|
||||
if (!event.abstained && event.primary_profile_id === label.profile_id) {
|
||||
|
|
@ -209,9 +226,13 @@ const buildCalibration = (
|
|||
count: bucket.probabilities.length,
|
||||
average_probability:
|
||||
bucket.probabilities.length > 0
|
||||
? round(bucket.probabilities.reduce((sum, value) => sum + value, 0) / bucket.probabilities.length)
|
||||
? round(
|
||||
bucket.probabilities.reduce((sum, value) => sum + value, 0) /
|
||||
bucket.probabilities.length
|
||||
)
|
||||
: 0,
|
||||
accuracy: bucket.probabilities.length > 0 ? round(bucket.correct / bucket.probabilities.length) : null
|
||||
accuracy:
|
||||
bucket.probabilities.length > 0 ? round(bucket.correct / bucket.probabilities.length) : null
|
||||
}));
|
||||
};
|
||||
|
||||
|
|
@ -223,7 +244,10 @@ const buildEconomicSanity = (
|
|||
sign: directionalSign(event.primary_direction),
|
||||
realized: label.realized_return_bps
|
||||
}))
|
||||
.filter((entry): entry is { sign: number; realized: number } => entry.sign !== 0 && Number.isFinite(entry.realized));
|
||||
.filter(
|
||||
(entry): entry is { sign: number; realized: number } =>
|
||||
entry.sign !== 0 && Number.isFinite(entry.realized)
|
||||
);
|
||||
|
||||
if (directional.length === 0) {
|
||||
return {
|
||||
|
|
@ -236,7 +260,12 @@ const buildEconomicSanity = (
|
|||
const signedReturns = directional.map((entry) => entry.sign * entry.realized);
|
||||
return {
|
||||
directional_count: directional.length,
|
||||
direction_hit_rate: round(signedReturns.filter((value) => value > 0).length / directional.length),
|
||||
average_signed_return_bps: round(signedReturns.reduce((sum, value) => sum + value, 0) / signedReturns.length, 2)
|
||||
direction_hit_rate: round(
|
||||
signedReturns.filter((value) => value > 0).length / directional.length
|
||||
),
|
||||
average_signed_return_bps: round(
|
||||
signedReturns.reduce((sum, value) => sum + value, 0) / signedReturns.length,
|
||||
2
|
||||
)
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -134,7 +134,9 @@ const dayDiff = (from: string | null, to: string | null): number | null => {
|
|||
};
|
||||
|
||||
const sameSizeLegSymmetry = (legs: LegEvidence[]): number => {
|
||||
const sizes = legs.map((leg) => leg.totalSize).filter((value) => Number.isFinite(value) && value > 0);
|
||||
const sizes = legs
|
||||
.map((leg) => leg.totalSize)
|
||||
.filter((value) => Number.isFinite(value) && value > 0);
|
||||
if (sizes.length < 2) {
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -146,7 +148,10 @@ const sameSizeLegSymmetry = (legs: LegEvidence[]): number => {
|
|||
return min / max;
|
||||
};
|
||||
|
||||
export const shouldEmitStructurePacket = (legs: LegEvidence[], currentLegContractId: string): boolean => {
|
||||
export const shouldEmitStructurePacket = (
|
||||
legs: LegEvidence[],
|
||||
currentLegContractId: string
|
||||
): boolean => {
|
||||
if (legs.length < 2) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -226,7 +231,8 @@ export const planStructurePacket = (
|
|||
const totalSize = legs.reduce((sum, leg) => sum + leg.totalSize, 0);
|
||||
const count = legs.reduce((sum, leg) => sum + leg.members.length, 0);
|
||||
const placements = mergePlacements(legs);
|
||||
const placementTotal = placements.aa + placements.a + placements.b + placements.bb + placements.mid;
|
||||
const placementTotal =
|
||||
placements.aa + placements.a + placements.b + placements.bb + placements.mid;
|
||||
const aggressiveTotal = placements.aa + placements.a + placements.b + placements.bb;
|
||||
const aggressiveBuy = placements.aa + placements.a;
|
||||
const aggressiveSell = placements.bb + placements.b;
|
||||
|
|
@ -235,7 +241,10 @@ export const planStructurePacket = (
|
|||
const nbboAggressiveSellRatio = aggressiveTotal > 0 ? aggressiveSell / aggressiveTotal : 0;
|
||||
const nbboAggressiveRatio = placementTotal > 0 ? aggressiveTotal / placementTotal : 0;
|
||||
|
||||
const source_ts = legs.reduce((min, leg) => Math.min(min, leg.source_ts), Number.POSITIVE_INFINITY);
|
||||
const source_ts = legs.reduce(
|
||||
(min, leg) => Math.min(min, leg.source_ts),
|
||||
Number.POSITIVE_INFINITY
|
||||
);
|
||||
const ingest_ts = legs.reduce((max, leg) => Math.max(max, leg.ingest_ts), 0);
|
||||
const seq = legs.reduce((max, leg) => Math.max(max, leg.seq), 0);
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,10 @@ export const summarizeStructure = (legs: ContractLeg[]): StructureSummary | null
|
|||
legs: legs.length,
|
||||
strikes: strikes.length,
|
||||
strikeSpan,
|
||||
rights: rights.size === 2 ? "C/P" : Array.from(rights)[0] ?? "",
|
||||
contractIds: legs.map((leg) => leg.contractId).slice().sort()
|
||||
rights: rights.size === 2 ? "C/P" : (Array.from(rights)[0] ?? ""),
|
||||
contractIds: legs
|
||||
.map((leg) => leg.contractId)
|
||||
.slice()
|
||||
.sort()
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -293,4 +293,3 @@ describe("compute classifiers", () => {
|
|||
expect(hit!.explanations[0]).toMatch(/Consistent with/i);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -17,16 +17,18 @@ export const TEST_CLASSIFIER_CONFIG: ClassifierConfig = {
|
|||
zeroDteMinSize: 400
|
||||
};
|
||||
|
||||
export const buildFlowPacket = (opts: {
|
||||
id?: string;
|
||||
source_ts?: number;
|
||||
ingest_ts?: number;
|
||||
seq?: number;
|
||||
trace_id?: string;
|
||||
members?: string[];
|
||||
features?: FlowPacket["features"];
|
||||
join_quality?: FlowPacket["join_quality"];
|
||||
} = {}): FlowPacket => {
|
||||
export const buildFlowPacket = (
|
||||
opts: {
|
||||
id?: string;
|
||||
source_ts?: number;
|
||||
ingest_ts?: number;
|
||||
seq?: number;
|
||||
trace_id?: string;
|
||||
members?: string[];
|
||||
features?: FlowPacket["features"];
|
||||
join_quality?: FlowPacket["join_quality"];
|
||||
} = {}
|
||||
): FlowPacket => {
|
||||
const id = opts.id ?? "flowpacket:test";
|
||||
const source_ts = opts.source_ts ?? Date.parse("2025-01-01T14:30:00Z");
|
||||
const ingest_ts = opts.ingest_ts ?? source_ts;
|
||||
|
|
@ -66,4 +68,3 @@ export const buildFlowPacket = (opts: {
|
|||
export const getHit = (hits: ClassifierHit[], id: string): ClassifierHit | null => {
|
||||
return hits.find((hit) => hit.classifier_id === id) ?? null;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,9 @@ const placements = (overrides?: Partial<LegEvidence["placements"]>): LegEvidence
|
|||
...overrides
|
||||
});
|
||||
|
||||
const leg = (input: Partial<LegEvidence> & Pick<LegEvidence, "contractId" | "right" | "strike">): LegEvidence => {
|
||||
const leg = (
|
||||
input: Partial<LegEvidence> & Pick<LegEvidence, "contractId" | "right" | "strike">
|
||||
): LegEvidence => {
|
||||
return {
|
||||
contractId: input.contractId,
|
||||
root: "SPY",
|
||||
|
|
|
|||
|
|
@ -85,10 +85,14 @@ const decodePayload = (data: WebSocket.RawData): unknown => {
|
|||
}
|
||||
|
||||
if (ArrayBuffer.isView(data)) {
|
||||
return JSON.parse(new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))) as unknown;
|
||||
return JSON.parse(
|
||||
new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))
|
||||
) as unknown;
|
||||
}
|
||||
|
||||
return JSON.parse(new TextDecoder().decode(new Uint8Array(data as unknown as ArrayBuffer))) as unknown;
|
||||
return JSON.parse(
|
||||
new TextDecoder().decode(new Uint8Array(data as unknown as ArrayBuffer))
|
||||
) as unknown;
|
||||
};
|
||||
|
||||
const extractExchangeMeta = (payload: unknown): AlpacaExchangeMetaEntry[] => {
|
||||
|
|
@ -103,8 +107,18 @@ const extractExchangeMeta = (payload: unknown): AlpacaExchangeMetaEntry[] => {
|
|||
continue;
|
||||
}
|
||||
const candidate = entry as Record<string, unknown>;
|
||||
const code = typeof candidate.code === "string" ? candidate.code : typeof candidate.exchange === "string" ? candidate.exchange : null;
|
||||
const name = typeof candidate.name === "string" ? candidate.name : typeof candidate.description === "string" ? candidate.description : null;
|
||||
const code =
|
||||
typeof candidate.code === "string"
|
||||
? candidate.code
|
||||
: typeof candidate.exchange === "string"
|
||||
? candidate.exchange
|
||||
: null;
|
||||
const name =
|
||||
typeof candidate.name === "string"
|
||||
? candidate.name
|
||||
: typeof candidate.description === "string"
|
||||
? candidate.description
|
||||
: null;
|
||||
if (!code || !name) {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -128,9 +142,19 @@ const buildExchangeNameMap = (entries: AlpacaExchangeMetaEntry[]): Map<string, s
|
|||
return map;
|
||||
};
|
||||
|
||||
const OFF_EXCHANGE_HINTS = ["FINRA", "TRF", "ADF", "OTC", "Trade Reporting Facility", "Alternative Display Facility"];
|
||||
const OFF_EXCHANGE_HINTS = [
|
||||
"FINRA",
|
||||
"TRF",
|
||||
"ADF",
|
||||
"OTC",
|
||||
"Trade Reporting Facility",
|
||||
"Alternative Display Facility"
|
||||
];
|
||||
|
||||
export const inferOffExchangeFlag = (exchangeCode: string | undefined, exchangeNameMap: Map<string, string>): boolean => {
|
||||
export const inferOffExchangeFlag = (
|
||||
exchangeCode: string | undefined,
|
||||
exchangeNameMap: Map<string, string>
|
||||
): boolean => {
|
||||
if (!exchangeCode) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -151,7 +175,9 @@ const buildWsUrl = (wsBaseUrl: string, feed: AlpacaEquitiesFeed): string => {
|
|||
return `${parsed.origin}/v2/${feed}`;
|
||||
};
|
||||
|
||||
const fetchExchangeMeta = async (config: AlpacaEquitiesAdapterConfig): Promise<Map<string, string>> => {
|
||||
const fetchExchangeMeta = async (
|
||||
config: AlpacaEquitiesAdapterConfig
|
||||
): Promise<Map<string, string>> => {
|
||||
const url = new URL("/v2/stocks/meta/exchanges", config.restUrl);
|
||||
|
||||
try {
|
||||
|
|
@ -243,7 +269,10 @@ export const createAlpacaEquitiesAdapter = (
|
|||
continue;
|
||||
}
|
||||
|
||||
const message = entry as (AlpacaTradeMessage | AlpacaQuoteMessage | { T?: string; msg?: string });
|
||||
const message = entry as
|
||||
| AlpacaTradeMessage
|
||||
| AlpacaQuoteMessage
|
||||
| { T?: string; msg?: string };
|
||||
const type = message.T;
|
||||
|
||||
if (type === "success") {
|
||||
|
|
|
|||
|
|
@ -89,11 +89,7 @@ const priceForPlacement = (
|
|||
return formatPrice(Math.max(0.01, price));
|
||||
};
|
||||
|
||||
const buildQuoteContext = (
|
||||
symbol: string,
|
||||
now: number,
|
||||
control: SyntheticControlState
|
||||
) => {
|
||||
const buildQuoteContext = (symbol: string, now: number, control: SyntheticControlState) => {
|
||||
const session = getSyntheticSessionState(now, control);
|
||||
const state = getSyntheticUnderlyingState(symbol, now, control, session);
|
||||
return {
|
||||
|
|
@ -184,7 +180,9 @@ export const createSyntheticEquitiesAdapter = (
|
|||
session.regime === "retail_chase";
|
||||
|
||||
if (allowDark) {
|
||||
const darkSymbol = focusSymbols[seq % focusSymbols.length] ?? SYNTHETIC_SYMBOLS[symbolCursor % SYNTHETIC_SYMBOLS.length]!;
|
||||
const darkSymbol =
|
||||
focusSymbols[seq % focusSymbols.length] ??
|
||||
SYNTHETIC_SYMBOLS[symbolCursor % SYNTHETIC_SYMBOLS.length]!;
|
||||
const darkQuote = buildQuoteContext(darkSymbol, now, control);
|
||||
const darkPlacement = pickDarkPlacement(
|
||||
darkQuote.state.driftBps,
|
||||
|
|
@ -203,13 +201,7 @@ export const createSyntheticEquitiesAdapter = (
|
|||
if (handlers.onQuote) {
|
||||
quoteSeq += 1;
|
||||
void handlers.onQuote(
|
||||
buildSyntheticQuote(
|
||||
quoteSeq,
|
||||
now - 2,
|
||||
darkSymbol,
|
||||
darkQuote.bid,
|
||||
darkQuote.ask
|
||||
)
|
||||
buildSyntheticQuote(quoteSeq, now - 2, darkSymbol, darkQuote.bid, darkQuote.ask)
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -236,11 +228,7 @@ export const createSyntheticEquitiesAdapter = (
|
|||
const eventTs = now + i * 4;
|
||||
const quote = buildQuoteContext(symbol, eventTs, control);
|
||||
const clustered = focusSet.has(symbol);
|
||||
const placement = pickPrimaryPlacement(
|
||||
quote.state.driftBps,
|
||||
session.regime,
|
||||
seq + i
|
||||
);
|
||||
const placement = pickPrimaryPlacement(quote.state.driftBps, session.regime, seq + i);
|
||||
const exchange = EXCHANGES[(seq + symbol.charCodeAt(0) + i) % EXCHANGES.length]!;
|
||||
const baseSize =
|
||||
throughput.litSizeBase +
|
||||
|
|
@ -255,13 +243,7 @@ export const createSyntheticEquitiesAdapter = (
|
|||
if (handlers.onQuote) {
|
||||
quoteSeq += 1;
|
||||
void handlers.onQuote(
|
||||
buildSyntheticQuote(
|
||||
quoteSeq,
|
||||
eventTs - 2,
|
||||
symbol,
|
||||
quote.bid,
|
||||
quote.ask
|
||||
)
|
||||
buildSyntheticQuote(quoteSeq, eventTs - 2, symbol, quote.bid, quote.ask)
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -240,10 +240,7 @@ const run = async () => {
|
|||
await ensureEquityQuotesTable(clickhouse);
|
||||
});
|
||||
|
||||
const adapter = selectAdapter(
|
||||
env.EQUITIES_INGEST_ADAPTER,
|
||||
() => syntheticControl
|
||||
);
|
||||
const adapter = selectAdapter(env.EQUITIES_INGEST_ADAPTER, () => syntheticControl);
|
||||
logger.info("ingest adapter selected", { adapter: adapter.name });
|
||||
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
|
||||
const allowQuotePublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
|
||||
|
|
|
|||
|
|
@ -126,9 +126,13 @@ const decodePayload = (data: WebSocket.RawData): unknown => {
|
|||
return JSON.parse(new TextDecoder().decode(new Uint8Array(data))) as unknown;
|
||||
}
|
||||
if (ArrayBuffer.isView(data)) {
|
||||
return JSON.parse(new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))) as unknown;
|
||||
return JSON.parse(
|
||||
new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))
|
||||
) as unknown;
|
||||
}
|
||||
return JSON.parse(new TextDecoder().decode(new Uint8Array(data as unknown as ArrayBuffer))) as unknown;
|
||||
return JSON.parse(
|
||||
new TextDecoder().decode(new Uint8Array(data as unknown as ArrayBuffer))
|
||||
) as unknown;
|
||||
};
|
||||
|
||||
const run = async () => {
|
||||
|
|
|
|||
|
|
@ -152,10 +152,7 @@ const normalizeUnderlyings = (value: string[]): string[] => {
|
|||
return result;
|
||||
};
|
||||
|
||||
const fetchJson = async <T>(
|
||||
url: URL,
|
||||
config: AlpacaOptionsAdapterConfig
|
||||
): Promise<T> => {
|
||||
const fetchJson = async <T>(url: URL, config: AlpacaOptionsAdapterConfig): Promise<T> => {
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: buildAlpacaAuthHeaders(config.credentials)
|
||||
});
|
||||
|
|
@ -235,10 +232,7 @@ const fetchOptionSnapshots = async (
|
|||
return contracts;
|
||||
};
|
||||
|
||||
const selectExpiries = (
|
||||
contracts: OptionContract[],
|
||||
maxDteDays: number
|
||||
): ExpiryInfo[] => {
|
||||
const selectExpiries = (contracts: OptionContract[], maxDteDays: number): ExpiryInfo[] => {
|
||||
const today = new Date();
|
||||
const expiryMap = new Map<string, ExpiryInfo>();
|
||||
|
||||
|
|
@ -332,7 +326,9 @@ const selectContractsForUnderlying = (
|
|||
const minStrike = price * (1 - config.moneynessPct);
|
||||
const maxStrike = price * (1 + config.moneynessPct);
|
||||
const strikePairs = Array.from(strikeMap.entries())
|
||||
.filter(([strike, pair]) => pair.call && pair.put && strike >= minStrike && strike <= maxStrike)
|
||||
.filter(
|
||||
([strike, pair]) => pair.call && pair.put && strike >= minStrike && strike <= maxStrike
|
||||
)
|
||||
.map(([strike, pair]) => ({
|
||||
strike,
|
||||
call: pair.call as string,
|
||||
|
|
@ -540,7 +536,10 @@ export const createAlpacaOptionsAdapter = (
|
|||
continue;
|
||||
}
|
||||
|
||||
const message = entry as AlpacaTradeMessage | AlpacaQuoteMessage | { T?: string; msg?: string };
|
||||
const message = entry as
|
||||
| AlpacaTradeMessage
|
||||
| AlpacaQuoteMessage
|
||||
| { T?: string; msg?: string };
|
||||
const type = message.T;
|
||||
|
||||
if (type === "t") {
|
||||
|
|
|
|||
|
|
@ -235,8 +235,7 @@ export const createDatabentoOptionsAdapter = (
|
|||
return;
|
||||
}
|
||||
|
||||
const scaledPrice =
|
||||
config.priceScale === 1 ? price : price / config.priceScale;
|
||||
const scaledPrice = config.priceScale === 1 ? price : price / config.priceScale;
|
||||
|
||||
const conditions = Array.isArray(payload.conditions)
|
||||
? payload.conditions.map((entry) => String(entry))
|
||||
|
|
|
|||
|
|
@ -59,9 +59,7 @@ const readLines = async (
|
|||
}
|
||||
};
|
||||
|
||||
export const createIbkrOptionsAdapter = (
|
||||
config: IbkrOptionsAdapterConfig
|
||||
): OptionIngestAdapter => {
|
||||
export const createIbkrOptionsAdapter = (config: IbkrOptionsAdapterConfig): OptionIngestAdapter => {
|
||||
return {
|
||||
name: "ibkr",
|
||||
start: (handlers: OptionIngestHandlers) => {
|
||||
|
|
|
|||
|
|
@ -715,10 +715,7 @@ const SYNTHETIC_PROFILES: Record<SyntheticMarketMode, SyntheticOptionsProfile> =
|
|||
...scenario,
|
||||
countRange: [scenario.countRange[0], scenario.countRange[1]],
|
||||
sizeRange: [scenario.sizeRange[0], scenario.sizeRange[1]],
|
||||
targetNotionalRange: [
|
||||
scenario.targetNotionalRange[0],
|
||||
scenario.targetNotionalRange[1]
|
||||
]
|
||||
targetNotionalRange: [scenario.targetNotionalRange[0], scenario.targetNotionalRange[1]]
|
||||
})),
|
||||
pricePlacements: PLACEMENTS
|
||||
},
|
||||
|
|
@ -743,10 +740,7 @@ const SYNTHETIC_PROFILES: Record<SyntheticMarketMode, SyntheticOptionsProfile> =
|
|||
scenarios: SCENARIO_LIBRARY.map((scenario) => ({
|
||||
...scenario,
|
||||
countRange: [scenario.countRange[0] + 2, scenario.countRange[1] + 4],
|
||||
sizeRange: [
|
||||
Math.round(scenario.sizeRange[0] * 1.8),
|
||||
Math.round(scenario.sizeRange[1] * 2.1)
|
||||
],
|
||||
sizeRange: [Math.round(scenario.sizeRange[0] * 1.8), Math.round(scenario.sizeRange[1] * 2.1)],
|
||||
targetNotionalRange: [
|
||||
Math.round(scenario.targetNotionalRange[0] * 1.7),
|
||||
Math.round(scenario.targetNotionalRange[1] * 2.0)
|
||||
|
|
@ -768,7 +762,7 @@ const SMART_MONEY_TEMPLATE_SCENARIOS: Record<
|
|||
hedge_reactive: "reactive_put_wall"
|
||||
};
|
||||
|
||||
const pick = <T,>(items: readonly T[], seed: number): T => {
|
||||
const pick = <T>(items: readonly T[], seed: number): T => {
|
||||
return items[Math.abs(seed) % items.length]!;
|
||||
};
|
||||
|
||||
|
|
@ -850,9 +844,7 @@ export const updateSyntheticIvForTest = (
|
|||
const sizeImpact = Math.log10(Math.max(10, input.size)) * 0.012;
|
||||
const notionalImpact = Math.log10(Math.max(1_000, input.notional)) * 0.01;
|
||||
pressure +=
|
||||
input.placement === "AA"
|
||||
? sizeImpact + notionalImpact
|
||||
: (sizeImpact + notionalImpact) * 0.65;
|
||||
input.placement === "AA" ? sizeImpact + notionalImpact : (sizeImpact + notionalImpact) * 0.65;
|
||||
} else if (input.placement === "MID") {
|
||||
pressure += 0.001;
|
||||
} else {
|
||||
|
|
@ -879,8 +871,7 @@ const estimateSyntheticOptionMid = (input: {
|
|||
: Math.max(0, input.strike - input.underlying);
|
||||
const timeYears = Math.max(1, input.dteDays + 1) / 365;
|
||||
const baselineIv = initializeSyntheticIv(input.dteDays, input.moneyness);
|
||||
const modeBoost =
|
||||
input.mode === "firehose" ? 1.18 : input.mode === "active" ? 1.08 : 0.96;
|
||||
const modeBoost = input.mode === "firehose" ? 1.18 : input.mode === "active" ? 1.08 : 0.96;
|
||||
const distance = Math.abs(input.moneyness - 1);
|
||||
const extrinsic =
|
||||
input.underlying *
|
||||
|
|
@ -939,12 +930,7 @@ const chooseScenario = (
|
|||
): Scenario => {
|
||||
const session = getSyntheticSessionState(now, control);
|
||||
const focusSymbol = session.focus_symbols[0] ?? SYNTHETIC_SYMBOLS[0]!;
|
||||
const familyWeights = getSyntheticScenarioWeights(
|
||||
focusSymbol,
|
||||
now,
|
||||
control,
|
||||
session
|
||||
);
|
||||
const familyWeights = getSyntheticScenarioWeights(focusSymbol, now, control, session);
|
||||
const coverageCounts = getCoverageCounts(coverageState, now, control);
|
||||
const weightedScenarios = profile.scenarios.map((scenario, index) => {
|
||||
const familyWeight = familyWeights[scenario.label];
|
||||
|
|
@ -964,7 +950,10 @@ const chooseScenario = (
|
|||
: 1;
|
||||
return {
|
||||
...scenario,
|
||||
weight: Math.max(1, Math.round(scenario.weight * familyWeight * coverageBoost * quietBias * 100))
|
||||
weight: Math.max(
|
||||
1,
|
||||
Math.round(scenario.weight * familyWeight * coverageBoost * quietBias * 100)
|
||||
)
|
||||
};
|
||||
});
|
||||
return pickWeighted(weightedScenarios, now + control.shared_seed * 31);
|
||||
|
|
@ -977,7 +966,8 @@ const pickScenarioSymbol = (
|
|||
): string => {
|
||||
const session = getSyntheticSessionState(now, control);
|
||||
const symbolPool =
|
||||
scenario.preferredSymbols?.length && (scenario.label === "event_driven" || Math.abs(now) % 4 === 0)
|
||||
scenario.preferredSymbols?.length &&
|
||||
(scenario.label === "event_driven" || Math.abs(now) % 4 === 0)
|
||||
? [...scenario.preferredSymbols]
|
||||
: session.focus_symbols.length > 0
|
||||
? [...session.focus_symbols, ...SYNTHETIC_SYMBOLS]
|
||||
|
|
@ -1033,11 +1023,12 @@ const buildDynamicFlowFeatures = (
|
|||
0,
|
||||
0.26
|
||||
),
|
||||
underlying_move_bps: Math.round(
|
||||
(Number(scenario.flowFeatures.underlying_move_bps ?? underlying.driftBps) +
|
||||
underlying.shockBps * 0.35) *
|
||||
100
|
||||
) / 100,
|
||||
underlying_move_bps:
|
||||
Math.round(
|
||||
(Number(scenario.flowFeatures.underlying_move_bps ?? underlying.driftBps) +
|
||||
underlying.shockBps * 0.35) *
|
||||
100
|
||||
) / 100,
|
||||
venue_count: Math.max(
|
||||
1,
|
||||
Math.round(
|
||||
|
|
@ -1059,18 +1050,14 @@ const buildBurst = (
|
|||
coverageState: CoverageWindowState,
|
||||
scenarioOverride?: Scenario
|
||||
): Burst => {
|
||||
const scenario =
|
||||
scenarioOverride ?? chooseScenario(profile, now, control, coverageState);
|
||||
const scenario = scenarioOverride ?? chooseScenario(profile, now, control, coverageState);
|
||||
const symbol = pickScenarioSymbol(scenario, now, control);
|
||||
const symbolHash = hashSyntheticSymbol(symbol);
|
||||
const seed = symbolHash + burstIndex * 7;
|
||||
const session = getSyntheticSessionState(now, control);
|
||||
const underlyingState = getSyntheticUnderlyingState(symbol, now, control, session);
|
||||
const baseUnderlying = underlyingState.mid;
|
||||
const expiryOffset = pick(
|
||||
scenario.expiryOffsets ?? EXPIRY_OFFSETS,
|
||||
symbolHash + burstIndex
|
||||
);
|
||||
const expiryOffset = pick(scenario.expiryOffsets ?? EXPIRY_OFFSETS, symbolHash + burstIndex);
|
||||
const strikeStep = baseUnderlying >= 200 ? 10 : baseUnderlying >= 100 ? 5 : 2.5;
|
||||
const right =
|
||||
scenario.right === "either"
|
||||
|
|
@ -1099,16 +1086,15 @@ const buildBurst = (
|
|||
const priceStep =
|
||||
scenario.priceTrend === "up" ? 0.01 : scenario.priceTrend === "down" ? -0.01 : 0;
|
||||
const flowFeatures = buildDynamicFlowFeatures(scenario, symbol, now, control);
|
||||
const legTemplates =
|
||||
scenario.legs?.length
|
||||
? scenario.legs
|
||||
: [
|
||||
{
|
||||
right,
|
||||
strikeMoneyness: scenario.strikeMoneyness,
|
||||
placementScenarioId: scenario.placementProfile ?? scenario.label
|
||||
}
|
||||
];
|
||||
const legTemplates = scenario.legs?.length
|
||||
? scenario.legs
|
||||
: [
|
||||
{
|
||||
right,
|
||||
strikeMoneyness: scenario.strikeMoneyness,
|
||||
placementScenarioId: scenario.placementProfile ?? scenario.label
|
||||
}
|
||||
];
|
||||
const targetNotionalPerLeg = targetNotional / legTemplates.length;
|
||||
|
||||
const legs = legTemplates.map((template, legIndex): BurstLeg => {
|
||||
|
|
@ -1127,8 +1113,7 @@ const buildBurst = (
|
|||
const strike = Math.max(
|
||||
1,
|
||||
templateStrike ??
|
||||
Math.round(baseUnderlying / strikeStep) * strikeStep +
|
||||
strikeOffset * strikeStep
|
||||
Math.round(baseUnderlying / strikeStep) * strikeStep + strikeOffset * strikeStep
|
||||
);
|
||||
const legSize = Math.max(1, Math.round(baseSize * (template.sizeMultiplier ?? 1)));
|
||||
const legMoneyness = strike / baseUnderlying;
|
||||
|
|
@ -1141,13 +1126,13 @@ const buildBurst = (
|
|||
mode
|
||||
});
|
||||
const targetMid =
|
||||
targetNotionalPerLeg /
|
||||
Math.max(1, legSize * cycles * OPTION_CONTRACT_MULTIPLIER);
|
||||
targetNotionalPerLeg / Math.max(1, legSize * cycles * OPTION_CONTRACT_MULTIPLIER);
|
||||
const cappedTheoreticalMid = Math.min(
|
||||
theoreticalMid,
|
||||
Math.max(0.35, targetMid * (scenario.label === "institutional_directional" ? 2.2 : 2.6))
|
||||
);
|
||||
const blendedMid = cappedTheoreticalMid * 0.45 + targetMid * 0.55 * (template.priceMultiplier ?? 1);
|
||||
const blendedMid =
|
||||
cappedTheoreticalMid * 0.45 + targetMid * 0.55 * (template.priceMultiplier ?? 1);
|
||||
return {
|
||||
contractId: `${symbol}-${expiry}-${formatStrike(strike)}-${template.right}`,
|
||||
right: template.right,
|
||||
|
|
@ -1184,8 +1169,7 @@ const buildBurst = (
|
|||
scenario.missingQuoteProbability ??
|
||||
clampValue((1 - session.quote_cleanliness) * 0.16, 0, 0.18),
|
||||
staleQuoteProbability:
|
||||
scenario.staleQuoteProbability ??
|
||||
clampValue((1 - session.quote_cleanliness) * 0.3, 0, 0.42)
|
||||
scenario.staleQuoteProbability ?? clampValue((1 - session.quote_cleanliness) * 0.3, 0, 0.42)
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -1202,7 +1186,9 @@ export const listSyntheticSmartMoneyScenariosForTest = (): SyntheticSmartMoneySc
|
|||
hiddenLabel:
|
||||
id === "neutral_noise"
|
||||
? "single_print_mid"
|
||||
: SMART_MONEY_TEMPLATE_SCENARIOS[id as Exclude<(typeof SMART_MONEY_SCENARIO_IDS)[number], "neutral_noise">]
|
||||
: SMART_MONEY_TEMPLATE_SCENARIOS[
|
||||
id as Exclude<(typeof SMART_MONEY_SCENARIO_IDS)[number], "neutral_noise">
|
||||
]
|
||||
}));
|
||||
|
||||
export const buildSyntheticSmartMoneyBurstForTest = (
|
||||
|
|
@ -1233,18 +1219,18 @@ export const buildSyntheticSmartMoneyBurstForTest = (
|
|||
updated_by: "system"
|
||||
} satisfies SyntheticControlState;
|
||||
const mode: SyntheticMarketMode =
|
||||
scenarioId === "retail_whale" || scenarioId === "neutral_noise"
|
||||
? "realistic"
|
||||
: "active";
|
||||
scenarioId === "retail_whale" || scenarioId === "neutral_noise" ? "realistic" : "active";
|
||||
const profile = SYNTHETIC_PROFILES[mode];
|
||||
const coverageState = createCoverageWindowState();
|
||||
const scenario =
|
||||
scenarioId === "neutral_noise"
|
||||
? profile.scenarios.find((candidate) => candidate.id === "single_print_mid")!
|
||||
: profile.scenarios.find(
|
||||
(candidate) => candidate.id === SMART_MONEY_TEMPLATE_SCENARIOS[
|
||||
scenarioId as Exclude<(typeof SMART_MONEY_SCENARIO_IDS)[number], "neutral_noise">
|
||||
]
|
||||
(candidate) =>
|
||||
candidate.id ===
|
||||
SMART_MONEY_TEMPLATE_SCENARIOS[
|
||||
scenarioId as Exclude<(typeof SMART_MONEY_SCENARIO_IDS)[number], "neutral_noise">
|
||||
]
|
||||
)!;
|
||||
return buildBurst(1, now, mode, profile, control, coverageState, scenario);
|
||||
};
|
||||
|
|
@ -1255,13 +1241,10 @@ export const buildSyntheticFlowPacketForTest = (
|
|||
): { packet: FlowPacket; hiddenLabel: string } => {
|
||||
const burst = buildSyntheticSmartMoneyBurstForTest(scenarioId, now);
|
||||
const primaryLeg = burst.legs[0]!;
|
||||
const corporateEventOffset = Number(
|
||||
burst.flowFeatures.corporate_event_ts_offset_days ?? 0
|
||||
);
|
||||
const corporateEventOffset = Number(burst.flowFeatures.corporate_event_ts_offset_days ?? 0);
|
||||
const totalSize = burst.legs.reduce((sum, leg) => sum + leg.baseSize * burst.cycles, 0);
|
||||
const totalPremium = burst.legs.reduce(
|
||||
(sum, leg) =>
|
||||
sum + leg.basePrice * leg.baseSize * burst.cycles * OPTION_CONTRACT_MULTIPLIER,
|
||||
(sum, leg) => sum + leg.basePrice * leg.baseSize * burst.cycles * OPTION_CONTRACT_MULTIPLIER,
|
||||
0
|
||||
);
|
||||
const flowFeatures: FlowPacket["features"] = {
|
||||
|
|
@ -1272,15 +1255,10 @@ export const buildSyntheticFlowPacketForTest = (
|
|||
window_ms: Math.max(0, (burst.printCount - 1) * 45),
|
||||
total_size: totalSize,
|
||||
total_premium: Number(totalPremium.toFixed(2)),
|
||||
total_notional: Number(
|
||||
(burst.underlying * totalSize * OPTION_CONTRACT_MULTIPLIER).toFixed(2)
|
||||
),
|
||||
total_notional: Number((burst.underlying * totalSize * OPTION_CONTRACT_MULTIPLIER).toFixed(2)),
|
||||
first_price: primaryLeg.basePrice,
|
||||
last_price: Number(
|
||||
(
|
||||
primaryLeg.basePrice *
|
||||
(1 + burst.priceStep * Math.max(0, burst.cycles - 1))
|
||||
).toFixed(2)
|
||||
(primaryLeg.basePrice * (1 + burst.priceStep * Math.max(0, burst.cycles - 1))).toFixed(2)
|
||||
),
|
||||
nbbo_missing_count: 0,
|
||||
nbbo_stale_count: 0,
|
||||
|
|
@ -1300,10 +1278,7 @@ export const buildSyntheticFlowPacketForTest = (
|
|||
Number(flowFeatures.total_premium ?? totalPremium),
|
||||
72_000
|
||||
);
|
||||
flowFeatures.execution_iv_shock = Math.max(
|
||||
Number(flowFeatures.execution_iv_shock ?? 0),
|
||||
0.22
|
||||
);
|
||||
flowFeatures.execution_iv_shock = Math.max(Number(flowFeatures.execution_iv_shock ?? 0), 0.22);
|
||||
}
|
||||
if (scenarioId === "event_driven") {
|
||||
flowFeatures.count = 2;
|
||||
|
|
@ -1411,14 +1386,7 @@ export const buildSyntheticBurstForTest = (
|
|||
return cached[burstIndex - 1]!;
|
||||
}
|
||||
for (let index = cached.length + 1; index <= burstIndex; index += 1) {
|
||||
const current = buildBurst(
|
||||
index,
|
||||
now + index * 1_000,
|
||||
mode,
|
||||
profile,
|
||||
control,
|
||||
coverageState
|
||||
);
|
||||
const current = buildBurst(index, now + index * 1_000, mode, profile, control, coverageState);
|
||||
recordCoverageHit(coverageState, current.label, now + index * 1_000);
|
||||
cached.push(current);
|
||||
}
|
||||
|
|
@ -1466,14 +1434,7 @@ export const createSyntheticOptionsAdapter = (
|
|||
};
|
||||
if (!currentBurst || remainingRuns <= 0) {
|
||||
burstIndex += 1;
|
||||
currentBurst = buildBurst(
|
||||
burstIndex,
|
||||
now,
|
||||
config.mode,
|
||||
profile,
|
||||
control,
|
||||
coverageState
|
||||
);
|
||||
currentBurst = buildBurst(burstIndex, now, config.mode, profile, control, coverageState);
|
||||
recordCoverageHit(coverageState, currentBurst.label, now);
|
||||
remainingRuns = pickInt(
|
||||
profile.burstRunRange[0],
|
||||
|
|
@ -1565,8 +1526,7 @@ export const createSyntheticOptionsAdapter = (
|
|||
const quoteSeed = Math.abs(burst.seed + i * 17) % 1000;
|
||||
const missingQuote = quoteSeed / 1000 < burst.missingQuoteProbability;
|
||||
const staleQuote =
|
||||
!missingQuote &&
|
||||
((quoteSeed + 233) % 1000) / 1000 < burst.staleQuoteProbability;
|
||||
!missingQuote && ((quoteSeed + 233) % 1000) / 1000 < burst.staleQuoteProbability;
|
||||
|
||||
if (handlers.onNBBO && !missingQuote) {
|
||||
nbboSeq += 1;
|
||||
|
|
|
|||
|
|
@ -48,7 +48,11 @@ export const selectAtOrBefore = <T extends { ts: number; seq: number }>(
|
|||
if (item.ts > ts) {
|
||||
continue;
|
||||
}
|
||||
if (!selected || item.ts > selected.ts || (item.ts === selected.ts && item.seq >= selected.seq)) {
|
||||
if (
|
||||
!selected ||
|
||||
item.ts > selected.ts ||
|
||||
(item.ts === selected.ts && item.seq >= selected.seq)
|
||||
) {
|
||||
selected = item;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,12 @@ import { createDatabentoOptionsAdapter } from "./adapters/databento";
|
|||
import { createIbkrOptionsAdapter } from "./adapters/ibkr";
|
||||
import { createSyntheticOptionsAdapter } from "./adapters/synthetic";
|
||||
import type { OptionIngestAdapter, StopHandler } from "./adapters/types";
|
||||
import { enrichOptionPrint, rememberContext, selectAtOrBefore, type ContextHistory } from "./enrichment";
|
||||
import {
|
||||
enrichOptionPrint,
|
||||
rememberContext,
|
||||
selectAtOrBefore,
|
||||
type ContextHistory
|
||||
} from "./enrichment";
|
||||
import { z } from "zod";
|
||||
|
||||
const service = "ingest-options";
|
||||
|
|
@ -87,7 +92,10 @@ const envSchema = z.object({
|
|||
IBKR_EXPIRY: z.string().min(1).default("20250117"),
|
||||
IBKR_STRIKE: z.coerce.number().positive().default(450),
|
||||
IBKR_RIGHT: z
|
||||
.preprocess((value) => (typeof value === "string" ? value.toUpperCase() : value), z.enum(["C", "P"]))
|
||||
.preprocess(
|
||||
(value) => (typeof value === "string" ? value.toUpperCase() : value),
|
||||
z.enum(["C", "P"])
|
||||
)
|
||||
.default("C"),
|
||||
IBKR_EXCHANGE: z.string().min(1).default("SMART"),
|
||||
IBKR_CURRENCY: z.string().min(1).default("USD"),
|
||||
|
|
@ -395,10 +403,7 @@ const run = async () => {
|
|||
await ensureOptionNBBOTable(clickhouse);
|
||||
});
|
||||
|
||||
const adapter = selectAdapter(
|
||||
env.OPTIONS_INGEST_ADAPTER,
|
||||
() => syntheticControl
|
||||
);
|
||||
const adapter = selectAdapter(env.OPTIONS_INGEST_ADAPTER, () => syntheticControl);
|
||||
logger.info("ingest adapter selected", { adapter: adapter.name });
|
||||
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
|
||||
const allowNbboPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
|
||||
|
|
@ -421,7 +426,10 @@ const run = async () => {
|
|||
rawPrint.ts
|
||||
);
|
||||
const equityQuote = parsedMetadata.underlying_id
|
||||
? selectAtOrBefore(equityQuoteHistoryByUnderlying.get(parsedMetadata.underlying_id), rawPrint.ts)
|
||||
? selectAtOrBefore(
|
||||
equityQuoteHistoryByUnderlying.get(parsedMetadata.underlying_id),
|
||||
rawPrint.ts
|
||||
)
|
||||
: null;
|
||||
const print = enrichOptionPrint(rawPrint, optionQuote, equityQuote, optionsSignalConfig);
|
||||
|
||||
|
|
@ -500,8 +508,16 @@ const run = async () => {
|
|||
|
||||
const pruneTimer = setInterval(() => {
|
||||
const removed =
|
||||
pruneContextHistory(nbboHistoryByContract, env.OPTION_CONTEXT_MAX_KEYS, env.OPTION_CONTEXT_TTL_MS) +
|
||||
pruneContextHistory(equityQuoteHistoryByUnderlying, env.OPTION_CONTEXT_MAX_KEYS, env.OPTION_CONTEXT_TTL_MS);
|
||||
pruneContextHistory(
|
||||
nbboHistoryByContract,
|
||||
env.OPTION_CONTEXT_MAX_KEYS,
|
||||
env.OPTION_CONTEXT_TTL_MS
|
||||
) +
|
||||
pruneContextHistory(
|
||||
equityQuoteHistoryByUnderlying,
|
||||
env.OPTION_CONTEXT_MAX_KEYS,
|
||||
env.OPTION_CONTEXT_TTL_MS
|
||||
);
|
||||
logger.info("option context cache summary", {
|
||||
nbbo_context_keys: nbboHistoryByContract.size,
|
||||
equity_quote_context_keys: equityQuoteHistoryByUnderlying.size,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
import { mkdir } from "node:fs/promises";
|
||||
|
||||
export type EventCalendarKind = "earnings" | "dividend" | "corporate_action" | "m_and_a" | "news" | "other";
|
||||
export type EventCalendarKind =
|
||||
| "earnings"
|
||||
| "dividend"
|
||||
| "corporate_action"
|
||||
| "m_and_a"
|
||||
| "news"
|
||||
| "other";
|
||||
|
||||
export type EventCalendarEntry = {
|
||||
underlying_id: string;
|
||||
|
|
@ -56,7 +62,8 @@ const asNumber = (value: unknown): number | null => {
|
|||
return null;
|
||||
};
|
||||
|
||||
const asString = (value: unknown): string | null => (typeof value === "string" && value.trim() ? value.trim() : null);
|
||||
const asString = (value: unknown): string | null =>
|
||||
typeof value === "string" && value.trim() ? value.trim() : null;
|
||||
|
||||
const parseCsvLine = (line: string): string[] => {
|
||||
const values: string[] = [];
|
||||
|
|
@ -139,9 +146,14 @@ export const parseEventCalendarEntries = (value: unknown): EventCalendarEntry[]
|
|||
const record = row as Record<string, unknown>;
|
||||
const underlying = asString(record.underlying_id ?? record.underlying ?? record.symbol);
|
||||
const eventTs = asNumber(record.event_ts ?? record.event_time ?? record.event_date);
|
||||
const announcedTs = asNumber(record.announced_ts ?? record.available_ts ?? record.as_of_ts ?? record.created_ts) ?? 0;
|
||||
const announcedTs =
|
||||
asNumber(
|
||||
record.announced_ts ?? record.available_ts ?? record.as_of_ts ?? record.created_ts
|
||||
) ?? 0;
|
||||
const rawKind = asString(record.event_kind ?? record.kind ?? record.type) ?? "other";
|
||||
const eventKind = EVENT_KINDS.has(rawKind as EventCalendarKind) ? (rawKind as EventCalendarKind) : "other";
|
||||
const eventKind = EVENT_KINDS.has(rawKind as EventCalendarKind)
|
||||
? (rawKind as EventCalendarKind)
|
||||
: "other";
|
||||
|
||||
if (!underlying || eventTs === null || eventTs < 0 || announcedTs < 0) {
|
||||
return [];
|
||||
|
|
@ -162,7 +174,9 @@ export const parseEventCalendarEntries = (value: unknown): EventCalendarEntry[]
|
|||
});
|
||||
};
|
||||
|
||||
export const createStaticEventCalendarProvider = (entries: EventCalendarEntry[]): EventCalendarProvider => {
|
||||
export const createStaticEventCalendarProvider = (
|
||||
entries: EventCalendarEntry[]
|
||||
): EventCalendarProvider => {
|
||||
const byUnderlying = new Map<string, EventCalendarEntry[]>();
|
||||
for (const entry of entries) {
|
||||
const key = normalizeUnderlying(entry.underlying_id);
|
||||
|
|
@ -184,15 +198,20 @@ export const createStaticEventCalendarProvider = (entries: EventCalendarEntry[])
|
|||
}
|
||||
|
||||
const bucket = byUnderlying.get(key) ?? [];
|
||||
const entry = bucket.find((candidate) => candidate.announced_ts <= asOfTs && candidate.event_ts >= asOfTs);
|
||||
const entry = bucket.find(
|
||||
(candidate) => candidate.announced_ts <= asOfTs && candidate.event_ts >= asOfTs
|
||||
);
|
||||
return entry ? { ...entry, days_to_event: (entry.event_ts - asOfTs) / MS_PER_DAY } : null;
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
export const createEmptyEventCalendarProvider = (): EventCalendarProvider => createStaticEventCalendarProvider([]);
|
||||
export const createEmptyEventCalendarProvider = (): EventCalendarProvider =>
|
||||
createStaticEventCalendarProvider([]);
|
||||
|
||||
export const loadEventCalendarProviderFromFile = async (path: string): Promise<EventCalendarProvider> => {
|
||||
export const loadEventCalendarProviderFromFile = async (
|
||||
path: string
|
||||
): Promise<EventCalendarProvider> => {
|
||||
const text = await Bun.file(path).text();
|
||||
return createStaticEventCalendarProvider(parseEventCalendarEntries(JSON.parse(text)));
|
||||
};
|
||||
|
|
@ -212,7 +231,9 @@ export const fetchAlphaVantageEarningsCalendar = async (
|
|||
const response = await (options.fetchFn ?? fetch)(url);
|
||||
const text = await response.text();
|
||||
if (!response.ok) {
|
||||
throw new Error(`Alpha Vantage earnings calendar request failed: ${response.status} ${text.slice(0, 160)}`);
|
||||
throw new Error(
|
||||
`Alpha Vantage earnings calendar request failed: ${response.status} ${text.slice(0, 160)}`
|
||||
);
|
||||
}
|
||||
if (/^(?:\s*\{|\s*Thank you for using Alpha Vantage)/i.test(text)) {
|
||||
throw new Error(`Alpha Vantage returned a non-calendar response: ${text.slice(0, 200)}`);
|
||||
|
|
@ -221,7 +242,10 @@ export const fetchAlphaVantageEarningsCalendar = async (
|
|||
return parseAlphaVantageEarningsCalendar(text, options.nowTs ?? Date.now());
|
||||
};
|
||||
|
||||
export const writeEventCalendarEntries = async (path: string, entries: EventCalendarEntry[]): Promise<void> => {
|
||||
export const writeEventCalendarEntries = async (
|
||||
path: string,
|
||||
entries: EventCalendarEntry[]
|
||||
): Promise<void> => {
|
||||
const directory = path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : "";
|
||||
if (directory) {
|
||||
await mkdir(directory, { recursive: true });
|
||||
|
|
|
|||
|
|
@ -12,9 +12,14 @@ const logger = createLogger({ service });
|
|||
|
||||
logger.info("service starting");
|
||||
|
||||
const eventCalendarPath = process.env.REFDATA_EVENT_CALENDAR_PATH ?? process.env.SMART_MONEY_EVENT_CALENDAR_PATH;
|
||||
const eventCalendarProvider = process.env.REFDATA_EVENT_CALENDAR_PROVIDER ?? process.env.EVENT_CALENDAR_PROVIDER;
|
||||
const refreshMs = Math.max(0, Number(process.env.REFDATA_EVENT_CALENDAR_REFRESH_MS ?? 86_400_000) || 0);
|
||||
const eventCalendarPath =
|
||||
process.env.REFDATA_EVENT_CALENDAR_PATH ?? process.env.SMART_MONEY_EVENT_CALENDAR_PATH;
|
||||
const eventCalendarProvider =
|
||||
process.env.REFDATA_EVENT_CALENDAR_PROVIDER ?? process.env.EVENT_CALENDAR_PROVIDER;
|
||||
const refreshMs = Math.max(
|
||||
0,
|
||||
Number(process.env.REFDATA_EVENT_CALENDAR_REFRESH_MS ?? 86_400_000) || 0
|
||||
);
|
||||
|
||||
const getAlphaVantageOptions = (): AlphaVantageEarningsCalendarOptions | null => {
|
||||
const apiKey = process.env.ALPHA_VANTAGE_API_KEY;
|
||||
|
|
@ -33,7 +38,9 @@ const getAlphaVantageOptions = (): AlphaVantageEarningsCalendarOptions | null =>
|
|||
|
||||
const refreshEventCalendar = async (): Promise<void> => {
|
||||
if (!eventCalendarPath) {
|
||||
logger.warn("event calendar refresh disabled; missing SMART_MONEY_EVENT_CALENDAR_PATH or REFDATA_EVENT_CALENDAR_PATH");
|
||||
logger.warn(
|
||||
"event calendar refresh disabled; missing SMART_MONEY_EVENT_CALENDAR_PATH or REFDATA_EVENT_CALENDAR_PATH"
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (eventCalendarProvider !== "alpha_vantage") {
|
||||
|
|
|
|||
|
|
@ -52,11 +52,7 @@ type ReplayStreamKind = "options" | "nbbo" | "equities" | "equity-quotes";
|
|||
|
||||
type ReplayEvent = OptionPrint | OptionNBBO | EquityPrint | EquityQuote;
|
||||
|
||||
type FetchAfter = (
|
||||
afterTs: number,
|
||||
afterSeq: number,
|
||||
limit: number
|
||||
) => Promise<ReplayEvent[]>;
|
||||
type FetchAfter = (afterTs: number, afterSeq: number, limit: number) => Promise<ReplayEvent[]>;
|
||||
|
||||
type ReplayStream = {
|
||||
kind: ReplayStreamKind;
|
||||
|
|
@ -79,7 +75,12 @@ const STREAM_DEFS: Record<
|
|||
subject: string;
|
||||
streamName: string;
|
||||
rank: number;
|
||||
fetchAfter: (client: ReturnType<typeof createClickHouseClient>, afterTs: number, afterSeq: number, limit: number) => Promise<ReplayEvent[]>;
|
||||
fetchAfter: (
|
||||
client: ReturnType<typeof createClickHouseClient>,
|
||||
afterTs: number,
|
||||
afterSeq: number,
|
||||
limit: number
|
||||
) => Promise<ReplayEvent[]>;
|
||||
}
|
||||
> = {
|
||||
options: {
|
||||
|
|
@ -196,7 +197,9 @@ const getEventIngestTs = (event: ReplayEvent): number =>
|
|||
|
||||
const getEventSeq = (event: ReplayEvent): number => (Number.isFinite(event.seq) ? event.seq : 0);
|
||||
|
||||
const pickNextEvent = (streams: ReplayStream[]): { stream: ReplayStream; event: ReplayEvent } | null => {
|
||||
const pickNextEvent = (
|
||||
streams: ReplayStream[]
|
||||
): { stream: ReplayStream; event: ReplayEvent } | null => {
|
||||
let choice: { stream: ReplayStream; event: ReplayEvent } | null = null;
|
||||
|
||||
for (const stream of streams) {
|
||||
|
|
@ -313,7 +316,8 @@ const run = async () => {
|
|||
kind,
|
||||
subject: def.subject,
|
||||
streamName: def.streamName,
|
||||
fetchAfter: (afterTs, afterSeq, limit) => def.fetchAfter(clickhouse, afterTs, afterSeq, limit),
|
||||
fetchAfter: (afterTs, afterSeq, limit) =>
|
||||
def.fetchAfter(clickhouse, afterTs, afterSeq, limit),
|
||||
buffer: [],
|
||||
cursor: { ...startCursor },
|
||||
done: false,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue