dirtydishes/islandflow
1
0
Fork 0
Code Issues 2 Pull requests 4 Projects Releases Packages Wiki Activity Actions 1

Add equity print quote joins

Browse source
This commit is contained in:
dirtydishes 2026-01-04 17:07:43 -05:00
parent 9908c431f0
commit 3164167bee
15 changed files with 1033 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

104
services/compute/src/equity-joins.ts Normal file
View file

@ -0,0 +1,104 @@
import type { EquityPrint, EquityPrintJoin, EquityQuote } from "@islandflow/types";
export type EquityQuoteJoin = {
quote: EquityQuote | null;
ageMs: number;
stale: boolean;
};
export type QuotePlacement = "AA" | "A" | "B" | "BB" | "MID" | "MISSING" | "STALE";
const roundTo = (value: number, digits = 4): number => {
if (!Number.isFinite(value)) {
return 0;
}
return Number(value.toFixed(digits));
};
export const classifyQuotePlacement = (
price: number,
join: EquityQuoteJoin
): QuotePlacement => {
if (!Number.isFinite(price)) {
return "MISSING";
}
if (!join.quote) {
return "MISSING";
}
if (join.stale) {
return "STALE";
}
const bid = join.quote.bid;
const ask = join.quote.ask;
if (!Number.isFinite(bid) || !Number.isFinite(ask) || ask <= 0) {
return "MISSING";
}
const spread = Math.max(0, ask - bid);
const epsilon = Math.max(0.01, spread * 0.05);
if (price > ask + epsilon) {
return "AA";
}
if (price >= ask - epsilon) {
return "A";
}
if (price < bid - epsilon) {
return "BB";
}
if (price <= bid + epsilon) {
return "B";
}
return "MID";
};
export const buildEquityPrintJoin = (
print: EquityPrint,
join: EquityQuoteJoin
): EquityPrintJoin => {
const joinQuality: Record<string, number> = {};
const placement = classifyQuotePlacement(print.price, join);
const features: Record<string, string | number | boolean> = {
underlying_id: print.underlying_id,
price: print.price,
size: print.size,
off_exchange_flag: print.offExchangeFlag,
print_ts: print.ts,
quote_placement: placement
};
if (!join.quote) {
joinQuality.quote_missing = 1;
} else {
joinQuality.quote_age_ms = join.ageMs;
if (join.stale) {
joinQuality.quote_stale = 1;
} else {
const bid = join.quote.bid;
const ask = join.quote.ask;
const mid = (bid + ask) / 2;
const spread = ask - bid;
features.quote_ts = join.quote.ts;
features.quote_bid = bid;
features.quote_ask = ask;
features.quote_mid = roundTo(mid);
features.quote_spread = roundTo(spread);
}
}
const joinId = `equityjoin:${print.trace_id}`;
return {
source_ts: print.source_ts,
ingest_ts: print.ingest_ts,
seq: print.seq,
trace_id: joinId,
id: joinId,
print_trace_id: print.trace_id,
quote_trace_id: join.quote?.trace_id ?? "",
features,
join_quality: joinQuality
};
};

277
services/compute/src/index.ts
View file

@ -3,11 +3,17 @@ import { createLogger } from "@islandflow/observability";
import {
SUBJECT_ALERTS,
SUBJECT_CLASSIFIER_HITS,
SUBJECT_EQUITY_JOINS,
SUBJECT_EQUITY_PRINTS,
SUBJECT_EQUITY_QUOTES,
SUBJECT_FLOW_PACKETS,
SUBJECT_OPTION_NBBO,
SUBJECT_OPTION_PRINTS,
STREAM_ALERTS,
STREAM_CLASSIFIER_HITS,
STREAM_EQUITY_JOINS,
STREAM_EQUITY_PRINTS,
STREAM_EQUITY_QUOTES,
STREAM_FLOW_PACKETS,
STREAM_OPTION_NBBO,
STREAM_OPTION_PRINTS,
@ -21,19 +27,27 @@ import {
createClickHouseClient,
ensureAlertsTable,
ensureClassifierHitsTable,
ensureEquityPrintJoinsTable,
ensureFlowPacketsTable,
insertAlert,
insertClassifierHit,
insertEquityPrintJoin,
insertFlowPacket
} from "@islandflow/storage";
import {
AlertEventSchema,
ClassifierHitEventSchema,
EquityPrintJoinSchema,
EquityPrintSchema,
EquityQuoteSchema,
FlowPacketSchema,
OptionNBBOSchema,
OptionPrintSchema,
type AlertEvent,
type ClassifierHitEvent,
type EquityPrint,
type EquityQuote,
type EquityPrintJoin,
type FlowPacket,
type OptionNBBO,
type OptionPrint
@ -41,6 +55,7 @@ import {
import { z } from "zod";
import { evaluateClassifiers, type ClassifierConfig } from "./classifiers";
import { parseContractId } from "./contracts";
import { buildEquityPrintJoin, type EquityQuoteJoin } from "./equity-joins";
import { createRedisClient, updateRollingStats, type RollingStatsConfig } from "./rolling-stats";
import { summarizeStructure, type ContractLeg } from "./structures";
@ -71,6 +86,7 @@ const envSchema = z.object({
}, z.boolean())
.default(false),
NBBO_MAX_AGE_MS: z.coerce.number().int().positive().default(1000),
EQUITY_QUOTE_MAX_AGE_MS: z.coerce.number().int().positive().default(1000),
CLASSIFIER_SWEEP_MIN_PREMIUM: z.coerce.number().positive().default(40_000),
CLASSIFIER_SWEEP_MIN_COUNT: z.coerce.number().int().positive().default(3),
CLASSIFIER_SWEEP_MIN_PREMIUM_Z: z.coerce.number().nonnegative().default(2),
@ -161,6 +177,7 @@ type ClusterState = {
const clusters = new Map<string, ClusterState>();
const nbboCache = new Map<string, OptionNBBO>();
const equityQuoteCache = new Map<string, EquityQuote>();
const recentLegsByKey = new Map<string, ContractLeg[]>();
const MAX_RECENT_LEGS = 20;
@ -341,6 +358,17 @@ const updateNbboCache = (nbbo: OptionNBBO): void => {
}
};
const updateEquityQuoteCache = (quote: EquityQuote): void => {
const existing = equityQuoteCache.get(quote.underlying_id);
if (
!existing ||
quote.ts > existing.ts ||
(quote.ts === existing.ts && quote.seq >= existing.seq)
) {
equityQuoteCache.set(quote.underlying_id, quote);
}
};
const selectNbbo = (contractId: string, ts: number): NbboJoin => {
const nbbo = nbboCache.get(contractId) ?? null;
if (!nbbo) {
@ -352,6 +380,17 @@ const selectNbbo = (contractId: string, ts: number): NbboJoin => {
return { nbbo, ageMs, stale };
};
const selectEquityQuote = (underlyingId: string, ts: number): EquityQuoteJoin => {
const quote = equityQuoteCache.get(underlyingId) ?? null;
if (!quote) {
return { quote: null, ageMs: env.EQUITY_QUOTE_MAX_AGE_MS + 1, stale: true };
}
const ageMs = Math.abs(ts - quote.ts);
const stale = ageMs > env.EQUITY_QUOTE_MAX_AGE_MS;
return { quote, ageMs, stale };
};
const classifyPlacement = (price: number, join: NbboJoin): NbboPlacement => {
if (!Number.isFinite(price)) {
return "MISSING";
@ -609,6 +648,25 @@ const emitClassifiers = async (
}
};
const emitEquityJoin = async (
clickhouse: ReturnType<typeof createClickHouseClient>,
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
print: EquityPrint
): Promise<void> => {
const join = selectEquityQuote(print.underlying_id, print.ts);
const payload: EquityPrintJoin = EquityPrintJoinSchema.parse(buildEquityPrintJoin(print, join));
try {
await insertEquityPrintJoin(clickhouse, payload);
await publishJson(js, SUBJECT_EQUITY_JOINS, payload);
} catch (error) {
logger.error("failed to emit equity print join", {
error: error instanceof Error ? error.message : String(error),
trace_id: payload.trace_id
});
}
};
const flushEligibleClusters = async (
clickhouse: ReturnType<typeof createClickHouseClient>,
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
@ -666,6 +724,32 @@ const run = async () => {
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_EQUITY_PRINTS,
subjects: [SUBJECT_EQUITY_PRINTS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_EQUITY_QUOTES,
subjects: [SUBJECT_EQUITY_QUOTES],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_FLOW_PACKETS,
subjects: [SUBJECT_FLOW_PACKETS],
@ -679,6 +763,19 @@ const run = async () => {
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_EQUITY_JOINS,
subjects: [SUBJECT_EQUITY_JOINS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_CLASSIFIER_HITS,
subjects: [SUBJECT_CLASSIFIER_HITS],
@ -726,12 +823,15 @@ const run = async () => {
await retry("clickhouse table init", 20, 500, async () => {
await ensureFlowPacketsTable(clickhouse);
await ensureEquityPrintJoinsTable(clickhouse);
await ensureClassifierHitsTable(clickhouse);
await ensureAlertsTable(clickhouse);
});
const durableName = "compute-option-prints";
const nbboDurableName = "compute-option-nbbo";
const equityPrintDurableName = "compute-equity-prints";
const equityQuoteDurableName = "compute-equity-quotes";
if (env.COMPUTE_CONSUMER_RESET) {
try {
@ -791,6 +891,76 @@ const run = async () => {
}
}
if (env.COMPUTE_CONSUMER_RESET) {
try {
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, equityPrintDurableName);
logger.warn("reset jetstream consumer", { durable: equityPrintDurableName });
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) {
logger.warn("failed to reset jetstream consumer", {
durable: equityPrintDurableName,
error: message
});
}
}
} else {
try {
const info = await jsm.consumers.info(STREAM_EQUITY_PRINTS, equityPrintDurableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
logger.warn("resetting consumer due to deliver policy change", {
durable: equityPrintDurableName,
current: info.config.deliver_policy,
desired: env.COMPUTE_DELIVER_POLICY
});
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, equityPrintDurableName);
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) {
logger.warn("failed to inspect jetstream consumer", {
durable: equityPrintDurableName,
error: message
});
}
}
}
if (env.COMPUTE_CONSUMER_RESET) {
try {
await jsm.consumers.delete(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
logger.warn("reset jetstream consumer", { durable: equityQuoteDurableName });
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) {
logger.warn("failed to reset jetstream consumer", {
durable: equityQuoteDurableName,
error: message
});
}
}
} else {
try {
const info = await jsm.consumers.info(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
logger.warn("resetting consumer due to deliver policy change", {
durable: equityQuoteDurableName,
current: info.config.deliver_policy,
desired: env.COMPUTE_DELIVER_POLICY
});
await jsm.consumers.delete(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) {
logger.warn("failed to inspect jetstream consumer", {
durable: equityQuoteDurableName,
error: message
});
}
}
}
const subscription = await (async () => {
const opts = buildDurableConsumer(durableName);
applyDeliverPolicy(opts, env.COMPUTE_DELIVER_POLICY);
@ -863,6 +1033,81 @@ const run = async () => {
}
})();
const equitySubscription = await (async () => {
const opts = buildDurableConsumer(equityPrintDurableName);
applyDeliverPolicy(opts, env.COMPUTE_DELIVER_POLICY);
try {
return await subscribeJson(js, SUBJECT_EQUITY_PRINTS, opts);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
const shouldReset =
message.includes("duplicate subscription") ||
message.includes("durable requires") ||
message.includes("subject does not match consumer");
if (!shouldReset) {
throw error;
}
logger.warn("resetting jetstream consumer", { durable: equityPrintDurableName, error: message });
try {
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, equityPrintDurableName);
} catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", {
durable: equityPrintDurableName,
error: deleteMessage
});
}
}
const resetOpts = buildDurableConsumer(equityPrintDurableName);
applyDeliverPolicy(resetOpts, env.COMPUTE_DELIVER_POLICY);
return await subscribeJson(js, SUBJECT_EQUITY_PRINTS, resetOpts);
}
})();
const equityQuoteSubscription = await (async () => {
const opts = buildDurableConsumer(equityQuoteDurableName);
applyDeliverPolicy(opts, env.COMPUTE_DELIVER_POLICY);
try {
return await subscribeJson(js, SUBJECT_EQUITY_QUOTES, opts);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
const shouldReset =
message.includes("duplicate subscription") ||
message.includes("durable requires") ||
message.includes("subject does not match consumer");
if (!shouldReset) {
throw error;
}
logger.warn("resetting jetstream consumer", {
durable: equityQuoteDurableName,
error: message
});
try {
await jsm.consumers.delete(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
} catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", {
durable: equityQuoteDurableName,
error: deleteMessage
});
}
}
const resetOpts = buildDurableConsumer(equityQuoteDurableName);
applyDeliverPolicy(resetOpts, env.COMPUTE_DELIVER_POLICY);
return await subscribeJson(js, SUBJECT_EQUITY_QUOTES, resetOpts);
}
})();
const nbboLoop = async () => {
for await (const msg of nbboSubscription.messages) {
try {
@ -878,7 +1123,39 @@ const run = async () => {
}
};
const equityQuoteLoop = async () => {
for await (const msg of equityQuoteSubscription.messages) {
try {
const quote = EquityQuoteSchema.parse(equityQuoteSubscription.decode(msg));
updateEquityQuoteCache(quote);
msg.ack();
} catch (error) {
logger.error("failed to process equity quote", {
error: error instanceof Error ? error.message : String(error)
});
msg.term();
}
}
};
const equityPrintLoop = async () => {
for await (const msg of equitySubscription.messages) {
try {
const print = EquityPrintSchema.parse(equitySubscription.decode(msg));
await emitEquityJoin(clickhouse, js, print);
msg.ack();
} catch (error) {
logger.error("failed to process equity print", {
error: error instanceof Error ? error.message : String(error)
});
msg.term();
}
}
};
void nbboLoop();
void equityQuoteLoop();
void equityPrintLoop();
const shutdown = async (signal: string) => {
logger.info("service stopping", { signal });