Add equity print quote joins

This commit is contained in:
dirtydishes 2026-01-04 17:07:43 -05:00
parent 9908c431f0
commit 3164167bee
15 changed files with 1033 additions and 7 deletions

View file

@ -4,6 +4,10 @@ export const STREAM_OPTION_NBBO = "OPTIONS_NBBO";
export const SUBJECT_OPTION_NBBO = "options.nbbo"; export const SUBJECT_OPTION_NBBO = "options.nbbo";
export const STREAM_EQUITY_PRINTS = "EQUITY_PRINTS"; export const STREAM_EQUITY_PRINTS = "EQUITY_PRINTS";
export const SUBJECT_EQUITY_PRINTS = "equities.prints"; export const SUBJECT_EQUITY_PRINTS = "equities.prints";
export const STREAM_EQUITY_QUOTES = "EQUITY_QUOTES";
export const SUBJECT_EQUITY_QUOTES = "equities.quotes";
export const STREAM_EQUITY_JOINS = "EQUITY_JOINS";
export const SUBJECT_EQUITY_JOINS = "equities.joins";
export const STREAM_FLOW_PACKETS = "FLOW_PACKETS"; export const STREAM_FLOW_PACKETS = "FLOW_PACKETS";
export const SUBJECT_FLOW_PACKETS = "flow.packets"; export const SUBJECT_FLOW_PACKETS = "flow.packets";
export const STREAM_CLASSIFIER_HITS = "CLASSIFIER_HITS"; export const STREAM_CLASSIFIER_HITS = "CLASSIFIER_HITS";

View file

@ -3,6 +3,8 @@ import {
AlertEventSchema, AlertEventSchema,
ClassifierHitEventSchema, ClassifierHitEventSchema,
EquityPrintSchema, EquityPrintSchema,
EquityQuoteSchema,
EquityPrintJoinSchema,
FlowPacketSchema, FlowPacketSchema,
OptionNBBOSchema, OptionNBBOSchema,
OptionPrintSchema OptionPrintSchema
@ -11,6 +13,8 @@ import type {
AlertEvent, AlertEvent,
ClassifierHitEvent, ClassifierHitEvent,
EquityPrint, EquityPrint,
EquityQuote,
EquityPrintJoin,
FlowPacket, FlowPacket,
OptionNBBO, OptionNBBO,
OptionPrint OptionPrint
@ -26,6 +30,18 @@ import {
EQUITY_PRINTS_TABLE, EQUITY_PRINTS_TABLE,
normalizeEquityPrint normalizeEquityPrint
} from "./equity-prints"; } from "./equity-prints";
import {
equityQuotesTableDDL,
EQUITY_QUOTES_TABLE,
normalizeEquityQuote
} from "./equity-quotes";
import {
equityPrintJoinsTableDDL,
EQUITY_PRINT_JOINS_TABLE,
fromEquityPrintJoinRecord,
toEquityPrintJoinRecord,
type EquityPrintJoinRecord
} from "./equity-print-joins";
import { import {
FLOW_PACKETS_TABLE, FLOW_PACKETS_TABLE,
flowPacketsTableDDL, flowPacketsTableDDL,
@ -88,6 +104,22 @@ export const ensureEquityPrintsTable = async (
}); });
}; };
export const ensureEquityQuotesTable = async (
client: ClickHouseClient
): Promise<void> => {
await client.exec({
query: equityQuotesTableDDL()
});
};
export const ensureEquityPrintJoinsTable = async (
client: ClickHouseClient
): Promise<void> => {
await client.exec({
query: equityPrintJoinsTableDDL()
});
};
export const ensureFlowPacketsTable = async ( export const ensureFlowPacketsTable = async (
client: ClickHouseClient client: ClickHouseClient
): Promise<void> => { ): Promise<void> => {
@ -146,6 +178,30 @@ export const insertEquityPrint = async (
}); });
}; };
export const insertEquityQuote = async (
client: ClickHouseClient,
quote: EquityQuote
): Promise<void> => {
const record = normalizeEquityQuote(quote);
await client.insert({
table: EQUITY_QUOTES_TABLE,
values: [record],
format: "JSONEachRow"
});
};
export const insertEquityPrintJoin = async (
client: ClickHouseClient,
join: EquityPrintJoin
): Promise<void> => {
const record = toEquityPrintJoinRecord(join);
await client.insert({
table: EQUITY_PRINT_JOINS_TABLE,
values: [record],
format: "JSONEachRow"
});
};
export const insertFlowPacket = async ( export const insertFlowPacket = async (
client: ClickHouseClient, client: ClickHouseClient,
packet: FlowPacket packet: FlowPacket
@ -253,6 +309,20 @@ const normalizeOptionNbboRow = (row: unknown): unknown => {
return row; return row;
}; };
const normalizeEquityQuoteRow = (row: unknown): unknown => {
if (row && typeof row === "object") {
return normalizeNumericFields(row as Record<string, unknown>, [
"source_ts",
"ingest_ts",
"seq",
"ts",
"bid",
"ask"
]);
}
return row;
};
const normalizeEquityRow = (row: unknown): unknown => { const normalizeEquityRow = (row: unknown): unknown => {
if (row && typeof row === "object") { if (row && typeof row === "object") {
@ -278,6 +348,25 @@ const normalizeEquityRow = (row: unknown): unknown => {
return row; return row;
}; };
const normalizeEquityPrintJoinRow = (row: unknown): EquityPrintJoinRecord | null => {
if (!row || typeof row !== "object") {
return null;
}
const record = row as Record<string, unknown>;
return {
source_ts: coerceNumber(record.source_ts) as number,
ingest_ts: coerceNumber(record.ingest_ts) as number,
seq: coerceNumber(record.seq) as number,
trace_id: String(record.trace_id ?? ""),
id: String(record.id ?? ""),
print_trace_id: String(record.print_trace_id ?? ""),
quote_trace_id: String(record.quote_trace_id ?? ""),
features_json: String(record.features_json ?? "{}"),
join_quality_json: String(record.join_quality_json ?? "{}")
};
};
const normalizeFlowPacketRow = (row: unknown): FlowPacketRecord | null => { const normalizeFlowPacketRow = (row: unknown): FlowPacketRecord | null => {
if (!row || typeof row !== "object") { if (!row || typeof row !== "object") {
return null; return null;
@ -376,6 +465,38 @@ export const fetchRecentEquityPrints = async (
return EquityPrintSchema.array().parse(rows.map(normalizeEquityRow)); return EquityPrintSchema.array().parse(rows.map(normalizeEquityRow));
}; };
export const fetchRecentEquityQuotes = async (
client: ClickHouseClient,
limit: number
): Promise<EquityQuote[]> => {
const safeLimit = clampLimit(limit);
const result = await client.query({
query: `SELECT * FROM ${EQUITY_QUOTES_TABLE} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
format: "JSONEachRow"
});
const rows = await result.json<unknown[]>();
return EquityQuoteSchema.array().parse(rows.map(normalizeEquityQuoteRow));
};
export const fetchRecentEquityPrintJoins = async (
client: ClickHouseClient,
limit: number
): Promise<EquityPrintJoin[]> => {
const safeLimit = clampLimit(limit);
const result = await client.query({
query: `SELECT * FROM ${EQUITY_PRINT_JOINS_TABLE} ORDER BY source_ts DESC, seq DESC LIMIT ${safeLimit}`,
format: "JSONEachRow"
});
const rows = await result.json<unknown[]>();
const records = rows
.map(normalizeEquityPrintJoinRow)
.filter((record): record is EquityPrintJoinRecord => record !== null);
const joins = records.map(fromEquityPrintJoinRecord);
return EquityPrintJoinSchema.array().parse(joins);
};
export const fetchRecentFlowPackets = async ( export const fetchRecentFlowPackets = async (
client: ClickHouseClient, client: ClickHouseClient,
limit: number limit: number
@ -486,3 +607,45 @@ export const fetchEquityPrintsAfter = async (
const rows = await result.json<unknown[]>(); const rows = await result.json<unknown[]>();
return EquityPrintSchema.array().parse(rows.map(normalizeEquityRow)); return EquityPrintSchema.array().parse(rows.map(normalizeEquityRow));
}; };
export const fetchEquityQuotesAfter = async (
client: ClickHouseClient,
afterTs: number,
afterSeq: number,
limit: number
): Promise<EquityQuote[]> => {
const safeLimit = clampLimit(limit);
const safeAfterTs = clampCursor(afterTs);
const safeAfterSeq = clampCursor(afterSeq);
const result = await client.query({
query: `SELECT * FROM ${EQUITY_QUOTES_TABLE} WHERE (ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
format: "JSONEachRow"
});
const rows = await result.json<unknown[]>();
return EquityQuoteSchema.array().parse(rows.map(normalizeEquityQuoteRow));
};
export const fetchEquityPrintJoinsAfter = async (
client: ClickHouseClient,
afterTs: number,
afterSeq: number,
limit: number
): Promise<EquityPrintJoin[]> => {
const safeLimit = clampLimit(limit);
const safeAfterTs = clampCursor(afterTs);
const safeAfterSeq = clampCursor(afterSeq);
const result = await client.query({
query: `SELECT * FROM ${EQUITY_PRINT_JOINS_TABLE} WHERE (source_ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY source_ts ASC, seq ASC LIMIT ${safeLimit}`,
format: "JSONEachRow"
});
const rows = await result.json<unknown[]>();
const records = rows
.map(normalizeEquityPrintJoinRow)
.filter((record): record is EquityPrintJoinRecord => record !== null);
const joins = records.map(fromEquityPrintJoinRecord);
return EquityPrintJoinSchema.array().parse(joins);
};

View file

@ -0,0 +1,74 @@
import type { EquityPrintJoin } from "@islandflow/types";
export const EQUITY_PRINT_JOINS_TABLE = "equity_print_joins";
export type EquityPrintJoinRecord = {
source_ts: number;
ingest_ts: number;
seq: number;
trace_id: string;
id: string;
print_trace_id: string;
quote_trace_id: string;
features_json: string;
join_quality_json: string;
};
export const equityPrintJoinsTableDDL = (): string => {
return `
CREATE TABLE IF NOT EXISTS ${EQUITY_PRINT_JOINS_TABLE} (
source_ts UInt64,
ingest_ts UInt64,
seq UInt64,
trace_id String,
id String,
print_trace_id String,
quote_trace_id String,
features_json String,
join_quality_json String
)
ENGINE = MergeTree
ORDER BY (source_ts, seq)
`;
};
export const toEquityPrintJoinRecord = (join: EquityPrintJoin): EquityPrintJoinRecord => {
return {
source_ts: join.source_ts,
ingest_ts: join.ingest_ts,
seq: join.seq,
trace_id: join.trace_id,
id: join.id,
print_trace_id: join.print_trace_id,
quote_trace_id: join.quote_trace_id,
features_json: JSON.stringify(join.features),
join_quality_json: JSON.stringify(join.join_quality)
};
};
const safeJson = (value: string, fallback: Record<string, unknown>): Record<string, unknown> => {
try {
const parsed = JSON.parse(value);
if (parsed && typeof parsed === "object") {
return parsed as Record<string, unknown>;
}
} catch {
// ignore
}
return fallback;
};
export const fromEquityPrintJoinRecord = (record: EquityPrintJoinRecord): EquityPrintJoin => {
return {
source_ts: record.source_ts,
ingest_ts: record.ingest_ts,
seq: record.seq,
trace_id: record.trace_id,
id: record.id,
print_trace_id: record.print_trace_id,
quote_trace_id: record.quote_trace_id,
features: safeJson(record.features_json, {}),
join_quality: safeJson(record.join_quality_json, {}) as Record<string, number>
};
};

View file

@ -0,0 +1,24 @@
import type { EquityQuote } from "@islandflow/types";
export const EQUITY_QUOTES_TABLE = "equity_quotes";
export const equityQuotesTableDDL = (): string => {
return `
CREATE TABLE IF NOT EXISTS ${EQUITY_QUOTES_TABLE} (
source_ts UInt64,
ingest_ts UInt64,
seq UInt64,
trace_id String,
ts UInt64,
underlying_id String,
bid Float64,
ask Float64
)
ENGINE = MergeTree
ORDER BY (ts, underlying_id)
`;
};
export const normalizeEquityQuote = (quote: EquityQuote): EquityQuote => {
return quote;
};

View file

@ -3,5 +3,7 @@ export * from "./classifier-hits";
export * from "./alerts"; export * from "./alerts";
export * from "./flow-packets"; export * from "./flow-packets";
export * from "./equity-prints"; export * from "./equity-prints";
export * from "./equity-quotes";
export * from "./equity-print-joins";
export * from "./option-prints"; export * from "./option-prints";
export * from "./option-nbbo"; export * from "./option-nbbo";

View file

@ -0,0 +1,42 @@
import { describe, expect, it } from "bun:test";
import {
equityPrintJoinsTableDDL,
EQUITY_PRINT_JOINS_TABLE,
fromEquityPrintJoinRecord,
toEquityPrintJoinRecord
} from "../src/equity-print-joins";
const join = {
source_ts: 100,
ingest_ts: 120,
seq: 1,
trace_id: "equityjoin:trace-1",
id: "equityjoin:trace-1",
print_trace_id: "trace-1",
quote_trace_id: "quote-1",
features: {
underlying_id: "SPY",
price: 450.12,
size: 200,
off_exchange_flag: true,
quote_placement: "A"
},
join_quality: {
quote_age_ms: 15
}
};
describe("equity-print-joins storage helpers", () => {
it("includes the correct table name in the DDL", () => {
const ddl = equityPrintJoinsTableDDL();
expect(ddl).toContain(EQUITY_PRINT_JOINS_TABLE);
expect(ddl).toContain("CREATE TABLE IF NOT EXISTS");
});
it("round-trips equity print join records", () => {
const record = toEquityPrintJoinRecord(join);
const restored = fromEquityPrintJoinRecord(record);
expect(restored.features).toEqual(join.features);
expect(restored.join_quality).toEqual(join.join_quality);
});
});

View file

@ -0,0 +1,30 @@
import { describe, expect, it } from "bun:test";
import {
equityQuotesTableDDL,
EQUITY_QUOTES_TABLE,
normalizeEquityQuote
} from "../src/equity-quotes";
const baseQuote = {
source_ts: 100,
ingest_ts: 200,
seq: 1,
trace_id: "trace-1",
ts: 100,
underlying_id: "SPY",
bid: 450.1,
ask: 450.2
};
describe("equity-quotes storage helpers", () => {
it("keeps required fields intact", () => {
const normalized = normalizeEquityQuote(baseQuote);
expect(normalized).toEqual(baseQuote);
});
it("includes the correct table name in the DDL", () => {
const ddl = equityQuotesTableDDL();
expect(ddl).toContain(EQUITY_QUOTES_TABLE);
expect(ddl).toContain("CREATE TABLE IF NOT EXISTS");
});
});

View file

@ -59,6 +59,18 @@ export const EquityQuoteSchema = EventMetaSchema.merge(
export type EquityQuote = z.infer<typeof EquityQuoteSchema>; export type EquityQuote = z.infer<typeof EquityQuoteSchema>;
export const EquityPrintJoinSchema = EventMetaSchema.merge(
z.object({
id: z.string().min(1),
print_trace_id: z.string().min(1),
quote_trace_id: z.string(),
features: z.record(z.union([z.string(), z.number(), z.boolean()])),
join_quality: z.record(z.number())
})
);
export type EquityPrintJoin = z.infer<typeof EquityPrintJoinSchema>;
export const FlowPacketSchema = EventMetaSchema.merge( export const FlowPacketSchema = EventMetaSchema.merge(
z.object({ z.object({
id: z.string().min(1), id: z.string().min(1),

View file

@ -3,13 +3,17 @@ import { createLogger } from "@islandflow/observability";
import { import {
SUBJECT_ALERTS, SUBJECT_ALERTS,
SUBJECT_CLASSIFIER_HITS, SUBJECT_CLASSIFIER_HITS,
SUBJECT_EQUITY_JOINS,
SUBJECT_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS,
SUBJECT_EQUITY_QUOTES,
SUBJECT_FLOW_PACKETS, SUBJECT_FLOW_PACKETS,
SUBJECT_OPTION_NBBO, SUBJECT_OPTION_NBBO,
SUBJECT_OPTION_PRINTS, SUBJECT_OPTION_PRINTS,
STREAM_ALERTS, STREAM_ALERTS,
STREAM_CLASSIFIER_HITS, STREAM_CLASSIFIER_HITS,
STREAM_EQUITY_JOINS,
STREAM_EQUITY_PRINTS, STREAM_EQUITY_PRINTS,
STREAM_EQUITY_QUOTES,
STREAM_FLOW_PACKETS, STREAM_FLOW_PACKETS,
STREAM_OPTION_NBBO, STREAM_OPTION_NBBO,
STREAM_OPTION_PRINTS, STREAM_OPTION_PRINTS,
@ -22,15 +26,21 @@ import {
createClickHouseClient, createClickHouseClient,
ensureAlertsTable, ensureAlertsTable,
ensureClassifierHitsTable, ensureClassifierHitsTable,
ensureEquityPrintJoinsTable,
ensureEquityPrintsTable, ensureEquityPrintsTable,
ensureEquityQuotesTable,
ensureFlowPacketsTable, ensureFlowPacketsTable,
ensureOptionNBBOTable, ensureOptionNBBOTable,
ensureOptionPrintsTable, ensureOptionPrintsTable,
fetchRecentAlerts, fetchRecentAlerts,
fetchRecentClassifierHits, fetchRecentClassifierHits,
fetchRecentEquityPrintJoins,
fetchRecentFlowPackets, fetchRecentFlowPackets,
fetchRecentEquityQuotes,
fetchRecentOptionNBBO, fetchRecentOptionNBBO,
fetchEquityPrintsAfter, fetchEquityPrintsAfter,
fetchEquityPrintJoinsAfter,
fetchEquityQuotesAfter,
fetchRecentEquityPrints, fetchRecentEquityPrints,
fetchOptionNBBOAfter, fetchOptionNBBOAfter,
fetchOptionPrintsAfter, fetchOptionPrintsAfter,
@ -40,6 +50,8 @@ import {
AlertEventSchema, AlertEventSchema,
ClassifierHitEventSchema, ClassifierHitEventSchema,
EquityPrintSchema, EquityPrintSchema,
EquityPrintJoinSchema,
EquityQuoteSchema,
FlowPacketSchema, FlowPacketSchema,
OptionNBBOSchema, OptionNBBOSchema,
OptionPrintSchema OptionPrintSchema
@ -93,7 +105,15 @@ const replayParamsSchema = z.object({
limit: z.coerce.number().int().positive().max(1000).default(200) limit: z.coerce.number().int().positive().max(1000).default(200)
}); });
type Channel = "options" | "options-nbbo" | "equities" | "flow" | "classifier-hits" | "alerts"; type Channel =
| "options"
| "options-nbbo"
| "equities"
| "equity-quotes"
| "equity-joins"
| "flow"
| "classifier-hits"
| "alerts";
type WsData = { type WsData = {
channel: Channel; channel: Channel;
@ -102,6 +122,8 @@ type WsData = {
const optionSockets = new Set<WebSocket<WsData>>(); const optionSockets = new Set<WebSocket<WsData>>();
const optionNbboSockets = new Set<WebSocket<WsData>>(); const optionNbboSockets = new Set<WebSocket<WsData>>();
const equitySockets = new Set<WebSocket<WsData>>(); const equitySockets = new Set<WebSocket<WsData>>();
const equityQuoteSockets = new Set<WebSocket<WsData>>();
const equityJoinSockets = new Set<WebSocket<WsData>>();
const flowSockets = new Set<WebSocket<WsData>>(); const flowSockets = new Set<WebSocket<WsData>>();
const classifierHitSockets = new Set<WebSocket<WsData>>(); const classifierHitSockets = new Set<WebSocket<WsData>>();
const alertSockets = new Set<WebSocket<WsData>>(); const alertSockets = new Set<WebSocket<WsData>>();
@ -202,6 +224,32 @@ const run = async () => {
num_replicas: 1 num_replicas: 1
}); });
await ensureStream(jsm, {
name: STREAM_EQUITY_QUOTES,
subjects: [SUBJECT_EQUITY_QUOTES],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_EQUITY_JOINS,
subjects: [SUBJECT_EQUITY_JOINS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, { await ensureStream(jsm, {
name: STREAM_FLOW_PACKETS, name: STREAM_FLOW_PACKETS,
subjects: [SUBJECT_FLOW_PACKETS], subjects: [SUBJECT_FLOW_PACKETS],
@ -250,6 +298,8 @@ const run = async () => {
await ensureOptionPrintsTable(clickhouse); await ensureOptionPrintsTable(clickhouse);
await ensureOptionNBBOTable(clickhouse); await ensureOptionNBBOTable(clickhouse);
await ensureEquityPrintsTable(clickhouse); await ensureEquityPrintsTable(clickhouse);
await ensureEquityQuotesTable(clickhouse);
await ensureEquityPrintJoinsTable(clickhouse);
await ensureFlowPacketsTable(clickhouse); await ensureFlowPacketsTable(clickhouse);
await ensureClassifierHitsTable(clickhouse); await ensureClassifierHitsTable(clickhouse);
await ensureAlertsTable(clickhouse); await ensureAlertsTable(clickhouse);
@ -311,6 +361,18 @@ const run = async () => {
"api-equity-prints" "api-equity-prints"
); );
const equityQuoteSubscription = await subscribeWithReset(
SUBJECT_EQUITY_QUOTES,
STREAM_EQUITY_QUOTES,
"api-equity-quotes"
);
const equityJoinSubscription = await subscribeWithReset(
SUBJECT_EQUITY_JOINS,
STREAM_EQUITY_JOINS,
"api-equity-joins"
);
const flowSubscription = await subscribeWithReset( const flowSubscription = await subscribeWithReset(
SUBJECT_FLOW_PACKETS, SUBJECT_FLOW_PACKETS,
STREAM_FLOW_PACKETS, STREAM_FLOW_PACKETS,
@ -374,6 +436,36 @@ const run = async () => {
} }
}; };
const pumpEquityQuotes = async () => {
for await (const msg of equityQuoteSubscription.messages) {
try {
const payload = EquityQuoteSchema.parse(equityQuoteSubscription.decode(msg));
broadcast(equityQuoteSockets, { type: "equity-quote", payload });
msg.ack();
} catch (error) {
logger.error("failed to process equity quote", {
error: error instanceof Error ? error.message : String(error)
});
msg.term();
}
}
};
const pumpEquityJoins = async () => {
for await (const msg of equityJoinSubscription.messages) {
try {
const payload = EquityPrintJoinSchema.parse(equityJoinSubscription.decode(msg));
broadcast(equityJoinSockets, { type: "equity-join", payload });
msg.ack();
} catch (error) {
logger.error("failed to process equity join", {
error: error instanceof Error ? error.message : String(error)
});
msg.term();
}
}
};
const pumpFlow = async () => { const pumpFlow = async () => {
for await (const msg of flowSubscription.messages) { for await (const msg of flowSubscription.messages) {
try { try {
@ -422,6 +514,8 @@ const run = async () => {
void pumpOptions(); void pumpOptions();
void pumpOptionNbbo(); void pumpOptionNbbo();
void pumpEquities(); void pumpEquities();
void pumpEquityQuotes();
void pumpEquityJoins();
void pumpFlow(); void pumpFlow();
void pumpClassifierHits(); void pumpClassifierHits();
void pumpAlerts(); void pumpAlerts();
@ -453,6 +547,18 @@ const run = async () => {
return jsonResponse({ data }); return jsonResponse({ data });
} }
if (req.method === "GET" && url.pathname === "/quotes/equities") {
const limit = parseLimit(url.searchParams.get("limit"));
const data = await fetchRecentEquityQuotes(clickhouse, limit);
return jsonResponse({ data });
}
if (req.method === "GET" && url.pathname === "/joins/equities") {
const limit = parseLimit(url.searchParams.get("limit"));
const data = await fetchRecentEquityPrintJoins(clickhouse, limit);
return jsonResponse({ data });
}
if (req.method === "GET" && url.pathname === "/flow/packets") { if (req.method === "GET" && url.pathname === "/flow/packets") {
const limit = parseLimit(url.searchParams.get("limit")); const limit = parseLimit(url.searchParams.get("limit"));
const data = await fetchRecentFlowPackets(clickhouse, limit); const data = await fetchRecentFlowPackets(clickhouse, limit);
@ -495,6 +601,22 @@ const run = async () => {
return jsonResponse({ data, next }); return jsonResponse({ data, next });
} }
if (req.method === "GET" && url.pathname === "/replay/equity-quotes") {
const { afterTs, afterSeq, limit } = parseReplayParams(url);
const data = await fetchEquityQuotesAfter(clickhouse, afterTs, afterSeq, limit);
const last = data.at(-1);
const next = last ? { ts: last.ts, seq: last.seq } : null;
return jsonResponse({ data, next });
}
if (req.method === "GET" && url.pathname === "/replay/equity-joins") {
const { afterTs, afterSeq, limit } = parseReplayParams(url);
const data = await fetchEquityPrintJoinsAfter(clickhouse, afterTs, afterSeq, limit);
const last = data.at(-1);
const next = last ? { ts: last.source_ts, seq: last.seq } : null;
return jsonResponse({ data, next });
}
if (req.method === "GET" && url.pathname === "/ws/options") { if (req.method === "GET" && url.pathname === "/ws/options") {
if (serverRef.upgrade(req, { data: { channel: "options" } })) { if (serverRef.upgrade(req, { data: { channel: "options" } })) {
return new Response(null, { status: 101 }); return new Response(null, { status: 101 });
@ -519,6 +641,22 @@ const run = async () => {
return jsonResponse({ error: "websocket upgrade failed" }, 400); return jsonResponse({ error: "websocket upgrade failed" }, 400);
} }
if (req.method === "GET" && url.pathname === "/ws/equity-quotes") {
if (serverRef.upgrade(req, { data: { channel: "equity-quotes" } })) {
return new Response(null, { status: 101 });
}
return jsonResponse({ error: "websocket upgrade failed" }, 400);
}
if (req.method === "GET" && url.pathname === "/ws/equity-joins") {
if (serverRef.upgrade(req, { data: { channel: "equity-joins" } })) {
return new Response(null, { status: 101 });
}
return jsonResponse({ error: "websocket upgrade failed" }, 400);
}
if (req.method === "GET" && url.pathname === "/ws/flow") { if (req.method === "GET" && url.pathname === "/ws/flow") {
if (serverRef.upgrade(req, { data: { channel: "flow" } })) { if (serverRef.upgrade(req, { data: { channel: "flow" } })) {
return new Response(null, { status: 101 }); return new Response(null, { status: 101 });
@ -553,6 +691,10 @@ const run = async () => {
optionNbboSockets.add(socket); optionNbboSockets.add(socket);
} else if (socket.data.channel === "equities") { } else if (socket.data.channel === "equities") {
equitySockets.add(socket); equitySockets.add(socket);
} else if (socket.data.channel === "equity-quotes") {
equityQuoteSockets.add(socket);
} else if (socket.data.channel === "equity-joins") {
equityJoinSockets.add(socket);
} else if (socket.data.channel === "flow") { } else if (socket.data.channel === "flow") {
flowSockets.add(socket); flowSockets.add(socket);
} else if (socket.data.channel === "classifier-hits") { } else if (socket.data.channel === "classifier-hits") {
@ -570,6 +712,10 @@ const run = async () => {
optionNbboSockets.delete(socket); optionNbboSockets.delete(socket);
} else if (socket.data.channel === "equities") { } else if (socket.data.channel === "equities") {
equitySockets.delete(socket); equitySockets.delete(socket);
} else if (socket.data.channel === "equity-quotes") {
equityQuoteSockets.delete(socket);
} else if (socket.data.channel === "equity-joins") {
equityJoinSockets.delete(socket);
} else if (socket.data.channel === "flow") { } else if (socket.data.channel === "flow") {
flowSockets.delete(socket); flowSockets.delete(socket);
} else if (socket.data.channel === "classifier-hits") { } else if (socket.data.channel === "classifier-hits") {

View file

@ -0,0 +1,104 @@
import type { EquityPrint, EquityPrintJoin, EquityQuote } from "@islandflow/types";
export type EquityQuoteJoin = {
quote: EquityQuote | null;
ageMs: number;
stale: boolean;
};
export type QuotePlacement = "AA" | "A" | "B" | "BB" | "MID" | "MISSING" | "STALE";
const roundTo = (value: number, digits = 4): number => {
if (!Number.isFinite(value)) {
return 0;
}
return Number(value.toFixed(digits));
};
export const classifyQuotePlacement = (
price: number,
join: EquityQuoteJoin
): QuotePlacement => {
if (!Number.isFinite(price)) {
return "MISSING";
}
if (!join.quote) {
return "MISSING";
}
if (join.stale) {
return "STALE";
}
const bid = join.quote.bid;
const ask = join.quote.ask;
if (!Number.isFinite(bid) || !Number.isFinite(ask) || ask <= 0) {
return "MISSING";
}
const spread = Math.max(0, ask - bid);
const epsilon = Math.max(0.01, spread * 0.05);
if (price > ask + epsilon) {
return "AA";
}
if (price >= ask - epsilon) {
return "A";
}
if (price < bid - epsilon) {
return "BB";
}
if (price <= bid + epsilon) {
return "B";
}
return "MID";
};
export const buildEquityPrintJoin = (
print: EquityPrint,
join: EquityQuoteJoin
): EquityPrintJoin => {
const joinQuality: Record<string, number> = {};
const placement = classifyQuotePlacement(print.price, join);
const features: Record<string, string | number | boolean> = {
underlying_id: print.underlying_id,
price: print.price,
size: print.size,
off_exchange_flag: print.offExchangeFlag,
print_ts: print.ts,
quote_placement: placement
};
if (!join.quote) {
joinQuality.quote_missing = 1;
} else {
joinQuality.quote_age_ms = join.ageMs;
if (join.stale) {
joinQuality.quote_stale = 1;
} else {
const bid = join.quote.bid;
const ask = join.quote.ask;
const mid = (bid + ask) / 2;
const spread = ask - bid;
features.quote_ts = join.quote.ts;
features.quote_bid = bid;
features.quote_ask = ask;
features.quote_mid = roundTo(mid);
features.quote_spread = roundTo(spread);
}
}
const joinId = `equityjoin:${print.trace_id}`;
return {
source_ts: print.source_ts,
ingest_ts: print.ingest_ts,
seq: print.seq,
trace_id: joinId,
id: joinId,
print_trace_id: print.trace_id,
quote_trace_id: join.quote?.trace_id ?? "",
features,
join_quality: joinQuality
};
};

View file

@ -3,11 +3,17 @@ import { createLogger } from "@islandflow/observability";
import { import {
SUBJECT_ALERTS, SUBJECT_ALERTS,
SUBJECT_CLASSIFIER_HITS, SUBJECT_CLASSIFIER_HITS,
SUBJECT_EQUITY_JOINS,
SUBJECT_EQUITY_PRINTS,
SUBJECT_EQUITY_QUOTES,
SUBJECT_FLOW_PACKETS, SUBJECT_FLOW_PACKETS,
SUBJECT_OPTION_NBBO, SUBJECT_OPTION_NBBO,
SUBJECT_OPTION_PRINTS, SUBJECT_OPTION_PRINTS,
STREAM_ALERTS, STREAM_ALERTS,
STREAM_CLASSIFIER_HITS, STREAM_CLASSIFIER_HITS,
STREAM_EQUITY_JOINS,
STREAM_EQUITY_PRINTS,
STREAM_EQUITY_QUOTES,
STREAM_FLOW_PACKETS, STREAM_FLOW_PACKETS,
STREAM_OPTION_NBBO, STREAM_OPTION_NBBO,
STREAM_OPTION_PRINTS, STREAM_OPTION_PRINTS,
@ -21,19 +27,27 @@ import {
createClickHouseClient, createClickHouseClient,
ensureAlertsTable, ensureAlertsTable,
ensureClassifierHitsTable, ensureClassifierHitsTable,
ensureEquityPrintJoinsTable,
ensureFlowPacketsTable, ensureFlowPacketsTable,
insertAlert, insertAlert,
insertClassifierHit, insertClassifierHit,
insertEquityPrintJoin,
insertFlowPacket insertFlowPacket
} from "@islandflow/storage"; } from "@islandflow/storage";
import { import {
AlertEventSchema, AlertEventSchema,
ClassifierHitEventSchema, ClassifierHitEventSchema,
EquityPrintJoinSchema,
EquityPrintSchema,
EquityQuoteSchema,
FlowPacketSchema, FlowPacketSchema,
OptionNBBOSchema, OptionNBBOSchema,
OptionPrintSchema, OptionPrintSchema,
type AlertEvent, type AlertEvent,
type ClassifierHitEvent, type ClassifierHitEvent,
type EquityPrint,
type EquityQuote,
type EquityPrintJoin,
type FlowPacket, type FlowPacket,
type OptionNBBO, type OptionNBBO,
type OptionPrint type OptionPrint
@ -41,6 +55,7 @@ import {
import { z } from "zod"; import { z } from "zod";
import { evaluateClassifiers, type ClassifierConfig } from "./classifiers"; import { evaluateClassifiers, type ClassifierConfig } from "./classifiers";
import { parseContractId } from "./contracts"; import { parseContractId } from "./contracts";
import { buildEquityPrintJoin, type EquityQuoteJoin } from "./equity-joins";
import { createRedisClient, updateRollingStats, type RollingStatsConfig } from "./rolling-stats"; import { createRedisClient, updateRollingStats, type RollingStatsConfig } from "./rolling-stats";
import { summarizeStructure, type ContractLeg } from "./structures"; import { summarizeStructure, type ContractLeg } from "./structures";
@ -71,6 +86,7 @@ const envSchema = z.object({
}, z.boolean()) }, z.boolean())
.default(false), .default(false),
NBBO_MAX_AGE_MS: z.coerce.number().int().positive().default(1000), NBBO_MAX_AGE_MS: z.coerce.number().int().positive().default(1000),
EQUITY_QUOTE_MAX_AGE_MS: z.coerce.number().int().positive().default(1000),
CLASSIFIER_SWEEP_MIN_PREMIUM: z.coerce.number().positive().default(40_000), CLASSIFIER_SWEEP_MIN_PREMIUM: z.coerce.number().positive().default(40_000),
CLASSIFIER_SWEEP_MIN_COUNT: z.coerce.number().int().positive().default(3), CLASSIFIER_SWEEP_MIN_COUNT: z.coerce.number().int().positive().default(3),
CLASSIFIER_SWEEP_MIN_PREMIUM_Z: z.coerce.number().nonnegative().default(2), CLASSIFIER_SWEEP_MIN_PREMIUM_Z: z.coerce.number().nonnegative().default(2),
@ -161,6 +177,7 @@ type ClusterState = {
const clusters = new Map<string, ClusterState>(); const clusters = new Map<string, ClusterState>();
const nbboCache = new Map<string, OptionNBBO>(); const nbboCache = new Map<string, OptionNBBO>();
const equityQuoteCache = new Map<string, EquityQuote>();
const recentLegsByKey = new Map<string, ContractLeg[]>(); const recentLegsByKey = new Map<string, ContractLeg[]>();
const MAX_RECENT_LEGS = 20; const MAX_RECENT_LEGS = 20;
@ -341,6 +358,17 @@ const updateNbboCache = (nbbo: OptionNBBO): void => {
} }
}; };
const updateEquityQuoteCache = (quote: EquityQuote): void => {
const existing = equityQuoteCache.get(quote.underlying_id);
if (
!existing ||
quote.ts > existing.ts ||
(quote.ts === existing.ts && quote.seq >= existing.seq)
) {
equityQuoteCache.set(quote.underlying_id, quote);
}
};
const selectNbbo = (contractId: string, ts: number): NbboJoin => { const selectNbbo = (contractId: string, ts: number): NbboJoin => {
const nbbo = nbboCache.get(contractId) ?? null; const nbbo = nbboCache.get(contractId) ?? null;
if (!nbbo) { if (!nbbo) {
@ -352,6 +380,17 @@ const selectNbbo = (contractId: string, ts: number): NbboJoin => {
return { nbbo, ageMs, stale }; return { nbbo, ageMs, stale };
}; };
const selectEquityQuote = (underlyingId: string, ts: number): EquityQuoteJoin => {
const quote = equityQuoteCache.get(underlyingId) ?? null;
if (!quote) {
return { quote: null, ageMs: env.EQUITY_QUOTE_MAX_AGE_MS + 1, stale: true };
}
const ageMs = Math.abs(ts - quote.ts);
const stale = ageMs > env.EQUITY_QUOTE_MAX_AGE_MS;
return { quote, ageMs, stale };
};
const classifyPlacement = (price: number, join: NbboJoin): NbboPlacement => { const classifyPlacement = (price: number, join: NbboJoin): NbboPlacement => {
if (!Number.isFinite(price)) { if (!Number.isFinite(price)) {
return "MISSING"; return "MISSING";
@ -609,6 +648,25 @@ const emitClassifiers = async (
} }
}; };
const emitEquityJoin = async (
clickhouse: ReturnType<typeof createClickHouseClient>,
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
print: EquityPrint
): Promise<void> => {
const join = selectEquityQuote(print.underlying_id, print.ts);
const payload: EquityPrintJoin = EquityPrintJoinSchema.parse(buildEquityPrintJoin(print, join));
try {
await insertEquityPrintJoin(clickhouse, payload);
await publishJson(js, SUBJECT_EQUITY_JOINS, payload);
} catch (error) {
logger.error("failed to emit equity print join", {
error: error instanceof Error ? error.message : String(error),
trace_id: payload.trace_id
});
}
};
const flushEligibleClusters = async ( const flushEligibleClusters = async (
clickhouse: ReturnType<typeof createClickHouseClient>, clickhouse: ReturnType<typeof createClickHouseClient>,
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"], js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
@ -666,6 +724,32 @@ const run = async () => {
num_replicas: 1 num_replicas: 1
}); });
await ensureStream(jsm, {
name: STREAM_EQUITY_PRINTS,
subjects: [SUBJECT_EQUITY_PRINTS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_EQUITY_QUOTES,
subjects: [SUBJECT_EQUITY_QUOTES],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, { await ensureStream(jsm, {
name: STREAM_FLOW_PACKETS, name: STREAM_FLOW_PACKETS,
subjects: [SUBJECT_FLOW_PACKETS], subjects: [SUBJECT_FLOW_PACKETS],
@ -679,6 +763,19 @@ const run = async () => {
num_replicas: 1 num_replicas: 1
}); });
await ensureStream(jsm, {
name: STREAM_EQUITY_JOINS,
subjects: [SUBJECT_EQUITY_JOINS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, { await ensureStream(jsm, {
name: STREAM_CLASSIFIER_HITS, name: STREAM_CLASSIFIER_HITS,
subjects: [SUBJECT_CLASSIFIER_HITS], subjects: [SUBJECT_CLASSIFIER_HITS],
@ -726,12 +823,15 @@ const run = async () => {
await retry("clickhouse table init", 20, 500, async () => { await retry("clickhouse table init", 20, 500, async () => {
await ensureFlowPacketsTable(clickhouse); await ensureFlowPacketsTable(clickhouse);
await ensureEquityPrintJoinsTable(clickhouse);
await ensureClassifierHitsTable(clickhouse); await ensureClassifierHitsTable(clickhouse);
await ensureAlertsTable(clickhouse); await ensureAlertsTable(clickhouse);
}); });
const durableName = "compute-option-prints"; const durableName = "compute-option-prints";
const nbboDurableName = "compute-option-nbbo"; const nbboDurableName = "compute-option-nbbo";
const equityPrintDurableName = "compute-equity-prints";
const equityQuoteDurableName = "compute-equity-quotes";
if (env.COMPUTE_CONSUMER_RESET) { if (env.COMPUTE_CONSUMER_RESET) {
try { try {
@ -791,6 +891,76 @@ const run = async () => {
} }
} }
if (env.COMPUTE_CONSUMER_RESET) {
try {
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, equityPrintDurableName);
logger.warn("reset jetstream consumer", { durable: equityPrintDurableName });
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) {
logger.warn("failed to reset jetstream consumer", {
durable: equityPrintDurableName,
error: message
});
}
}
} else {
try {
const info = await jsm.consumers.info(STREAM_EQUITY_PRINTS, equityPrintDurableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
logger.warn("resetting consumer due to deliver policy change", {
durable: equityPrintDurableName,
current: info.config.deliver_policy,
desired: env.COMPUTE_DELIVER_POLICY
});
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, equityPrintDurableName);
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) {
logger.warn("failed to inspect jetstream consumer", {
durable: equityPrintDurableName,
error: message
});
}
}
}
if (env.COMPUTE_CONSUMER_RESET) {
try {
await jsm.consumers.delete(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
logger.warn("reset jetstream consumer", { durable: equityQuoteDurableName });
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) {
logger.warn("failed to reset jetstream consumer", {
durable: equityQuoteDurableName,
error: message
});
}
}
} else {
try {
const info = await jsm.consumers.info(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
logger.warn("resetting consumer due to deliver policy change", {
durable: equityQuoteDurableName,
current: info.config.deliver_policy,
desired: env.COMPUTE_DELIVER_POLICY
});
await jsm.consumers.delete(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) {
logger.warn("failed to inspect jetstream consumer", {
durable: equityQuoteDurableName,
error: message
});
}
}
}
const subscription = await (async () => { const subscription = await (async () => {
const opts = buildDurableConsumer(durableName); const opts = buildDurableConsumer(durableName);
applyDeliverPolicy(opts, env.COMPUTE_DELIVER_POLICY); applyDeliverPolicy(opts, env.COMPUTE_DELIVER_POLICY);
@ -863,6 +1033,81 @@ const run = async () => {
} }
})(); })();
const equitySubscription = await (async () => {
const opts = buildDurableConsumer(equityPrintDurableName);
applyDeliverPolicy(opts, env.COMPUTE_DELIVER_POLICY);
try {
return await subscribeJson(js, SUBJECT_EQUITY_PRINTS, opts);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
const shouldReset =
message.includes("duplicate subscription") ||
message.includes("durable requires") ||
message.includes("subject does not match consumer");
if (!shouldReset) {
throw error;
}
logger.warn("resetting jetstream consumer", { durable: equityPrintDurableName, error: message });
try {
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, equityPrintDurableName);
} catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", {
durable: equityPrintDurableName,
error: deleteMessage
});
}
}
const resetOpts = buildDurableConsumer(equityPrintDurableName);
applyDeliverPolicy(resetOpts, env.COMPUTE_DELIVER_POLICY);
return await subscribeJson(js, SUBJECT_EQUITY_PRINTS, resetOpts);
}
})();
const equityQuoteSubscription = await (async () => {
const opts = buildDurableConsumer(equityQuoteDurableName);
applyDeliverPolicy(opts, env.COMPUTE_DELIVER_POLICY);
try {
return await subscribeJson(js, SUBJECT_EQUITY_QUOTES, opts);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
const shouldReset =
message.includes("duplicate subscription") ||
message.includes("durable requires") ||
message.includes("subject does not match consumer");
if (!shouldReset) {
throw error;
}
logger.warn("resetting jetstream consumer", {
durable: equityQuoteDurableName,
error: message
});
try {
await jsm.consumers.delete(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
} catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", {
durable: equityQuoteDurableName,
error: deleteMessage
});
}
}
const resetOpts = buildDurableConsumer(equityQuoteDurableName);
applyDeliverPolicy(resetOpts, env.COMPUTE_DELIVER_POLICY);
return await subscribeJson(js, SUBJECT_EQUITY_QUOTES, resetOpts);
}
})();
const nbboLoop = async () => { const nbboLoop = async () => {
for await (const msg of nbboSubscription.messages) { for await (const msg of nbboSubscription.messages) {
try { try {
@ -878,7 +1123,39 @@ const run = async () => {
} }
}; };
const equityQuoteLoop = async () => {
for await (const msg of equityQuoteSubscription.messages) {
try {
const quote = EquityQuoteSchema.parse(equityQuoteSubscription.decode(msg));
updateEquityQuoteCache(quote);
msg.ack();
} catch (error) {
logger.error("failed to process equity quote", {
error: error instanceof Error ? error.message : String(error)
});
msg.term();
}
}
};
const equityPrintLoop = async () => {
for await (const msg of equitySubscription.messages) {
try {
const print = EquityPrintSchema.parse(equitySubscription.decode(msg));
await emitEquityJoin(clickhouse, js, print);
msg.ack();
} catch (error) {
logger.error("failed to process equity print", {
error: error instanceof Error ? error.message : String(error)
});
msg.term();
}
}
};
void nbboLoop(); void nbboLoop();
void equityQuoteLoop();
void equityPrintLoop();
const shutdown = async (signal: string) => { const shutdown = async (signal: string) => {
logger.info("service stopping", { signal }); logger.info("service stopping", { signal });

View file

@ -0,0 +1,72 @@
import { describe, expect, it } from "bun:test";
import { buildEquityPrintJoin, classifyQuotePlacement } from "../src/equity-joins";
const basePrint = {
source_ts: 100,
ingest_ts: 110,
seq: 1,
trace_id: "print-1",
ts: 100,
underlying_id: "SPY",
price: 100,
size: 50,
exchange: "TEST",
offExchangeFlag: false
};
const baseQuote = {
source_ts: 95,
ingest_ts: 105,
seq: 2,
trace_id: "quote-1",
ts: 98,
underlying_id: "SPY",
bid: 99.9,
ask: 100.1
};
describe("equity join helpers", () => {
it("classifies placements with stale and missing quotes", () => {
const missing = classifyQuotePlacement(basePrint.price, {
quote: null,
ageMs: 1500,
stale: true
});
const stale = classifyQuotePlacement(basePrint.price, {
quote: baseQuote,
ageMs: 1500,
stale: true
});
expect(missing).toBe("MISSING");
expect(stale).toBe("STALE");
});
it("builds join payloads with quote features when fresh", () => {
const join = buildEquityPrintJoin(basePrint, {
quote: baseQuote,
ageMs: 5,
stale: false
});
expect(join.id).toBe("equityjoin:print-1");
expect(join.quote_trace_id).toBe("quote-1");
expect(join.join_quality.quote_age_ms).toBe(5);
expect(join.features.quote_bid).toBe(99.9);
expect(join.features.quote_ask).toBe(100.1);
expect(join.features.quote_mid).toBeCloseTo(100, 2);
expect(join.features.quote_spread).toBeCloseTo(0.2, 2);
});
it("marks missing quotes in join quality", () => {
const join = buildEquityPrintJoin(basePrint, {
quote: null,
ageMs: 2000,
stale: true
});
expect(join.quote_trace_id).toBe("");
expect(join.join_quality.quote_missing).toBe(1);
expect(join.features.quote_placement).toBe("MISSING");
});
});

View file

@ -1,4 +1,4 @@
import { SP500_SYMBOLS, type EquityPrint } from "@islandflow/types"; import { SP500_SYMBOLS, type EquityPrint, type EquityQuote } from "@islandflow/types";
import type { EquityIngestAdapter, EquityIngestHandlers } from "./types"; import type { EquityIngestAdapter, EquityIngestHandlers } from "./types";
type SyntheticEquitiesAdapterConfig = { type SyntheticEquitiesAdapterConfig = {
@ -38,6 +38,25 @@ const buildSyntheticPrint = (
}; };
}; };
const buildSyntheticQuote = (
seq: number,
now: number,
symbol: string,
bid: number,
ask: number
): EquityQuote => {
return {
source_ts: now,
ingest_ts: now,
seq,
trace_id: `synthetic-equity-quote-${seq}`,
ts: now,
underlying_id: symbol,
bid,
ask
};
};
export const createSyntheticEquitiesAdapter = ( export const createSyntheticEquitiesAdapter = (
config: SyntheticEquitiesAdapterConfig config: SyntheticEquitiesAdapterConfig
): EquityIngestAdapter => { ): EquityIngestAdapter => {
@ -45,6 +64,7 @@ export const createSyntheticEquitiesAdapter = (
name: "synthetic", name: "synthetic",
start: (handlers: EquityIngestHandlers) => { start: (handlers: EquityIngestHandlers) => {
let seq = 0; let seq = 0;
let quoteSeq = 0;
let timer: ReturnType<typeof setInterval> | null = null; let timer: ReturnType<typeof setInterval> | null = null;
let stopped = false; let stopped = false;
@ -65,8 +85,18 @@ export const createSyntheticEquitiesAdapter = (
const size = 10 + (seq % 600); const size = 10 + (seq % 600);
const exchange = EXCHANGES[(seq + symbolHash) % EXCHANGES.length]; const exchange = EXCHANGES[(seq + symbolHash) % EXCHANGES.length];
const offExchangeFlag = (seq + i) % 6 === 0; const offExchangeFlag = (seq + i) % 6 === 0;
const print = buildSyntheticPrint(seq, now + i * 4, symbol, price, size, exchange, offExchangeFlag); const eventTs = now + i * 4;
const print = buildSyntheticPrint(seq, eventTs, symbol, price, size, exchange, offExchangeFlag);
void handlers.onTrade(print); void handlers.onTrade(print);
if (handlers.onQuote) {
quoteSeq += 1;
const spread = Math.max(0.02, Number((price * 0.002).toFixed(2)));
const bid = Math.max(0.01, Number((price - spread / 2).toFixed(2)));
const ask = Math.max(bid + 0.01, Number((price + spread / 2).toFixed(2)));
const quote = buildSyntheticQuote(quoteSeq, eventTs, symbol, bid, ask);
void handlers.onQuote(quote);
}
} }
}; };

View file

@ -2,7 +2,9 @@ import { readEnv } from "@islandflow/config";
import { createLogger } from "@islandflow/observability"; import { createLogger } from "@islandflow/observability";
import { import {
SUBJECT_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS,
SUBJECT_EQUITY_QUOTES,
STREAM_EQUITY_PRINTS, STREAM_EQUITY_PRINTS,
STREAM_EQUITY_QUOTES,
connectJetStreamWithRetry, connectJetStreamWithRetry,
ensureStream, ensureStream,
publishJson publishJson
@ -10,9 +12,16 @@ import {
import { import {
createClickHouseClient, createClickHouseClient,
ensureEquityPrintsTable, ensureEquityPrintsTable,
insertEquityPrint ensureEquityQuotesTable,
insertEquityPrint,
insertEquityQuote
} from "@islandflow/storage"; } from "@islandflow/storage";
import { EquityPrintSchema, type EquityPrint } from "@islandflow/types"; import {
EquityPrintSchema,
EquityQuoteSchema,
type EquityPrint,
type EquityQuote
} from "@islandflow/types";
import { createSyntheticEquitiesAdapter } from "./adapters/synthetic"; import { createSyntheticEquitiesAdapter } from "./adapters/synthetic";
import type { EquityIngestAdapter, StopHandler } from "./adapters/types"; import type { EquityIngestAdapter, StopHandler } from "./adapters/types";
import { z } from "zod"; import { z } from "zod";
@ -136,6 +145,19 @@ const run = async () => {
num_replicas: 1 num_replicas: 1
}); });
await ensureStream(jsm, {
name: STREAM_EQUITY_QUOTES,
subjects: [SUBJECT_EQUITY_QUOTES],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
const clickhouse = createClickHouseClient({ const clickhouse = createClickHouseClient({
url: env.CLICKHOUSE_URL, url: env.CLICKHOUSE_URL,
database: env.CLICKHOUSE_DATABASE database: env.CLICKHOUSE_DATABASE
@ -143,11 +165,13 @@ const run = async () => {
await retry("clickhouse table init", 20, 500, async () => { await retry("clickhouse table init", 20, 500, async () => {
await ensureEquityPrintsTable(clickhouse); await ensureEquityPrintsTable(clickhouse);
await ensureEquityQuotesTable(clickhouse);
}); });
const adapter = selectAdapter(env.EQUITIES_INGEST_ADAPTER); const adapter = selectAdapter(env.EQUITIES_INGEST_ADAPTER);
logger.info("ingest adapter selected", { adapter: adapter.name }); logger.info("ingest adapter selected", { adapter: adapter.name });
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS); const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
const allowQuotePublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
const stopAdapter: StopHandler = await adapter.start({ const stopAdapter: StopHandler = await adapter.start({
onTrade: async (candidate: EquityPrint) => { onTrade: async (candidate: EquityPrint) => {
@ -176,6 +200,28 @@ const run = async () => {
trace_id: print.trace_id trace_id: print.trace_id
}); });
} }
},
onQuote: async (candidate: EquityQuote) => {
if (state.shuttingDown) {
return;
}
const now = Date.now();
if (!allowQuotePublish(now)) {
return;
}
const quote = EquityQuoteSchema.parse(candidate);
try {
await insertEquityQuote(clickhouse, quote);
await publishJson(js, SUBJECT_EQUITY_QUOTES, quote);
} catch (error) {
logger.error("failed to publish equity quote", {
error: error instanceof Error ? error.message : String(error),
trace_id: quote.trace_id
});
}
} }
}); });

View file

@ -8,6 +8,6 @@
"isolatedModules": true, "isolatedModules": true,
"resolveJsonModule": true, "resolveJsonModule": true,
"skipLibCheck": true, "skipLibCheck": true,
"noEmit": true "noEmit": true,
} },
} }