Add smart-money option signal path and tape filters
This commit is contained in:
parent
758f111d7e
commit
27b0a399e6
23 changed files with 1827 additions and 175 deletions
|
|
@ -10,7 +10,7 @@ import {
|
|||
SUBJECT_INFERRED_DARK,
|
||||
SUBJECT_FLOW_PACKETS,
|
||||
SUBJECT_OPTION_NBBO,
|
||||
SUBJECT_OPTION_PRINTS,
|
||||
SUBJECT_OPTION_SIGNAL_PRINTS,
|
||||
STREAM_ALERTS,
|
||||
STREAM_CLASSIFIER_HITS,
|
||||
STREAM_EQUITY_CANDLES,
|
||||
|
|
@ -20,7 +20,7 @@ import {
|
|||
STREAM_INFERRED_DARK,
|
||||
STREAM_FLOW_PACKETS,
|
||||
STREAM_OPTION_NBBO,
|
||||
STREAM_OPTION_PRINTS,
|
||||
STREAM_OPTION_SIGNAL_PRINTS,
|
||||
buildDurableConsumer,
|
||||
connectJetStreamWithRetry,
|
||||
ensureStream,
|
||||
|
|
@ -85,6 +85,13 @@ import {
|
|||
LiveServerMessage,
|
||||
LiveSubscription,
|
||||
LiveSubscriptionSchema,
|
||||
matchesFlowPacketFilters,
|
||||
matchesOptionPrintFilters,
|
||||
OptionFlowFilters,
|
||||
OptionFlowViewSchema,
|
||||
OptionNbboSideSchema,
|
||||
OptionSecurityTypeSchema,
|
||||
OptionTypeSchema,
|
||||
FlowPacketSchema,
|
||||
OptionNBBOSchema,
|
||||
OptionPrintSchema,
|
||||
|
|
@ -199,6 +206,32 @@ const equityPrintRangeSchema = z.object({
|
|||
end_ts: z.coerce.number().int().nonnegative(),
|
||||
limit: limitSchema.optional()
|
||||
});
|
||||
const optionSideListSchema = z
|
||||
.string()
|
||||
.transform((value) =>
|
||||
value
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean)
|
||||
)
|
||||
.pipe(z.array(OptionNbboSideSchema));
|
||||
const optionTypeListSchema = z
|
||||
.string()
|
||||
.transform((value) =>
|
||||
value
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean)
|
||||
)
|
||||
.pipe(z.array(OptionTypeSchema));
|
||||
const optionSecuritySchema = z.enum(["stock", "etf", "all"]);
|
||||
const optionFilterQuerySchema = z.object({
|
||||
view: OptionFlowViewSchema.optional(),
|
||||
security: optionSecuritySchema.optional(),
|
||||
side: optionSideListSchema.optional(),
|
||||
type: optionTypeListSchema.optional(),
|
||||
min_notional: z.coerce.number().nonnegative().optional()
|
||||
});
|
||||
|
||||
type Channel =
|
||||
| "options"
|
||||
|
|
@ -235,6 +268,7 @@ const classifierHitSockets = new Set<LegacySocket>();
|
|||
const alertSockets = new Set<LegacySocket>();
|
||||
const liveSocketSubscriptions = new Map<LiveSocket, Set<string>>();
|
||||
const subscriptionSockets = new Map<string, Set<LiveSocket>>();
|
||||
const subscriptionDefinitions = new Map<string, LiveSubscription>();
|
||||
const liveHeartbeats = new Map<LiveSocket, ReturnType<typeof setInterval>>();
|
||||
|
||||
const jsonResponse = (body: unknown, status = 200): Response => {
|
||||
|
|
@ -254,6 +288,43 @@ const parseLimit = (value: string | null): number => {
|
|||
return limitSchema.parse(value);
|
||||
};
|
||||
|
||||
const parseOptionPrintFilters = (
|
||||
url: URL
|
||||
): {
|
||||
view: z.infer<typeof OptionFlowViewSchema>;
|
||||
storageFilters: Parameters<typeof fetchRecentOptionPrints>[3];
|
||||
liveFilters: OptionFlowFilters;
|
||||
} => {
|
||||
const parsed = optionFilterQuerySchema.parse({
|
||||
view: url.searchParams.get("view") ?? undefined,
|
||||
security: url.searchParams.get("security") ?? undefined,
|
||||
side: url.searchParams.get("side") ?? undefined,
|
||||
type: url.searchParams.get("type") ?? undefined,
|
||||
min_notional: url.searchParams.get("min_notional") ?? undefined
|
||||
});
|
||||
const view = parsed.view ?? "signal";
|
||||
const security = parsed.security ?? (view === "raw" ? "all" : "stock");
|
||||
const storageFilters = {
|
||||
view,
|
||||
security,
|
||||
minNotional: parsed.min_notional,
|
||||
nbboSides: parsed.side,
|
||||
optionTypes: parsed.type
|
||||
} as const;
|
||||
const liveFilters: OptionFlowFilters = {
|
||||
view,
|
||||
securityTypes:
|
||||
security === "all"
|
||||
? undefined
|
||||
: ([security] as Array<z.infer<typeof OptionSecurityTypeSchema>>),
|
||||
nbboSides: parsed.side,
|
||||
optionTypes: parsed.type,
|
||||
minNotional: parsed.min_notional
|
||||
};
|
||||
|
||||
return { view, storageFilters, liveFilters };
|
||||
};
|
||||
|
||||
const parseReplayParams = (url: URL): { afterTs: number; afterSeq: number; limit: number } => {
|
||||
const params = replayParamsSchema.parse({
|
||||
after_ts: url.searchParams.get("after_ts") ?? undefined,
|
||||
|
|
@ -412,6 +483,7 @@ const subscribeSocket = (socket: LiveSocket, subscription: LiveSubscription): vo
|
|||
const sockets = subscriptionSockets.get(key) ?? new Set<LiveSocket>();
|
||||
sockets.add(socket);
|
||||
subscriptionSockets.set(key, sockets);
|
||||
subscriptionDefinitions.set(key, subscription);
|
||||
};
|
||||
|
||||
const unsubscribeSocket = (socket: LiveSocket, subscription: LiveSubscription): void => {
|
||||
|
|
@ -425,6 +497,7 @@ const unsubscribeSocket = (socket: LiveSocket, subscription: LiveSubscription):
|
|||
sockets.delete(socket);
|
||||
if (sockets.size === 0) {
|
||||
subscriptionSockets.delete(key);
|
||||
subscriptionDefinitions.delete(key);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -436,6 +509,7 @@ const cleanupLiveSocket = (socket: LiveSocket): void => {
|
|||
sockets?.delete(socket);
|
||||
if (sockets && sockets.size === 0) {
|
||||
subscriptionSockets.delete(key);
|
||||
subscriptionDefinitions.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -504,8 +578,8 @@ const run = async () => {
|
|||
);
|
||||
|
||||
await ensureStream(jsm, {
|
||||
name: STREAM_OPTION_PRINTS,
|
||||
subjects: [SUBJECT_OPTION_PRINTS],
|
||||
name: STREAM_OPTION_SIGNAL_PRINTS,
|
||||
subjects: [SUBJECT_OPTION_SIGNAL_PRINTS],
|
||||
retention: "limits",
|
||||
storage: "file",
|
||||
discard: "old",
|
||||
|
|
@ -722,8 +796,8 @@ const run = async () => {
|
|||
};
|
||||
|
||||
const optionSubscription = await subscribeWithReset(
|
||||
SUBJECT_OPTION_PRINTS,
|
||||
STREAM_OPTION_PRINTS,
|
||||
SUBJECT_OPTION_SIGNAL_PRINTS,
|
||||
STREAM_OPTION_SIGNAL_PRINTS,
|
||||
"api-option-prints"
|
||||
);
|
||||
|
||||
|
|
@ -786,20 +860,44 @@ const run = async () => {
|
|||
item: unknown,
|
||||
ingestChannel: "options" | "nbbo" | "equities" | "equity-candles" | "equity-overlay" | "equity-joins" | "flow" | "classifier-hits" | "alerts" | "inferred-dark"
|
||||
) => {
|
||||
const key = getSubscriptionKey(subscription);
|
||||
const sockets = subscriptionSockets.get(key);
|
||||
const watermark = await liveState.ingest(ingestChannel, item);
|
||||
if (!sockets || sockets.size === 0) {
|
||||
const matchingSubscriptions =
|
||||
subscription.channel === "options" || subscription.channel === "flow"
|
||||
? [...subscriptionDefinitions.entries()].filter(([, candidate]) => candidate.channel === subscription.channel)
|
||||
: [[getSubscriptionKey(subscription), subscription] as const];
|
||||
|
||||
if (matchingSubscriptions.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const socket of sockets) {
|
||||
sendLiveMessage(socket, {
|
||||
op: "event",
|
||||
subscription,
|
||||
item,
|
||||
watermark
|
||||
});
|
||||
for (const [key, candidate] of matchingSubscriptions) {
|
||||
const sockets = subscriptionSockets.get(key);
|
||||
if (!sockets || sockets.size === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
candidate.channel === "options" &&
|
||||
!matchesOptionPrintFilters(OptionPrintSchema.parse(item), candidate.filters)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
candidate.channel === "flow" &&
|
||||
!matchesFlowPacketFilters(FlowPacketSchema.parse(item), candidate.filters)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const socket of sockets) {
|
||||
sendLiveMessage(socket, {
|
||||
op: "event",
|
||||
subscription: candidate,
|
||||
item,
|
||||
watermark
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -996,10 +1094,21 @@ const run = async () => {
|
|||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/prints/options") {
|
||||
const limit = parseLimit(url.searchParams.get("limit"));
|
||||
const source = parseReplaySource(url) ?? undefined;
|
||||
const data = await fetchRecentOptionPrints(clickhouse, limit, source);
|
||||
return jsonResponse({ data });
|
||||
try {
|
||||
const limit = parseLimit(url.searchParams.get("limit"));
|
||||
const source = parseReplaySource(url) ?? undefined;
|
||||
const { storageFilters } = parseOptionPrintFilters(url);
|
||||
const data = await fetchRecentOptionPrints(clickhouse, limit, source, storageFilters);
|
||||
return jsonResponse({ data });
|
||||
} catch (error) {
|
||||
return jsonResponse(
|
||||
{
|
||||
error: "invalid options query",
|
||||
detail: error instanceof Error ? error.message : String(error)
|
||||
},
|
||||
400
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/nbbo/options") {
|
||||
|
|
@ -1105,10 +1214,28 @@ const run = async () => {
|
|||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/history/options") {
|
||||
const { beforeTs, beforeSeq, limit } = parseBeforeParams(url);
|
||||
const source = parseReplaySource(url) ?? undefined;
|
||||
const data = await fetchOptionPrintsBefore(clickhouse, beforeTs, beforeSeq, limit, source);
|
||||
return jsonResponse(buildHistoryResponse(data, (item) => ({ ts: item.ts, seq: item.seq })));
|
||||
try {
|
||||
const { beforeTs, beforeSeq, limit } = parseBeforeParams(url);
|
||||
const source = parseReplaySource(url) ?? undefined;
|
||||
const { storageFilters } = parseOptionPrintFilters(url);
|
||||
const data = await fetchOptionPrintsBefore(
|
||||
clickhouse,
|
||||
beforeTs,
|
||||
beforeSeq,
|
||||
limit,
|
||||
source,
|
||||
storageFilters
|
||||
);
|
||||
return jsonResponse(buildHistoryResponse(data, (item) => ({ ts: item.ts, seq: item.seq })));
|
||||
} catch (error) {
|
||||
return jsonResponse(
|
||||
{
|
||||
error: "invalid options history query",
|
||||
detail: error instanceof Error ? error.message : String(error)
|
||||
},
|
||||
400
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/history/nbbo") {
|
||||
|
|
@ -1183,12 +1310,30 @@ const run = async () => {
|
|||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/replay/options") {
|
||||
const { afterTs, afterSeq, limit } = parseReplayParams(url);
|
||||
const source = parseReplaySource(url) ?? undefined;
|
||||
const data = await fetchOptionPrintsAfter(clickhouse, afterTs, afterSeq, limit, source);
|
||||
const last = data.at(-1);
|
||||
const next = last ? { ts: last.ts, seq: last.seq } : null;
|
||||
return jsonResponse({ data, next });
|
||||
try {
|
||||
const { afterTs, afterSeq, limit } = parseReplayParams(url);
|
||||
const source = parseReplaySource(url) ?? undefined;
|
||||
const { storageFilters } = parseOptionPrintFilters(url);
|
||||
const data = await fetchOptionPrintsAfter(
|
||||
clickhouse,
|
||||
afterTs,
|
||||
afterSeq,
|
||||
limit,
|
||||
source,
|
||||
storageFilters
|
||||
);
|
||||
const last = data.at(-1);
|
||||
const next = last ? { ts: last.ts, seq: last.seq } : null;
|
||||
return jsonResponse({ data, next });
|
||||
} catch (error) {
|
||||
return jsonResponse(
|
||||
{
|
||||
error: "invalid options replay query",
|
||||
detail: error instanceof Error ? error.message : String(error)
|
||||
},
|
||||
400
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/replay/nbbo") {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import {
|
||||
fetchRecentOptionPrints,
|
||||
fetchRecentAlerts,
|
||||
fetchRecentClassifierHits,
|
||||
fetchRecentEquityCandles,
|
||||
|
|
@ -7,9 +8,9 @@ import {
|
|||
fetchRecentFlowPackets,
|
||||
fetchRecentInferredDark,
|
||||
fetchRecentOptionNBBO,
|
||||
fetchRecentOptionPrints,
|
||||
type ClickHouseClient
|
||||
} from "@islandflow/storage";
|
||||
import type { OptionPrintQueryFilters } from "@islandflow/storage";
|
||||
import {
|
||||
AlertEventSchema,
|
||||
ClassifierHitEventSchema,
|
||||
|
|
@ -22,8 +23,11 @@ import {
|
|||
InferredDarkEventSchema,
|
||||
LiveGenericChannel,
|
||||
LiveSubscription,
|
||||
matchesFlowPacketFilters,
|
||||
matchesOptionPrintFilters,
|
||||
OptionNBBOSchema,
|
||||
OptionPrintSchema,
|
||||
type OptionFlowFilters,
|
||||
type Cursor,
|
||||
type EquityCandle,
|
||||
type EquityPrint,
|
||||
|
|
@ -124,7 +128,8 @@ const getGenericConfig = (limits: GenericLiveLimits): {
|
|||
limit: limits.options,
|
||||
parse: (value) => OptionPrintSchema.parse(value),
|
||||
cursor: (item) => ({ ts: item.ts, seq: item.seq }),
|
||||
fetchRecent: fetchRecentOptionPrints
|
||||
fetchRecent: (clickhouse, limit) =>
|
||||
fetchRecentOptionPrints(clickhouse, limit, undefined, { view: "signal" })
|
||||
},
|
||||
nbbo: {
|
||||
redisKey: "live:nbbo",
|
||||
|
|
@ -279,6 +284,55 @@ export class LiveStateManager {
|
|||
|
||||
async getSnapshot(subscription: LiveSubscription): Promise<FeedSnapshot<unknown>> {
|
||||
switch (subscription.channel) {
|
||||
case "options": {
|
||||
if (subscription.filters?.view === "raw") {
|
||||
const storageFilters: OptionPrintQueryFilters = {
|
||||
view: "raw",
|
||||
security:
|
||||
subscription.filters.securityTypes?.length === 1
|
||||
? subscription.filters.securityTypes[0]
|
||||
: "all",
|
||||
nbboSides: subscription.filters.nbboSides,
|
||||
optionTypes: subscription.filters.optionTypes,
|
||||
minNotional: subscription.filters.minNotional
|
||||
};
|
||||
const items = await fetchRecentOptionPrints(
|
||||
this.clickhouse,
|
||||
this.generic.options.limit,
|
||||
undefined,
|
||||
storageFilters
|
||||
);
|
||||
return {
|
||||
subscription,
|
||||
items,
|
||||
watermark: items[0] ? { ts: items[0].ts, seq: items[0].seq } : null,
|
||||
next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq }))
|
||||
};
|
||||
}
|
||||
|
||||
const config = this.generic.options;
|
||||
const items = (this.genericItems.get("options") ?? []).filter((item) =>
|
||||
matchesOptionPrintFilters(item, subscription.filters)
|
||||
);
|
||||
return {
|
||||
subscription,
|
||||
items,
|
||||
watermark: this.genericCursors.get(config.cursorField) ?? null,
|
||||
next_before: nextBeforeForItems(items, config.cursor)
|
||||
};
|
||||
}
|
||||
case "flow": {
|
||||
const config = this.generic.flow;
|
||||
const items = (this.genericItems.get("flow") ?? []).filter((item) =>
|
||||
matchesFlowPacketFilters(item, subscription.filters)
|
||||
);
|
||||
return {
|
||||
subscription,
|
||||
items,
|
||||
watermark: this.genericCursors.get(config.cursorField) ?? null,
|
||||
next_before: nextBeforeForItems(items, config.cursor)
|
||||
};
|
||||
}
|
||||
case "equity-candles": {
|
||||
const key = candleRedisKey(subscription.underlying_id, subscription.interval_ms);
|
||||
const cursorField = candleCursorField(subscription.underlying_id, subscription.interval_ms);
|
||||
|
|
|
|||
|
|
@ -196,4 +196,81 @@ describe("LiveStateManager", () => {
|
|||
expect(stats.trimOperations).toBeGreaterThan(0);
|
||||
expect(stats.cacheDepthByKey["live:flow"]).toBe(2);
|
||||
});
|
||||
|
||||
it("filters option and flow snapshots using subscription filters", async () => {
|
||||
const manager = new LiveStateManager(makeClickHouse(), null);
|
||||
|
||||
await manager.ingest("options", {
|
||||
source_ts: 100,
|
||||
ingest_ts: 101,
|
||||
seq: 1,
|
||||
trace_id: "opt-1",
|
||||
ts: 100,
|
||||
option_contract_id: "AAPL-2025-01-17-200-C",
|
||||
price: 1,
|
||||
size: 100,
|
||||
exchange: "X",
|
||||
underlying_id: "AAPL",
|
||||
option_type: "call",
|
||||
notional: 10000,
|
||||
nbbo_side: "A",
|
||||
is_etf: false,
|
||||
signal_pass: true,
|
||||
signal_reasons: ["keep:ask-side"],
|
||||
signal_profile: "smart-money"
|
||||
});
|
||||
await manager.ingest("options", {
|
||||
source_ts: 110,
|
||||
ingest_ts: 111,
|
||||
seq: 2,
|
||||
trace_id: "opt-2",
|
||||
ts: 110,
|
||||
option_contract_id: "SPY-2025-01-17-500-P",
|
||||
price: 1,
|
||||
size: 100,
|
||||
exchange: "X",
|
||||
underlying_id: "SPY",
|
||||
option_type: "put",
|
||||
notional: 10000,
|
||||
nbbo_side: "B",
|
||||
is_etf: true,
|
||||
signal_pass: true,
|
||||
signal_reasons: ["keep:ask-side"],
|
||||
signal_profile: "smart-money"
|
||||
});
|
||||
await manager.ingest("flow", {
|
||||
source_ts: 120,
|
||||
ingest_ts: 121,
|
||||
seq: 3,
|
||||
trace_id: "flow-1",
|
||||
id: "flow-1",
|
||||
members: ["opt-1"],
|
||||
features: {
|
||||
option_contract_id: "AAPL-2025-01-17-200-C",
|
||||
total_notional: 10000,
|
||||
is_etf: false,
|
||||
option_type: "call",
|
||||
nbbo_a_count: 1,
|
||||
nbbo_aa_count: 0,
|
||||
nbbo_mid_count: 0,
|
||||
nbbo_b_count: 0,
|
||||
nbbo_bb_count: 0,
|
||||
nbbo_missing_count: 0,
|
||||
nbbo_stale_count: 0
|
||||
},
|
||||
join_quality: {}
|
||||
});
|
||||
|
||||
const optionSnapshot = await manager.getSnapshot({
|
||||
channel: "options",
|
||||
filters: { securityTypes: ["stock"], nbboSides: ["A"], optionTypes: ["call"] }
|
||||
});
|
||||
const flowSnapshot = await manager.getSnapshot({
|
||||
channel: "flow",
|
||||
filters: { securityTypes: ["stock"], nbboSides: ["A"], optionTypes: ["call"] }
|
||||
});
|
||||
|
||||
expect(optionSnapshot.items).toHaveLength(1);
|
||||
expect(flowSnapshot.items).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import {
|
|||
SUBJECT_INFERRED_DARK,
|
||||
SUBJECT_FLOW_PACKETS,
|
||||
SUBJECT_OPTION_NBBO,
|
||||
SUBJECT_OPTION_PRINTS,
|
||||
SUBJECT_OPTION_SIGNAL_PRINTS,
|
||||
STREAM_ALERTS,
|
||||
STREAM_CLASSIFIER_HITS,
|
||||
STREAM_EQUITY_JOINS,
|
||||
|
|
@ -18,7 +18,7 @@ import {
|
|||
STREAM_INFERRED_DARK,
|
||||
STREAM_FLOW_PACKETS,
|
||||
STREAM_OPTION_NBBO,
|
||||
STREAM_OPTION_PRINTS,
|
||||
STREAM_OPTION_SIGNAL_PRINTS,
|
||||
buildDurableConsumer,
|
||||
connectJetStreamWithRetry,
|
||||
ensureStream,
|
||||
|
|
@ -231,6 +231,9 @@ type NbboPlacementCounts = {
|
|||
|
||||
type ClusterState = {
|
||||
contractId: string;
|
||||
underlyingId: string | null;
|
||||
optionType: string | null;
|
||||
isEtf: boolean | null;
|
||||
startTs: number;
|
||||
endTs: number;
|
||||
startSourceTs: number;
|
||||
|
|
@ -530,6 +533,9 @@ const buildCluster = (print: OptionPrint): ClusterState => {
|
|||
recordPlacement(placements, classifyPlacement(print.price, selectNbbo(print.option_contract_id, print.ts)));
|
||||
return {
|
||||
contractId: print.option_contract_id,
|
||||
underlyingId: print.underlying_id ?? null,
|
||||
optionType: print.option_type ?? null,
|
||||
isEtf: typeof print.is_etf === "boolean" ? print.is_etf : null,
|
||||
startTs: print.ts,
|
||||
endTs: print.ts,
|
||||
startSourceTs: print.source_ts,
|
||||
|
|
@ -546,6 +552,15 @@ const buildCluster = (print: OptionPrint): ClusterState => {
|
|||
};
|
||||
|
||||
const updateCluster = (cluster: ClusterState, print: OptionPrint): ClusterState => {
|
||||
if (!cluster.underlyingId && print.underlying_id) {
|
||||
cluster.underlyingId = print.underlying_id;
|
||||
}
|
||||
if (!cluster.optionType && print.option_type) {
|
||||
cluster.optionType = print.option_type;
|
||||
}
|
||||
if (cluster.isEtf === null && typeof print.is_etf === "boolean") {
|
||||
cluster.isEtf = print.is_etf;
|
||||
}
|
||||
cluster.endTs = Math.max(cluster.endTs, print.ts);
|
||||
cluster.endIngestTs = Math.max(cluster.endIngestTs, print.ingest_ts);
|
||||
cluster.endSeq = Math.max(cluster.endSeq, print.seq);
|
||||
|
|
@ -705,6 +720,15 @@ const flushCluster = async (
|
|||
}
|
||||
}
|
||||
}
|
||||
if (cluster.underlyingId) {
|
||||
features.underlying_id = cluster.underlyingId;
|
||||
}
|
||||
if (cluster.optionType) {
|
||||
features.option_type = cluster.optionType;
|
||||
}
|
||||
if (cluster.isEtf !== null) {
|
||||
features.is_etf = cluster.isEtf;
|
||||
}
|
||||
|
||||
const placementTotal =
|
||||
cluster.placements.aa +
|
||||
|
|
@ -1012,8 +1036,8 @@ const run = async () => {
|
|||
);
|
||||
|
||||
await ensureStream(jsm, {
|
||||
name: STREAM_OPTION_PRINTS,
|
||||
subjects: [SUBJECT_OPTION_PRINTS],
|
||||
name: STREAM_OPTION_SIGNAL_PRINTS,
|
||||
subjects: [SUBJECT_OPTION_SIGNAL_PRINTS],
|
||||
retention: "limits",
|
||||
storage: "file",
|
||||
discard: "old",
|
||||
|
|
@ -1162,7 +1186,7 @@ const run = async () => {
|
|||
|
||||
if (env.COMPUTE_CONSUMER_RESET) {
|
||||
try {
|
||||
await jsm.consumers.delete(STREAM_OPTION_PRINTS, durableName);
|
||||
await jsm.consumers.delete(STREAM_OPTION_SIGNAL_PRINTS, durableName);
|
||||
logger.warn("reset jetstream consumer", { durable: durableName });
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
|
|
@ -1172,14 +1196,14 @@ const run = async () => {
|
|||
}
|
||||
} else {
|
||||
try {
|
||||
const info = await jsm.consumers.info(STREAM_OPTION_PRINTS, durableName);
|
||||
const info = await jsm.consumers.info(STREAM_OPTION_SIGNAL_PRINTS, durableName);
|
||||
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
|
||||
logger.warn("resetting consumer due to deliver policy change", {
|
||||
durable: durableName,
|
||||
current: info.config.deliver_policy,
|
||||
desired: env.COMPUTE_DELIVER_POLICY
|
||||
});
|
||||
await jsm.consumers.delete(STREAM_OPTION_PRINTS, durableName);
|
||||
await jsm.consumers.delete(STREAM_OPTION_SIGNAL_PRINTS, durableName);
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
|
|
@ -1292,7 +1316,7 @@ const run = async () => {
|
|||
const opts = buildDurableConsumer(durableName);
|
||||
applyDeliverPolicy(opts, env.COMPUTE_DELIVER_POLICY);
|
||||
try {
|
||||
return await subscribeJson(js, SUBJECT_OPTION_PRINTS, opts);
|
||||
return await subscribeJson(js, SUBJECT_OPTION_SIGNAL_PRINTS, opts);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const shouldReset =
|
||||
|
|
@ -1307,7 +1331,7 @@ const run = async () => {
|
|||
logger.warn("resetting jetstream consumer", { durable: durableName, error: message });
|
||||
|
||||
try {
|
||||
await jsm.consumers.delete(STREAM_OPTION_PRINTS, durableName);
|
||||
await jsm.consumers.delete(STREAM_OPTION_SIGNAL_PRINTS, durableName);
|
||||
} catch (deleteError) {
|
||||
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
if (!deleteMessage.includes("not found")) {
|
||||
|
|
@ -1320,7 +1344,7 @@ const run = async () => {
|
|||
|
||||
const resetOpts = buildDurableConsumer(durableName);
|
||||
applyDeliverPolicy(resetOpts, env.COMPUTE_DELIVER_POLICY);
|
||||
return await subscribeJson(js, SUBJECT_OPTION_PRINTS, resetOpts);
|
||||
return await subscribeJson(js, SUBJECT_OPTION_SIGNAL_PRINTS, resetOpts);
|
||||
}
|
||||
})();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
import { SP500_SYMBOLS, type EquityPrint, type EquityQuote } from "@islandflow/types";
|
||||
import {
|
||||
SP500_SYMBOLS,
|
||||
type EquityPrint,
|
||||
type EquityQuote,
|
||||
type SyntheticMarketMode
|
||||
} from "@islandflow/types";
|
||||
import type { EquityIngestAdapter, EquityIngestHandlers } from "./types";
|
||||
|
||||
type SyntheticEquitiesAdapterConfig = {
|
||||
emitIntervalMs: number;
|
||||
mode: SyntheticMarketMode;
|
||||
};
|
||||
|
||||
const EXCHANGES = ["NYSE", "NASDAQ", "ARCA", "BATS", "IEX", "TEST"];
|
||||
|
|
@ -22,10 +28,7 @@ const DARK_SEQUENCE: DarkScenario[] = [
|
|||
"sell",
|
||||
"sell"
|
||||
];
|
||||
const SYNTHETIC_SYMBOLS = [
|
||||
"SPY",
|
||||
...SP500_SYMBOLS.filter((symbol) => symbol !== "SPY")
|
||||
];
|
||||
const SYNTHETIC_SYMBOLS = ["SPY", ...(SP500_SYMBOLS as readonly string[])];
|
||||
|
||||
const hashSymbol = (value: string): number => {
|
||||
let hash = 0;
|
||||
|
|
@ -124,6 +127,30 @@ const priceForPlacement = (
|
|||
export const createSyntheticEquitiesAdapter = (
|
||||
config: SyntheticEquitiesAdapterConfig
|
||||
): EquityIngestAdapter => {
|
||||
const profile =
|
||||
config.mode === "firehose"
|
||||
? {
|
||||
batchSize: 10,
|
||||
darkEvery: true,
|
||||
offExchangeMod: 2,
|
||||
litSizeBase: 40,
|
||||
litSizeRange: 1400
|
||||
}
|
||||
: config.mode === "active"
|
||||
? {
|
||||
batchSize: 5,
|
||||
darkEvery: true,
|
||||
offExchangeMod: 4,
|
||||
litSizeBase: 20,
|
||||
litSizeRange: 900
|
||||
}
|
||||
: {
|
||||
batchSize: 2,
|
||||
darkEvery: false,
|
||||
offExchangeMod: 8,
|
||||
litSizeBase: 10,
|
||||
litSizeRange: 300
|
||||
};
|
||||
return {
|
||||
name: "synthetic",
|
||||
start: (handlers: EquityIngestHandlers) => {
|
||||
|
|
@ -140,7 +167,7 @@ export const createSyntheticEquitiesAdapter = (
|
|||
}
|
||||
|
||||
const now = Date.now();
|
||||
const batchSize = 3;
|
||||
const batchSize = profile.batchSize;
|
||||
|
||||
const darkSymbol = SYNTHETIC_SYMBOLS[darkSymbolIndex % SYNTHETIC_SYMBOLS.length];
|
||||
const darkHash = hashSymbol(darkSymbol);
|
||||
|
|
@ -151,44 +178,46 @@ export const createSyntheticEquitiesAdapter = (
|
|||
const scenario = DARK_SEQUENCE[darkStep % DARK_SEQUENCE.length];
|
||||
const darkTs = now;
|
||||
|
||||
if (handlers.onQuote) {
|
||||
quoteSeq += 1;
|
||||
const quoteEvent = buildSyntheticQuote(
|
||||
quoteSeq,
|
||||
darkTs - 2,
|
||||
if (profile.darkEvery) {
|
||||
if (handlers.onQuote) {
|
||||
quoteSeq += 1;
|
||||
const quoteEvent = buildSyntheticQuote(
|
||||
quoteSeq,
|
||||
darkTs - 2,
|
||||
darkSymbol,
|
||||
darkQuote.bid,
|
||||
darkQuote.ask
|
||||
);
|
||||
void handlers.onQuote(quoteEvent);
|
||||
}
|
||||
|
||||
seq += 1;
|
||||
let darkPlacement: PricePlacement = "MID";
|
||||
let darkSize = config.mode === "firehose" ? 4000 : 2600;
|
||||
if (scenario === "buy") {
|
||||
darkPlacement = darkStep % 2 === 0 ? "A" : "AA";
|
||||
darkSize = config.mode === "firehose" ? 1500 : 800;
|
||||
} else if (scenario === "sell") {
|
||||
darkPlacement = darkStep % 2 === 0 ? "B" : "BB";
|
||||
darkSize = config.mode === "firehose" ? 1500 : 800;
|
||||
}
|
||||
const darkPrice = priceForPlacement(darkMid, darkQuote, darkPlacement);
|
||||
const darkPrint = buildSyntheticPrint(
|
||||
seq,
|
||||
darkTs,
|
||||
darkSymbol,
|
||||
darkQuote.bid,
|
||||
darkQuote.ask
|
||||
darkPrice,
|
||||
darkSize,
|
||||
DARK_EXCHANGE,
|
||||
true
|
||||
);
|
||||
void handlers.onQuote(quoteEvent);
|
||||
}
|
||||
void handlers.onTrade(darkPrint);
|
||||
|
||||
seq += 1;
|
||||
let darkPlacement: PricePlacement = "MID";
|
||||
let darkSize = 2600;
|
||||
if (scenario === "buy") {
|
||||
darkPlacement = darkStep % 2 === 0 ? "A" : "AA";
|
||||
darkSize = 800;
|
||||
} else if (scenario === "sell") {
|
||||
darkPlacement = darkStep % 2 === 0 ? "B" : "BB";
|
||||
darkSize = 800;
|
||||
}
|
||||
const darkPrice = priceForPlacement(darkMid, darkQuote, darkPlacement);
|
||||
const darkPrint = buildSyntheticPrint(
|
||||
seq,
|
||||
darkTs,
|
||||
darkSymbol,
|
||||
darkPrice,
|
||||
darkSize,
|
||||
DARK_EXCHANGE,
|
||||
true
|
||||
);
|
||||
void handlers.onTrade(darkPrint);
|
||||
|
||||
darkStep += 1;
|
||||
if (darkStep >= DARK_SEQUENCE.length) {
|
||||
darkStep = 0;
|
||||
darkSymbolIndex += 1;
|
||||
darkStep += 1;
|
||||
if (darkStep >= DARK_SEQUENCE.length) {
|
||||
darkStep = 0;
|
||||
darkSymbolIndex += 1;
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < batchSize; i += 1) {
|
||||
|
|
@ -201,9 +230,9 @@ export const createSyntheticEquitiesAdapter = (
|
|||
const placement: PricePlacement =
|
||||
seq % 11 === 0 ? "A" : seq % 13 === 0 ? "B" : "MID";
|
||||
const price = priceForPlacement(mid, quote, placement);
|
||||
const size = 10 + (seq % 600);
|
||||
const size = profile.litSizeBase + (seq % profile.litSizeRange);
|
||||
const exchange = EXCHANGES[(seq + symbolHash) % EXCHANGES.length];
|
||||
const offExchangeFlag = (seq + i) % 6 === 0;
|
||||
const offExchangeFlag = (seq + i) % profile.offExchangeMod === 0;
|
||||
const eventTs = now + i * 4;
|
||||
|
||||
if (handlers.onQuote) {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import {
|
|||
import {
|
||||
EquityPrintSchema,
|
||||
EquityQuoteSchema,
|
||||
resolveSyntheticMarketModes,
|
||||
type EquityPrint,
|
||||
type EquityQuote
|
||||
} from "@islandflow/types";
|
||||
|
|
@ -36,6 +37,8 @@ const envSchema = z.object({
|
|||
CLICKHOUSE_DATABASE: z.string().default("default"),
|
||||
EQUITIES_INGEST_ADAPTER: z.string().min(1).default("synthetic"),
|
||||
EMIT_INTERVAL_MS: z.coerce.number().int().positive().default(1000),
|
||||
SYNTHETIC_MARKET_MODE: z.string().default("realistic"),
|
||||
SYNTHETIC_EQUITIES_MODE: z.string().default(""),
|
||||
|
||||
// Alpaca (equities)
|
||||
ALPACA_KEY_ID: z.string().default(""),
|
||||
|
|
@ -63,6 +66,10 @@ const envSchema = z.object({
|
|||
});
|
||||
|
||||
const env = readEnv(envSchema);
|
||||
const syntheticModes = resolveSyntheticMarketModes({
|
||||
syntheticMarketMode: env.SYNTHETIC_MARKET_MODE,
|
||||
syntheticEquitiesMode: env.SYNTHETIC_EQUITIES_MODE
|
||||
});
|
||||
|
||||
const state = {
|
||||
shuttingDown: false,
|
||||
|
|
@ -153,7 +160,10 @@ const parseSymbolList = (value: string): string[] => {
|
|||
|
||||
const selectAdapter = (name: string): EquityIngestAdapter => {
|
||||
if (name === "synthetic") {
|
||||
return createSyntheticEquitiesAdapter({ emitIntervalMs: env.EMIT_INTERVAL_MS });
|
||||
return createSyntheticEquitiesAdapter({
|
||||
emitIntervalMs: env.EMIT_INTERVAL_MS,
|
||||
mode: syntheticModes.equities
|
||||
});
|
||||
}
|
||||
|
||||
if (name === "alpaca") {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
import { SP500_SYMBOLS, type OptionNBBO, type OptionPrint } from "@islandflow/types";
|
||||
import {
|
||||
SP500_SYMBOLS,
|
||||
type OptionNBBO,
|
||||
type OptionPrint,
|
||||
type SyntheticMarketMode
|
||||
} from "@islandflow/types";
|
||||
import type { OptionIngestAdapter, OptionIngestHandlers } from "./types";
|
||||
|
||||
type SyntheticOptionsAdapterConfig = {
|
||||
emitIntervalMs: number;
|
||||
mode: SyntheticMarketMode;
|
||||
};
|
||||
|
||||
type Burst = {
|
||||
|
|
@ -17,17 +23,18 @@ type Burst = {
|
|||
seed: number;
|
||||
};
|
||||
|
||||
const SYNTHETIC_SYMBOLS = [
|
||||
"SPY",
|
||||
...SP500_SYMBOLS.filter((symbol) => symbol !== "SPY")
|
||||
];
|
||||
const SYNTHETIC_SYMBOLS = ["SPY", ...(SP500_SYMBOLS as readonly string[])];
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
||||
const EXPIRY_OFFSETS = [0, 1, 7, 14, 28, 45, 60, 90];
|
||||
const EXCHANGES = ["CBOE", "PHLX", "ISE", "ARCA", "BOX", "MIAX"];
|
||||
const CONDITIONS = ["SWEEP", "ISO", "FILL", "TEST"];
|
||||
const BURST_RUN_RANGE: [number, number] = [2, 4];
|
||||
type SyntheticOptionsProfile = {
|
||||
burstRunRange: [number, number];
|
||||
scenarios: Scenario[];
|
||||
pricePlacements: Record<string, WeightedValue<PricePlacement>[]>;
|
||||
};
|
||||
|
||||
type PricePlacement = "AA" | "A" | "B" | "BB";
|
||||
type PricePlacement = "AA" | "A" | "MID" | "B" | "BB";
|
||||
|
||||
type WeightedValue<T> = {
|
||||
value: T;
|
||||
|
|
@ -45,7 +52,70 @@ type Scenario = {
|
|||
conditions?: string[];
|
||||
};
|
||||
|
||||
const SCENARIOS: Scenario[] = [
|
||||
const REALISTIC_SCENARIOS: Scenario[] = [
|
||||
{
|
||||
id: "ask_lift",
|
||||
weight: 18,
|
||||
right: "either",
|
||||
countRange: [1, 2],
|
||||
sizeRange: [30, 180],
|
||||
premiumRange: [9_000, 35_000],
|
||||
priceTrend: "flat",
|
||||
conditions: ["FILL"]
|
||||
},
|
||||
{
|
||||
id: "mid_block",
|
||||
weight: 14,
|
||||
right: "either",
|
||||
countRange: [1, 2],
|
||||
sizeRange: [120, 480],
|
||||
premiumRange: [12_000, 45_000],
|
||||
priceTrend: "flat",
|
||||
conditions: ["FILL"]
|
||||
},
|
||||
{
|
||||
id: "bullish_sweep",
|
||||
weight: 8,
|
||||
right: "C",
|
||||
countRange: [2, 3],
|
||||
sizeRange: [180, 520],
|
||||
premiumRange: [25_000, 90_000],
|
||||
priceTrend: "up",
|
||||
conditions: ["SWEEP"]
|
||||
},
|
||||
{
|
||||
id: "bearish_sweep",
|
||||
weight: 8,
|
||||
right: "P",
|
||||
countRange: [2, 3],
|
||||
sizeRange: [180, 520],
|
||||
premiumRange: [25_000, 90_000],
|
||||
priceTrend: "up",
|
||||
conditions: ["SWEEP"]
|
||||
},
|
||||
{
|
||||
id: "contract_spike",
|
||||
weight: 6,
|
||||
right: "either",
|
||||
countRange: [2, 3],
|
||||
sizeRange: [500, 900],
|
||||
premiumRange: [18_000, 70_000],
|
||||
priceTrend: "flat",
|
||||
conditions: ["ISO"]
|
||||
},
|
||||
{
|
||||
id: "noise",
|
||||
weight: 46,
|
||||
right: "either",
|
||||
countRange: [1, 2],
|
||||
sizeRange: [5, 60],
|
||||
premiumRange: [500, 6_000],
|
||||
priceTrend: "flat",
|
||||
conditions: ["FILL"]
|
||||
}
|
||||
];
|
||||
|
||||
const ACTIVE_SCENARIOS: Scenario[] = [
|
||||
{
|
||||
id: "bullish_sweep",
|
||||
weight: 35,
|
||||
|
|
@ -88,7 +158,50 @@ const SCENARIOS: Scenario[] = [
|
|||
}
|
||||
];
|
||||
|
||||
const PRICE_PLACEMENTS: Record<string, WeightedValue<PricePlacement>[]> = {
|
||||
const REALISTIC_PRICE_PLACEMENTS: Record<string, WeightedValue<PricePlacement>[]> = {
|
||||
ask_lift: [
|
||||
{ value: "A", weight: 45 },
|
||||
{ value: "AA", weight: 20 },
|
||||
{ value: "MID", weight: 25 },
|
||||
{ value: "B", weight: 8 },
|
||||
{ value: "BB", weight: 2 }
|
||||
],
|
||||
mid_block: [
|
||||
{ value: "MID", weight: 60 },
|
||||
{ value: "A", weight: 20 },
|
||||
{ value: "B", weight: 20 }
|
||||
],
|
||||
bullish_sweep: [
|
||||
{ value: "AA", weight: 20 },
|
||||
{ value: "A", weight: 50 },
|
||||
{ value: "MID", weight: 15 },
|
||||
{ value: "B", weight: 10 },
|
||||
{ value: "BB", weight: 5 }
|
||||
],
|
||||
bearish_sweep: [
|
||||
{ value: "AA", weight: 10 },
|
||||
{ value: "A", weight: 20 },
|
||||
{ value: "MID", weight: 15 },
|
||||
{ value: "B", weight: 35 },
|
||||
{ value: "BB", weight: 20 }
|
||||
],
|
||||
contract_spike: [
|
||||
{ value: "A", weight: 25 },
|
||||
{ value: "MID", weight: 40 },
|
||||
{ value: "B", weight: 25 },
|
||||
{ value: "AA", weight: 5 },
|
||||
{ value: "BB", weight: 5 }
|
||||
],
|
||||
noise: [
|
||||
{ value: "MID", weight: 40 },
|
||||
{ value: "A", weight: 20 },
|
||||
{ value: "B", weight: 20 },
|
||||
{ value: "AA", weight: 10 },
|
||||
{ value: "BB", weight: 10 }
|
||||
]
|
||||
};
|
||||
|
||||
const ACTIVE_PRICE_PLACEMENTS: Record<string, WeightedValue<PricePlacement>[]> = {
|
||||
bullish_sweep: [
|
||||
{ value: "AA", weight: 25 },
|
||||
{ value: "A", weight: 40 },
|
||||
|
|
@ -115,7 +228,52 @@ const PRICE_PLACEMENTS: Record<string, WeightedValue<PricePlacement>[]> = {
|
|||
]
|
||||
};
|
||||
|
||||
const PLACEMENT_PATTERN: PricePlacement[] = ["A", "AA", "B", "BB"];
|
||||
const FIREHOSE_PRICE_PLACEMENTS: Record<string, WeightedValue<PricePlacement>[]> = {
|
||||
...ACTIVE_PRICE_PLACEMENTS,
|
||||
noise: [
|
||||
{ value: "A", weight: 20 },
|
||||
{ value: "AA", weight: 20 },
|
||||
{ value: "MID", weight: 20 },
|
||||
{ value: "B", weight: 20 },
|
||||
{ value: "BB", weight: 20 }
|
||||
]
|
||||
};
|
||||
|
||||
const PLACEMENT_PATTERN: PricePlacement[] = ["A", "AA", "MID", "B", "BB"];
|
||||
|
||||
const SYNTHETIC_PROFILES: Record<SyntheticMarketMode, SyntheticOptionsProfile> = {
|
||||
realistic: {
|
||||
burstRunRange: [1, 2],
|
||||
scenarios: REALISTIC_SCENARIOS,
|
||||
pricePlacements: REALISTIC_PRICE_PLACEMENTS
|
||||
},
|
||||
active: {
|
||||
burstRunRange: [2, 4],
|
||||
scenarios: ACTIVE_SCENARIOS,
|
||||
pricePlacements: ACTIVE_PRICE_PLACEMENTS
|
||||
},
|
||||
firehose: {
|
||||
burstRunRange: [4, 7],
|
||||
scenarios: ACTIVE_SCENARIOS.map((scenario): Scenario =>
|
||||
scenario.id === "noise"
|
||||
? {
|
||||
...scenario,
|
||||
weight: 20,
|
||||
countRange: [5, 8],
|
||||
sizeRange: [20, 300],
|
||||
premiumRange: [800, 12_000]
|
||||
}
|
||||
: {
|
||||
...scenario,
|
||||
weight: scenario.weight + 10,
|
||||
countRange: [scenario.countRange[0] + 2, scenario.countRange[1] + 3],
|
||||
sizeRange: [scenario.sizeRange[0], scenario.sizeRange[1] * 2],
|
||||
premiumRange: [scenario.premiumRange[0], scenario.premiumRange[1] * 1.5]
|
||||
}
|
||||
),
|
||||
pricePlacements: FIREHOSE_PRICE_PLACEMENTS
|
||||
}
|
||||
};
|
||||
|
||||
const pick = <T,>(items: T[], seed: number): T => {
|
||||
return items[Math.abs(seed) % items.length];
|
||||
|
|
@ -153,8 +311,12 @@ const pickWeightedValue = <T>(items: WeightedValue<T>[], seed: number): T => {
|
|||
return pickWeighted(items, seed).value;
|
||||
};
|
||||
|
||||
const pickPlacement = (burst: Burst, index: number): PricePlacement => {
|
||||
const placementOptions = PRICE_PLACEMENTS[burst.scenarioId] ?? PRICE_PLACEMENTS.noise;
|
||||
const pickPlacement = (
|
||||
burst: Burst,
|
||||
index: number,
|
||||
profile: SyntheticOptionsProfile
|
||||
): PricePlacement => {
|
||||
const placementOptions = profile.pricePlacements[burst.scenarioId] ?? profile.pricePlacements.noise;
|
||||
const offset = Math.abs(burst.seed) % PLACEMENT_PATTERN.length;
|
||||
if (index < PLACEMENT_PATTERN.length) {
|
||||
return PLACEMENT_PATTERN[(offset + index) % PLACEMENT_PATTERN.length];
|
||||
|
|
@ -180,11 +342,11 @@ const formatExpiry = (now: number, offsetDays: number): string => {
|
|||
return expiryDate.toISOString().slice(0, 10);
|
||||
};
|
||||
|
||||
const buildBurst = (burstIndex: number, now: number): Burst => {
|
||||
const buildBurst = (burstIndex: number, now: number, profile: SyntheticOptionsProfile): Burst => {
|
||||
const symbol = SYNTHETIC_SYMBOLS[burstIndex % SYNTHETIC_SYMBOLS.length];
|
||||
const symbolHash = hashSymbol(symbol);
|
||||
const seed = symbolHash + burstIndex * 7;
|
||||
const scenario = pickWeighted(SCENARIOS, seed);
|
||||
const scenario = pickWeighted(profile.scenarios, seed);
|
||||
const baseUnderlying = 30 + (symbolHash % 470);
|
||||
const expiryOffset = pick(EXPIRY_OFFSETS, symbolHash + burstIndex);
|
||||
const expiry = formatExpiry(now, expiryOffset);
|
||||
|
|
@ -231,6 +393,7 @@ const buildBurst = (burstIndex: number, now: number): Burst => {
|
|||
export const createSyntheticOptionsAdapter = (
|
||||
config: SyntheticOptionsAdapterConfig
|
||||
): OptionIngestAdapter => {
|
||||
const profile = SYNTHETIC_PROFILES[config.mode];
|
||||
return {
|
||||
name: "synthetic",
|
||||
start: (handlers: OptionIngestHandlers) => {
|
||||
|
|
@ -250,8 +413,12 @@ export const createSyntheticOptionsAdapter = (
|
|||
const now = Date.now();
|
||||
if (!currentBurst || remainingRuns <= 0) {
|
||||
burstIndex += 1;
|
||||
currentBurst = buildBurst(burstIndex, now);
|
||||
remainingRuns = pickInt(BURST_RUN_RANGE[0], BURST_RUN_RANGE[1], burstIndex * 23);
|
||||
currentBurst = buildBurst(burstIndex, now, profile);
|
||||
remainingRuns = pickInt(
|
||||
profile.burstRunRange[0],
|
||||
profile.burstRunRange[1],
|
||||
burstIndex * 23
|
||||
);
|
||||
}
|
||||
|
||||
const burst = currentBurst;
|
||||
|
|
@ -267,13 +434,15 @@ export const createSyntheticOptionsAdapter = (
|
|||
const bid = Math.max(0.01, Number((mid - spread / 2).toFixed(2)));
|
||||
const ask = Math.max(bid + 0.01, Number((mid + spread / 2).toFixed(2)));
|
||||
const tick = Math.max(0.01, Number((spread * 0.25).toFixed(2)));
|
||||
const placement = pickPlacement(burst, i);
|
||||
const placement = pickPlacement(burst, i, profile);
|
||||
let tradePrice = mid;
|
||||
|
||||
if (placement === "AA") {
|
||||
tradePrice = ask + tick;
|
||||
} else if (placement === "A") {
|
||||
tradePrice = ask;
|
||||
} else if (placement === "MID") {
|
||||
tradePrice = mid;
|
||||
} else if (placement === "BB") {
|
||||
tradePrice = Math.max(0.01, bid - tick);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@ import { createLogger } from "@islandflow/observability";
|
|||
import {
|
||||
SUBJECT_OPTION_NBBO,
|
||||
SUBJECT_OPTION_PRINTS,
|
||||
SUBJECT_OPTION_SIGNAL_PRINTS,
|
||||
STREAM_OPTION_NBBO,
|
||||
STREAM_OPTION_PRINTS,
|
||||
STREAM_OPTION_SIGNAL_PRINTS,
|
||||
connectJetStreamWithRetry,
|
||||
ensureStream,
|
||||
publishJson
|
||||
|
|
@ -16,7 +18,16 @@ import {
|
|||
insertOptionNBBO,
|
||||
insertOptionPrint
|
||||
} from "@islandflow/storage";
|
||||
import { OptionNBBOSchema, OptionPrintSchema, type OptionNBBO, type OptionPrint } from "@islandflow/types";
|
||||
import {
|
||||
OptionNBBOSchema,
|
||||
OptionPrintSchema,
|
||||
evaluateOptionSignal,
|
||||
deriveOptionPrintMetadata,
|
||||
resolveSyntheticMarketModes,
|
||||
type OptionNBBO,
|
||||
type OptionPrint,
|
||||
type OptionsSignalConfig
|
||||
} from "@islandflow/types";
|
||||
import { createAlpacaOptionsAdapter } from "./adapters/alpaca";
|
||||
import { createDatabentoOptionsAdapter } from "./adapters/databento";
|
||||
import { createIbkrOptionsAdapter } from "./adapters/ibkr";
|
||||
|
|
@ -68,6 +79,17 @@ const envSchema = z.object({
|
|||
IBKR_CURRENCY: z.string().min(1).default("USD"),
|
||||
IBKR_PYTHON_BIN: z.string().min(1).default("python3"),
|
||||
EMIT_INTERVAL_MS: z.coerce.number().int().positive().default(1000),
|
||||
SYNTHETIC_MARKET_MODE: z.string().default("realistic"),
|
||||
SYNTHETIC_OPTIONS_MODE: z.string().default(""),
|
||||
OPTIONS_SIGNAL_MODE: z.enum(["smart-money", "balanced", "all"]).default("smart-money"),
|
||||
OPTIONS_SIGNAL_MIN_NOTIONAL: z.coerce.number().nonnegative().default(10_000),
|
||||
OPTIONS_SIGNAL_ETF_MIN_NOTIONAL: z.coerce.number().nonnegative().default(50_000),
|
||||
OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL: z.coerce.number().nonnegative().default(25_000),
|
||||
OPTIONS_SIGNAL_MID_MIN_NOTIONAL: z.coerce.number().nonnegative().default(20_000),
|
||||
OPTIONS_SIGNAL_NBBO_MAX_AGE_MS: z.coerce.number().int().positive().default(1500),
|
||||
OPTIONS_SIGNAL_ETF_UNDERLYINGS: z
|
||||
.string()
|
||||
.default("SPY,QQQ,IWM,DIA,TLT,GLD,SLV,XLF,XLE,XLV,XLI,XLP,XLU,XLY,SMH,ARKK"),
|
||||
TESTING_MODE: z
|
||||
.preprocess((value) => {
|
||||
if (typeof value === "string") {
|
||||
|
|
@ -86,11 +108,34 @@ const envSchema = z.object({
|
|||
});
|
||||
|
||||
const env = readEnv(envSchema);
|
||||
const syntheticModes = resolveSyntheticMarketModes({
|
||||
syntheticMarketMode: env.SYNTHETIC_MARKET_MODE,
|
||||
syntheticOptionsMode: env.SYNTHETIC_OPTIONS_MODE
|
||||
});
|
||||
const optionsSignalConfig: OptionsSignalConfig = {
|
||||
mode: env.OPTIONS_SIGNAL_MODE,
|
||||
minNotional: env.OPTIONS_SIGNAL_MIN_NOTIONAL,
|
||||
etfMinNotional: env.OPTIONS_SIGNAL_ETF_MIN_NOTIONAL,
|
||||
bidSideMinNotional: env.OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL,
|
||||
midMinNotional: env.OPTIONS_SIGNAL_MID_MIN_NOTIONAL,
|
||||
missingNbboMinNotional: 50_000,
|
||||
largePrintMinSize: 500,
|
||||
largePrintMinNotional: env.OPTIONS_SIGNAL_MIN_NOTIONAL,
|
||||
sweepMinNotional: env.OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL,
|
||||
autoKeepMinNotional: 100_000,
|
||||
nbboMaxAgeMs: env.OPTIONS_SIGNAL_NBBO_MAX_AGE_MS,
|
||||
etfUnderlyings: new Set(
|
||||
env.OPTIONS_SIGNAL_ETF_UNDERLYINGS.split(",")
|
||||
.map((value) => value.trim().toUpperCase())
|
||||
.filter(Boolean)
|
||||
)
|
||||
};
|
||||
|
||||
const state = {
|
||||
shuttingDown: false,
|
||||
shutdownPromise: null as Promise<void> | null
|
||||
};
|
||||
const latestNbboByContract = new Map<string, OptionNBBO>();
|
||||
|
||||
const getErrorMessage = (error: unknown): string => {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
|
|
@ -169,7 +214,10 @@ const retry = async <T>(
|
|||
|
||||
const selectAdapter = (name: string): OptionIngestAdapter => {
|
||||
if (name === "synthetic") {
|
||||
return createSyntheticOptionsAdapter({ emitIntervalMs: env.EMIT_INTERVAL_MS });
|
||||
return createSyntheticOptionsAdapter({
|
||||
emitIntervalMs: env.EMIT_INTERVAL_MS,
|
||||
mode: syntheticModes.options
|
||||
});
|
||||
}
|
||||
|
||||
if (name === "alpaca") {
|
||||
|
|
@ -277,6 +325,19 @@ const run = async () => {
|
|||
num_replicas: 1
|
||||
});
|
||||
|
||||
await ensureStream(jsm, {
|
||||
name: STREAM_OPTION_SIGNAL_PRINTS,
|
||||
subjects: [SUBJECT_OPTION_SIGNAL_PRINTS],
|
||||
retention: "limits",
|
||||
storage: "file",
|
||||
discard: "old",
|
||||
max_msgs_per_subject: -1,
|
||||
max_msgs: -1,
|
||||
max_bytes: -1,
|
||||
max_age: 0,
|
||||
num_replicas: 1
|
||||
});
|
||||
|
||||
const clickhouse = createClickHouseClient({
|
||||
url: env.CLICKHOUSE_URL,
|
||||
database: env.CLICKHOUSE_DATABASE
|
||||
|
|
@ -303,15 +364,41 @@ const run = async () => {
|
|||
return;
|
||||
}
|
||||
|
||||
const print = OptionPrintSchema.parse(candidate);
|
||||
const rawPrint = OptionPrintSchema.parse(candidate);
|
||||
const derived = deriveOptionPrintMetadata(
|
||||
rawPrint,
|
||||
latestNbboByContract.get(rawPrint.option_contract_id),
|
||||
optionsSignalConfig
|
||||
);
|
||||
const signalDecision = evaluateOptionSignal(
|
||||
{
|
||||
...rawPrint,
|
||||
...derived,
|
||||
signal_profile: optionsSignalConfig.mode
|
||||
},
|
||||
optionsSignalConfig
|
||||
);
|
||||
const print = OptionPrintSchema.parse({
|
||||
...rawPrint,
|
||||
...derived,
|
||||
signal_pass: signalDecision.signalPass,
|
||||
signal_reasons: signalDecision.signalReasons,
|
||||
signal_profile: signalDecision.signalProfile
|
||||
});
|
||||
|
||||
try {
|
||||
await insertOptionPrint(clickhouse, print);
|
||||
await publishJson(js, SUBJECT_OPTION_PRINTS, print);
|
||||
if (print.signal_pass) {
|
||||
await publishJson(js, SUBJECT_OPTION_SIGNAL_PRINTS, print);
|
||||
}
|
||||
logger.info("published option print", {
|
||||
trace_id: print.trace_id,
|
||||
seq: print.seq,
|
||||
option_contract_id: print.option_contract_id
|
||||
option_contract_id: print.option_contract_id,
|
||||
signal_pass: print.signal_pass,
|
||||
nbbo_side: print.nbbo_side,
|
||||
notional: print.notional
|
||||
});
|
||||
} catch (error) {
|
||||
if (isExpectedShutdownError(error)) {
|
||||
|
|
@ -335,6 +422,14 @@ const run = async () => {
|
|||
}
|
||||
|
||||
const nbbo = OptionNBBOSchema.parse(candidate);
|
||||
const existing = latestNbboByContract.get(nbbo.option_contract_id);
|
||||
if (
|
||||
!existing ||
|
||||
nbbo.ts > existing.ts ||
|
||||
(nbbo.ts === existing.ts && nbbo.seq >= existing.seq)
|
||||
) {
|
||||
latestNbboByContract.set(nbbo.option_contract_id, nbbo);
|
||||
}
|
||||
|
||||
try {
|
||||
await insertOptionNBBO(clickhouse, nbbo);
|
||||
|
|
|
|||
|
|
@ -5,10 +5,12 @@ import {
|
|||
SUBJECT_EQUITY_QUOTES,
|
||||
SUBJECT_OPTION_NBBO,
|
||||
SUBJECT_OPTION_PRINTS,
|
||||
SUBJECT_OPTION_SIGNAL_PRINTS,
|
||||
STREAM_EQUITY_PRINTS,
|
||||
STREAM_EQUITY_QUOTES,
|
||||
STREAM_OPTION_NBBO,
|
||||
STREAM_OPTION_PRINTS,
|
||||
STREAM_OPTION_SIGNAL_PRINTS,
|
||||
connectJetStreamWithRetry,
|
||||
ensureStream,
|
||||
publishJson
|
||||
|
|
@ -304,6 +306,9 @@ const run = async () => {
|
|||
const def = STREAM_DEFS[kind];
|
||||
await ensureStream(jsm, buildStreamConfig(def.streamName, def.subject));
|
||||
}
|
||||
if (streamKinds.includes("options")) {
|
||||
await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS));
|
||||
}
|
||||
|
||||
const clickhouse = createClickHouseClient({
|
||||
url: env.CLICKHOUSE_URL,
|
||||
|
|
@ -411,6 +416,9 @@ const run = async () => {
|
|||
|
||||
try {
|
||||
await publishJson(js, stream.subject, event);
|
||||
if (stream.kind === "options" && (event as OptionPrint).signal_pass) {
|
||||
await publishJson(js, SUBJECT_OPTION_SIGNAL_PRINTS, event as OptionPrint);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error("failed to publish replay event", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue