Add NBBO persistence, API/WS streaming, and UI context
This commit is contained in:
parent
15fce370ef
commit
fc7065792f
12 changed files with 768 additions and 46 deletions
|
|
@ -52,7 +52,9 @@ TESTING_THROTTLE_MS=200
|
||||||
# Compute consumer behavior
|
# Compute consumer behavior
|
||||||
COMPUTE_DELIVER_POLICY=new
|
COMPUTE_DELIVER_POLICY=new
|
||||||
COMPUTE_CONSUMER_RESET=false
|
COMPUTE_CONSUMER_RESET=false
|
||||||
CLASSIFIER_SWEEP_MIN_PREMIUM=50000
|
NBBO_MAX_AGE_MS=1000
|
||||||
|
NEXT_PUBLIC_NBBO_MAX_AGE_MS=1000
|
||||||
|
CLASSIFIER_SWEEP_MIN_PREMIUM=40000
|
||||||
CLASSIFIER_SWEEP_MIN_COUNT=3
|
CLASSIFIER_SWEEP_MIN_COUNT=3
|
||||||
CLASSIFIER_SPIKE_MIN_PREMIUM=25000
|
CLASSIFIER_SPIKE_MIN_PREMIUM=20000
|
||||||
CLASSIFIER_SPIKE_MIN_SIZE=500
|
CLASSIFIER_SPIKE_MIN_SIZE=400
|
||||||
|
|
|
||||||
|
|
@ -426,6 +426,96 @@ h1 {
|
||||||
background: rgba(111, 91, 57, 0.12);
|
background: rgba(111, 91, 57, 0.12);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.nbbo-meta {
|
||||||
|
font-size: 0.72rem;
|
||||||
|
color: #6f5b39;
|
||||||
|
}
|
||||||
|
|
||||||
|
.nbbo-side {
|
||||||
|
position: relative;
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
margin-left: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.nbbo-tag {
|
||||||
|
padding: 2px 6px;
|
||||||
|
border-radius: 999px;
|
||||||
|
border: 1px solid rgba(111, 91, 57, 0.35);
|
||||||
|
font-size: 0.7rem;
|
||||||
|
letter-spacing: 0.08em;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
.nbbo-tag-a {
|
||||||
|
border-color: rgba(47, 109, 79, 0.5);
|
||||||
|
color: #2f6d4f;
|
||||||
|
background: rgba(47, 109, 79, 0.16);
|
||||||
|
}
|
||||||
|
|
||||||
|
.nbbo-tag-aa {
|
||||||
|
border-color: rgba(26, 87, 60, 0.6);
|
||||||
|
color: #1a573c;
|
||||||
|
background: rgba(26, 87, 60, 0.2);
|
||||||
|
}
|
||||||
|
|
||||||
|
.nbbo-tag-b {
|
||||||
|
border-color: rgba(140, 74, 22, 0.5);
|
||||||
|
color: #8c4a16;
|
||||||
|
background: rgba(196, 111, 42, 0.18);
|
||||||
|
}
|
||||||
|
|
||||||
|
.nbbo-tag-bb {
|
||||||
|
border-color: rgba(110, 44, 12, 0.6);
|
||||||
|
color: #6e2c0c;
|
||||||
|
background: rgba(110, 44, 12, 0.2);
|
||||||
|
}
|
||||||
|
|
||||||
|
.nbbo-tooltip {
|
||||||
|
position: absolute;
|
||||||
|
right: 0;
|
||||||
|
bottom: 100%;
|
||||||
|
transform: translateY(-6px);
|
||||||
|
display: grid;
|
||||||
|
gap: 4px;
|
||||||
|
padding: 8px 10px;
|
||||||
|
border-radius: 10px;
|
||||||
|
border: 1px solid rgba(217, 205, 184, 0.8);
|
||||||
|
background: #fffdf7;
|
||||||
|
box-shadow: 0 12px 26px rgba(66, 45, 18, 0.18);
|
||||||
|
opacity: 0;
|
||||||
|
pointer-events: none;
|
||||||
|
transition: opacity 0.15s ease, transform 0.15s ease;
|
||||||
|
z-index: 2;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.nbbo-tooltip-row {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
font-size: 0.68rem;
|
||||||
|
color: #6f5b39;
|
||||||
|
}
|
||||||
|
|
||||||
|
.nbbo-side:hover .nbbo-tooltip,
|
||||||
|
.nbbo-side:focus-within .nbbo-tooltip {
|
||||||
|
opacity: 1;
|
||||||
|
transform: translateY(-10px);
|
||||||
|
}
|
||||||
|
|
||||||
|
.nbbo-missing {
|
||||||
|
border-color: rgba(136, 58, 17, 0.4);
|
||||||
|
color: #8c3a11;
|
||||||
|
background: rgba(196, 111, 42, 0.16);
|
||||||
|
}
|
||||||
|
|
||||||
|
.nbbo-stale {
|
||||||
|
border-color: rgba(31, 74, 123, 0.4);
|
||||||
|
color: #1f4a7b;
|
||||||
|
background: rgba(31, 74, 123, 0.12);
|
||||||
|
}
|
||||||
|
|
||||||
.severity-high {
|
.severity-high {
|
||||||
border-color: rgba(136, 58, 17, 0.6);
|
border-color: rgba(136, 58, 17, 0.6);
|
||||||
color: #8c3a11;
|
color: #8c3a11;
|
||||||
|
|
|
||||||
|
|
@ -1,16 +1,32 @@
|
||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
|
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
|
||||||
import type { AlertEvent, ClassifierHitEvent, EquityPrint, FlowPacket, OptionPrint } from "@islandflow/types";
|
import type {
|
||||||
|
AlertEvent,
|
||||||
|
ClassifierHitEvent,
|
||||||
|
EquityPrint,
|
||||||
|
FlowPacket,
|
||||||
|
OptionNBBO,
|
||||||
|
OptionPrint
|
||||||
|
} from "@islandflow/types";
|
||||||
|
|
||||||
const MAX_ITEMS = 500;
|
const MAX_ITEMS = 500;
|
||||||
|
const NBBO_MAX_AGE_MS = Number(process.env.NEXT_PUBLIC_NBBO_MAX_AGE_MS);
|
||||||
|
const NBBO_MAX_AGE_MS_SAFE =
|
||||||
|
Number.isFinite(NBBO_MAX_AGE_MS) && NBBO_MAX_AGE_MS > 0 ? NBBO_MAX_AGE_MS : 1000;
|
||||||
const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1"]);
|
const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1"]);
|
||||||
|
|
||||||
type WsStatus = "connecting" | "connected" | "disconnected";
|
type WsStatus = "connecting" | "connected" | "disconnected";
|
||||||
|
|
||||||
type TapeMode = "live" | "replay";
|
type TapeMode = "live" | "replay";
|
||||||
|
|
||||||
type MessageType = "option-print" | "equity-print" | "flow-packet" | "classifier-hit" | "alert";
|
type MessageType =
|
||||||
|
| "option-print"
|
||||||
|
| "option-nbbo"
|
||||||
|
| "equity-print"
|
||||||
|
| "flow-packet"
|
||||||
|
| "classifier-hit"
|
||||||
|
| "alert";
|
||||||
|
|
||||||
type StreamMessage<T> = {
|
type StreamMessage<T> = {
|
||||||
type: MessageType;
|
type: MessageType;
|
||||||
|
|
@ -220,6 +236,39 @@ const parseNumber = (value: unknown, fallback: number): number => {
|
||||||
return fallback;
|
return fallback;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type NbboSide = "AA" | "A" | "B" | "BB";
|
||||||
|
|
||||||
|
const classifyNbboSide = (price: number, quote: OptionNBBO | null | undefined): NbboSide | null => {
|
||||||
|
if (!quote || !Number.isFinite(price)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const bid = quote.bid;
|
||||||
|
const ask = quote.ask;
|
||||||
|
if (!Number.isFinite(bid) || !Number.isFinite(ask) || ask <= 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const spread = Math.max(0, ask - bid);
|
||||||
|
const epsilon = Math.max(0.01, spread * 0.05);
|
||||||
|
|
||||||
|
if (price > ask + epsilon) {
|
||||||
|
return "AA";
|
||||||
|
}
|
||||||
|
if (price >= ask - epsilon) {
|
||||||
|
return "A";
|
||||||
|
}
|
||||||
|
if (price < bid - epsilon) {
|
||||||
|
return "BB";
|
||||||
|
}
|
||||||
|
if (price <= bid + epsilon) {
|
||||||
|
return "B";
|
||||||
|
}
|
||||||
|
|
||||||
|
const mid = (bid + ask) / 2;
|
||||||
|
return price >= mid ? "A" : "B";
|
||||||
|
};
|
||||||
|
|
||||||
type ListScrollState = {
|
type ListScrollState = {
|
||||||
listRef: React.RefObject<HTMLDivElement>;
|
listRef: React.RefObject<HTMLDivElement>;
|
||||||
isAtTop: boolean;
|
isAtTop: boolean;
|
||||||
|
|
@ -1242,6 +1291,16 @@ export default function HomePage() {
|
||||||
onNewItems: equitiesScroll.onNewItems
|
onNewItems: equitiesScroll.onNewItems
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const nbbo = useTape<OptionNBBO>({
|
||||||
|
mode,
|
||||||
|
wsPath: "/ws/options-nbbo",
|
||||||
|
replayPath: "/replay/nbbo",
|
||||||
|
latestPath: "/nbbo/options",
|
||||||
|
expectedType: "option-nbbo",
|
||||||
|
batchSize: mode === "replay" ? 120 : undefined,
|
||||||
|
pollMs: mode === "replay" ? 200 : undefined
|
||||||
|
});
|
||||||
|
|
||||||
const flow = useFlowStream(mode === "live", flowScroll.onNewItems, flowAnchor.capture);
|
const flow = useFlowStream(mode === "live", flowScroll.onNewItems, flowAnchor.capture);
|
||||||
const alerts = useLiveStream<AlertEvent>({
|
const alerts = useLiveStream<AlertEvent>({
|
||||||
enabled: mode === "live",
|
enabled: mode === "live",
|
||||||
|
|
@ -1288,6 +1347,21 @@ export default function HomePage() {
|
||||||
|
|
||||||
const tickerSet = useMemo(() => new Set(activeTickers), [activeTickers]);
|
const tickerSet = useMemo(() => new Set(activeTickers), [activeTickers]);
|
||||||
|
|
||||||
|
const nbboMap = useMemo(() => {
|
||||||
|
const map = new Map<string, OptionNBBO>();
|
||||||
|
for (const quote of nbbo.items) {
|
||||||
|
const existing = map.get(quote.option_contract_id);
|
||||||
|
if (
|
||||||
|
!existing ||
|
||||||
|
quote.ts > existing.ts ||
|
||||||
|
(quote.ts === existing.ts && quote.seq >= existing.seq)
|
||||||
|
) {
|
||||||
|
map.set(quote.option_contract_id, quote);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}, [nbbo.items]);
|
||||||
|
|
||||||
const optionPrintMap = useMemo(() => {
|
const optionPrintMap = useMemo(() => {
|
||||||
const map = new Map<string, OptionPrint>();
|
const map = new Map<string, OptionPrint>();
|
||||||
for (const print of options.items) {
|
for (const print of options.items) {
|
||||||
|
|
@ -1541,7 +1615,14 @@ export default function HomePage() {
|
||||||
: "Replay queue empty. Ensure ClickHouse has data."}
|
: "Replay queue empty. Ensure ClickHouse has data."}
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
filteredOptions.map((print) => (
|
filteredOptions.map((print) => {
|
||||||
|
const quote = nbboMap.get(print.option_contract_id);
|
||||||
|
const nbboAge = quote ? Math.abs(print.ts - quote.ts) : null;
|
||||||
|
const nbboStale = nbboAge !== null && nbboAge > NBBO_MAX_AGE_MS_SAFE;
|
||||||
|
const nbboMid = quote ? (quote.bid + quote.ask) / 2 : null;
|
||||||
|
const nbboSide = classifyNbboSide(print.price, quote);
|
||||||
|
|
||||||
|
return (
|
||||||
<div className="row" key={`${print.trace_id}-${print.seq}`}>
|
<div className="row" key={`${print.trace_id}-${print.seq}`}>
|
||||||
<div>
|
<div>
|
||||||
<div className="contract">{print.option_contract_id}</div>
|
<div className="contract">{print.option_contract_id}</div>
|
||||||
|
|
@ -1553,10 +1634,49 @@ export default function HomePage() {
|
||||||
<span>{print.conditions.join(", ")}</span>
|
<span>{print.conditions.join(", ")}</span>
|
||||||
) : null}
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
|
{quote ? (
|
||||||
|
<div className="meta nbbo-meta">
|
||||||
|
<span>Bid ${formatPrice(quote.bid)}</span>
|
||||||
|
<span>Ask ${formatPrice(quote.ask)}</span>
|
||||||
|
<span>Mid ${formatPrice(nbboMid ?? 0)}</span>
|
||||||
|
<span>{Math.round(nbboAge ?? 0)}ms</span>
|
||||||
|
{nbboSide ? (
|
||||||
|
<span className="nbbo-side" tabIndex={0} aria-label="NBBO side legend">
|
||||||
|
<span className={`nbbo-tag nbbo-tag-${nbboSide.toLowerCase()}`}>
|
||||||
|
{nbboSide}
|
||||||
|
</span>
|
||||||
|
<span className="nbbo-tooltip" role="tooltip">
|
||||||
|
<span className="nbbo-tooltip-row">
|
||||||
|
<span className="nbbo-tag nbbo-tag-a">A</span>
|
||||||
|
<span>Ask</span>
|
||||||
|
</span>
|
||||||
|
<span className="nbbo-tooltip-row">
|
||||||
|
<span className="nbbo-tag nbbo-tag-aa">AA</span>
|
||||||
|
<span>Above Ask</span>
|
||||||
|
</span>
|
||||||
|
<span className="nbbo-tooltip-row">
|
||||||
|
<span className="nbbo-tag nbbo-tag-b">B</span>
|
||||||
|
<span>Bid</span>
|
||||||
|
</span>
|
||||||
|
<span className="nbbo-tooltip-row">
|
||||||
|
<span className="nbbo-tag nbbo-tag-bb">BB</span>
|
||||||
|
<span>Below Bid</span>
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
) : null}
|
||||||
|
{nbboStale ? <span className="pill nbbo-stale">Stale</span> : null}
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="meta nbbo-meta">
|
||||||
|
<span className="pill nbbo-missing">NBBO missing</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
<div className="time">{formatTime(print.ts)}</div>
|
<div className="time">{formatTime(print.ts)}</div>
|
||||||
</div>
|
</div>
|
||||||
))
|
);
|
||||||
|
})
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
@ -1664,6 +1784,13 @@ export default function HomePage() {
|
||||||
const startTs = parseNumber(features.start_ts, packet.source_ts);
|
const startTs = parseNumber(features.start_ts, packet.source_ts);
|
||||||
const endTs = parseNumber(features.end_ts, startTs);
|
const endTs = parseNumber(features.end_ts, startTs);
|
||||||
const windowMs = parseNumber(features.window_ms, 0);
|
const windowMs = parseNumber(features.window_ms, 0);
|
||||||
|
const nbboBid = parseNumber(features.nbbo_bid, Number.NaN);
|
||||||
|
const nbboAsk = parseNumber(features.nbbo_ask, Number.NaN);
|
||||||
|
const nbboMid = parseNumber(features.nbbo_mid, Number.NaN);
|
||||||
|
const nbboSpread = parseNumber(features.nbbo_spread, Number.NaN);
|
||||||
|
const nbboAge = parseNumber(packet.join_quality.nbbo_age_ms, Number.NaN);
|
||||||
|
const nbboStale = parseNumber(packet.join_quality.nbbo_stale, 0) > 0;
|
||||||
|
const nbboMissing = parseNumber(packet.join_quality.nbbo_missing, 0) > 0;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="row" key={packet.id}>
|
<div className="row" key={packet.id}>
|
||||||
|
|
@ -1677,6 +1804,24 @@ export default function HomePage() {
|
||||||
{windowMs > 0 ? (
|
{windowMs > 0 ? (
|
||||||
<span>{formatFlowMetric(windowMs, "ms")}</span>
|
<span>{formatFlowMetric(windowMs, "ms")}</span>
|
||||||
) : null}
|
) : null}
|
||||||
|
{Number.isFinite(nbboBid) && Number.isFinite(nbboAsk) ? (
|
||||||
|
<span>
|
||||||
|
NBBO ${formatPrice(nbboBid)} x ${formatPrice(nbboAsk)}
|
||||||
|
</span>
|
||||||
|
) : null}
|
||||||
|
{Number.isFinite(nbboMid) ? (
|
||||||
|
<span>Mid ${formatPrice(nbboMid)}</span>
|
||||||
|
) : null}
|
||||||
|
{Number.isFinite(nbboSpread) ? (
|
||||||
|
<span>Spread ${formatPrice(nbboSpread)}</span>
|
||||||
|
) : null}
|
||||||
|
{Number.isFinite(nbboAge) ? (
|
||||||
|
<span>{Math.round(nbboAge)}ms</span>
|
||||||
|
) : null}
|
||||||
|
{nbboStale ? <span className="pill nbbo-stale">NBBO stale</span> : null}
|
||||||
|
{nbboMissing ? (
|
||||||
|
<span className="pill nbbo-missing">NBBO missing</span>
|
||||||
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div className="time">
|
<div className="time">
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
export const STREAM_OPTION_PRINTS = "OPTIONS_PRINTS";
|
export const STREAM_OPTION_PRINTS = "OPTIONS_PRINTS";
|
||||||
export const SUBJECT_OPTION_PRINTS = "options.prints";
|
export const SUBJECT_OPTION_PRINTS = "options.prints";
|
||||||
|
export const STREAM_OPTION_NBBO = "OPTIONS_NBBO";
|
||||||
|
export const SUBJECT_OPTION_NBBO = "options.nbbo";
|
||||||
export const STREAM_EQUITY_PRINTS = "EQUITY_PRINTS";
|
export const STREAM_EQUITY_PRINTS = "EQUITY_PRINTS";
|
||||||
export const SUBJECT_EQUITY_PRINTS = "equities.prints";
|
export const SUBJECT_EQUITY_PRINTS = "equities.prints";
|
||||||
export const STREAM_FLOW_PACKETS = "FLOW_PACKETS";
|
export const STREAM_FLOW_PACKETS = "FLOW_PACKETS";
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import {
|
||||||
ClassifierHitEventSchema,
|
ClassifierHitEventSchema,
|
||||||
EquityPrintSchema,
|
EquityPrintSchema,
|
||||||
FlowPacketSchema,
|
FlowPacketSchema,
|
||||||
|
OptionNBBOSchema,
|
||||||
OptionPrintSchema
|
OptionPrintSchema
|
||||||
} from "@islandflow/types";
|
} from "@islandflow/types";
|
||||||
import type {
|
import type {
|
||||||
|
|
@ -11,6 +12,7 @@ import type {
|
||||||
ClassifierHitEvent,
|
ClassifierHitEvent,
|
||||||
EquityPrint,
|
EquityPrint,
|
||||||
FlowPacket,
|
FlowPacket,
|
||||||
|
OptionNBBO,
|
||||||
OptionPrint
|
OptionPrint
|
||||||
} from "@islandflow/types";
|
} from "@islandflow/types";
|
||||||
import {
|
import {
|
||||||
|
|
@ -18,6 +20,7 @@ import {
|
||||||
optionPrintsTableDDL,
|
optionPrintsTableDDL,
|
||||||
OPTION_PRINTS_TABLE
|
OPTION_PRINTS_TABLE
|
||||||
} from "./option-prints";
|
} from "./option-prints";
|
||||||
|
import { normalizeOptionNBBO, optionNBBOTableDDL, OPTION_NBBO_TABLE } from "./option-nbbo";
|
||||||
import {
|
import {
|
||||||
equityPrintsTableDDL,
|
equityPrintsTableDDL,
|
||||||
EQUITY_PRINTS_TABLE,
|
EQUITY_PRINTS_TABLE,
|
||||||
|
|
@ -69,6 +72,14 @@ export const ensureOptionPrintsTable = async (
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const ensureOptionNBBOTable = async (
|
||||||
|
client: ClickHouseClient
|
||||||
|
): Promise<void> => {
|
||||||
|
await client.exec({
|
||||||
|
query: optionNBBOTableDDL()
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
export const ensureEquityPrintsTable = async (
|
export const ensureEquityPrintsTable = async (
|
||||||
client: ClickHouseClient
|
client: ClickHouseClient
|
||||||
): Promise<void> => {
|
): Promise<void> => {
|
||||||
|
|
@ -111,6 +122,18 @@ export const insertOptionPrint = async (
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const insertOptionNBBO = async (
|
||||||
|
client: ClickHouseClient,
|
||||||
|
nbbo: OptionNBBO
|
||||||
|
): Promise<void> => {
|
||||||
|
const record = normalizeOptionNBBO(nbbo);
|
||||||
|
await client.insert({
|
||||||
|
table: OPTION_NBBO_TABLE,
|
||||||
|
values: [record],
|
||||||
|
format: "JSONEachRow"
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
export const insertEquityPrint = async (
|
export const insertEquityPrint = async (
|
||||||
client: ClickHouseClient,
|
client: ClickHouseClient,
|
||||||
print: EquityPrint
|
print: EquityPrint
|
||||||
|
|
@ -213,6 +236,24 @@ const normalizeOptionRow = (row: unknown): unknown => {
|
||||||
return row;
|
return row;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const normalizeOptionNbboRow = (row: unknown): unknown => {
|
||||||
|
if (row && typeof row === "object") {
|
||||||
|
return normalizeNumericFields(row as Record<string, unknown>, [
|
||||||
|
"source_ts",
|
||||||
|
"ingest_ts",
|
||||||
|
"seq",
|
||||||
|
"ts",
|
||||||
|
"bid",
|
||||||
|
"ask",
|
||||||
|
"bidSize",
|
||||||
|
"askSize"
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return row;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
const normalizeEquityRow = (row: unknown): unknown => {
|
const normalizeEquityRow = (row: unknown): unknown => {
|
||||||
if (row && typeof row === "object") {
|
if (row && typeof row === "object") {
|
||||||
const record = normalizeNumericFields(row as Record<string, unknown>, [
|
const record = normalizeNumericFields(row as Record<string, unknown>, [
|
||||||
|
|
@ -307,6 +348,20 @@ export const fetchRecentOptionPrints = async (
|
||||||
return OptionPrintSchema.array().parse(rows.map(normalizeOptionRow));
|
return OptionPrintSchema.array().parse(rows.map(normalizeOptionRow));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const fetchRecentOptionNBBO = async (
|
||||||
|
client: ClickHouseClient,
|
||||||
|
limit: number
|
||||||
|
): Promise<OptionNBBO[]> => {
|
||||||
|
const safeLimit = clampLimit(limit);
|
||||||
|
const result = await client.query({
|
||||||
|
query: `SELECT * FROM ${OPTION_NBBO_TABLE} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
|
||||||
|
format: "JSONEachRow"
|
||||||
|
});
|
||||||
|
|
||||||
|
const rows = await result.json<unknown[]>();
|
||||||
|
return OptionNBBOSchema.array().parse(rows.map(normalizeOptionNbboRow));
|
||||||
|
};
|
||||||
|
|
||||||
export const fetchRecentEquityPrints = async (
|
export const fetchRecentEquityPrints = async (
|
||||||
client: ClickHouseClient,
|
client: ClickHouseClient,
|
||||||
limit: number
|
limit: number
|
||||||
|
|
@ -394,6 +449,25 @@ export const fetchOptionPrintsAfter = async (
|
||||||
return OptionPrintSchema.array().parse(rows.map(normalizeOptionRow));
|
return OptionPrintSchema.array().parse(rows.map(normalizeOptionRow));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const fetchOptionNBBOAfter = async (
|
||||||
|
client: ClickHouseClient,
|
||||||
|
afterTs: number,
|
||||||
|
afterSeq: number,
|
||||||
|
limit: number
|
||||||
|
): Promise<OptionNBBO[]> => {
|
||||||
|
const safeLimit = clampLimit(limit);
|
||||||
|
const safeAfterTs = clampCursor(afterTs);
|
||||||
|
const safeAfterSeq = clampCursor(afterSeq);
|
||||||
|
|
||||||
|
const result = await client.query({
|
||||||
|
query: `SELECT * FROM ${OPTION_NBBO_TABLE} WHERE (ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
|
||||||
|
format: "JSONEachRow"
|
||||||
|
});
|
||||||
|
|
||||||
|
const rows = await result.json<unknown[]>();
|
||||||
|
return OptionNBBOSchema.array().parse(rows.map(normalizeOptionNbboRow));
|
||||||
|
};
|
||||||
|
|
||||||
export const fetchEquityPrintsAfter = async (
|
export const fetchEquityPrintsAfter = async (
|
||||||
client: ClickHouseClient,
|
client: ClickHouseClient,
|
||||||
afterTs: number,
|
afterTs: number,
|
||||||
|
|
|
||||||
|
|
@ -4,3 +4,4 @@ export * from "./alerts";
|
||||||
export * from "./flow-packets";
|
export * from "./flow-packets";
|
||||||
export * from "./equity-prints";
|
export * from "./equity-prints";
|
||||||
export * from "./option-prints";
|
export * from "./option-prints";
|
||||||
|
export * from "./option-nbbo";
|
||||||
|
|
|
||||||
26
packages/storage/src/option-nbbo.ts
Normal file
26
packages/storage/src/option-nbbo.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
import type { OptionNBBO } from "@islandflow/types";
|
||||||
|
|
||||||
|
export const OPTION_NBBO_TABLE = "option_nbbo";
|
||||||
|
|
||||||
|
export const optionNBBOTableDDL = (): string => {
|
||||||
|
return `
|
||||||
|
CREATE TABLE IF NOT EXISTS ${OPTION_NBBO_TABLE} (
|
||||||
|
source_ts UInt64,
|
||||||
|
ingest_ts UInt64,
|
||||||
|
seq UInt64,
|
||||||
|
trace_id String,
|
||||||
|
ts UInt64,
|
||||||
|
option_contract_id String,
|
||||||
|
bid Float64,
|
||||||
|
ask Float64,
|
||||||
|
bidSize UInt32,
|
||||||
|
askSize UInt32
|
||||||
|
)
|
||||||
|
ENGINE = MergeTree
|
||||||
|
ORDER BY (ts, option_contract_id)
|
||||||
|
`;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const normalizeOptionNBBO = (nbbo: OptionNBBO): OptionNBBO => {
|
||||||
|
return nbbo;
|
||||||
|
};
|
||||||
28
packages/storage/tests/option-nbbo.test.ts
Normal file
28
packages/storage/tests/option-nbbo.test.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
import { describe, expect, it } from "bun:test";
|
||||||
|
import { normalizeOptionNBBO, optionNBBOTableDDL, OPTION_NBBO_TABLE } from "../src/option-nbbo";
|
||||||
|
|
||||||
|
const baseNbbo = {
|
||||||
|
source_ts: 100,
|
||||||
|
ingest_ts: 200,
|
||||||
|
seq: 1,
|
||||||
|
trace_id: "trace-1",
|
||||||
|
ts: 100,
|
||||||
|
option_contract_id: "SPY-2025-01-17-450-C",
|
||||||
|
bid: 1.2,
|
||||||
|
ask: 1.3,
|
||||||
|
bidSize: 10,
|
||||||
|
askSize: 12
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("option-nbbo storage helpers", () => {
|
||||||
|
it("keeps required fields intact", () => {
|
||||||
|
const normalized = normalizeOptionNBBO(baseNbbo);
|
||||||
|
expect(normalized).toEqual(baseNbbo);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("includes the correct table name in the DDL", () => {
|
||||||
|
const ddl = optionNBBOTableDDL();
|
||||||
|
expect(ddl).toContain(OPTION_NBBO_TABLE);
|
||||||
|
expect(ddl).toContain("CREATE TABLE IF NOT EXISTS");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -5,11 +5,13 @@ import {
|
||||||
SUBJECT_CLASSIFIER_HITS,
|
SUBJECT_CLASSIFIER_HITS,
|
||||||
SUBJECT_EQUITY_PRINTS,
|
SUBJECT_EQUITY_PRINTS,
|
||||||
SUBJECT_FLOW_PACKETS,
|
SUBJECT_FLOW_PACKETS,
|
||||||
|
SUBJECT_OPTION_NBBO,
|
||||||
SUBJECT_OPTION_PRINTS,
|
SUBJECT_OPTION_PRINTS,
|
||||||
STREAM_ALERTS,
|
STREAM_ALERTS,
|
||||||
STREAM_CLASSIFIER_HITS,
|
STREAM_CLASSIFIER_HITS,
|
||||||
STREAM_EQUITY_PRINTS,
|
STREAM_EQUITY_PRINTS,
|
||||||
STREAM_FLOW_PACKETS,
|
STREAM_FLOW_PACKETS,
|
||||||
|
STREAM_OPTION_NBBO,
|
||||||
STREAM_OPTION_PRINTS,
|
STREAM_OPTION_PRINTS,
|
||||||
buildDurableConsumer,
|
buildDurableConsumer,
|
||||||
connectJetStreamWithRetry,
|
connectJetStreamWithRetry,
|
||||||
|
|
@ -22,12 +24,15 @@ import {
|
||||||
ensureClassifierHitsTable,
|
ensureClassifierHitsTable,
|
||||||
ensureEquityPrintsTable,
|
ensureEquityPrintsTable,
|
||||||
ensureFlowPacketsTable,
|
ensureFlowPacketsTable,
|
||||||
|
ensureOptionNBBOTable,
|
||||||
ensureOptionPrintsTable,
|
ensureOptionPrintsTable,
|
||||||
fetchRecentAlerts,
|
fetchRecentAlerts,
|
||||||
fetchRecentClassifierHits,
|
fetchRecentClassifierHits,
|
||||||
fetchRecentFlowPackets,
|
fetchRecentFlowPackets,
|
||||||
|
fetchRecentOptionNBBO,
|
||||||
fetchEquityPrintsAfter,
|
fetchEquityPrintsAfter,
|
||||||
fetchRecentEquityPrints,
|
fetchRecentEquityPrints,
|
||||||
|
fetchOptionNBBOAfter,
|
||||||
fetchOptionPrintsAfter,
|
fetchOptionPrintsAfter,
|
||||||
fetchRecentOptionPrints
|
fetchRecentOptionPrints
|
||||||
} from "@islandflow/storage";
|
} from "@islandflow/storage";
|
||||||
|
|
@ -36,6 +41,7 @@ import {
|
||||||
ClassifierHitEventSchema,
|
ClassifierHitEventSchema,
|
||||||
EquityPrintSchema,
|
EquityPrintSchema,
|
||||||
FlowPacketSchema,
|
FlowPacketSchema,
|
||||||
|
OptionNBBOSchema,
|
||||||
OptionPrintSchema
|
OptionPrintSchema
|
||||||
} from "@islandflow/types";
|
} from "@islandflow/types";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
|
@ -87,13 +93,14 @@ const replayParamsSchema = z.object({
|
||||||
limit: z.coerce.number().int().positive().max(1000).default(200)
|
limit: z.coerce.number().int().positive().max(1000).default(200)
|
||||||
});
|
});
|
||||||
|
|
||||||
type Channel = "options" | "equities" | "flow" | "classifier-hits" | "alerts";
|
type Channel = "options" | "options-nbbo" | "equities" | "flow" | "classifier-hits" | "alerts";
|
||||||
|
|
||||||
type WsData = {
|
type WsData = {
|
||||||
channel: Channel;
|
channel: Channel;
|
||||||
};
|
};
|
||||||
|
|
||||||
const optionSockets = new Set<WebSocket<WsData>>();
|
const optionSockets = new Set<WebSocket<WsData>>();
|
||||||
|
const optionNbboSockets = new Set<WebSocket<WsData>>();
|
||||||
const equitySockets = new Set<WebSocket<WsData>>();
|
const equitySockets = new Set<WebSocket<WsData>>();
|
||||||
const flowSockets = new Set<WebSocket<WsData>>();
|
const flowSockets = new Set<WebSocket<WsData>>();
|
||||||
const classifierHitSockets = new Set<WebSocket<WsData>>();
|
const classifierHitSockets = new Set<WebSocket<WsData>>();
|
||||||
|
|
@ -169,6 +176,19 @@ const run = async () => {
|
||||||
num_replicas: 1
|
num_replicas: 1
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await ensureStream(jsm, {
|
||||||
|
name: STREAM_OPTION_NBBO,
|
||||||
|
subjects: [SUBJECT_OPTION_NBBO],
|
||||||
|
retention: "limits",
|
||||||
|
storage: "file",
|
||||||
|
discard: "old",
|
||||||
|
max_msgs_per_subject: -1,
|
||||||
|
max_msgs: -1,
|
||||||
|
max_bytes: -1,
|
||||||
|
max_age: 0,
|
||||||
|
num_replicas: 1
|
||||||
|
});
|
||||||
|
|
||||||
await ensureStream(jsm, {
|
await ensureStream(jsm, {
|
||||||
name: STREAM_EQUITY_PRINTS,
|
name: STREAM_EQUITY_PRINTS,
|
||||||
subjects: [SUBJECT_EQUITY_PRINTS],
|
subjects: [SUBJECT_EQUITY_PRINTS],
|
||||||
|
|
@ -228,6 +248,7 @@ const run = async () => {
|
||||||
|
|
||||||
await retry("clickhouse table init", 20, 500, async () => {
|
await retry("clickhouse table init", 20, 500, async () => {
|
||||||
await ensureOptionPrintsTable(clickhouse);
|
await ensureOptionPrintsTable(clickhouse);
|
||||||
|
await ensureOptionNBBOTable(clickhouse);
|
||||||
await ensureEquityPrintsTable(clickhouse);
|
await ensureEquityPrintsTable(clickhouse);
|
||||||
await ensureFlowPacketsTable(clickhouse);
|
await ensureFlowPacketsTable(clickhouse);
|
||||||
await ensureClassifierHitsTable(clickhouse);
|
await ensureClassifierHitsTable(clickhouse);
|
||||||
|
|
@ -240,6 +261,12 @@ const run = async () => {
|
||||||
buildDurableConsumer("api-option-prints")
|
buildDurableConsumer("api-option-prints")
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const optionNbboSubscription = await subscribeJson(
|
||||||
|
js,
|
||||||
|
SUBJECT_OPTION_NBBO,
|
||||||
|
buildDurableConsumer("api-option-nbbo")
|
||||||
|
);
|
||||||
|
|
||||||
const equitySubscription = await subscribeJson(
|
const equitySubscription = await subscribeJson(
|
||||||
js,
|
js,
|
||||||
SUBJECT_EQUITY_PRINTS,
|
SUBJECT_EQUITY_PRINTS,
|
||||||
|
|
@ -279,6 +306,21 @@ const run = async () => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const pumpOptionNbbo = async () => {
|
||||||
|
for await (const msg of optionNbboSubscription.messages) {
|
||||||
|
try {
|
||||||
|
const payload = OptionNBBOSchema.parse(optionNbboSubscription.decode(msg));
|
||||||
|
broadcast(optionNbboSockets, { type: "option-nbbo", payload });
|
||||||
|
msg.ack();
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("failed to process option nbbo", {
|
||||||
|
error: error instanceof Error ? error.message : String(error)
|
||||||
|
});
|
||||||
|
msg.term();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const pumpEquities = async () => {
|
const pumpEquities = async () => {
|
||||||
for await (const msg of equitySubscription.messages) {
|
for await (const msg of equitySubscription.messages) {
|
||||||
try {
|
try {
|
||||||
|
|
@ -340,6 +382,7 @@ const run = async () => {
|
||||||
};
|
};
|
||||||
|
|
||||||
void pumpOptions();
|
void pumpOptions();
|
||||||
|
void pumpOptionNbbo();
|
||||||
void pumpEquities();
|
void pumpEquities();
|
||||||
void pumpFlow();
|
void pumpFlow();
|
||||||
void pumpClassifierHits();
|
void pumpClassifierHits();
|
||||||
|
|
@ -360,6 +403,12 @@ const run = async () => {
|
||||||
return jsonResponse({ data });
|
return jsonResponse({ data });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname === "/nbbo/options") {
|
||||||
|
const limit = parseLimit(url.searchParams.get("limit"));
|
||||||
|
const data = await fetchRecentOptionNBBO(clickhouse, limit);
|
||||||
|
return jsonResponse({ data });
|
||||||
|
}
|
||||||
|
|
||||||
if (req.method === "GET" && url.pathname === "/prints/equities") {
|
if (req.method === "GET" && url.pathname === "/prints/equities") {
|
||||||
const limit = parseLimit(url.searchParams.get("limit"));
|
const limit = parseLimit(url.searchParams.get("limit"));
|
||||||
const data = await fetchRecentEquityPrints(clickhouse, limit);
|
const data = await fetchRecentEquityPrints(clickhouse, limit);
|
||||||
|
|
@ -392,6 +441,14 @@ const run = async () => {
|
||||||
return jsonResponse({ data, next });
|
return jsonResponse({ data, next });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname === "/replay/nbbo") {
|
||||||
|
const { afterTs, afterSeq, limit } = parseReplayParams(url);
|
||||||
|
const data = await fetchOptionNBBOAfter(clickhouse, afterTs, afterSeq, limit);
|
||||||
|
const last = data.at(-1);
|
||||||
|
const next = last ? { ts: last.ts, seq: last.seq } : null;
|
||||||
|
return jsonResponse({ data, next });
|
||||||
|
}
|
||||||
|
|
||||||
if (req.method === "GET" && url.pathname === "/replay/equities") {
|
if (req.method === "GET" && url.pathname === "/replay/equities") {
|
||||||
const { afterTs, afterSeq, limit } = parseReplayParams(url);
|
const { afterTs, afterSeq, limit } = parseReplayParams(url);
|
||||||
const data = await fetchEquityPrintsAfter(clickhouse, afterTs, afterSeq, limit);
|
const data = await fetchEquityPrintsAfter(clickhouse, afterTs, afterSeq, limit);
|
||||||
|
|
@ -408,6 +465,14 @@ const run = async () => {
|
||||||
return jsonResponse({ error: "websocket upgrade failed" }, 400);
|
return jsonResponse({ error: "websocket upgrade failed" }, 400);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname === "/ws/options-nbbo") {
|
||||||
|
if (serverRef.upgrade(req, { data: { channel: "options-nbbo" } })) {
|
||||||
|
return new Response(null, { status: 101 });
|
||||||
|
}
|
||||||
|
|
||||||
|
return jsonResponse({ error: "websocket upgrade failed" }, 400);
|
||||||
|
}
|
||||||
|
|
||||||
if (req.method === "GET" && url.pathname === "/ws/equities") {
|
if (req.method === "GET" && url.pathname === "/ws/equities") {
|
||||||
if (serverRef.upgrade(req, { data: { channel: "equities" } })) {
|
if (serverRef.upgrade(req, { data: { channel: "equities" } })) {
|
||||||
return new Response(null, { status: 101 });
|
return new Response(null, { status: 101 });
|
||||||
|
|
@ -446,6 +511,8 @@ const run = async () => {
|
||||||
open: (socket) => {
|
open: (socket) => {
|
||||||
if (socket.data.channel === "options") {
|
if (socket.data.channel === "options") {
|
||||||
optionSockets.add(socket);
|
optionSockets.add(socket);
|
||||||
|
} else if (socket.data.channel === "options-nbbo") {
|
||||||
|
optionNbboSockets.add(socket);
|
||||||
} else if (socket.data.channel === "equities") {
|
} else if (socket.data.channel === "equities") {
|
||||||
equitySockets.add(socket);
|
equitySockets.add(socket);
|
||||||
} else if (socket.data.channel === "flow") {
|
} else if (socket.data.channel === "flow") {
|
||||||
|
|
@ -461,6 +528,8 @@ const run = async () => {
|
||||||
close: (socket) => {
|
close: (socket) => {
|
||||||
if (socket.data.channel === "options") {
|
if (socket.data.channel === "options") {
|
||||||
optionSockets.delete(socket);
|
optionSockets.delete(socket);
|
||||||
|
} else if (socket.data.channel === "options-nbbo") {
|
||||||
|
optionNbboSockets.delete(socket);
|
||||||
} else if (socket.data.channel === "equities") {
|
} else if (socket.data.channel === "equities") {
|
||||||
equitySockets.delete(socket);
|
equitySockets.delete(socket);
|
||||||
} else if (socket.data.channel === "flow") {
|
} else if (socket.data.channel === "flow") {
|
||||||
|
|
|
||||||
|
|
@ -4,10 +4,12 @@ import {
|
||||||
SUBJECT_ALERTS,
|
SUBJECT_ALERTS,
|
||||||
SUBJECT_CLASSIFIER_HITS,
|
SUBJECT_CLASSIFIER_HITS,
|
||||||
SUBJECT_FLOW_PACKETS,
|
SUBJECT_FLOW_PACKETS,
|
||||||
|
SUBJECT_OPTION_NBBO,
|
||||||
SUBJECT_OPTION_PRINTS,
|
SUBJECT_OPTION_PRINTS,
|
||||||
STREAM_ALERTS,
|
STREAM_ALERTS,
|
||||||
STREAM_CLASSIFIER_HITS,
|
STREAM_CLASSIFIER_HITS,
|
||||||
STREAM_FLOW_PACKETS,
|
STREAM_FLOW_PACKETS,
|
||||||
|
STREAM_OPTION_NBBO,
|
||||||
STREAM_OPTION_PRINTS,
|
STREAM_OPTION_PRINTS,
|
||||||
buildDurableConsumer,
|
buildDurableConsumer,
|
||||||
connectJetStreamWithRetry,
|
connectJetStreamWithRetry,
|
||||||
|
|
@ -28,10 +30,12 @@ import {
|
||||||
AlertEventSchema,
|
AlertEventSchema,
|
||||||
ClassifierHitEventSchema,
|
ClassifierHitEventSchema,
|
||||||
FlowPacketSchema,
|
FlowPacketSchema,
|
||||||
|
OptionNBBOSchema,
|
||||||
OptionPrintSchema,
|
OptionPrintSchema,
|
||||||
type AlertEvent,
|
type AlertEvent,
|
||||||
type ClassifierHitEvent,
|
type ClassifierHitEvent,
|
||||||
type FlowPacket,
|
type FlowPacket,
|
||||||
|
type OptionNBBO,
|
||||||
type OptionPrint
|
type OptionPrint
|
||||||
} from "@islandflow/types";
|
} from "@islandflow/types";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
|
@ -60,6 +64,7 @@ const envSchema = z.object({
|
||||||
return value;
|
return value;
|
||||||
}, z.boolean())
|
}, z.boolean())
|
||||||
.default(false),
|
.default(false),
|
||||||
|
NBBO_MAX_AGE_MS: z.coerce.number().int().positive().default(1000),
|
||||||
CLASSIFIER_SWEEP_MIN_PREMIUM: z.coerce.number().positive().default(40_000),
|
CLASSIFIER_SWEEP_MIN_PREMIUM: z.coerce.number().positive().default(40_000),
|
||||||
CLASSIFIER_SWEEP_MIN_COUNT: z.coerce.number().int().positive().default(3),
|
CLASSIFIER_SWEEP_MIN_COUNT: z.coerce.number().int().positive().default(3),
|
||||||
CLASSIFIER_SPIKE_MIN_PREMIUM: z.coerce.number().positive().default(20_000),
|
CLASSIFIER_SPIKE_MIN_PREMIUM: z.coerce.number().positive().default(20_000),
|
||||||
|
|
@ -117,6 +122,7 @@ type ClusterState = {
|
||||||
};
|
};
|
||||||
|
|
||||||
const clusters = new Map<string, ClusterState>();
|
const clusters = new Map<string, ClusterState>();
|
||||||
|
const nbboCache = new Map<string, OptionNBBO>();
|
||||||
|
|
||||||
const applyDeliverPolicy = (
|
const applyDeliverPolicy = (
|
||||||
opts: ReturnType<typeof buildDurableConsumer>,
|
opts: ReturnType<typeof buildDurableConsumer>,
|
||||||
|
|
@ -166,12 +172,43 @@ const updateCluster = (cluster: ClusterState, print: OptionPrint): ClusterState
|
||||||
return cluster;
|
return cluster;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type NbboJoin = {
|
||||||
|
nbbo: OptionNBBO | null;
|
||||||
|
ageMs: number;
|
||||||
|
stale: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
const updateNbboCache = (nbbo: OptionNBBO): void => {
|
||||||
|
const existing = nbboCache.get(nbbo.option_contract_id);
|
||||||
|
if (
|
||||||
|
!existing ||
|
||||||
|
nbbo.ts > existing.ts ||
|
||||||
|
(nbbo.ts === existing.ts && nbbo.seq >= existing.seq)
|
||||||
|
) {
|
||||||
|
nbboCache.set(nbbo.option_contract_id, nbbo);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const selectNbbo = (contractId: string, ts: number): NbboJoin => {
|
||||||
|
const nbbo = nbboCache.get(contractId) ?? null;
|
||||||
|
if (!nbbo) {
|
||||||
|
return { nbbo: null, ageMs: env.NBBO_MAX_AGE_MS + 1, stale: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
const ageMs = Math.abs(ts - nbbo.ts);
|
||||||
|
const stale = ageMs > env.NBBO_MAX_AGE_MS;
|
||||||
|
return { nbbo, ageMs, stale };
|
||||||
|
};
|
||||||
|
|
||||||
const flushCluster = async (
|
const flushCluster = async (
|
||||||
clickhouse: ReturnType<typeof createClickHouseClient>,
|
clickhouse: ReturnType<typeof createClickHouseClient>,
|
||||||
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
|
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
|
||||||
cluster: ClusterState
|
cluster: ClusterState
|
||||||
): Promise<void> => {
|
): Promise<void> => {
|
||||||
const features = {
|
const joinQuality: Record<string, number> = {};
|
||||||
|
const nbboJoin = selectNbbo(cluster.contractId, cluster.endTs);
|
||||||
|
|
||||||
|
const features: Record<string, string | number | boolean> = {
|
||||||
option_contract_id: cluster.contractId,
|
option_contract_id: cluster.contractId,
|
||||||
count: cluster.members.length,
|
count: cluster.members.length,
|
||||||
total_size: cluster.totalSize,
|
total_size: cluster.totalSize,
|
||||||
|
|
@ -183,6 +220,23 @@ const flushCluster = async (
|
||||||
window_ms: env.CLUSTER_WINDOW_MS
|
window_ms: env.CLUSTER_WINDOW_MS
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if (!nbboJoin.nbbo) {
|
||||||
|
joinQuality.nbbo_missing = 1;
|
||||||
|
} else {
|
||||||
|
joinQuality.nbbo_age_ms = nbboJoin.ageMs;
|
||||||
|
if (nbboJoin.stale) {
|
||||||
|
joinQuality.nbbo_stale = 1;
|
||||||
|
} else {
|
||||||
|
const mid = (nbboJoin.nbbo.bid + nbboJoin.nbbo.ask) / 2;
|
||||||
|
features.nbbo_bid = nbboJoin.nbbo.bid;
|
||||||
|
features.nbbo_ask = nbboJoin.nbbo.ask;
|
||||||
|
features.nbbo_mid = Number(mid.toFixed(4));
|
||||||
|
features.nbbo_spread = Number((nbboJoin.nbbo.ask - nbboJoin.nbbo.bid).toFixed(4));
|
||||||
|
features.nbbo_bid_size = nbboJoin.nbbo.bidSize;
|
||||||
|
features.nbbo_ask_size = nbboJoin.nbbo.askSize;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const packet: FlowPacket = {
|
const packet: FlowPacket = {
|
||||||
source_ts: cluster.startSourceTs,
|
source_ts: cluster.startSourceTs,
|
||||||
ingest_ts: cluster.endIngestTs,
|
ingest_ts: cluster.endIngestTs,
|
||||||
|
|
@ -191,7 +245,7 @@ const flushCluster = async (
|
||||||
id: `flowpacket:${cluster.contractId}:${cluster.startTs}:${cluster.endTs}`,
|
id: `flowpacket:${cluster.contractId}:${cluster.startTs}:${cluster.endTs}`,
|
||||||
members: cluster.members,
|
members: cluster.members,
|
||||||
features,
|
features,
|
||||||
join_quality: {}
|
join_quality: joinQuality
|
||||||
};
|
};
|
||||||
|
|
||||||
const validated = FlowPacketSchema.parse(packet);
|
const validated = FlowPacketSchema.parse(packet);
|
||||||
|
|
@ -323,6 +377,19 @@ const run = async () => {
|
||||||
num_replicas: 1
|
num_replicas: 1
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await ensureStream(jsm, {
|
||||||
|
name: STREAM_OPTION_NBBO,
|
||||||
|
subjects: [SUBJECT_OPTION_NBBO],
|
||||||
|
retention: "limits",
|
||||||
|
storage: "file",
|
||||||
|
discard: "old",
|
||||||
|
max_msgs_per_subject: -1,
|
||||||
|
max_msgs: -1,
|
||||||
|
max_bytes: -1,
|
||||||
|
max_age: 0,
|
||||||
|
num_replicas: 1
|
||||||
|
});
|
||||||
|
|
||||||
await ensureStream(jsm, {
|
await ensureStream(jsm, {
|
||||||
name: STREAM_FLOW_PACKETS,
|
name: STREAM_FLOW_PACKETS,
|
||||||
subjects: [SUBJECT_FLOW_PACKETS],
|
subjects: [SUBJECT_FLOW_PACKETS],
|
||||||
|
|
@ -374,6 +441,7 @@ const run = async () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
const durableName = "compute-option-prints";
|
const durableName = "compute-option-prints";
|
||||||
|
const nbboDurableName = "compute-option-nbbo";
|
||||||
|
|
||||||
if (env.COMPUTE_CONSUMER_RESET) {
|
if (env.COMPUTE_CONSUMER_RESET) {
|
||||||
try {
|
try {
|
||||||
|
|
@ -404,6 +472,35 @@ const run = async () => {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (env.COMPUTE_CONSUMER_RESET) {
|
||||||
|
try {
|
||||||
|
await jsm.consumers.delete(STREAM_OPTION_NBBO, nbboDurableName);
|
||||||
|
logger.warn("reset jetstream consumer", { durable: nbboDurableName });
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
if (!message.includes("not found")) {
|
||||||
|
logger.warn("failed to reset jetstream consumer", { durable: nbboDurableName, error: message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
const info = await jsm.consumers.info(STREAM_OPTION_NBBO, nbboDurableName);
|
||||||
|
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
|
||||||
|
logger.warn("resetting consumer due to deliver policy change", {
|
||||||
|
durable: nbboDurableName,
|
||||||
|
current: info.config.deliver_policy,
|
||||||
|
desired: env.COMPUTE_DELIVER_POLICY
|
||||||
|
});
|
||||||
|
await jsm.consumers.delete(STREAM_OPTION_NBBO, nbboDurableName);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
if (!message.includes("not found")) {
|
||||||
|
logger.warn("failed to inspect jetstream consumer", { durable: nbboDurableName, error: message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const subscription = await (async () => {
|
const subscription = await (async () => {
|
||||||
const opts = buildDurableConsumer(durableName);
|
const opts = buildDurableConsumer(durableName);
|
||||||
applyDeliverPolicy(opts, env.COMPUTE_DELIVER_POLICY);
|
applyDeliverPolicy(opts, env.COMPUTE_DELIVER_POLICY);
|
||||||
|
|
@ -440,6 +537,59 @@ const run = async () => {
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
const nbboSubscription = await (async () => {
|
||||||
|
const opts = buildDurableConsumer(nbboDurableName);
|
||||||
|
applyDeliverPolicy(opts, env.COMPUTE_DELIVER_POLICY);
|
||||||
|
try {
|
||||||
|
return await subscribeJson(js, SUBJECT_OPTION_NBBO, opts);
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
const shouldReset =
|
||||||
|
message.includes("duplicate subscription") ||
|
||||||
|
message.includes("durable requires") ||
|
||||||
|
message.includes("subject does not match consumer");
|
||||||
|
|
||||||
|
if (!shouldReset) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.warn("resetting jetstream consumer", { durable: nbboDurableName, error: message });
|
||||||
|
|
||||||
|
try {
|
||||||
|
await jsm.consumers.delete(STREAM_OPTION_NBBO, nbboDurableName);
|
||||||
|
} catch (deleteError) {
|
||||||
|
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||||
|
if (!deleteMessage.includes("not found")) {
|
||||||
|
logger.warn("failed to delete jetstream consumer", {
|
||||||
|
durable: nbboDurableName,
|
||||||
|
error: deleteMessage
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const resetOpts = buildDurableConsumer(nbboDurableName);
|
||||||
|
applyDeliverPolicy(resetOpts, env.COMPUTE_DELIVER_POLICY);
|
||||||
|
return await subscribeJson(js, SUBJECT_OPTION_NBBO, resetOpts);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
|
const nbboLoop = async () => {
|
||||||
|
for await (const msg of nbboSubscription.messages) {
|
||||||
|
try {
|
||||||
|
const nbbo = OptionNBBOSchema.parse(nbboSubscription.decode(msg));
|
||||||
|
updateNbboCache(nbbo);
|
||||||
|
msg.ack();
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("failed to process option nbbo", {
|
||||||
|
error: error instanceof Error ? error.message : String(error)
|
||||||
|
});
|
||||||
|
msg.term();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
void nbboLoop();
|
||||||
|
|
||||||
const shutdown = async (signal: string) => {
|
const shutdown = async (signal: string) => {
|
||||||
logger.info("service stopping", { signal });
|
logger.info("service stopping", { signal });
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { SP500_SYMBOLS, type OptionPrint } from "@islandflow/types";
|
import { SP500_SYMBOLS, type OptionNBBO, type OptionPrint } from "@islandflow/types";
|
||||||
import type { OptionIngestAdapter, OptionIngestHandlers } from "./types";
|
import type { OptionIngestAdapter, OptionIngestHandlers } from "./types";
|
||||||
|
|
||||||
type SyntheticOptionsAdapterConfig = {
|
type SyntheticOptionsAdapterConfig = {
|
||||||
|
|
@ -13,6 +13,8 @@ type Burst = {
|
||||||
conditions?: string[];
|
conditions?: string[];
|
||||||
printCount: number;
|
printCount: number;
|
||||||
priceStep: number;
|
priceStep: number;
|
||||||
|
scenarioId: string;
|
||||||
|
seed: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
||||||
|
|
@ -21,6 +23,13 @@ const EXCHANGES = ["CBOE", "PHLX", "ISE", "ARCA", "BOX", "MIAX"];
|
||||||
const CONDITIONS = ["SWEEP", "ISO", "FILL", "TEST"];
|
const CONDITIONS = ["SWEEP", "ISO", "FILL", "TEST"];
|
||||||
const BURST_RUN_RANGE: [number, number] = [2, 4];
|
const BURST_RUN_RANGE: [number, number] = [2, 4];
|
||||||
|
|
||||||
|
type PricePlacement = "AA" | "A" | "B" | "BB";
|
||||||
|
|
||||||
|
type WeightedValue<T> = {
|
||||||
|
value: T;
|
||||||
|
weight: number;
|
||||||
|
};
|
||||||
|
|
||||||
type Scenario = {
|
type Scenario = {
|
||||||
id: string;
|
id: string;
|
||||||
weight: number;
|
weight: number;
|
||||||
|
|
@ -75,6 +84,35 @@ const SCENARIOS: Scenario[] = [
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
|
|
||||||
|
const PRICE_PLACEMENTS: Record<string, WeightedValue<PricePlacement>[]> = {
|
||||||
|
bullish_sweep: [
|
||||||
|
{ value: "AA", weight: 25 },
|
||||||
|
{ value: "A", weight: 40 },
|
||||||
|
{ value: "B", weight: 20 },
|
||||||
|
{ value: "BB", weight: 15 }
|
||||||
|
],
|
||||||
|
bearish_sweep: [
|
||||||
|
{ value: "AA", weight: 15 },
|
||||||
|
{ value: "A", weight: 20 },
|
||||||
|
{ value: "B", weight: 40 },
|
||||||
|
{ value: "BB", weight: 25 }
|
||||||
|
],
|
||||||
|
contract_spike: [
|
||||||
|
{ value: "AA", weight: 25 },
|
||||||
|
{ value: "A", weight: 25 },
|
||||||
|
{ value: "B", weight: 25 },
|
||||||
|
{ value: "BB", weight: 25 }
|
||||||
|
],
|
||||||
|
noise: [
|
||||||
|
{ value: "AA", weight: 25 },
|
||||||
|
{ value: "A", weight: 25 },
|
||||||
|
{ value: "B", weight: 25 },
|
||||||
|
{ value: "BB", weight: 25 }
|
||||||
|
]
|
||||||
|
};
|
||||||
|
|
||||||
|
const PLACEMENT_PATTERN: PricePlacement[] = ["A", "AA", "B", "BB"];
|
||||||
|
|
||||||
const pick = <T,>(items: T[], seed: number): T => {
|
const pick = <T,>(items: T[], seed: number): T => {
|
||||||
return items[Math.abs(seed) % items.length];
|
return items[Math.abs(seed) % items.length];
|
||||||
};
|
};
|
||||||
|
|
@ -107,6 +145,19 @@ const pickWeighted = <T extends { weight: number }>(items: T[], seed: number): T
|
||||||
return items[0];
|
return items[0];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const pickWeightedValue = <T>(items: WeightedValue<T>[], seed: number): T => {
|
||||||
|
return pickWeighted(items, seed).value;
|
||||||
|
};
|
||||||
|
|
||||||
|
const pickPlacement = (burst: Burst, index: number): PricePlacement => {
|
||||||
|
const placementOptions = PRICE_PLACEMENTS[burst.scenarioId] ?? PRICE_PLACEMENTS.noise;
|
||||||
|
const offset = Math.abs(burst.seed) % PLACEMENT_PATTERN.length;
|
||||||
|
if (index < PLACEMENT_PATTERN.length) {
|
||||||
|
return PLACEMENT_PATTERN[(offset + index) % PLACEMENT_PATTERN.length];
|
||||||
|
}
|
||||||
|
return pickWeightedValue(placementOptions, burst.seed + index * 11);
|
||||||
|
};
|
||||||
|
|
||||||
const hashSymbol = (value: string): number => {
|
const hashSymbol = (value: string): number => {
|
||||||
let hash = 0;
|
let hash = 0;
|
||||||
for (let i = 0; i < value.length; i += 1) {
|
for (let i = 0; i < value.length; i += 1) {
|
||||||
|
|
@ -128,7 +179,8 @@ const formatExpiry = (now: number, offsetDays: number): string => {
|
||||||
const buildBurst = (burstIndex: number, now: number): Burst => {
|
const buildBurst = (burstIndex: number, now: number): Burst => {
|
||||||
const symbol = SP500_SYMBOLS[burstIndex % SP500_SYMBOLS.length];
|
const symbol = SP500_SYMBOLS[burstIndex % SP500_SYMBOLS.length];
|
||||||
const symbolHash = hashSymbol(symbol);
|
const symbolHash = hashSymbol(symbol);
|
||||||
const scenario = pickWeighted(SCENARIOS, symbolHash + burstIndex * 7);
|
const seed = symbolHash + burstIndex * 7;
|
||||||
|
const scenario = pickWeighted(SCENARIOS, seed);
|
||||||
const baseUnderlying = 30 + (symbolHash % 470);
|
const baseUnderlying = 30 + (symbolHash % 470);
|
||||||
const expiryOffset = pick(EXPIRY_OFFSETS, symbolHash + burstIndex);
|
const expiryOffset = pick(EXPIRY_OFFSETS, symbolHash + burstIndex);
|
||||||
const expiry = formatExpiry(now, expiryOffset);
|
const expiry = formatExpiry(now, expiryOffset);
|
||||||
|
|
@ -166,7 +218,9 @@ const buildBurst = (burstIndex: number, now: number): Burst => {
|
||||||
exchange,
|
exchange,
|
||||||
conditions,
|
conditions,
|
||||||
printCount,
|
printCount,
|
||||||
priceStep
|
priceStep,
|
||||||
|
scenarioId: scenario.id,
|
||||||
|
seed
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -177,6 +231,7 @@ export const createSyntheticOptionsAdapter = (
|
||||||
name: "synthetic",
|
name: "synthetic",
|
||||||
start: (handlers: OptionIngestHandlers) => {
|
start: (handlers: OptionIngestHandlers) => {
|
||||||
let seq = 0;
|
let seq = 0;
|
||||||
|
let nbboSeq = 0;
|
||||||
let burstIndex = 0;
|
let burstIndex = 0;
|
||||||
let currentBurst: Burst | null = null;
|
let currentBurst: Burst | null = null;
|
||||||
let remainingRuns = 0;
|
let remainingRuns = 0;
|
||||||
|
|
@ -203,6 +258,24 @@ export const createSyntheticOptionsAdapter = (
|
||||||
const priceJitter = ((i % 3) - 1) * 0.004;
|
const priceJitter = ((i % 3) - 1) * 0.004;
|
||||||
const sizeJitter = ((i % 3) - 1) * 0.08;
|
const sizeJitter = ((i % 3) - 1) * 0.08;
|
||||||
const priceMultiplier = 1 + burst.priceStep * i + priceJitter;
|
const priceMultiplier = 1 + burst.priceStep * i + priceJitter;
|
||||||
|
const mid = Math.max(0.05, Number((burst.basePrice * priceMultiplier).toFixed(2)));
|
||||||
|
const spread = Math.max(0.02, Number((mid * 0.02).toFixed(2)));
|
||||||
|
const bid = Math.max(0.01, Number((mid - spread / 2).toFixed(2)));
|
||||||
|
const ask = Math.max(bid + 0.01, Number((mid + spread / 2).toFixed(2)));
|
||||||
|
const tick = Math.max(0.01, Number((spread * 0.25).toFixed(2)));
|
||||||
|
const placement = pickPlacement(burst, i);
|
||||||
|
let tradePrice = mid;
|
||||||
|
|
||||||
|
if (placement === "AA") {
|
||||||
|
tradePrice = ask + tick;
|
||||||
|
} else if (placement === "A") {
|
||||||
|
tradePrice = ask;
|
||||||
|
} else if (placement === "BB") {
|
||||||
|
tradePrice = Math.max(0.01, bid - tick);
|
||||||
|
} else {
|
||||||
|
tradePrice = bid;
|
||||||
|
}
|
||||||
|
|
||||||
const print: OptionPrint = {
|
const print: OptionPrint = {
|
||||||
source_ts: now + i * 5,
|
source_ts: now + i * 5,
|
||||||
ingest_ts: now + i * 5,
|
ingest_ts: now + i * 5,
|
||||||
|
|
@ -210,13 +283,34 @@ export const createSyntheticOptionsAdapter = (
|
||||||
trace_id: `synthetic-options-${seq}`,
|
trace_id: `synthetic-options-${seq}`,
|
||||||
ts: now + i * 5,
|
ts: now + i * 5,
|
||||||
option_contract_id: burst.contractId,
|
option_contract_id: burst.contractId,
|
||||||
price: Math.max(0.05, Number((burst.basePrice * priceMultiplier).toFixed(2))),
|
price: tradePrice,
|
||||||
size: Math.max(1, Math.round(burst.baseSize * (1 + sizeJitter))),
|
size: Math.max(1, Math.round(burst.baseSize * (1 + sizeJitter))),
|
||||||
exchange: burst.exchange,
|
exchange: burst.exchange,
|
||||||
conditions: burst.conditions
|
conditions: burst.conditions
|
||||||
};
|
};
|
||||||
|
|
||||||
void handlers.onTrade(print);
|
void handlers.onTrade(print);
|
||||||
|
|
||||||
|
if (handlers.onNBBO) {
|
||||||
|
nbboSeq += 1;
|
||||||
|
const sizeBase = Math.max(1, Math.round(burst.baseSize * 0.4));
|
||||||
|
const bidSize = Math.max(1, Math.round(sizeBase * (1 + sizeJitter)));
|
||||||
|
const askSize = Math.max(1, Math.round(sizeBase * (1 - sizeJitter)));
|
||||||
|
const nbbo: OptionNBBO = {
|
||||||
|
source_ts: print.ts,
|
||||||
|
ingest_ts: print.ingest_ts,
|
||||||
|
seq: nbboSeq,
|
||||||
|
trace_id: `synthetic-nbbo-${nbboSeq}`,
|
||||||
|
ts: print.ts,
|
||||||
|
option_contract_id: burst.contractId,
|
||||||
|
bid,
|
||||||
|
ask,
|
||||||
|
bidSize,
|
||||||
|
askSize
|
||||||
|
};
|
||||||
|
|
||||||
|
void handlers.onNBBO(nbbo);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
remainingRuns -= 1;
|
remainingRuns -= 1;
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,9 @@
|
||||||
import { readEnv } from "@islandflow/config";
|
import { readEnv } from "@islandflow/config";
|
||||||
import { createLogger } from "@islandflow/observability";
|
import { createLogger } from "@islandflow/observability";
|
||||||
import {
|
import {
|
||||||
|
SUBJECT_OPTION_NBBO,
|
||||||
SUBJECT_OPTION_PRINTS,
|
SUBJECT_OPTION_PRINTS,
|
||||||
|
STREAM_OPTION_NBBO,
|
||||||
STREAM_OPTION_PRINTS,
|
STREAM_OPTION_PRINTS,
|
||||||
connectJetStreamWithRetry,
|
connectJetStreamWithRetry,
|
||||||
ensureStream,
|
ensureStream,
|
||||||
|
|
@ -9,10 +11,12 @@ import {
|
||||||
} from "@islandflow/bus";
|
} from "@islandflow/bus";
|
||||||
import {
|
import {
|
||||||
createClickHouseClient,
|
createClickHouseClient,
|
||||||
|
ensureOptionNBBOTable,
|
||||||
ensureOptionPrintsTable,
|
ensureOptionPrintsTable,
|
||||||
|
insertOptionNBBO,
|
||||||
insertOptionPrint
|
insertOptionPrint
|
||||||
} from "@islandflow/storage";
|
} from "@islandflow/storage";
|
||||||
import { OptionPrintSchema, type OptionPrint } from "@islandflow/types";
|
import { OptionNBBOSchema, OptionPrintSchema, type OptionNBBO, type OptionPrint } from "@islandflow/types";
|
||||||
import { createAlpacaOptionsAdapter } from "./adapters/alpaca";
|
import { createAlpacaOptionsAdapter } from "./adapters/alpaca";
|
||||||
import { createDatabentoOptionsAdapter } from "./adapters/databento";
|
import { createDatabentoOptionsAdapter } from "./adapters/databento";
|
||||||
import { createIbkrOptionsAdapter } from "./adapters/ibkr";
|
import { createIbkrOptionsAdapter } from "./adapters/ibkr";
|
||||||
|
|
@ -237,6 +241,19 @@ const run = async () => {
|
||||||
num_replicas: 1
|
num_replicas: 1
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await ensureStream(jsm, {
|
||||||
|
name: STREAM_OPTION_NBBO,
|
||||||
|
subjects: [SUBJECT_OPTION_NBBO],
|
||||||
|
retention: "limits",
|
||||||
|
storage: "file",
|
||||||
|
discard: "old",
|
||||||
|
max_msgs_per_subject: -1,
|
||||||
|
max_msgs: -1,
|
||||||
|
max_bytes: -1,
|
||||||
|
max_age: 0,
|
||||||
|
num_replicas: 1
|
||||||
|
});
|
||||||
|
|
||||||
const clickhouse = createClickHouseClient({
|
const clickhouse = createClickHouseClient({
|
||||||
url: env.CLICKHOUSE_URL,
|
url: env.CLICKHOUSE_URL,
|
||||||
database: env.CLICKHOUSE_DATABASE
|
database: env.CLICKHOUSE_DATABASE
|
||||||
|
|
@ -244,11 +261,13 @@ const run = async () => {
|
||||||
|
|
||||||
await retry("clickhouse table init", 20, 500, async () => {
|
await retry("clickhouse table init", 20, 500, async () => {
|
||||||
await ensureOptionPrintsTable(clickhouse);
|
await ensureOptionPrintsTable(clickhouse);
|
||||||
|
await ensureOptionNBBOTable(clickhouse);
|
||||||
});
|
});
|
||||||
|
|
||||||
const adapter = selectAdapter(env.OPTIONS_INGEST_ADAPTER);
|
const adapter = selectAdapter(env.OPTIONS_INGEST_ADAPTER);
|
||||||
logger.info("ingest adapter selected", { adapter: adapter.name });
|
logger.info("ingest adapter selected", { adapter: adapter.name });
|
||||||
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
|
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
|
||||||
|
const allowNbboPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
|
||||||
|
|
||||||
const stopAdapter: StopHandler = await adapter.start({
|
const stopAdapter: StopHandler = await adapter.start({
|
||||||
onTrade: async (candidate: OptionPrint) => {
|
onTrade: async (candidate: OptionPrint) => {
|
||||||
|
|
@ -277,6 +296,28 @@ const run = async () => {
|
||||||
trace_id: print.trace_id
|
trace_id: print.trace_id
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
onNBBO: async (candidate: OptionNBBO) => {
|
||||||
|
if (state.shuttingDown) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
if (!allowNbboPublish(now)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const nbbo = OptionNBBOSchema.parse(candidate);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await insertOptionNBBO(clickhouse, nbbo);
|
||||||
|
await publishJson(js, SUBJECT_OPTION_NBBO, nbbo);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("failed to publish option nbbo", {
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
trace_id: nbbo.trace_id
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue