Add smart-money option signal path and tape filters
This commit is contained in:
parent
758f111d7e
commit
27b0a399e6
23 changed files with 1827 additions and 175 deletions
2
.beads/issues.jsonl
Normal file
2
.beads/issues.jsonl
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
{"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-biq","title":"Finish raw live options delivery and filter/backpressure observability","description":"The smart-money signal path and Tape filters are in place, but the next firehose pass should finish server-side selective raw live delivery for options subscriptions and add explicit filtered-out/backpressure observability for API/web counters. This was discovered while landing islandflow-e4r.\n","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:28:58Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:28:58Z","dependencies":[{"issue_id":"islandflow-biq","depends_on_id":"islandflow-e4r","type":"discovered-from","created_at":"2026-04-28T16:28:58Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
76
README.md
76
README.md
|
|
@ -125,6 +125,15 @@ All runtime configuration comes from `.env`.
|
|||
- `OPTIONS_INGEST_ADAPTER` (`synthetic` | `alpaca` | `ibkr` | `databento`)
|
||||
- `EQUITIES_INGEST_ADAPTER` (`synthetic` | `alpaca`)
|
||||
- `EMIT_INTERVAL_MS` (synthetic emit cadence)
|
||||
- `SYNTHETIC_MARKET_MODE` (`realistic` | `active` | `firehose`, default `realistic`)
|
||||
- `SYNTHETIC_OPTIONS_MODE` (optional per-service override; falls back to `SYNTHETIC_MARKET_MODE`)
|
||||
- `SYNTHETIC_EQUITIES_MODE` (optional per-service override; falls back to `SYNTHETIC_MARKET_MODE`)
|
||||
|
||||
### Synthetic mode profiles
|
||||
|
||||
- `realistic` is the default local mode. Options produce materially more ordinary prints, fewer repeated bursts, and fewer alert-driving sweeps/spikes. Equities produce smaller batches and less relentless off-exchange activity.
|
||||
- `active` is a busier demo mode that still leaves meaningful visible history in the UI.
|
||||
- `firehose` is the stress profile for backpressure, hot-window eviction, and Databento-readiness validation.
|
||||
|
||||
### Options adapter settings
|
||||
|
||||
|
|
@ -142,6 +151,30 @@ All runtime configuration comes from `.env`.
|
|||
- Classifiers: `CLASSIFIER_SWEEP_MIN_PREMIUM`, `CLASSIFIER_SWEEP_MIN_COUNT`, `CLASSIFIER_SWEEP_MIN_PREMIUM_Z`, `CLASSIFIER_SPIKE_MIN_PREMIUM`, `CLASSIFIER_SPIKE_MIN_SIZE`, `CLASSIFIER_SPIKE_MIN_PREMIUM_Z`, `CLASSIFIER_SPIKE_MIN_SIZE_Z`, `CLASSIFIER_Z_MIN_SAMPLES`, `CLASSIFIER_MIN_NBBO_COVERAGE`, `CLASSIFIER_MIN_AGGRESSOR_RATIO`, `CLASSIFIER_0DTE_MAX_ATM_PCT`, `CLASSIFIER_0DTE_MIN_PREMIUM`, `CLASSIFIER_0DTE_MIN_SIZE`
|
||||
- Dark inference: `EQUITY_QUOTE_MAX_AGE_MS`, `DARK_INFER_WINDOW_MS`, `DARK_INFER_COOLDOWN_MS`, `DARK_INFER_MIN_BLOCK_SIZE`, `DARK_INFER_MIN_ACCUM_SIZE`, `DARK_INFER_MIN_ACCUM_COUNT`, `DARK_INFER_MIN_PRINT_SIZE`, `DARK_INFER_MAX_EVIDENCE`, `DARK_INFER_MAX_SPREAD_PCT`
|
||||
|
||||
### Options signal filtering
|
||||
|
||||
- `OPTIONS_SIGNAL_MODE` (`smart-money` | `balanced` | `all`, default `smart-money`)
|
||||
- `OPTIONS_SIGNAL_MIN_NOTIONAL` (default `10000`)
|
||||
- `OPTIONS_SIGNAL_ETF_MIN_NOTIONAL` (default `50000`)
|
||||
- `OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL` (default `25000`)
|
||||
- `OPTIONS_SIGNAL_MID_MIN_NOTIONAL` (default `20000`)
|
||||
- `OPTIONS_SIGNAL_NBBO_MAX_AGE_MS` (default `1500`)
|
||||
- `OPTIONS_SIGNAL_ETF_UNDERLYINGS` (default `SPY,QQQ,IWM,DIA,TLT,GLD,SLV,XLF,XLE,XLV,XLI,XLP,XLU,XLY,SMH,ARKK`)
|
||||
|
||||
Default `smart-money` behavior:
|
||||
|
||||
- reject sub-`10k` options prints,
|
||||
- reject ETF prints below `50k`,
|
||||
- reject `B` / `BB` prints below `25k`,
|
||||
- reject non-`SWEEP` / non-`ISO` `MID` prints below `20k`,
|
||||
- require `50k` when NBBO is missing or stale,
|
||||
- auto-keep `100k+`,
|
||||
- keep ask-side `A` / `AA` prints at `10k+`,
|
||||
- keep `SWEEP` / `ISO` prints at `25k+`,
|
||||
- keep `500+` contract prints at `10k+`.
|
||||
|
||||
`balanced` uses the same shape with lower thresholds. `all` marks every option print as signal-passing.
|
||||
|
||||
### Candles
|
||||
|
||||
- `CANDLE_INTERVALS_MS`, `CANDLE_MAX_LATE_MS`, `CANDLE_CACHE_LIMIT`, `CANDLE_DELIVER_POLICY`, `CANDLE_CONSUMER_RESET`
|
||||
|
|
@ -156,6 +189,7 @@ All runtime configuration comes from `.env`.
|
|||
- `NEXT_PUBLIC_LIVE_HOT_WINDOW` (frontend hot live window cap; default `2000`)
|
||||
- `NEXT_PUBLIC_PINNED_EVIDENCE_TTL_MS` (pinned evidence TTL; default `1200000`)
|
||||
- `NEXT_PUBLIC_PINNED_EVIDENCE_MAX_ITEMS` (pinned evidence cache guardrail; default `4000`)
|
||||
- `NEXT_PUBLIC_FLOW_FILTER_PRESET` (`smart-money` | `balanced` | `all`, default `smart-money`)
|
||||
|
||||
### Replay service
|
||||
|
||||
|
|
@ -170,8 +204,48 @@ All runtime configuration comes from `.env`.
|
|||
|
||||
- Python dependencies are required only for IBKR/Databento sidecars (`services/ingest-options/py/requirements.txt`).
|
||||
- Candle construction is server-side; the client consumes prebuilt OHLC events.
|
||||
- Option prints now persist as enriched raw rows and can be queried as either:
|
||||
- `view=signal` — default live/UI path and compute input.
|
||||
- `view=raw` — audit/debug path that preserves every stored print.
|
||||
- The default Tape page options/packets posture is now stock-only, hides `B` / `BB`, keeps calls and puts visible, and applies in-memory min-notional controls immediately.
|
||||
- Live retention uses a two-tier model:
|
||||
- API/Redis maintain a bounded hot cache per live generic channel.
|
||||
- UI keeps a bounded hot window for rendering performance.
|
||||
- UI keeps a bounded hot window for rendering performance around the signal view rather than raw noise.
|
||||
- Alert/drawer evidence is pinned and hydrated by id/trace so details remain inspectable after hot-window eviction.
|
||||
- Firehose-readiness strategy:
|
||||
- preserve raw ingest for storage/replay,
|
||||
- feed compute and default live UI from the filtered signal path,
|
||||
- add filterable live subscription contracts now so selective delivery can move server-side without reshaping the protocol later.
|
||||
- This repository is for personal, non-redistributed usage.
|
||||
|
||||
## Useful Examples
|
||||
|
||||
Realistic local demo:
|
||||
|
||||
```bash
|
||||
SYNTHETIC_MARKET_MODE=realistic \
|
||||
OPTIONS_SIGNAL_MODE=smart-money \
|
||||
bun run dev
|
||||
```
|
||||
|
||||
Active demo:
|
||||
|
||||
```bash
|
||||
SYNTHETIC_MARKET_MODE=active bun run dev
|
||||
```
|
||||
|
||||
Firehose stress test:
|
||||
|
||||
```bash
|
||||
SYNTHETIC_MARKET_MODE=firehose \
|
||||
NEXT_PUBLIC_LIVE_HOT_WINDOW=2000 \
|
||||
bun run dev
|
||||
```
|
||||
|
||||
Show raw options flow for debugging:
|
||||
|
||||
```text
|
||||
/prints/options?view=raw&security=all
|
||||
/history/options?view=raw&security=all&before_ts=<ts>&before_seq=<seq>
|
||||
/replay/options?view=raw&security=all&after_ts=<ts>&after_seq=<seq>
|
||||
```
|
||||
|
|
|
|||
|
|
@ -419,6 +419,55 @@ h3 {
|
|||
gap: 10px;
|
||||
}
|
||||
|
||||
.flow-filter-panel {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: flex-end;
|
||||
gap: 10px 16px;
|
||||
padding: 10px 12px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 12px;
|
||||
background: linear-gradient(180deg, rgba(255, 255, 255, 0.04), rgba(255, 255, 255, 0.02));
|
||||
}
|
||||
|
||||
.flow-filter-group {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.flow-filter-label {
|
||||
color: var(--muted);
|
||||
font-size: 0.72rem;
|
||||
letter-spacing: 0.08em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.flow-filter-check {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 0.84rem;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.filter-chip {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 999px;
|
||||
background: rgba(255, 255, 255, 0.03);
|
||||
color: var(--text);
|
||||
padding: 6px 10px;
|
||||
font: inherit;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.filter-chip.is-active {
|
||||
border-color: rgba(127, 234, 170, 0.6);
|
||||
background: rgba(127, 234, 170, 0.14);
|
||||
color: var(--accent-strong);
|
||||
}
|
||||
|
||||
.overview-strip,
|
||||
.replay-matrix {
|
||||
display: grid;
|
||||
|
|
|
|||
|
|
@ -24,9 +24,18 @@ import type {
|
|||
InferredDarkEvent,
|
||||
LiveServerMessage,
|
||||
LiveSubscription,
|
||||
OptionFlowFilters,
|
||||
OptionNbboSide,
|
||||
OptionSecurityType,
|
||||
OptionType,
|
||||
OptionNBBO,
|
||||
OptionPrint
|
||||
} from "@islandflow/types";
|
||||
import {
|
||||
getSubscriptionKey as getLiveSubscriptionKey,
|
||||
matchesFlowPacketFilters,
|
||||
matchesOptionPrintFilters
|
||||
} from "@islandflow/types";
|
||||
import { createChart, type IChartApi, type SeriesMarker, type UTCTimestamp } from "lightweight-charts";
|
||||
|
||||
const parseBoundedInt = (
|
||||
|
|
@ -61,6 +70,7 @@ const PINNED_EVIDENCE_MAX_ITEMS = parseBoundedInt(
|
|||
const NBBO_MAX_AGE_MS = Number(process.env.NEXT_PUBLIC_NBBO_MAX_AGE_MS);
|
||||
const NBBO_MAX_AGE_MS_SAFE =
|
||||
Number.isFinite(NBBO_MAX_AGE_MS) && NBBO_MAX_AGE_MS > 0 ? NBBO_MAX_AGE_MS : 1000;
|
||||
const FLOW_FILTER_PRESET = process.env.NEXT_PUBLIC_FLOW_FILTER_PRESET ?? "smart-money";
|
||||
const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1"]);
|
||||
const CANDLE_INTERVALS = [
|
||||
{ label: "1m", ms: 60000 },
|
||||
|
|
@ -614,6 +624,33 @@ const getJoinBoolean = (join: EquityPrintJoin, key: string): boolean => {
|
|||
|
||||
type NbboSide = "AA" | "A" | "B" | "BB";
|
||||
|
||||
const DEFAULT_FLOW_SIDES: OptionNbboSide[] = ["AA", "A", "MID"];
|
||||
const DEFAULT_FLOW_OPTION_TYPES: OptionType[] = ["call", "put"];
|
||||
const DEFAULT_FLOW_SECURITY_TYPES: OptionSecurityType[] = ["stock"];
|
||||
|
||||
const buildDefaultFlowFilters = (): OptionFlowFilters => ({
|
||||
view: "signal",
|
||||
securityTypes: DEFAULT_FLOW_SECURITY_TYPES,
|
||||
nbboSides: DEFAULT_FLOW_SIDES,
|
||||
optionTypes: DEFAULT_FLOW_OPTION_TYPES,
|
||||
minNotional:
|
||||
FLOW_FILTER_PRESET === "all"
|
||||
? undefined
|
||||
: FLOW_FILTER_PRESET === "balanced"
|
||||
? 5_000
|
||||
: undefined
|
||||
});
|
||||
|
||||
const toggleFilterValue = <T extends string>(values: T[] | undefined, value: T, enabled: boolean): T[] => {
|
||||
const current = new Set(values ?? []);
|
||||
if (enabled) {
|
||||
current.add(value);
|
||||
} else {
|
||||
current.delete(value);
|
||||
}
|
||||
return [...current].sort();
|
||||
};
|
||||
|
||||
const classifyNbboSide = (price: number, quote: OptionNBBO | null | undefined): NbboSide | null => {
|
||||
if (!quote || !Number.isFinite(price)) {
|
||||
return null;
|
||||
|
|
@ -935,6 +972,7 @@ type TapeConfig<T> = {
|
|||
getReplayKey?: (item: T) => string | null;
|
||||
replaySourceKey?: string | null;
|
||||
onReplaySourceKey?: (key: string | null) => void;
|
||||
queryParams?: Record<string, string | null | undefined>;
|
||||
};
|
||||
|
||||
const useTape = <T extends SortableItem & { seq: number }>(
|
||||
|
|
@ -947,6 +985,7 @@ const useTape = <T extends SortableItem & { seq: number }>(
|
|||
const getReplayKey = config.getReplayKey ?? extractTracePrefix;
|
||||
const replaySourceKey = config.replaySourceKey ?? null;
|
||||
const onReplaySourceKey = config.onReplaySourceKey;
|
||||
const queryParams = config.queryParams;
|
||||
const [status, setStatus] = useState<WsStatus>("connecting");
|
||||
const [items, setItems] = useState<T[]>([]);
|
||||
const [lastUpdate, setLastUpdate] = useState<number | null>(null);
|
||||
|
|
@ -1053,6 +1092,11 @@ const useTape = <T extends SortableItem & { seq: number }>(
|
|||
try {
|
||||
const url = new URL(buildApiUrl(latestPath));
|
||||
url.searchParams.set("limit", "1");
|
||||
for (const [key, value] of Object.entries(queryParams ?? {})) {
|
||||
if (value) {
|
||||
url.searchParams.set(key, value);
|
||||
}
|
||||
}
|
||||
if (replaySourceKey) {
|
||||
url.searchParams.set("source", replaySourceKey);
|
||||
}
|
||||
|
|
@ -1076,7 +1120,7 @@ const useTape = <T extends SortableItem & { seq: number }>(
|
|||
return () => {
|
||||
active = false;
|
||||
};
|
||||
}, [mode, latestPath, getItemTs, replaySourceKey]);
|
||||
}, [mode, latestPath, getItemTs, replaySourceKey, queryParams]);
|
||||
|
||||
useEffect(() => {
|
||||
if (mode !== "live" || config.liveEnabled === false) {
|
||||
|
|
@ -1196,6 +1240,11 @@ const useTape = <T extends SortableItem & { seq: number }>(
|
|||
url.searchParams.set("after_ts", cursor.ts.toString());
|
||||
url.searchParams.set("after_seq", cursor.seq.toString());
|
||||
url.searchParams.set("limit", batchSize.toString());
|
||||
for (const [key, value] of Object.entries(queryParams ?? {})) {
|
||||
if (value) {
|
||||
url.searchParams.set(key, value);
|
||||
}
|
||||
}
|
||||
const desiredSource = replaySourceKey ?? replaySourceRef.current;
|
||||
if (desiredSource) {
|
||||
url.searchParams.set("source", desiredSource);
|
||||
|
|
@ -1309,7 +1358,8 @@ const useTape = <T extends SortableItem & { seq: number }>(
|
|||
getItemTs,
|
||||
getReplayKey,
|
||||
replaySourceKey,
|
||||
onReplaySourceKey
|
||||
onReplaySourceKey,
|
||||
queryParams
|
||||
]);
|
||||
|
||||
return {
|
||||
|
|
@ -1589,21 +1639,11 @@ type LiveSessionState = {
|
|||
chartOverlay: EquityPrint[];
|
||||
};
|
||||
|
||||
const getLiveSubscriptionKey = (subscription: LiveSubscription): string => {
|
||||
switch (subscription.channel) {
|
||||
case "equity-candles":
|
||||
return `${subscription.channel}|${subscription.underlying_id}|${subscription.interval_ms}`;
|
||||
case "equity-overlay":
|
||||
return `${subscription.channel}|${subscription.underlying_id}`;
|
||||
default:
|
||||
return subscription.channel;
|
||||
}
|
||||
};
|
||||
|
||||
const getLiveManifest = (
|
||||
pathname: string,
|
||||
chartTicker: string,
|
||||
chartIntervalMs: number
|
||||
chartIntervalMs: number,
|
||||
flowFilters: OptionFlowFilters
|
||||
): LiveSubscription[] => {
|
||||
const chartSubs: LiveSubscription[] = [
|
||||
{ channel: "equity-candles", underlying_id: chartTicker, interval_ms: chartIntervalMs },
|
||||
|
|
@ -1612,10 +1652,10 @@ const getLiveManifest = (
|
|||
|
||||
if (pathname === "/tape") {
|
||||
return [
|
||||
{ channel: "options" },
|
||||
{ channel: "options", filters: flowFilters },
|
||||
{ channel: "nbbo" },
|
||||
{ channel: "equities" },
|
||||
{ channel: "flow" }
|
||||
{ channel: "flow", filters: flowFilters }
|
||||
];
|
||||
}
|
||||
|
||||
|
|
@ -1645,7 +1685,8 @@ const useLiveSession = (
|
|||
enabled: boolean,
|
||||
pathname: string,
|
||||
chartTicker: string,
|
||||
chartIntervalMs: number
|
||||
chartIntervalMs: number,
|
||||
flowFilters: OptionFlowFilters
|
||||
): LiveSessionState => {
|
||||
const [status, setStatus] = useState<WsStatus>(enabled ? "connecting" : "disconnected");
|
||||
const [lastUpdate, setLastUpdate] = useState<number | null>(null);
|
||||
|
|
@ -1664,8 +1705,8 @@ const useLiveSession = (
|
|||
const subscribedKeysRef = useRef<Set<string>>(new Set());
|
||||
const subscribedMapRef = useRef<Map<string, LiveSubscription>>(new Map());
|
||||
const manifest = useMemo(
|
||||
() => getLiveManifest(pathname, chartTicker.toUpperCase(), chartIntervalMs),
|
||||
[pathname, chartTicker, chartIntervalMs]
|
||||
() => getLiveManifest(pathname, chartTicker.toUpperCase(), chartIntervalMs, flowFilters),
|
||||
[pathname, chartTicker, chartIntervalMs, flowFilters]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -3079,6 +3120,7 @@ const useTerminalState = () => {
|
|||
const [selectedDarkEvent, setSelectedDarkEvent] = useState<InferredDarkEvent | null>(null);
|
||||
const [selectedClassifierHit, setSelectedClassifierHit] = useState<ClassifierHitEvent | null>(null);
|
||||
const [filterInput, setFilterInput] = useState<string>("");
|
||||
const [flowFilters, setFlowFilters] = useState<OptionFlowFilters>(() => buildDefaultFlowFilters());
|
||||
const [chartIntervalMs, setChartIntervalMs] = useState<number>(CANDLE_INTERVALS[0].ms);
|
||||
const activeTickers = useMemo(() => {
|
||||
const parts = filterInput
|
||||
|
|
@ -3089,7 +3131,13 @@ const useTerminalState = () => {
|
|||
}, [filterInput]);
|
||||
const tickerSet = useMemo(() => new Set(activeTickers), [activeTickers]);
|
||||
const chartTicker = useMemo(() => activeTickers[0] ?? "SPY", [activeTickers]);
|
||||
const liveSession = useLiveSession(mode === "live", pathname, chartTicker, chartIntervalMs);
|
||||
const liveSession = useLiveSession(
|
||||
mode === "live",
|
||||
pathname,
|
||||
chartTicker,
|
||||
chartIntervalMs,
|
||||
flowFilters
|
||||
);
|
||||
|
||||
const handleReplaySource = useCallback((value: string | null) => {
|
||||
setReplaySource(value);
|
||||
|
|
@ -3115,6 +3163,20 @@ const useTerminalState = () => {
|
|||
classifierScroll.isAtTopRef
|
||||
);
|
||||
const disableReplayGrouping = useCallback(() => null, []);
|
||||
const optionQueryParams = useMemo<Record<string, string | undefined>>(
|
||||
() => ({
|
||||
view: flowFilters.view ?? "signal",
|
||||
security:
|
||||
flowFilters.securityTypes?.length === 1 ? flowFilters.securityTypes[0] : undefined,
|
||||
side: flowFilters.nbboSides?.length ? flowFilters.nbboSides.join(",") : undefined,
|
||||
type: flowFilters.optionTypes?.length ? flowFilters.optionTypes.join(",") : undefined,
|
||||
min_notional:
|
||||
typeof flowFilters.minNotional === "number"
|
||||
? String(flowFilters.minNotional)
|
||||
: undefined
|
||||
}),
|
||||
[flowFilters]
|
||||
);
|
||||
|
||||
const options = useTape<OptionPrint>({
|
||||
mode,
|
||||
|
|
@ -3128,7 +3190,8 @@ const useTerminalState = () => {
|
|||
captureScroll: optionsAnchor.capture,
|
||||
onNewItems: optionsScroll.onNewItems,
|
||||
getReplayKey: extractReplaySource,
|
||||
onReplaySourceKey: handleReplaySource
|
||||
onReplaySourceKey: handleReplaySource,
|
||||
queryParams: optionQueryParams
|
||||
});
|
||||
|
||||
const equities = useTape<EquityPrint>({
|
||||
|
|
@ -3672,13 +3735,16 @@ const useTerminalState = () => {
|
|||
);
|
||||
|
||||
const filteredOptions = useMemo(() => {
|
||||
if (tickerSet.size === 0) {
|
||||
return optionsFeed.items;
|
||||
return optionsFeed.items.filter((print) => {
|
||||
if (!matchesOptionPrintFilters(print, flowFilters)) {
|
||||
return false;
|
||||
}
|
||||
return optionsFeed.items.filter((print) =>
|
||||
matchesTicker(extractUnderlying(normalizeContractId(print.option_contract_id)))
|
||||
);
|
||||
}, [optionsFeed.items, matchesTicker, tickerSet]);
|
||||
if (tickerSet.size === 0) {
|
||||
return true;
|
||||
}
|
||||
return matchesTicker(extractUnderlying(normalizeContractId(print.option_contract_id)));
|
||||
});
|
||||
}, [flowFilters, optionsFeed.items, matchesTicker, tickerSet]);
|
||||
|
||||
const filteredEquities = useMemo(() => {
|
||||
if (tickerSet.size === 0) {
|
||||
|
|
@ -3698,13 +3764,16 @@ const useTerminalState = () => {
|
|||
}, [resolvedEquityJoinMap, equityPrintMap, inferredDarkFeed.items, matchesTicker, tickerSet]);
|
||||
|
||||
const filteredFlow = useMemo(() => {
|
||||
if (tickerSet.size === 0) {
|
||||
return flowFeed.items;
|
||||
return flowFeed.items.filter((packet) => {
|
||||
if (!matchesFlowPacketFilters(packet, flowFilters)) {
|
||||
return false;
|
||||
}
|
||||
return flowFeed.items.filter((packet) =>
|
||||
matchesTicker(extractUnderlying(extractPacketContract(packet)))
|
||||
);
|
||||
}, [flowFeed.items, extractPacketContract, matchesTicker, tickerSet]);
|
||||
if (tickerSet.size === 0) {
|
||||
return true;
|
||||
}
|
||||
return matchesTicker(extractUnderlying(extractPacketContract(packet)));
|
||||
});
|
||||
}, [flowFeed.items, flowFilters, extractPacketContract, matchesTicker, tickerSet]);
|
||||
|
||||
const filteredAlerts = useMemo(() => {
|
||||
if (tickerSet.size === 0) {
|
||||
|
|
@ -4000,6 +4069,8 @@ const useTerminalState = () => {
|
|||
setSelectedClassifierHit,
|
||||
filterInput,
|
||||
setFilterInput,
|
||||
flowFilters,
|
||||
setFlowFilters,
|
||||
chartIntervalMs,
|
||||
setChartIntervalMs,
|
||||
optionsScroll,
|
||||
|
|
@ -4088,6 +4159,101 @@ const PageFrame = ({ title, actions, children }: PageFrameProps) => {
|
|||
);
|
||||
};
|
||||
|
||||
const FlowFilterControls = () => {
|
||||
const state = useTerminal();
|
||||
const filters = state.flowFilters;
|
||||
|
||||
const toggleSecurity = (value: OptionSecurityType, enabled: boolean) => {
|
||||
state.setFlowFilters((prev) => ({
|
||||
...prev,
|
||||
securityTypes: toggleFilterValue(prev.securityTypes, value, enabled)
|
||||
}));
|
||||
};
|
||||
|
||||
const toggleSide = (value: OptionNbboSide, enabled: boolean) => {
|
||||
state.setFlowFilters((prev) => ({
|
||||
...prev,
|
||||
nbboSides: toggleFilterValue(prev.nbboSides, value, enabled)
|
||||
}));
|
||||
};
|
||||
|
||||
const toggleOptionType = (value: OptionType, enabled: boolean) => {
|
||||
state.setFlowFilters((prev) => ({
|
||||
...prev,
|
||||
optionTypes: toggleFilterValue(prev.optionTypes, value, enabled)
|
||||
}));
|
||||
};
|
||||
|
||||
const applyMinNotional = (value: number | undefined) => {
|
||||
state.setFlowFilters((prev) => ({
|
||||
...prev,
|
||||
minNotional: value
|
||||
}));
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flow-filter-panel">
|
||||
<div className="flow-filter-group">
|
||||
<span className="flow-filter-label">Security</span>
|
||||
{(["stock", "etf"] as OptionSecurityType[]).map((value) => (
|
||||
<label className="flow-filter-check" key={value}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={(filters.securityTypes ?? []).includes(value)}
|
||||
onChange={(event) => toggleSecurity(value, event.target.checked)}
|
||||
/>
|
||||
<span>{value.toUpperCase()}</span>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
<div className="flow-filter-group">
|
||||
<span className="flow-filter-label">Side</span>
|
||||
{(["AA", "A", "MID", "B", "BB"] as OptionNbboSide[]).map((value) => (
|
||||
<label className="flow-filter-check" key={value}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={(filters.nbboSides ?? []).includes(value)}
|
||||
onChange={(event) => toggleSide(value, event.target.checked)}
|
||||
/>
|
||||
<span>{value}</span>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
<div className="flow-filter-group">
|
||||
<span className="flow-filter-label">Type</span>
|
||||
{(["call", "put"] as OptionType[]).map((value) => (
|
||||
<label className="flow-filter-check" key={value}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={(filters.optionTypes ?? []).includes(value)}
|
||||
onChange={(event) => toggleOptionType(value, event.target.checked)}
|
||||
/>
|
||||
<span>{value}</span>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
<div className="flow-filter-group">
|
||||
<span className="flow-filter-label">Min Notional</span>
|
||||
{[
|
||||
{ label: "All signal", value: undefined },
|
||||
{ label: ">= 25k", value: 25_000 },
|
||||
{ label: ">= 50k", value: 50_000 },
|
||||
{ label: ">= 100k", value: 100_000 }
|
||||
].map((preset) => (
|
||||
<button
|
||||
className={`filter-chip ${filters.minNotional === preset.value ? "is-active" : ""}`}
|
||||
key={preset.label}
|
||||
type="button"
|
||||
onClick={() => applyMinNotional(preset.value)}
|
||||
>
|
||||
{preset.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
type PaneProps = {
|
||||
title: string;
|
||||
status?: ReactNode;
|
||||
|
|
@ -4250,8 +4416,8 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => {
|
|||
const nbboAge = quote ? Math.abs(print.ts - quote.ts) : null;
|
||||
const nbboStale = nbboAge !== null && nbboAge > NBBO_MAX_AGE_MS_SAFE;
|
||||
const nbboMid = quote ? (quote.bid + quote.ask) / 2 : null;
|
||||
const nbboSide = classifyNbboSide(print.price, quote);
|
||||
const notional = print.price * print.size * 100;
|
||||
const nbboSide = print.nbbo_side ?? classifyNbboSide(print.price, quote);
|
||||
const notional = print.notional ?? print.price * print.size * 100;
|
||||
|
||||
return (
|
||||
<div className="row" key={`${print.trace_id}-${print.seq}`}>
|
||||
|
|
@ -4295,11 +4461,13 @@ const OptionsPane = ({ limit }: OptionsPaneProps) => {
|
|||
</span>
|
||||
</span>
|
||||
) : null}
|
||||
{nbboStale ? <span className="pill nbbo-stale">Stale</span> : null}
|
||||
{print.nbbo_side === "STALE" || nbboStale ? <span className="pill nbbo-stale">Stale</span> : null}
|
||||
</div>
|
||||
) : (
|
||||
<div className="meta nbbo-meta">
|
||||
<span className="pill nbbo-missing">NBBO missing</span>
|
||||
<span className="pill nbbo-missing">
|
||||
{print.nbbo_side === "STALE" ? "NBBO stale" : "NBBO missing"}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
|
@ -5051,7 +5219,7 @@ export function OverviewRoute() {
|
|||
|
||||
export function TapeRoute() {
|
||||
return (
|
||||
<PageFrame title="Tape">
|
||||
<PageFrame title="Tape" actions={<FlowFilterControls />}>
|
||||
<div className="page-grid page-grid-tape">
|
||||
<OptionsPane />
|
||||
<EquitiesPane />
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
export const STREAM_OPTION_PRINTS = "OPTIONS_PRINTS";
|
||||
export const SUBJECT_OPTION_PRINTS = "options.prints";
|
||||
export const STREAM_OPTION_SIGNAL_PRINTS = "OPTIONS_SIGNAL_PRINTS";
|
||||
export const SUBJECT_OPTION_SIGNAL_PRINTS = "options.prints.signal";
|
||||
export const STREAM_OPTION_NBBO = "OPTIONS_NBBO";
|
||||
export const SUBJECT_OPTION_NBBO = "options.nbbo";
|
||||
export const STREAM_EQUITY_PRINTS = "EQUITY_PRINTS";
|
||||
|
|
|
|||
|
|
@ -20,11 +20,14 @@ import type {
|
|||
InferredDarkEvent,
|
||||
FlowPacket,
|
||||
OptionNBBO,
|
||||
OptionPrint
|
||||
OptionPrint,
|
||||
OptionFlowFilters,
|
||||
OptionFlowView
|
||||
} from "@islandflow/types";
|
||||
import {
|
||||
normalizeOptionPrint,
|
||||
optionPrintsTableDDL,
|
||||
optionPrintsTableMigrations,
|
||||
OPTION_PRINTS_TABLE
|
||||
} from "./option-prints";
|
||||
import { normalizeOptionNBBO, optionNBBOTableDDL, OPTION_NBBO_TABLE } from "./option-nbbo";
|
||||
|
|
@ -221,6 +224,9 @@ export const ensureOptionPrintsTable = async (
|
|||
await client.exec({
|
||||
query: optionPrintsTableDDL()
|
||||
});
|
||||
for (const query of optionPrintsTableMigrations()) {
|
||||
await client.exec({ query });
|
||||
}
|
||||
};
|
||||
|
||||
export const ensureOptionNBBOTable = async (
|
||||
|
|
@ -499,19 +505,78 @@ const normalizeNumericFields = (
|
|||
|
||||
const normalizeOptionRow = (row: unknown): unknown => {
|
||||
if (row && typeof row === "object") {
|
||||
return normalizeNumericFields(row as Record<string, unknown>, [
|
||||
const record = normalizeNumericFields(row as Record<string, unknown>, [
|
||||
"source_ts",
|
||||
"ingest_ts",
|
||||
"seq",
|
||||
"ts",
|
||||
"price",
|
||||
"size"
|
||||
"size",
|
||||
"notional"
|
||||
]);
|
||||
|
||||
if ("is_etf" in record) {
|
||||
record.is_etf = Boolean(record.is_etf);
|
||||
}
|
||||
if ("signal_pass" in record) {
|
||||
record.signal_pass = Boolean(record.signal_pass);
|
||||
}
|
||||
if (record.signal_reasons == null) {
|
||||
record.signal_reasons = [];
|
||||
}
|
||||
return record;
|
||||
}
|
||||
|
||||
return row;
|
||||
};
|
||||
|
||||
export type OptionPrintQueryFilters = {
|
||||
view?: OptionFlowView;
|
||||
minNotional?: number;
|
||||
security?: "stock" | "etf" | "all";
|
||||
optionTypes?: string[];
|
||||
nbboSides?: string[];
|
||||
};
|
||||
|
||||
const buildOptionPrintFilterConditions = (
|
||||
filters: OptionPrintQueryFilters | undefined,
|
||||
tracePrefix: string | undefined
|
||||
): string[] => {
|
||||
const conditions: string[] = [];
|
||||
const traceCondition = buildTracePrefixCondition(tracePrefix);
|
||||
if (traceCondition) {
|
||||
conditions.push(traceCondition);
|
||||
}
|
||||
|
||||
if (!filters) {
|
||||
return conditions;
|
||||
}
|
||||
|
||||
if ((filters.view ?? "signal") === "signal") {
|
||||
conditions.push("signal_pass = 1");
|
||||
}
|
||||
|
||||
if (typeof filters.minNotional === "number" && Number.isFinite(filters.minNotional)) {
|
||||
conditions.push(`notional >= ${filters.minNotional}`);
|
||||
}
|
||||
|
||||
if (filters.security === "stock") {
|
||||
conditions.push("(is_etf = 0 OR is_etf IS NULL)");
|
||||
} else if (filters.security === "etf") {
|
||||
conditions.push("is_etf = 1");
|
||||
}
|
||||
|
||||
if (filters.optionTypes && filters.optionTypes.length > 0) {
|
||||
conditions.push(`option_type IN (${buildStringList(filters.optionTypes)})`);
|
||||
}
|
||||
|
||||
if (filters.nbboSides && filters.nbboSides.length > 0) {
|
||||
conditions.push(`nbbo_side IN (${buildStringList(filters.nbboSides)})`);
|
||||
}
|
||||
|
||||
return conditions;
|
||||
};
|
||||
|
||||
const normalizeOptionNbboRow = (row: unknown): unknown => {
|
||||
if (row && typeof row === "object") {
|
||||
return normalizeNumericFields(row as Record<string, unknown>, [
|
||||
|
|
@ -683,11 +748,12 @@ const normalizeAlertRow = (row: unknown): AlertRecord | null => {
|
|||
export const fetchRecentOptionPrints = async (
|
||||
client: ClickHouseClient,
|
||||
limit: number,
|
||||
tracePrefix?: string
|
||||
tracePrefix?: string,
|
||||
filters?: OptionPrintQueryFilters
|
||||
): Promise<OptionPrint[]> => {
|
||||
const safeLimit = clampLimit(limit);
|
||||
const condition = buildTracePrefixCondition(tracePrefix);
|
||||
const whereClause = condition ? ` WHERE ${condition}` : "";
|
||||
const conditions = buildOptionPrintFilterConditions(filters, tracePrefix);
|
||||
const whereClause = conditions.length > 0 ? ` WHERE ${conditions.join(" AND ")}` : "";
|
||||
const result = await client.query({
|
||||
query: `SELECT * FROM ${OPTION_PRINTS_TABLE}${whereClause} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
|
||||
format: "JSONEachRow"
|
||||
|
|
@ -855,16 +921,19 @@ export const fetchOptionPrintsAfter = async (
|
|||
afterTs: number,
|
||||
afterSeq: number,
|
||||
limit: number,
|
||||
tracePrefix?: string
|
||||
tracePrefix?: string,
|
||||
filters?: OptionPrintQueryFilters
|
||||
): Promise<OptionPrint[]> => {
|
||||
const safeLimit = clampLimit(limit);
|
||||
const safeAfterTs = clampCursor(afterTs);
|
||||
const safeAfterSeq = clampCursor(afterSeq);
|
||||
const traceCondition = buildTracePrefixCondition(tracePrefix);
|
||||
const traceClause = traceCondition ? ` AND ${traceCondition}` : "";
|
||||
const conditions = [
|
||||
`((ts, seq) > (${safeAfterTs}, ${safeAfterSeq}))`,
|
||||
...buildOptionPrintFilterConditions(filters, tracePrefix)
|
||||
];
|
||||
|
||||
const result = await client.query({
|
||||
query: `SELECT * FROM ${OPTION_PRINTS_TABLE} WHERE (ts, seq) > (${safeAfterTs}, ${safeAfterSeq})${traceClause} ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
|
||||
query: `SELECT * FROM ${OPTION_PRINTS_TABLE} WHERE ${conditions.join(" AND ")} ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
|
||||
format: "JSONEachRow"
|
||||
});
|
||||
|
||||
|
|
@ -1122,14 +1191,14 @@ export const fetchOptionPrintsBefore = async (
|
|||
beforeTs: number,
|
||||
beforeSeq: number,
|
||||
limit: number,
|
||||
tracePrefix?: string
|
||||
tracePrefix?: string,
|
||||
filters?: OptionPrintQueryFilters
|
||||
): Promise<OptionPrint[]> => {
|
||||
const safeLimit = clampLimit(limit);
|
||||
const conditions = [buildBeforeTupleCondition("ts", "seq", beforeTs, beforeSeq)];
|
||||
const traceCondition = buildTracePrefixCondition(tracePrefix);
|
||||
if (traceCondition) {
|
||||
conditions.push(traceCondition);
|
||||
}
|
||||
const conditions = [
|
||||
buildBeforeTupleCondition("ts", "seq", beforeTs, beforeSeq),
|
||||
...buildOptionPrintFilterConditions(filters, tracePrefix)
|
||||
];
|
||||
|
||||
const result = await client.query({
|
||||
query: `SELECT * FROM ${OPTION_PRINTS_TABLE} WHERE ${conditions.join(" AND ")} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
|
||||
|
|
|
|||
|
|
@ -14,16 +14,38 @@ CREATE TABLE IF NOT EXISTS ${OPTION_PRINTS_TABLE} (
|
|||
price Float64,
|
||||
size UInt32,
|
||||
exchange String,
|
||||
conditions Array(String)
|
||||
conditions Array(String),
|
||||
underlying_id Nullable(String),
|
||||
option_type Nullable(String),
|
||||
notional Nullable(Float64),
|
||||
nbbo_side Nullable(String),
|
||||
is_etf Nullable(Bool),
|
||||
signal_pass Nullable(Bool),
|
||||
signal_reasons Array(String) DEFAULT [],
|
||||
signal_profile Nullable(String)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (ts, option_contract_id)
|
||||
`;
|
||||
};
|
||||
|
||||
export const optionPrintsTableMigrations = (): string[] => {
|
||||
return [
|
||||
`ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS underlying_id Nullable(String)`,
|
||||
`ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS option_type Nullable(String)`,
|
||||
`ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS notional Nullable(Float64)`,
|
||||
`ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS nbbo_side Nullable(String)`,
|
||||
`ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS is_etf Nullable(Bool)`,
|
||||
`ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS signal_pass Nullable(Bool)`,
|
||||
`ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS signal_reasons Array(String) DEFAULT []`,
|
||||
`ALTER TABLE ${OPTION_PRINTS_TABLE} ADD COLUMN IF NOT EXISTS signal_profile Nullable(String)`
|
||||
];
|
||||
};
|
||||
|
||||
export const normalizeOptionPrint = (print: OptionPrint): OptionPrint => {
|
||||
return {
|
||||
...print,
|
||||
conditions: print.conditions ?? []
|
||||
conditions: print.conditions ?? [],
|
||||
signal_reasons: print.signal_reasons ?? []
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
import { describe, expect, it } from "bun:test";
|
||||
import { createClickHouseClient, fetchOptionPrintsBefore, fetchOptionPrintsByTraceIds } from "../src/clickhouse";
|
||||
import {
|
||||
createClickHouseClient,
|
||||
fetchOptionPrintsBefore,
|
||||
fetchOptionPrintsByTraceIds,
|
||||
fetchRecentOptionPrints
|
||||
} from "../src/clickhouse";
|
||||
import { normalizeOptionPrint, optionPrintsTableDDL, OPTION_PRINTS_TABLE } from "../src/option-prints";
|
||||
|
||||
const basePrint = {
|
||||
|
|
@ -38,12 +43,24 @@ describe("option-prints storage helpers", () => {
|
|||
};
|
||||
};
|
||||
|
||||
await fetchRecentOptionPrints(client, 25, undefined, {
|
||||
view: "signal",
|
||||
security: "stock",
|
||||
nbboSides: ["AA", "A"],
|
||||
optionTypes: ["call"],
|
||||
minNotional: 25_000
|
||||
});
|
||||
await fetchOptionPrintsBefore(client, 100, 5, 20, "alpaca");
|
||||
await fetchOptionPrintsByTraceIds(client, ["trace-1", "trace-2"]);
|
||||
|
||||
expect(queries[0]).toContain("(ts, seq) < (100, 5)");
|
||||
expect(queries[0]).toContain("startsWith(trace_id, 'alpaca')");
|
||||
expect(queries[0]).toContain("ORDER BY ts DESC, seq DESC LIMIT 20");
|
||||
expect(queries[1]).toContain("trace_id IN ('trace-1', 'trace-2')");
|
||||
expect(queries[0]).toContain("signal_pass = 1");
|
||||
expect(queries[0]).toContain("(is_etf = 0 OR is_etf IS NULL)");
|
||||
expect(queries[0]).toContain("nbbo_side IN ('AA', 'A')");
|
||||
expect(queries[0]).toContain("option_type IN ('call')");
|
||||
expect(queries[0]).toContain("notional >= 25000");
|
||||
expect(queries[1]).toContain("(ts, seq) < (100, 5)");
|
||||
expect(queries[1]).toContain("startsWith(trace_id, 'alpaca')");
|
||||
expect(queries[1]).toContain("ORDER BY ts DESC, seq DESC LIMIT 20");
|
||||
expect(queries[2]).toContain("trace_id IN ('trace-1', 'trace-2')");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { z } from "zod";
|
||||
import { OptionNbboSideSchema, OptionTypeSchema, OptionsSignalModeSchema } from "./options-flow";
|
||||
|
||||
export const EventMetaSchema = z.object({
|
||||
source_ts: z.number().int().nonnegative(),
|
||||
|
|
@ -16,7 +17,18 @@ export const OptionPrintSchema = EventMetaSchema.merge(
|
|||
price: z.number().nonnegative(),
|
||||
size: z.number().int().positive(),
|
||||
exchange: z.string().min(1),
|
||||
conditions: z.array(z.string().min(1)).optional()
|
||||
conditions: z.array(z.string().min(1)).optional(),
|
||||
underlying_id: z.preprocess((value) => (value === null ? undefined : value), z.string().min(1).optional()),
|
||||
option_type: z.preprocess((value) => (value === null ? undefined : value), OptionTypeSchema.optional()),
|
||||
notional: z.preprocess((value) => (value === null ? undefined : value), z.number().nonnegative().optional()),
|
||||
nbbo_side: z.preprocess((value) => (value === null ? undefined : value), OptionNbboSideSchema.optional()),
|
||||
is_etf: z.preprocess((value) => (value === null ? undefined : value), z.boolean().optional()),
|
||||
signal_pass: z.preprocess((value) => (value === null ? undefined : value), z.boolean().optional()),
|
||||
signal_reasons: z.array(z.string().min(1)).optional(),
|
||||
signal_profile: z.preprocess(
|
||||
(value) => (value === null ? undefined : value),
|
||||
OptionsSignalModeSchema.optional()
|
||||
)
|
||||
})
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
export * from "./events";
|
||||
export * from "./live";
|
||||
export * from "./options-flow";
|
||||
export * from "./sp500";
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@ import {
|
|||
OptionNBBOSchema,
|
||||
OptionPrintSchema
|
||||
} from "./events";
|
||||
import {
|
||||
OptionFlowFiltersSchema,
|
||||
optionFlowFilterKey
|
||||
} from "./options-flow";
|
||||
|
||||
export const CursorSchema = z.object({
|
||||
ts: z.number().int().nonnegative(),
|
||||
|
|
@ -47,7 +51,15 @@ export type LiveGenericChannel = z.infer<typeof LiveGenericChannelSchema>;
|
|||
|
||||
export const LiveSubscriptionSchema = z.discriminatedUnion("channel", [
|
||||
z.object({
|
||||
channel: LiveGenericChannelSchema
|
||||
channel: z.literal("options"),
|
||||
filters: OptionFlowFiltersSchema.optional()
|
||||
}),
|
||||
z.object({
|
||||
channel: z.literal("flow"),
|
||||
filters: OptionFlowFiltersSchema.optional()
|
||||
}),
|
||||
z.object({
|
||||
channel: z.enum(["nbbo", "equities", "equity-joins", "classifier-hits", "alerts", "inferred-dark"])
|
||||
}),
|
||||
z.object({
|
||||
channel: z.literal("equity-candles"),
|
||||
|
|
@ -165,6 +177,9 @@ export type LiveServerMessage = z.infer<typeof LiveServerMessageSchema>;
|
|||
|
||||
export const getSubscriptionKey = (subscription: LiveSubscription): string => {
|
||||
switch (subscription.channel) {
|
||||
case "options":
|
||||
case "flow":
|
||||
return `${subscription.channel}|${optionFlowFilterKey(subscription.filters)}`;
|
||||
case "equity-candles":
|
||||
return `${subscription.channel}|${subscription.underlying_id}|${subscription.interval_ms}`;
|
||||
case "equity-overlay":
|
||||
|
|
|
|||
464
packages/types/src/options-flow.ts
Normal file
464
packages/types/src/options-flow.ts
Normal file
|
|
@ -0,0 +1,464 @@
|
|||
import { z } from "zod";
|
||||
import type { FlowPacket, OptionNBBO, OptionPrint } from "./events";
|
||||
|
||||
export const SyntheticMarketModeSchema = z.enum(["realistic", "active", "firehose"]);
|
||||
export type SyntheticMarketMode = z.infer<typeof SyntheticMarketModeSchema>;
|
||||
|
||||
export const OptionTypeSchema = z.enum(["call", "put"]);
|
||||
export type OptionType = z.infer<typeof OptionTypeSchema>;
|
||||
|
||||
export const OptionNbboSideSchema = z.enum(["AA", "A", "MID", "B", "BB", "MISSING", "STALE"]);
|
||||
export type OptionNbboSide = z.infer<typeof OptionNbboSideSchema>;
|
||||
|
||||
export const OptionFlowViewSchema = z.enum(["signal", "raw"]);
|
||||
export type OptionFlowView = z.infer<typeof OptionFlowViewSchema>;
|
||||
|
||||
export const OptionSecurityTypeSchema = z.enum(["stock", "etf"]);
|
||||
export type OptionSecurityType = z.infer<typeof OptionSecurityTypeSchema>;
|
||||
|
||||
export const OptionsSignalModeSchema = z.enum(["smart-money", "balanced", "all"]);
|
||||
export type OptionsSignalMode = z.infer<typeof OptionsSignalModeSchema>;
|
||||
|
||||
export const OptionFlowFiltersSchema = z.object({
|
||||
view: OptionFlowViewSchema.optional(),
|
||||
securityTypes: z.array(OptionSecurityTypeSchema).optional(),
|
||||
nbboSides: z.array(OptionNbboSideSchema).optional(),
|
||||
optionTypes: z.array(OptionTypeSchema).optional(),
|
||||
minNotional: z.number().nonnegative().optional()
|
||||
});
|
||||
|
||||
export type OptionFlowFilters = z.infer<typeof OptionFlowFiltersSchema>;
|
||||
|
||||
export type ParsedOptionContract = {
|
||||
root: string;
|
||||
expiry: string;
|
||||
strike: number;
|
||||
right: "C" | "P";
|
||||
};
|
||||
|
||||
export type SyntheticModeResolution = {
|
||||
market: SyntheticMarketMode;
|
||||
options: SyntheticMarketMode;
|
||||
equities: SyntheticMarketMode;
|
||||
};
|
||||
|
||||
export type OptionsSignalConfig = {
|
||||
mode: OptionsSignalMode;
|
||||
minNotional: number;
|
||||
etfMinNotional: number;
|
||||
bidSideMinNotional: number;
|
||||
midMinNotional: number;
|
||||
missingNbboMinNotional: number;
|
||||
largePrintMinSize: number;
|
||||
largePrintMinNotional: number;
|
||||
sweepMinNotional: number;
|
||||
autoKeepMinNotional: number;
|
||||
nbboMaxAgeMs: number;
|
||||
etfUnderlyings: Set<string>;
|
||||
};
|
||||
|
||||
export type DerivedOptionPrintMetadata = {
|
||||
underlying_id?: string;
|
||||
option_type?: OptionType;
|
||||
notional?: number;
|
||||
nbbo_side?: OptionNbboSide;
|
||||
is_etf?: boolean;
|
||||
};
|
||||
|
||||
export type OptionSignalDecision = {
|
||||
signalPass: boolean;
|
||||
signalReasons: string[];
|
||||
signalProfile: OptionsSignalMode;
|
||||
};
|
||||
|
||||
const parseDashedContract = (value: string): ParsedOptionContract | null => {
|
||||
const parts = value.split("-");
|
||||
if (parts.length < 6) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const rightRaw = parts.at(-1) ?? "";
|
||||
if (rightRaw !== "C" && rightRaw !== "P") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const strikeRaw = parts.at(-2) ?? "";
|
||||
const strike = Number(strikeRaw);
|
||||
const expiryParts = parts.slice(-5, -2);
|
||||
const expiry = expiryParts.join("-");
|
||||
const root = parts.slice(0, -5).join("-");
|
||||
|
||||
if (!root || !expiry || !Number.isFinite(strike)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
root,
|
||||
expiry,
|
||||
strike,
|
||||
right: rightRaw
|
||||
};
|
||||
};
|
||||
|
||||
const parseOccContract = (value: string): ParsedOptionContract | null => {
|
||||
if (value.length < 15) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const tail = value.slice(-15);
|
||||
const root = value.slice(0, -15).trim();
|
||||
const expiryRaw = tail.slice(0, 6);
|
||||
const right = tail.slice(6, 7);
|
||||
const strikeRaw = tail.slice(7);
|
||||
|
||||
if (!/^\d{6}$/.test(expiryRaw) || !/^\d{8}$/.test(strikeRaw)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (right !== "C" && right !== "P") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const year = 2000 + Number(expiryRaw.slice(0, 2));
|
||||
const month = Number(expiryRaw.slice(2, 4)) - 1;
|
||||
const day = Number(expiryRaw.slice(4, 6));
|
||||
const expiryDate = new Date(Date.UTC(year, month, day));
|
||||
const expiry = expiryDate.toISOString().slice(0, 10);
|
||||
const strike = Number(strikeRaw) / 1000;
|
||||
|
||||
if (!root || !Number.isFinite(strike)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
root,
|
||||
expiry,
|
||||
strike,
|
||||
right
|
||||
};
|
||||
};
|
||||
|
||||
export const parseOptionContractId = (value: string | undefined): ParsedOptionContract | null => {
|
||||
if (!value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return parseDashedContract(value) ?? parseOccContract(value);
|
||||
};
|
||||
|
||||
export const resolveSyntheticMarketModes = (input: {
|
||||
syntheticMarketMode?: string | null | undefined;
|
||||
syntheticOptionsMode?: string | null | undefined;
|
||||
syntheticEquitiesMode?: string | null | undefined;
|
||||
}): SyntheticModeResolution => {
|
||||
const market = SyntheticMarketModeSchema.catch("realistic").parse(
|
||||
input.syntheticMarketMode ?? "realistic"
|
||||
);
|
||||
const options = SyntheticMarketModeSchema.catch(market).parse(
|
||||
input.syntheticOptionsMode ?? market
|
||||
);
|
||||
const equities = SyntheticMarketModeSchema.catch(market).parse(
|
||||
input.syntheticEquitiesMode ?? market
|
||||
);
|
||||
|
||||
return { market, options, equities };
|
||||
};
|
||||
|
||||
export const classifyOptionNbboSide = (
|
||||
price: number,
|
||||
quote: Pick<OptionNBBO, "bid" | "ask" | "ts"> | null | undefined,
|
||||
tradeTs: number,
|
||||
maxAgeMs: number
|
||||
): OptionNbboSide => {
|
||||
if (!quote || !Number.isFinite(price)) {
|
||||
return "MISSING";
|
||||
}
|
||||
|
||||
const bid = quote.bid;
|
||||
const ask = quote.ask;
|
||||
if (!Number.isFinite(bid) || !Number.isFinite(ask) || ask <= 0) {
|
||||
return "MISSING";
|
||||
}
|
||||
|
||||
const ageMs = Math.abs(tradeTs - quote.ts);
|
||||
if (ageMs > maxAgeMs) {
|
||||
return "STALE";
|
||||
}
|
||||
|
||||
const spread = Math.max(0, ask - bid);
|
||||
const epsilon = Math.max(0.01, spread * 0.05);
|
||||
|
||||
if (price > ask + epsilon) {
|
||||
return "AA";
|
||||
}
|
||||
if (price >= ask - epsilon) {
|
||||
return "A";
|
||||
}
|
||||
if (price < bid - epsilon) {
|
||||
return "BB";
|
||||
}
|
||||
if (price <= bid + epsilon) {
|
||||
return "B";
|
||||
}
|
||||
|
||||
return "MID";
|
||||
};
|
||||
|
||||
export const deriveOptionPrintMetadata = (
|
||||
print: Pick<OptionPrint, "option_contract_id" | "price" | "size" | "ts">,
|
||||
quote: Pick<OptionNBBO, "bid" | "ask" | "ts"> | null | undefined,
|
||||
config: Pick<OptionsSignalConfig, "nbboMaxAgeMs" | "etfUnderlyings">
|
||||
): DerivedOptionPrintMetadata => {
|
||||
const parsed = parseOptionContractId(print.option_contract_id);
|
||||
const underlying = parsed?.root?.toUpperCase();
|
||||
const optionType = parsed?.right === "C" ? "call" : parsed?.right === "P" ? "put" : undefined;
|
||||
const notional = Number.isFinite(print.price) && Number.isFinite(print.size)
|
||||
? Number((print.price * print.size * 100).toFixed(2))
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
underlying_id: underlying,
|
||||
option_type: optionType,
|
||||
notional,
|
||||
nbbo_side: classifyOptionNbboSide(print.price, quote, print.ts, config.nbboMaxAgeMs),
|
||||
is_etf: underlying ? config.etfUnderlyings.has(underlying) : undefined
|
||||
};
|
||||
};
|
||||
|
||||
const hasCondition = (conditions: string[] | undefined, expected: string): boolean => {
|
||||
return (conditions ?? []).some((condition) => condition.toUpperCase() === expected);
|
||||
};
|
||||
|
||||
const balancedThresholds = (config: OptionsSignalConfig): OptionsSignalConfig => ({
|
||||
...config,
|
||||
minNotional: Math.min(config.minNotional, 5_000),
|
||||
etfMinNotional: Math.min(config.etfMinNotional, 25_000),
|
||||
bidSideMinNotional: Math.min(config.bidSideMinNotional, 15_000),
|
||||
midMinNotional: Math.min(config.midMinNotional, 12_500),
|
||||
missingNbboMinNotional: Math.min(config.missingNbboMinNotional, 25_000),
|
||||
sweepMinNotional: Math.min(config.sweepMinNotional, 15_000),
|
||||
autoKeepMinNotional: Math.min(config.autoKeepMinNotional, 75_000)
|
||||
});
|
||||
|
||||
export const evaluateOptionSignal = (
|
||||
print: Pick<
|
||||
OptionPrint,
|
||||
"size" | "conditions" | "signal_profile" | "underlying_id" | "option_type" | "notional" | "nbbo_side" | "is_etf"
|
||||
>,
|
||||
baseConfig: OptionsSignalConfig
|
||||
): OptionSignalDecision => {
|
||||
const mode = print.signal_profile ?? baseConfig.mode;
|
||||
if (mode === "all") {
|
||||
return {
|
||||
signalPass: true,
|
||||
signalReasons: ["mode:all"],
|
||||
signalProfile: "all"
|
||||
};
|
||||
}
|
||||
|
||||
const config = mode === "balanced" ? balancedThresholds(baseConfig) : baseConfig;
|
||||
const reasons: string[] = [];
|
||||
const notional = print.notional ?? 0;
|
||||
const side = print.nbbo_side ?? "MISSING";
|
||||
const isSweepOrIso = hasCondition(print.conditions, "SWEEP") || hasCondition(print.conditions, "ISO");
|
||||
|
||||
if (notional < config.minNotional) {
|
||||
return {
|
||||
signalPass: false,
|
||||
signalReasons: ["reject:min-notional"],
|
||||
signalProfile: mode
|
||||
};
|
||||
}
|
||||
|
||||
if (notional >= config.autoKeepMinNotional) {
|
||||
reasons.push("keep:auto-large");
|
||||
}
|
||||
|
||||
if (print.is_etf && notional < config.etfMinNotional) {
|
||||
return {
|
||||
signalPass: false,
|
||||
signalReasons: ["reject:etf-min-notional"],
|
||||
signalProfile: mode
|
||||
};
|
||||
}
|
||||
|
||||
if ((side === "B" || side === "BB") && notional < config.bidSideMinNotional) {
|
||||
return {
|
||||
signalPass: false,
|
||||
signalReasons: ["reject:bid-side-min-notional"],
|
||||
signalProfile: mode
|
||||
};
|
||||
}
|
||||
|
||||
if (side === "MID" && !isSweepOrIso && notional < config.midMinNotional) {
|
||||
return {
|
||||
signalPass: false,
|
||||
signalReasons: ["reject:mid-min-notional"],
|
||||
signalProfile: mode
|
||||
};
|
||||
}
|
||||
|
||||
if ((side === "MISSING" || side === "STALE") && notional < config.missingNbboMinNotional) {
|
||||
return {
|
||||
signalPass: false,
|
||||
signalReasons: ["reject:missing-nbbo-min-notional"],
|
||||
signalProfile: mode
|
||||
};
|
||||
}
|
||||
|
||||
if ((side === "A" || side === "AA") && notional >= config.minNotional) {
|
||||
reasons.push("keep:ask-side");
|
||||
}
|
||||
|
||||
if (isSweepOrIso && notional >= config.sweepMinNotional) {
|
||||
reasons.push("keep:sweep-or-iso");
|
||||
}
|
||||
|
||||
if (print.size >= config.largePrintMinSize && notional >= config.largePrintMinNotional) {
|
||||
reasons.push("keep:large-contract-count");
|
||||
}
|
||||
|
||||
if (reasons.length === 0) {
|
||||
return {
|
||||
signalPass: false,
|
||||
signalReasons: ["reject:no-signal-rule"],
|
||||
signalProfile: mode
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
signalPass: true,
|
||||
signalReasons: reasons,
|
||||
signalProfile: mode
|
||||
};
|
||||
};
|
||||
|
||||
const sortStrings = (values: string[] | undefined): string[] | undefined => {
|
||||
if (!values || values.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return [...new Set(values)].sort();
|
||||
};
|
||||
|
||||
export const normalizeOptionFlowFilters = (
|
||||
filters: OptionFlowFilters | undefined
|
||||
): OptionFlowFilters | undefined => {
|
||||
if (!filters) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
view: filters.view,
|
||||
securityTypes: sortStrings(filters.securityTypes) as OptionSecurityType[] | undefined,
|
||||
nbboSides: sortStrings(filters.nbboSides) as OptionNbboSide[] | undefined,
|
||||
optionTypes: sortStrings(filters.optionTypes) as OptionType[] | undefined,
|
||||
minNotional:
|
||||
typeof filters.minNotional === "number" && Number.isFinite(filters.minNotional)
|
||||
? filters.minNotional
|
||||
: undefined
|
||||
};
|
||||
};
|
||||
|
||||
export const optionFlowFilterKey = (filters: OptionFlowFilters | undefined): string => {
|
||||
return JSON.stringify(normalizeOptionFlowFilters(filters) ?? {});
|
||||
};
|
||||
|
||||
export const matchesOptionPrintFilters = (
|
||||
print: Pick<OptionPrint, "is_etf" | "nbbo_side" | "option_type" | "notional" | "signal_pass">,
|
||||
filters: OptionFlowFilters | undefined
|
||||
): boolean => {
|
||||
if (!filters) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const view = filters.view ?? "signal";
|
||||
if (view === "signal" && print.signal_pass === false) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (filters.securityTypes?.length) {
|
||||
const securityType: OptionSecurityType = print.is_etf ? "etf" : "stock";
|
||||
if (!filters.securityTypes.includes(securityType)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (filters.nbboSides?.length) {
|
||||
const side = print.nbbo_side ?? "MISSING";
|
||||
if (!filters.nbboSides.includes(side)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (filters.optionTypes?.length) {
|
||||
const optionType = print.option_type;
|
||||
if (!optionType || !filters.optionTypes.includes(optionType)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof filters.minNotional === "number" && (print.notional ?? 0) < filters.minNotional) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
export const matchesFlowPacketFilters = (
|
||||
packet: FlowPacket,
|
||||
filters: OptionFlowFilters | undefined
|
||||
): boolean => {
|
||||
if (!filters) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const features = packet.features ?? {};
|
||||
const totalNotional = typeof features.total_notional === "number" ? features.total_notional : Number(features.total_notional ?? 0);
|
||||
if (typeof filters.minNotional === "number" && (!Number.isFinite(totalNotional) || totalNotional < filters.minNotional)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (filters.securityTypes?.length) {
|
||||
const isEtf = typeof features.is_etf === "boolean" ? features.is_etf : features.is_etf === 1;
|
||||
const securityType: OptionSecurityType = isEtf ? "etf" : "stock";
|
||||
if (!filters.securityTypes.includes(securityType)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (filters.optionTypes?.length) {
|
||||
const optionType =
|
||||
typeof features.option_type === "string"
|
||||
? features.option_type
|
||||
: typeof features.structure_rights === "string"
|
||||
? features.structure_rights.toLowerCase()
|
||||
: null;
|
||||
if (
|
||||
!optionType ||
|
||||
!filters.optionTypes.some((selected) => optionType.includes(selected))
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (filters.nbboSides?.length) {
|
||||
const sideToFeature: Record<OptionNbboSide, string> = {
|
||||
AA: "nbbo_aa_count",
|
||||
A: "nbbo_a_count",
|
||||
MID: "nbbo_mid_count",
|
||||
B: "nbbo_b_count",
|
||||
BB: "nbbo_bb_count",
|
||||
MISSING: "nbbo_missing_count",
|
||||
STALE: "nbbo_stale_count"
|
||||
};
|
||||
const matchesSide = filters.nbboSides.some((side) => {
|
||||
const value = features[sideToFeature[side]];
|
||||
return typeof value === "number" ? value > 0 : Number(value ?? 0) > 0;
|
||||
});
|
||||
if (!matchesSide) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
|
@ -8,7 +8,21 @@ import {
|
|||
|
||||
describe("live protocol types", () => {
|
||||
it("builds stable keys for generic and parameterized subscriptions", () => {
|
||||
expect(getSubscriptionKey({ channel: "flow" })).toBe("flow");
|
||||
expect(getSubscriptionKey({ channel: "flow" })).toBe("flow|{}");
|
||||
expect(
|
||||
getSubscriptionKey({
|
||||
channel: "options",
|
||||
filters: {
|
||||
view: "signal",
|
||||
securityTypes: ["stock"],
|
||||
nbboSides: ["A", "AA"],
|
||||
optionTypes: ["call", "put"],
|
||||
minNotional: 25000
|
||||
}
|
||||
})
|
||||
).toBe(
|
||||
'options|{"view":"signal","securityTypes":["stock"],"nbboSides":["A","AA"],"optionTypes":["call","put"],"minNotional":25000}'
|
||||
);
|
||||
expect(
|
||||
getSubscriptionKey({
|
||||
channel: "equity-candles",
|
||||
|
|
@ -25,7 +39,7 @@ describe("live protocol types", () => {
|
|||
const parsed = LiveClientMessageSchema.parse({
|
||||
op: "subscribe",
|
||||
subscriptions: [
|
||||
{ channel: "flow" },
|
||||
{ channel: "flow", filters: { nbboSides: ["AA", "A"], minNotional: 50000 } },
|
||||
{ channel: "equity-candles", underlying_id: "SPY", interval_ms: 60000 }
|
||||
]
|
||||
});
|
||||
|
|
|
|||
132
packages/types/tests/options-flow.test.ts
Normal file
132
packages/types/tests/options-flow.test.ts
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
deriveOptionPrintMetadata,
|
||||
evaluateOptionSignal,
|
||||
resolveSyntheticMarketModes,
|
||||
type OptionsSignalConfig
|
||||
} from "../src/options-flow";
|
||||
|
||||
const baseConfig: OptionsSignalConfig = {
|
||||
mode: "smart-money",
|
||||
minNotional: 10_000,
|
||||
etfMinNotional: 50_000,
|
||||
bidSideMinNotional: 25_000,
|
||||
midMinNotional: 20_000,
|
||||
missingNbboMinNotional: 50_000,
|
||||
largePrintMinSize: 500,
|
||||
largePrintMinNotional: 10_000,
|
||||
sweepMinNotional: 25_000,
|
||||
autoKeepMinNotional: 100_000,
|
||||
nbboMaxAgeMs: 1_500,
|
||||
etfUnderlyings: new Set(["SPY", "QQQ"])
|
||||
};
|
||||
|
||||
describe("options-flow helpers", () => {
|
||||
it("resolves synthetic modes with per-service overrides", () => {
|
||||
expect(
|
||||
resolveSyntheticMarketModes({
|
||||
syntheticMarketMode: "active",
|
||||
syntheticOptionsMode: "firehose"
|
||||
})
|
||||
).toEqual({
|
||||
market: "active",
|
||||
options: "firehose",
|
||||
equities: "active"
|
||||
});
|
||||
});
|
||||
|
||||
it("derives underlying, notional, nbbo side, and etf metadata", () => {
|
||||
const metadata = deriveOptionPrintMetadata(
|
||||
{
|
||||
option_contract_id: "SPY-2025-01-17-450-C",
|
||||
price: 2.5,
|
||||
size: 100,
|
||||
ts: 5_000
|
||||
},
|
||||
{
|
||||
bid: 2.3,
|
||||
ask: 2.5,
|
||||
ts: 4_500
|
||||
},
|
||||
baseConfig
|
||||
);
|
||||
|
||||
expect(metadata.underlying_id).toBe("SPY");
|
||||
expect(metadata.option_type).toBe("call");
|
||||
expect(metadata.notional).toBe(25_000);
|
||||
expect(metadata.nbbo_side).toBe("A");
|
||||
expect(metadata.is_etf).toBe(true);
|
||||
});
|
||||
|
||||
it("accepts and rejects smart-money thresholds at boundaries", () => {
|
||||
const acceptedAsk = evaluateOptionSignal(
|
||||
{
|
||||
size: 100,
|
||||
conditions: [],
|
||||
underlying_id: "AAPL",
|
||||
option_type: "call",
|
||||
notional: 10_000,
|
||||
nbbo_side: "A",
|
||||
is_etf: false
|
||||
},
|
||||
baseConfig
|
||||
);
|
||||
expect(acceptedAsk.signalPass).toBe(true);
|
||||
|
||||
const rejectedLow = evaluateOptionSignal(
|
||||
{
|
||||
size: 100,
|
||||
conditions: [],
|
||||
underlying_id: "AAPL",
|
||||
option_type: "call",
|
||||
notional: 9_999,
|
||||
nbbo_side: "A",
|
||||
is_etf: false
|
||||
},
|
||||
baseConfig
|
||||
);
|
||||
expect(rejectedLow.signalPass).toBe(false);
|
||||
|
||||
const rejectedBid = evaluateOptionSignal(
|
||||
{
|
||||
size: 100,
|
||||
conditions: [],
|
||||
underlying_id: "AAPL",
|
||||
option_type: "put",
|
||||
notional: 24_999,
|
||||
nbbo_side: "B",
|
||||
is_etf: false
|
||||
},
|
||||
baseConfig
|
||||
);
|
||||
expect(rejectedBid.signalPass).toBe(false);
|
||||
|
||||
const acceptedSweep = evaluateOptionSignal(
|
||||
{
|
||||
size: 100,
|
||||
conditions: ["SWEEP"],
|
||||
underlying_id: "AAPL",
|
||||
option_type: "call",
|
||||
notional: 25_000,
|
||||
nbbo_side: "MID",
|
||||
is_etf: false
|
||||
},
|
||||
baseConfig
|
||||
);
|
||||
expect(acceptedSweep.signalPass).toBe(true);
|
||||
|
||||
const rejectedEtf = evaluateOptionSignal(
|
||||
{
|
||||
size: 100,
|
||||
conditions: [],
|
||||
underlying_id: "SPY",
|
||||
option_type: "call",
|
||||
notional: 49_999,
|
||||
nbbo_side: "A",
|
||||
is_etf: true
|
||||
},
|
||||
baseConfig
|
||||
);
|
||||
expect(rejectedEtf.signalPass).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -10,7 +10,7 @@ import {
|
|||
SUBJECT_INFERRED_DARK,
|
||||
SUBJECT_FLOW_PACKETS,
|
||||
SUBJECT_OPTION_NBBO,
|
||||
SUBJECT_OPTION_PRINTS,
|
||||
SUBJECT_OPTION_SIGNAL_PRINTS,
|
||||
STREAM_ALERTS,
|
||||
STREAM_CLASSIFIER_HITS,
|
||||
STREAM_EQUITY_CANDLES,
|
||||
|
|
@ -20,7 +20,7 @@ import {
|
|||
STREAM_INFERRED_DARK,
|
||||
STREAM_FLOW_PACKETS,
|
||||
STREAM_OPTION_NBBO,
|
||||
STREAM_OPTION_PRINTS,
|
||||
STREAM_OPTION_SIGNAL_PRINTS,
|
||||
buildDurableConsumer,
|
||||
connectJetStreamWithRetry,
|
||||
ensureStream,
|
||||
|
|
@ -85,6 +85,13 @@ import {
|
|||
LiveServerMessage,
|
||||
LiveSubscription,
|
||||
LiveSubscriptionSchema,
|
||||
matchesFlowPacketFilters,
|
||||
matchesOptionPrintFilters,
|
||||
OptionFlowFilters,
|
||||
OptionFlowViewSchema,
|
||||
OptionNbboSideSchema,
|
||||
OptionSecurityTypeSchema,
|
||||
OptionTypeSchema,
|
||||
FlowPacketSchema,
|
||||
OptionNBBOSchema,
|
||||
OptionPrintSchema,
|
||||
|
|
@ -199,6 +206,32 @@ const equityPrintRangeSchema = z.object({
|
|||
end_ts: z.coerce.number().int().nonnegative(),
|
||||
limit: limitSchema.optional()
|
||||
});
|
||||
const optionSideListSchema = z
|
||||
.string()
|
||||
.transform((value) =>
|
||||
value
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean)
|
||||
)
|
||||
.pipe(z.array(OptionNbboSideSchema));
|
||||
const optionTypeListSchema = z
|
||||
.string()
|
||||
.transform((value) =>
|
||||
value
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean)
|
||||
)
|
||||
.pipe(z.array(OptionTypeSchema));
|
||||
const optionSecuritySchema = z.enum(["stock", "etf", "all"]);
|
||||
const optionFilterQuerySchema = z.object({
|
||||
view: OptionFlowViewSchema.optional(),
|
||||
security: optionSecuritySchema.optional(),
|
||||
side: optionSideListSchema.optional(),
|
||||
type: optionTypeListSchema.optional(),
|
||||
min_notional: z.coerce.number().nonnegative().optional()
|
||||
});
|
||||
|
||||
type Channel =
|
||||
| "options"
|
||||
|
|
@ -235,6 +268,7 @@ const classifierHitSockets = new Set<LegacySocket>();
|
|||
const alertSockets = new Set<LegacySocket>();
|
||||
const liveSocketSubscriptions = new Map<LiveSocket, Set<string>>();
|
||||
const subscriptionSockets = new Map<string, Set<LiveSocket>>();
|
||||
const subscriptionDefinitions = new Map<string, LiveSubscription>();
|
||||
const liveHeartbeats = new Map<LiveSocket, ReturnType<typeof setInterval>>();
|
||||
|
||||
const jsonResponse = (body: unknown, status = 200): Response => {
|
||||
|
|
@ -254,6 +288,43 @@ const parseLimit = (value: string | null): number => {
|
|||
return limitSchema.parse(value);
|
||||
};
|
||||
|
||||
const parseOptionPrintFilters = (
|
||||
url: URL
|
||||
): {
|
||||
view: z.infer<typeof OptionFlowViewSchema>;
|
||||
storageFilters: Parameters<typeof fetchRecentOptionPrints>[3];
|
||||
liveFilters: OptionFlowFilters;
|
||||
} => {
|
||||
const parsed = optionFilterQuerySchema.parse({
|
||||
view: url.searchParams.get("view") ?? undefined,
|
||||
security: url.searchParams.get("security") ?? undefined,
|
||||
side: url.searchParams.get("side") ?? undefined,
|
||||
type: url.searchParams.get("type") ?? undefined,
|
||||
min_notional: url.searchParams.get("min_notional") ?? undefined
|
||||
});
|
||||
const view = parsed.view ?? "signal";
|
||||
const security = parsed.security ?? (view === "raw" ? "all" : "stock");
|
||||
const storageFilters = {
|
||||
view,
|
||||
security,
|
||||
minNotional: parsed.min_notional,
|
||||
nbboSides: parsed.side,
|
||||
optionTypes: parsed.type
|
||||
} as const;
|
||||
const liveFilters: OptionFlowFilters = {
|
||||
view,
|
||||
securityTypes:
|
||||
security === "all"
|
||||
? undefined
|
||||
: ([security] as Array<z.infer<typeof OptionSecurityTypeSchema>>),
|
||||
nbboSides: parsed.side,
|
||||
optionTypes: parsed.type,
|
||||
minNotional: parsed.min_notional
|
||||
};
|
||||
|
||||
return { view, storageFilters, liveFilters };
|
||||
};
|
||||
|
||||
const parseReplayParams = (url: URL): { afterTs: number; afterSeq: number; limit: number } => {
|
||||
const params = replayParamsSchema.parse({
|
||||
after_ts: url.searchParams.get("after_ts") ?? undefined,
|
||||
|
|
@ -412,6 +483,7 @@ const subscribeSocket = (socket: LiveSocket, subscription: LiveSubscription): vo
|
|||
const sockets = subscriptionSockets.get(key) ?? new Set<LiveSocket>();
|
||||
sockets.add(socket);
|
||||
subscriptionSockets.set(key, sockets);
|
||||
subscriptionDefinitions.set(key, subscription);
|
||||
};
|
||||
|
||||
const unsubscribeSocket = (socket: LiveSocket, subscription: LiveSubscription): void => {
|
||||
|
|
@ -425,6 +497,7 @@ const unsubscribeSocket = (socket: LiveSocket, subscription: LiveSubscription):
|
|||
sockets.delete(socket);
|
||||
if (sockets.size === 0) {
|
||||
subscriptionSockets.delete(key);
|
||||
subscriptionDefinitions.delete(key);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -436,6 +509,7 @@ const cleanupLiveSocket = (socket: LiveSocket): void => {
|
|||
sockets?.delete(socket);
|
||||
if (sockets && sockets.size === 0) {
|
||||
subscriptionSockets.delete(key);
|
||||
subscriptionDefinitions.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -504,8 +578,8 @@ const run = async () => {
|
|||
);
|
||||
|
||||
await ensureStream(jsm, {
|
||||
name: STREAM_OPTION_PRINTS,
|
||||
subjects: [SUBJECT_OPTION_PRINTS],
|
||||
name: STREAM_OPTION_SIGNAL_PRINTS,
|
||||
subjects: [SUBJECT_OPTION_SIGNAL_PRINTS],
|
||||
retention: "limits",
|
||||
storage: "file",
|
||||
discard: "old",
|
||||
|
|
@ -722,8 +796,8 @@ const run = async () => {
|
|||
};
|
||||
|
||||
const optionSubscription = await subscribeWithReset(
|
||||
SUBJECT_OPTION_PRINTS,
|
||||
STREAM_OPTION_PRINTS,
|
||||
SUBJECT_OPTION_SIGNAL_PRINTS,
|
||||
STREAM_OPTION_SIGNAL_PRINTS,
|
||||
"api-option-prints"
|
||||
);
|
||||
|
||||
|
|
@ -786,21 +860,45 @@ const run = async () => {
|
|||
item: unknown,
|
||||
ingestChannel: "options" | "nbbo" | "equities" | "equity-candles" | "equity-overlay" | "equity-joins" | "flow" | "classifier-hits" | "alerts" | "inferred-dark"
|
||||
) => {
|
||||
const key = getSubscriptionKey(subscription);
|
||||
const sockets = subscriptionSockets.get(key);
|
||||
const watermark = await liveState.ingest(ingestChannel, item);
|
||||
if (!sockets || sockets.size === 0) {
|
||||
const matchingSubscriptions =
|
||||
subscription.channel === "options" || subscription.channel === "flow"
|
||||
? [...subscriptionDefinitions.entries()].filter(([, candidate]) => candidate.channel === subscription.channel)
|
||||
: [[getSubscriptionKey(subscription), subscription] as const];
|
||||
|
||||
if (matchingSubscriptions.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [key, candidate] of matchingSubscriptions) {
|
||||
const sockets = subscriptionSockets.get(key);
|
||||
if (!sockets || sockets.size === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
candidate.channel === "options" &&
|
||||
!matchesOptionPrintFilters(OptionPrintSchema.parse(item), candidate.filters)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
candidate.channel === "flow" &&
|
||||
!matchesFlowPacketFilters(FlowPacketSchema.parse(item), candidate.filters)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const socket of sockets) {
|
||||
sendLiveMessage(socket, {
|
||||
op: "event",
|
||||
subscription,
|
||||
subscription: candidate,
|
||||
item,
|
||||
watermark
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const pumpOptions = async () => {
|
||||
|
|
@ -996,10 +1094,21 @@ const run = async () => {
|
|||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/prints/options") {
|
||||
try {
|
||||
const limit = parseLimit(url.searchParams.get("limit"));
|
||||
const source = parseReplaySource(url) ?? undefined;
|
||||
const data = await fetchRecentOptionPrints(clickhouse, limit, source);
|
||||
const { storageFilters } = parseOptionPrintFilters(url);
|
||||
const data = await fetchRecentOptionPrints(clickhouse, limit, source, storageFilters);
|
||||
return jsonResponse({ data });
|
||||
} catch (error) {
|
||||
return jsonResponse(
|
||||
{
|
||||
error: "invalid options query",
|
||||
detail: error instanceof Error ? error.message : String(error)
|
||||
},
|
||||
400
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/nbbo/options") {
|
||||
|
|
@ -1105,10 +1214,28 @@ const run = async () => {
|
|||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/history/options") {
|
||||
try {
|
||||
const { beforeTs, beforeSeq, limit } = parseBeforeParams(url);
|
||||
const source = parseReplaySource(url) ?? undefined;
|
||||
const data = await fetchOptionPrintsBefore(clickhouse, beforeTs, beforeSeq, limit, source);
|
||||
const { storageFilters } = parseOptionPrintFilters(url);
|
||||
const data = await fetchOptionPrintsBefore(
|
||||
clickhouse,
|
||||
beforeTs,
|
||||
beforeSeq,
|
||||
limit,
|
||||
source,
|
||||
storageFilters
|
||||
);
|
||||
return jsonResponse(buildHistoryResponse(data, (item) => ({ ts: item.ts, seq: item.seq })));
|
||||
} catch (error) {
|
||||
return jsonResponse(
|
||||
{
|
||||
error: "invalid options history query",
|
||||
detail: error instanceof Error ? error.message : String(error)
|
||||
},
|
||||
400
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/history/nbbo") {
|
||||
|
|
@ -1183,12 +1310,30 @@ const run = async () => {
|
|||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/replay/options") {
|
||||
try {
|
||||
const { afterTs, afterSeq, limit } = parseReplayParams(url);
|
||||
const source = parseReplaySource(url) ?? undefined;
|
||||
const data = await fetchOptionPrintsAfter(clickhouse, afterTs, afterSeq, limit, source);
|
||||
const { storageFilters } = parseOptionPrintFilters(url);
|
||||
const data = await fetchOptionPrintsAfter(
|
||||
clickhouse,
|
||||
afterTs,
|
||||
afterSeq,
|
||||
limit,
|
||||
source,
|
||||
storageFilters
|
||||
);
|
||||
const last = data.at(-1);
|
||||
const next = last ? { ts: last.ts, seq: last.seq } : null;
|
||||
return jsonResponse({ data, next });
|
||||
} catch (error) {
|
||||
return jsonResponse(
|
||||
{
|
||||
error: "invalid options replay query",
|
||||
detail: error instanceof Error ? error.message : String(error)
|
||||
},
|
||||
400
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (req.method === "GET" && url.pathname === "/replay/nbbo") {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import {
|
||||
fetchRecentOptionPrints,
|
||||
fetchRecentAlerts,
|
||||
fetchRecentClassifierHits,
|
||||
fetchRecentEquityCandles,
|
||||
|
|
@ -7,9 +8,9 @@ import {
|
|||
fetchRecentFlowPackets,
|
||||
fetchRecentInferredDark,
|
||||
fetchRecentOptionNBBO,
|
||||
fetchRecentOptionPrints,
|
||||
type ClickHouseClient
|
||||
} from "@islandflow/storage";
|
||||
import type { OptionPrintQueryFilters } from "@islandflow/storage";
|
||||
import {
|
||||
AlertEventSchema,
|
||||
ClassifierHitEventSchema,
|
||||
|
|
@ -22,8 +23,11 @@ import {
|
|||
InferredDarkEventSchema,
|
||||
LiveGenericChannel,
|
||||
LiveSubscription,
|
||||
matchesFlowPacketFilters,
|
||||
matchesOptionPrintFilters,
|
||||
OptionNBBOSchema,
|
||||
OptionPrintSchema,
|
||||
type OptionFlowFilters,
|
||||
type Cursor,
|
||||
type EquityCandle,
|
||||
type EquityPrint,
|
||||
|
|
@ -124,7 +128,8 @@ const getGenericConfig = (limits: GenericLiveLimits): {
|
|||
limit: limits.options,
|
||||
parse: (value) => OptionPrintSchema.parse(value),
|
||||
cursor: (item) => ({ ts: item.ts, seq: item.seq }),
|
||||
fetchRecent: fetchRecentOptionPrints
|
||||
fetchRecent: (clickhouse, limit) =>
|
||||
fetchRecentOptionPrints(clickhouse, limit, undefined, { view: "signal" })
|
||||
},
|
||||
nbbo: {
|
||||
redisKey: "live:nbbo",
|
||||
|
|
@ -279,6 +284,55 @@ export class LiveStateManager {
|
|||
|
||||
async getSnapshot(subscription: LiveSubscription): Promise<FeedSnapshot<unknown>> {
|
||||
switch (subscription.channel) {
|
||||
case "options": {
|
||||
if (subscription.filters?.view === "raw") {
|
||||
const storageFilters: OptionPrintQueryFilters = {
|
||||
view: "raw",
|
||||
security:
|
||||
subscription.filters.securityTypes?.length === 1
|
||||
? subscription.filters.securityTypes[0]
|
||||
: "all",
|
||||
nbboSides: subscription.filters.nbboSides,
|
||||
optionTypes: subscription.filters.optionTypes,
|
||||
minNotional: subscription.filters.minNotional
|
||||
};
|
||||
const items = await fetchRecentOptionPrints(
|
||||
this.clickhouse,
|
||||
this.generic.options.limit,
|
||||
undefined,
|
||||
storageFilters
|
||||
);
|
||||
return {
|
||||
subscription,
|
||||
items,
|
||||
watermark: items[0] ? { ts: items[0].ts, seq: items[0].seq } : null,
|
||||
next_before: nextBeforeForItems(items, (item) => ({ ts: item.ts, seq: item.seq }))
|
||||
};
|
||||
}
|
||||
|
||||
const config = this.generic.options;
|
||||
const items = (this.genericItems.get("options") ?? []).filter((item) =>
|
||||
matchesOptionPrintFilters(item, subscription.filters)
|
||||
);
|
||||
return {
|
||||
subscription,
|
||||
items,
|
||||
watermark: this.genericCursors.get(config.cursorField) ?? null,
|
||||
next_before: nextBeforeForItems(items, config.cursor)
|
||||
};
|
||||
}
|
||||
case "flow": {
|
||||
const config = this.generic.flow;
|
||||
const items = (this.genericItems.get("flow") ?? []).filter((item) =>
|
||||
matchesFlowPacketFilters(item, subscription.filters)
|
||||
);
|
||||
return {
|
||||
subscription,
|
||||
items,
|
||||
watermark: this.genericCursors.get(config.cursorField) ?? null,
|
||||
next_before: nextBeforeForItems(items, config.cursor)
|
||||
};
|
||||
}
|
||||
case "equity-candles": {
|
||||
const key = candleRedisKey(subscription.underlying_id, subscription.interval_ms);
|
||||
const cursorField = candleCursorField(subscription.underlying_id, subscription.interval_ms);
|
||||
|
|
|
|||
|
|
@ -196,4 +196,81 @@ describe("LiveStateManager", () => {
|
|||
expect(stats.trimOperations).toBeGreaterThan(0);
|
||||
expect(stats.cacheDepthByKey["live:flow"]).toBe(2);
|
||||
});
|
||||
|
||||
it("filters option and flow snapshots using subscription filters", async () => {
|
||||
const manager = new LiveStateManager(makeClickHouse(), null);
|
||||
|
||||
await manager.ingest("options", {
|
||||
source_ts: 100,
|
||||
ingest_ts: 101,
|
||||
seq: 1,
|
||||
trace_id: "opt-1",
|
||||
ts: 100,
|
||||
option_contract_id: "AAPL-2025-01-17-200-C",
|
||||
price: 1,
|
||||
size: 100,
|
||||
exchange: "X",
|
||||
underlying_id: "AAPL",
|
||||
option_type: "call",
|
||||
notional: 10000,
|
||||
nbbo_side: "A",
|
||||
is_etf: false,
|
||||
signal_pass: true,
|
||||
signal_reasons: ["keep:ask-side"],
|
||||
signal_profile: "smart-money"
|
||||
});
|
||||
await manager.ingest("options", {
|
||||
source_ts: 110,
|
||||
ingest_ts: 111,
|
||||
seq: 2,
|
||||
trace_id: "opt-2",
|
||||
ts: 110,
|
||||
option_contract_id: "SPY-2025-01-17-500-P",
|
||||
price: 1,
|
||||
size: 100,
|
||||
exchange: "X",
|
||||
underlying_id: "SPY",
|
||||
option_type: "put",
|
||||
notional: 10000,
|
||||
nbbo_side: "B",
|
||||
is_etf: true,
|
||||
signal_pass: true,
|
||||
signal_reasons: ["keep:ask-side"],
|
||||
signal_profile: "smart-money"
|
||||
});
|
||||
await manager.ingest("flow", {
|
||||
source_ts: 120,
|
||||
ingest_ts: 121,
|
||||
seq: 3,
|
||||
trace_id: "flow-1",
|
||||
id: "flow-1",
|
||||
members: ["opt-1"],
|
||||
features: {
|
||||
option_contract_id: "AAPL-2025-01-17-200-C",
|
||||
total_notional: 10000,
|
||||
is_etf: false,
|
||||
option_type: "call",
|
||||
nbbo_a_count: 1,
|
||||
nbbo_aa_count: 0,
|
||||
nbbo_mid_count: 0,
|
||||
nbbo_b_count: 0,
|
||||
nbbo_bb_count: 0,
|
||||
nbbo_missing_count: 0,
|
||||
nbbo_stale_count: 0
|
||||
},
|
||||
join_quality: {}
|
||||
});
|
||||
|
||||
const optionSnapshot = await manager.getSnapshot({
|
||||
channel: "options",
|
||||
filters: { securityTypes: ["stock"], nbboSides: ["A"], optionTypes: ["call"] }
|
||||
});
|
||||
const flowSnapshot = await manager.getSnapshot({
|
||||
channel: "flow",
|
||||
filters: { securityTypes: ["stock"], nbboSides: ["A"], optionTypes: ["call"] }
|
||||
});
|
||||
|
||||
expect(optionSnapshot.items).toHaveLength(1);
|
||||
expect(flowSnapshot.items).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import {
|
|||
SUBJECT_INFERRED_DARK,
|
||||
SUBJECT_FLOW_PACKETS,
|
||||
SUBJECT_OPTION_NBBO,
|
||||
SUBJECT_OPTION_PRINTS,
|
||||
SUBJECT_OPTION_SIGNAL_PRINTS,
|
||||
STREAM_ALERTS,
|
||||
STREAM_CLASSIFIER_HITS,
|
||||
STREAM_EQUITY_JOINS,
|
||||
|
|
@ -18,7 +18,7 @@ import {
|
|||
STREAM_INFERRED_DARK,
|
||||
STREAM_FLOW_PACKETS,
|
||||
STREAM_OPTION_NBBO,
|
||||
STREAM_OPTION_PRINTS,
|
||||
STREAM_OPTION_SIGNAL_PRINTS,
|
||||
buildDurableConsumer,
|
||||
connectJetStreamWithRetry,
|
||||
ensureStream,
|
||||
|
|
@ -231,6 +231,9 @@ type NbboPlacementCounts = {
|
|||
|
||||
type ClusterState = {
|
||||
contractId: string;
|
||||
underlyingId: string | null;
|
||||
optionType: string | null;
|
||||
isEtf: boolean | null;
|
||||
startTs: number;
|
||||
endTs: number;
|
||||
startSourceTs: number;
|
||||
|
|
@ -530,6 +533,9 @@ const buildCluster = (print: OptionPrint): ClusterState => {
|
|||
recordPlacement(placements, classifyPlacement(print.price, selectNbbo(print.option_contract_id, print.ts)));
|
||||
return {
|
||||
contractId: print.option_contract_id,
|
||||
underlyingId: print.underlying_id ?? null,
|
||||
optionType: print.option_type ?? null,
|
||||
isEtf: typeof print.is_etf === "boolean" ? print.is_etf : null,
|
||||
startTs: print.ts,
|
||||
endTs: print.ts,
|
||||
startSourceTs: print.source_ts,
|
||||
|
|
@ -546,6 +552,15 @@ const buildCluster = (print: OptionPrint): ClusterState => {
|
|||
};
|
||||
|
||||
const updateCluster = (cluster: ClusterState, print: OptionPrint): ClusterState => {
|
||||
if (!cluster.underlyingId && print.underlying_id) {
|
||||
cluster.underlyingId = print.underlying_id;
|
||||
}
|
||||
if (!cluster.optionType && print.option_type) {
|
||||
cluster.optionType = print.option_type;
|
||||
}
|
||||
if (cluster.isEtf === null && typeof print.is_etf === "boolean") {
|
||||
cluster.isEtf = print.is_etf;
|
||||
}
|
||||
cluster.endTs = Math.max(cluster.endTs, print.ts);
|
||||
cluster.endIngestTs = Math.max(cluster.endIngestTs, print.ingest_ts);
|
||||
cluster.endSeq = Math.max(cluster.endSeq, print.seq);
|
||||
|
|
@ -705,6 +720,15 @@ const flushCluster = async (
|
|||
}
|
||||
}
|
||||
}
|
||||
if (cluster.underlyingId) {
|
||||
features.underlying_id = cluster.underlyingId;
|
||||
}
|
||||
if (cluster.optionType) {
|
||||
features.option_type = cluster.optionType;
|
||||
}
|
||||
if (cluster.isEtf !== null) {
|
||||
features.is_etf = cluster.isEtf;
|
||||
}
|
||||
|
||||
const placementTotal =
|
||||
cluster.placements.aa +
|
||||
|
|
@ -1012,8 +1036,8 @@ const run = async () => {
|
|||
);
|
||||
|
||||
await ensureStream(jsm, {
|
||||
name: STREAM_OPTION_PRINTS,
|
||||
subjects: [SUBJECT_OPTION_PRINTS],
|
||||
name: STREAM_OPTION_SIGNAL_PRINTS,
|
||||
subjects: [SUBJECT_OPTION_SIGNAL_PRINTS],
|
||||
retention: "limits",
|
||||
storage: "file",
|
||||
discard: "old",
|
||||
|
|
@ -1162,7 +1186,7 @@ const run = async () => {
|
|||
|
||||
if (env.COMPUTE_CONSUMER_RESET) {
|
||||
try {
|
||||
await jsm.consumers.delete(STREAM_OPTION_PRINTS, durableName);
|
||||
await jsm.consumers.delete(STREAM_OPTION_SIGNAL_PRINTS, durableName);
|
||||
logger.warn("reset jetstream consumer", { durable: durableName });
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
|
|
@ -1172,14 +1196,14 @@ const run = async () => {
|
|||
}
|
||||
} else {
|
||||
try {
|
||||
const info = await jsm.consumers.info(STREAM_OPTION_PRINTS, durableName);
|
||||
const info = await jsm.consumers.info(STREAM_OPTION_SIGNAL_PRINTS, durableName);
|
||||
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) {
|
||||
logger.warn("resetting consumer due to deliver policy change", {
|
||||
durable: durableName,
|
||||
current: info.config.deliver_policy,
|
||||
desired: env.COMPUTE_DELIVER_POLICY
|
||||
});
|
||||
await jsm.consumers.delete(STREAM_OPTION_PRINTS, durableName);
|
||||
await jsm.consumers.delete(STREAM_OPTION_SIGNAL_PRINTS, durableName);
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
|
|
@ -1292,7 +1316,7 @@ const run = async () => {
|
|||
const opts = buildDurableConsumer(durableName);
|
||||
applyDeliverPolicy(opts, env.COMPUTE_DELIVER_POLICY);
|
||||
try {
|
||||
return await subscribeJson(js, SUBJECT_OPTION_PRINTS, opts);
|
||||
return await subscribeJson(js, SUBJECT_OPTION_SIGNAL_PRINTS, opts);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const shouldReset =
|
||||
|
|
@ -1307,7 +1331,7 @@ const run = async () => {
|
|||
logger.warn("resetting jetstream consumer", { durable: durableName, error: message });
|
||||
|
||||
try {
|
||||
await jsm.consumers.delete(STREAM_OPTION_PRINTS, durableName);
|
||||
await jsm.consumers.delete(STREAM_OPTION_SIGNAL_PRINTS, durableName);
|
||||
} catch (deleteError) {
|
||||
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||
if (!deleteMessage.includes("not found")) {
|
||||
|
|
@ -1320,7 +1344,7 @@ const run = async () => {
|
|||
|
||||
const resetOpts = buildDurableConsumer(durableName);
|
||||
applyDeliverPolicy(resetOpts, env.COMPUTE_DELIVER_POLICY);
|
||||
return await subscribeJson(js, SUBJECT_OPTION_PRINTS, resetOpts);
|
||||
return await subscribeJson(js, SUBJECT_OPTION_SIGNAL_PRINTS, resetOpts);
|
||||
}
|
||||
})();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
import { SP500_SYMBOLS, type EquityPrint, type EquityQuote } from "@islandflow/types";
|
||||
import {
|
||||
SP500_SYMBOLS,
|
||||
type EquityPrint,
|
||||
type EquityQuote,
|
||||
type SyntheticMarketMode
|
||||
} from "@islandflow/types";
|
||||
import type { EquityIngestAdapter, EquityIngestHandlers } from "./types";
|
||||
|
||||
type SyntheticEquitiesAdapterConfig = {
|
||||
emitIntervalMs: number;
|
||||
mode: SyntheticMarketMode;
|
||||
};
|
||||
|
||||
const EXCHANGES = ["NYSE", "NASDAQ", "ARCA", "BATS", "IEX", "TEST"];
|
||||
|
|
@ -22,10 +28,7 @@ const DARK_SEQUENCE: DarkScenario[] = [
|
|||
"sell",
|
||||
"sell"
|
||||
];
|
||||
const SYNTHETIC_SYMBOLS = [
|
||||
"SPY",
|
||||
...SP500_SYMBOLS.filter((symbol) => symbol !== "SPY")
|
||||
];
|
||||
const SYNTHETIC_SYMBOLS = ["SPY", ...(SP500_SYMBOLS as readonly string[])];
|
||||
|
||||
const hashSymbol = (value: string): number => {
|
||||
let hash = 0;
|
||||
|
|
@ -124,6 +127,30 @@ const priceForPlacement = (
|
|||
export const createSyntheticEquitiesAdapter = (
|
||||
config: SyntheticEquitiesAdapterConfig
|
||||
): EquityIngestAdapter => {
|
||||
const profile =
|
||||
config.mode === "firehose"
|
||||
? {
|
||||
batchSize: 10,
|
||||
darkEvery: true,
|
||||
offExchangeMod: 2,
|
||||
litSizeBase: 40,
|
||||
litSizeRange: 1400
|
||||
}
|
||||
: config.mode === "active"
|
||||
? {
|
||||
batchSize: 5,
|
||||
darkEvery: true,
|
||||
offExchangeMod: 4,
|
||||
litSizeBase: 20,
|
||||
litSizeRange: 900
|
||||
}
|
||||
: {
|
||||
batchSize: 2,
|
||||
darkEvery: false,
|
||||
offExchangeMod: 8,
|
||||
litSizeBase: 10,
|
||||
litSizeRange: 300
|
||||
};
|
||||
return {
|
||||
name: "synthetic",
|
||||
start: (handlers: EquityIngestHandlers) => {
|
||||
|
|
@ -140,7 +167,7 @@ export const createSyntheticEquitiesAdapter = (
|
|||
}
|
||||
|
||||
const now = Date.now();
|
||||
const batchSize = 3;
|
||||
const batchSize = profile.batchSize;
|
||||
|
||||
const darkSymbol = SYNTHETIC_SYMBOLS[darkSymbolIndex % SYNTHETIC_SYMBOLS.length];
|
||||
const darkHash = hashSymbol(darkSymbol);
|
||||
|
|
@ -151,6 +178,7 @@ export const createSyntheticEquitiesAdapter = (
|
|||
const scenario = DARK_SEQUENCE[darkStep % DARK_SEQUENCE.length];
|
||||
const darkTs = now;
|
||||
|
||||
if (profile.darkEvery) {
|
||||
if (handlers.onQuote) {
|
||||
quoteSeq += 1;
|
||||
const quoteEvent = buildSyntheticQuote(
|
||||
|
|
@ -165,13 +193,13 @@ export const createSyntheticEquitiesAdapter = (
|
|||
|
||||
seq += 1;
|
||||
let darkPlacement: PricePlacement = "MID";
|
||||
let darkSize = 2600;
|
||||
let darkSize = config.mode === "firehose" ? 4000 : 2600;
|
||||
if (scenario === "buy") {
|
||||
darkPlacement = darkStep % 2 === 0 ? "A" : "AA";
|
||||
darkSize = 800;
|
||||
darkSize = config.mode === "firehose" ? 1500 : 800;
|
||||
} else if (scenario === "sell") {
|
||||
darkPlacement = darkStep % 2 === 0 ? "B" : "BB";
|
||||
darkSize = 800;
|
||||
darkSize = config.mode === "firehose" ? 1500 : 800;
|
||||
}
|
||||
const darkPrice = priceForPlacement(darkMid, darkQuote, darkPlacement);
|
||||
const darkPrint = buildSyntheticPrint(
|
||||
|
|
@ -190,6 +218,7 @@ export const createSyntheticEquitiesAdapter = (
|
|||
darkStep = 0;
|
||||
darkSymbolIndex += 1;
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < batchSize; i += 1) {
|
||||
seq += 1;
|
||||
|
|
@ -201,9 +230,9 @@ export const createSyntheticEquitiesAdapter = (
|
|||
const placement: PricePlacement =
|
||||
seq % 11 === 0 ? "A" : seq % 13 === 0 ? "B" : "MID";
|
||||
const price = priceForPlacement(mid, quote, placement);
|
||||
const size = 10 + (seq % 600);
|
||||
const size = profile.litSizeBase + (seq % profile.litSizeRange);
|
||||
const exchange = EXCHANGES[(seq + symbolHash) % EXCHANGES.length];
|
||||
const offExchangeFlag = (seq + i) % 6 === 0;
|
||||
const offExchangeFlag = (seq + i) % profile.offExchangeMod === 0;
|
||||
const eventTs = now + i * 4;
|
||||
|
||||
if (handlers.onQuote) {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import {
|
|||
import {
|
||||
EquityPrintSchema,
|
||||
EquityQuoteSchema,
|
||||
resolveSyntheticMarketModes,
|
||||
type EquityPrint,
|
||||
type EquityQuote
|
||||
} from "@islandflow/types";
|
||||
|
|
@ -36,6 +37,8 @@ const envSchema = z.object({
|
|||
CLICKHOUSE_DATABASE: z.string().default("default"),
|
||||
EQUITIES_INGEST_ADAPTER: z.string().min(1).default("synthetic"),
|
||||
EMIT_INTERVAL_MS: z.coerce.number().int().positive().default(1000),
|
||||
SYNTHETIC_MARKET_MODE: z.string().default("realistic"),
|
||||
SYNTHETIC_EQUITIES_MODE: z.string().default(""),
|
||||
|
||||
// Alpaca (equities)
|
||||
ALPACA_KEY_ID: z.string().default(""),
|
||||
|
|
@ -63,6 +66,10 @@ const envSchema = z.object({
|
|||
});
|
||||
|
||||
const env = readEnv(envSchema);
|
||||
const syntheticModes = resolveSyntheticMarketModes({
|
||||
syntheticMarketMode: env.SYNTHETIC_MARKET_MODE,
|
||||
syntheticEquitiesMode: env.SYNTHETIC_EQUITIES_MODE
|
||||
});
|
||||
|
||||
const state = {
|
||||
shuttingDown: false,
|
||||
|
|
@ -153,7 +160,10 @@ const parseSymbolList = (value: string): string[] => {
|
|||
|
||||
const selectAdapter = (name: string): EquityIngestAdapter => {
|
||||
if (name === "synthetic") {
|
||||
return createSyntheticEquitiesAdapter({ emitIntervalMs: env.EMIT_INTERVAL_MS });
|
||||
return createSyntheticEquitiesAdapter({
|
||||
emitIntervalMs: env.EMIT_INTERVAL_MS,
|
||||
mode: syntheticModes.equities
|
||||
});
|
||||
}
|
||||
|
||||
if (name === "alpaca") {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
import { SP500_SYMBOLS, type OptionNBBO, type OptionPrint } from "@islandflow/types";
|
||||
import {
|
||||
SP500_SYMBOLS,
|
||||
type OptionNBBO,
|
||||
type OptionPrint,
|
||||
type SyntheticMarketMode
|
||||
} from "@islandflow/types";
|
||||
import type { OptionIngestAdapter, OptionIngestHandlers } from "./types";
|
||||
|
||||
type SyntheticOptionsAdapterConfig = {
|
||||
emitIntervalMs: number;
|
||||
mode: SyntheticMarketMode;
|
||||
};
|
||||
|
||||
type Burst = {
|
||||
|
|
@ -17,17 +23,18 @@ type Burst = {
|
|||
seed: number;
|
||||
};
|
||||
|
||||
const SYNTHETIC_SYMBOLS = [
|
||||
"SPY",
|
||||
...SP500_SYMBOLS.filter((symbol) => symbol !== "SPY")
|
||||
];
|
||||
const SYNTHETIC_SYMBOLS = ["SPY", ...(SP500_SYMBOLS as readonly string[])];
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
||||
const EXPIRY_OFFSETS = [0, 1, 7, 14, 28, 45, 60, 90];
|
||||
const EXCHANGES = ["CBOE", "PHLX", "ISE", "ARCA", "BOX", "MIAX"];
|
||||
const CONDITIONS = ["SWEEP", "ISO", "FILL", "TEST"];
|
||||
const BURST_RUN_RANGE: [number, number] = [2, 4];
|
||||
type SyntheticOptionsProfile = {
|
||||
burstRunRange: [number, number];
|
||||
scenarios: Scenario[];
|
||||
pricePlacements: Record<string, WeightedValue<PricePlacement>[]>;
|
||||
};
|
||||
|
||||
type PricePlacement = "AA" | "A" | "B" | "BB";
|
||||
type PricePlacement = "AA" | "A" | "MID" | "B" | "BB";
|
||||
|
||||
type WeightedValue<T> = {
|
||||
value: T;
|
||||
|
|
@ -45,7 +52,70 @@ type Scenario = {
|
|||
conditions?: string[];
|
||||
};
|
||||
|
||||
const SCENARIOS: Scenario[] = [
|
||||
const REALISTIC_SCENARIOS: Scenario[] = [
|
||||
{
|
||||
id: "ask_lift",
|
||||
weight: 18,
|
||||
right: "either",
|
||||
countRange: [1, 2],
|
||||
sizeRange: [30, 180],
|
||||
premiumRange: [9_000, 35_000],
|
||||
priceTrend: "flat",
|
||||
conditions: ["FILL"]
|
||||
},
|
||||
{
|
||||
id: "mid_block",
|
||||
weight: 14,
|
||||
right: "either",
|
||||
countRange: [1, 2],
|
||||
sizeRange: [120, 480],
|
||||
premiumRange: [12_000, 45_000],
|
||||
priceTrend: "flat",
|
||||
conditions: ["FILL"]
|
||||
},
|
||||
{
|
||||
id: "bullish_sweep",
|
||||
weight: 8,
|
||||
right: "C",
|
||||
countRange: [2, 3],
|
||||
sizeRange: [180, 520],
|
||||
premiumRange: [25_000, 90_000],
|
||||
priceTrend: "up",
|
||||
conditions: ["SWEEP"]
|
||||
},
|
||||
{
|
||||
id: "bearish_sweep",
|
||||
weight: 8,
|
||||
right: "P",
|
||||
countRange: [2, 3],
|
||||
sizeRange: [180, 520],
|
||||
premiumRange: [25_000, 90_000],
|
||||
priceTrend: "up",
|
||||
conditions: ["SWEEP"]
|
||||
},
|
||||
{
|
||||
id: "contract_spike",
|
||||
weight: 6,
|
||||
right: "either",
|
||||
countRange: [2, 3],
|
||||
sizeRange: [500, 900],
|
||||
premiumRange: [18_000, 70_000],
|
||||
priceTrend: "flat",
|
||||
conditions: ["ISO"]
|
||||
},
|
||||
{
|
||||
id: "noise",
|
||||
weight: 46,
|
||||
right: "either",
|
||||
countRange: [1, 2],
|
||||
sizeRange: [5, 60],
|
||||
premiumRange: [500, 6_000],
|
||||
priceTrend: "flat",
|
||||
conditions: ["FILL"]
|
||||
}
|
||||
];
|
||||
|
||||
const ACTIVE_SCENARIOS: Scenario[] = [
|
||||
{
|
||||
id: "bullish_sweep",
|
||||
weight: 35,
|
||||
|
|
@ -88,7 +158,50 @@ const SCENARIOS: Scenario[] = [
|
|||
}
|
||||
];
|
||||
|
||||
const PRICE_PLACEMENTS: Record<string, WeightedValue<PricePlacement>[]> = {
|
||||
const REALISTIC_PRICE_PLACEMENTS: Record<string, WeightedValue<PricePlacement>[]> = {
|
||||
ask_lift: [
|
||||
{ value: "A", weight: 45 },
|
||||
{ value: "AA", weight: 20 },
|
||||
{ value: "MID", weight: 25 },
|
||||
{ value: "B", weight: 8 },
|
||||
{ value: "BB", weight: 2 }
|
||||
],
|
||||
mid_block: [
|
||||
{ value: "MID", weight: 60 },
|
||||
{ value: "A", weight: 20 },
|
||||
{ value: "B", weight: 20 }
|
||||
],
|
||||
bullish_sweep: [
|
||||
{ value: "AA", weight: 20 },
|
||||
{ value: "A", weight: 50 },
|
||||
{ value: "MID", weight: 15 },
|
||||
{ value: "B", weight: 10 },
|
||||
{ value: "BB", weight: 5 }
|
||||
],
|
||||
bearish_sweep: [
|
||||
{ value: "AA", weight: 10 },
|
||||
{ value: "A", weight: 20 },
|
||||
{ value: "MID", weight: 15 },
|
||||
{ value: "B", weight: 35 },
|
||||
{ value: "BB", weight: 20 }
|
||||
],
|
||||
contract_spike: [
|
||||
{ value: "A", weight: 25 },
|
||||
{ value: "MID", weight: 40 },
|
||||
{ value: "B", weight: 25 },
|
||||
{ value: "AA", weight: 5 },
|
||||
{ value: "BB", weight: 5 }
|
||||
],
|
||||
noise: [
|
||||
{ value: "MID", weight: 40 },
|
||||
{ value: "A", weight: 20 },
|
||||
{ value: "B", weight: 20 },
|
||||
{ value: "AA", weight: 10 },
|
||||
{ value: "BB", weight: 10 }
|
||||
]
|
||||
};
|
||||
|
||||
const ACTIVE_PRICE_PLACEMENTS: Record<string, WeightedValue<PricePlacement>[]> = {
|
||||
bullish_sweep: [
|
||||
{ value: "AA", weight: 25 },
|
||||
{ value: "A", weight: 40 },
|
||||
|
|
@ -115,7 +228,52 @@ const PRICE_PLACEMENTS: Record<string, WeightedValue<PricePlacement>[]> = {
|
|||
]
|
||||
};
|
||||
|
||||
const PLACEMENT_PATTERN: PricePlacement[] = ["A", "AA", "B", "BB"];
|
||||
const FIREHOSE_PRICE_PLACEMENTS: Record<string, WeightedValue<PricePlacement>[]> = {
|
||||
...ACTIVE_PRICE_PLACEMENTS,
|
||||
noise: [
|
||||
{ value: "A", weight: 20 },
|
||||
{ value: "AA", weight: 20 },
|
||||
{ value: "MID", weight: 20 },
|
||||
{ value: "B", weight: 20 },
|
||||
{ value: "BB", weight: 20 }
|
||||
]
|
||||
};
|
||||
|
||||
const PLACEMENT_PATTERN: PricePlacement[] = ["A", "AA", "MID", "B", "BB"];
|
||||
|
||||
const SYNTHETIC_PROFILES: Record<SyntheticMarketMode, SyntheticOptionsProfile> = {
|
||||
realistic: {
|
||||
burstRunRange: [1, 2],
|
||||
scenarios: REALISTIC_SCENARIOS,
|
||||
pricePlacements: REALISTIC_PRICE_PLACEMENTS
|
||||
},
|
||||
active: {
|
||||
burstRunRange: [2, 4],
|
||||
scenarios: ACTIVE_SCENARIOS,
|
||||
pricePlacements: ACTIVE_PRICE_PLACEMENTS
|
||||
},
|
||||
firehose: {
|
||||
burstRunRange: [4, 7],
|
||||
scenarios: ACTIVE_SCENARIOS.map((scenario): Scenario =>
|
||||
scenario.id === "noise"
|
||||
? {
|
||||
...scenario,
|
||||
weight: 20,
|
||||
countRange: [5, 8],
|
||||
sizeRange: [20, 300],
|
||||
premiumRange: [800, 12_000]
|
||||
}
|
||||
: {
|
||||
...scenario,
|
||||
weight: scenario.weight + 10,
|
||||
countRange: [scenario.countRange[0] + 2, scenario.countRange[1] + 3],
|
||||
sizeRange: [scenario.sizeRange[0], scenario.sizeRange[1] * 2],
|
||||
premiumRange: [scenario.premiumRange[0], scenario.premiumRange[1] * 1.5]
|
||||
}
|
||||
),
|
||||
pricePlacements: FIREHOSE_PRICE_PLACEMENTS
|
||||
}
|
||||
};
|
||||
|
||||
const pick = <T,>(items: T[], seed: number): T => {
|
||||
return items[Math.abs(seed) % items.length];
|
||||
|
|
@ -153,8 +311,12 @@ const pickWeightedValue = <T>(items: WeightedValue<T>[], seed: number): T => {
|
|||
return pickWeighted(items, seed).value;
|
||||
};
|
||||
|
||||
const pickPlacement = (burst: Burst, index: number): PricePlacement => {
|
||||
const placementOptions = PRICE_PLACEMENTS[burst.scenarioId] ?? PRICE_PLACEMENTS.noise;
|
||||
const pickPlacement = (
|
||||
burst: Burst,
|
||||
index: number,
|
||||
profile: SyntheticOptionsProfile
|
||||
): PricePlacement => {
|
||||
const placementOptions = profile.pricePlacements[burst.scenarioId] ?? profile.pricePlacements.noise;
|
||||
const offset = Math.abs(burst.seed) % PLACEMENT_PATTERN.length;
|
||||
if (index < PLACEMENT_PATTERN.length) {
|
||||
return PLACEMENT_PATTERN[(offset + index) % PLACEMENT_PATTERN.length];
|
||||
|
|
@ -180,11 +342,11 @@ const formatExpiry = (now: number, offsetDays: number): string => {
|
|||
return expiryDate.toISOString().slice(0, 10);
|
||||
};
|
||||
|
||||
const buildBurst = (burstIndex: number, now: number): Burst => {
|
||||
const buildBurst = (burstIndex: number, now: number, profile: SyntheticOptionsProfile): Burst => {
|
||||
const symbol = SYNTHETIC_SYMBOLS[burstIndex % SYNTHETIC_SYMBOLS.length];
|
||||
const symbolHash = hashSymbol(symbol);
|
||||
const seed = symbolHash + burstIndex * 7;
|
||||
const scenario = pickWeighted(SCENARIOS, seed);
|
||||
const scenario = pickWeighted(profile.scenarios, seed);
|
||||
const baseUnderlying = 30 + (symbolHash % 470);
|
||||
const expiryOffset = pick(EXPIRY_OFFSETS, symbolHash + burstIndex);
|
||||
const expiry = formatExpiry(now, expiryOffset);
|
||||
|
|
@ -231,6 +393,7 @@ const buildBurst = (burstIndex: number, now: number): Burst => {
|
|||
export const createSyntheticOptionsAdapter = (
|
||||
config: SyntheticOptionsAdapterConfig
|
||||
): OptionIngestAdapter => {
|
||||
const profile = SYNTHETIC_PROFILES[config.mode];
|
||||
return {
|
||||
name: "synthetic",
|
||||
start: (handlers: OptionIngestHandlers) => {
|
||||
|
|
@ -250,8 +413,12 @@ export const createSyntheticOptionsAdapter = (
|
|||
const now = Date.now();
|
||||
if (!currentBurst || remainingRuns <= 0) {
|
||||
burstIndex += 1;
|
||||
currentBurst = buildBurst(burstIndex, now);
|
||||
remainingRuns = pickInt(BURST_RUN_RANGE[0], BURST_RUN_RANGE[1], burstIndex * 23);
|
||||
currentBurst = buildBurst(burstIndex, now, profile);
|
||||
remainingRuns = pickInt(
|
||||
profile.burstRunRange[0],
|
||||
profile.burstRunRange[1],
|
||||
burstIndex * 23
|
||||
);
|
||||
}
|
||||
|
||||
const burst = currentBurst;
|
||||
|
|
@ -267,13 +434,15 @@ export const createSyntheticOptionsAdapter = (
|
|||
const bid = Math.max(0.01, Number((mid - spread / 2).toFixed(2)));
|
||||
const ask = Math.max(bid + 0.01, Number((mid + spread / 2).toFixed(2)));
|
||||
const tick = Math.max(0.01, Number((spread * 0.25).toFixed(2)));
|
||||
const placement = pickPlacement(burst, i);
|
||||
const placement = pickPlacement(burst, i, profile);
|
||||
let tradePrice = mid;
|
||||
|
||||
if (placement === "AA") {
|
||||
tradePrice = ask + tick;
|
||||
} else if (placement === "A") {
|
||||
tradePrice = ask;
|
||||
} else if (placement === "MID") {
|
||||
tradePrice = mid;
|
||||
} else if (placement === "BB") {
|
||||
tradePrice = Math.max(0.01, bid - tick);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@ import { createLogger } from "@islandflow/observability";
|
|||
import {
|
||||
SUBJECT_OPTION_NBBO,
|
||||
SUBJECT_OPTION_PRINTS,
|
||||
SUBJECT_OPTION_SIGNAL_PRINTS,
|
||||
STREAM_OPTION_NBBO,
|
||||
STREAM_OPTION_PRINTS,
|
||||
STREAM_OPTION_SIGNAL_PRINTS,
|
||||
connectJetStreamWithRetry,
|
||||
ensureStream,
|
||||
publishJson
|
||||
|
|
@ -16,7 +18,16 @@ import {
|
|||
insertOptionNBBO,
|
||||
insertOptionPrint
|
||||
} from "@islandflow/storage";
|
||||
import { OptionNBBOSchema, OptionPrintSchema, type OptionNBBO, type OptionPrint } from "@islandflow/types";
|
||||
import {
|
||||
OptionNBBOSchema,
|
||||
OptionPrintSchema,
|
||||
evaluateOptionSignal,
|
||||
deriveOptionPrintMetadata,
|
||||
resolveSyntheticMarketModes,
|
||||
type OptionNBBO,
|
||||
type OptionPrint,
|
||||
type OptionsSignalConfig
|
||||
} from "@islandflow/types";
|
||||
import { createAlpacaOptionsAdapter } from "./adapters/alpaca";
|
||||
import { createDatabentoOptionsAdapter } from "./adapters/databento";
|
||||
import { createIbkrOptionsAdapter } from "./adapters/ibkr";
|
||||
|
|
@ -68,6 +79,17 @@ const envSchema = z.object({
|
|||
IBKR_CURRENCY: z.string().min(1).default("USD"),
|
||||
IBKR_PYTHON_BIN: z.string().min(1).default("python3"),
|
||||
EMIT_INTERVAL_MS: z.coerce.number().int().positive().default(1000),
|
||||
SYNTHETIC_MARKET_MODE: z.string().default("realistic"),
|
||||
SYNTHETIC_OPTIONS_MODE: z.string().default(""),
|
||||
OPTIONS_SIGNAL_MODE: z.enum(["smart-money", "balanced", "all"]).default("smart-money"),
|
||||
OPTIONS_SIGNAL_MIN_NOTIONAL: z.coerce.number().nonnegative().default(10_000),
|
||||
OPTIONS_SIGNAL_ETF_MIN_NOTIONAL: z.coerce.number().nonnegative().default(50_000),
|
||||
OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL: z.coerce.number().nonnegative().default(25_000),
|
||||
OPTIONS_SIGNAL_MID_MIN_NOTIONAL: z.coerce.number().nonnegative().default(20_000),
|
||||
OPTIONS_SIGNAL_NBBO_MAX_AGE_MS: z.coerce.number().int().positive().default(1500),
|
||||
OPTIONS_SIGNAL_ETF_UNDERLYINGS: z
|
||||
.string()
|
||||
.default("SPY,QQQ,IWM,DIA,TLT,GLD,SLV,XLF,XLE,XLV,XLI,XLP,XLU,XLY,SMH,ARKK"),
|
||||
TESTING_MODE: z
|
||||
.preprocess((value) => {
|
||||
if (typeof value === "string") {
|
||||
|
|
@ -86,11 +108,34 @@ const envSchema = z.object({
|
|||
});
|
||||
|
||||
const env = readEnv(envSchema);
|
||||
const syntheticModes = resolveSyntheticMarketModes({
|
||||
syntheticMarketMode: env.SYNTHETIC_MARKET_MODE,
|
||||
syntheticOptionsMode: env.SYNTHETIC_OPTIONS_MODE
|
||||
});
|
||||
const optionsSignalConfig: OptionsSignalConfig = {
|
||||
mode: env.OPTIONS_SIGNAL_MODE,
|
||||
minNotional: env.OPTIONS_SIGNAL_MIN_NOTIONAL,
|
||||
etfMinNotional: env.OPTIONS_SIGNAL_ETF_MIN_NOTIONAL,
|
||||
bidSideMinNotional: env.OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL,
|
||||
midMinNotional: env.OPTIONS_SIGNAL_MID_MIN_NOTIONAL,
|
||||
missingNbboMinNotional: 50_000,
|
||||
largePrintMinSize: 500,
|
||||
largePrintMinNotional: env.OPTIONS_SIGNAL_MIN_NOTIONAL,
|
||||
sweepMinNotional: env.OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL,
|
||||
autoKeepMinNotional: 100_000,
|
||||
nbboMaxAgeMs: env.OPTIONS_SIGNAL_NBBO_MAX_AGE_MS,
|
||||
etfUnderlyings: new Set(
|
||||
env.OPTIONS_SIGNAL_ETF_UNDERLYINGS.split(",")
|
||||
.map((value) => value.trim().toUpperCase())
|
||||
.filter(Boolean)
|
||||
)
|
||||
};
|
||||
|
||||
const state = {
|
||||
shuttingDown: false,
|
||||
shutdownPromise: null as Promise<void> | null
|
||||
};
|
||||
const latestNbboByContract = new Map<string, OptionNBBO>();
|
||||
|
||||
const getErrorMessage = (error: unknown): string => {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
|
|
@ -169,7 +214,10 @@ const retry = async <T>(
|
|||
|
||||
const selectAdapter = (name: string): OptionIngestAdapter => {
|
||||
if (name === "synthetic") {
|
||||
return createSyntheticOptionsAdapter({ emitIntervalMs: env.EMIT_INTERVAL_MS });
|
||||
return createSyntheticOptionsAdapter({
|
||||
emitIntervalMs: env.EMIT_INTERVAL_MS,
|
||||
mode: syntheticModes.options
|
||||
});
|
||||
}
|
||||
|
||||
if (name === "alpaca") {
|
||||
|
|
@ -277,6 +325,19 @@ const run = async () => {
|
|||
num_replicas: 1
|
||||
});
|
||||
|
||||
await ensureStream(jsm, {
|
||||
name: STREAM_OPTION_SIGNAL_PRINTS,
|
||||
subjects: [SUBJECT_OPTION_SIGNAL_PRINTS],
|
||||
retention: "limits",
|
||||
storage: "file",
|
||||
discard: "old",
|
||||
max_msgs_per_subject: -1,
|
||||
max_msgs: -1,
|
||||
max_bytes: -1,
|
||||
max_age: 0,
|
||||
num_replicas: 1
|
||||
});
|
||||
|
||||
const clickhouse = createClickHouseClient({
|
||||
url: env.CLICKHOUSE_URL,
|
||||
database: env.CLICKHOUSE_DATABASE
|
||||
|
|
@ -303,15 +364,41 @@ const run = async () => {
|
|||
return;
|
||||
}
|
||||
|
||||
const print = OptionPrintSchema.parse(candidate);
|
||||
const rawPrint = OptionPrintSchema.parse(candidate);
|
||||
const derived = deriveOptionPrintMetadata(
|
||||
rawPrint,
|
||||
latestNbboByContract.get(rawPrint.option_contract_id),
|
||||
optionsSignalConfig
|
||||
);
|
||||
const signalDecision = evaluateOptionSignal(
|
||||
{
|
||||
...rawPrint,
|
||||
...derived,
|
||||
signal_profile: optionsSignalConfig.mode
|
||||
},
|
||||
optionsSignalConfig
|
||||
);
|
||||
const print = OptionPrintSchema.parse({
|
||||
...rawPrint,
|
||||
...derived,
|
||||
signal_pass: signalDecision.signalPass,
|
||||
signal_reasons: signalDecision.signalReasons,
|
||||
signal_profile: signalDecision.signalProfile
|
||||
});
|
||||
|
||||
try {
|
||||
await insertOptionPrint(clickhouse, print);
|
||||
await publishJson(js, SUBJECT_OPTION_PRINTS, print);
|
||||
if (print.signal_pass) {
|
||||
await publishJson(js, SUBJECT_OPTION_SIGNAL_PRINTS, print);
|
||||
}
|
||||
logger.info("published option print", {
|
||||
trace_id: print.trace_id,
|
||||
seq: print.seq,
|
||||
option_contract_id: print.option_contract_id
|
||||
option_contract_id: print.option_contract_id,
|
||||
signal_pass: print.signal_pass,
|
||||
nbbo_side: print.nbbo_side,
|
||||
notional: print.notional
|
||||
});
|
||||
} catch (error) {
|
||||
if (isExpectedShutdownError(error)) {
|
||||
|
|
@ -335,6 +422,14 @@ const run = async () => {
|
|||
}
|
||||
|
||||
const nbbo = OptionNBBOSchema.parse(candidate);
|
||||
const existing = latestNbboByContract.get(nbbo.option_contract_id);
|
||||
if (
|
||||
!existing ||
|
||||
nbbo.ts > existing.ts ||
|
||||
(nbbo.ts === existing.ts && nbbo.seq >= existing.seq)
|
||||
) {
|
||||
latestNbboByContract.set(nbbo.option_contract_id, nbbo);
|
||||
}
|
||||
|
||||
try {
|
||||
await insertOptionNBBO(clickhouse, nbbo);
|
||||
|
|
|
|||
|
|
@ -5,10 +5,12 @@ import {
|
|||
SUBJECT_EQUITY_QUOTES,
|
||||
SUBJECT_OPTION_NBBO,
|
||||
SUBJECT_OPTION_PRINTS,
|
||||
SUBJECT_OPTION_SIGNAL_PRINTS,
|
||||
STREAM_EQUITY_PRINTS,
|
||||
STREAM_EQUITY_QUOTES,
|
||||
STREAM_OPTION_NBBO,
|
||||
STREAM_OPTION_PRINTS,
|
||||
STREAM_OPTION_SIGNAL_PRINTS,
|
||||
connectJetStreamWithRetry,
|
||||
ensureStream,
|
||||
publishJson
|
||||
|
|
@ -304,6 +306,9 @@ const run = async () => {
|
|||
const def = STREAM_DEFS[kind];
|
||||
await ensureStream(jsm, buildStreamConfig(def.streamName, def.subject));
|
||||
}
|
||||
if (streamKinds.includes("options")) {
|
||||
await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS));
|
||||
}
|
||||
|
||||
const clickhouse = createClickHouseClient({
|
||||
url: env.CLICKHOUSE_URL,
|
||||
|
|
@ -411,6 +416,9 @@ const run = async () => {
|
|||
|
||||
try {
|
||||
await publishJson(js, stream.subject, event);
|
||||
if (stream.kind === "options" && (event as OptionPrint).signal_pass) {
|
||||
await publishJson(js, SUBJECT_OPTION_SIGNAL_PRINTS, event as OptionPrint);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error("failed to publish replay event", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue