Implement options snapshot tape table

This commit is contained in:
dirtydishes 2026-05-04 01:14:52 -04:00
parent 6abfff30d3
commit e78387130a
15 changed files with 904 additions and 128 deletions

View file

@ -13,6 +13,9 @@ type SyntheticOptionsAdapterConfig = {
type Burst = {
contractId: string;
underlying: number;
expiryOffsetDays: number;
strike: number;
basePrice: number;
baseSize: number;
exchange: string;
@ -23,7 +26,16 @@ type Burst = {
seed: number;
};
export type SyntheticContractIvState = {
iv: number;
pressure: number;
lastTs: number;
};
const OPTION_CONTRACT_MULTIPLIER = 100;
const IV_MIN = 0.05;
const IV_MAX = 2.5;
const IV_DECAY_HALF_LIFE_MS = 60_000;
const SYNTHETIC_SYMBOLS = ["SPY", ...(SP500_SYMBOLS as readonly string[])];
const MS_PER_DAY = 24 * 60 * 60 * 1000;
@ -36,7 +48,7 @@ type SyntheticOptionsProfile = {
pricePlacements: Record<string, WeightedValue<PricePlacement>[]>;
};
type PricePlacement = "AA" | "A" | "MID" | "B" | "BB";
export type PricePlacement = "AA" | "A" | "MID" | "B" | "BB";
type WeightedValue<T> = {
value: T;
@ -347,6 +359,55 @@ const formatExpiry = (now: number, offsetDays: number): string => {
return expiryDate.toISOString().slice(0, 10);
};
const clampValue = (value: number, min: number, max: number): number => {
if (!Number.isFinite(value)) {
return min;
}
return Math.max(min, Math.min(max, value));
};
const initializeSyntheticIv = (dteDays: number, moneyness: number): number => {
const dteBoost = dteDays <= 0 ? 0.22 : dteDays <= 7 ? 0.14 : dteDays <= 30 ? 0.06 : 0;
const moneynessBoost = clampValue(Math.abs(moneyness - 1) * 0.8, 0, 0.2);
return clampValue(0.24 + dteBoost + moneynessBoost, 0.18, 0.65);
};
export const updateSyntheticIvForTest = (
state: SyntheticContractIvState | undefined,
input: {
ts: number;
placement: PricePlacement;
size: number;
notional: number;
dteDays: number;
moneyness: number;
}
): SyntheticContractIvState => {
const previous = state ?? {
iv: initializeSyntheticIv(input.dteDays, input.moneyness),
pressure: 0,
lastTs: input.ts
};
const elapsed = Math.max(0, input.ts - previous.lastTs);
const decay = Math.pow(0.5, elapsed / IV_DECAY_HALF_LIFE_MS);
let pressure = previous.pressure * decay;
if (input.placement === "AA" || input.placement === "A") {
const sizeImpact = Math.log10(Math.max(10, input.size)) * 0.012;
const notionalImpact = Math.log10(Math.max(1_000, input.notional)) * 0.01;
pressure += input.placement === "AA" ? sizeImpact + notionalImpact : (sizeImpact + notionalImpact) * 0.65;
} else if (input.placement === "MID") {
pressure += 0.001;
} else {
pressure -= input.placement === "BB" ? 0.018 : 0.01;
}
pressure = clampValue(pressure, -0.25, 1.85);
const baseline = initializeSyntheticIv(input.dteDays, input.moneyness);
const iv = clampValue(baseline + pressure * 0.42, IV_MIN, IV_MAX);
return { iv: Number(iv.toFixed(4)), pressure, lastTs: input.ts };
};
const buildBurst = (burstIndex: number, now: number, profile: SyntheticOptionsProfile): Burst => {
const symbol = SYNTHETIC_SYMBOLS[burstIndex % SYNTHETIC_SYMBOLS.length];
const symbolHash = hashSymbol(symbol);
@ -392,6 +453,9 @@ const buildBurst = (burstIndex: number, now: number, profile: SyntheticOptionsPr
return {
contractId,
underlying: baseUnderlying,
expiryOffsetDays: expiryOffset,
strike,
basePrice: basePricePer,
baseSize,
exchange,
@ -420,6 +484,7 @@ export const createSyntheticOptionsAdapter = (
let nbboSeq = 0;
let burstIndex = 0;
let currentBurst: Burst | null = null;
const ivByContract = new Map<string, SyntheticContractIvState>();
let remainingRuns = 0;
let timer: ReturnType<typeof setInterval> | null = null;
let stopped = false;
@ -448,12 +513,28 @@ export const createSyntheticOptionsAdapter = (
const priceJitter = ((i % 3) - 1) * 0.004;
const sizeJitter = ((i % 3) - 1) * 0.08;
const priceMultiplier = 1 + burst.priceStep * i + priceJitter;
const mid = Math.max(0.05, Number((burst.basePrice * priceMultiplier).toFixed(2)));
const spread = Math.max(0.02, Number((mid * 0.02).toFixed(2)));
const placement = pickPlacement(burst, i, profile);
const size = Math.max(1, Math.round(burst.baseSize * (1 + sizeJitter)));
const previousIv = ivByContract.get(burst.contractId);
const provisionalNotional = burst.basePrice * size * OPTION_CONTRACT_MULTIPLIER;
const ivState = updateSyntheticIvForTest(previousIv, {
ts: now + i * 5,
placement,
size,
notional: provisionalNotional,
dteDays: burst.expiryOffsetDays,
moneyness: burst.strike / burst.underlying
});
ivByContract.set(burst.contractId, ivState);
const ivDrift = Math.max(0, ivState.iv - initializeSyntheticIv(burst.expiryOffsetDays, burst.strike / burst.underlying));
const mid = Math.max(
0.05,
Number((burst.basePrice * priceMultiplier * (1 + ivDrift * 1.15)).toFixed(2))
);
const spread = Math.max(0.02, Number((mid * (0.02 + Math.min(0.035, ivState.iv * 0.01))).toFixed(2)));
const bid = Math.max(0.01, Number((mid - spread / 2).toFixed(2)));
const ask = Math.max(bid + 0.01, Number((mid + spread / 2).toFixed(2)));
const tick = Math.max(0.01, Number((spread * 0.25).toFixed(2)));
const placement = pickPlacement(burst, i, profile);
let tradePrice = mid;
if (placement === "AA") {
@ -476,9 +557,11 @@ export const createSyntheticOptionsAdapter = (
ts: now + i * 5,
option_contract_id: burst.contractId,
price: tradePrice,
size: Math.max(1, Math.round(burst.baseSize * (1 + sizeJitter))),
size,
exchange: burst.exchange,
conditions: burst.conditions
conditions: burst.conditions,
execution_iv: ivState.iv,
execution_iv_source: "synthetic_pressure_model"
};
if (handlers.onNBBO) {

View file

@ -0,0 +1,125 @@
import {
OptionPrintSchema,
classifyOptionNbboSide,
deriveOptionPrintMetadata,
evaluateOptionSignal,
type EquityQuote,
type OptionNBBO,
type OptionPrint,
type OptionsSignalConfig
} from "@islandflow/types";
export const MAX_CONTEXT_HISTORY = 64;
export type ContextHistory<T extends { ts: number; seq: number }> = Map<string, T[]>;
export const rememberContext = <T extends { ts: number; seq: number }>(
history: ContextHistory<T>,
key: string,
value: T
): void => {
const bucket = history.get(key) ?? [];
const existingIndex = bucket.findIndex((item) => item.ts === value.ts && item.seq === value.seq);
if (existingIndex >= 0) {
bucket[existingIndex] = value;
} else {
bucket.push(value);
}
bucket.sort((a, b) => {
const delta = a.ts - b.ts;
return delta !== 0 ? delta : a.seq - b.seq;
});
if (bucket.length > MAX_CONTEXT_HISTORY) {
bucket.splice(0, bucket.length - MAX_CONTEXT_HISTORY);
}
history.set(key, bucket);
};
export const selectAtOrBefore = <T extends { ts: number; seq: number }>(
items: readonly T[] | undefined,
ts: number
): T | null => {
if (!items?.length) {
return null;
}
let selected: T | null = null;
for (const item of items) {
if (item.ts > ts) {
continue;
}
if (!selected || item.ts > selected.ts || (item.ts === selected.ts && item.seq >= selected.seq)) {
selected = item;
}
}
return selected;
};
export const enrichOptionPrint = (
rawPrint: OptionPrint,
optionQuote: OptionNBBO | null | undefined,
equityQuote: EquityQuote | null | undefined,
config: OptionsSignalConfig
): OptionPrint => {
const derived = deriveOptionPrintMetadata(rawPrint, optionQuote, config);
const executionNbboSide = optionQuote
? classifyOptionNbboSide(rawPrint.price, optionQuote, rawPrint.ts, config.nbboMaxAgeMs)
: undefined;
const nbboMid =
optionQuote && Number.isFinite(optionQuote.bid) && Number.isFinite(optionQuote.ask)
? Number(((optionQuote.bid + optionQuote.ask) / 2).toFixed(4))
: undefined;
const nbboSpread =
optionQuote && Number.isFinite(optionQuote.bid) && Number.isFinite(optionQuote.ask)
? Number(Math.max(0, optionQuote.ask - optionQuote.bid).toFixed(4))
: undefined;
const underlyingMid =
equityQuote && Number.isFinite(equityQuote.bid) && Number.isFinite(equityQuote.ask)
? Number(((equityQuote.bid + equityQuote.ask) / 2).toFixed(4))
: undefined;
const underlyingSpread =
equityQuote && Number.isFinite(equityQuote.bid) && Number.isFinite(equityQuote.ask)
? Number(Math.max(0, equityQuote.ask - equityQuote.bid).toFixed(4))
: undefined;
const enrichedForSignal: OptionPrint = {
...rawPrint,
...derived,
nbbo_side: executionNbboSide ?? derived.nbbo_side,
...(optionQuote
? {
execution_nbbo_bid: optionQuote.bid,
execution_nbbo_ask: optionQuote.ask,
execution_nbbo_mid: nbboMid,
execution_nbbo_spread: nbboSpread,
execution_nbbo_bid_size: optionQuote.bidSize,
execution_nbbo_ask_size: optionQuote.askSize,
execution_nbbo_ts: optionQuote.ts,
execution_nbbo_age_ms: rawPrint.ts - optionQuote.ts,
execution_nbbo_side: executionNbboSide,
nbbo_side: executionNbboSide
}
: {}),
...(equityQuote && underlyingMid !== undefined
? {
execution_underlying_spot: underlyingMid,
execution_underlying_bid: equityQuote.bid,
execution_underlying_ask: equityQuote.ask,
execution_underlying_mid: underlyingMid,
execution_underlying_spread: underlyingSpread,
execution_underlying_ts: equityQuote.ts,
execution_underlying_age_ms: rawPrint.ts - equityQuote.ts,
execution_underlying_source: "equity_quote_mid" as const
}
: {}),
signal_profile: config.mode
};
const signalDecision = evaluateOptionSignal(enrichedForSignal, config);
return OptionPrintSchema.parse({
...enrichedForSignal,
signal_pass: signalDecision.signalPass,
signal_reasons: signalDecision.signalReasons,
signal_profile: signalDecision.signalProfile
});
};

View file

@ -4,12 +4,16 @@ import {
SUBJECT_OPTION_NBBO,
SUBJECT_OPTION_PRINTS,
SUBJECT_OPTION_SIGNAL_PRINTS,
SUBJECT_EQUITY_QUOTES,
STREAM_EQUITY_QUOTES,
STREAM_OPTION_NBBO,
STREAM_OPTION_PRINTS,
STREAM_OPTION_SIGNAL_PRINTS,
buildDurableConsumer,
connectJetStreamWithRetry,
ensureStream,
publishJson
publishJson,
subscribeJson
} from "@islandflow/bus";
import {
createClickHouseClient,
@ -21,9 +25,10 @@ import {
import {
OptionNBBOSchema,
OptionPrintSchema,
evaluateOptionSignal,
EquityQuoteSchema,
deriveOptionPrintMetadata,
resolveSyntheticMarketModes,
type EquityQuote,
type OptionNBBO,
type OptionPrint,
type OptionsSignalConfig
@ -33,6 +38,7 @@ import { createDatabentoOptionsAdapter } from "./adapters/databento";
import { createIbkrOptionsAdapter } from "./adapters/ibkr";
import { createSyntheticOptionsAdapter } from "./adapters/synthetic";
import type { OptionIngestAdapter, StopHandler } from "./adapters/types";
import { enrichOptionPrint, rememberContext, selectAtOrBefore, type ContextHistory } from "./enrichment";
import { z } from "zod";
const service = "ingest-options";
@ -135,7 +141,9 @@ const state = {
shuttingDown: false,
shutdownPromise: null as Promise<void> | null
};
const latestNbboByContract = new Map<string, OptionNBBO>();
const nbboHistoryByContract: ContextHistory<OptionNBBO> = new Map();
const equityQuoteHistoryByUnderlying: ContextHistory<EquityQuote> = new Map();
const getErrorMessage = (error: unknown): string => {
return error instanceof Error ? error.message : String(error);
@ -338,6 +346,19 @@ const run = async () => {
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_EQUITY_QUOTES,
subjects: [SUBJECT_EQUITY_QUOTES],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
const clickhouse = createClickHouseClient({
url: env.CLICKHOUSE_URL,
database: env.CLICKHOUSE_DATABASE
@ -365,26 +386,15 @@ const run = async () => {
}
const rawPrint = OptionPrintSchema.parse(candidate);
const derived = deriveOptionPrintMetadata(
rawPrint,
latestNbboByContract.get(rawPrint.option_contract_id),
optionsSignalConfig
const parsedMetadata = deriveOptionPrintMetadata(rawPrint, null, optionsSignalConfig);
const optionQuote = selectAtOrBefore(
nbboHistoryByContract.get(rawPrint.option_contract_id),
rawPrint.ts
);
const signalDecision = evaluateOptionSignal(
{
...rawPrint,
...derived,
signal_profile: optionsSignalConfig.mode
},
optionsSignalConfig
);
const print = OptionPrintSchema.parse({
...rawPrint,
...derived,
signal_pass: signalDecision.signalPass,
signal_reasons: signalDecision.signalReasons,
signal_profile: signalDecision.signalProfile
});
const equityQuote = parsedMetadata.underlying_id
? selectAtOrBefore(equityQuoteHistoryByUnderlying.get(parsedMetadata.underlying_id), rawPrint.ts)
: null;
const print = enrichOptionPrint(rawPrint, optionQuote, equityQuote, optionsSignalConfig);
try {
await insertOptionPrint(clickhouse, print);
@ -422,14 +432,7 @@ const run = async () => {
}
const nbbo = OptionNBBOSchema.parse(candidate);
const existing = latestNbboByContract.get(nbbo.option_contract_id);
if (
!existing ||
nbbo.ts > existing.ts ||
(nbbo.ts === existing.ts && nbbo.seq >= existing.seq)
) {
latestNbboByContract.set(nbbo.option_contract_id, nbbo);
}
rememberContext(nbboHistoryByContract, nbbo.option_contract_id, nbbo);
try {
await insertOptionNBBO(clickhouse, nbbo);
@ -447,6 +450,33 @@ const run = async () => {
}
});
const equityQuoteConsumer = buildDurableConsumer("ingest-options-equity-quotes");
equityQuoteConsumer.deliverAll();
const equityQuoteSubscription = await subscribeJson<EquityQuote>(
js,
SUBJECT_EQUITY_QUOTES,
equityQuoteConsumer
);
void (async () => {
for await (const msg of equityQuoteSubscription.messages) {
if (state.shuttingDown) {
msg.ack();
continue;
}
try {
const quote = EquityQuoteSchema.parse(equityQuoteSubscription.decode(msg));
rememberContext(equityQuoteHistoryByUnderlying, quote.underlying_id.toUpperCase(), quote);
msg.ack();
} catch (error) {
logger.error("failed to process equity quote context", {
error: getErrorMessage(error)
});
msg.ack();
}
}
})();
const shutdown = async (signal: string) => {
if (state.shutdownPromise) {
return state.shutdownPromise;

View file

@ -0,0 +1,88 @@
import { describe, expect, it } from "bun:test";
import type { EquityQuote, OptionNBBO, OptionPrint, OptionsSignalConfig } from "@islandflow/types";
import { enrichOptionPrint, selectAtOrBefore } from "../src/enrichment";
const config: OptionsSignalConfig = {
mode: "all",
minNotional: 0,
etfMinNotional: 0,
bidSideMinNotional: 0,
midMinNotional: 0,
missingNbboMinNotional: 0,
largePrintMinSize: 1,
largePrintMinNotional: 0,
sweepMinNotional: 0,
autoKeepMinNotional: 100_000,
nbboMaxAgeMs: 1_500,
etfUnderlyings: new Set(["SPY"])
};
const print: OptionPrint = {
source_ts: 1_000,
ingest_ts: 1_000,
seq: 1,
trace_id: "print-1",
ts: 1_000,
option_contract_id: "SPY-2025-01-17-450-C",
price: 1.3,
size: 10,
exchange: "TEST"
};
const nbbo = (overrides: Partial<OptionNBBO> = {}): OptionNBBO => ({
source_ts: 990,
ingest_ts: 990,
seq: 1,
trace_id: "nbbo-1",
ts: 990,
option_contract_id: "SPY-2025-01-17-450-C",
bid: 1.2,
ask: 1.3,
bidSize: 20,
askSize: 30,
...overrides
});
const equityQuote = (overrides: Partial<EquityQuote> = {}): EquityQuote => ({
source_ts: 980,
ingest_ts: 980,
seq: 1,
trace_id: "eq-1",
ts: 980,
underlying_id: "SPY",
bid: 450,
ask: 450.1,
...overrides
});
describe("option print enrichment", () => {
it("attaches preserved NBBO context and mirrors nbbo_side", () => {
const enriched = enrichOptionPrint(print, nbbo(), null, config);
expect(enriched.execution_nbbo_bid).toBe(1.2);
expect(enriched.execution_nbbo_ask).toBe(1.3);
expect(enriched.execution_nbbo_mid).toBe(1.25);
expect(enriched.execution_nbbo_age_ms).toBe(10);
expect(enriched.execution_nbbo_side).toBe("A");
expect(enriched.nbbo_side).toBe(enriched.execution_nbbo_side);
});
it("attaches preserved underlying quote mid as spot", () => {
const enriched = enrichOptionPrint(print, null, equityQuote(), config);
expect(enriched.execution_underlying_spot).toBe(450.05);
expect(enriched.execution_underlying_mid).toBe(450.05);
expect(enriched.execution_underlying_source).toBe("equity_quote_mid");
expect(enriched.execution_underlying_age_ms).toBe(20);
});
it("selects context at or before the print timestamp only", () => {
const selected = selectAtOrBefore(
[nbbo({ ts: 900, seq: 1, bid: 1 }), nbbo({ ts: 1_001, seq: 2, bid: 2 })],
print.ts
);
expect(selected?.ts).toBe(900);
expect(selected?.bid).toBe(1);
});
});

View file

@ -1,5 +1,5 @@
import { describe, expect, it } from "bun:test";
import { buildSyntheticBurstForTest } from "../src/adapters/synthetic";
import { buildSyntheticBurstForTest, updateSyntheticIvForTest } from "../src/adapters/synthetic";
const totalBurstNotional = (burst: {
basePrice: number;
@ -24,3 +24,66 @@ describe("synthetic options burst sizing", () => {
expect(totalBurstNotional(burst)).toBeLessThanOrEqual(240_000);
});
});
describe("synthetic options IV model", () => {
it("increases under repeated same-contract ask buying", () => {
let state = updateSyntheticIvForTest(undefined, {
ts: 1_000,
placement: "A",
size: 100,
notional: 20_000,
dteDays: 1,
moneyness: 1.02
});
const firstIv = state.iv;
state = updateSyntheticIvForTest(state, {
ts: 1_100,
placement: "AA",
size: 300,
notional: 80_000,
dteDays: 1,
moneyness: 1.02
});
expect(state.iv).toBeGreaterThan(firstIv);
});
it("decays after inactivity", () => {
const active = updateSyntheticIvForTest(undefined, {
ts: 1_000,
placement: "AA",
size: 500,
notional: 120_000,
dteDays: 7,
moneyness: 1.1
});
const decayed = updateSyntheticIvForTest(active, {
ts: 181_000,
placement: "MID",
size: 10,
notional: 1_000,
dteDays: 7,
moneyness: 1.1
});
expect(decayed.iv).toBeLessThan(active.iv);
});
it("keeps IV within clamps", () => {
let state = undefined;
for (let i = 0; i < 80; i += 1) {
state = updateSyntheticIvForTest(state, {
ts: 1_000 + i * 10,
placement: "AA",
size: 10_000,
notional: 5_000_000,
dteDays: 0,
moneyness: 1.8
});
}
expect(state.iv).toBeGreaterThanOrEqual(0.05);
expect(state.iv).toBeLessThanOrEqual(2.5);
});
});