Add hosted synthetic control plane

This commit is contained in:
dirtydishes 2026-05-13 22:10:05 -04:00
parent af04875107
commit 8dcbcd2201
21 changed files with 3695 additions and 772 deletions

View file

@ -25,8 +25,12 @@ import {
STREAM_OPTION_SIGNAL_PRINTS,
buildDurableConsumer,
connectJetStreamWithRetry,
ensureSyntheticControlState,
ensureKnownStreams,
subscribeJson
openSyntheticControlKv,
subscribeJson,
watchSyntheticControlState,
writeSyntheticControlState
} from "@islandflow/bus";
import {
createClickHouseClient,
@ -100,6 +104,7 @@ import {
matchesFlowPacketFilters,
matchesOptionPrintFilters,
FlowPacketSchema,
SyntheticControlStateSchema,
SmartMoneyEventSchema,
OptionNBBOSchema,
OptionPrintSchema,
@ -114,6 +119,13 @@ import {
shouldFanoutLiveEvent
} from "./live";
import { parseOptionPrintQuery } from "./option-queries";
import {
buildSyntheticDerivedStatus,
createRollingSyntheticProfileHits,
getSyntheticBackendDisabledReason,
recordSyntheticProfileHit,
resolveSyntheticBackendMode
} from "./synthetic-control";
const service = "api";
const logger = createLogger({ service });
@ -127,10 +139,27 @@ const envSchema = z.object({
CLICKHOUSE_URL: z.string().default("http://127.0.0.1:8123"),
CLICKHOUSE_DATABASE: z.string().default("default"),
REDIS_URL: z.string().default("redis://127.0.0.1:6379"),
OPTIONS_INGEST_ADAPTER: z.string().min(1).default("synthetic"),
EQUITIES_INGEST_ADAPTER: z.string().min(1).default("synthetic"),
REST_DEFAULT_LIMIT: z.coerce.number().int().positive().default(200),
API_DELIVER_POLICY: DeliverPolicySchema.default("new"),
API_CONSUMER_RESET: z.coerce.boolean().default(false),
LIVE_LAG_WARN_MS: z.coerce.number().int().positive().default(120_000)
LIVE_LAG_WARN_MS: z.coerce.number().int().positive().default(120_000),
SYNTHETIC_CONTROL_ENABLED: z
.preprocess((value) => {
if (typeof value === "string") {
const normalized = value.trim().toLowerCase();
if (["1", "true", "yes", "on"].includes(normalized)) {
return true;
}
if (["0", "false", "no", "off"].includes(normalized)) {
return false;
}
}
return value;
}, z.boolean())
.default(false),
SYNTHETIC_ADMIN_TOKEN: z.string().default("")
});
const env = readEnv(envSchema);
@ -283,6 +312,14 @@ const readJsonBody = async (req: Request): Promise<unknown> => {
return JSON.parse(text);
};
const getBearerToken = (req: Request): string => {
const authorization = req.headers.get("authorization") ?? "";
if (authorization.toLowerCase().startsWith("bearer ")) {
return authorization.slice(7).trim();
}
return req.headers.get("x-synthetic-admin-token")?.trim() ?? "";
};
const optionsSupportLookupSchema = z.object({
trace_ids: z.array(z.string().min(1)).default([]),
nbbo_context: z
@ -641,6 +678,27 @@ const run = async () => {
{ logger }
);
const syntheticBackendMode = resolveSyntheticBackendMode(
env.OPTIONS_INGEST_ADAPTER,
env.EQUITIES_INGEST_ADAPTER
);
const syntheticBackendDisabledReason =
getSyntheticBackendDisabledReason(syntheticBackendMode);
const syntheticControlKv = await openSyntheticControlKv(js);
let syntheticControl = await ensureSyntheticControlState(syntheticControlKv);
const syntheticProfileHits = createRollingSyntheticProfileHits();
const stopSyntheticControlWatch = await watchSyntheticControlState(
syntheticControlKv,
(nextControl) => {
syntheticControl = nextControl;
},
(error) => {
logger.warn("synthetic control watch failed", {
error: getErrorMessage(error)
});
}
);
const clickhouse = createClickHouseClient({
url: env.CLICKHOUSE_URL,
database: env.CLICKHOUSE_DATABASE
@ -1146,6 +1204,7 @@ const run = async () => {
for await (const msg of smartMoneySubscription.messages) {
try {
const payload = SmartMoneyEventSchema.parse(smartMoneySubscription.decode(msg));
recordSyntheticProfileHit(syntheticProfileHits, payload);
broadcast(smartMoneySockets, { type: "smart-money", payload });
await fanoutLive({ channel: "smart-money" }, payload, "smart-money");
msg.ack();
@ -1202,6 +1261,54 @@ const run = async () => {
void pumpClassifierHits();
void pumpAlerts();
const buildSyntheticStatusBody = () => {
const derived =
syntheticBackendMode === "synthetic"
? buildSyntheticDerivedStatus(Date.now(), syntheticControl, syntheticProfileHits)
: null;
return {
enabled: env.SYNTHETIC_CONTROL_ENABLED && syntheticBackendMode === "synthetic",
backend_mode: syntheticBackendMode,
adapters: {
options: env.OPTIONS_INGEST_ADAPTER,
equities: env.EQUITIES_INGEST_ADAPTER
},
control: syntheticBackendMode === "synthetic" ? syntheticControl : null,
derived,
...(syntheticBackendDisabledReason
? { disabled_reason: syntheticBackendDisabledReason }
: {})
};
};
const authenticateSyntheticAdminRequest = (req: Request): Response | null => {
if (!env.SYNTHETIC_CONTROL_ENABLED) {
return jsonResponse({ error: "not found" }, 404);
}
if (!env.SYNTHETIC_ADMIN_TOKEN) {
return jsonResponse(
{
error: "synthetic admin misconfigured",
detail: "SYNTHETIC_ADMIN_TOKEN is required when synthetic control is enabled."
},
500
);
}
if (getBearerToken(req) !== env.SYNTHETIC_ADMIN_TOKEN) {
return jsonResponse({ error: "unauthorized" }, 401);
}
if (syntheticBackendMode !== "synthetic") {
return jsonResponse(
{
error: "synthetic backend unavailable",
...buildSyntheticStatusBody()
},
409
);
}
return null;
};
const server = Bun.serve<WsData | LiveWsData>({
port: env.API_PORT,
fetch: async (req: Request, serverRef: any) => {
@ -1211,6 +1318,49 @@ const run = async () => {
return jsonResponse({ status: "ok" });
}
if (req.method === "GET" && url.pathname === "/admin/synthetic/status") {
const authError = authenticateSyntheticAdminRequest(req);
if (authError) {
return authError;
}
return jsonResponse(buildSyntheticStatusBody());
}
if (req.method === "GET" && url.pathname === "/admin/synthetic/control") {
const authError = authenticateSyntheticAdminRequest(req);
if (authError) {
return authError;
}
return jsonResponse({ control: syntheticControl });
}
if (req.method === "PUT" && url.pathname === "/admin/synthetic/control") {
const authError = authenticateSyntheticAdminRequest(req);
if (authError) {
return authError;
}
try {
const payload = SyntheticControlStateSchema.parse(await readJsonBody(req));
syntheticControl = await writeSyntheticControlState(syntheticControlKv, payload);
return jsonResponse({
control: syntheticControl,
derived: buildSyntheticDerivedStatus(
Date.now(),
syntheticControl,
syntheticProfileHits
)
});
} catch (error) {
return jsonResponse(
{
error: "invalid synthetic control payload",
detail: getErrorMessage(error)
},
400
);
}
}
if (req.method === "GET" && url.pathname === "/prints/options") {
try {
const limit = parseLimit(url.searchParams.get("limit"));
@ -1824,6 +1974,7 @@ const run = async () => {
logger.info("service stopping", { signal });
server.stop();
clearInterval(liveStateMetricsTimer);
await stopSyntheticControlWatch();
await liveState.close();
if (redis && redis.isOpen) {

View file

@ -0,0 +1,93 @@
import {
SyntheticDerivedStatusSchema,
buildEmptySyntheticProfileHitCounts,
getSyntheticSessionState,
type SmartMoneyEvent,
type SmartMoneyProfileId,
type SyntheticControlState,
type SyntheticDerivedStatus
} from "@islandflow/types";
export type SyntheticBackendMode = "synthetic" | "mixed" | "live";
export type RollingSyntheticProfileHits = Record<SmartMoneyProfileId, number[]>;
export const createRollingSyntheticProfileHits = (): RollingSyntheticProfileHits => ({
institutional_directional: [],
retail_whale: [],
event_driven: [],
vol_seller: [],
arbitrage: [],
hedge_reactive: []
});
export const resolveSyntheticBackendMode = (
optionsAdapter: string,
equitiesAdapter: string
): SyntheticBackendMode => {
const optionsSynthetic = optionsAdapter === "synthetic";
const equitiesSynthetic = equitiesAdapter === "synthetic";
if (optionsSynthetic && equitiesSynthetic) {
return "synthetic";
}
if (optionsSynthetic || equitiesSynthetic) {
return "mixed";
}
return "live";
};
export const getSyntheticBackendDisabledReason = (
mode: SyntheticBackendMode
): string | undefined => {
if (mode === "synthetic") {
return undefined;
}
if (mode === "mixed") {
return "Synthetic control requires both hosted ingest adapters to run in synthetic mode.";
}
return "Hosted ingest adapters are not synthetic, so the internal synthetic control surface is unavailable.";
};
export const recordSyntheticProfileHit = (
state: RollingSyntheticProfileHits,
event: Pick<SmartMoneyEvent, "primary_profile_id" | "source_ts">
): void => {
if (!event.primary_profile_id) {
return;
}
state[event.primary_profile_id].push(event.source_ts);
};
export const getSyntheticProfileHitCounts = (
state: RollingSyntheticProfileHits,
now: number,
coverageWindowMinutes: number
): Record<SmartMoneyProfileId, number> => {
const floorTs = now - coverageWindowMinutes * 60_000;
const counts = buildEmptySyntheticProfileHitCounts();
for (const profileId of Object.keys(state) as SmartMoneyProfileId[]) {
const retained = state[profileId].filter((ts) => ts >= floorTs);
state[profileId] = retained;
counts[profileId] = retained.length;
}
return counts;
};
export const buildSyntheticDerivedStatus = (
now: number,
control: SyntheticControlState,
state: RollingSyntheticProfileHits
): SyntheticDerivedStatus => {
const session = getSyntheticSessionState(now, control);
return SyntheticDerivedStatusSchema.parse({
session_phase: session.session_phase,
regime: session.regime,
focus_symbols: session.focus_symbols,
profile_hit_counts: getSyntheticProfileHitCounts(
state,
now,
control.coverage_window_minutes
),
coverage_window_minutes: control.coverage_window_minutes
});
};

View file

@ -0,0 +1,69 @@
import { describe, expect, it } from "bun:test";
import { DEFAULT_SYNTHETIC_CONTROL_STATE } from "@islandflow/types";
import {
buildSyntheticDerivedStatus,
createRollingSyntheticProfileHits,
getSyntheticBackendDisabledReason,
getSyntheticProfileHitCounts,
recordSyntheticProfileHit,
resolveSyntheticBackendMode
} from "../src/synthetic-control";
describe("synthetic control backend mode", () => {
it("detects synthetic, mixed, and live hosted modes", () => {
expect(resolveSyntheticBackendMode("synthetic", "synthetic")).toBe("synthetic");
expect(resolveSyntheticBackendMode("synthetic", "alpaca")).toBe("mixed");
expect(resolveSyntheticBackendMode("alpaca", "alpaca")).toBe("live");
});
it("provides a useful disabled reason for non-synthetic modes", () => {
expect(getSyntheticBackendDisabledReason("mixed")).toContain("both hosted ingest adapters");
expect(getSyntheticBackendDisabledReason("live")).toContain("not synthetic");
});
});
describe("synthetic control rolling status", () => {
it("tracks public-profile hits inside the rolling coverage window", () => {
const hits = createRollingSyntheticProfileHits();
recordSyntheticProfileHit(hits, {
primary_profile_id: "event_driven",
source_ts: 1_000
});
recordSyntheticProfileHit(hits, {
primary_profile_id: "event_driven",
source_ts: 60_000
});
recordSyntheticProfileHit(hits, {
primary_profile_id: "arbitrage",
source_ts: 70_000
});
expect(getSyntheticProfileHitCounts(hits, 11 * 60_000, 10)).toEqual({
institutional_directional: 0,
retail_whale: 0,
event_driven: 1,
vol_seller: 0,
arbitrage: 1,
hedge_reactive: 0
});
});
it("builds derived status from the shared session engine", () => {
const hits = createRollingSyntheticProfileHits();
recordSyntheticProfileHit(hits, {
primary_profile_id: "hedge_reactive",
source_ts: Date.parse("2026-01-14T18:00:00Z")
});
const derived = buildSyntheticDerivedStatus(
Date.parse("2026-01-14T18:05:00Z"),
DEFAULT_SYNTHETIC_CONTROL_STATE,
hits
);
expect(derived.coverage_window_minutes).toBe(20);
expect(derived.focus_symbols.length).toBeGreaterThan(0);
expect(derived.profile_hit_counts.hedge_reactive).toBe(1);
});
});

View file

@ -1,7 +1,10 @@
import {
SP500_SYMBOLS,
getSyntheticSessionState,
getSyntheticUnderlyingState,
type EquityPrint,
type EquityQuote,
type SyntheticControlState,
type SyntheticMarketMode
} from "@islandflow/types";
import type { EquityIngestAdapter, EquityIngestHandlers } from "./types";
@ -9,34 +12,14 @@ import type { EquityIngestAdapter, EquityIngestHandlers } from "./types";
type SyntheticEquitiesAdapterConfig = {
emitIntervalMs: number;
mode: SyntheticMarketMode;
getControl: () => SyntheticControlState;
};
const EXCHANGES = ["NYSE", "NASDAQ", "ARCA", "BATS", "IEX", "TEST"];
const EXCHANGES = ["NYSE", "NASDAQ", "ARCA", "BATS", "IEX", "MEMX"];
const DARK_EXCHANGE = "OTC";
type PricePlacement = "MID" | "A" | "AA" | "B" | "BB";
type DarkScenario = "block" | "buy" | "sell";
const DARK_SEQUENCE: DarkScenario[] = [
"block",
"buy",
"buy",
"buy",
"buy",
"sell",
"sell",
"sell",
"sell"
];
const SYNTHETIC_SYMBOLS = ["SPY", ...(SP500_SYMBOLS as readonly string[])];
const hashSymbol = (value: string): number => {
let hash = 0;
for (let i = 0; i < value.length; i += 1) {
hash = (hash * 31 + value.charCodeAt(i)) >>> 0;
}
return hash;
};
type PricePlacement = "MID" | "A" | "AA" | "B" | "BB";
const buildSyntheticPrint = (
seq: number,
@ -46,20 +29,18 @@ const buildSyntheticPrint = (
size: number,
exchange: string,
offExchangeFlag: boolean
): EquityPrint => {
return {
source_ts: now,
ingest_ts: now,
seq,
trace_id: `synthetic-equities-${seq}`,
ts: now,
underlying_id: symbol,
price,
size,
exchange,
offExchangeFlag
};
};
): EquityPrint => ({
source_ts: now,
ingest_ts: now,
seq,
trace_id: `synthetic-equities-${seq}`,
ts: now,
underlying_id: symbol,
price,
size,
exchange,
offExchangeFlag
});
const buildSyntheticQuote = (
seq: number,
@ -67,32 +48,18 @@ const buildSyntheticQuote = (
symbol: string,
bid: number,
ask: number
): EquityQuote => {
return {
source_ts: now,
ingest_ts: now,
seq,
trace_id: `synthetic-equity-quote-${seq}`,
ts: now,
underlying_id: symbol,
bid,
ask
};
};
): EquityQuote => ({
source_ts: now,
ingest_ts: now,
seq,
trace_id: `synthetic-equity-quote-${seq}`,
ts: now,
underlying_id: symbol,
bid,
ask
});
const formatPrice = (value: number): number => {
return Number(value.toFixed(2));
};
const buildQuoteFromMid = (mid: number) => {
const spread = Math.max(0.05, Number((mid * 0.002).toFixed(2)));
const half = spread / 2;
const bid = formatPrice(Math.max(0.01, mid - half));
const ask = formatPrice(Math.max(bid + 0.01, mid + half));
const epsilon = Math.max(0.01, spread * 0.05);
return { bid, ask, spread, epsilon };
};
const formatPrice = (value: number): number => Number(value.toFixed(2));
const priceForPlacement = (
mid: number,
@ -100,7 +67,6 @@ const priceForPlacement = (
placement: PricePlacement
): number => {
const { bid, ask, epsilon } = quote;
let price = mid;
switch (placement) {
case "AA":
@ -120,44 +86,83 @@ const priceForPlacement = (
price = mid;
break;
}
return formatPrice(Math.max(0.01, price));
};
const buildQuoteContext = (
symbol: string,
now: number,
control: SyntheticControlState
) => {
const session = getSyntheticSessionState(now, control);
const state = getSyntheticUnderlyingState(symbol, now, control, session);
return {
session,
state,
mid: state.mid,
bid: formatPrice(state.bid),
ask: formatPrice(state.ask),
spread: state.spread,
epsilon: Math.max(0.01, state.spread * 0.08)
};
};
const pickPrimaryPlacement = (
driftBps: number,
regime: ReturnType<typeof getSyntheticSessionState>["regime"],
seq: number
): PricePlacement => {
if (regime === "dealer_gamma") {
return seq % 4 === 0 ? "A" : seq % 3 === 0 ? "B" : "MID";
}
if (regime === "arb_calm" || regime === "mean_revert") {
return seq % 11 === 0 ? "A" : seq % 13 === 0 ? "B" : "MID";
}
if (regime === "event_ramp" || regime === "retail_chase") {
if (driftBps >= 0) {
return seq % 3 === 0 ? "AA" : "A";
}
return seq % 3 === 0 ? "BB" : "B";
}
if (driftBps >= 0) {
return seq % 5 === 0 ? "A" : "MID";
}
return seq % 5 === 0 ? "B" : "MID";
};
const pickDarkPlacement = (
driftBps: number,
regime: ReturnType<typeof getSyntheticSessionState>["regime"],
seq: number
): PricePlacement => {
if (regime === "dealer_gamma") {
return seq % 2 === 0 ? "A" : "B";
}
if (regime === "arb_calm" || regime === "mean_revert") {
return "MID";
}
if (regime === "event_ramp" || regime === "retail_chase") {
return driftBps >= 0 ? (seq % 2 === 0 ? "A" : "AA") : seq % 2 === 0 ? "B" : "BB";
}
return driftBps >= 0 ? "A" : "B";
};
export const createSyntheticEquitiesAdapter = (
config: SyntheticEquitiesAdapterConfig
): EquityIngestAdapter => {
const profile =
const throughput =
config.mode === "firehose"
? {
batchSize: 10,
darkEvery: true,
offExchangeMod: 2,
litSizeBase: 40,
litSizeRange: 1400
}
? { batchSize: 10, litSizeBase: 48, litSizeRange: 1800, darkSizeBase: 2800 }
: config.mode === "active"
? {
batchSize: 5,
darkEvery: true,
offExchangeMod: 4,
litSizeBase: 20,
litSizeRange: 900
}
: {
batchSize: 2,
darkEvery: false,
offExchangeMod: 8,
litSizeBase: 10,
litSizeRange: 300
};
? { batchSize: 5, litSizeBase: 22, litSizeRange: 980, darkSizeBase: 1800 }
: { batchSize: 2, litSizeBase: 12, litSizeRange: 340, darkSizeBase: 900 };
return {
name: "synthetic",
start: (handlers: EquityIngestHandlers) => {
let seq = 0;
let quoteSeq = 0;
let darkStep = 0;
let darkSymbolIndex = 0;
let symbolCursor = 0;
let timer: ReturnType<typeof setInterval> | null = null;
let stopped = false;
@ -167,84 +172,113 @@ export const createSyntheticEquitiesAdapter = (
}
const now = Date.now();
const batchSize = profile.batchSize;
const control = config.getControl();
const session = getSyntheticSessionState(now, control);
const focusSymbols =
session.focus_symbols.length > 0 ? session.focus_symbols : SYNTHETIC_SYMBOLS.slice(0, 3);
const focusSet = new Set(focusSymbols);
const allowDark =
config.mode !== "realistic" ||
session.regime === "event_ramp" ||
session.regime === "dealer_gamma" ||
session.regime === "retail_chase";
const darkSymbol = SYNTHETIC_SYMBOLS[darkSymbolIndex % SYNTHETIC_SYMBOLS.length];
const darkHash = hashSymbol(darkSymbol);
const darkBase = 25 + (darkHash % 475);
const darkDrift = ((darkStep % 24) - 12) * 0.08;
const darkMid = formatPrice(darkBase + darkDrift);
const darkQuote = buildQuoteFromMid(darkMid);
const scenario = DARK_SEQUENCE[darkStep % DARK_SEQUENCE.length];
const darkTs = now;
if (profile.darkEvery) {
if (handlers.onQuote) {
quoteSeq += 1;
const quoteEvent = buildSyntheticQuote(
quoteSeq,
darkTs - 2,
darkSymbol,
darkQuote.bid,
darkQuote.ask
);
void handlers.onQuote(quoteEvent);
}
seq += 1;
let darkPlacement: PricePlacement = "MID";
let darkSize = config.mode === "firehose" ? 4000 : 2600;
if (scenario === "buy") {
darkPlacement = darkStep % 2 === 0 ? "A" : "AA";
darkSize = config.mode === "firehose" ? 1500 : 800;
} else if (scenario === "sell") {
darkPlacement = darkStep % 2 === 0 ? "B" : "BB";
darkSize = config.mode === "firehose" ? 1500 : 800;
}
const darkPrice = priceForPlacement(darkMid, darkQuote, darkPlacement);
const darkPrint = buildSyntheticPrint(
seq,
darkTs,
darkSymbol,
darkPrice,
darkSize,
DARK_EXCHANGE,
true
if (allowDark) {
const darkSymbol = focusSymbols[seq % focusSymbols.length] ?? SYNTHETIC_SYMBOLS[symbolCursor % SYNTHETIC_SYMBOLS.length]!;
const darkQuote = buildQuoteContext(darkSymbol, now, control);
const darkPlacement = pickDarkPlacement(
darkQuote.state.driftBps,
session.regime,
seq + 1
);
const darkBias = darkQuote.state.offExchangeBias;
const darkSize = Math.max(
250,
Math.round(
throughput.darkSizeBase *
(0.65 + darkBias * 0.9 + darkQuote.state.sessionVolatility * 0.2)
)
);
void handlers.onTrade(darkPrint);
darkStep += 1;
if (darkStep >= DARK_SEQUENCE.length) {
darkStep = 0;
darkSymbolIndex += 1;
}
}
for (let i = 0; i < batchSize; i += 1) {
seq += 1;
const symbol = SYNTHETIC_SYMBOLS[(seq + i) % SYNTHETIC_SYMBOLS.length];
const symbolHash = hashSymbol(symbol);
const basePrice = 25 + (symbolHash % 475);
const mid = formatPrice(basePrice + ((seq % 40) - 20) * 0.05);
const quote = buildQuoteFromMid(mid);
const placement: PricePlacement =
seq % 11 === 0 ? "A" : seq % 13 === 0 ? "B" : "MID";
const price = priceForPlacement(mid, quote, placement);
const size = profile.litSizeBase + (seq % profile.litSizeRange);
const exchange = EXCHANGES[(seq + symbolHash) % EXCHANGES.length];
const offExchangeFlag = (seq + i) % profile.offExchangeMod === 0;
const eventTs = now + i * 4;
if (handlers.onQuote) {
quoteSeq += 1;
const quoteEventTs = eventTs - 2;
const quoteEvent = buildSyntheticQuote(quoteSeq, quoteEventTs, symbol, quote.bid, quote.ask);
void handlers.onQuote(quoteEvent);
void handlers.onQuote(
buildSyntheticQuote(
quoteSeq,
now - 2,
darkSymbol,
darkQuote.bid,
darkQuote.ask
)
);
}
const print = buildSyntheticPrint(seq, eventTs, symbol, price, size, exchange, offExchangeFlag);
void handlers.onTrade(print);
seq += 1;
void handlers.onTrade(
buildSyntheticPrint(
seq,
now,
darkSymbol,
priceForPlacement(darkQuote.mid, darkQuote, darkPlacement),
darkSize,
DARK_EXCHANGE,
true
)
);
}
for (let i = 0; i < throughput.batchSize; i += 1) {
seq += 1;
const symbol =
i < focusSymbols.length
? focusSymbols[i]!
: SYNTHETIC_SYMBOLS[(symbolCursor + i) % SYNTHETIC_SYMBOLS.length]!;
const eventTs = now + i * 4;
const quote = buildQuoteContext(symbol, eventTs, control);
const clustered = focusSet.has(symbol);
const placement = pickPrimaryPlacement(
quote.state.driftBps,
session.regime,
seq + i
);
const exchange = EXCHANGES[(seq + symbol.charCodeAt(0) + i) % EXCHANGES.length]!;
const baseSize =
throughput.litSizeBase +
((seq + i) % throughput.litSizeRange) +
Math.round(quote.state.sessionVolatility * 140);
const size = clustered
? Math.round(baseSize * (1 + quote.state.clusteringScore * 0.35))
: baseSize;
const offExchangeFlag =
((seq + i * 3) % 10) / 10 < quote.state.offExchangeBias * (clustered ? 1.12 : 0.86);
if (handlers.onQuote) {
quoteSeq += 1;
void handlers.onQuote(
buildSyntheticQuote(
quoteSeq,
eventTs - 2,
symbol,
quote.bid,
quote.ask
)
);
}
void handlers.onTrade(
buildSyntheticPrint(
seq,
eventTs,
symbol,
priceForPlacement(quote.mid, quote, placement),
size,
exchange,
offExchangeFlag
)
);
}
symbolCursor = (symbolCursor + throughput.batchSize) % SYNTHETIC_SYMBOLS.length;
};
timer = setInterval(emit, config.emitIntervalMs);

View file

@ -6,7 +6,10 @@ import {
STREAM_EQUITY_PRINTS,
STREAM_EQUITY_QUOTES,
connectJetStreamWithRetry,
ensureSyntheticControlState,
ensureKnownStreams,
openSyntheticControlKv,
watchSyntheticControlState,
publishJson
} from "@islandflow/bus";
import {
@ -19,9 +22,11 @@ import {
import {
EquityPrintSchema,
EquityQuoteSchema,
DEFAULT_SYNTHETIC_CONTROL_STATE,
resolveSyntheticMarketModes,
type EquityPrint,
type EquityQuote
type EquityQuote,
type SyntheticControlState
} from "@islandflow/types";
import { createAlpacaEquitiesAdapter } from "./adapters/alpaca";
import { createSyntheticEquitiesAdapter } from "./adapters/synthetic";
@ -157,11 +162,15 @@ const parseSymbolList = (value: string): string[] => {
.filter(Boolean);
};
const selectAdapter = (name: string): EquityIngestAdapter => {
const selectAdapter = (
name: string,
getSyntheticControl: () => SyntheticControlState
): EquityIngestAdapter => {
if (name === "synthetic") {
return createSyntheticEquitiesAdapter({
emitIntervalMs: env.EMIT_INTERVAL_MS,
mode: syntheticModes.equities
mode: syntheticModes.equities,
getControl: getSyntheticControl
});
}
@ -196,6 +205,24 @@ const run = async () => {
await ensureKnownStreams(jsm, [STREAM_EQUITY_PRINTS, STREAM_EQUITY_QUOTES], { logger });
let syntheticControl = DEFAULT_SYNTHETIC_CONTROL_STATE;
let stopSyntheticControlWatch = async () => {};
if (env.EQUITIES_INGEST_ADAPTER === "synthetic") {
const syntheticControlKv = await openSyntheticControlKv(js);
syntheticControl = await ensureSyntheticControlState(syntheticControlKv);
stopSyntheticControlWatch = await watchSyntheticControlState(
syntheticControlKv,
(nextControl) => {
syntheticControl = nextControl;
},
(error) => {
logger.warn("synthetic control watch failed", {
error: getErrorMessage(error)
});
}
);
}
const clickhouse = createClickHouseClient({
url: env.CLICKHOUSE_URL,
database: env.CLICKHOUSE_DATABASE
@ -206,7 +233,10 @@ const run = async () => {
await ensureEquityQuotesTable(clickhouse);
});
const adapter = selectAdapter(env.EQUITIES_INGEST_ADAPTER);
const adapter = selectAdapter(
env.EQUITIES_INGEST_ADAPTER,
() => syntheticControl
);
logger.info("ingest adapter selected", { adapter: adapter.name });
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
const allowQuotePublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
@ -274,6 +304,7 @@ const run = async () => {
state.shuttingDown = true;
state.shutdownPromise = (async () => {
logger.info("service stopping", { signal });
await stopSyntheticControlWatch();
await stopAdapter();
try {

File diff suppressed because it is too large Load diff

View file

@ -11,9 +11,12 @@ import {
STREAM_OPTION_SIGNAL_PRINTS,
buildDurableConsumer,
connectJetStreamWithRetry,
ensureSyntheticControlState,
ensureKnownStreams,
openSyntheticControlKv,
publishJson,
subscribeJson
subscribeJson,
watchSyntheticControlState
} from "@islandflow/bus";
import {
createClickHouseClient,
@ -26,12 +29,14 @@ import {
OptionNBBOSchema,
OptionPrintSchema,
EquityQuoteSchema,
DEFAULT_SYNTHETIC_CONTROL_STATE,
deriveOptionPrintMetadata,
resolveSyntheticMarketModes,
type EquityQuote,
type OptionNBBO,
type OptionPrint,
type OptionsSignalConfig
type OptionsSignalConfig,
type SyntheticControlState
} from "@islandflow/types";
import { createAlpacaOptionsAdapter } from "./adapters/alpaca";
import { createDatabentoOptionsAdapter } from "./adapters/databento";
@ -259,11 +264,15 @@ const retry = async <T>(
throw lastError ?? new Error(`${label} failed after retries`);
};
const selectAdapter = (name: string): OptionIngestAdapter => {
const selectAdapter = (
name: string,
getSyntheticControl: () => SyntheticControlState
): OptionIngestAdapter => {
if (name === "synthetic") {
return createSyntheticOptionsAdapter({
emitIntervalMs: env.EMIT_INTERVAL_MS,
mode: syntheticModes.options
mode: syntheticModes.options,
getControl: getSyntheticControl
});
}
@ -351,6 +360,24 @@ const run = async () => {
{ logger }
);
let syntheticControl = DEFAULT_SYNTHETIC_CONTROL_STATE;
let stopSyntheticControlWatch = async () => {};
if (env.OPTIONS_INGEST_ADAPTER === "synthetic") {
const syntheticControlKv = await openSyntheticControlKv(js);
syntheticControl = await ensureSyntheticControlState(syntheticControlKv);
stopSyntheticControlWatch = await watchSyntheticControlState(
syntheticControlKv,
(nextControl) => {
syntheticControl = nextControl;
},
(error) => {
logger.warn("synthetic control watch failed", {
error: getErrorMessage(error)
});
}
);
}
const clickhouse = createClickHouseClient({
url: env.CLICKHOUSE_URL,
database: env.CLICKHOUSE_DATABASE
@ -361,7 +388,10 @@ const run = async () => {
await ensureOptionNBBOTable(clickhouse);
});
const adapter = selectAdapter(env.OPTIONS_INGEST_ADAPTER);
const adapter = selectAdapter(
env.OPTIONS_INGEST_ADAPTER,
() => syntheticControl
);
logger.info("ingest adapter selected", { adapter: adapter.name });
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
const allowNbboPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
@ -482,6 +512,7 @@ const run = async () => {
state.shutdownPromise = (async () => {
logger.info("service stopping", { signal });
clearInterval(pruneTimer);
await stopSyntheticControlWatch();
await stopAdapter();
try {

View file

@ -10,26 +10,43 @@ import {
} from "../src/adapters/synthetic";
const totalBurstNotional = (burst: {
basePrice: number;
baseSize: number;
printCount: number;
}): number => burst.basePrice * burst.baseSize * burst.printCount * 100;
legs: Array<{
basePrice: number;
baseSize: number;
}>;
cycles: number;
}): number =>
burst.legs.reduce((sum, leg) => sum + leg.basePrice * leg.baseSize * burst.cycles * 100, 0);
const findBurst = (
mode: "realistic" | "active",
scenarioId: string,
now = Date.UTC(2026, 0, 2)
) => {
for (let i = 1; i <= 360; i += 1) {
const burst = buildSyntheticBurstForTest(i, now + i * 1_000, mode);
if (burst.scenarioId === scenarioId) {
return burst;
}
}
throw new Error(`Unable to find synthetic scenario ${scenarioId} in mode ${mode}`);
};
describe("synthetic options burst sizing", () => {
it("keeps realistic-mode ask lifts inside the configured notional band", () => {
const burst = buildSyntheticBurstForTest(2, Date.UTC(2026, 0, 2), "realistic");
it("keeps realistic-mode ask-lift accumulation inside the configured notional band", () => {
const burst = findBurst("realistic", "ask_lift_accumulation");
expect(burst.scenarioId).toBe("ask_lift");
expect(totalBurstNotional(burst)).toBeGreaterThanOrEqual(9_000);
expect(totalBurstNotional(burst)).toBeLessThanOrEqual(35_000);
expect(burst.scenarioId).toBe("ask_lift_accumulation");
expect(totalBurstNotional(burst)).toBeGreaterThanOrEqual(12_000);
expect(totalBurstNotional(burst)).toBeLessThanOrEqual(90_000);
});
it("keeps active-mode sweeps inside the configured notional band", () => {
const burst = buildSyntheticBurstForTest(1, Date.UTC(2026, 0, 2), "active");
it("keeps active-mode call sweeps inside the configured notional band", () => {
const burst = findBurst("active", "call_sweep");
expect(burst.scenarioId).toBe("bearish_sweep");
expect(totalBurstNotional(burst)).toBeGreaterThanOrEqual(120_000);
expect(totalBurstNotional(burst)).toBeLessThanOrEqual(240_000);
expect(burst.scenarioId).toBe("call_sweep");
expect(totalBurstNotional(burst)).toBeGreaterThanOrEqual(70_000);
expect(totalBurstNotional(burst)).toBeLessThanOrEqual(420_000);
});
});
@ -114,7 +131,7 @@ describe("synthetic smart-money scenarios", () => {
it("scores each labeled scenario as its intended primary profile", () => {
const now = Date.parse("2026-01-02T15:00:00Z");
const scenarios = listSyntheticSmartMoneyScenariosForTest().filter(
(scenario) => scenario.hiddenLabel !== "neutral_noise"
(scenario) => scenario.label !== "neutral_noise"
);
for (const scenario of scenarios) {
@ -122,17 +139,62 @@ describe("synthetic smart-money scenarios", () => {
const event = buildSmartMoneyEventFromPacket(packet);
const winningScore = event.profile_scores[0];
const nearbyWrongScores = event.profile_scores.filter(
(score) => score.profile_id !== hiddenLabel && score.probability >= 0.5
(score) => score.profile_id !== scenario.label && score.probability >= 0.5
);
expect(event.abstained, scenario.id).toBe(false);
expect(event.primary_profile_id, scenario.id).toBe(hiddenLabel);
expect(winningScore?.profile_id, scenario.id).toBe(hiddenLabel);
expect(event.primary_profile_id, scenario.id).toBe(scenario.label);
expect(winningScore?.profile_id, scenario.id).toBe(scenario.label);
expect(winningScore?.probability ?? 0, scenario.id).toBeGreaterThanOrEqual(0.5);
expect(hiddenLabel.length, scenario.id).toBeGreaterThan(0);
expect(nearbyWrongScores, scenario.id).toEqual([]);
}
});
it("covers every smart-money label in active runtime mode over a deterministic sample", () => {
const seen = new Set<string>();
const now = Date.parse("2026-01-02T15:00:00Z");
for (let i = 1; i <= 120; i += 1) {
const burst = buildSyntheticBurstForTest(i, now + i * 1_000, "active");
seen.add(burst.label);
}
expect(seen).toEqual(
new Set([
"institutional_directional",
"retail_whale",
"event_driven",
"vol_seller",
"arbitrage",
"hedge_reactive",
"neutral_noise"
])
);
});
it("covers every smart-money label in realistic mode within a default twenty-minute window", () => {
const seen = new Set<string>();
const now = Date.parse("2026-01-02T15:00:00Z");
for (let i = 1; i <= 120; i += 1) {
const burst = buildSyntheticBurstForTest(i, now + i * 10_000, "realistic");
seen.add(burst.label);
}
expect(seen).toEqual(
new Set([
"institutional_directional",
"retail_whale",
"event_driven",
"vol_seller",
"arbitrage",
"hedge_reactive",
"neutral_noise"
])
);
});
it("keeps neutral background noise below the emission threshold", () => {
const { packet } = buildSyntheticFlowPacketForTest(
"neutral_noise",