dirtydishes/islandflow
1
0
Fork 0
Code Issues 2 Pull requests 4 Projects Releases Packages Wiki Activity Actions 1

Add Databento historical replay adapter + symbol mapping; speed up replay UI + completion state

Browse source 
- add Databento options adapter (TS) with Python sidecar and env wiring
- add  to stream historical trades and resolve instrument_id -> raw_symbol via symbology
- include Databento + typing_extensions in ingest-options Python requirements
- expose Databento env settings in ingest-options index (dataset/schema/start/end/stype/limit/price scale/python bin)
- update README with Databento replay usage and env docs
- speed up UI replay polling/drain, add per-card replay time display
- stop replay at end and prevent fallback to synthetic by pinning replay to initial trace source
This commit is contained in:
dirtydishes 2025-12-28 21:30:24 -05:00
parent 6dc279099f
commit baaadcf105
6 changed files with 799 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

36
services/ingest-options/src/index.ts
View file

@ -14,6 +14,7 @@ import {
} from "@islandflow/storage";
import { OptionPrintSchema, type OptionPrint } from "@islandflow/types";
import { createAlpacaOptionsAdapter } from "./adapters/alpaca";
import { createDatabentoOptionsAdapter } from "./adapters/databento";
import { createIbkrOptionsAdapter } from "./adapters/ibkr";
import { createSyntheticOptionsAdapter } from "./adapters/synthetic";
import type { OptionIngestAdapter, StopHandler } from "./adapters/types";
@ -38,6 +39,17 @@ const envSchema = z.object({
ALPACA_MONEYNESS_PCT: z.coerce.number().positive().default(0.06),
ALPACA_MONEYNESS_FALLBACK_PCT: z.coerce.number().positive().default(0.1),
ALPACA_MAX_QUOTES: z.coerce.number().int().positive().default(200),
DATABENTO_API_KEY: z.string().default(""),
DATABENTO_DATASET: z.string().default("OPRA.PILLAR"),
DATABENTO_SCHEMA: z.string().default("trades"),
DATABENTO_START: z.string().default(""),
DATABENTO_END: z.string().default(""),
DATABENTO_SYMBOLS: z.string().default("ALL"),
DATABENTO_STYPE_IN: z.string().default("raw_symbol"),
DATABENTO_STYPE_OUT: z.string().default("raw_symbol"),
DATABENTO_LIMIT: z.coerce.number().int().nonnegative().default(0),
DATABENTO_PRICE_SCALE: z.coerce.number().positive().default(1),
DATABENTO_PYTHON_BIN: z.string().default("python3"),
IBKR_HOST: z.string().default("127.0.0.1"),
IBKR_PORT: z.coerce.number().int().positive().default(7497),
IBKR_CLIENT_ID: z.coerce.number().int().nonnegative().default(0),
@ -113,6 +125,30 @@ const selectAdapter = (name: string): OptionIngestAdapter => {
});
}
if (name === "databento") {
if (!env.DATABENTO_API_KEY) {
throw new Error("DATABENTO_API_KEY is required for the databento adapter.");
}
if (!env.DATABENTO_START) {
throw new Error("DATABENTO_START is required for the databento adapter.");
}
return createDatabentoOptionsAdapter({
apiKey: env.DATABENTO_API_KEY,
dataset: env.DATABENTO_DATASET,
schema: env.DATABENTO_SCHEMA,
start: env.DATABENTO_START,
end: env.DATABENTO_END || undefined,
symbols: env.DATABENTO_SYMBOLS,
stypeIn: env.DATABENTO_STYPE_IN,
stypeOut: env.DATABENTO_STYPE_OUT,
limit: env.DATABENTO_LIMIT,
priceScale: env.DATABENTO_PRICE_SCALE,
pythonBin: env.DATABENTO_PYTHON_BIN
});
}
if (name === "ibkr") {
return createIbkrOptionsAdapter({
host: env.IBKR_HOST,