diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..0e77d57 --- /dev/null +++ b/.env.example @@ -0,0 +1,45 @@ +NATS_URL=nats://localhost:4222 +CLICKHOUSE_URL=http://localhost:8123 +CLICKHOUSE_DATABASE=default + +# Options ingest +INGEST_ADAPTER=alpaca +ALPACA_KEY_ID= +ALPACA_SECRET_KEY= +ALPACA_REST_URL=https://data.alpaca.markets +ALPACA_WS_BASE_URL=wss://stream.data.alpaca.markets/v1beta1 +ALPACA_FEED=indicative +ALPACA_UNDERLYINGS=SPY +ALPACA_STRIKES_PER_SIDE=8 +ALPACA_MAX_DTE_DAYS=30 +ALPACA_MONEYNESS_PCT=0.06 +ALPACA_MONEYNESS_FALLBACK_PCT=0.1 +ALPACA_MAX_QUOTES=200 + +# Databento replay +DATABENTO_API_KEY= +DATABENTO_DATASET=OPRA.PILLAR +DATABENTO_SCHEMA=trades +DATABENTO_START= +DATABENTO_END= +DATABENTO_SYMBOLS=SPY.OPT +DATABENTO_STYPE_IN=parent +DATABENTO_STYPE_OUT=instrument_id +DATABENTO_LIMIT=0 +DATABENTO_PRICE_SCALE=1 +DATABENTO_PYTHON_BIN=py/.venv/bin/python + +# IBKR adapter (options) +IBKR_HOST=127.0.0.1 +IBKR_PORT=7497 +IBKR_CLIENT_ID=0 +IBKR_SYMBOL=SPY +IBKR_EXPIRY=20250117 +IBKR_STRIKE=450 +IBKR_RIGHT=C +IBKR_EXCHANGE=SMART +IBKR_CURRENCY=USD +IBKR_PYTHON_BIN=python3 + +# Equities ingest +EMIT_INTERVAL_MS=1000 diff --git a/.gitignore b/.gitignore index 4ff1317..89ac89b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ node_modules/ dist/ .env .env.* +!.env.example coverage/ logs/ .tmp/ diff --git a/README.md b/README.md index 722a536..a558da6 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,12 @@ Done now (in repo): - Synthetic options/equity prints published to NATS and persisted to ClickHouse - Deterministic option FlowPacket clustering (time window) + persistence - API: REST for prints/flow packets, WS for live options/equities/flow, replay endpoints -- UI: live tapes for options/equities/flow + replay toggle + pause controls +- UI: live tapes for options/equities/flow + replay toggle + pause controls + replay time/completion +- Databento historical replay adapter (options) with symbol mapping +- Alpaca options adapter (dev-only, bounded contract list) In progress / blocked: -- Live data adapters (requires licensed data source) +- Live data adapters beyond dev-only feeds (requires licensed data source) - Rolling stats and advanced clustering Not started: @@ -84,6 +86,9 @@ Install dependencies: Start infra: - `docker compose up -d` +Create env file: +- Copy `.env.example` to `.env` and fill in the API keys you plan to use. + Start everything (infra + services + web): - `bun run dev` diff --git a/services/ingest-options/src/index.ts b/services/ingest-options/src/index.ts index 82cce12..c4b4be5 100644 --- a/services/ingest-options/src/index.ts +++ b/services/ingest-options/src/index.ts @@ -28,8 +28,8 @@ const envSchema = z.object({ CLICKHOUSE_URL: z.string().default("http://localhost:8123"), CLICKHOUSE_DATABASE: z.string().default("default"), INGEST_ADAPTER: z.string().min(1).default("alpaca"), - ALPACA_KEY_ID: z.string().default("PKQDUYKNHDYCPONSMWIXZHT6QV"), - ALPACA_SECRET_KEY: z.string().default("5ktmszfCiWg125GtPguuFpSeTB2zHNewScncAaY4hnKo"), + ALPACA_KEY_ID: z.string().default(""), + ALPACA_SECRET_KEY: z.string().default(""), ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"), ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets/v1beta1"), ALPACA_FEED: z.enum(["indicative", "opra"]).default("indicative"), @@ -105,6 +105,7 @@ const selectAdapter = (name: string): OptionIngestAdapter => { if (name === "alpaca") { if (!env.ALPACA_KEY_ID || !env.ALPACA_SECRET_KEY) { + logger.warn("alpaca credentials missing; set ALPACA_KEY_ID and ALPACA_SECRET_KEY"); throw new Error("ALPACA_KEY_ID and ALPACA_SECRET_KEY are required for the alpaca adapter."); } @@ -127,10 +128,12 @@ const selectAdapter = (name: string): OptionIngestAdapter => { if (name === "databento") { if (!env.DATABENTO_API_KEY) { + logger.warn("databento api key missing; set DATABENTO_API_KEY"); throw new Error("DATABENTO_API_KEY is required for the databento adapter."); } if (!env.DATABENTO_START) { + logger.warn("databento start missing; set DATABENTO_START"); throw new Error("DATABENTO_START is required for the databento adapter."); }