diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0daa08d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.git +.github +.DS_Store +.bun +.tmp +node_modules +dist +coverage +logs +apps/web/.next +.env +.env.* +session-ses_*.md +token-usage-output.txt +!.env.example +!**/.env.example diff --git a/deployment/docker/.env.example b/deployment/docker/.env.example new file mode 100644 index 0000000..bbaf268 --- /dev/null +++ b/deployment/docker/.env.example @@ -0,0 +1,106 @@ +NATS_URL=nats://nats:4222 +CLICKHOUSE_URL=http://clickhouse:8123 +CLICKHOUSE_DATABASE=default +REDIS_URL=redis://redis:6379 + +API_PORT=4000 +REST_DEFAULT_LIMIT=200 + +# Recommended with NPM on the same Docker network: +# app. -> web:3000 +# api. -> api:4000 +NEXT_PUBLIC_API_URL=https://api.example.com +NEXT_PUBLIC_NBBO_MAX_AGE_MS=1000 + +# Options ingest +OPTIONS_INGEST_ADAPTER=synthetic +ALPACA_KEY_ID= +ALPACA_SECRET_KEY= +ALPACA_REST_URL=https://data.alpaca.markets +ALPACA_WS_BASE_URL=wss://stream.data.alpaca.markets/v1beta1 +ALPACA_FEED=indicative +ALPACA_UNDERLYINGS=SPY,NVDA,AAPL +ALPACA_STRIKES_PER_SIDE=8 +ALPACA_MAX_DTE_DAYS=30 +ALPACA_MONEYNESS_PCT=0.06 +ALPACA_MONEYNESS_FALLBACK_PCT=0.1 +ALPACA_MAX_QUOTES=200 + +# Databento replay +DATABENTO_API_KEY= +DATABENTO_DATASET=OPRA.PILLAR +DATABENTO_SCHEMA=trades +DATABENTO_NBBO_SCHEMA=tbbo +DATABENTO_START= +DATABENTO_END= +DATABENTO_SYMBOLS=ALL +DATABENTO_STYPE_IN=raw_symbol +DATABENTO_STYPE_OUT=raw_symbol +DATABENTO_LIMIT=0 +DATABENTO_PRICE_SCALE=1 +DATABENTO_PYTHON_BIN=python3 + +# IBKR adapter (options) +IBKR_HOST=host.docker.internal +IBKR_PORT=7497 +IBKR_CLIENT_ID=0 +IBKR_SYMBOL=SPY +IBKR_EXPIRY=20250117 +IBKR_STRIKE=450 +IBKR_RIGHT=C +IBKR_EXCHANGE=SMART +IBKR_CURRENCY=USD +IBKR_PYTHON_BIN=python3 + +# Equities ingest +EQUITIES_INGEST_ADAPTER=synthetic +EMIT_INTERVAL_MS=1000 +ALPACA_EQUITIES_FEED=iex + +# Testing mode +TESTING_MODE=false +TESTING_THROTTLE_MS=200 + +# Compute and inference +COMPUTE_DELIVER_POLICY=new +COMPUTE_CONSUMER_RESET=false +NBBO_MAX_AGE_MS=1000 +ROLLING_WINDOW_SIZE=50 +ROLLING_TTL_SEC=86400 +EQUITY_QUOTE_MAX_AGE_MS=1000 +DARK_INFER_WINDOW_MS=60000 +DARK_INFER_COOLDOWN_MS=30000 +DARK_INFER_MIN_BLOCK_SIZE=2000 +DARK_INFER_MIN_ACCUM_SIZE=3000 +DARK_INFER_MIN_ACCUM_COUNT=4 +DARK_INFER_MIN_PRINT_SIZE=200 +DARK_INFER_MAX_EVIDENCE=20 +DARK_INFER_MAX_SPREAD_PCT=0.005 +CLASSIFIER_SWEEP_MIN_PREMIUM=40000 +CLASSIFIER_SWEEP_MIN_COUNT=3 +CLASSIFIER_SWEEP_MIN_PREMIUM_Z=2 +CLASSIFIER_SPIKE_MIN_PREMIUM=20000 +CLASSIFIER_SPIKE_MIN_SIZE=400 +CLASSIFIER_SPIKE_MIN_PREMIUM_Z=2.5 +CLASSIFIER_SPIKE_MIN_SIZE_Z=2 +CLASSIFIER_Z_MIN_SAMPLES=12 +CLASSIFIER_MIN_NBBO_COVERAGE=0.5 +CLASSIFIER_MIN_AGGRESSOR_RATIO=0.55 +CLASSIFIER_0DTE_MAX_ATM_PCT=0.01 +CLASSIFIER_0DTE_MIN_PREMIUM=20000 +CLASSIFIER_0DTE_MIN_SIZE=400 + +# Candles +CANDLE_INTERVALS_MS=60000,300000 +CANDLE_MAX_LATE_MS=0 +CANDLE_CACHE_LIMIT=2000 +CANDLE_DELIVER_POLICY=new +CANDLE_CONSUMER_RESET=false + +# Replay profile +REPLAY_STREAMS=options,nbbo,equities,equity-quotes +REPLAY_START_TS=0 +REPLAY_END_TS=0 +REPLAY_SPEED=1 +REPLAY_BATCH_SIZE=200 +REPLAY_LOG_EVERY=1000 diff --git a/deployment/docker/Dockerfile.ingest-options b/deployment/docker/Dockerfile.ingest-options new file mode 100644 index 0000000..a7efdd2 --- /dev/null +++ b/deployment/docker/Dockerfile.ingest-options @@ -0,0 +1,15 @@ +FROM oven/bun:1.3.11 + +WORKDIR /app + +ENV NODE_ENV=production + +COPY . . + +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-pip \ + && rm -rf /var/lib/apt/lists/* \ + && pip3 install --no-cache-dir -r services/ingest-options/py/requirements.txt \ + && bun install --frozen-lockfile + +ENTRYPOINT ["bun"] diff --git a/deployment/docker/Dockerfile.service b/deployment/docker/Dockerfile.service new file mode 100644 index 0000000..4c32bbe --- /dev/null +++ b/deployment/docker/Dockerfile.service @@ -0,0 +1,11 @@ +FROM oven/bun:1.3.11 + +WORKDIR /app + +ENV NODE_ENV=production + +COPY . . + +RUN bun install --frozen-lockfile + +ENTRYPOINT ["bun"] diff --git a/deployment/docker/Dockerfile.web b/deployment/docker/Dockerfile.web new file mode 100644 index 0000000..038e0a4 --- /dev/null +++ b/deployment/docker/Dockerfile.web @@ -0,0 +1,34 @@ +FROM oven/bun:1.3.11 AS build + +WORKDIR /app + +ARG NEXT_PUBLIC_API_URL="" +ARG NEXT_PUBLIC_NBBO_MAX_AGE_MS=1000 + +ENV NODE_ENV=production +ENV NEXT_TELEMETRY_DISABLED=1 +ENV NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL} +ENV NEXT_PUBLIC_NBBO_MAX_AGE_MS=${NEXT_PUBLIC_NBBO_MAX_AGE_MS} + +COPY . . + +RUN bun install --frozen-lockfile +RUN bun run --cwd apps/web build + +FROM oven/bun:1.3.11 AS runtime + +WORKDIR /app + +ENV NODE_ENV=production +ENV NEXT_TELEMETRY_DISABLED=1 + +COPY --from=build /app/package.json ./package.json +COPY --from=build /app/bun.lock ./bun.lock +COPY --from=build /app/tsconfig.base.json ./tsconfig.base.json +COPY --from=build /app/node_modules ./node_modules +COPY --from=build /app/apps/web ./apps/web +COPY --from=build /app/packages ./packages + +EXPOSE 3000 + +CMD ["bun", "run", "--cwd", "apps/web", "start"] diff --git a/deployment/docker/README.md b/deployment/docker/README.md new file mode 100644 index 0000000..33066bc --- /dev/null +++ b/deployment/docker/README.md @@ -0,0 +1,216 @@ +# Docker Deployment + +This directory contains a VPS-oriented Docker deployment for the full Islandflow stack. + +It is separate from the repo-root `docker-compose.yml`, which is still the lightweight local infra stack for development. + +## What this stack does + +- Assumes Nginx Proxy Manager is the edge proxy and runs on the same Docker network. +- Keeps `web` and `api` internal to the Docker network instead of publishing host ports. +- Targets a two-subdomain routing model by default: + - `app.` -> `web:3000` + - `api.` -> `api:4000` +- Runs ClickHouse, Redis, and NATS JetStream with persistent Docker volumes. +- Runs the core runtime services: `ingest-options`, `ingest-equities`, `compute`, `candles`, `api`, and `web`. +- Keeps `replay` opt-in through a Compose profile, because the current replay service starts immediately when the container is enabled. + +## Files + +- `deployment/docker/docker-compose.yml`: production-style stack for a single VPS +- `deployment/docker/Dockerfile.service`: shared Bun runtime image for most services +- `deployment/docker/Dockerfile.ingest-options`: Bun runtime plus Python dependencies for Databento and IBKR adapters +- `deployment/docker/Dockerfile.web`: multi-stage build for the Next.js web app +- `deployment/docker/.env.example`: container-oriented environment template + +## Prerequisites + +- A Linux VPS with Docker Engine and Docker Compose v2 installed +- Enough RAM for ClickHouse plus the Bun services +- Nginx Proxy Manager running in Docker on the same host/network path you plan to use + +Optional: + +- A DNS record pointed at the VPS +- Alpaca, Databento, or IBKR credentials if you are not using the synthetic adapters + +## First deployment + +1. Copy the env template: + +```bash +cd deployment/docker +cp .env.example .env +``` + +2. Edit `.env`. + +Important defaults: + +- `NATS_URL`, `CLICKHOUSE_URL`, and `REDIS_URL` should stay on the internal container hostnames unless you intentionally split infra out. +- `OPTIONS_INGEST_ADAPTER=synthetic` and `EQUITIES_INGEST_ADAPTER=synthetic` are the safest first boot settings. +- `NEXT_PUBLIC_API_URL=https://api.example.com` is the recommended production shape when using NPM with two subdomains. + +3. Build and start the stack: + +```bash +docker compose up -d --build +``` + +4. Confirm the containers are healthy: + +```bash +docker compose ps +docker compose logs -f api web compute candles ingest-options ingest-equities +``` + +5. Make sure NPM can reach the stack network. + +The Compose project name is pinned to `islandflow-vps`, so the default network name will be: + +```bash +islandflow-vps_default +``` + +If your NPM container is separate, connect it once: + +```bash +docker network connect islandflow-vps_default +``` + +6. Create these NPM proxy hosts: + +- `app.example.com` -> forward to `web`, port `3000` +- `api.example.com` -> forward to `api`, port `4000` + +For the API host, enable websocket support. + +7. Open the app: + +- `https://app.example.com/` +- Health check: `https://api.example.com/health` + +## Replay service + +Replay is disabled by default in this stack. + +Start it only when you want it: + +```bash +docker compose --profile replay up -d replay +``` + +Stop it again: + +```bash +docker compose stop replay +``` + +## Adapter notes + +### Synthetic mode + +This is the easiest way to smoke-test the deployment: + +- `OPTIONS_INGEST_ADAPTER=synthetic` +- `EQUITIES_INGEST_ADAPTER=synthetic` + +### Alpaca mode + +Set the adapter values and credentials in `.env`: + +- `OPTIONS_INGEST_ADAPTER=alpaca` +- `EQUITIES_INGEST_ADAPTER=alpaca` +- `ALPACA_KEY_ID=...` +- `ALPACA_SECRET_KEY=...` + +### Databento mode + +The `ingest-options` image in this deployment includes Python plus the repo’s sidecar dependencies, so Databento can run without a custom image. Set the Databento env vars in `.env`, especially: + +- `OPTIONS_INGEST_ADAPTER=databento` +- `DATABENTO_API_KEY=...` +- `DATABENTO_START=...` + +### IBKR mode + +If TWS or IB Gateway is running on the VPS host, the default `.env.example` already points `IBKR_HOST` at `host.docker.internal`, and the Compose stack adds the required host gateway mapping. + +If IBKR is running somewhere else, change: + +- `IBKR_HOST` +- `IBKR_PORT` + +## NPM routing + +Recommended proxy hosts: + +- `app.` -> `web:3000` +- `api.` -> `api:4000` + +The web app should be built with `NEXT_PUBLIC_API_URL=https://api.` so browser REST and websocket traffic goes straight to the API host through NPM. + +The API host needs websocket support enabled because the app uses `/ws/*` endpoints for live streams. + +## Updating the deployment + +When you pull new code: + +```bash +cd deployment/docker +docker compose up -d --build +``` + +If you changed only env values for the Bun services: + +```bash +docker compose up -d +``` + +If you changed `NEXT_PUBLIC_API_URL` or `NEXT_PUBLIC_NBBO_MAX_AGE_MS`, rebuild the web image because those are public Next.js build-time values: + +```bash +docker compose build web +docker compose up -d web +``` + +## Backups and persistence + +Persistent data lives in Docker volumes: + +- `clickhouse-data` +- `redis-data` +- `nats-data` + +Before destructive maintenance, back up those volumes or the underlying Docker data directory for the host. + +## Shutdown + +Stop everything while keeping data: + +```bash +docker compose down +``` + +Stop everything and remove volumes too: + +```bash +docker compose down -v +``` + +Only use `-v` if you intentionally want to wipe ClickHouse, Redis, and JetStream state. + +## Known caveats + +- The root `.env.example` still contains a `REPLAY_ENABLED` comment, but the current replay service does not read that variable. Use the Compose replay profile instead. +- This stack does not publish `web` or `api` to host ports. NPM must be able to resolve `web` and `api` over the shared Docker network. +- The stack assumes a single-node VPS deployment. If you later split infra or add external managed services, update the three core connection URLs in `.env`. + +## Smoke checks + +After NPM is wired up: + +- `https://app./` should load the UI. +- Browser network requests from the UI should target `https://api./...`. +- Live feeds should connect over `wss://api./ws/...`. +- `docker compose ps` should show no service publishing host port `80`. diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml new file mode 100644 index 0000000..7849c15 --- /dev/null +++ b/deployment/docker/docker-compose.yml @@ -0,0 +1,126 @@ +name: islandflow-vps + +x-service-common: &service-common + build: + context: ../.. + dockerfile: deployment/docker/Dockerfile.service + env_file: + - ./.env + restart: unless-stopped + init: true + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + - nats + - clickhouse + - redis + +services: + web: + build: + context: ../.. + dockerfile: deployment/docker/Dockerfile.web + args: + NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-} + NEXT_PUBLIC_NBBO_MAX_AGE_MS: ${NEXT_PUBLIC_NBBO_MAX_AGE_MS:-1000} + env_file: + - ./.env + restart: unless-stopped + init: true + expose: + - "3000" + depends_on: + api: + condition: service_healthy + healthcheck: + test: + [ + "CMD", + "bun", + "-e", + "const r=await fetch('http://127.0.0.1:3000/'); if(!r.ok) throw new Error('web healthcheck failed: '+r.status);" + ] + interval: 30s + timeout: 10s + retries: 5 + start_period: 45s + + api: + <<: *service-common + command: ["services/api/src/index.ts"] + expose: + - "4000" + healthcheck: + test: + [ + "CMD", + "bun", + "-e", + "const r=await fetch('http://127.0.0.1:4000/health'); if(!r.ok) throw new Error('api healthcheck failed: '+r.status);" + ] + interval: 30s + timeout: 10s + retries: 5 + start_period: 20s + + compute: + <<: *service-common + command: ["services/compute/src/index.ts"] + + candles: + <<: *service-common + command: ["services/candles/src/index.ts"] + + ingest-options: + build: + context: ../.. + dockerfile: deployment/docker/Dockerfile.ingest-options + env_file: + - ./.env + restart: unless-stopped + init: true + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + - nats + - clickhouse + - redis + command: ["services/ingest-options/src/index.ts"] + + ingest-equities: + <<: *service-common + command: ["services/ingest-equities/src/index.ts"] + + replay: + <<: *service-common + profiles: ["replay"] + command: ["services/replay/src/index.ts"] + + clickhouse: + image: clickhouse/clickhouse-server:23.8 + restart: unless-stopped + ulimits: + nofile: + soft: 262144 + hard: 262144 + volumes: + - clickhouse-data:/var/lib/clickhouse + + redis: + image: redis:7.2 + restart: unless-stopped + command: ["redis-server", "--appendonly", "yes"] + volumes: + - redis-data:/data + + nats: + image: nats:2.10 + restart: unless-stopped + command: ["-js", "-sd", "/data"] + volumes: + - nats-data:/data + +volumes: + clickhouse-data: + redis-data: + nats-data: