Merge pull request #10 from dirtydishes/lavender/prepare-docker-deployment

add docker deployment assets for vps hosting

.dockerignore

@@ -0,0 +1,16 @@
+.git
+.github
+.DS_Store
+.bun
+.tmp
+node_modules
+dist
+coverage
+logs
+apps/web/.next
+.env
+.env.*
+session-ses_*.md
+token-usage-output.txt
+!.env.example
+!**/.env.example

deployment/docker/.env.example

@@ -0,0 +1,106 @@
+NATS_URL=nats://nats:4222
+CLICKHOUSE_URL=http://clickhouse:8123
+CLICKHOUSE_DATABASE=default
+REDIS_URL=redis://redis:6379
+
+API_PORT=4000
+REST_DEFAULT_LIMIT=200
+
+# Recommended with NPM on the same Docker network:
+# app.<domain> -> web:3000
+# api.<domain> -> api:4000
+NEXT_PUBLIC_API_URL=https://api.example.com
+NEXT_PUBLIC_NBBO_MAX_AGE_MS=1000
+
+# Options ingest
+OPTIONS_INGEST_ADAPTER=synthetic
+ALPACA_KEY_ID=
+ALPACA_SECRET_KEY=
+ALPACA_REST_URL=https://data.alpaca.markets
+ALPACA_WS_BASE_URL=wss://stream.data.alpaca.markets/v1beta1
+ALPACA_FEED=indicative
+ALPACA_UNDERLYINGS=SPY,NVDA,AAPL
+ALPACA_STRIKES_PER_SIDE=8
+ALPACA_MAX_DTE_DAYS=30
+ALPACA_MONEYNESS_PCT=0.06
+ALPACA_MONEYNESS_FALLBACK_PCT=0.1
+ALPACA_MAX_QUOTES=200
+
+# Databento replay
+DATABENTO_API_KEY=
+DATABENTO_DATASET=OPRA.PILLAR
+DATABENTO_SCHEMA=trades
+DATABENTO_NBBO_SCHEMA=tbbo
+DATABENTO_START=
+DATABENTO_END=
+DATABENTO_SYMBOLS=ALL
+DATABENTO_STYPE_IN=raw_symbol
+DATABENTO_STYPE_OUT=raw_symbol
+DATABENTO_LIMIT=0
+DATABENTO_PRICE_SCALE=1
+DATABENTO_PYTHON_BIN=python3
+
+# IBKR adapter (options)
+IBKR_HOST=host.docker.internal
+IBKR_PORT=7497
+IBKR_CLIENT_ID=0
+IBKR_SYMBOL=SPY
+IBKR_EXPIRY=20250117
+IBKR_STRIKE=450
+IBKR_RIGHT=C
+IBKR_EXCHANGE=SMART
+IBKR_CURRENCY=USD
+IBKR_PYTHON_BIN=python3
+
+# Equities ingest
+EQUITIES_INGEST_ADAPTER=synthetic
+EMIT_INTERVAL_MS=1000
+ALPACA_EQUITIES_FEED=iex
+
+# Testing mode
+TESTING_MODE=false
+TESTING_THROTTLE_MS=200
+
+# Compute and inference
+COMPUTE_DELIVER_POLICY=new
+COMPUTE_CONSUMER_RESET=false
+NBBO_MAX_AGE_MS=1000
+ROLLING_WINDOW_SIZE=50
+ROLLING_TTL_SEC=86400
+EQUITY_QUOTE_MAX_AGE_MS=1000
+DARK_INFER_WINDOW_MS=60000
+DARK_INFER_COOLDOWN_MS=30000
+DARK_INFER_MIN_BLOCK_SIZE=2000
+DARK_INFER_MIN_ACCUM_SIZE=3000
+DARK_INFER_MIN_ACCUM_COUNT=4
+DARK_INFER_MIN_PRINT_SIZE=200
+DARK_INFER_MAX_EVIDENCE=20
+DARK_INFER_MAX_SPREAD_PCT=0.005
+CLASSIFIER_SWEEP_MIN_PREMIUM=40000
+CLASSIFIER_SWEEP_MIN_COUNT=3
+CLASSIFIER_SWEEP_MIN_PREMIUM_Z=2
+CLASSIFIER_SPIKE_MIN_PREMIUM=20000
+CLASSIFIER_SPIKE_MIN_SIZE=400
+CLASSIFIER_SPIKE_MIN_PREMIUM_Z=2.5
+CLASSIFIER_SPIKE_MIN_SIZE_Z=2
+CLASSIFIER_Z_MIN_SAMPLES=12
+CLASSIFIER_MIN_NBBO_COVERAGE=0.5
+CLASSIFIER_MIN_AGGRESSOR_RATIO=0.55
+CLASSIFIER_0DTE_MAX_ATM_PCT=0.01
+CLASSIFIER_0DTE_MIN_PREMIUM=20000
+CLASSIFIER_0DTE_MIN_SIZE=400
+
+# Candles
+CANDLE_INTERVALS_MS=60000,300000
+CANDLE_MAX_LATE_MS=0
+CANDLE_CACHE_LIMIT=2000
+CANDLE_DELIVER_POLICY=new
+CANDLE_CONSUMER_RESET=false
+
+# Replay profile
+REPLAY_STREAMS=options,nbbo,equities,equity-quotes
+REPLAY_START_TS=0
+REPLAY_END_TS=0
+REPLAY_SPEED=1
+REPLAY_BATCH_SIZE=200
+REPLAY_LOG_EVERY=1000

deployment/docker/Dockerfile.ingest-options

@@ -0,0 +1,15 @@
+FROM oven/bun:1.3.11
+
+WORKDIR /app
+
+ENV NODE_ENV=production
+
+COPY . .
+
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends python3 python3-pip \
+  && rm -rf /var/lib/apt/lists/* \
+  && pip3 install --no-cache-dir -r services/ingest-options/py/requirements.txt \
+  && bun install --frozen-lockfile
+
+ENTRYPOINT ["bun"]

deployment/docker/Dockerfile.service

@@ -0,0 +1,11 @@
+FROM oven/bun:1.3.11
+
+WORKDIR /app
+
+ENV NODE_ENV=production
+
+COPY . .
+
+RUN bun install --frozen-lockfile
+
+ENTRYPOINT ["bun"]

deployment/docker/Dockerfile.web

@@ -0,0 +1,34 @@
+FROM oven/bun:1.3.11 AS build
+
+WORKDIR /app
+
+ARG NEXT_PUBLIC_API_URL=""
+ARG NEXT_PUBLIC_NBBO_MAX_AGE_MS=1000
+
+ENV NODE_ENV=production
+ENV NEXT_TELEMETRY_DISABLED=1
+ENV NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL}
+ENV NEXT_PUBLIC_NBBO_MAX_AGE_MS=${NEXT_PUBLIC_NBBO_MAX_AGE_MS}
+
+COPY . .
+
+RUN bun install --frozen-lockfile
+RUN bun run --cwd apps/web build
+
+FROM oven/bun:1.3.11 AS runtime
+
+WORKDIR /app
+
+ENV NODE_ENV=production
+ENV NEXT_TELEMETRY_DISABLED=1
+
+COPY --from=build /app/package.json ./package.json
+COPY --from=build /app/bun.lock ./bun.lock
+COPY --from=build /app/tsconfig.base.json ./tsconfig.base.json
+COPY --from=build /app/node_modules ./node_modules
+COPY --from=build /app/apps/web ./apps/web
+COPY --from=build /app/packages ./packages
+
+EXPOSE 3000
+
+CMD ["bun", "run", "--cwd", "apps/web", "start"]

deployment/docker/README.md

@@ -0,0 +1,216 @@
+# Docker Deployment
+
+This directory contains a VPS-oriented Docker deployment for the full Islandflow stack.
+
+It is separate from the repo-root `docker-compose.yml`, which is still the lightweight local infra stack for development.
+
+## What this stack does
+
+- Assumes Nginx Proxy Manager is the edge proxy and runs on the same Docker network.
+- Keeps `web` and `api` internal to the Docker network instead of publishing host ports.
+- Targets a two-subdomain routing model by default:
+  - `app.<domain>` -> `web:3000`
+  - `api.<domain>` -> `api:4000`
+- Runs ClickHouse, Redis, and NATS JetStream with persistent Docker volumes.
+- Runs the core runtime services: `ingest-options`, `ingest-equities`, `compute`, `candles`, `api`, and `web`.
+- Keeps `replay` opt-in through a Compose profile, because the current replay service starts immediately when the container is enabled.
+
+## Files
+
+- `deployment/docker/docker-compose.yml`: production-style stack for a single VPS
+- `deployment/docker/Dockerfile.service`: shared Bun runtime image for most services
+- `deployment/docker/Dockerfile.ingest-options`: Bun runtime plus Python dependencies for Databento and IBKR adapters
+- `deployment/docker/Dockerfile.web`: multi-stage build for the Next.js web app
+- `deployment/docker/.env.example`: container-oriented environment template
+
+## Prerequisites
+
+- A Linux VPS with Docker Engine and Docker Compose v2 installed
+- Enough RAM for ClickHouse plus the Bun services
+- Nginx Proxy Manager running in Docker on the same host/network path you plan to use
+
+Optional:
+
+- A DNS record pointed at the VPS
+- Alpaca, Databento, or IBKR credentials if you are not using the synthetic adapters
+
+## First deployment
+
+1. Copy the env template:
+
+```bash
+cd deployment/docker
+cp .env.example .env
+```
+
+2. Edit `.env`.
+
+Important defaults:
+
+- `NATS_URL`, `CLICKHOUSE_URL`, and `REDIS_URL` should stay on the internal container hostnames unless you intentionally split infra out.
+- `OPTIONS_INGEST_ADAPTER=synthetic` and `EQUITIES_INGEST_ADAPTER=synthetic` are the safest first boot settings.
+- `NEXT_PUBLIC_API_URL=https://api.example.com` is the recommended production shape when using NPM with two subdomains.
+
+3. Build and start the stack:
+
+```bash
+docker compose up -d --build
+```
+
+4. Confirm the containers are healthy:
+
+```bash
+docker compose ps
+docker compose logs -f api web compute candles ingest-options ingest-equities
+```
+
+5. Make sure NPM can reach the stack network.
+
+The Compose project name is pinned to `islandflow-vps`, so the default network name will be:
+
+```bash
+islandflow-vps_default
+```
+
+If your NPM container is separate, connect it once:
+
+```bash
+docker network connect islandflow-vps_default <npm-container-name>
+```
+
+6. Create these NPM proxy hosts:
+
+- `app.example.com` -> forward to `web`, port `3000`
+- `api.example.com` -> forward to `api`, port `4000`
+
+For the API host, enable websocket support.
+
+7. Open the app:
+
+- `https://app.example.com/`
+- Health check: `https://api.example.com/health`
+
+## Replay service
+
+Replay is disabled by default in this stack.
+
+Start it only when you want it:
+
+```bash
+docker compose --profile replay up -d replay
+```
+
+Stop it again:
+
+```bash
+docker compose stop replay
+```
+
+## Adapter notes
+
+### Synthetic mode
+
+This is the easiest way to smoke-test the deployment:
+
+- `OPTIONS_INGEST_ADAPTER=synthetic`
+- `EQUITIES_INGEST_ADAPTER=synthetic`
+
+### Alpaca mode
+
+Set the adapter values and credentials in `.env`:
+
+- `OPTIONS_INGEST_ADAPTER=alpaca`
+- `EQUITIES_INGEST_ADAPTER=alpaca`
+- `ALPACA_KEY_ID=...`
+- `ALPACA_SECRET_KEY=...`
+
+### Databento mode
+
+The `ingest-options` image in this deployment includes Python plus the repo’s sidecar dependencies, so Databento can run without a custom image. Set the Databento env vars in `.env`, especially:
+
+- `OPTIONS_INGEST_ADAPTER=databento`
+- `DATABENTO_API_KEY=...`
+- `DATABENTO_START=...`
+
+### IBKR mode
+
+If TWS or IB Gateway is running on the VPS host, the default `.env.example` already points `IBKR_HOST` at `host.docker.internal`, and the Compose stack adds the required host gateway mapping.
+
+If IBKR is running somewhere else, change:
+
+- `IBKR_HOST`
+- `IBKR_PORT`
+
+## NPM routing
+
+Recommended proxy hosts:
+
+- `app.<domain>` -> `web:3000`
+- `api.<domain>` -> `api:4000`
+
+The web app should be built with `NEXT_PUBLIC_API_URL=https://api.<domain>` so browser REST and websocket traffic goes straight to the API host through NPM.
+
+The API host needs websocket support enabled because the app uses `/ws/*` endpoints for live streams.
+
+## Updating the deployment
+
+When you pull new code:
+
+```bash
+cd deployment/docker
+docker compose up -d --build
+```
+
+If you changed only env values for the Bun services:
+
+```bash
+docker compose up -d
+```
+
+If you changed `NEXT_PUBLIC_API_URL` or `NEXT_PUBLIC_NBBO_MAX_AGE_MS`, rebuild the web image because those are public Next.js build-time values:
+
+```bash
+docker compose build web
+docker compose up -d web
+```
+
+## Backups and persistence
+
+Persistent data lives in Docker volumes:
+
+- `clickhouse-data`
+- `redis-data`
+- `nats-data`
+
+Before destructive maintenance, back up those volumes or the underlying Docker data directory for the host.
+
+## Shutdown
+
+Stop everything while keeping data:
+
+```bash
+docker compose down
+```
+
+Stop everything and remove volumes too:
+
+```bash
+docker compose down -v
+```
+
+Only use `-v` if you intentionally want to wipe ClickHouse, Redis, and JetStream state.
+
+## Known caveats
+
+- The root `.env.example` still contains a `REPLAY_ENABLED` comment, but the current replay service does not read that variable. Use the Compose replay profile instead.
+- This stack does not publish `web` or `api` to host ports. NPM must be able to resolve `web` and `api` over the shared Docker network.
+- The stack assumes a single-node VPS deployment. If you later split infra or add external managed services, update the three core connection URLs in `.env`.
+
+## Smoke checks
+
+After NPM is wired up:
+
+- `https://app.<domain>/` should load the UI.
+- Browser network requests from the UI should target `https://api.<domain>/...`.
+- Live feeds should connect over `wss://api.<domain>/ws/...`.
+- `docker compose ps` should show no service publishing host port `80`.

deployment/docker/docker-compose.yml

@@ -0,0 +1,126 @@
+name: islandflow-vps
+
+x-service-common: &service-common
+  build:
+    context: ../..
+    dockerfile: deployment/docker/Dockerfile.service
+  env_file:
+    - ./.env
+  restart: unless-stopped
+  init: true
+  extra_hosts:
+    - "host.docker.internal:host-gateway"
+  depends_on:
+    - nats
+    - clickhouse
+    - redis
+
+services:
+  web:
+    build:
+      context: ../..
+      dockerfile: deployment/docker/Dockerfile.web
+      args:
+        NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-}
+        NEXT_PUBLIC_NBBO_MAX_AGE_MS: ${NEXT_PUBLIC_NBBO_MAX_AGE_MS:-1000}
+    env_file:
+      - ./.env
+    restart: unless-stopped
+    init: true
+    expose:
+      - "3000"
+    depends_on:
+      api:
+        condition: service_healthy
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "bun",
+          "-e",
+          "const r=await fetch('http://127.0.0.1:3000/'); if(!r.ok) throw new Error('web healthcheck failed: '+r.status);"
+        ]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 45s
+
+  api:
+    <<: *service-common
+    command: ["services/api/src/index.ts"]
+    expose:
+      - "4000"
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "bun",
+          "-e",
+          "const r=await fetch('http://127.0.0.1:4000/health'); if(!r.ok) throw new Error('api healthcheck failed: '+r.status);"
+        ]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 20s
+
+  compute:
+    <<: *service-common
+    command: ["services/compute/src/index.ts"]
+
+  candles:
+    <<: *service-common
+    command: ["services/candles/src/index.ts"]
+
+  ingest-options:
+    build:
+      context: ../..
+      dockerfile: deployment/docker/Dockerfile.ingest-options
+    env_file:
+      - ./.env
+    restart: unless-stopped
+    init: true
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    depends_on:
+      - nats
+      - clickhouse
+      - redis
+    command: ["services/ingest-options/src/index.ts"]
+
+  ingest-equities:
+    <<: *service-common
+    command: ["services/ingest-equities/src/index.ts"]
+
+  replay:
+    <<: *service-common
+    profiles: ["replay"]
+    command: ["services/replay/src/index.ts"]
+
+  clickhouse:
+    image: clickhouse/clickhouse-server:23.8
+    restart: unless-stopped
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    volumes:
+      - clickhouse-data:/var/lib/clickhouse
+
+  redis:
+    image: redis:7.2
+    restart: unless-stopped
+    command: ["redis-server", "--appendonly", "yes"]
+    volumes:
+      - redis-data:/data
+
+  nats:
+    image: nats:2.10
+    restart: unless-stopped
+    command: ["-js", "-sd", "/data"]
+    volumes:
+      - nats-data:/data
+
+volumes:
+  clickhouse-data:
+  redis-data:
+  nats-data: