diff --git a/.env.example b/.env.example index d42f715..be20b62 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,10 @@ REDIS_URL=redis://127.0.0.1:6379 # Options ingest OPTIONS_INGEST_ADAPTER=synthetic ALPACA_API_KEY= +ALPACA_API_KEY_ID= +ALPACA_KEY_ID= +ALPACA_API_SECRET_KEY= +ALPACA_SECRET_KEY= ALPACA_REST_URL=https://data.alpaca.markets ALPACA_WS_BASE_URL=wss://stream.data.alpaca.markets/v1beta1 ALPACA_FEED=indicative diff --git a/.github/workflows/docs-pages.yml b/.github/workflows/docs-pages.yml new file mode 100644 index 0000000..9c4db98 --- /dev/null +++ b/.github/workflows/docs-pages.yml @@ -0,0 +1,56 @@ +name: Publish Docs + +on: + push: + branches: + - main + paths: + - "docs/**" + - "scripts/generate-docs-index.mjs" + - ".github/workflows/docs-pages.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure Pages + uses: actions/configure-pages@v5 + + - name: Build docs index + run: node scripts/generate-docs-index.mjs + + - name: Prepare static site payload + run: | + mkdir -p site/docs + cp -R docs/. site/docs/ + printf '%s\n' 'Islandflow DocsContinue to docs' > site/index.html + touch site/.nojekyll + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + needs: build + runs-on: ubuntu-latest + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/AGENTS server.md b/AGENTS server.md new file mode 100644 index 0000000..08a484a --- /dev/null +++ b/AGENTS server.md @@ -0,0 +1,174 @@ + +## Beads Issue Tracker + +This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands. + +### Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work +bd close # Complete work +``` + +### Rules + +- Use `bd` for ALL task tracking — do NOT use TodoWrite, TaskCreate, or markdown TODO lists +- Run `bd prime` for detailed command reference and session close protocol +- Use `bd remember` for persistent knowledge — do NOT use MEMORY.md files + +## Session Completion + +**When ending a work session**, you MUST complete ALL steps below. Work is NOT complete until `git push` succeeds. + +**MANDATORY WORKFLOW:** + +1. **File issues for remaining work** - Create issues for anything that needs follow-up +2. **Run quality gates** (if code changed) - Tests, linters, builds +3. **Update issue status** - Close finished work, update in-progress items +4. **PUSH TO REMOTE** - This is MANDATORY: + ```bash + git pull --rebase + bd dolt push + git push + git status # MUST show "up to date with origin" + ``` +5. **Clean up** - Clear stashes, prune remote branches +6. **Verify** - All changes committed AND pushed +7. **Hand off** - Provide context for next session + +**CRITICAL RULES:** +- Work is NOT complete until `git push` succeeds +- NEVER stop before pushing - that leaves work stranded locally +- NEVER say "ready to push when you are" - YOU must push +- If push fails, resolve and retry until it succeeds + + +## Minimal Repo Operating Instructions + +This is a Bun + TypeScript monorepo for an event-sourced market-data pipeline: +- Flow: ingest services publish to NATS/JetStream, compute/candles derive events, API serves REST/WS, web consumes live/replay streams. +- Main folders: `services/*` (runtime services), `packages/*` (shared libs/types/storage), `apps/web` (Next.js UI). +- Infra dependency: local dev assumes Docker services (NATS, ClickHouse, Redis) are available. + +Use these repo-specific commands: +- Install deps: `bun install` +- Start full stack: `bun run dev` +- Start infra only: `bun run dev:infra` +- Start backend services only: `bun run dev:services` +- Start web only: `bun run dev:web` + +Testing and validation in this repo are Bun-first: +- Run tests: `bun test` +- Run scoped tests: `bun test services/compute/tests` (or another package/service path) +- Validate web production build when UI code changes: `bun --cwd=apps/web run build` + +Working style that avoids common problems here: +- Prefer editing in the touched workspace (`services/`, `packages/`, `apps/web`) and keep shared contract changes in `packages/types`. +- Keep `.env` aligned with `.env.example`; adapters default to synthetic modes for local development. +- Dev runners persist child PID state in `.tmp/`; if a previous run crashed, restart via the standard `bun run dev*` commands so stale processes are cleaned up. + +## Required Turn Documentation + +At the end of every completed implementation task, before final handoff, create a user-readable HTML document describing the work. + +This documentation is mandatory whenever code, configuration, tests, or project files were changed. + +### Location + +Save the document in: + +```text +docs/turns/ +``` +## Important: If you are not working inside a git repository, save the document to `~/dev/docs/turns/` + +Use a clear timestamped filename: + +```text +docs/turns/YYYY-MM-DD-short-task-name.html +``` + +Example: + +```text +docs/turns/2026-05-14-add-market-replay-controls.html +``` + +### Format + +Use the impeccable skill to structure the document as clean, readable HTML. + +If the impeccable skill is unavailable, still create a well-structured standalone HTML file with: + +- A concise summary at the top +- A detailed explanation of what changed +- Relevant context or background +- Specific code snippets or examples when helpful +- Issues, limitations, tradeoffs, or mitigations +- Validation performed, including tests, builds, linters, or manual checks +- Any remaining follow-up work, with corresponding Beads issue IDs when applicable + +### Required Sections + +Each turn document must include these sections: + +1. **Summary** +2. **Changes Made** +3. **Context** +4. **Important Implementation Details** +5. **Relevant Diff Snippets** +6. **Expected Impact for End-Users** +7. **Validation** +8. **Issues, Limitations, and Mitigations** +9. **Follow-up Work** + +### Completion Rule + +A task is not complete until: + +1. The Beads workflow is updated +2. The turn document is created in `docs/turns` +3. Relevant quality gates have passed or failures are documented +4. Changes are committed +5. `bd dolt push` succeeds +6. `git push` succeeds +7. `git status` shows the branch is up to date with origin + +For trivial changes, the document may be brief, but it must still exist and clearly explain what changed and how it was validated. + +## Plan Mode Documentation + +When working in plan mode, do not modify implementation files. + +At the end of plan mode, provide a concise summary of the plan and ask the user whether they want to proceed with implementation. + +If the user asks to save the plan, create a user-readable HTML plan document in: + +```text +docs/plans/ +``` + +Use a clear timestamped filename: + +```text +docs/plans/YYYY-MM-DD-short-plan-name.html +``` + +The plan document should be labeled clearly as a plan and should include: + +1. **Plan Summary** +2. **Goals** +3. **Proposed Changes** +4. **Relevant Context** +5. **Implementation Steps** +6. **Risks, Limitations, and Mitigations** +7. **Open Questions** + +Always do the following when you finish a task, finish the beads workflow and and make a commit: +- Document the changes in a user-readable format +- Use the impeccable skill to structure the document as HTML +- Create a clear, concise summary of the changes at the top, followed by a detailed description of the changes, including any relevant context or background as well as specific code snippets or examples. +- Note any relevant issues or limitations that were addressed or mitigated by the changes. +- The HTML file should be stored in the `docs/turns` directory. It should include the current date and time, as well as a brief explanation of changes. e.g. docs/turns/YYYY-MM-DD-{description}.html diff --git a/AGENTS.md b/AGENTS.md index 3ab1cf0..fe8ffca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -97,9 +97,11 @@ docs/turns/2026-05-14-add-market-replay-controls.html ### Format -Use the impeccable skill to structure the document as clean, readable HTML. +Use the `impeccable` skill to structure and style the document as clean, readable HTML. -If the impeccable skill is unavailable, still create a well-structured standalone HTML file with: +For this repository, `impeccable` is the styling and layout authority for turn documents when available. Do not apply global non-repo computer-task house styling to repository turn documents. + +If the `impeccable` skill is unavailable or blocked by an actual tool/file error, still create a well-structured standalone HTML file with: - A concise summary at the top - A detailed explanation of what changed @@ -117,10 +119,11 @@ Each turn document must include these sections: 2. **Changes Made** 3. **Context** 4. **Important Implementation Details** -5. **Expected Impact for End-Users** -5. **Validation** -6. **Issues, Limitations, and Mitigations** -7. **Follow-up Work** +5. **Relevant Diff Snippets** +6. **Expected Impact for End-Users** +7. **Validation** +8. **Issues, Limitations, and Mitigations** +9. **Follow-up Work** ### Completion Rule diff --git a/README.md b/README.md index 50063d9..6b3b7fc 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,12 @@ > **Pre-alpha warning** This project is in an early pre-alpha state. It will not perform consistently or as expected, and APIs, behavior, and data contracts may change without notice. -This repository contains a Bun + TypeScript monorepo for a personal-use, event-sourced market microstructure research platform focused on: +Islandflow is a Bun + TypeScript monorepo for a personal-use, event-sourced market microstructure research platform focused on: - options prints + NBBO, - off-exchange equity prints, -- explainable rule-based flow classification, +- market news context, +- explainable smart-money flow classification, - deterministic replay, - evidence-linked UI inspection. @@ -19,124 +20,176 @@ This repository contains a Bun + TypeScript monorepo for a personal-use, event-s Implemented now: - Bun workspaces with shared packages for schemas, bus, config, observability, and ClickHouse access. -- Infra orchestration via Docker Compose (NATS JetStream, ClickHouse, Redis). -- Options ingest service with adapters: - - synthetic stream, - - Alpaca options (dev-focused, bounded contracts), - - IBKR bridge (Python sidecar), - - Databento historical replay adapter (Python sidecar). -- Equities ingest service with adapters: - - synthetic stream, - - Alpaca equities trades/quotes. -- Compute service: - - deterministic option print clustering into `FlowPacket`s, - - NBBO join quality features and aggressor-mix metrics, - - rolling baselines in Redis, - - structure summarization and structure packet emission, - - rule-based classifiers + confidence-scored alert events, - - dark-style inferred events from equity prints/quotes, - - equity print-to-quote join events. -- Candles service: - - server-side equity candle aggregation, - - ClickHouse persistence, - - optional Redis hot cache, - - NATS publication. -- Replay service: - - deterministic republishing from ClickHouse to NATS, - - multi-stream merge with stable tie-break ordering, - - speed/start/end controls. -- API service: - - REST endpoints for recent + cursor pagination, - - REST range endpoints for chart windows, - - REST replay-oriented endpoints, - - WebSocket channels for options, NBBO, equities, quotes, joins, flow, classifier hits, alerts, inferred dark, and candles. -- Next.js web app: - - live tape/workspace views, - - replay controls and status, - - signals and chart-focused routes, - - evidence-centric terminal UI. -- Refdata + EOD enricher service entrypoints are present but currently scaffolds (lifecycle/logging only). +- Infra orchestration via Docker Compose for local NATS JetStream, ClickHouse, and Redis. +- Options ingest service with synthetic, Alpaca options, IBKR bridge, and Databento historical replay adapters. +- Equities ingest service with synthetic and Alpaca equities trades/quotes adapters. +- News ingest service for Alpaca news backfill and websocket publication. +- Compute service for deterministic parent-event reconstruction, flow packets, NBBO quality features, rolling baselines, smart-money profile scoring, compatibility classifier hits, alerts, inferred dark-style events, and equity print-to-quote joins. +- Candles service for server-side equity candle aggregation, ClickHouse persistence, optional Redis hot cache, and NATS publication. +- Replay service for deterministic ClickHouse-to-NATS republishing with multi-stream merge, stable tie-break ordering, speed, start, and end controls. +- API service with REST endpoints, cursor pagination, replay/history endpoints, live hot-cache hydration, and WebSocket channels for options, NBBO, equities, quotes, joins, flow, classifier hits, alerts, smart-money events, inferred dark, candles, and news. +- Next.js web app upgraded to Next.js `16.2.6`, React `19.2.0`, and React DOM `19.2.0`. +- Evidence-centric terminal UI, live/replay controls, chart-focused routes, news view, profile-aware smart-money display, and alert-context hydration. +- Thin Electron desktop shell in `apps/desktop` that can wrap the hosted app or local web UI. +- Refdata + EOD enricher service entrypoints are present, with refdata able to validate or refresh the event-calendar cache. Planned / not yet complete: - production-grade licensed feed integrations and entitlement workflow, - richer refdata/corp-action enrichment, - secure deployment/auth hardening, -- deeper structure + calibration workflows from `PLAN.md`. +- native deployment unit templates and rollback helpers, +- signed/notarized desktop distribution and richer desktop-native features, +- deeper calibration workflows from `PLAN.md` and `SMART_MONEY_REBUILD_PLAN.md`. ## Core Principles -- **Explainability first** — inferred outputs are evidence-backed and human-readable. -- **Event sourcing** — raw and derived events persist to support replay. -- **Determinism** — replay behavior tracks live pipeline logic. -- **Microstructure awareness** — bounded joins, confidence scoring, and explicit uncertainty. -- **Bun-first tooling** — runtime/package/scripts all use Bun. +- **Explainability first**: inferred outputs are evidence-backed and human-readable. +- **Event sourcing**: raw and derived events persist to support replay. +- **Determinism**: replay behavior tracks live pipeline logic. +- **Microstructure awareness**: bounded joins, confidence scoring, and explicit uncertainty. +- **Taxonomy over folklore**: "smart money" is modeled as participant-style hypotheses, not a single binary label. +- **Bun-first tooling**: runtime, package management, scripts, and tests use Bun. + +## Smart-Money Classification Taxonomy + +Islandflow now emits first-class `SmartMoneyEvent` records instead of treating old classifier hits as the final semantic object. `FlowPacket` remains the clustering bridge, while smart-money events carry typed features, profile scores, confidence bands, directions, reason codes, abstention state, and suppression reasons. + +Public profile IDs: + +| Profile ID | Meaning | Common evidence | +| --- | --- | --- | +| `institutional_directional` | Large directional parent flow with stronger institutional-style conviction. | premium, size, sweep/burst behavior, aggressor imbalance, quote quality, not short-dated retail-chase context | +| `retail_whale` | Large retail-style speculative bursts, often short-dated or attention-driven. | short-dated OTM concentration, burst prints, IV shock, lower premium than institutional blocks | +| `event_driven` | Flow aligned to known upcoming events. | event-calendar proximity, expiry after event, pre-event concentration, spread/IV pressure | +| `vol_seller` | Premium-selling or short-volatility structure evidence. | sell-side premium, straddles/strangles, neutral direction | +| `arbitrage` | Multi-leg or symmetric structures with low directional exposure. | matched leg symmetry, same-size legs, near-flat directional bias | +| `hedge_reactive` | Hedge or dealer-reaction style flow around short-dated ATM/gamma context. | 0-2 DTE, near-ATM contracts, underlying move linkage, size | + +Compatibility surfaces remain in place: + +- `ClassifierHitEvent` is derived from `SmartMoneyEvent.primary_profile_id`. +- `AlertEvent` may include `primary_profile_id` and `profile_scores`. +- Legacy classifier and alert endpoints still work. + +Primary smart-money access paths: + +```text +/flow/smart-money +/history/smart-money +/replay/smart-money +/ws/smart-money +``` + +The classifier intentionally abstains when evidence is weak or quote context is stale/missing. Suppression guards cover stale quotes, complex/special prints, retail-frenzy directional confusion, hedge-reactive short-dated ATM contexts, and arbitrage symmetry. ## Monorepo Layout - `apps/web` — Next.js UI shell/routes. -- `apps/desktop` — Electron desktop shell that loads the hosted Islandflow app. +- `apps/desktop` — Electron desktop shell that loads the hosted or local Islandflow app. - `services/ingest-options` — options print/NBBO ingest adapters. - `services/ingest-equities` — equity print/quote ingest adapters. -- `services/compute` — clustering, structures, classifiers, alerts, inferred dark. +- `services/ingest-news` — Alpaca news backfill and websocket ingest. +- `services/compute` — parent-event reconstruction, flow packets, smart-money scoring, alerts, inferred dark. - `services/candles` — server-side candle aggregation + cache. -- `services/replay` — ClickHouse → NATS replay streamer. +- `services/replay` — ClickHouse to NATS replay streamer. - `services/api` — REST + WebSocket gateway. -- `services/refdata` — scaffold service. +- `services/refdata` — event-calendar validation/provider refresh scaffolding. - `services/eod-enricher` — scaffold service. - `packages/types` — shared event schemas/types. - `packages/storage` — ClickHouse tables/queries. - `packages/bus` — NATS/JetStream helpers. - `packages/config` — env parsing. - `packages/observability` — logger + metrics facade. +- `deployment/docker` — supported VPS Docker Compose runtime. +- `deployment/native` — experimental host-native Bun + systemd deployment notes. ## Build and Run Install dependencies: -- `bun install` +```bash +bun install +``` Start infrastructure only: -- `docker compose up -d` +```bash +bun run dev:infra +``` Create env file: -- copy `.env.example` to `.env` and set provider credentials as needed. +```bash +cp .env.example .env +``` Start infra + all services + web: -- `bun run dev` +```bash +bun run dev +``` -Start services only (assumes infra is already running): +Start services only, assuming infra is already running: -- `bun run dev:services` +```bash +bun run dev:services +``` Start web only: -- `bun run dev:web` +```bash +bun run dev:web +``` Recommended fast iteration loop: -- `bun run dev:infra` for Docker-backed infra only -- `bun run dev:services` for native Bun backend services -- `bun run dev:web` for the local Next.js UI +```bash +bun run dev:infra +bun run dev:services +bun run dev:web +``` -This keeps Docker in the local workflow where it helps most (NATS, ClickHouse, Redis) without forcing the app services themselves into slower container rebuild/restart loops. +This keeps Docker in the local workflow where it helps most, for NATS, ClickHouse, and Redis, while keeping the app services in native Bun/Next.js loops. ## Deployment Workflow -- `./deploy main` keeps the current VPS Docker rollout path as the default and recommended path. -- Do not run the repo-root `docker-compose.yml` on the VPS. That file is for local infra only and can create duplicate exposed NATS, ClickHouse, and Redis containers on the server. -- `./deploy main --runtime native` targets an experimental host-native Bun + systemd deployment. -- `./deploy current-branch` and `./deploy current-branch --runtime native` keep branch deploys available during the transition, but Docker remains the supported path for the current VPS. -- Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, and `--no-build`. -- Docker runtime details live in `deployment/docker/README.md`. -- Native runtime expectations and prerequisites live in `deployment/native/README.md`. +Docker remains the supported and recommended path for the current VPS. + +```bash +./deploy main +./deploy main --runtime docker +./deploy current-branch +./deploy current-branch --runtime docker +``` + +Important deployment notes: + +- Run the deploy helper from the local repo checkout, not from the VPS shell. +- Do not run the repo-root `docker-compose.yml` on the VPS. It is local infra only and can create duplicate exposed NATS, ClickHouse, and Redis containers on the server. +- The Docker stack lives in `deployment/docker` and is separate from local development infra. +- Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, `--workers-only`, `--fast`, `--no-build`, and `--force-recreate`. +- `--fast` defaults to a services-only Docker rollout when no explicit scope is provided and trims public API route-suite verification while preserving remote service health checks. +- `./deploy current-branch` requires a clean local working tree and pushes the branch before moving the server checkout. +- The helper has Forgejo-aware remote resolution for deployments and branch pushes. +- When run from `/home/delta/islandflow` on the VPS itself, `./deploy` can execute locally instead of SSHing back into the same server. +- Native deployment is opt-in and experimental: + +```bash +./deploy main --runtime native +./deploy current-branch --runtime native +``` + +Native deployment expects Bun, systemd units, host-reachable infra, and deliberate reverse-proxy changes. Native deploys are intended primarily for worker-only fast iteration until the public edge is cut over deliberately. + +Read more: + +- `deployment/docker/README.md` +- `deployment/native/README.md` ## Desktop Shell -Islandflow also includes a thin Electron desktop shell in `apps/desktop`. +Islandflow includes a thin Electron desktop shell in `apps/desktop`. What it is: @@ -144,37 +197,35 @@ What it is: - a native app window plus packaging/distribution shell, - a way to run the existing web UI inside Electron without local backend services. -What it is not: +What it is not yet: - a bundled backend runtime, -- a packaged local Next.js frontend in v1, -- a desktop feature layer with notifications, preferences, or auto-updates yet. +- a packaged local Next.js frontend, +- a desktop feature layer with notifications, preferences, auto-updates, signing, or notarization. Run the desktop shell against a local web UI: -- `bun run dev:desktop` - -This starts the local Next.js app, defaults `NEXT_PUBLIC_API_URL` to `https://flow.deltaisland.io` unless you already set it, waits for port `3000`, and then launches Electron against `http://127.0.0.1:3000`. +```bash +bun run dev:desktop +``` Run the desktop shell directly against the hosted app: -- `bun run dev:desktop:remote` +```bash +bun run dev:desktop:remote +``` Package the desktop shell: -- `bun run package:desktop` -- `bun run make:desktop` +```bash +bun run package:desktop +bun run make:desktop +``` Desktop-specific environment: - `ISLANDFLOW_DESKTOP_START_URL` is only used by the Electron shell and is restricted to trusted Islandflow app origins. -- `NEXT_PUBLIC_API_URL` remains the web app's API/WebSocket origin control and should usually point at `https://flow.deltaisland.io` when developing the local UI inside Electron. - -Current desktop limitations: - -- v1 builds are unsigned internal macOS artifacts only, -- Forge currently makes a simple zip distributable for the current host architecture, -- signing, notarization, auto-updates, remembered window state, and richer native integrations are intentionally deferred. +- `NEXT_PUBLIC_API_URL` remains the web app API/WebSocket origin control and usually points at `https://flow.deltaisland.io` when developing local UI inside Electron. ## Environment Configuration @@ -196,32 +247,31 @@ All runtime configuration comes from `.env`. | `OPTIONS_INGEST_ADAPTER` | `synthetic` | Options ingest source: `synthetic`, `alpaca`, `ibkr`, or `databento`. | | `EQUITIES_INGEST_ADAPTER` | `synthetic` | Equities ingest source: `synthetic` or `alpaca`. | | `EMIT_INTERVAL_MS` | `1000` | Emit cadence for synthetic ingest adapters. | -| `SYNTHETIC_MARKET_MODE` | `realistic` | Shared synthetic profile (`realistic`, `active`, `firehose`) used when per-service override is unset. | -| `SYNTHETIC_OPTIONS_MODE` | empty | Options-only synthetic profile override; falls back to `SYNTHETIC_MARKET_MODE`. | -| `SYNTHETIC_EQUITIES_MODE` | empty | Equities-only synthetic profile override; falls back to `SYNTHETIC_MARKET_MODE`. | +| `SYNTHETIC_MARKET_MODE` | `realistic` | Shared synthetic profile: `realistic`, `active`, or `firehose`. | +| `SYNTHETIC_OPTIONS_MODE` | empty | Options-only synthetic profile override. | +| `SYNTHETIC_EQUITIES_MODE` | empty | Equities-only synthetic profile override. | -Synthetic profile intent: -- `realistic`: default local mode with lower synthetic burstiness/noise. -- `active`: busier demo flow while still readable. -- `firehose`: stress mode for throughput/backpressure/hot-window behavior. - -### Options ingest adapter configuration +### Alpaca and news configuration | Variable | Default | What it controls | | --- | --- | --- | -| `ALPACA_API_KEY` | empty | Single-token Alpaca API auth for options/equities adapters. Use this when your account provides one API key value. | -| `ALPACA_REST_URL` | `https://data.alpaca.markets` | Alpaca REST base URL for contract discovery/reference calls. | -| `ALPACA_WS_BASE_URL` | `wss://stream.data.alpaca.markets/v1beta1` (options), `wss://stream.data.alpaca.markets` (equities) | Alpaca websocket base URL. | -| `ALPACA_FEED` | `indicative` | Options feed tier for Alpaca options (`indicative` or `opra`). | +| `ALPACA_API_KEY` | empty | Legacy single-token fallback kept for older Alpaca setups. Prefer explicit key ID + secret vars for current Alpaca auth. | +| `ALPACA_API_KEY_ID` | empty | Preferred Alpaca key ID used for market-data REST and websocket auth. | +| `ALPACA_KEY_ID` | empty | Alternate name accepted for the Alpaca key ID. | +| `ALPACA_API_SECRET_KEY` | empty | Preferred Alpaca secret key paired with `ALPACA_API_KEY_ID`. | +| `ALPACA_SECRET_KEY` | empty | Alternate name accepted for the Alpaca secret key. | +| `ALPACA_REST_URL` | `https://data.alpaca.markets` | Alpaca REST base URL. | +| `ALPACA_WS_BASE_URL` | `wss://stream.data.alpaca.markets/v1beta1` for options, `wss://stream.data.alpaca.markets` for equities/news | Alpaca websocket base URL. | +| `ALPACA_FEED` | `indicative` | Options feed tier: `indicative` or `opra`. | | `ALPACA_UNDERLYINGS` | `SPY,NVDA,AAPL` | Comma-separated symbols targeted by Alpaca ingest. | | `ALPACA_STRIKES_PER_SIDE` | `8` | Contracts selected per side of spot for Alpaca options chain sampling. | | `ALPACA_MAX_DTE_DAYS` | `30` | Max days-to-expiry included for Alpaca options contract selection. | | `ALPACA_MONEYNESS_PCT` | `0.06` | Primary moneyness filter for Alpaca options contract selection. | | `ALPACA_MONEYNESS_FALLBACK_PCT` | `0.1` | Wider fallback moneyness filter if candidate set is too sparse. | | `ALPACA_MAX_QUOTES` | `200` | Upper bound on selected Alpaca options contracts/quotes per cycle. | -| `ALPACA_EQUITIES_FEED` | `iex` | Alpaca equities feed (`iex` free tier, `sip` paid consolidated feed). | - -For Alpaca adapters, configure `ALPACA_API_KEY`. +| `ALPACA_EQUITIES_FEED` | `iex` | Alpaca equities feed: `iex` or `sip`. | +| `ALPACA_NEWS_BACKFILL_LIMIT` | `50` | Alpaca news stories fetched on startup, capped at 50 by the Alpaca News API. | +| `ALPACA_NEWS_WEBSOCKET_PATH` | `/v1beta1/news` | Alpaca news websocket path. | ### Databento replay adapter configuration @@ -236,7 +286,7 @@ For Alpaca adapters, configure `ALPACA_API_KEY`. | `DATABENTO_SYMBOLS` | `ALL` | Symbol selection forwarded to Databento sidecar query. | | `DATABENTO_STYPE_IN` | `raw_symbol` | Databento input symbology type. | | `DATABENTO_STYPE_OUT` | `raw_symbol` | Databento output symbology type. | -| `DATABENTO_LIMIT` | `0` | Max Databento records (`0` means no explicit limit). | +| `DATABENTO_LIMIT` | `0` | Max Databento records, where `0` means no explicit limit. | | `DATABENTO_PRICE_SCALE` | `1` | Multiplier applied to decoded prices from sidecar output. | | `DATABENTO_PYTHON_BIN` | `python3` | Python executable used to run Databento sidecar script. | @@ -248,9 +298,9 @@ For Alpaca adapters, configure `ALPACA_API_KEY`. | `IBKR_PORT` | `7497` | TWS/Gateway port for IBKR bridge. | | `IBKR_CLIENT_ID` | `0` | IBKR client id used by the bridge connection. | | `IBKR_SYMBOL` | `SPY` | Underlying symbol requested from IBKR. | -| `IBKR_EXPIRY` | `20250117` | Option expiry (YYYYMMDD) requested from IBKR. | +| `IBKR_EXPIRY` | `20250117` | Option expiry requested from IBKR. | | `IBKR_STRIKE` | `450` | Strike requested from IBKR. | -| `IBKR_RIGHT` | `C` | Option side (`C` or `P`). | +| `IBKR_RIGHT` | `C` | Option side: `C` or `P`. | | `IBKR_EXCHANGE` | `SMART` | IBKR exchange routing code. | | `IBKR_CURRENCY` | `USD` | Contract currency. | | `IBKR_PYTHON_BIN` | `python3` | Python executable used for IBKR sidecar. | @@ -259,133 +309,77 @@ For Alpaca adapters, configure `ALPACA_API_KEY`. | Variable | Default | What it controls | | --- | --- | --- | -| `OPTIONS_SIGNAL_MODE` | `smart-money` | Signal pass policy (`smart-money`, `balanced`, `all`) for options prints. | +| `OPTIONS_SIGNAL_MODE` | `smart-money` | Signal pass policy: `smart-money`, `balanced`, or `all`. | | `OPTIONS_SIGNAL_MIN_NOTIONAL` | `10000` | Base minimum notional for most signal candidates. | | `OPTIONS_SIGNAL_ETF_MIN_NOTIONAL` | `50000` | ETF-specific minimum notional for signal inclusion. | -| `OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL` | `25000` | Minimum notional for bid-side (`B`/`BB`) or sweep/ISO thresholds. | +| `OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL` | `25000` | Minimum notional for bid-side or sweep/ISO thresholds. | | `OPTIONS_SIGNAL_MID_MIN_NOTIONAL` | `20000` | Minimum notional for non-sweep/non-ISO `MID` prints. | | `OPTIONS_SIGNAL_NBBO_MAX_AGE_MS` | `1500` | NBBO freshness threshold used during signal classification. | -| `OPTIONS_SIGNAL_ETF_UNDERLYINGS` | `SPY,QQQ,IWM,DIA,TLT,GLD,SLV,XLF,XLE,XLV,XLI,XLP,XLU,XLY,SMH,ARKK` | Comma-separated underlyings treated as ETFs by signal filters. | +| `OPTIONS_SIGNAL_ETF_UNDERLYINGS` | `SPY,QQQ,IWM,DIA,TLT,GLD,SLV,XLF,XLE,XLV,XLI,XLP,XLU,XLY,SMH,ARKK` | ETF underlyings treated specially by signal filters. | -Default `smart-money` policy rejects lower-information prints and keeps high-confidence/high-notional/sweep-style flow; `balanced` lowers thresholds; `all` bypasses filtering. +Default `smart-money` policy rejects lower-information prints and keeps higher-confidence, higher-notional, sweep-style flow. `balanced` lowers thresholds. `all` bypasses filtering. -### Compute/classifier/dark-inference configuration +### Compute, classifier, and dark-inference configuration | Variable | Default | What it controls | | --- | --- | --- | -| `CLUSTER_WINDOW_MS` | `500` | Time window used to cluster nearby option prints into a packet candidate. | -| `COMPUTE_DELIVER_POLICY` | `new` | Consumer start policy for compute stream subscriptions (`new`, `all`, `last`, `last_per_subject`). | -| `COMPUTE_CONSUMER_RESET` | `false` | If true, resets durable consumer position for compute on startup. | +| `CLUSTER_WINDOW_MS` | `500` | Time window used to cluster nearby option prints into packet candidates. | +| `COMPUTE_DELIVER_POLICY` | `new` | Consumer start policy for compute subscriptions. | +| `COMPUTE_CONSUMER_RESET` | `false` | Resets durable consumer position for compute on startup when true. | | `NBBO_MAX_AGE_MS` | `1000` | Max NBBO age accepted when enriching option prints in compute. | | `ROLLING_WINDOW_SIZE` | `50` | Number of observations retained per rolling metric key. | | `ROLLING_TTL_SEC` | `86400` | Redis TTL for rolling metric keys. | | `EQUITY_QUOTE_MAX_AGE_MS` | `1000` | Max quote staleness when joining equity prints for inference. | | `DARK_INFER_WINDOW_MS` | `60000` | Sliding window length for dark-style inference accumulation. | -| `DARK_INFER_COOLDOWN_MS` | `30000` | Cooldown before emitting repeated dark inferences for same symbol/pattern. | -| `DARK_INFER_MIN_BLOCK_SIZE` | `2000` | Minimum single-print size for block-style dark inference evidence. | -| `DARK_INFER_MIN_ACCUM_SIZE` | `3000` | Minimum aggregate size for accumulation-style dark inference evidence. | -| `DARK_INFER_MIN_ACCUM_COUNT` | `4` | Minimum print count for accumulation-style dark inference. | -| `DARK_INFER_MIN_PRINT_SIZE` | `200` | Minimum print size considered as dark inference evidence. | -| `DARK_INFER_MAX_EVIDENCE` | `20` | Max evidence items attached to one inferred dark event. | -| `DARK_INFER_MAX_SPREAD_PCT` | `0.005` | Maximum spread percentage allowed for dark inference confidence. | -| `CLASSIFIER_SWEEP_MIN_PREMIUM` | `40000` | Minimum premium to trigger sweep classifier logic. | -| `CLASSIFIER_SWEEP_MIN_COUNT` | `3` | Minimum child prints in cluster for sweep classifier hit. | -| `CLASSIFIER_SWEEP_MIN_PREMIUM_Z` | `2` | Min premium z-score for sweep classifier confirmation. | -| `CLASSIFIER_SPIKE_MIN_PREMIUM` | `20000` | Minimum premium for spike classifier logic. | -| `CLASSIFIER_SPIKE_MIN_SIZE` | `400` | Minimum total size for spike classifier logic. | -| `CLASSIFIER_SPIKE_MIN_PREMIUM_Z` | `2.5` | Min premium z-score for spike classifier confirmation. | -| `CLASSIFIER_SPIKE_MIN_SIZE_Z` | `2` | Min size z-score for spike classifier confirmation. | -| `CLASSIFIER_Z_MIN_SAMPLES` | `12` | Minimum rolling sample count before z-score gating applies. | -| `CLASSIFIER_MIN_NBBO_COVERAGE` | `0.5` | Required fraction of prints in cluster with valid NBBO context. | -| `CLASSIFIER_MIN_AGGRESSOR_RATIO` | `0.55` | Minimum aggressor-side ratio for classifier confidence. | -| `CLASSIFIER_0DTE_MAX_ATM_PCT` | `0.01` | Max distance-from-ATM to qualify as near-ATM 0DTE event. | -| `CLASSIFIER_0DTE_MIN_PREMIUM` | `20000` | Minimum premium for 0DTE classifier events. | -| `CLASSIFIER_0DTE_MIN_SIZE` | `400` | Minimum size for 0DTE classifier events. | -| `SMART_MONEY_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file used by compute to enrich event-driven smart-money profile features. | -| `REFDATA_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file for refdata service startup validation; falls back to `SMART_MONEY_EVENT_CALENDAR_PATH` when unset. | -| `REFDATA_EVENT_CALENDAR_PROVIDER` | empty | Set to `alpha_vantage` to have refdata refresh the calendar cache from Alpha Vantage. | -| `ALPHA_VANTAGE_API_KEY` | empty | Alpha Vantage key used when `REFDATA_EVENT_CALENDAR_PROVIDER=alpha_vantage`. | -| `ALPHA_VANTAGE_EARNINGS_HORIZON` | `3month` | Alpha Vantage earnings horizon: `3month`, `6month`, or `12month`. | -| `ALPHA_VANTAGE_EARNINGS_SYMBOL` | empty | Optional single-symbol Alpha Vantage earnings query; empty fetches the full scheduled earnings list. | -| `REFDATA_EVENT_CALENDAR_REFRESH_MS` | `86400000` | Refdata refresh cadence for provider-backed event-calendar cache writes. | +| `DARK_INFER_COOLDOWN_MS` | `30000` | Cooldown before repeated dark inferences for same symbol/pattern. | +| `SMART_MONEY_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file used by compute. | +| `REFDATA_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar path for refdata; falls back to `SMART_MONEY_EVENT_CALENDAR_PATH`. | +| `REFDATA_EVENT_CALENDAR_PROVIDER` | empty | Set to `alpha_vantage` to refresh event-calendar cache from Alpha Vantage. | +| `ALPHA_VANTAGE_API_KEY` | empty | Alpha Vantage key for provider-backed event-calendar refresh. | -Event-calendar rows may use `symbol`, `underlying`, or `underlying_id`; `event_date`, `event_time`, or `event_ts`; and `announced_ts`, `available_ts`, `as_of_ts`, or `created_ts`. Compute only uses events already available at the packet timestamp, so missing or unavailable rows leave event-alignment features as neutral `null` values. - -### Candle service configuration - -| Variable | Default | What it controls | -| --- | --- | --- | -| `CANDLE_INTERVALS_MS` | `60000,300000` | Comma-separated candle intervals generated from equity prints. | -| `CANDLE_MAX_LATE_MS` | `0` | Allowed lateness for out-of-order prints before candle rejection/roll policy applies. | -| `CANDLE_CACHE_LIMIT` | `2000` | Max cached candles per `(underlying, interval)` in Redis (`0` disables cache). | -| `CANDLE_DELIVER_POLICY` | `new` | Consumer start policy for candle service (`new`, `all`, `last`, `last_per_subject`). | -| `CANDLE_CONSUMER_RESET` | `false` | If true, resets candle durable consumer position on startup. | - -### API + live cache configuration +### API, live cache, and web client | Variable | Default | What it controls | | --- | --- | --- | | `API_PORT` | `4000` | API service listen port. | -| `REST_DEFAULT_LIMIT` | `200` | Default record count when a REST endpoint omits `limit`. | -| `API_DELIVER_POLICY` | `new` | JetStream consumer start policy used by API live subscribers (`new`, `all`, `last`, `last_per_subject`). | -| `API_CONSUMER_RESET` | `false` | If true, API resets/recreates its live durable consumers on startup. | -| `LIVE_LIMIT_OPTIONS` | `10000` | In-memory/Redis live cache depth for options channel (clamped `1..100000`). | -| `LIVE_LIMIT_NBBO` | `10000` | Live cache depth for options NBBO channel (clamped `1..100000`). | -| `LIVE_LIMIT_EQUITIES` | `10000` | Live cache depth for equities channel (clamped `1..100000`). | -| `LIVE_LIMIT_EQUITY_QUOTES` | `10000` | Live cache depth for equity quotes channel (clamped `1..100000`). | -| `LIVE_LIMIT_EQUITY_JOINS` | `10000` | Live cache depth for equity join channel (clamped `1..100000`). | -| `LIVE_LIMIT_FLOW` | `10000` | Live cache depth for flow packet channel (clamped `1..100000`). | -| `LIVE_LIMIT_CLASSIFIER_HITS` | `10000` | Live cache depth for classifier hits channel (clamped `1..100000`). | -| `LIVE_LIMIT_ALERTS` | `10000` | Live cache depth for alerts channel (clamped `1..100000`). | -| `LIVE_LIMIT_INFERRED_DARK` | `10000` | Live cache depth for inferred dark channel (clamped `1..100000`). | - -### Web client configuration (`NEXT_PUBLIC_*`) - -| Variable | Default | What it controls | -| --- | --- | --- | -| `NEXT_PUBLIC_API_URL` | auto-detected (`window.location.origin` in browser; `http://127.0.0.1:4000` fallback) | Explicit base URL for API/WS calls from the web app. | -| `NEXT_PUBLIC_LIVE_HOT_WINDOW` | `2000` | Max hot-window items retained for non-options live streams in UI state (`100..100000`). | -| `NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS` | `25000` | Dedicated max hot-window items retained for options prints (`100..100000`). | -| `NEXT_PUBLIC_NBBO_MAX_AGE_MS` | `1000` | Frontend NBBO staleness threshold used for UI status/placement logic. | -| `NEXT_PUBLIC_LIVE_EQUITIES_SILENT_WARNING_MS` | `25000` | Delay before warning when equities stream is quiet (`5000..300000`). | -| `NEXT_PUBLIC_PINNED_EVIDENCE_TTL_MS` | `1200000` | TTL for pinned evidence objects in UI (`60000..7200000`). | -| `NEXT_PUBLIC_PINNED_EVIDENCE_MAX_ITEMS` | `4000` | Maximum pinned evidence cache size in UI (`100..50000`). | -| `NEXT_PUBLIC_FLOW_FILTER_PRESET` | `smart-money` | Default flow filter preset applied on page load (`smart-money`, `balanced`, `all`). | +| `REST_DEFAULT_LIMIT` | `200` | Default REST record count. | +| `API_DELIVER_POLICY` | `new` | JetStream consumer start policy used by API live subscribers. | +| `API_CONSUMER_RESET` | `false` | Resets/recreates API live durable consumers on startup when true. | +| `LIVE_LIMIT_DEFAULT` | `1000` | Optional generic live cache depth default. | +| `LIVE_LIMIT_FLOW` | `500` | Live cache depth for flow packet events unless overridden. | +| `LIVE_LIMIT_SMART_MONEY` | `300` | Live cache depth for smart-money events unless overridden. | +| `LIVE_LIMIT_OPTIONS` | `1000` | Live cache depth for options channel unless overridden. | +| `LIVE_LIMIT_ALERTS` | `300` | Live cache depth for alerts channel unless overridden. | +| `LIVE_LIMIT_NEWS` | `100` | Live cache depth for news channel unless overridden. | +| `NEXT_PUBLIC_API_URL` | auto-detected in browser, `http://127.0.0.1:4000` fallback | Explicit base URL for API/WS calls from the web app. | +| `NEXT_PUBLIC_LIVE_HOT_WINDOW` | `600` | Max hot-window items retained for non-options live streams in UI state. | +| `NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS` | `1200` | Dedicated max hot-window items retained for options prints. | +| `NEXT_PUBLIC_NBBO_MAX_AGE_MS` | `1000` | Frontend NBBO staleness threshold. | +| `NEXT_PUBLIC_FLOW_FILTER_PRESET` | `smart-money` | Default flow filter preset: `smart-money`, `balanced`, or `all`. | ### Replay and testing controls | Variable | Default | What it controls | | --- | --- | --- | -| `REPLAY_ENABLED` | `false` | Dev-script toggle: starts replay service in `bun run dev` when truthy. | -| `REPLAY_STREAMS` | `options,nbbo,equities,equity-quotes` | Replay stream selection (`all` or comma list of supported aliases). | -| `REPLAY_START_TS` | `0` | Replay lower-bound timestamp; `0` means from earliest stored data. | -| `REPLAY_END_TS` | `0` | Replay upper-bound timestamp; `0` means no explicit end bound. | -| `REPLAY_SPEED` | `1` | Replay speed multiplier relative to original event timing. | -| `REPLAY_BATCH_SIZE` | `200` | Batch fetch size per replay stream pull. | -| `REPLAY_LOG_EVERY` | `1000` | Progress log interval (emitted event count). | +| `REPLAY_ENABLED` | `false` | Starts replay service in `bun run dev` when truthy. | +| `REPLAY_STREAMS` | `options,nbbo,equities,equity-quotes` | Replay stream selection. | +| `REPLAY_START_TS` | `0` | Replay lower-bound timestamp. | +| `REPLAY_END_TS` | `0` | Replay upper-bound timestamp. | +| `REPLAY_SPEED` | `1` | Replay speed multiplier. | +| `REPLAY_BATCH_SIZE` | `200` | Batch fetch size per stream. | +| `REPLAY_LOG_EVERY` | `1000` | Progress log interval. | | `TESTING_MODE` | `false` | Enables ingest publish throttling for deterministic/lower-volume test runs. | | `TESTING_THROTTLE_MS` | `200` | Minimum delay between emitted events while `TESTING_MODE=true`. | ## Quick Notes -- Python dependencies are required only for IBKR/Databento sidecars (`services/ingest-options/py/requirements.txt`). +- Python dependencies are required only for IBKR/Databento sidecars: `services/ingest-options/py/requirements.txt`. - Candle construction is server-side; the client consumes prebuilt OHLC events. -- Option prints now persist as enriched raw rows and can be queried as either: - - `view=signal` — default live/UI path and compute input. - - `view=raw` — audit/debug path that preserves every stored print. -- The default Tape page options/packets posture is now stock-only, hides `B` / `BB`, keeps calls and puts visible, and applies in-memory min-notional controls immediately. -- Live retention uses a two-tier model: - - ClickHouse is durable server history; Redis is a bounded hot cache per live generic channel. - - `LIVE_LIMIT_*` controls initial snapshot/hot-cache depth, not total persisted history. - - Browser state is only a rendering window and UI preferences, not a market-data database. - - Devices connected to the same API hydrate from the same server-seen history. - - UI keeps a bounded hot window for rendering performance around the signal view rather than raw noise. - - Options prints can use a deeper dedicated cap via `NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS` without raising every other feed. - - Alert/drawer evidence is pinned and hydrated by id/trace so details remain inspectable after hot-window eviction. -- Firehose-readiness strategy: - - preserve raw ingest for storage/replay, - - feed compute and default live UI from the filtered signal path, - - add filterable live subscription contracts now so selective delivery can move server-side without reshaping the protocol later. +- Option prints persist as enriched raw rows and can be queried as `view=signal` or `view=raw`. +- The default Tape page options/packets posture is stock-only, hides `B` / `BB`, keeps calls and puts visible, and applies in-memory min-notional controls immediately. +- Live retention uses ClickHouse for durable server history, Redis for bounded hot cache, and browser state for rendering windows/preferences. +- Alert and drawer evidence is pinned and hydrated by id/trace so details remain inspectable after hot-window eviction. +- Firehose readiness keeps raw ingest for storage/replay, routes default compute/UI through filtered signals, and keeps subscription contracts ready for server-side selective delivery. - This repository is for personal, non-redistributed usage. ## Useful Examples diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index d9a291a..13c0ff4 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -727,12 +727,17 @@ h3 { grid-template-columns: repeat(3, minmax(0, 1fr)); } +.page-grid-news { + grid-template-columns: minmax(0, 1fr); +} + .page-grid-settings { grid-template-columns: minmax(0, 1.25fr) minmax(320px, 0.9fr); align-items: start; } .page-grid-home > :nth-child(3), +.page-grid-home > :nth-child(4), .page-grid-tape > :nth-child(1), .page-grid-replay > :nth-child(1) { grid-column: 1 / -1; @@ -961,6 +966,7 @@ h3 { } .page-grid-home > :nth-child(3), +.page-grid-home > :nth-child(4), .page-grid-replay > :not(:first-child) { height: clamp(430px, 58vh, 760px); } @@ -2094,6 +2100,72 @@ h3 { gap: 10px; } +.terminal-link-button { + text-decoration: none; +} + +.news-list { + display: flex; + flex-direction: column; + gap: 10px; +} + +.news-row { + width: 100%; + display: flex; + flex-direction: column; + gap: 8px; + padding: 14px 16px; + border: 1px solid var(--border); + border-radius: 12px; + background: oklch(0.18 0.012 250 / 0.6); + color: var(--text); + text-align: left; + transition: border-color 150ms ease, background 150ms ease; +} + +.news-row:hover { + border-color: var(--accent-soft); + background: oklch(0.2 0.015 250 / 0.75); +} + +.news-row-head, +.news-row-meta { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + flex-wrap: wrap; +} + +.news-row h3 { + margin: 0; + font-size: 0.96rem; + font-weight: 600; +} + +.news-row-time { + color: var(--text-dim); + font-family: var(--font-mono), monospace; + font-size: 0.78rem; +} + +.news-row-meta { + color: var(--text-dim); + font-size: 0.78rem; +} + +.news-drawer-body a { + color: var(--accent); +} + +.news-drawer-body p, +.news-drawer-body ul, +.news-drawer-body ol, +.news-drawer-body blockquote { + margin: 0 0 12px; +} + .synthetic-status-grid strong, .synthetic-hit-row strong { font-family: var(--font-mono), monospace; @@ -2312,6 +2384,7 @@ h3 { } .page-grid-home > :nth-child(3), + .page-grid-home > :nth-child(4), .page-grid-tape > :nth-child(1), .page-grid-replay > :nth-child(1) { grid-column: auto; @@ -2321,6 +2394,7 @@ h3 { .page-grid-home > :nth-child(1), .page-grid-home > :nth-child(2), .page-grid-home > :nth-child(3), + .page-grid-home > :nth-child(4), .page-grid-signals > .terminal-pane, .page-grid-replay > :not(:first-child), .page-grid-tape > :first-child, diff --git a/apps/web/app/news/page.tsx b/apps/web/app/news/page.tsx new file mode 100644 index 0000000..7e06aa8 --- /dev/null +++ b/apps/web/app/news/page.tsx @@ -0,0 +1,7 @@ +import { NewsRoute } from "../terminal"; + +export const dynamic = "force-dynamic"; + +export default function Page() { + return ; +} diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index 9297f1b..f039c95 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -43,6 +43,8 @@ import { shouldClearOptionFocusSeed, smartMoneyProfileLabel, smartMoneyToneForProfile, + getAlertFlowPacketRefs, + resolveAlertFlowPacket, statusLabel, toggleFilterValue } from "./terminal"; @@ -133,6 +135,33 @@ describe("alert context hydration helpers", () => { expect(evidence.prints.get("print:1")?.execution_nbbo_bid).toBe(1.2); expect(evidence.prints.get("print:1")?.execution_underlying_spot).toBe(450.05); }); + + it("finds flow-packet refs even when they are not first in alert evidence", () => { + const alert = makeAlert({ + evidence_refs: ["smartmoney:single_leg_event:flowpacket:1", "flowpacket:1", "print:1"] + }); + + expect(getAlertFlowPacketRefs(alert)).toEqual(["flowpacket:1"]); + }); + + it("resolves the primary alert flow packet from hydrated historical context", () => { + const packet = { + trace_id: "flowpacket:1", + id: "flowpacket:1", + members: ["print:1"], + source_ts: 1, + ingest_ts: 2, + seq: 1, + features: {}, + join_quality: {} + } as any; + const alert = makeAlert({ + evidence_refs: ["smartmoney:single_leg_event:flowpacket:1", "flowpacket:1", "print:1"] + }); + const packets = new Map([[packet.id, packet]]); + + expect(resolveAlertFlowPacket(alert, packets)).toBe(packet); + }); }); describe("live manifest", () => { @@ -247,6 +276,15 @@ describe("live manifest", () => { ]); }); + it("includes news subscriptions on home and /news", () => { + expect(getLiveManifest("/", "SPY", 60000, buildDefaultFlowFilters()).map((subscription) => subscription.channel)).toContain( + "news" + ); + expect(getLiveManifest("/news", "SPY", 60000, buildDefaultFlowFilters()).map((subscription) => subscription.channel)).toEqual([ + "news" + ]); + }); + it("scopes /charts subscriptions to chart channels only", () => { const channels = getLiveManifest("/charts", "SPY", 60000, buildDefaultFlowFilters()).map( (subscription) => subscription.channel @@ -432,6 +470,13 @@ describe("route feature map", () => { expect(features.alerts).toBe(false); }); + it("maps /news to the dedicated news pane", () => { + const features = getRouteFeatures("/news"); + expect(features.news).toBe(true); + expect(features.showNewsPane).toBe(true); + expect(features.showAlertsPane).toBe(false); + }); + it("maps /replay to replay panes and dependencies", () => { const features = getRouteFeatures("/replay"); expect(features.showReplayConsole).toBe(true); @@ -483,6 +528,7 @@ describe("terminal navigation", () => { expect(NAV_ITEMS).toEqual([ { href: "/", label: "Home" }, { href: "/tape", label: "Tape" }, + { href: "/news", label: "News" }, { href: "/signals", label: "Signals" }, { href: "/charts", label: "Charts" }, { href: "/replay", label: "Replay" }, diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index b7bfba1..c26a486 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -33,6 +33,7 @@ import type { LiveServerMessage, LiveHotChannelHealthMap, LiveSubscription, + NewsStory, OptionFlowFilters, OptionFlowView, OptionNbboSide, @@ -165,6 +166,7 @@ type RouteFeatures = { nbbo: boolean; equities: boolean; flow: boolean; + news: boolean; alerts: boolean; smartMoney: boolean; classifierHits: boolean; @@ -175,6 +177,7 @@ type RouteFeatures = { showOptionsPane: boolean; showEquitiesPane: boolean; showFlowPane: boolean; + showNewsPane: boolean; showAlertsPane: boolean; showClassifierPane: boolean; showDarkPane: boolean; @@ -194,6 +197,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { const includeEquitiesFallback = shouldIncludeEquitiesForDarkUnderlyingFallback(); const normalizedPath = pathname === "/tape" || + pathname === "/news" || pathname === "/signals" || pathname === "/charts" || pathname === "/replay" || @@ -208,6 +212,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { nbbo: true, equities: true, flow: true, + news: false, alerts: false, smartMoney: false, classifierHits: false, @@ -218,6 +223,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { showOptionsPane: true, showEquitiesPane: true, showFlowPane: true, + showNewsPane: false, showAlertsPane: false, showClassifierPane: false, showDarkPane: false, @@ -228,12 +234,41 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { needsAlertEvidencePrefetch: false, needsDarkUnderlying: false }; + case "/news": + return { + options: false, + nbbo: false, + equities: false, + flow: false, + news: true, + alerts: false, + smartMoney: false, + classifierHits: false, + inferredDark: false, + equityJoins: false, + equityCandles: false, + equityOverlay: false, + showOptionsPane: false, + showEquitiesPane: false, + showFlowPane: false, + showNewsPane: true, + showAlertsPane: false, + showClassifierPane: false, + showDarkPane: false, + showChartPane: false, + showFocusPane: false, + showReplayConsole: false, + needsClassifierDecor: false, + needsAlertEvidencePrefetch: false, + needsDarkUnderlying: false + }; case "/signals": return { options: false, nbbo: false, equities: includeEquitiesFallback, flow: false, + news: false, alerts: true, smartMoney: true, classifierHits: true, @@ -244,6 +279,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { showOptionsPane: false, showEquitiesPane: false, showFlowPane: false, + showNewsPane: false, showAlertsPane: true, showClassifierPane: true, showDarkPane: true, @@ -260,6 +296,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { nbbo: false, equities: includeEquitiesFallback, flow: false, + news: false, alerts: false, smartMoney: true, classifierHits: false, @@ -270,6 +307,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { showOptionsPane: false, showEquitiesPane: false, showFlowPane: false, + showNewsPane: false, showAlertsPane: false, showClassifierPane: false, showDarkPane: false, @@ -286,6 +324,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { nbbo: false, equities: false, flow: false, + news: false, alerts: false, smartMoney: false, classifierHits: false, @@ -296,6 +335,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { showOptionsPane: true, showEquitiesPane: false, showFlowPane: true, + showNewsPane: false, showAlertsPane: true, showClassifierPane: false, showDarkPane: false, @@ -312,6 +352,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { nbbo: false, equities: false, flow: false, + news: false, alerts: false, smartMoney: false, classifierHits: false, @@ -322,6 +363,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { showOptionsPane: false, showEquitiesPane: false, showFlowPane: false, + showNewsPane: false, showAlertsPane: false, showClassifierPane: false, showDarkPane: false, @@ -339,6 +381,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { nbbo: false, equities: true, flow: false, + news: true, alerts: true, smartMoney: true, classifierHits: false, @@ -349,6 +392,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { showOptionsPane: false, showEquitiesPane: true, showFlowPane: false, + showNewsPane: true, showAlertsPane: true, showClassifierPane: false, showDarkPane: false, @@ -366,6 +410,7 @@ const EMPTY_ALERT_EVENTS: AlertEvent[] = []; const EMPTY_CLASSIFIER_HIT_EVENTS: ClassifierHitEvent[] = []; const EMPTY_SMART_MONEY_EVENTS: SmartMoneyEvent[] = []; const EMPTY_INFERRED_DARK_EVENTS: InferredDarkEvent[] = []; +const EMPTY_NEWS_STORIES: NewsStory[] = []; type CandlestickSeries = ReturnType; @@ -1228,6 +1273,44 @@ const formatDateTime = (ts: number): string => { return `${date.toLocaleDateString()} ${date.toLocaleTimeString()}`; }; +const isSameLocalDay = (left: number, right: number): boolean => { + const a = new Date(left); + const b = new Date(right); + return ( + a.getFullYear() === b.getFullYear() && + a.getMonth() === b.getMonth() && + a.getDate() === b.getDate() + ); +}; + +export const formatNewsTimestamp = (ts: number, now = Date.now()): string => { + const date = new Date(ts); + return isSameLocalDay(ts, now) + ? date.toLocaleTimeString([], { hour: "numeric", minute: "2-digit" }) + : date.toLocaleString([], { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" }); +}; + +const sanitizeNewsHtml = (value: string): { html: string; fallbackText: string; sanitized: boolean } => { + const fallbackText = value + .replace(//gi, " ") + .replace(//gi, " ") + .replace(/<[^>]+>/g, " ") + .replace(/\s+/g, " ") + .trim(); + + try { + const sanitized = value + .replace(//gi, "") + .replace(//gi, "") + .replace(/\son\w+=(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, "") + .replace(/\shref=(["'])javascript:[\s\S]*?\1/gi, ' href="#"') + .replace(/<(?!\/?(p|div|section|article|span|strong|em|b|i|ul|ol|li|br|a|h1|h2|h3|h4|blockquote)\b)[^>]*>/gi, ""); + return { html: sanitized, fallbackText, sanitized: true }; + } catch { + return { html: "", fallbackText, sanitized: false }; + } +}; + const humanizeClassifierId = (value: string): string => { if (!value) { return "Classifier"; @@ -1702,7 +1785,7 @@ export const getOptionTableSnapshot = ( }; type ListScrollState = { - listRef: React.RefObject; + listRef: React.RefObject; listNode: HTMLDivElement | null; setListRef: (node: HTMLDivElement | null) => void; isAtTop: boolean; @@ -1807,7 +1890,7 @@ const useListScroll = (): ListScrollState => { }; const useScrollAnchor = ( - listRef: React.RefObject, + listRef: React.RefObject, isAtTopRef: React.MutableRefObject ) => { const pendingRef = useRef<{ @@ -1949,7 +2032,7 @@ type TapeVirtualRow = { const useTapeVirtualList = ( items: T[], - listRef: React.RefObject, + listRef: React.RefObject, config: TapeVirtualListConfig ): TapeVirtualListResult => { const virtualizer = useVirtualizer({ @@ -2904,6 +2987,7 @@ type LiveSessionState = { smartMoneyHistory: SmartMoneyEvent[]; classifierHitsHistory: ClassifierHitEvent[]; alertsHistory: AlertEvent[]; + newsHistory: NewsStory[]; inferredDarkHistory: InferredDarkEvent[]; options: OptionPrint[]; nbbo: OptionNBBO[]; @@ -2914,6 +2998,7 @@ type LiveSessionState = { smartMoney: SmartMoneyEvent[]; classifierHits: ClassifierHitEvent[]; alerts: AlertEvent[]; + news: NewsStory[]; inferredDark: InferredDarkEvent[]; chartCandles: EquityCandle[]; chartOverlay: EquityPrint[]; @@ -2934,6 +3019,7 @@ const LIVE_HISTORY_ENDPOINTS: Partial([]); const [classifierHits, setClassifierHits] = useState([]); const [alerts, setAlerts] = useState([]); + const [news, setNews] = useState([]); const [inferredDark, setInferredDark] = useState([]); const [optionsHistory, setOptionsHistory] = useState([]); const [nbboHistory, setNbboHistory] = useState([]); @@ -3176,6 +3266,7 @@ const useLiveSession = ( const [smartMoneyHistory, setSmartMoneyHistory] = useState([]); const [classifierHitsHistory, setClassifierHitsHistory] = useState([]); const [alertsHistory, setAlertsHistory] = useState([]); + const [newsHistory, setNewsHistory] = useState([]); const [inferredDarkHistory, setInferredDarkHistory] = useState([]); const [chartCandles, setChartCandles] = useState([]); const [chartOverlay, setChartOverlay] = useState([]); @@ -3188,6 +3279,7 @@ const useLiveSession = ( const smartMoneyRef = useRef([]); const classifierHitsRef = useRef([]); const alertsRef = useRef([]); + const newsRef = useRef([]); const inferredDarkRef = useRef([]); const chartCandlesRef = useRef([]); const chartOverlayRef = useRef([]); @@ -3199,6 +3291,7 @@ const useLiveSession = ( const smartMoneyHistoryRef = useRef([]); const classifierHitsHistoryRef = useRef([]); const alertsHistoryRef = useRef([]); + const newsHistoryRef = useRef([]); const inferredDarkHistoryRef = useRef([]); const socketRef = useRef(null); const reconnectRef = useRef(null); @@ -3252,6 +3345,7 @@ const useLiveSession = ( setSmartMoney([]); setClassifierHits([]); setAlerts([]); + setNews([]); setInferredDark([]); setOptionsHistory([]); setNbboHistory([]); @@ -3261,6 +3355,7 @@ const useLiveSession = ( setSmartMoneyHistory([]); setClassifierHitsHistory([]); setAlertsHistory([]); + setNewsHistory([]); setInferredDarkHistory([]); setChartCandles([]); setChartOverlay([]); @@ -3273,6 +3368,7 @@ const useLiveSession = ( smartMoneyRef.current = []; classifierHitsRef.current = []; alertsRef.current = []; + newsRef.current = []; inferredDarkRef.current = []; chartCandlesRef.current = []; chartOverlayRef.current = []; @@ -3284,6 +3380,7 @@ const useLiveSession = ( smartMoneyHistoryRef.current = []; classifierHitsHistoryRef.current = []; alertsHistoryRef.current = []; + newsHistoryRef.current = []; inferredDarkHistoryRef.current = []; subscribedKeysRef.current = new Set(); subscribedMapRef.current = new Map(); @@ -3437,6 +3534,12 @@ const useLiveSession = ( ref: alertsHistoryRef }); break; + case "news": + mergeItems(setNews, newsRef, items as NewsStory[], LIVE_OPTIONS_HEAD_LIMIT, { + setter: setNewsHistory, + ref: newsHistoryRef + }); + break; case "inferred-dark": mergeItems(setInferredDark, inferredDarkRef, items as InferredDarkEvent[], LIVE_HOT_WINDOW, { setter: setInferredDarkHistory, @@ -3728,6 +3831,9 @@ const useLiveSession = ( case "alerts": mergeOlder(setAlertsHistory, alertsHistoryRef, alertsRef.current); break; + case "news": + mergeOlder(setNewsHistory, newsHistoryRef, newsRef.current); + break; case "inferred-dark": mergeOlder(setInferredDarkHistory, inferredDarkHistoryRef, inferredDarkRef.current); break; @@ -3769,6 +3875,7 @@ const useLiveSession = ( smartMoneyHistory, classifierHitsHistory, alertsHistory, + newsHistory, inferredDarkHistory, options, nbbo, @@ -3779,6 +3886,7 @@ const useLiveSession = ( smartMoney, classifierHits, alerts, + news, inferredDark, chartCandles, chartOverlay @@ -4681,6 +4789,26 @@ export const collectAlertContextEvidence = ( return { packets, prints }; }; +export const getAlertFlowPacketRefs = ( + alert: Pick +): string[] => { + return alert.evidence_refs.filter((ref) => ref.startsWith("flowpacket:")); +}; + +export const resolveAlertFlowPacket = ( + alert: Pick, + packets: Map +): FlowPacket | null => { + for (const ref of getAlertFlowPacketRefs(alert)) { + const packet = packets.get(ref); + if (packet) { + return packet; + } + } + + return null; +}; + type DarkEvidenceItem = | { kind: "join"; id: string; join: EquityPrintJoin } | { kind: "unknown"; id: string }; @@ -4856,6 +4984,69 @@ const AlertDrawer = ({ alert, flowPacket, evidence, contextStatus, onClose }: Al ); }; +type NewsDrawerProps = { + story: NewsStory; + onClose: () => void; +}; + +const NewsDrawer = ({ story, onClose }: NewsDrawerProps) => { + const body = sanitizeNewsHtml(story.content_html); + + return ( + + ); +}; + type ClassifierHitDrawerProps = { hit: ClassifierHitEvent; flowPacket: FlowPacket | null; @@ -5222,6 +5413,7 @@ const useTerminalState = () => { const [mode, setMode] = useState("live"); const [replaySource, setReplaySource] = useState(null); const [selectedAlert, setSelectedAlert] = useState(null); + const [selectedNewsStory, setSelectedNewsStory] = useState(null); const [selectedDarkEvent, setSelectedDarkEvent] = useState(null); const [selectedClassifierHit, setSelectedClassifierHit] = useState(null); const [selectedSmartMoneyEvent, setSelectedSmartMoneyEvent] = useState(null); @@ -5318,12 +5510,13 @@ const useTerminalState = () => { }, [mode]); useEffect(() => { - if (!selectedAlert && !selectedClassifierHit && !selectedDarkEvent && !selectedSmartMoneyEvent) { + if (!selectedAlert && !selectedNewsStory && !selectedClassifierHit && !selectedDarkEvent && !selectedSmartMoneyEvent) { return; } const dismissDrawers = () => { setSelectedAlert(null); + setSelectedNewsStory(null); setSelectedClassifierHit(null); setSelectedSmartMoneyEvent(null); setSelectedDarkEvent(null); @@ -5349,7 +5542,7 @@ const useTerminalState = () => { document.removeEventListener("mousedown", handlePointerDown); document.removeEventListener("keydown", handleKeyDown); }; - }, [selectedAlert, selectedClassifierHit, selectedDarkEvent, selectedSmartMoneyEvent]); + }, [selectedAlert, selectedNewsStory, selectedClassifierHit, selectedDarkEvent, selectedSmartMoneyEvent]); const optionsScroll = useListScroll(); const equitiesScroll = useListScroll(); @@ -5584,6 +5777,14 @@ const useTerminalState = () => { ) : equityJoins; const flowFeed = mode === "live" ? liveFlow : flow; + const newsFeed = + mode === "live" + ? toStaticTapeState( + liveSession.status, + composeTapeItems([], liveSession.news, liveSession.newsHistory), + liveSession.lastUpdate + ) + : toStaticTapeState("disconnected", [], null); const alertsFeed = mode === "live" ? toStaticTapeState( @@ -5879,8 +6080,7 @@ const useTerminalState = () => { if (!selectedAlert) { return null; } - const packetId = selectedAlert.evidence_refs[0]; - return packetId ? resolvedFlowPacketMap.get(packetId) ?? null : null; + return resolveAlertFlowPacket(selectedAlert, resolvedFlowPacketMap); }, [selectedAlert, resolvedFlowPacketMap]); const selectedDarkEvidence = useMemo((): DarkEvidenceItem[] => { @@ -6307,12 +6507,9 @@ const useTerminalState = () => { return fromTrace; } - const packetId = alert.evidence_refs[0]; - if (packetId) { - const packet = resolvedFlowPacketMap.get(packetId); - if (packet) { - return extractUnderlying(extractPacketContract(packet)); - } + const packet = resolveAlertFlowPacket(alert, resolvedFlowPacketMap); + if (packet) { + return extractUnderlying(extractPacketContract(packet)); } for (const ref of alert.evidence_refs) { @@ -6549,6 +6746,16 @@ const useTerminalState = () => { routeFeatures.needsAlertEvidencePrefetch ]); + const filteredNews = useMemo(() => { + if (!routeFeatures.news && !routeFeatures.showNewsPane) { + return EMPTY_NEWS_STORIES; + } + if (tickerSet.size === 0) { + return newsFeed.items; + } + return newsFeed.items.filter((story) => story.resolved_symbols.some((symbol) => matchesTicker(symbol))); + }, [matchesTicker, newsFeed.items, routeFeatures.news, routeFeatures.showNewsPane, tickerSet]); + const visibleAlerts = useMemo(() => { if (routeFeatures.needsAlertEvidencePrefetch) { return filteredAlerts.slice(0, 12); @@ -6574,9 +6781,7 @@ const useTerminalState = () => { return; } - const visiblePacketIds = visibleAlerts - .map((alert) => alert.evidence_refs[0] ?? null) - .filter((id): id is string => Boolean(id) && id.startsWith("flowpacket:")); + const visiblePacketIds = visibleAlerts.flatMap((alert) => getAlertFlowPacketRefs(alert)); const missingPacketIds = Array.from(new Set(visiblePacketIds)).filter( (id) => !resolvedFlowPacketMap.has(id) ); @@ -6658,9 +6863,10 @@ const useTerminalState = () => { const activePinnedFlowKeys = useMemo(() => { const keys = new Set(); - const selectedAlertPacketId = selectedAlert?.evidence_refs[0]; - if (selectedAlertPacketId) { - keys.add(selectedAlertPacketId); + if (selectedAlert) { + for (const packetId of getAlertFlowPacketRefs(selectedAlert)) { + keys.add(packetId); + } } if (selectedClassifierPacketId) { keys.add(selectedClassifierPacketId); @@ -6669,8 +6875,7 @@ const useTerminalState = () => { keys.add(packetId); } for (const alert of visibleAlerts) { - const packetId = alert.evidence_refs[0]; - if (packetId) { + for (const packetId of getAlertFlowPacketRefs(alert)) { keys.add(packetId); } } @@ -6815,7 +7020,7 @@ const useTerminalState = () => { const desiredTrace = `alert:${packetId}`; return ( alertsFeed.items.find( - (item) => item.trace_id === desiredTrace || item.evidence_refs[0] === packetId + (item) => item.trace_id === desiredTrace || getAlertFlowPacketRefs(item).includes(packetId) ) ?? null ); }, @@ -6826,6 +7031,7 @@ const useTerminalState = () => { (hit: ClassifierHitEvent) => { const alert = findAlertForClassifierHit(hit); if (alert) { + setSelectedNewsStory(null); setSelectedClassifierHit(null); setSelectedDarkEvent(null); setSelectedSmartMoneyEvent(null); @@ -6833,6 +7039,7 @@ const useTerminalState = () => { return; } + setSelectedNewsStory(null); setSelectedAlert(null); setSelectedDarkEvent(null); setSelectedSmartMoneyEvent(null); @@ -6842,6 +7049,7 @@ const useTerminalState = () => { ); const openFromSmartMoneyEvent = useCallback((event: SmartMoneyEvent) => { + setSelectedNewsStory(null); setSelectedAlert(null); setSelectedClassifierHit(null); setSelectedDarkEvent(null); @@ -6856,6 +7064,7 @@ const useTerminalState = () => { ); const handleDarkMarkerClick = useCallback((event: InferredDarkEvent) => { + setSelectedNewsStory(null); setSelectedAlert(null); setSelectedClassifierHit(null); setSelectedSmartMoneyEvent(null); @@ -6876,6 +7085,9 @@ const useTerminalState = () => { if (routeFeatures.flow || routeFeatures.showFlowPane) { updates.push(flowFeed.lastUpdate); } + if (routeFeatures.news || routeFeatures.showNewsPane) { + updates.push(newsFeed.lastUpdate); + } if (routeFeatures.alerts || routeFeatures.showAlertsPane) { updates.push(alertsFeed.lastUpdate); } @@ -6898,6 +7110,8 @@ const useTerminalState = () => { routeFeatures.showFocusPane, routeFeatures.flow, routeFeatures.showFlowPane, + routeFeatures.news, + routeFeatures.showNewsPane, routeFeatures.alerts, routeFeatures.showAlertsPane, routeFeatures.smartMoney, @@ -6908,6 +7122,7 @@ const useTerminalState = () => { equitiesFeed.lastUpdate, inferredDarkFeed.lastUpdate, flowFeed.lastUpdate, + newsFeed.lastUpdate, alertsFeed.lastUpdate, smartMoneyFeed.lastUpdate, classifierHitsFeed.lastUpdate @@ -6920,6 +7135,8 @@ const useTerminalState = () => { setReplaySource, selectedAlert, setSelectedAlert, + selectedNewsStory, + setSelectedNewsStory, selectedDarkEvent, setSelectedDarkEvent, selectedClassifierHit, @@ -6946,6 +7163,7 @@ const useTerminalState = () => { equityJoins: equityJoinsFeed, nbbo: nbboFeed, inferredDark: inferredDarkFeed, + news: newsFeed, flow: flowFeed, alerts: alertsFeed, smartMoney: smartMoneyFeed, @@ -6980,6 +7198,7 @@ const useTerminalState = () => { equitiesScopedQuiet, equitiesSilentWarning, filteredInferredDark, + filteredNews, filteredFlow, filteredAlerts, filteredSmartMoneyEvents, @@ -7014,6 +7233,7 @@ const useTerminal = (): TerminalState => { export const NAV_ITEMS = [ { href: "/", label: "Home" }, { href: "/tape", label: "Tape" }, + { href: "/news", label: "News" }, { href: "/signals", label: "Signals" }, { href: "/charts", label: "Charts" }, { href: "/replay", label: "Replay" }, @@ -7844,6 +8064,7 @@ const AlertsPane = memo(({ state, limit, withStrip = false, className }: AlertsP data-tape-key={key} style={{ transform: `translateY(${start}px)` }} onClick={() => { + state.setSelectedNewsStory(null); state.setSelectedDarkEvent(null); state.setSelectedClassifierHit(null); state.setSelectedSmartMoneyEvent(null); @@ -7870,6 +8091,83 @@ const AlertsPane = memo(({ state, limit, withStrip = false, className }: AlertsP ); }); +type NewsPaneProps = { + state: TerminalState; + limit?: number; + className?: string; +}; + +const NewsPane = memo(({ state, limit, className }: NewsPaneProps) => { + const items = limit ? state.filteredNews.slice(0, limit) : state.filteredNews; + const canLoadOlder = state.mode === "live" && !limit && items.length > 0; + + return ( + + View all + + ) : ( +
+ + {state.mode === "live" ? "Live wire" : "Live-only in v1"} +
+ ) + } + actions={ + canLoadOlder ? ( + + ) : null + } + > + {state.mode === "replay" ? ( +
News is live-only in v1.
+ ) : items.length === 0 ? ( +
+ {state.tickerSet.size > 0 ? "No news stories match the current filter." : "Waiting for live news stories."} +
+ ) : ( +
+ {items.map((story) => ( + + ))} +
+ )} + + ); +}); + type ClassifierPaneProps = { state: TerminalState; limit?: number; @@ -8080,6 +8378,7 @@ const DarkPane = memo(({ state, limit, className }: DarkPaneProps) => { data-tape-key={key} style={{ transform: `translateY(${start}px)` }} onClick={() => { + state.setSelectedNewsStory(null); state.setSelectedAlert(null); state.setSelectedClassifierHit(null); state.setSelectedSmartMoneyEvent(null); @@ -8720,6 +9019,10 @@ export function TerminalAppShell({ children }: { children: ReactNode }) { /> ) : null} + {state.selectedNewsStory ? ( + state.setSelectedNewsStory(null)} /> + ) : null} + {state.selectedClassifierHit ? ( + ); } +export function NewsRoute() { + const state = useTerminal(); + return ( + +
+ +
+ + ); +} + export function TapeRoute() { const state = useTerminal(); return ( diff --git a/apps/web/package.json b/apps/web/package.json index 8ab6906..5179852 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -5,19 +5,20 @@ "scripts": { "dev": "bun run scripts/dev.ts", "build": "next build", - "start": "next start -p 3000" + "start": "next start" }, "dependencies": { "@islandflow/types": "workspace:*", "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", - "next": "^14.2.4", - "react": "^18.3.1", - "react-dom": "^18.3.1" + "next": "^16.2.6", + "react": "^19.2.0", + "react-dom": "^19.2.0" }, "devDependencies": { "@types/node": "^20.14.10", - "@types/react": "^18.3.3", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", "typescript": "^5.5.4" } } diff --git a/bun.lock b/bun.lock index d312fe7..b76fe82 100644 --- a/bun.lock +++ b/bun.lock @@ -29,13 +29,14 @@ "@islandflow/types": "workspace:*", "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", - "next": "^14.2.4", - "react": "^18.3.1", - "react-dom": "^18.3.1", + "next": "^16.2.6", + "react": "^19.2.0", + "react-dom": "^19.2.0", }, "devDependencies": { "@types/node": "^20.14.10", - "@types/react": "^18.3.3", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", "typescript": "^5.5.4", }, }, @@ -124,6 +125,17 @@ "zod": "^3.23.8", }, }, + "services/ingest-news": { + "name": "@islandflow/ingest-news", + "dependencies": { + "@islandflow/bus": "workspace:*", + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + "@islandflow/types": "workspace:*", + "ws": "^8.18.3", + "zod": "^3.23.8", + }, + }, "services/ingest-options": { "name": "@islandflow/ingest-options", "dependencies": { @@ -207,8 +219,60 @@ "@electron/windows-sign": ["@electron/windows-sign@1.2.2", "", { "dependencies": { "cross-dirname": "^0.1.0", "debug": "^4.3.4", "fs-extra": "^11.1.1", "minimist": "^1.2.8", "postject": "^1.0.0-alpha.6" }, "bin": { "electron-windows-sign": "bin/electron-windows-sign.js" } }, "sha512-dfZeox66AvdPtb2lD8OsIIQh12Tp0GNCRUDfBHIKGpbmopZto2/A8nSpYYLoedPIHpqkeblZ/k8OV0Gy7PYuyQ=="], + "@emnapi/runtime": ["@emnapi/runtime@1.10.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="], + "@gar/promisify": ["@gar/promisify@1.1.3", "", {}, "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw=="], + "@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.2.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="], + + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.2.4", "", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.2.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="], + + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="], + + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="], + + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], + "@inquirer/checkbox": ["@inquirer/checkbox@3.0.1", "", { "dependencies": { "@inquirer/core": "^9.2.1", "@inquirer/figures": "^1.0.6", "@inquirer/type": "^2.0.0", "ansi-escapes": "^4.3.2", "yoctocolors-cjs": "^2.1.2" } }, "sha512-0hm2nrToWUdD6/UHnel/UKGdk1//ke5zGUpHIvk5ZWmaKezlGxZkOJXNSWsdxO/rEqTkbB3lNC2J6nBElV2aAQ=="], "@inquirer/confirm": ["@inquirer/confirm@4.0.1", "", { "dependencies": { "@inquirer/core": "^9.2.1", "@inquirer/type": "^2.0.0" } }, "sha512-46yL28o2NJ9doViqOy0VDcoTzng7rAb6yPQKU7VDLqkmbCaH4JqK4yk4XqlzNWy9PVC5pG1ZUXPBQv+VqnYs2w=="], @@ -253,6 +317,8 @@ "@islandflow/ingest-equities": ["@islandflow/ingest-equities@workspace:services/ingest-equities"], + "@islandflow/ingest-news": ["@islandflow/ingest-news@workspace:services/ingest-news"], + "@islandflow/ingest-options": ["@islandflow/ingest-options@workspace:services/ingest-options"], "@islandflow/observability": ["@islandflow/observability@workspace:packages/observability"], @@ -283,25 +349,23 @@ "@msgpack/msgpack": ["@msgpack/msgpack@3.1.3", "", {}, "sha512-47XIizs9XZXvuJgoaJUIE2lFoID8ugvc0jzSHP+Ptfk8nTbnR8g788wv48N03Kx0UkAv559HWRQ3yzOgzlRNUA=="], - "@next/env": ["@next/env@14.2.35", "", {}, "sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ=="], + "@next/env": ["@next/env@16.2.6", "", {}, "sha512-gd8HoHN4ufj73WmR3JmVolrpJR47ILK6LouP5xElPglaVxir6e1a7VzvTvDWkOoPXT9rkkTzyCxBu4yeZfZwcw=="], - "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@14.2.33", "", { "os": "darwin", "cpu": "arm64" }, "sha512-HqYnb6pxlsshoSTubdXKu15g3iivcbsMXg4bYpjL2iS/V6aQot+iyF4BUc2qA/J/n55YtvE4PHMKWBKGCF/+wA=="], + "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@16.2.6", "", { "os": "darwin", "cpu": "arm64" }, "sha512-ZJGkkcNfYgrrMkqOdZ7zoLa1TOy0qpcMfk/z4Mh/FKUz40gVO+HNQWqmLxf67Z5WB64DRp0dhEbyHfel+6sJUg=="], - "@next/swc-darwin-x64": ["@next/swc-darwin-x64@14.2.33", "", { "os": "darwin", "cpu": "x64" }, "sha512-8HGBeAE5rX3jzKvF593XTTFg3gxeU4f+UWnswa6JPhzaR6+zblO5+fjltJWIZc4aUalqTclvN2QtTC37LxvZAA=="], + "@next/swc-darwin-x64": ["@next/swc-darwin-x64@16.2.6", "", { "os": "darwin", "cpu": "x64" }, "sha512-v/YLBHIY132Ced3puBJ7YJKw1lqsCrgcNo2aRJlCEyQrrCeRJlvGlnmxhPxNQI3KE3N1DN5r9TPNPvka3nq5RQ=="], - "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@14.2.33", "", { "os": "linux", "cpu": "arm64" }, "sha512-JXMBka6lNNmqbkvcTtaX8Gu5by9547bukHQvPoLe9VRBx1gHwzf5tdt4AaezW85HAB3pikcvyqBToRTDA4DeLw=="], + "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@16.2.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-RPOvqlYBbcQjkz9VQQDZ2T2bARIjXZV1KFlt+V2Mr6SW/e4I9fcKsaA0hdyf2FHoTlsV2xnBd5Y912rP/1Ce6w=="], - "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@14.2.33", "", { "os": "linux", "cpu": "arm64" }, "sha512-Bm+QulsAItD/x6Ih8wGIMfRJy4G73tu1HJsrccPW6AfqdZd0Sfm5Imhgkgq2+kly065rYMnCOxTBvmvFY1BKfg=="], + "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@16.2.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-URUTu1+dMkxJsPFgm+OeEvq9wf5sujw0EvgYy80TDGHTSLTnIHeqb0Eu8A3sC95IRgjejQL+kC4mw+4yPxiAXA=="], - "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@14.2.33", "", { "os": "linux", "cpu": "x64" }, "sha512-FnFn+ZBgsVMbGDsTqo8zsnRzydvsGV8vfiWwUo1LD8FTmPTdV+otGSWKc4LJec0oSexFnCYVO4hX8P8qQKaSlg=="], + "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@16.2.6", "", { "os": "linux", "cpu": "x64" }, "sha512-DOj182mPV8G3UkrayLoREM5YEYI+Dk5wv7Ox9xl1fFibAELEsFD0lDPfHIeILlutMMfdyhlzYPELG3peuKaurw=="], - "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@14.2.33", "", { "os": "linux", "cpu": "x64" }, "sha512-345tsIWMzoXaQndUTDv1qypDRiebFxGYx9pYkhwY4hBRaOLt8UGfiWKr9FSSHs25dFIf8ZqIFaPdy5MljdoawA=="], + "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@16.2.6", "", { "os": "linux", "cpu": "x64" }, "sha512-HKQ5SP/V/ub73UvF7n/zeJlxk2kLmtL7Wzrg4WfmkjmNos5onJ2tKu7yZOPdL18A6Svfn3max29ym+ry7NkK4g=="], - "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@14.2.33", "", { "os": "win32", "cpu": "arm64" }, "sha512-nscpt0G6UCTkrT2ppnJnFsYbPDQwmum4GNXYTeoTIdsmMydSKFz9Iny2jpaRupTb+Wl298+Rh82WKzt9LCcqSQ=="], + "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@16.2.6", "", { "os": "win32", "cpu": "arm64" }, "sha512-LZXpTlPyS5v7HhSmnvsLGP3iIYgYOBnc8r8ArlT55sGHV89bR2HlDdBjWQ+PY6SJMmk8TuVGFuxalnP3k/0Dwg=="], - "@next/swc-win32-ia32-msvc": ["@next/swc-win32-ia32-msvc@14.2.33", "", { "os": "win32", "cpu": "ia32" }, "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q=="], - - "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@14.2.33", "", { "os": "win32", "cpu": "x64" }, "sha512-nOjfZMy8B94MdisuzZo9/57xuFVLHJaDj5e/xrduJp9CV2/HrfxTRH2fbyLe+K9QT41WBLUd4iXX3R7jBp0EUg=="], + "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@16.2.6", "", { "os": "win32", "cpu": "x64" }, "sha512-F0+4i0h9J6C4eE3EAPWsoCk7UW/dbzOjyzxY0qnDUOYFu6FFmdZ6l97/XdV3/Nz3VYyO7UWjyEJUXkGqcoXfMA=="], "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], @@ -325,9 +389,7 @@ "@sindresorhus/is": ["@sindresorhus/is@4.6.0", "", {}, "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw=="], - "@swc/counter": ["@swc/counter@0.1.3", "", {}, "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="], - - "@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="], + "@swc/helpers": ["@swc/helpers@0.5.15", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g=="], "@szmarczak/http-timer": ["@szmarczak/http-timer@4.0.6", "", { "dependencies": { "defer-to-connect": "^2.0.0" } }, "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w=="], @@ -355,9 +417,9 @@ "@types/node": ["@types/node@24.12.4", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GUUEShf+PBCGW2KaXwcIt3Yk+e3pkKwWKb9GSyM9WQVE+ep2jzmHdGsHzu4wgcZy5fN9FBdVzjpBQsYlpfpgLA=="], - "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], + "@types/react": ["@types/react@19.2.14", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w=="], - "@types/react": ["@types/react@18.3.27", "", { "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" } }, "sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w=="], + "@types/react-dom": ["@types/react-dom@19.2.3", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="], "@types/responselike": ["@types/responselike@1.0.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw=="], @@ -455,15 +517,13 @@ "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], - "busboy": ["busboy@1.6.0", "", { "dependencies": { "streamsearch": "^1.1.0" } }, "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA=="], - "cacache": ["cacache@16.1.3", "", { "dependencies": { "@npmcli/fs": "^2.1.0", "@npmcli/move-file": "^2.0.0", "chownr": "^2.0.0", "fs-minipass": "^2.1.0", "glob": "^8.0.1", "infer-owner": "^1.0.4", "lru-cache": "^7.7.1", "minipass": "^3.1.6", "minipass-collect": "^1.0.2", "minipass-flush": "^1.0.5", "minipass-pipeline": "^1.2.4", "mkdirp": "^1.0.4", "p-map": "^4.0.0", "promise-inflight": "^1.0.1", "rimraf": "^3.0.2", "ssri": "^9.0.0", "tar": "^6.1.11", "unique-filename": "^2.0.0" } }, "sha512-/+Emcj9DAXxX4cwlLmRI9c166RuL3w30zp4R7Joiv2cQTtTtA+jeuCAjH3ZlGnYS3tKENSrKhAzVVP9GVyzeYQ=="], "cacheable-lookup": ["cacheable-lookup@5.0.4", "", {}, "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA=="], "cacheable-request": ["cacheable-request@7.0.4", "", { "dependencies": { "clone-response": "^1.0.2", "get-stream": "^5.1.0", "http-cache-semantics": "^4.0.0", "keyv": "^4.0.0", "lowercase-keys": "^2.0.0", "normalize-url": "^6.0.1", "responselike": "^2.0.0" } }, "sha512-v+p6ongsrp0yTGbJXjgxPow2+DL93DASP4kXCDKb8/bwRtt9OEF3whggkkDkGNzgcWy2XaF4a8nZglC7uElscg=="], - "caniuse-lite": ["caniuse-lite@1.0.30001761", "", {}, "sha512-JF9ptu1vP2coz98+5051jZ4PwQgd2ni8A+gYSN7EA7dPKIMf0pDlSUxhdmVOaV3/fYK5uWBkgSXJaRLr4+3A6g=="], + "caniuse-lite": ["caniuse-lite@1.0.30001792", "", {}, "sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw=="], "chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], @@ -703,8 +763,6 @@ "jiti": ["jiti@2.7.0", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="], - "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], - "json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="], "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], @@ -735,8 +793,6 @@ "log-update": ["log-update@5.0.1", "", { "dependencies": { "ansi-escapes": "^5.0.0", "cli-cursor": "^4.0.0", "slice-ansi": "^5.0.0", "strip-ansi": "^7.0.1", "wrap-ansi": "^8.0.1" } }, "sha512-5UtUDQ/6edw4ofyljDNcOVJQ4c7OjDro4h3y8e1GQL5iYElYclVHJ3zeWchylvMaKnDbDilC8irOVyexnA/Slw=="], - "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], - "lowercase-keys": ["lowercase-keys@2.0.0", "", {}, "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="], "lru-cache": ["lru-cache@7.18.3", "", {}, "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="], @@ -793,7 +849,7 @@ "neo-async": ["neo-async@2.6.2", "", {}, "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="], - "next": ["next@14.2.35", "", { "dependencies": { "@next/env": "14.2.35", "@swc/helpers": "0.5.5", "busboy": "1.6.0", "caniuse-lite": "^1.0.30001579", "graceful-fs": "^4.2.11", "postcss": "8.4.31", "styled-jsx": "5.1.1" }, "optionalDependencies": { "@next/swc-darwin-arm64": "14.2.33", "@next/swc-darwin-x64": "14.2.33", "@next/swc-linux-arm64-gnu": "14.2.33", "@next/swc-linux-arm64-musl": "14.2.33", "@next/swc-linux-x64-gnu": "14.2.33", "@next/swc-linux-x64-musl": "14.2.33", "@next/swc-win32-arm64-msvc": "14.2.33", "@next/swc-win32-ia32-msvc": "14.2.33", "@next/swc-win32-x64-msvc": "14.2.33" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.41.2", "react": "^18.2.0", "react-dom": "^18.2.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-KhYd2Hjt/O1/1aZVX3dCwGXM1QmOV4eNM2UTacK5gipDdPN/oHHK/4oVGy7X8GMfPMsUTUEmGlsy0EY1YGAkig=="], + "next": ["next@16.2.6", "", { "dependencies": { "@next/env": "16.2.6", "@swc/helpers": "0.5.15", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", "styled-jsx": "5.1.6" }, "optionalDependencies": { "@next/swc-darwin-arm64": "16.2.6", "@next/swc-darwin-x64": "16.2.6", "@next/swc-linux-arm64-gnu": "16.2.6", "@next/swc-linux-arm64-musl": "16.2.6", "@next/swc-linux-x64-gnu": "16.2.6", "@next/swc-linux-x64-musl": "16.2.6", "@next/swc-win32-arm64-msvc": "16.2.6", "@next/swc-win32-x64-msvc": "16.2.6", "sharp": "^0.34.5" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "babel-plugin-react-compiler", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-qOVgKJg1+At15NpeUP+eJgCHvTCgXsogweq87Ri/Ix7PkqQHg4sdaXmSFqKlgaIXE4kW0g25LE68W87UANlHtw=="], "nice-try": ["nice-try@1.0.5", "", {}, "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ=="], @@ -887,9 +943,9 @@ "quick-lru": ["quick-lru@5.1.1", "", {}, "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA=="], - "react": ["react@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="], + "react": ["react@19.2.6", "", {}, "sha512-sfWGGfavi0xr8Pg0sVsyHMAOziVYKgPLNrS7ig+ivMNb3wbCBw3KxtflsGBAwD3gYQlE/AEZsTLgToRrSCjb0Q=="], - "react-dom": ["react-dom@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" }, "peerDependencies": { "react": "^18.3.1" } }, "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw=="], + "react-dom": ["react-dom@19.2.6", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.6" } }, "sha512-0prMI+hvBbPjsWnxDLxlCGyM8PN6UuWjEUCYmZhO67xIV9Xasa/r/vDnq+Xyq4Lo27g8QSbO5YzARu0D1Sps3g=="], "read-binary-file-arch": ["read-binary-file-arch@1.0.6", "", { "dependencies": { "debug": "^4.3.4" }, "bin": { "read-binary-file-arch": "cli.js" } }, "sha512-BNg9EN3DD3GsDXX7Aa8O4p92sryjkmzYYgmgTAc6CA4uGLEDzFfxOxugu21akOxpcXHiEgsYkC6nPsQvLLLmEg=="], @@ -933,7 +989,7 @@ "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], - "scheduler": ["scheduler@0.23.2", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ=="], + "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], "schema-utils": ["schema-utils@4.3.3", "", { "dependencies": { "@types/json-schema": "^7.0.9", "ajv": "^8.9.0", "ajv-formats": "^2.1.1", "ajv-keywords": "^5.1.0" } }, "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA=="], @@ -943,6 +999,8 @@ "serialize-error": ["serialize-error@7.0.1", "", { "dependencies": { "type-fest": "^0.13.1" } }, "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw=="], + "sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], @@ -975,8 +1033,6 @@ "ssri": ["ssri@9.0.1", "", { "dependencies": { "minipass": "^3.1.1" } }, "sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q=="], - "streamsearch": ["streamsearch@1.1.0", "", {}, "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg=="], - "string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], @@ -989,7 +1045,7 @@ "strip-outer": ["strip-outer@1.0.1", "", { "dependencies": { "escape-string-regexp": "^1.0.2" } }, "sha512-k55yxKHwaXnpYGsOzg4Vl8+tDrWylxDEpknGjhTiZB8dFRU5rTo9CAzeycivxV3s+zlTKwrs6WxMxR95n26kwg=="], - "styled-jsx": ["styled-jsx@5.1.1", "", { "dependencies": { "client-only": "0.0.1" }, "peerDependencies": { "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0" } }, "sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw=="], + "styled-jsx": ["styled-jsx@5.1.6", "", { "dependencies": { "client-only": "0.0.1" }, "peerDependencies": { "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0" } }, "sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA=="], "sumchecker": ["sumchecker@3.0.1", "", { "dependencies": { "debug": "^4.1.0" } }, "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg=="], @@ -1131,8 +1187,6 @@ "ansi-escapes/type-fest": ["type-fest@0.21.3", "", {}, "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w=="], - "browserslist/caniuse-lite": ["caniuse-lite@1.0.30001792", "", {}, "sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw=="], - "cacache/glob": ["glob@8.1.0", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^5.0.1", "once": "^1.3.0" } }, "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ=="], "cacache/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="], diff --git a/deployment/docker/.dockerignore b/deployment/docker/.dockerignore new file mode 100644 index 0000000..8fd5de7 --- /dev/null +++ b/deployment/docker/.dockerignore @@ -0,0 +1,23 @@ +.git +.github +.DS_Store +.bun +.tmp +node_modules +dist +coverage +logs +apps/web/.next +.env +.env.* +session-ses_*.md +token-usage-output.txt +signal-cli-*.tar.gz +*.tar +*.tar.gz +*.tgz +*.zip +__pycache__ +.pytest_cache +!.env.example +!**/.env.example diff --git a/deployment/docker/.env.example b/deployment/docker/.env.example index eee9cef..4972ada 100644 --- a/deployment/docker/.env.example +++ b/deployment/docker/.env.example @@ -4,8 +4,10 @@ NATS_URL=nats://nats:4222 CLICKHOUSE_URL=http://clickhouse:8123 CLICKHOUSE_DATABASE=default REDIS_URL=redis://redis:6379 +ISLANDFLOW_DATA_ROOT=/var/lib/islandflow API_PORT=4000 +API_HOST=0.0.0.0 API_BIND_IP=127.0.0.1 API_HOST_PORT=4000 WEB_BIND_IP=127.0.0.1 @@ -25,6 +27,10 @@ NEXT_PUBLIC_NBBO_MAX_AGE_MS=1000 # Options ingest OPTIONS_INGEST_ADAPTER=synthetic ALPACA_API_KEY= +ALPACA_API_KEY_ID= +ALPACA_KEY_ID= +ALPACA_API_SECRET_KEY= +ALPACA_SECRET_KEY= ALPACA_REST_URL=https://data.alpaca.markets ALPACA_WS_BASE_URL=wss://stream.data.alpaca.markets/v1beta1 ALPACA_FEED=indicative diff --git a/deployment/docker/Dockerfile.ingest-options b/deployment/docker/Dockerfile.ingest-options index 52cba59..212b96b 100644 --- a/deployment/docker/Dockerfile.ingest-options +++ b/deployment/docker/Dockerfile.ingest-options @@ -31,6 +31,7 @@ COPY --from=services candles/package.json ./services/candles/package.json COPY --from=services compute/package.json ./services/compute/package.json COPY --from=services eod-enricher/package.json ./services/eod-enricher/package.json COPY --from=services ingest-equities/package.json ./services/ingest-equities/package.json +COPY --from=services ingest-news/package.json ./services/ingest-news/package.json COPY --from=services ingest-options/package.json ./services/ingest-options/package.json COPY --from=services ingest-options/py/requirements.txt ./services/ingest-options/py/requirements.txt COPY --from=services refdata/package.json ./services/refdata/package.json diff --git a/deployment/docker/Dockerfile.service b/deployment/docker/Dockerfile.service index e0fcf72..4a7d9f1 100644 --- a/deployment/docker/Dockerfile.service +++ b/deployment/docker/Dockerfile.service @@ -24,6 +24,7 @@ COPY --from=services candles/package.json ./services/candles/package.json COPY --from=services compute/package.json ./services/compute/package.json COPY --from=services eod-enricher/package.json ./services/eod-enricher/package.json COPY --from=services ingest-equities/package.json ./services/ingest-equities/package.json +COPY --from=services ingest-news/package.json ./services/ingest-news/package.json COPY --from=services ingest-options/package.json ./services/ingest-options/package.json COPY --from=services refdata/package.json ./services/refdata/package.json COPY --from=services replay/package.json ./services/replay/package.json diff --git a/deployment/docker/Dockerfile.web b/deployment/docker/Dockerfile.web index 33723ae..54dbd16 100644 --- a/deployment/docker/Dockerfile.web +++ b/deployment/docker/Dockerfile.web @@ -30,6 +30,7 @@ COPY --from=services candles/package.json ./services/candles/package.json COPY --from=services compute/package.json ./services/compute/package.json COPY --from=services eod-enricher/package.json ./services/eod-enricher/package.json COPY --from=services ingest-equities/package.json ./services/ingest-equities/package.json +COPY --from=services ingest-news/package.json ./services/ingest-news/package.json COPY --from=services ingest-options/package.json ./services/ingest-options/package.json COPY --from=services refdata/package.json ./services/refdata/package.json COPY --from=services replay/package.json ./services/replay/package.json @@ -59,4 +60,4 @@ COPY --from=build /app/packages ./packages EXPOSE 3000 -CMD ["bun", "run", "--cwd", "apps/web", "start"] +CMD ["bun", "run", "--cwd", "apps/web", "start", "--", "-H", "0.0.0.0", "-p", "3000"] diff --git a/deployment/docker/README.md b/deployment/docker/README.md index 2b167da..644798b 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -2,12 +2,12 @@ This directory contains the Docker runtime for Islandflow VPS deployments. -Docker remains the default and recommended server rollout path, but the repo-root `deploy` helper can now target either: +Docker remains the default rollout path before native cutover and the rollback path after cutover. The repo-root `deploy` helper can target either: - `--runtime docker` for this Docker Compose stack -- `--runtime native` for an experimental host-native Bun + systemd rollout described in `deployment/native/README.md` +- `--runtime native` for the host-native Bun + systemd rollout described in `deployment/native/README.md` -The repo no longer ships or supports a separate `deployment/npm` stack. If you want a reverse proxy, point it at the host ports published by this stack. +The public VPS edge remains Nginx Proxy Manager. Docker fallback can be reached either through the shared Docker network service names or the host ports published by this stack. It is separate from the repo-root `docker-compose.yml`, which remains the lightweight local infra stack for development. @@ -17,7 +17,7 @@ Do not run the repo-root `docker-compose.yml` on the VPS. On the live server tha - Builds and runs the full Islandflow stack with Docker Compose. - Publishes `web` and `api` to host ports, bound to loopback by default. -- Runs ClickHouse, Redis, and NATS JetStream with persistent Docker volumes. +- Runs ClickHouse, Redis, and NATS JetStream with persistent host data under `ISLANDFLOW_DATA_ROOT`. - Runs the core runtime services: `ingest-options`, `ingest-equities`, `compute`, `candles`, `api`, and `web`. - Keeps `replay` opt-in through a Compose profile, because the current replay service starts immediately when the container is enabled. @@ -56,6 +56,7 @@ cp .env.example .env Important defaults: - `NATS_URL`, `CLICKHOUSE_URL`, and `REDIS_URL` should stay on the internal container hostnames unless you intentionally split infra out. +- `ISLANDFLOW_DATA_ROOT=/var/lib/islandflow` matches the native infra data root used by the VPS cutover helpers. - `OPTIONS_INGEST_ADAPTER=synthetic` and `EQUITIES_INGEST_ADAPTER=synthetic` are the safest first-boot settings. - `WEB_BIND_IP=127.0.0.1` and `API_BIND_IP=127.0.0.1` keep the published ports local to the host by default. - `WEB_HOST_PORT=3000` and `API_HOST_PORT=4000` control the host-side published ports. @@ -160,8 +161,10 @@ Set the adapter values and credentials in `.env`: - `OPTIONS_INGEST_ADAPTER=alpaca` - `EQUITIES_INGEST_ADAPTER=alpaca` -- `ALPACA_KEY_ID=...` -- `ALPACA_SECRET_KEY=...` +- `ALPACA_API_KEY_ID=...` +- `ALPACA_API_SECRET_KEY=...` + +The older single-variable `ALPACA_API_KEY` fallback is still accepted for legacy setups, but Alpaca's current market-data auth expects a key ID plus secret key pair. ### Databento mode @@ -213,17 +216,19 @@ BuildKit cache mounts require a modern Docker Engine with Dockerfile frontend su ## Safe rollouts on `152.53.80.229` -The current live VPS uses Nginx Proxy Manager on the shared Docker network and routes public traffic to the Docker `web` and `api` containers by container name. Because of that, this Docker path remains the operationally correct default for the live server today. +The current live VPS uses Nginx Proxy Manager as the outer edge. Before native cutover, NPM routes Islandflow traffic to Docker service names. During cutover, `deployment/native/switch-npm-edge.sh native` retargets only the Islandflow proxy hosts to the NPM bridge gateway IP so NPM can reach native host ports. If needed, override the detected target with `ISLANDFLOW_NATIVE_HOST=`. The deploy helper also warns if it detects a second compose project named `islandflow` on the server, because that usually means the repo-root local-infra stack was started on the VPS by mistake. -The checked-in deploy helper is meant to run from your local repo checkout, not from the VPS shell. It always targets: +The checked-in deploy helper normally runs from your local repo checkout and targets: - SSH host: `delta@152.53.80.229` -- SSH key: `~/.ssh/delta_ed25519` +- SSH key: `~/.ssh/delta_ed25519` by default - Live repo checkout: `/home/delta/islandflow` - Live compose directory: `/home/delta/islandflow/deployment/docker` +If you run `./deploy` from `/home/delta/islandflow` on the VPS itself, it now executes the remote steps locally instead of SSHing back into the same machine. You can still force SSH with `DEPLOY_FORCE_SSH=1`, or override the key path with `DEPLOY_SSH_KEY_PATH=/path/to/key`. + It preserves the current Docker Compose project and avoids destructive cleanup on the server. ### Deploy `origin/main` @@ -271,6 +276,7 @@ Examples: ./deploy main --runtime docker --web-only ./deploy main --runtime docker --api-only ./deploy current-branch --runtime docker --services-only +./deploy main --runtime docker --workers-only ./deploy main --runtime docker --fast ./deploy main --runtime docker --web-only --no-build ``` @@ -280,6 +286,7 @@ Scoped Docker deploys now build only the selected image set and then restart onl - `--web-only`: `docker compose build web`, then `docker compose up -d web` - `--api-only`: `docker compose build api`, then `docker compose up -d api` - `--services-only`: builds and restarts `api`, `compute`, `candles`, `ingest-options`, and `ingest-equities` +- `--workers-only`: builds and restarts `compute`, `candles`, `ingest-options`, `ingest-equities`, and `ingest-news` without touching `web` or `api` - `--fast`: when no explicit scope flag is given, treats the deploy as `--services-only` and skips the public API route suite for quicker completion. It still runs remote service health checks. Use `--no-build` only when the image is already correct and you need Compose to recreate or restart containers, such as after changing server-side environment values that do not affect a Next.js build-time variable. Do not use `--no-build` for dependency changes, application source changes, or `NEXT_PUBLIC_*` changes. diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index 96598ba..ec9cd36 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -42,6 +42,8 @@ services: init: true expose: - "3000" + ports: + - "${WEB_BIND_IP:-127.0.0.1}:${WEB_HOST_PORT:-3000}:3000" networks: - default - shared @@ -64,8 +66,13 @@ services: api: <<: *service-common command: ["services/api/src/index.ts"] + environment: + LOG_LEVEL: ${LOG_LEVEL:-warn} + API_HOST: 0.0.0.0 expose: - "4000" + ports: + - "${API_BIND_IP:-127.0.0.1}:${API_HOST_PORT:-4000}:4000" networks: - default - shared @@ -115,6 +122,10 @@ services: <<: *service-common command: ["services/ingest-equities/src/index.ts"] + ingest-news: + <<: *service-common + command: ["services/ingest-news/src/index.ts"] + replay: <<: *service-common profiles: ["replay"] @@ -128,7 +139,7 @@ services: soft: 262144 hard: 262144 volumes: - - clickhouse-data:/var/lib/clickhouse + - ${ISLANDFLOW_DATA_ROOT:-/var/lib/islandflow}/clickhouse:/var/lib/clickhouse - ./clickhouse/listen.xml:/etc/clickhouse-server/config.d/listen.xml:ro healthcheck: test: @@ -146,7 +157,7 @@ services: restart: unless-stopped command: ["redis-server", "--appendonly", "yes"] volumes: - - redis-data:/data + - ${ISLANDFLOW_DATA_ROOT:-/var/lib/islandflow}/redis:/data healthcheck: test: [ @@ -164,14 +175,9 @@ services: restart: unless-stopped command: ["-js", "-sd", "/data"] volumes: - - nats-data:/data + - ${ISLANDFLOW_DATA_ROOT:-/var/lib/islandflow}/nats:/data networks: shared: external: true name: ${NPM_SHARED_NETWORK:-npm-shared} - -volumes: - clickhouse-data: - redis-data: - nats-data: diff --git a/deployment/docker/workspace-root/bun.lock b/deployment/docker/workspace-root/bun.lock index 46160a7..80788c9 100644 --- a/deployment/docker/workspace-root/bun.lock +++ b/deployment/docker/workspace-root/bun.lock @@ -26,13 +26,14 @@ "@islandflow/types": "workspace:*", "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", - "next": "^14.2.4", - "react": "^18.3.1", - "react-dom": "^18.3.1", + "next": "^16.2.6", + "react": "^19.2.0", + "react-dom": "^19.2.0", }, "devDependencies": { "@types/node": "^20.14.10", - "@types/react": "^18.3.3", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", "typescript": "^5.5.4", }, }, @@ -121,6 +122,17 @@ "zod": "^3.23.8", }, }, + "services/ingest-news": { + "name": "@islandflow/ingest-news", + "dependencies": { + "@islandflow/bus": "workspace:*", + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + "@islandflow/types": "workspace:*", + "ws": "^8.18.3", + "zod": "^3.23.8", + }, + }, "services/ingest-options": { "name": "@islandflow/ingest-options", "dependencies": { @@ -204,8 +216,60 @@ "@electron/windows-sign": ["@electron/windows-sign@1.2.2", "", { "dependencies": { "cross-dirname": "^0.1.0", "debug": "^4.3.4", "fs-extra": "^11.1.1", "minimist": "^1.2.8", "postject": "^1.0.0-alpha.6" }, "bin": { "electron-windows-sign": "bin/electron-windows-sign.js" } }, "sha512-dfZeox66AvdPtb2lD8OsIIQh12Tp0GNCRUDfBHIKGpbmopZto2/A8nSpYYLoedPIHpqkeblZ/k8OV0Gy7PYuyQ=="], + "@emnapi/runtime": ["@emnapi/runtime@1.10.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="], + "@gar/promisify": ["@gar/promisify@1.1.3", "", {}, "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw=="], + "@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.2.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="], + + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.2.4", "", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.2.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="], + + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="], + + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="], + + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], + "@inquirer/checkbox": ["@inquirer/checkbox@3.0.1", "", { "dependencies": { "@inquirer/core": "^9.2.1", "@inquirer/figures": "^1.0.6", "@inquirer/type": "^2.0.0", "ansi-escapes": "^4.3.2", "yoctocolors-cjs": "^2.1.2" } }, "sha512-0hm2nrToWUdD6/UHnel/UKGdk1//ke5zGUpHIvk5ZWmaKezlGxZkOJXNSWsdxO/rEqTkbB3lNC2J6nBElV2aAQ=="], "@inquirer/confirm": ["@inquirer/confirm@4.0.1", "", { "dependencies": { "@inquirer/core": "^9.2.1", "@inquirer/type": "^2.0.0" } }, "sha512-46yL28o2NJ9doViqOy0VDcoTzng7rAb6yPQKU7VDLqkmbCaH4JqK4yk4XqlzNWy9PVC5pG1ZUXPBQv+VqnYs2w=="], @@ -250,6 +314,8 @@ "@islandflow/ingest-equities": ["@islandflow/ingest-equities@workspace:services/ingest-equities"], + "@islandflow/ingest-news": ["@islandflow/ingest-news@workspace:services/ingest-news"], + "@islandflow/ingest-options": ["@islandflow/ingest-options@workspace:services/ingest-options"], "@islandflow/observability": ["@islandflow/observability@workspace:packages/observability"], @@ -280,25 +346,23 @@ "@msgpack/msgpack": ["@msgpack/msgpack@3.1.3", "", {}, "sha512-47XIizs9XZXvuJgoaJUIE2lFoID8ugvc0jzSHP+Ptfk8nTbnR8g788wv48N03Kx0UkAv559HWRQ3yzOgzlRNUA=="], - "@next/env": ["@next/env@14.2.35", "", {}, "sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ=="], + "@next/env": ["@next/env@16.2.6", "", {}, "sha512-gd8HoHN4ufj73WmR3JmVolrpJR47ILK6LouP5xElPglaVxir6e1a7VzvTvDWkOoPXT9rkkTzyCxBu4yeZfZwcw=="], - "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@14.2.33", "", { "os": "darwin", "cpu": "arm64" }, "sha512-HqYnb6pxlsshoSTubdXKu15g3iivcbsMXg4bYpjL2iS/V6aQot+iyF4BUc2qA/J/n55YtvE4PHMKWBKGCF/+wA=="], + "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@16.2.6", "", { "os": "darwin", "cpu": "arm64" }, "sha512-ZJGkkcNfYgrrMkqOdZ7zoLa1TOy0qpcMfk/z4Mh/FKUz40gVO+HNQWqmLxf67Z5WB64DRp0dhEbyHfel+6sJUg=="], - "@next/swc-darwin-x64": ["@next/swc-darwin-x64@14.2.33", "", { "os": "darwin", "cpu": "x64" }, "sha512-8HGBeAE5rX3jzKvF593XTTFg3gxeU4f+UWnswa6JPhzaR6+zblO5+fjltJWIZc4aUalqTclvN2QtTC37LxvZAA=="], + "@next/swc-darwin-x64": ["@next/swc-darwin-x64@16.2.6", "", { "os": "darwin", "cpu": "x64" }, "sha512-v/YLBHIY132Ced3puBJ7YJKw1lqsCrgcNo2aRJlCEyQrrCeRJlvGlnmxhPxNQI3KE3N1DN5r9TPNPvka3nq5RQ=="], - "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@14.2.33", "", { "os": "linux", "cpu": "arm64" }, "sha512-JXMBka6lNNmqbkvcTtaX8Gu5by9547bukHQvPoLe9VRBx1gHwzf5tdt4AaezW85HAB3pikcvyqBToRTDA4DeLw=="], + "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@16.2.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-RPOvqlYBbcQjkz9VQQDZ2T2bARIjXZV1KFlt+V2Mr6SW/e4I9fcKsaA0hdyf2FHoTlsV2xnBd5Y912rP/1Ce6w=="], - "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@14.2.33", "", { "os": "linux", "cpu": "arm64" }, "sha512-Bm+QulsAItD/x6Ih8wGIMfRJy4G73tu1HJsrccPW6AfqdZd0Sfm5Imhgkgq2+kly065rYMnCOxTBvmvFY1BKfg=="], + "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@16.2.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-URUTu1+dMkxJsPFgm+OeEvq9wf5sujw0EvgYy80TDGHTSLTnIHeqb0Eu8A3sC95IRgjejQL+kC4mw+4yPxiAXA=="], - "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@14.2.33", "", { "os": "linux", "cpu": "x64" }, "sha512-FnFn+ZBgsVMbGDsTqo8zsnRzydvsGV8vfiWwUo1LD8FTmPTdV+otGSWKc4LJec0oSexFnCYVO4hX8P8qQKaSlg=="], + "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@16.2.6", "", { "os": "linux", "cpu": "x64" }, "sha512-DOj182mPV8G3UkrayLoREM5YEYI+Dk5wv7Ox9xl1fFibAELEsFD0lDPfHIeILlutMMfdyhlzYPELG3peuKaurw=="], - "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@14.2.33", "", { "os": "linux", "cpu": "x64" }, "sha512-345tsIWMzoXaQndUTDv1qypDRiebFxGYx9pYkhwY4hBRaOLt8UGfiWKr9FSSHs25dFIf8ZqIFaPdy5MljdoawA=="], + "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@16.2.6", "", { "os": "linux", "cpu": "x64" }, "sha512-HKQ5SP/V/ub73UvF7n/zeJlxk2kLmtL7Wzrg4WfmkjmNos5onJ2tKu7yZOPdL18A6Svfn3max29ym+ry7NkK4g=="], - "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@14.2.33", "", { "os": "win32", "cpu": "arm64" }, "sha512-nscpt0G6UCTkrT2ppnJnFsYbPDQwmum4GNXYTeoTIdsmMydSKFz9Iny2jpaRupTb+Wl298+Rh82WKzt9LCcqSQ=="], + "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@16.2.6", "", { "os": "win32", "cpu": "arm64" }, "sha512-LZXpTlPyS5v7HhSmnvsLGP3iIYgYOBnc8r8ArlT55sGHV89bR2HlDdBjWQ+PY6SJMmk8TuVGFuxalnP3k/0Dwg=="], - "@next/swc-win32-ia32-msvc": ["@next/swc-win32-ia32-msvc@14.2.33", "", { "os": "win32", "cpu": "ia32" }, "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q=="], - - "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@14.2.33", "", { "os": "win32", "cpu": "x64" }, "sha512-nOjfZMy8B94MdisuzZo9/57xuFVLHJaDj5e/xrduJp9CV2/HrfxTRH2fbyLe+K9QT41WBLUd4iXX3R7jBp0EUg=="], + "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@16.2.6", "", { "os": "win32", "cpu": "x64" }, "sha512-F0+4i0h9J6C4eE3EAPWsoCk7UW/dbzOjyzxY0qnDUOYFu6FFmdZ6l97/XdV3/Nz3VYyO7UWjyEJUXkGqcoXfMA=="], "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], @@ -322,9 +386,7 @@ "@sindresorhus/is": ["@sindresorhus/is@4.6.0", "", {}, "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw=="], - "@swc/counter": ["@swc/counter@0.1.3", "", {}, "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="], - - "@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="], + "@swc/helpers": ["@swc/helpers@0.5.15", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g=="], "@szmarczak/http-timer": ["@szmarczak/http-timer@4.0.6", "", { "dependencies": { "defer-to-connect": "^2.0.0" } }, "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w=="], @@ -352,9 +414,9 @@ "@types/node": ["@types/node@24.12.4", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GUUEShf+PBCGW2KaXwcIt3Yk+e3pkKwWKb9GSyM9WQVE+ep2jzmHdGsHzu4wgcZy5fN9FBdVzjpBQsYlpfpgLA=="], - "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], + "@types/react": ["@types/react@19.2.14", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w=="], - "@types/react": ["@types/react@18.3.27", "", { "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" } }, "sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w=="], + "@types/react-dom": ["@types/react-dom@19.2.3", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="], "@types/responselike": ["@types/responselike@1.0.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw=="], @@ -452,15 +514,13 @@ "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], - "busboy": ["busboy@1.6.0", "", { "dependencies": { "streamsearch": "^1.1.0" } }, "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA=="], - "cacache": ["cacache@16.1.3", "", { "dependencies": { "@npmcli/fs": "^2.1.0", "@npmcli/move-file": "^2.0.0", "chownr": "^2.0.0", "fs-minipass": "^2.1.0", "glob": "^8.0.1", "infer-owner": "^1.0.4", "lru-cache": "^7.7.1", "minipass": "^3.1.6", "minipass-collect": "^1.0.2", "minipass-flush": "^1.0.5", "minipass-pipeline": "^1.2.4", "mkdirp": "^1.0.4", "p-map": "^4.0.0", "promise-inflight": "^1.0.1", "rimraf": "^3.0.2", "ssri": "^9.0.0", "tar": "^6.1.11", "unique-filename": "^2.0.0" } }, "sha512-/+Emcj9DAXxX4cwlLmRI9c166RuL3w30zp4R7Joiv2cQTtTtA+jeuCAjH3ZlGnYS3tKENSrKhAzVVP9GVyzeYQ=="], "cacheable-lookup": ["cacheable-lookup@5.0.4", "", {}, "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA=="], "cacheable-request": ["cacheable-request@7.0.4", "", { "dependencies": { "clone-response": "^1.0.2", "get-stream": "^5.1.0", "http-cache-semantics": "^4.0.0", "keyv": "^4.0.0", "lowercase-keys": "^2.0.0", "normalize-url": "^6.0.1", "responselike": "^2.0.0" } }, "sha512-v+p6ongsrp0yTGbJXjgxPow2+DL93DASP4kXCDKb8/bwRtt9OEF3whggkkDkGNzgcWy2XaF4a8nZglC7uElscg=="], - "caniuse-lite": ["caniuse-lite@1.0.30001761", "", {}, "sha512-JF9ptu1vP2coz98+5051jZ4PwQgd2ni8A+gYSN7EA7dPKIMf0pDlSUxhdmVOaV3/fYK5uWBkgSXJaRLr4+3A6g=="], + "caniuse-lite": ["caniuse-lite@1.0.30001792", "", {}, "sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw=="], "chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], @@ -700,8 +760,6 @@ "jiti": ["jiti@2.7.0", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="], - "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], - "json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="], "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], @@ -732,8 +790,6 @@ "log-update": ["log-update@5.0.1", "", { "dependencies": { "ansi-escapes": "^5.0.0", "cli-cursor": "^4.0.0", "slice-ansi": "^5.0.0", "strip-ansi": "^7.0.1", "wrap-ansi": "^8.0.1" } }, "sha512-5UtUDQ/6edw4ofyljDNcOVJQ4c7OjDro4h3y8e1GQL5iYElYclVHJ3zeWchylvMaKnDbDilC8irOVyexnA/Slw=="], - "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], - "lowercase-keys": ["lowercase-keys@2.0.0", "", {}, "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="], "lru-cache": ["lru-cache@7.18.3", "", {}, "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="], @@ -790,7 +846,7 @@ "neo-async": ["neo-async@2.6.2", "", {}, "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="], - "next": ["next@14.2.35", "", { "dependencies": { "@next/env": "14.2.35", "@swc/helpers": "0.5.5", "busboy": "1.6.0", "caniuse-lite": "^1.0.30001579", "graceful-fs": "^4.2.11", "postcss": "8.4.31", "styled-jsx": "5.1.1" }, "optionalDependencies": { "@next/swc-darwin-arm64": "14.2.33", "@next/swc-darwin-x64": "14.2.33", "@next/swc-linux-arm64-gnu": "14.2.33", "@next/swc-linux-arm64-musl": "14.2.33", "@next/swc-linux-x64-gnu": "14.2.33", "@next/swc-linux-x64-musl": "14.2.33", "@next/swc-win32-arm64-msvc": "14.2.33", "@next/swc-win32-ia32-msvc": "14.2.33", "@next/swc-win32-x64-msvc": "14.2.33" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.41.2", "react": "^18.2.0", "react-dom": "^18.2.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-KhYd2Hjt/O1/1aZVX3dCwGXM1QmOV4eNM2UTacK5gipDdPN/oHHK/4oVGy7X8GMfPMsUTUEmGlsy0EY1YGAkig=="], + "next": ["next@16.2.6", "", { "dependencies": { "@next/env": "16.2.6", "@swc/helpers": "0.5.15", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", "styled-jsx": "5.1.6" }, "optionalDependencies": { "@next/swc-darwin-arm64": "16.2.6", "@next/swc-darwin-x64": "16.2.6", "@next/swc-linux-arm64-gnu": "16.2.6", "@next/swc-linux-arm64-musl": "16.2.6", "@next/swc-linux-x64-gnu": "16.2.6", "@next/swc-linux-x64-musl": "16.2.6", "@next/swc-win32-arm64-msvc": "16.2.6", "@next/swc-win32-x64-msvc": "16.2.6", "sharp": "^0.34.5" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "babel-plugin-react-compiler", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-qOVgKJg1+At15NpeUP+eJgCHvTCgXsogweq87Ri/Ix7PkqQHg4sdaXmSFqKlgaIXE4kW0g25LE68W87UANlHtw=="], "nice-try": ["nice-try@1.0.5", "", {}, "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ=="], @@ -884,9 +940,9 @@ "quick-lru": ["quick-lru@5.1.1", "", {}, "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA=="], - "react": ["react@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="], + "react": ["react@19.2.6", "", {}, "sha512-sfWGGfavi0xr8Pg0sVsyHMAOziVYKgPLNrS7ig+ivMNb3wbCBw3KxtflsGBAwD3gYQlE/AEZsTLgToRrSCjb0Q=="], - "react-dom": ["react-dom@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" }, "peerDependencies": { "react": "^18.3.1" } }, "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw=="], + "react-dom": ["react-dom@19.2.6", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.6" } }, "sha512-0prMI+hvBbPjsWnxDLxlCGyM8PN6UuWjEUCYmZhO67xIV9Xasa/r/vDnq+Xyq4Lo27g8QSbO5YzARu0D1Sps3g=="], "read-binary-file-arch": ["read-binary-file-arch@1.0.6", "", { "dependencies": { "debug": "^4.3.4" }, "bin": { "read-binary-file-arch": "cli.js" } }, "sha512-BNg9EN3DD3GsDXX7Aa8O4p92sryjkmzYYgmgTAc6CA4uGLEDzFfxOxugu21akOxpcXHiEgsYkC6nPsQvLLLmEg=="], @@ -930,7 +986,7 @@ "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], - "scheduler": ["scheduler@0.23.2", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ=="], + "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], "schema-utils": ["schema-utils@4.3.3", "", { "dependencies": { "@types/json-schema": "^7.0.9", "ajv": "^8.9.0", "ajv-formats": "^2.1.1", "ajv-keywords": "^5.1.0" } }, "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA=="], @@ -940,6 +996,8 @@ "serialize-error": ["serialize-error@7.0.1", "", { "dependencies": { "type-fest": "^0.13.1" } }, "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw=="], + "sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], @@ -972,8 +1030,6 @@ "ssri": ["ssri@9.0.1", "", { "dependencies": { "minipass": "^3.1.1" } }, "sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q=="], - "streamsearch": ["streamsearch@1.1.0", "", {}, "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg=="], - "string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], @@ -986,7 +1042,7 @@ "strip-outer": ["strip-outer@1.0.1", "", { "dependencies": { "escape-string-regexp": "^1.0.2" } }, "sha512-k55yxKHwaXnpYGsOzg4Vl8+tDrWylxDEpknGjhTiZB8dFRU5rTo9CAzeycivxV3s+zlTKwrs6WxMxR95n26kwg=="], - "styled-jsx": ["styled-jsx@5.1.1", "", { "dependencies": { "client-only": "0.0.1" }, "peerDependencies": { "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0" } }, "sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw=="], + "styled-jsx": ["styled-jsx@5.1.6", "", { "dependencies": { "client-only": "0.0.1" }, "peerDependencies": { "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0" } }, "sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA=="], "sumchecker": ["sumchecker@3.0.1", "", { "dependencies": { "debug": "^4.1.0" } }, "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg=="], @@ -1128,8 +1184,6 @@ "ansi-escapes/type-fest": ["type-fest@0.21.3", "", {}, "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w=="], - "browserslist/caniuse-lite": ["caniuse-lite@1.0.30001792", "", {}, "sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw=="], - "cacache/glob": ["glob@8.1.0", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^5.0.1", "once": "^1.3.0" } }, "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ=="], "cacache/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="], diff --git a/deployment/native/README.md b/deployment/native/README.md index a9903cc..219f952 100644 --- a/deployment/native/README.md +++ b/deployment/native/README.md @@ -1,29 +1,170 @@ # Native Deployment -This directory documents the experimental host-native Islandflow rollout path used by: +This directory documents the host-native Islandflow rollout path used by: ```bash ./deploy main --runtime native ./deploy current-branch --runtime native ``` -This runtime is intended for faster server iteration during the transition away from Docker-only app rollouts. It is not the recommended path for the current production VPS, which still uses Nginx Proxy Manager to reach the Docker `web` and `api` containers by container name on the shared Docker network. Local development should still prefer: +## Current operating model -- Docker for infra (`bun run dev:infra`) -- native Bun services (`bun run dev:services`) -- native Next.js web (`bun run dev:web`) +Native runtime is now intended for a phased VPS cutover. Docker remains the supported rollback runtime, but Docker and native app services must not own the same Islandflow scope at the same time because the workers and API use durable JetStream consumers. + +Today, the recommended split is: + +- **Nginx Proxy Manager** remains the public `:80/:443` edge +- **Native system services** own NATS, Redis, and ClickHouse after infra cutover +- **Native user services** own `web`, `api`, and workers after app cutover +- **Docker Compose** remains available as the rollback runtime +- local development stays: + - Docker infra: `bun run dev:infra` + - native backend services: `bun run dev:services` + - native web: `bun run dev:web` ## What native deploy means here The checked-in `deploy` helper assumes: -- the live repo checkout is still `/home/delta/islandflow` +- the live repo checkout is `/home/delta/islandflow` - Bun is installed on the VPS -- app processes are managed by `systemd` -- infrastructure services such as NATS, ClickHouse, and Redis are already reachable from the host +- app processes are managed by `systemd --user` +- infrastructure services such as NATS, ClickHouse, and Redis are reachable from the host - the web app runs from `apps/web` and is served with `next start -p 3000` -The deploy script updates the repo checkout, optionally runs `bun install --frozen-lockfile`, optionally rebuilds the web app, restarts the target systemd units, and then verifies the services locally on the VPS plus through the public app URL. +The deploy script updates the repo checkout, optionally runs `bun install --frozen-lockfile`, optionally rebuilds the web app, restarts the target user units, verifies local health, and then runs public verification when the selected scope includes the public edge. + +## Live audit status on 2026-05-18 + +The plan assumptions were audited on the VPS: + +- `bun` is installed and available at `/home/delta/.bun/bin/bun` +- `systemctl --user` is available and the `delta` user has lingering enabled +- `/home/delta/islandflow/.env` exists +- public `https://flow.deltaisland.io/replay/options` routing is healthy again +- the previously reported duplicate `islandflow` compose project is not currently present in `docker compose ls` +- native Islandflow user units were not installed at the start of the audit; this change now provides and installs the checked-in user unit files, but they remain disabled until an operator enables a scope intentionally + +That means native worker deploy support is now provisioned on the host, but native runtime should still be enabled scope-by-scope rather than started wholesale. + +## Checked-in native ops assets + +### Infra system units + +Checked-in system service units and config live under: + +- `deployment/native/systemd/system/islandflow-nats.service` +- `deployment/native/systemd/system/islandflow-redis.service` +- `deployment/native/systemd/system/islandflow-clickhouse.service` +- `deployment/native/config/redis.conf` +- `deployment/native/config/clickhouse-listen.xml` + +Install and start them on the VPS with: + +```bash +./deployment/native/bootstrap-infra.sh +``` + +Or install and start manually: + +```bash +sudo ./deployment/native/install-infra-units.sh +sudo ./deployment/native/start-infra.sh +./deployment/native/check-native-infra.sh +``` + +The native infra services bind to loopback and use stable host data paths: + +- NATS JetStream: `/var/lib/islandflow/nats` +- Redis: `/var/lib/islandflow/redis` +- ClickHouse: `/var/lib/islandflow/clickhouse` + +The Docker fallback compose file uses the same `ISLANDFLOW_DATA_ROOT` default of `/var/lib/islandflow`, so rollback can preserve durable state when only one runtime is active. + +### User unit templates + +Checked-in unit files live under: + +- `deployment/native/systemd/user/islandflow-web.service` +- `deployment/native/systemd/user/islandflow-api.service` +- `deployment/native/systemd/user/islandflow-compute.service` +- `deployment/native/systemd/user/islandflow-candles.service` +- `deployment/native/systemd/user/islandflow-ingest-options.service` +- `deployment/native/systemd/user/islandflow-ingest-equities.service` +- `deployment/native/systemd/user/islandflow-ingest-news.service` + +These are written for the current VPS layout: + +- repo root: `/home/delta/islandflow` +- Bun binary: `/home/delta/.bun/bin/bun` +- env file: `/home/delta/islandflow/.env` + +Important: treat `/home/delta/islandflow/.env` as the effective source of truth for adapter selection. The Bun-launched services read that file directly at runtime, so a conflicting `OPTIONS_INGEST_ADAPTER` value in `.env` can still win over a systemd-only override and push `ingest-options` onto the wrong provider path. + +### Install the units + +```bash +./deployment/native/install-user-units.sh +./deployment/native/install-user-units.sh workers +systemctl --user start islandflow-compute.service +``` + +Install script behavior: + +- copies the checked-in unit files into `~/.config/systemd/user` +- reloads the user systemd daemon +- enables only the scope you explicitly request +- defaults to installing without enabling anything yet + +### Smoke test helper + +```bash +./deployment/native/check-native-health.sh workers +./deployment/native/check-native-health.sh services +./deployment/native/check-native-health.sh full +``` + +This validates: + +- native infra health for `full`, `api`, `services`, and `workers` +- `systemctl --user is-active` for the selected units +- local API health at `http://127.0.0.1:4000/health` when API scope is included +- local web health at `http://127.0.0.1:3000/` when web scope is included + +### App cutover and edge switch helpers + +```bash +./deployment/native/cutover.sh full +./deployment/native/switch-npm-edge.sh native +./deployment/native/full-rollback.sh +``` + +The edge switch helper updates the Nginx Proxy Manager database entries for `flow.deltaisland.io` and `api.flow.deltaisland.io`, preserving the same-origin Islandflow API location matcher: + +```nginx +^/(ws|replay|prints|joins|nbbo|dark|flow|candles|history)/ +``` + +For native cutover, the helper targets the NPM bridge gateway IP by default, not `host.docker.internal`. NPM generates `proxy_pass` with a runtime-resolved `$server` variable, so Docker's `/etc/hosts` alias is not sufficient for these proxy hosts. On the current VPS that native target resolves to `172.18.0.1`, which reaches the host-native `3000` and `4000` listeners from the NPM container. + +Switching back to Docker restores upstreams to the Compose service names `web:3000` and `api:4000`. + +### Rollback helper + +```bash +./deployment/native/rollback.sh workers +./deployment/native/rollback.sh services +``` + +Rollback helper behavior: + +- requires a clean repo state +- fetches refs +- switches the checkout to a detached target ref +- reruns `bun install --frozen-lockfile` +- rebuilds the web app only when web scope is included +- restarts the selected user units +- runs the native smoke checks ## Expected unit names @@ -35,6 +176,7 @@ Default unit names used by `scripts/deploy.ts`: - `islandflow-candles` - `islandflow-ingest-options` - `islandflow-ingest-equities` +- `islandflow-ingest-news` Override them from your local shell before running `./deploy` if the server uses different names: @@ -51,90 +193,108 @@ Available overrides: - `DEPLOY_NATIVE_CANDLES_UNIT` - `DEPLOY_NATIVE_INGEST_OPTIONS_UNIT` - `DEPLOY_NATIVE_INGEST_EQUITIES_UNIT` +- `DEPLOY_NATIVE_INGEST_NEWS_UNIT` ## systemctl invocation -By default the deploy helper uses: - -```bash -sudo -n systemctl -``` - -If the server uses user units or another wrapper, override it locally before invoking `./deploy`: +For the checked-in user units, use: ```bash export DEPLOY_NATIVE_SYSTEMCTL_PREFIX="systemctl --user" -./deploy main --runtime native ``` +The deploy helper defaults to `sudo -n systemctl`, but that is only appropriate if you intentionally install matching system units. + ## Partial native rollouts Examples: ```bash -./deploy main --runtime native --web-only -./deploy main --runtime native --api-only -./deploy current-branch --runtime native --services-only +./deploy main --runtime native --workers-only ./deploy main --runtime native --fast -./deploy main --runtime native --web-only --no-build +./deploy main --runtime native --services-only +./deploy main --runtime native --web-only +./deploy current-branch --runtime native --workers-only --no-build ``` Scope behavior: -- default: restart web + API + backend services +- default: restart web + API + worker services - `--web-only`: rebuild/restart only the web unit - `--api-only`: restart only the API unit -- `--services-only`: restart API + backend units without touching the web unit -- `--fast`: when no explicit scope flag is provided, uses the same `--services-only` scope and trims verbose verification output for quicker completion +- `--services-only`: restart API + worker units without touching the web unit +- `--workers-only`: restart only `compute`, `candles`, `ingest-options`, `ingest-equities`, and `ingest-news` +- `--fast`: when no explicit scope flag is provided, native deploys now default to `--workers-only` - `--no-build`: skip `bun install --frozen-lockfile` and skip the web build step -## Current status +## Edge-cutover guardrail -On the current live VPS, native deploys should be treated as opt-in infrastructure work, not the default rollout path. Before a native deploy can succeed there, all of the following must be true at the same time: - -- Bun is installed on the host. -- The selected `systemctl` command works non-interactively. -- Islandflow systemd units exist for the requested scope. -- Host-native services can reach the intended NATS, ClickHouse, and Redis endpoints. -- If `web` or `api` move native, the reverse proxy topology is updated deliberately. - -Until that is prepared intentionally, prefer: +Native deploys that touch the public web or API edge are intentionally blocked unless you acknowledge cutover readiness: ```bash -./deploy main --runtime docker -./deploy current-branch --runtime docker +export DEPLOY_NATIVE_EDGE_READY=1 ``` -## Server preparation checklist +Without that variable, these commands are refused: -Before the first native rollout, ensure the VPS has: +- `./deploy main --runtime native` +- `./deploy main --runtime native --web-only` +- `./deploy main --runtime native --api-only` +- `./deploy main --runtime native --services-only` -1. Bun installed and on `PATH` -2. a working `/home/delta/islandflow/.env` (or unit-managed equivalent env source) -3. systemd units for each target service -4. the web unit configured to serve the built app on port `3000` -5. the API unit configured to serve health checks on port `4000` -6. infrastructure endpoints configured so the native services can reach NATS, ClickHouse, and Redis +This keeps native app ownership explicit until infra, app health, and proxy routing are switched deliberately. -## Verification +## Running deploy from the VPS itself -Native deploys verify: +If you run `./deploy` from `/home/delta/islandflow` on the live server, the deploy helper now executes the remote steps locally instead of SSHing back into the same machine. -- target units are active via `systemctl` -- recent unit status and journal output can be collected -- local `http://127.0.0.1:4000/health` when API scope is included -- local `http://127.0.0.1:3000/` when web scope is included -- the public app URL from the local machine after the rollout finishes +That means: -## Rollback +- no SSH key is required for on-server deploy execution +- timing and verification behavior stay the same +- you can still force SSH with `DEPLOY_FORCE_SSH=1` +- you can override the SSH key path with `DEPLOY_SSH_KEY_PATH=/path/to/key` -Rollback remains manual for now: +## Validation matrix -1. switch the server checkout back to the last known-good branch or commit -2. rerun the appropriate native deploy command -3. if needed, restart only the affected units with `systemctl` +| Area | Native workers-only | Native edge cutover | +| --- | --- | --- | +| Bun installed | required | required | +| `systemctl --user` works | required | required | +| Islandflow user units installed | worker units only | all units | +| Host access to NATS/ClickHouse/Redis | required | required | +| Proxy routes updated for `/prints`, `/history`, `/replay`, `/nbbo`, `/ws`, `/flow`, `/candles` | not required | required | +| Public app check | not required | required | +| Public API route suite | not required | required | -Docker remains the fallback and currently recommended runtime during the transition: +## Staged cutover plan + +1. **Stage 1: native workers only** + - install user units + - validate `./deployment/native/check-native-health.sh workers` + - use `./deploy main --runtime native --fast` +2. **Stage 2: native API behind local-only verification** + - start `islandflow-api.service` + - confirm `curl http://127.0.0.1:4000/health` + - do not switch public routing yet +3. **Stage 3: deliberate public edge cutover** + - update proxy routing to native `web`/`api` + - export `DEPLOY_NATIVE_EDGE_READY=1` + - run full native deploy + - validate `bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io` +4. **Stage 4: decide final default runtime** + - keep Docker as fallback until native edge has proven stable + +## Recommended current commands + +Fast backend iteration before edge cutover: + +```bash +export DEPLOY_NATIVE_SYSTEMCTL_PREFIX="systemctl --user" +./deploy main --runtime native --fast +``` + +Supported production path today: ```bash ./deploy main --runtime docker diff --git a/deployment/native/bootstrap-infra.sh b/deployment/native/bootstrap-infra.sh new file mode 100755 index 0000000..dfc3422 --- /dev/null +++ b/deployment/native/bootstrap-infra.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +if [[ "${EUID}" -eq 0 ]]; then + "$repo_root/deployment/native/install-infra-units.sh" +else + sudo "$repo_root/deployment/native/install-infra-units.sh" +fi + +echo "Stopping Docker Islandflow services before native infra opens durable data." +( + cd "$repo_root/deployment/docker" + docker compose stop web api compute candles ingest-options ingest-equities nats redis clickhouse +) + +if [[ "${EUID}" -eq 0 ]]; then + "$repo_root/deployment/native/start-infra.sh" +else + sudo "$repo_root/deployment/native/start-infra.sh" +fi + +"$repo_root/deployment/native/check-native-infra.sh" diff --git a/deployment/native/check-native-health.sh b/deployment/native/check-native-health.sh new file mode 100755 index 0000000..e78270a --- /dev/null +++ b/deployment/native/check-native-health.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +scope="${1:-full}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +units=() + +case "$scope" in + full) + units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) + ;; + web) + units=(islandflow-web.service) + ;; + api) + units=(islandflow-api.service) + ;; + services) + units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) + ;; + workers) + units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) + ;; + *) + echo "Unknown scope: $scope" >&2 + echo "Expected one of: full, web, api, services, workers" >&2 + exit 1 + ;; +esac + +case "$scope" in + full|api|services|workers) + "$repo_root/deployment/native/check-native-infra.sh" + ;; +esac + +for unit in "${units[@]}"; do + systemctl --user is-active --quiet "$unit" + echo "ok $unit" +done + +if [[ " ${units[*]} " == *" islandflow-api.service "* ]]; then + curl -fksS http://127.0.0.1:4000/health >/dev/null + echo "ok api-health" +fi + +if [[ " ${units[*]} " == *" islandflow-web.service "* ]]; then + curl -I -fksS http://127.0.0.1:3000/ >/dev/null + echo "ok web-health" +fi diff --git a/deployment/native/check-native-infra.sh b/deployment/native/check-native-infra.sh new file mode 100755 index 0000000..bfdc998 --- /dev/null +++ b/deployment/native/check-native-infra.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +systemctl is-active --quiet islandflow-nats.service +echo "ok islandflow-nats.service" + +systemctl is-active --quiet islandflow-redis.service +echo "ok islandflow-redis.service" + +systemctl is-active --quiet islandflow-clickhouse.service +echo "ok islandflow-clickhouse.service" + +if command -v redis-cli >/dev/null 2>&1; then + redis-cli -h 127.0.0.1 -p 6379 ping | grep -q PONG +else + timeout 2 bash -c ' + 127.0.0.1 + /var/lib/islandflow/clickhouse/ + /var/lib/islandflow/clickhouse/tmp/ + /var/lib/islandflow/clickhouse/user_files/ + diff --git a/deployment/native/config/redis.conf b/deployment/native/config/redis.conf new file mode 100644 index 0000000..8a39ba6 --- /dev/null +++ b/deployment/native/config/redis.conf @@ -0,0 +1,10 @@ +bind 127.0.0.1 +protected-mode yes +port 6379 +dir /var/lib/islandflow/redis +appendonly yes +save 900 1 +save 300 10 +save 60 10000 +loglevel notice +databases 16 diff --git a/deployment/native/cutover.sh b/deployment/native/cutover.sh new file mode 100755 index 0000000..5971f12 --- /dev/null +++ b/deployment/native/cutover.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +scope="${1:-full}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +case "$scope" in + full|services|workers|api|web) + ;; + *) + echo "Usage: deployment/native/cutover.sh [full|services|workers|api|web]" >&2 + exit 1 + ;; +esac + +echo "Stopping Docker-owned Islandflow app services before native ownership starts." +( + cd "$repo_root/deployment/docker" + docker compose stop web api compute candles ingest-options ingest-equities ingest-news +) + +if [[ "$scope" == "full" || "$scope" == "services" || "$scope" == "api" || "$scope" == "web" ]]; then + "$repo_root/deployment/native/check-native-infra.sh" +fi + +systemctl --user restart $(case "$scope" in + full) echo islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service ;; + services) echo islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service ;; + workers) echo islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service ;; + api) echo islandflow-api.service ;; + web) echo islandflow-web.service ;; +esac) + +"$repo_root/deployment/native/check-native-health.sh" "$scope" diff --git a/deployment/native/full-rollback.sh b/deployment/native/full-rollback.sh new file mode 100755 index 0000000..9cac62b --- /dev/null +++ b/deployment/native/full-rollback.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +echo "Stopping native app services." +systemctl --user stop islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service || true + +echo "Stopping native infra before Docker reopens durable data." +if [[ "${EUID}" -eq 0 ]]; then + systemctl stop islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service || true +else + sudo systemctl stop islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service || true +fi + +echo "Switching NPM Islandflow upstreams back to Docker service names." +"$repo_root/deployment/native/switch-npm-edge.sh" docker + +echo "Restarting Docker Islandflow runtime." +( + cd "$repo_root/deployment/docker" + docker compose up -d web api compute candles ingest-options ingest-equities ingest-news +) + +curl -I -fksS "${DEPLOY_PUBLIC_APP_URL:-https://flow.deltaisland.io}" >/dev/null +curl -fksS "${DEPLOY_PUBLIC_API_HEALTH_URL:-https://api.flow.deltaisland.io/health}" >/dev/null +echo "Rollback validation passed." diff --git a/deployment/native/install-infra-units.sh b/deployment/native/install-infra-units.sh new file mode 100755 index 0000000..2a9ab85 --- /dev/null +++ b/deployment/native/install-infra-units.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +system_unit_source_dir="$repo_root/deployment/native/systemd/system" +config_source_dir="$repo_root/deployment/native/config" + +if [[ "${EUID}" -ne 0 ]]; then + echo "Run as root: sudo $0" >&2 + exit 1 +fi + +resolve_binary() { + local name="$1" + local path="" + + path="$(command -v "$name" 2>/dev/null || true)" + if [[ -n "$path" ]]; then + printf '%s\n' "$path" + return 0 + fi + + for candidate in "/usr/bin/$name" "/usr/sbin/$name" "/usr/local/bin/$name" "/usr/local/sbin/$name"; do + if [[ -x "$candidate" ]]; then + printf '%s\n' "$candidate" + return 0 + fi + done + + return 1 +} + +missing=() +for command in nats-server redis-server clickhouse-server; do + if ! resolve_binary "$command" >/dev/null; then + missing+=("$command") + fi +done + +if [[ ${#missing[@]} -gt 0 ]]; then + echo "Missing native infra binaries: ${missing[*]}" >&2 + echo "Install NATS Server, Redis Server, and ClickHouse Server before bootstrapping native infra." >&2 + echo "On Debian, Redis is usually available as redis-server; ClickHouse and NATS may require their vendor repositories or packaged binaries." >&2 + exit 1 +fi + +ensure_system_user() { + local name="$1" + local home="$2" + + getent group "$name" >/dev/null || groupadd --system "$name" + getent passwd "$name" >/dev/null || useradd --system --gid "$name" --home-dir "$home" --shell /usr/sbin/nologin "$name" +} + +ensure_system_user nats /var/lib/islandflow/nats +ensure_system_user redis /var/lib/islandflow/redis +ensure_system_user clickhouse /var/lib/islandflow/clickhouse + +install -d -m 0755 /etc/islandflow +install -m 0644 "$config_source_dir/redis.conf" /etc/islandflow/redis.conf +install -d -m 0755 /etc/clickhouse-server/config.d +install -m 0644 "$config_source_dir/clickhouse-listen.xml" /etc/clickhouse-server/config.d/islandflow-listen.xml + +install -d -o nats -g nats -m 0750 /var/lib/islandflow/nats +install -d -o redis -g redis -m 0750 /var/lib/islandflow/redis +install -d -o clickhouse -g clickhouse -m 0750 /var/lib/islandflow/clickhouse + +install -m 0644 "$system_unit_source_dir"/islandflow-*.service /etc/systemd/system/ +systemctl daemon-reload + +echo "Installed native infra system units and config." +echo "Start infra with: sudo deployment/native/start-infra.sh" diff --git a/deployment/native/install-user-units.sh b/deployment/native/install-user-units.sh new file mode 100755 index 0000000..558ff93 --- /dev/null +++ b/deployment/native/install-user-units.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +scope="${1:-none}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +unit_source_dir="$repo_root/deployment/native/systemd/user" +unit_target_dir="${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user" +units=() + +case "$scope" in + none) + ;; + full) + units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) + ;; + web) + units=(islandflow-web.service) + ;; + api) + units=(islandflow-api.service) + ;; + services) + units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) + ;; + workers) + units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) + ;; + *) + echo "Unknown scope: $scope" >&2 + echo "Expected one of: none, full, web, api, services, workers" >&2 + exit 1 + ;; +esac + +mkdir -p "$unit_target_dir" +cp "$unit_source_dir"/*.service "$unit_target_dir"/ + +systemctl --user daemon-reload + +if [[ ${#units[@]} -gt 0 ]]; then + systemctl --user enable "${units[@]}" +fi + +echo "Installed Islandflow user units into $unit_target_dir" +if [[ ${#units[@]} -gt 0 ]]; then + echo "Enabled scope: $scope" +else + echo "No units enabled yet. Pass a scope such as workers when you are ready." +fi diff --git a/deployment/native/rollback.sh b/deployment/native/rollback.sh new file mode 100755 index 0000000..0721b50 --- /dev/null +++ b/deployment/native/rollback.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 || $# -gt 2 ]]; then + echo "Usage: deployment/native/rollback.sh [full|web|api|services|workers]" >&2 + exit 1 +fi + +ref="$1" +scope="${2:-services}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +cd "$repo_root" + +if [[ -n "$(git status --porcelain=v1)" ]]; then + echo "Refusing rollback with a dirty working tree." >&2 + exit 1 +fi + +current_ref="$(git rev-parse --short HEAD)" +echo "Rolling back from $current_ref to $ref (scope: $scope)" + +git fetch --all --prune +git switch --detach "$ref" +bun install --frozen-lockfile + +if [[ "$scope" == "full" || "$scope" == "web" ]]; then + bun --cwd=apps/web run build +fi + +case "$scope" in + full) + units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) + ;; + web) + units=(islandflow-web.service) + ;; + api) + units=(islandflow-api.service) + ;; + services) + units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) + ;; + workers) + units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) + ;; + *) + echo "Unknown scope: $scope" >&2 + exit 1 + ;; +esac + +systemctl --user restart "${units[@]}" +"$repo_root/deployment/native/check-native-health.sh" "$scope" + +echo "Rollback complete. Repo is now detached at $(git rev-parse --short HEAD)." +echo "Return to tracked main later with: git switch main && git pull --ff-only main" diff --git a/deployment/native/start-infra.sh b/deployment/native/start-infra.sh new file mode 100755 index 0000000..8f78791 --- /dev/null +++ b/deployment/native/start-infra.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${EUID}" -ne 0 ]]; then + echo "Run as root: sudo $0" >&2 + exit 1 +fi + +for unit in redis-server.service nats-server.service clickhouse-server.service; do + if systemctl list-unit-files "$unit" >/dev/null 2>&1; then + systemctl disable --now "$unit" >/dev/null 2>&1 || true + fi +done + +systemctl reset-failed islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service || true +systemctl enable --now islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service +"$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/check-native-infra.sh" diff --git a/deployment/native/stop-infra.sh b/deployment/native/stop-infra.sh new file mode 100755 index 0000000..91a488d --- /dev/null +++ b/deployment/native/stop-infra.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${EUID}" -ne 0 ]]; then + echo "Run as root: sudo $0" >&2 + exit 1 +fi + +systemctl stop islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service diff --git a/deployment/native/switch-npm-edge.sh b/deployment/native/switch-npm-edge.sh new file mode 100755 index 0000000..c9fcd93 --- /dev/null +++ b/deployment/native/switch-npm-edge.sh @@ -0,0 +1,285 @@ +#!/usr/bin/env bash +set -euo pipefail + +target="${1:-native}" +npm_root="${NPM_ROOT:-/home/delta/nginx-proxy-manager}" +db_path="${NPM_DB_PATH:-$npm_root/data/database.sqlite}" +app_domain="${ISLANDFLOW_APP_DOMAIN:-flow.deltaisland.io}" +api_domain="${ISLANDFLOW_API_DOMAIN:-api.flow.deltaisland.io}" +native_host="${ISLANDFLOW_NATIVE_HOST:-}" +docker_web_host="${ISLANDFLOW_DOCKER_WEB_HOST:-web}" +docker_api_host="${ISLANDFLOW_DOCKER_API_HOST:-api}" +web_port="${ISLANDFLOW_WEB_PORT:-3000}" +api_port="${ISLANDFLOW_API_PORT:-4000}" +restart_npm="${NPM_RESTART:-1}" +npm_container="${NPM_CONTAINER_NAME:-nginx-proxy-manager}" +sudo_cmd=() + +case "$target" in + native|docker) + ;; + *) + echo "Usage: deployment/native/switch-npm-edge.sh [native|docker]" >&2 + exit 1 + ;; +esac + +resolve_native_host() { + if [[ -n "$native_host" ]]; then + printf '%s\n' "$native_host" + return + fi + + if command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' | grep -qx "$npm_container"; then + native_host="$(docker inspect "$npm_container" --format '{{range .NetworkSettings.Networks}}{{println .Gateway}}{{end}}' | sed '/^$/d' | head -n1)" + if [[ -n "$native_host" ]]; then + printf '%s\n' "$native_host" + return + fi + fi + + echo "Unable to determine the native upstream host for NPM." >&2 + echo "Set ISLANDFLOW_NATIVE_HOST explicitly or start the $npm_container container first." >&2 + exit 1 +} + +if [[ "$target" == "native" ]]; then + native_host="$(resolve_native_host)" +fi + +if [[ ! -w "$db_path" || ! -w "$(dirname "$db_path")" ]]; then + if [[ "${EUID}" -eq 0 ]]; then + sudo_cmd=() + elif command -v sudo >/dev/null 2>&1; then + sudo_cmd=(sudo) + else + echo "NPM database path is not writable and sudo is unavailable: $db_path" >&2 + exit 1 + fi +fi + +if [[ ! -f "$db_path" ]]; then + echo "NPM database not found: $db_path" >&2 + exit 1 +fi + +backup="$db_path.before-islandflow-$target-$(date +%Y%m%d%H%M%S)" +"${sudo_cmd[@]}" cp "$db_path" "$backup" +echo "Backed up NPM database to $backup" + +"${sudo_cmd[@]}" python3 - "$db_path" "$target" "$app_domain" "$api_domain" "$native_host" "$docker_web_host" "$docker_api_host" "$web_port" "$api_port" <<'PY' +import json +import sqlite3 +import sys + +db_path, target, app_domain, api_domain, native_host, docker_web_host, docker_api_host, web_port, api_port = sys.argv[1:] +web_host = native_host if target == "native" else docker_web_host +api_host = native_host if target == "native" else docker_api_host + +advanced_config = f"""location ~ ^/(ws|replay|prints|joins|nbbo|dark|flow|candles|history)/ {{ + set $forward_scheme http; + set $server "{api_host}"; + set $port {api_port}; + + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $http_connection; + proxy_http_version 1.1; + + include conf.d/include/proxy.conf; +}}""" + +def has_domain(raw, domain): + try: + return domain in json.loads(raw) + except Exception: + return domain in raw + +con = sqlite3.connect(db_path) +cur = con.cursor() +rows = list(cur.execute("select id, domain_names from proxy_host where is_deleted = 0")) +app_ids = [row_id for row_id, domains in rows if has_domain(domains, app_domain)] +api_ids = [row_id for row_id, domains in rows if has_domain(domains, api_domain)] + +if len(app_ids) != 1 or len(api_ids) != 1: + raise SystemExit(f"Expected one app and one API proxy host, found app={app_ids} api={api_ids}") + +cur.execute( + "update proxy_host set forward_scheme = 'http', forward_host = ?, forward_port = ?, allow_websocket_upgrade = 1, advanced_config = ?, modified_on = datetime('now') where id = ?", + (web_host, int(web_port), advanced_config, app_ids[0]), +) +cur.execute( + "update proxy_host set forward_scheme = 'http', forward_host = ?, forward_port = ?, allow_websocket_upgrade = 1, modified_on = datetime('now') where id = ?", + (api_host, int(api_port), api_ids[0]), +) +con.commit() +print(f"Updated {app_domain} -> {web_host}:{web_port}") +print(f"Updated {api_domain} -> {api_host}:{api_port}") +PY + +if command -v python3 >/dev/null 2>&1; then + "${sudo_cmd[@]}" python3 - "$npm_root" "$db_path" "$target" "$app_domain" "$api_domain" "$native_host" "$docker_web_host" "$docker_api_host" "$web_port" "$api_port" <<'PY' +import json +import re +import sqlite3 +import sys +from pathlib import Path + +( + npm_root, + db_path, + target, + app_domain, + api_domain, + native_host, + docker_web_host, + docker_api_host, + web_port, + api_port, +) = sys.argv[1:] + +web_host = native_host if target == "native" else docker_web_host +api_host = native_host if target == "native" else docker_api_host + +def has_domain(raw, domain): + try: + return domain in json.loads(raw) + except Exception: + return domain in raw + +def replace_nth(text, pattern, replacement, index): + matches = list(pattern.finditer(text)) + if len(matches) < index: + raise SystemExit(f"Unable to rewrite generated proxy config; expected match {index} for {pattern.pattern!r}") + match = matches[index - 1] + return text[:match.start()] + replacement(match) + text[match.end():] + +server_pattern = re.compile(r'^(?P\s*set \$server\s+)".*?";\s*$', re.M) +port_pattern = re.compile(r'^(?P\s*set \$port\s+)\d+;\s*$', re.M) + +def replace_server(text, host, index): + return replace_nth(text, server_pattern, lambda m: f'{m.group("prefix")}"{host}";', index) + +def replace_port(text, port, index): + return replace_nth(text, port_pattern, lambda m: f'{m.group("prefix")}{port};', index) + +con = sqlite3.connect(db_path) +rows = list(con.execute("select id, domain_names from proxy_host where is_deleted = 0")) +app_ids = [row_id for row_id, domains in rows if has_domain(domains, app_domain)] +api_ids = [row_id for row_id, domains in rows if has_domain(domains, api_domain)] +if len(app_ids) != 1 or len(api_ids) != 1: + raise SystemExit(f"Expected one app and one API proxy host, found app={app_ids} api={api_ids}") + +api_conf = Path(npm_root) / "data/nginx/proxy_host" / f"{api_ids[0]}.conf" +app_conf = Path(npm_root) / "data/nginx/proxy_host" / f"{app_ids[0]}.conf" + +if api_conf.exists(): + text = api_conf.read_text() + text = replace_server(text, api_host, 1) + text = replace_port(text, int(api_port), 1) + api_conf.write_text(text) + print(f"Synchronized {api_conf.name} -> {api_host}:{api_port}") + +if app_conf.exists(): + text = app_conf.read_text() + text = replace_server(text, web_host, 1) + text = replace_port(text, int(web_port), 1) + text = replace_server(text, api_host, 2) + text = replace_port(text, int(api_port), 2) + app_conf.write_text(text) + print(f"Synchronized {app_conf.name} -> {web_host}:{web_port} and API matcher -> {api_host}:{api_port}") +PY +fi + +if [[ "$restart_npm" == "0" ]]; then + echo "NPM container restart skipped because NPM_RESTART=0." +elif command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' | grep -qx nginx-proxy-manager; then + docker restart nginx-proxy-manager >/dev/null + echo "Restarted nginx-proxy-manager" +else + echo "NPM container restart skipped; restart it manually if it is not managed by Docker on this host." +fi + +if command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' | grep -qx "$npm_container"; then + "${sudo_cmd[@]}" python3 - "$npm_root" "$db_path" "$target" "$app_domain" "$api_domain" "$native_host" "$docker_web_host" "$docker_api_host" "$web_port" "$api_port" <<'PY' +import json +import re +import sqlite3 +import sys +from pathlib import Path + +( + npm_root, + db_path, + target, + app_domain, + api_domain, + native_host, + docker_web_host, + docker_api_host, + web_port, + api_port, +) = sys.argv[1:] + +web_host = native_host if target == "native" else docker_web_host +api_host = native_host if target == "native" else docker_api_host + +def has_domain(raw, domain): + try: + return domain in json.loads(raw) + except Exception: + return domain in raw + +def replace_nth(text, pattern, replacement, index): + matches = list(pattern.finditer(text)) + if len(matches) < index: + raise SystemExit(f"Unable to rewrite generated proxy config; expected match {index} for {pattern.pattern!r}") + match = matches[index - 1] + return text[:match.start()] + replacement(match) + text[match.end():] + +server_pattern = re.compile(r'^(?P\s*set \$server\s+)".*?";\s*$', re.M) +port_pattern = re.compile(r'^(?P\s*set \$port\s+)\d+;\s*$', re.M) + +def replace_server(text, host, index): + return replace_nth(text, server_pattern, lambda m: f'{m.group("prefix")}"{host}";', index) + +def replace_port(text, port, index): + return replace_nth(text, port_pattern, lambda m: f'{m.group("prefix")}{port};', index) + +con = sqlite3.connect(db_path) +rows = list(con.execute("select id, domain_names from proxy_host where is_deleted = 0")) +app_ids = [row_id for row_id, domains in rows if has_domain(domains, app_domain)] +api_ids = [row_id for row_id, domains in rows if has_domain(domains, api_domain)] +if len(app_ids) != 1 or len(api_ids) != 1: + raise SystemExit(f"Expected one app and one API proxy host, found app={app_ids} api={api_ids}") + +api_conf = Path(npm_root) / "data/nginx/proxy_host" / f"{api_ids[0]}.conf" +app_conf = Path(npm_root) / "data/nginx/proxy_host" / f"{app_ids[0]}.conf" + +if api_conf.exists(): + text = api_conf.read_text() + text = replace_server(text, api_host, 1) + text = replace_port(text, int(api_port), 1) + api_conf.write_text(text) + +if app_conf.exists(): + text = app_conf.read_text() + text = replace_server(text, web_host, 1) + text = replace_port(text, int(web_port), 1) + text = replace_server(text, api_host, 2) + text = replace_port(text, int(api_port), 2) + app_conf.write_text(text) +PY + reloaded=0 + for _ in 1 2 3 4 5; do + if docker exec "$npm_container" nginx -s reload >/dev/null 2>&1; then + reloaded=1 + break + fi + sleep 1 + done + if [[ "$reloaded" == "1" ]]; then + echo "Reloaded nginx-proxy-manager" + else + echo "Warning: nginx-proxy-manager reload did not succeed after restart; verify the container is healthy." >&2 + fi +fi diff --git a/deployment/native/systemd/system/islandflow-clickhouse.service b/deployment/native/systemd/system/islandflow-clickhouse.service new file mode 100644 index 0000000..79f8ed2 --- /dev/null +++ b/deployment/native/systemd/system/islandflow-clickhouse.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow ClickHouse +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/bin/env clickhouse-server --config-file=/etc/clickhouse-server/config.xml +Restart=always +RestartSec=5 +User=clickhouse +Group=clickhouse +StateDirectory=clickhouse +LimitNOFILE=262144 + +[Install] +WantedBy=multi-user.target diff --git a/deployment/native/systemd/system/islandflow-nats.service b/deployment/native/systemd/system/islandflow-nats.service new file mode 100644 index 0000000..a23eefc --- /dev/null +++ b/deployment/native/systemd/system/islandflow-nats.service @@ -0,0 +1,18 @@ +[Unit] +Description=Islandflow NATS JetStream +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/sbin/nats-server -js -sd /var/lib/islandflow/nats -a 127.0.0.1 -p 4222 -m 8222 +Restart=always +RestartSec=2 +User=nats +Group=nats +RuntimeDirectory=islandflow-nats +StateDirectory=islandflow/nats +LimitNOFILE=1048576 + +[Install] +WantedBy=multi-user.target diff --git a/deployment/native/systemd/system/islandflow-redis.service b/deployment/native/systemd/system/islandflow-redis.service new file mode 100644 index 0000000..3e63d74 --- /dev/null +++ b/deployment/native/systemd/system/islandflow-redis.service @@ -0,0 +1,18 @@ +[Unit] +Description=Islandflow Redis +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +ExecStart=/usr/bin/env redis-server /etc/islandflow/redis.conf --supervised systemd --daemonize no +Restart=always +RestartSec=2 +User=redis +Group=redis +RuntimeDirectory=islandflow-redis +StateDirectory=islandflow/redis +LimitNOFILE=65535 + +[Install] +WantedBy=multi-user.target diff --git a/deployment/native/systemd/user/islandflow-api.service b/deployment/native/systemd/user/islandflow-api.service new file mode 100644 index 0000000..1e6cc99 --- /dev/null +++ b/deployment/native/systemd/user/islandflow-api.service @@ -0,0 +1,19 @@ +[Unit] +Description=Islandflow API +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +Environment=API_HOST=0.0.0.0 +Environment=API_PORT=4000 +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun services/api/src/index.ts +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/deployment/native/systemd/user/islandflow-candles.service b/deployment/native/systemd/user/islandflow-candles.service new file mode 100644 index 0000000..585b37c --- /dev/null +++ b/deployment/native/systemd/user/islandflow-candles.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow candles +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun services/candles/src/index.ts +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/deployment/native/systemd/user/islandflow-compute.service b/deployment/native/systemd/user/islandflow-compute.service new file mode 100644 index 0000000..603f252 --- /dev/null +++ b/deployment/native/systemd/user/islandflow-compute.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow compute +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun services/compute/src/index.ts +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/deployment/native/systemd/user/islandflow-ingest-equities.service b/deployment/native/systemd/user/islandflow-ingest-equities.service new file mode 100644 index 0000000..837a04f --- /dev/null +++ b/deployment/native/systemd/user/islandflow-ingest-equities.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow ingest-equities +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun services/ingest-equities/src/index.ts +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/deployment/native/systemd/user/islandflow-ingest-news.service b/deployment/native/systemd/user/islandflow-ingest-news.service new file mode 100644 index 0000000..bca11a3 --- /dev/null +++ b/deployment/native/systemd/user/islandflow-ingest-news.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow ingest-news +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun services/ingest-news/src/index.ts +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/deployment/native/systemd/user/islandflow-ingest-options.service b/deployment/native/systemd/user/islandflow-ingest-options.service new file mode 100644 index 0000000..eac0a6c --- /dev/null +++ b/deployment/native/systemd/user/islandflow-ingest-options.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow ingest-options +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun services/ingest-options/src/index.ts +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/deployment/native/systemd/user/islandflow-web.service b/deployment/native/systemd/user/islandflow-web.service new file mode 100644 index 0000000..ce75e0b --- /dev/null +++ b/deployment/native/systemd/user/islandflow-web.service @@ -0,0 +1,19 @@ +[Unit] +Description=Islandflow web +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +Environment=WEB_HOST=0.0.0.0 +Environment=WEB_PORT=3000 +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/bin/sh -lc 'cd /home/delta/islandflow/apps/web && exec /home/delta/.bun/bin/bun x next start -H "$WEB_HOST" -p "$WEB_PORT"' +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html b/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html new file mode 100644 index 0000000..1d6e914 --- /dev/null +++ b/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html @@ -0,0 +1,482 @@ + + + + + + Daily Git Summary for 2026-05-18 (Merged View) + + + +
+
+ Daily Git Summary +

Standup summary for Monday, May 18, 2026 (after merge)

+

+ This regenerated report uses merged history for the full May 18 local-day window + (2026-05-18 00:00 -0400 through 2026-05-19 00:00 -0400). It now includes eight commits, + including native deployment work and the merge commit that landed that line of work on main. +

+
+
+ Commits +
8
+
+
+ Unique Files +
68
+
+
+ Insertions +
4244
+
+
+ Deletions +
194
+
+
+
+ +
+

Summary

+
+
+ User-facing delivery +

+ Commit 906fe411 added Alpaca news wire support across ingest, storage, API, and web terminal/news + route surfaces. +

+
+
+ Platform and deployment delivery +

+ Commits d589858c and bdb9d9a9 added native deployment workflow, infra/user units, + cutover, rollback, and health-check scripts, then merged via 8f0794dd (PR #2). +

+
+
+ Workflow and docs updates +

+ Commits 687a2170, 62aae708, 48095fce, and 04baeceb updated + beads/docs instructions and added turn/standup documentation. +

+
+
+
+ +
+

Changes Made

+
+
+
+ update beads + 687a2170 + 2026-05-18 03:15 -0400 + 1 file +
+

Touched deployment/docker/workspace-root/package.json with one-line change.

+
+ +
+
+ Implement native fast iterative deploy workflow + d589858c + 2026-05-18 03:34 -0400 + 17 files + +873 / -110 +
+
    +
  • Expanded scripts/deploy.ts for native deploy runtime behavior.
    • +
  • Added native user-unit templates and rollback/health tooling in deployment/native/.
    • +
  • Added associated plan and turn documents in docs/plans and docs/turns.
    • +
+
+ +
+
+ fix(api): remove duplicate alert context import + 48095fce + 2026-05-18 09:04 -0400 + 2 files +
+

Removed duplicate import in services/api/src/index.ts and added a turn doc.

+
+ +
+
+ docs(general): add 2026-05-17 standup summary + 62aae708 + 2026-05-18 09:05 -0400 + 2 files +
+

Added docs/general/2026-05-18-standup-summary-2026-05-17.html and updated beads state.

+
+ +
+
+ add alpaca news wire across ingest api and web + 906fe411 + 2026-05-18 16:55 -0400 + 31 files + +1407 / -50 +
+
    +
  • Created services/ingest-news and wired Alpaca backfill/websocket ingestion.
    • +
  • Added news types/storage contracts in packages/types and packages/storage.
    • +
  • Extended API live/history endpoints and web terminal/news route rendering.
    • +
+
+ +
+
+ Implement native public edge cutover + bdb9d9a9 + 2026-05-18 19:55 -0400 + 29 files + +1215 / -31 +
+
    +
  • Added native infra system units and scripts for bootstrap/start/stop/cutover/full rollback.
    • +
  • Updated deploy docs and runtime config files under deployment/native/config.
    • +
  • Added turn doc docs/turns/2026-05-18-native-public-edge-cutover.html.
    • +
+
+ +
+
+ Merge pull request 'Native public edge cutover with Docker rollback path' (#2) + 8f0794dd + 2026-05-19 00:09 +0000 + merge commit +
+

Merged native-deploy into main within the May 18 US/Eastern day window.

+
+ +
+
+ update turn docs and beads workflow + 04baeceb + 2026-05-18 21:32 -0400 + 1 file +
+

Updated repository-level instructions in AGENTS.md.

+
+
+
+ +
+

Context

+

+ The earlier report was generated before merged history included the native deployment branch on main. + This recreation uses git log --all over the same date window, so it captures both feature work and + merged operational/deployment work visible after PR merge. +

+
+ +
+

Important Implementation Details

+
+
+

News wire ingestion and delivery path

+

+ The news pipeline added a new ingest service and API fanout channel, then exposed UI surfaces in + /news and terminal panes. +

+ 
if (features.news) {
+  subscriptions.push({ channel: "news", snapshot_limit: LIVE_OPTIONS_HEAD_LIMIT });
+}
+
+
+

Native deployment hardening

+

+ Deployment scripts and unit templates now include direct scripts for cutover and rollback, with infra and + service checks under deployment/native/. +

+ 
deployment/native/cutover.sh
+deployment/native/full-rollback.sh
+deployment/native/install-infra-units.sh
+
+
+

Merged history effect on standup scope

+

+ The merged view increased the standup scope from 4 to 8 commits and from 35 to 68 unique files touched for the + same local-day window. +

+
+
+
+ +
+

Expected Impact for End-Users

+
+
+ Trading UI users +

Live news wire data is now available in terminal surfaces alongside existing market/event feeds.

+
+
+ Operators +

Native deployment and rollback procedures now have first-class scripted and documented paths.

+
+
+ Team reporting +

This standup report now matches merged repository history instead of pre-merge branch-local history.

+
+
+
+ +
+

Validation

+
    +
  • Used git fetch --all --prune before recomputing history.
    • +
  • Used git log --all over the May 18 ET window to include merged commits.
    • +
  • Used git log --stat --summary and --numstat to ground file and line-count statements.
    • +
  • No build/test commands were run because this task only regenerates reporting documentation.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • This report describes commit history only and does not infer intent beyond commit messages and touched files.
    • +
  • Commit 8f0794dd is timestamped in UTC; it still falls on May 18 in US/Eastern, so it is included.
    • +
  • Metrics are based on local git history at regeneration time and can change if additional backdated commits appear.
    • +
+
+ +
+

Follow-up Work

+
    +
  • This regeneration is tracked by beads issue islandflow-0ty.
    • +
  • No additional follow-up work was identified during this documentation-only task.
    • +
+
+
+ + diff --git a/docs/general/2026-05-18-standup-summary-2026-05-17.html b/docs/general/2026-05-18-standup-summary-2026-05-17.html new file mode 100644 index 0000000..ba21b1b --- /dev/null +++ b/docs/general/2026-05-18-standup-summary-2026-05-17.html @@ -0,0 +1,549 @@ + + + + + + Standup Summary for 2026-05-17 + + + +
+
+ Git Standup Summary +

Repository activity recorded for 2026-05-17

+

+ Yesterday's git history shows three main themes: frontend and API work to hydrate alert evidence from + ClickHouse, deploy workflow changes in scripts/deploy.ts, and Beads/Dolt remote setup plus + documentation updates. This summary is grounded in the commits, merged PRs, and touched files visible in the + repository history for 2026-05-17. +

+
+
+ Commit Count + 20 commits on 2026-05-17 +
+
+ Merges + 5 pull request merges +
+
+ File Footprint + 22 distinct paths touched +
+
+ Most Revisited + .beads/issues.jsonl, scripts/deploy.ts, apps/web/app/terminal.tsx +
+
+
+ +
+

Summary

+
+
+ Alert context from ClickHouse landed and was merged twice through follow-up PRs. + The core implementation appeared in commit c0b5b6d and merge PR #41 + (3e08955), then was extended in 58e57fa and merged through #43 + (a27d499) and a documentation polish PR #44 (49efc24). +
+
+ Deploy tooling changed in three steps. + The day included an allowlist tightening in 5ddfbfa, a new fast deploy mode in + 75ed6f3, and Forgejo-aware remote resolution in 6e6788b, all centered on + scripts/deploy.ts. +
+
+ Process and reporting work was visible alongside feature work. + Beads Dolt remote configuration was added in 37bd393, revised in d0d8bd4 and + cd0a1dd, and yesterday's prior standup report was added in 0416194. +
+
+
+ +
+

Changes Made

+
+
+
+ Frontend + API + c0b5b6d + 11:02 EDT +
+

Hydrate alert evidence from ClickHouse

+

+ Commit c0b5b6d added ClickHouse-backed alert context across storage, API, tests, and the + terminal UI. The same change set was merged as PR #41 in 3e08955. +

+
+ packages/storage/src/clickhouse.ts + services/api/src/alert-context.ts + services/api/src/index.ts + apps/web/app/terminal.tsx + apps/web/app/terminal.test.ts + packages/storage/tests/alerts.test.ts +
+
+ +
+
+ Deploy workflow + 5ddfbfa + 11:45 EDT +
+

Tighten deploy remote untracked allowlist

+

+ Commit 5ddfbfa, later merged as PR #42 in 8b166a5, narrowed the + remote untracked allowlist in scripts/deploy.ts. Two follow-up documentation commits, + 8631a53 and 219d3fd, recorded and corrected the validation notes for that + change. +

+
+ scripts/deploy.ts + docs/turns/2026-05-17-deploy-allowlist-pr-packaging.html +
+
+ +
+
+ Integration + 58e57fa + 20:18 EDT +
+

Add ClickHouse alert context hydration for alert drawers

+

+ Commit 58e57fa extended the earlier alert-context work, adding drawer-specific hydration in + the web app and API. A merge-conflict resolution commit dc932cf combined this with the + deploy allowlist branch before PR #43 merged in a27d499. +

+
+ apps/web/app/terminal.tsx + packages/storage/src/clickhouse.ts + services/api/src/index.ts + docs/turns/2026-05-17-clickhouse-alert-context.html +
+
+ +
+
+ Deploy workflow + 75ed6f3 + 22:53 EDT +
+

Add fast deploy mode for routine rollouts

+

+ Commit 75ed6f3 added a faster deploy path and updated both deployment readmes. Minutes + later, commit 6e6788b made deploy remote resolution Forgejo-aware, again in + scripts/deploy.ts. +

+
+ scripts/deploy.ts + deployment/docker/README.md + deployment/native/README.md + docs/turns/2026-05-17-add-fast-deploy-mode.html + docs/turns/2026-05-17-forgejo-deploy-remote-resolution.html +
+
+ +
+
+ Repo operations + 37bd393 + 06:41 EDT +
+

Beads remote setup and daily reporting

+

+ Commit 37bd393 configured the Beads Dolt remote in .beads/config.yaml, then + commits d0d8bd4 and cd0a1dd revised the same sync settings. Commit + 0416194 added the standup summary document for 2026-05-16 activity in + docs/general. +

+
+ .beads/config.yaml + .beads/issues.jsonl + docs/general/2026-05-17-standup-summary-2026-05-16.html +
+
+
+
+ +
+

Context

+
+
+ Merged PRs +
    +
  • #40 merged in 88b2c33: live tape scroll stability and related deploy/image work.
    • +
  • #41 merged in 3e08955: initial ClickHouse alert evidence hydration.
    • +
  • #42 merged in 8b166a5: deploy allowlist packaging follow-through.
    • +
  • #43 merged in a27d499: alert drawer hydration follow-up.
    • +
  • #44 merged in 49efc24: turn-document polish for alert context.
    • +
+
+
+ Most Touched Areas +
    +
  • .beads/issues.jsonl changed in 9 commits, reflecting issue tracking churn throughout the day.
    • +
  • scripts/deploy.ts changed in 3 direct commits tied to deploy safety and speed.
    • +
  • apps/web/app/terminal.tsx changed in 3 direct commits tied to live tape behavior and alert context.
    • +
  • Documentation output expanded across docs/turns and docs/general alongside implementation work.
    • +
+
+
+
+ +
+

Important Implementation Details

+
    +
  • + The ClickHouse alert-context work was not isolated to one layer. Commits c0b5b6d and + 58e57fa touched storage access, API wiring, UI presentation, and dedicated tests, which + makes this the clearest full-stack change in yesterday's history. +
    • +
  • + The deploy changes were incremental rather than a single rewrite. The history shows a narrowing change in + 5ddfbfa, an operator-speed path in 75ed6f3, and remote detection logic in + 6e6788b. +
    • +
  • + Merge commit dc932cf explicitly resolved conflicts between the alert-context and deploy + allowlist branches before later PR merges landed, so yesterday's main branch activity included integration + work as well as feature work. +
    • +
  • + Commit 073c1de created an empty forgejo.test path. The git history shows the file + creation, but no test content in that commit. +
    • +
+
+ +
+

Expected Impact for End-Users

+
    +
  • + User-facing terminal behavior changed in two visible ways: live tape scroll stability from + d334e16/#40 and richer alert evidence context from c0b5b6d, + 58e57fa, and the follow-up merges. +
    • +
  • + Deploy workflow commits affected operator tooling rather than customer-facing product screens. Those changes + should matter most to maintainers using scripts/deploy.ts and the deployment readmes. +
    • +
  • + Beads remote configuration and standup-report commits affected internal workflow and documentation, not + runtime product behavior. +
    • +
+
+ +
+

Validation

+
    +
  • + The turn document added in d334e16 records + bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts passing and + bun --cwd=apps/web run build passing. +
    • +
  • + The turn document added in c0b5b6d records + bun test packages/storage/tests, + bun test services/api/tests, + bun test apps/web/app/terminal.test.ts, and + bun --cwd=apps/web run build. +
    • +
  • + The polished turn document merged in 49efc24 records those alert-context validations as + passing. +
    • +
  • + The deploy allowlist turn document created in 8631a53 and corrected in 219d3fd + explicitly notes that a repository-wide bun test run reported failures at that point. +
    • +
  • + Later deploy-related turn documents added in 75ed6f3 and 6e6788b record full + bun test passing, with the Forgejo remote document stating 232 passing, + 0 failing. +
    • +
  • + This automation run only created documentation. No additional code validation command was run for this + summary itself. +
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • + This document summarizes repository history only. It does not infer goals beyond what commit subjects, PR + titles, merge structure, and touched files show. +
    • +
  • + Some PR context is visible only through merge commits. For example, PR #40 bundles scroll + stability with deploy and Docker-path changes, so the summary reports the merged file footprint rather than + inferring which portion dominated the review. +
    • +
  • + Validation evidence comes from committed turn documents, not from re-running every historical command during + this automation. +
    • +
+
+ +
+

Follow-up Work

+

+ No new follow-up Beads issue was created from the git summary itself. The Beads task for this automation run + is islandflow-x70, which tracks creation of this standup document and will be closed as part of + the session sync. +

+
+
+ + diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..211c5ac --- /dev/null +++ b/docs/index.html @@ -0,0 +1,638 @@ + + + + + + Islandflow Docs + + + +
+
+

Islandflow docs index

+

A browsable index of files under docs/ with filtering and grouped navigation.

+
+ +
+
35 of 35 files shown
+ + +
+ +
+
+

turns 28

+ +
+ + +
+

daily-git 1

+ +
+ + +
+

general 2

+ +
+ + +
+

plans 2

+ +
+ + +
+

root 2

+ +
+
+

No files match that filter.

+
+ + + + diff --git a/docs/plans/2026-05-18-native-fast-iterative-deploy-plan.html b/docs/plans/2026-05-18-native-fast-iterative-deploy-plan.html new file mode 100644 index 0000000..98fff10 --- /dev/null +++ b/docs/plans/2026-05-18-native-fast-iterative-deploy-plan.html @@ -0,0 +1,93 @@ + + + + + + Plan: Native Fast Iterative Deployment + + + +

Plan: Native, Fast, Iterative Deployment (Docker Optional)

+

Date: 2026-05-18

+ +
+

Plan Summary

+

Define and execute a fast iteration deployment path centered on host-native services, while preserving Docker as a fallback/runtime option.

+
+ +
+

Goals

+
    +
  • Reduce deploy turnaround time immediately.
    • +
  • Identify concrete bottlenecks with timing evidence.
    • +
  • Stabilize proxy/runtime topology for reliable production rollouts.
    • +
  • Support both native and Docker strategies with explicit guardrails.
    • +
+
+ +
+

Proposed Changes

+
    +
  • Use scoped fast deploys short-term.
    • +
  • Audit and remediate server-state blockers (duplicate compose/project drift).
    • +
  • Prepare native runtime prerequisites and checked-in operational assets.
    • +
  • Add deployment strategy prechecks, validation matrix, and staged cutover.
    • +
+
+ +
+

Relevant Context

+
    +
  • Open issue islandflow-2db: stale duplicate compose stack cleanup.
    • +
  • Open issue islandflow-sz8: public /replay/options proxy regression.
    • +
  • Open issue islandflow-38p: native unit templates and rollback helpers.
    • +
+
+ +
+

Implementation Steps

+
    +
  1. Stop the bleeding immediately (current deploy loop).
    2. +
  2. Get hard timing data per deploy phase.
    3. +
  3. Live server state audit (when plan mode is off).
    4. +
  4. Resolve duplicate compose stack first (islandflow-2db).
    5. +
  5. Fix NPM proxy route regression (islandflow-sz8).
    6. +
  6. Define target iterative deployment model.
    7. +
  7. Prepare native runtime prerequisites on VPS.
    8. +
  8. Checked-in native ops assets (islandflow-38p).
    9. +
  9. Switch proxy topology for native mode carefully.
    10. +
  10. Deploy strategy guardrails.
    11. +
  11. Validation matrix.
    12. +
  12. Staged cutover plan.
    13. +
  13. Decision: final default runtime.
    14. +
  14. Decision: optimization priority.
    15. +
  15. Decision: immediate live audit kickoff.
    16. +
+
+ +
+

Risks, Limitations, and Mitigations

+
    +
  • Risk: native runtime not yet production-hardened. Mitigation: keep Docker fallback and explicit gating.
    • +
  • Risk: proxy misrouting breaks API routes. Mitigation: route checks and post-change smoke validation.
    • +
  • Risk: operational drift on VPS. Mitigation: preflight audits and documented rollback steps.
    • +
+
+ +
+

Open Questions

+
    +
  • Should native become the default runtime now, or after hardening milestones?
    • +
  • Should backend iteration speed be prioritized ahead of web deploy speed?
    • +
  • Do we start immediate live server audit as soon as plan mode is disabled?
    • +
+
+ + diff --git a/docs/turns/2026-05-18-native-fast-iterative-deploy.html b/docs/turns/2026-05-18-native-fast-iterative-deploy.html new file mode 100644 index 0000000..45cba6c --- /dev/null +++ b/docs/turns/2026-05-18-native-fast-iterative-deploy.html @@ -0,0 +1,153 @@ + + + + + + 2026-05-18: Native fast iterative deploy + + + +
+
Turn document · 2026-05-18 03:29 EDT · Issues: islandflow-9rc, islandflow-38p, islandflow-bsg, islandflow-2db
+

Native fast iterative deploy

+

Implemented the native-first iterative deploy plan by adding deploy timing output, a safe worker-only native fast path, checked-in systemd user units and rollback helpers, server-local deploy execution, and updated live-operational documentation based on a fresh VPS audit.

+ +
+

Summary

+

The deploy flow now supports a safer native worker iteration model without requiring public edge cutover first. It can run directly from the VPS checkout without SSH, emits phase timings, includes checked-in native unit files plus install/rollback/smoke-test helpers, and documents the staged cutover path. During live audit, the previously reported /replay/options proxy issue and duplicate islandflow compose stack were both confirmed resolved on the host.

+
+ +
+

Changes Made

+
    +
  • Extended scripts/deploy.ts with deploy timing summaries for precheck, rollout, and verification phases.
    • +
  • Added --workers-only deploy scope for Docker and native runtimes.
    • +
  • Changed native --fast behavior so default full-scope fast deploys become worker-only instead of touching web/API.
    • +
  • Added native edge guardrails via DEPLOY_NATIVE_EDGE_READY=1 before web/API native deploys are allowed.
    • +
  • Added local-server execution mode so ./deploy can run from /home/delta/islandflow without SSHing back into the same host.
    • +
  • Added DEPLOY_SSH_KEY_PATH and DEPLOY_FORCE_SSH overrides for operators with non-default SSH setups.
    • +
  • Checked in native ops assets under deployment/native/:
    • +
  • install-user-units.sh, check-native-health.sh, rollback.sh
    • +
  • six user unit files in deployment/native/systemd/user/
    • +
  • Updated README.md, deployment/docker/README.md, and deployment/native/README.md to document the worker-first model, local execution mode, validation matrix, and staged cutover guidance.
    • +
  • Synced deployment/docker/workspace-root/package.json so Docker workspace validation passes again.
    • +
  • Installed the checked-in user unit files onto the live VPS in disabled form under ~/.config/systemd/user.
    • +
+
+ +
+

Context

+

The plan targeted faster deployment iteration while avoiding a premature move of the public edge away from the current Docker + Nginx Proxy Manager topology. The practical target was to make native runtime useful immediately for backend-worker iteration, while leaving web/API cutover deliberate and reversible.

+
+ +
+

Important Implementation Details

+
    +
  • Native fast mode now defaults to --workers-only; Docker fast mode still defaults to --services-only.
    • +
  • Native deploys that include public web/API scope now fail fast unless DEPLOY_NATIVE_EDGE_READY=1 is set.
    • +
  • Running from the live VPS checkout automatically switches deploy execution from SSH mode to local mode.
    • +
  • The checked-in native unit files are user units aimed at the current VPS layout: /home/delta/islandflow and /home/delta/.bun/bin/bun.
    • +
  • install-user-units.sh now installs units safely without enabling anything by default; enabling is explicit and scope-based.
    • +
  • rollback.sh intentionally uses a detached git ref to make one-off native rollback practical without rewriting branch history.
    • +
+ 
export DEPLOY_NATIVE_SYSTEMCTL_PREFIX="systemctl --user"
+./deploy main --runtime native --fast
+# resolves to worker-only native deploy before public edge cutover
+
+ +
+

Expected Impact for End-Users

+

End-users should see indirect benefits first: faster backend iteration, safer operational changes, and clearer rollback paths. Public traffic behavior should remain unchanged until a deliberate native edge cutover is performed.

+
+ +
+

Validation

+
    +
  • Passed: bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io
    • +
  • Passed: direct public /replay/options curl returned JSON
    • +
  • Passed: live Nginx Proxy Manager config contains /replay in the API route matcher
    • +
  • Passed: docker compose ls shows no duplicate islandflow project
    • +
  • Passed: bash -n deployment/native/install-user-units.sh deployment/native/check-native-health.sh deployment/native/rollback.sh
    • +
  • Passed: systemd-analyze verify deployment/native/systemd/user/*.service
    • +
  • Passed: bun run check:docker-workspace after syncing workspace snapshot
    • +
  • Passed: native edge guard refusal for bun run scripts/deploy.ts main --runtime native --web-only --no-build
    • +
  • Passed: ./deployment/native/install-user-units.sh followed by systemctl --user list-unit-files 'islandflow*'
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • Native units were installed but not enabled or started. This is intentional to avoid conflicting with the current Docker production edge.
    • +
  • Public web/API native deploys are still gated. Mitigation: explicit DEPLOY_NATIVE_EDGE_READY=1 acknowledgment and staged cutover documentation.
    • +
  • Native worker runtime has not yet been exercised live against the existing Docker worker stack. Mitigation: follow-up issue to soak worker-only native units before any default-runtime decision.
    • +
  • The known untracked Signal CLI tarball remains in the repo checkout. This is already tolerated by the deploy helper allowlist and was not changed here.
    • +
+
+ +
+

Follow-up Work

+
    +
  • Open follow-up: islandflow-vvw — stage native public-edge cutover after worker soak.
    • +
  • Decide whether native should ever replace Docker as the default runtime only after worker soak data and deliberate edge cutover validation.
    • +
+
+
+ + diff --git a/docs/turns/2026-05-18-native-public-edge-cutover.html b/docs/turns/2026-05-18-native-public-edge-cutover.html new file mode 100644 index 0000000..8d2d2b1 --- /dev/null +++ b/docs/turns/2026-05-18-native-public-edge-cutover.html @@ -0,0 +1,521 @@ + + + + + + Turn Document - Native Public Edge Cutover + + + +
+
+
Islandflow Turn Document
+

Native Public Edge Cutover

+

+ Completed the VPS native-first cutover for Islandflow infrastructure and app services while keeping Nginx + Proxy Manager as the outer edge and Docker as the rollback path. The final state now serves + flow.deltaisland.io and api.flow.deltaisland.io from the native web and API + processes, with verified public routing and a documented follow-up for the long-term API Cloudflare posture. +

+
+
+
Generated
+
2026-05-18 19:52 EDT
+
+
+
Primary Issue
+
islandflow-vvw
+
+
+
Follow-up
+
islandflow-fl5
+
+
+
Runtime State
+
Native active, Docker retained for rollback
+
+
+
+ +
+

Summary

+

+ The repository now contains the native infra units, native cutover scripts, Docker fallback adjustments, and + public-edge retargeting logic required to run Islandflow natively on the VPS. During validation, the live NPM + edge was switched from Docker container-name upstreams to native host ports, the host firewall was adjusted so + the NPM bridge could reach the native API, and the separate public API TLS problem was resolved by correcting + the Cloudflare DNS state for api.flow.deltaisland.io. +

+
+ +
+

Changes Made

+
    +
  • + Added checked-in native infra operations under deployment/native/, including + bootstrap-infra.sh, check-native-infra.sh, cutover.sh, + full-rollback.sh, start-infra.sh, and the native system units for NATS, Redis, + and ClickHouse. +
    • +
  • + Extended native app runtime units so the web and API bind on host-reachable interfaces, and forced the + native options ingest service to use the synthetic adapter during the cutover. +
    • +
  • + Updated services/api to support explicit host binding through API_HOST, and fixed + JetStream retention conversion in packages/bus so native services can start cleanly with the + configured max-age values. +
    • +
  • + Updated the Docker fallback assets to publish loopback web/API ports, share durable host data under + /var/lib/islandflow, and document the native-to-Docker rollback path. +
    • +
  • + Reworked deployment/native/switch-npm-edge.sh so it targets the NPM bridge gateway IP instead + of host.docker.internal, handles the root-owned NPM SQLite database, synchronizes generated + proxy_host configs, and reloads NPM deterministically after the edge switch. +
    • +
  • + Created Beads follow-up issue islandflow-fl5 for the remaining decision about whether + api.flow.deltaisland.io should remain DNS-only or be re-proxied through Cloudflare. +
    • +
+
+ +
+

Context

+

+ The migration started from a Docker-owned production baseline where NATS, Redis, ClickHouse, API, workers, and + web all ran in Compose, while NPM routed Islandflow traffic to Docker service names. That setup blocked a safe + native cutover for two reasons: the native services could not reach Docker-only infra reliably, and NPM could + not send public traffic to host-native processes without a deliberate upstream retarget. +

+

+ The runtime model for this work is exclusive ownership. Native and Docker are not allowed to run the same API + or worker scopes in parallel because JetStream durable consumers would conflict. The objective was therefore a + phased handoff, not a mixed soak for the same queues. +

+
+ +
+

Important Implementation Details

+
+
+

NPM edge targeting

+

+ NPM generates proxy_pass from a runtime-resolved $server variable, so the + Docker /etc/hosts alias for host.docker.internal was not sufficient. The switch + helper now detects the NPM bridge gateway and uses that IP for native upstreams. +

+
+
+

Firewall path

+

+ The host UFW policy already allowed port 3000 but not 4000. The live fix was a + source-scoped allow for the NPM bridge subnet so the containerized edge could reach the native API. +

+
+
+

Cloudflare API hostname

+

+ The API hostname failure was separate from the native cutover. The hostname is now a DNS-only + A record pointing at the VPS, which restored public TLS and health responses. +

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AreaImplementation detail
Native API + services/api/src/index.ts now accepts API_HOST and passes it to + Bun.serve. The native unit sets API_HOST=0.0.0.0 and + API_PORT=4000. +
Native web + The native web unit now starts from apps/web with + bun x next start -H "$WEB_HOST" -p "$WEB_PORT", avoiding the earlier repo-root startup + failure and binding the service on 0.0.0.0:3000. +
JetStream retention + Native startup exposed a retention-unit bug. The shared bus layer now converts stream max-age values with + nanos(...) and formats them back with millis(...). +
Docker fallback + Docker Compose now uses ISLANDFLOW_DATA_ROOT=/var/lib/islandflow, publishes loopback + ports, and keeps the fallback runtime compatible with the same durable data directories as the native + services. +
NPM switch helper + The helper now updates both the NPM database and the generated + /data/nginx/proxy_host/*.conf files, because a DB-only restart did not reliably rewrite the + live configs for Islandflow. +
+ + 
sudo ufw allow proto tcp from 172.18.0.0/16 to any port 4000 comment 'npm bridge to native api'
+
+ +
+

Expected Impact for End-Users

+
    +
  • + Public web and API traffic now reaches the native Islandflow services, which removes Docker from the primary + live request path while keeping the outer edge unchanged. +
    • +
  • + Same-origin public API routes such as /prints, /history, /replay, + /nbbo, and /ws/live continue to resolve correctly through the main app hostname. +
    • +
  • + Rollback remains fast and explicit: NPM can be pointed back at Docker service names and the Docker runtime + can reclaim the same durable data directories if native operation needs to be abandoned. +
    • +
+
+ +
+

Validation

+
+
+
Static checks
+
    +
  • bun run check:docker-workspace
    • +
  • docker compose -f deployment/docker/docker-compose.yml config --quiet
    • +
  • docker compose -f /home/delta/nginx-proxy-manager/docker-compose.yml config --quiet
    • +
  • bash -n deployment/native/*.sh
    • +
  • systemd-analyze verify deployment/native/systemd/user/*.service deployment/native/systemd/system/*.service
    • +
  • bun build services/api/src/index.ts --target=bun
    • +
  • bun build scripts/deploy.ts --target=bun
    • +
+
+
+
Native runtime
+
    +
  • ./deployment/native/check-native-health.sh full
    • +
  • curl http://127.0.0.1:4000/health
    • +
  • curl -I http://127.0.0.1:3000/
    • +
+
+
+
Public edge
+
    +
  • curl -I -fksS https://flow.deltaisland.io
    • +
  • curl -fksS https://api.flow.deltaisland.io/health
    • +
  • bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io
    • +
+
+
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • + The native ingest-options service required an explicit synthetic-adapter override because the environment file + still pointed at an Alpaca adapter that was returning 401 responses. The service now starts + cleanly for native cutover, but production adapter selection remains an operational decision. +
    • +
  • + The NPM helper still relies on direct config synchronization because NPM did not reliably regenerate the + Islandflow proxy files from SQLite changes alone. This is mitigated by keeping the synchronization logic + checked in and by reloading NPM as part of the helper itself. +
    • +
  • + The final public API recovery currently leaves api.flow.deltaisland.io as a DNS-only hostname. + That restored service, but it changes the edge posture relative to the web hostname and should be reviewed + deliberately. +
    • +
  • + A temporary Cloudflare API token was used to inspect and correct zone state during validation. That token + should be rotated outside this repository workflow. +
    • +
+
+ +
+

Follow-up Work

+
    +
  • + islandflow-fl5: decide whether api.flow.deltaisland.io should remain DNS-only or + be re-proxied through Cloudflare, then re-validate TLS, websocket, and operational behavior for the chosen + posture. +
    • +
  • + After operational soak, decide whether native should become the default production runtime or remain a + supported alternative with Docker as the preferred steady-state runtime. +
    • +
+
+
+ + diff --git a/docs/turns/2026-05-18-news-wire-view.html b/docs/turns/2026-05-18-news-wire-view.html new file mode 100644 index 0000000..be02f26 --- /dev/null +++ b/docs/turns/2026-05-18-news-wire-view.html @@ -0,0 +1,152 @@ + + + + + + Turn Report: News Wire View via Alpaca Feed + + + +
+

Created 2026-05-18 · Task: News Wire View via Alpaca Feed

+

News Wire View via Alpaca Feed

+
+ Summary +

+ Added an Alpaca-backed live news pipeline end to end: normalized NewsStory types, + a dedicated JetStream subject/stream, ClickHouse storage helpers with latest-revision semantics, + a new services/ingest-news service, API endpoints and live fanout, and a web + /news route plus Home preview with a right-side story drawer. +

+
+ +
+

Changes Made

+
    +
  • Added NewsStorySchema, the news live channel, and subscription parsing support in packages/types.
    • +
  • Added bus constants for the flow.news subject and NEWS stream.
    • +
  • Added ClickHouse news storage helpers, including recent, before-cursor, and after-cursor queries that collapse provider revisions to the latest row per provider + story_id.
    • +
  • Created services/ingest-news with Alpaca REST backfill, Alpaca websocket streaming, normalization, and deterministic ticker resolution.
    • +
  • Extended the API service to persist live news in the shared cache, expose GET /news and GET /history/news, and fan out news events on /ws/live.
    • +
  • Added a top-level /news route, primary nav entry, Home preview pane, replay-mode live-only empty states, and a sanitized full-story drawer.
    • +
  • Updated dev and deployment wiring so the new service is included in local runners and the Docker workspace snapshot.
    • +
+
+ +
+

Context

+

+ The plan called for a free-provider v1 news surface that behaves like the rest of Islandflow: + compact, evidence-first, and live-native. The implementation keeps replay intentionally out of scope + for news while still integrating news into the same live manifest, history pagination, rail navigation, + and drawer language used elsewhere in the terminal. +

+
+ +
+

Important Implementation Details

+
    +
  • Ticker resolution prefers provider symbols first, then falls back only to structured patterns in provider HTML: ticker anchors, EXCHANGE:SYM, and $SYM.
    • +
  • News history uses published_ts as the visible cursor while revisions are collapsed with a window function over provider, story_id ordered by updated_ts, ingest_ts, and seq.
    • +
  • The web drawer sanitizes provider HTML by removing scripts, inline event handlers, and unsupported tags; if sanitization yields nothing useful, the drawer falls back to stripped plain text.
    • +
  • Replay mode intentionally renders a clear empty state for news on both Home and /news instead of pretending news is replay-synced.
    • +
+ 
resolved_symbols = provider_symbols
+  or ticker anchors in content_html
+  or EXCHANGE:SYM matches
+  or $SYM matches
+
+ +
+

Expected Impact for End-Users

+

+ Traders can now monitor a dedicated live news wire inside Islandflow, spot symbol-linked headlines from + the Home view, and open full stories in-context without leaving the app. The displayed ticker chips are + grounded in stored provider and derived symbol metadata, which makes the feed safer to filter and trust. +

+
+ +
+

Validation

+
    +
  • Ran targeted Bun tests covering types, storage, API live-state behavior, ingest-news symbol resolution, route wiring, and terminal helpers.
    • +
  • Built the Next.js web app with bun --cwd=apps/web run build.
    • +
  • Ran bun run check:docker-workspace after syncing the deployment workspace snapshot.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • Replay support remains intentionally absent in v1; the UI now states that explicitly instead of showing misleading empty historical behavior.
    • +
  • The sanitizer is intentionally conservative and custom, which keeps dependencies light but may strip some harmless provider formatting.
    • +
  • The ingest service assumes Alpaca’s current REST and websocket news contracts; if Alpaca changes those payload shapes, the normalization layer will need adjustment.
    • +
+
+ +
+

Follow-up Work

+
    +
  • No additional follow-up issue was required during this turn.
    • +
  • Future extensions are still available behind the same contract: multi-provider aggregation, server-side symbol filtering, and replay-aware news history.
    • +
+
+
+ + diff --git a/docs/turns/2026-05-19-0739-update-readme-current-state.html b/docs/turns/2026-05-19-0739-update-readme-current-state.html new file mode 100644 index 0000000..77e0a2a --- /dev/null +++ b/docs/turns/2026-05-19-0739-update-readme-current-state.html @@ -0,0 +1,259 @@ + + + + + + README Current-State Update + + + +
+
+
Turn document · 2026-05-19 07:39 America/New_York
+

README Current-State Update

+

+ Resolved the README merge conflict and rewrote the project overview so it matches the current Islandflow codebase, including the smart-money taxonomy, Next.js 16 update, news ingest, desktop shell, and current deployment posture. +

+
+ README.md + smart-money taxonomy + Next.js 16.2.6 + deployment docs +
+
+ +
+

Summary

+

+ The README no longer contains conflict markers. It now gives a concise but current description of the platform, its runtime services, public smart-money categories, environment knobs, and supported deployment workflow. +

+
+ +
+

Changes Made

+
    +
  • Resolved the conflicted README by preserving the useful project-state content and removing stale simplified sections.
    • +
  • Added a first-class smart-money taxonomy section for the six public profiles: institutional_directional, retail_whale, event_driven, vol_seller, arbitrage, and hedge_reactive.
    • +
  • Documented that smart-money events are now the semantic object, while legacy classifier hits and alerts remain compatibility surfaces.
    • +
  • Updated the current implementation state to include Alpaca news ingest, profile-aware UI behavior, alert-context hydration, and the Electron shell.
    • +
  • Recorded the Next.js update to 16.2.6 with React and React DOM 19.2.0.
    • +
  • Clarified deployment: Docker is still the supported VPS path, native Bun/systemd rollout is experimental, and scoped deploy flags are available.
    • +
  • Aligned live-cache and web hot-window defaults with the current env examples and API defaults.
    • +
+
+ +
+

Context

+

+ Recent commits showed the README branch was carrying a Next.js upgrade, Alpaca news support, smart-money event work, and deployment helper changes. The prior README mixed both sides of a merge conflict and did not explain the newer taxonomy-driven classifier model. +

+
+ +
+

Important Implementation Details

+

+ The README intentionally treats FlowPacket as an intermediate clustering bridge and SmartMoneyEvent as the current semantic surface. It also documents abstention and suppression behavior so readers do not mistake every large print for a forced smart-money label. +

+

+ Deployment language now matches the current operations docs: ./deploy main defaults to the Docker path, --runtime native is available but experimental, and native rollout still depends on systemd units and reverse-proxy preparation. +

+
+ +
+

Relevant Diff Snippets

+

+ Diff snippets are formatted for readability in the same spirit as diffs.com, with only the most relevant README changes shown here. +

+ 
+## Smart-Money Classification Taxonomy
++
++Islandflow now emits first-class `SmartMoneyEvent` records instead of treating old classifier hits as the final semantic object.
++
++| Profile ID | Meaning | Common evidence |
++| --- | --- | --- |
++| `institutional_directional` | Large directional parent flow with stronger institutional-style conviction. | premium, size, sweep/burst behavior, aggressor imbalance, quote quality |
++| `retail_whale` | Large retail-style speculative bursts, often short-dated or attention-driven. | short-dated OTM concentration, burst prints, IV shock |
++| `event_driven` | Flow aligned to known upcoming events. | event-calendar proximity, expiry after event, pre-event concentration |
++| `vol_seller` | Premium-selling or short-volatility structure evidence. | sell-side premium, straddles/strangles |
++| `arbitrage` | Multi-leg or symmetric structures with low directional exposure. | matched leg symmetry, near-flat directional bias |
++| `hedge_reactive` | Hedge or dealer-reaction style flow around short-dated ATM/gamma context. | 0-2 DTE, near-ATM contracts, underlying move linkage |
+ 
+## Deployment Workflow
++
++Docker remains the supported and recommended path for the current VPS.
++
++./deploy main
++./deploy main --runtime docker
++./deploy current-branch
++./deploy current-branch --runtime docker
++
++Native deployment is opt-in and experimental:
++
++./deploy main --runtime native
++./deploy current-branch --runtime native
+
+ +
+

Expected Impact for End-Users

+

+ New contributors or future sessions should be able to read the README and understand what Islandflow currently does, which service owns each capability, how the smart-money labels should be interpreted, and which deployment command is appropriate for the VPS. +

+
+ +
+

Validation

+
    +
  • Confirmed no merge conflict markers remain with rg -n "<<<<<<<|=======|>>>>>>>" README.md.
    • +
  • Ran git diff --check; no whitespace or patch-format issues were reported.
    • +
  • Ran focused tests: bun test packages/types/tests/options-flow.test.ts packages/types/tests/live.test.ts packages/storage/tests/smart-money-events.test.ts services/compute/tests/parent-events.test.ts.
    • +
  • Focused test result: 12 pass, 0 fail.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • This was documentation-only, so no full production web build was run. The focused tests cover the smart-money/type/storage claims most relevant to the README update.
    • +
  • The README summarizes environment variables instead of listing every low-level classifier and dark-inference threshold. Detailed knobs remain available in .env.example and service code.
    • +
  • Native deployment remains experimental; the README calls that out directly and points to the dedicated native deployment document.
    • +
+
+ +
+

Follow-up Work

+
    +
  • islandflow-38p: add native deployment unit templates and rollback helpers.
    • +
  • islandflow-932: continue desktop follow-up native features.
    • +
  • islandflow-2db: manually remove stale local-infra containers from the VPS when doing server hygiene.
    • +
+
+
+ + diff --git a/docs/turns/2026-05-19-0805-clarify-repo-turn-doc-rules.html b/docs/turns/2026-05-19-0805-clarify-repo-turn-doc-rules.html new file mode 100644 index 0000000..9342851 --- /dev/null +++ b/docs/turns/2026-05-19-0805-clarify-repo-turn-doc-rules.html @@ -0,0 +1,200 @@ + + + + + + Clarify Repo Turn Documentation Rules + + + +
+
+
Turn document · 2026-05-19 08:05 America/New_York
+

Clarify Repo Turn Documentation Rules

+

+ Updated the repository instructions so Islandflow turn documents are clearly repo-local and styled through impeccable, without inheriting global non-repo computer-task styling. +

+
+ +
+

Summary

+

+ The repo AGENTS.md now removes a stray non-repo location rule and explicitly states that impeccable is the styling and layout authority for Islandflow turn documents when available. +

+
+ +
+

Changes Made

+
    +
  • Removed the confusing instruction to save non-repo documentation under ~/dev/docs/turns/.
    • +
  • Clarified that repository turn documents stay in docs/turns/.
    • +
  • Updated the format rule to say impeccable handles both structure and styling.
    • +
  • Added an explicit guard against applying global non-repo computer-task house styling to this repository's turn documents.
    • +
  • Clarified that the fallback standalone HTML path only applies when impeccable is unavailable or blocked by an actual error.
    • +
+
+ +
+

Context

+

+ The global agent instructions now distinguish repository implementation documentation from non-repo computer-task documentation. This repo file needed a small cleanup so it would not reintroduce ambiguity about location or styling. +

+
+ +
+

Important Implementation Details

+

+ This was a documentation-only change in AGENTS.md. It changes future agent behavior but does not alter runtime code, tests, deployment scripts, or application behavior. +

+
+ +
+

Relevant Diff Snippets

+ 
-## Important: If you are not working inside a git repository, save the document to `~/dev/docs/turns/`
+
+-Use the impeccable skill to structure the document as clean, readable HTML.
++Use the `impeccable` skill to structure and style the document as clean, readable HTML.
++
++For this repository, `impeccable` is the styling and layout authority for turn documents when available. Do not apply global non-repo computer-task house styling to repository turn documents.
+
+-If the impeccable skill is unavailable, still create a well-structured standalone HTML file with:
++If the `impeccable` skill is unavailable or blocked by an actual tool/file error, still create a well-structured standalone HTML file with:
+
+ +
+

Expected Impact for End-Users

+

+ Future Islandflow turns should produce documentation in the repo's docs/turns/ folder and let impeccable drive the visual treatment, making repo documentation less likely to inherit global computer-task styling. +

+
+ +
+

Validation

+
    +
  • Reviewed the AGENTS.md diff after patching.
    • +
  • Ran git diff --check with no whitespace errors.
    • +
  • No application test suite was run because this change only updates repository instructions.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+

+ This clarification depends on future agents reading both global and repo instructions. The new wording is intentionally direct about repo scope, location, and styling to reduce that risk. +

+
+ +
+

Follow-up Work

+

+ No follow-up issue is required for this patch. The related Beads task for this documentation cleanup is islandflow-lm6. +

+
+
+ + diff --git a/docs/turns/2026-05-19-fix-native-alpaca-news.html b/docs/turns/2026-05-19-fix-native-alpaca-news.html new file mode 100644 index 0000000..ddecc1a --- /dev/null +++ b/docs/turns/2026-05-19-fix-native-alpaca-news.html @@ -0,0 +1,233 @@ + + + + + + Turn Report: Fix Native Alpaca News + + + +
+

Created 2026-05-19 20:05 EDT · Branch: alpaca-news · Issue: islandflow-laq

+

Fix Native Alpaca News

+
+

+ Restored the native Alpaca news pipeline on the VPS by correcting Alpaca auth to use key ID + secret, + adding the missing native islandflow-ingest-news unit and worker-scope wiring, fixing the + Alpaca news backfill defaults to match the current API contract, requesting article content explicitly, + and repairing API-side news persistence so the feed is both live and queryable. +

+
+ VPS unit installed and enabled + Alpaca auth aligned to current docs + Live news confirmed + ClickHouse news history confirmed +
+
+ +
+

Summary

+

+ The original native news rollout failed for two separate reasons: the repo never fully wired + ingest-news into the native worker templates, and the service was still using bearer-style + Alpaca auth plus an oversized backfill limit that Alpaca's current News API rejects. After the service + started flowing again, one more pipeline gap appeared: the API fanned news out live but never persisted it + to ClickHouse, so /news stayed empty even when headlines showed up in the UI. +

+
+ +
+

Changes Made

+
    +
  • Added shared Alpaca credential helpers in packages/config with support for official key ID + secret auth and a legacy bearer fallback.
    • +
  • Rewired the Alpaca news, options, and equities adapters to use the shared auth model instead of hardcoded bearer headers and empty websocket secrets.
    • +
  • Added the checked-in native user unit deployment/native/systemd/user/islandflow-ingest-news.service.
    • +
  • Updated native install, health, cutover, rollback, and deploy-scope scripts so worker/native rollouts include ingest-news.
    • +
  • Corrected the native and Docker env/docs story to advertise current Alpaca credential names.
    • +
  • Lowered the default Alpaca news backfill limit from 100 to 50 to match the current endpoint contract.
    • +
  • Requested include_content=true for Alpaca news backfill and added a safe summary fallback when article content is missing.
    • +
  • Fixed API-side persistence by inserting each consumed news story into ClickHouse before live fanout.
    • +
  • On the VPS, created a fresh .env backup, added ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY, set ALPACA_NEWS_BACKFILL_LIMIT=50, switched the server checkout to alpaca-news, installed the new user unit, and restarted api plus ingest-news.
    • +
+
+ +
+

Context

+

+ Alpaca's current official auth docs require the APCA-API-KEY-ID and + APCA-API-SECRET-KEY header pair for market-data requests, and the current News endpoint + documents a limit range of 1..50 plus optional + include_content. This turn aligned Islandflow's native news path with those present-day + contracts instead of relying on the older single-token assumption that had drifted into the repo. +

+
+ +
+

Important Implementation Details

+
    +
  • The shared helper prefers ALPACA_API_KEY_ID + ALPACA_API_SECRET_KEY, also accepts ALPACA_KEY_ID + ALPACA_SECRET_KEY, and only falls back to legacy bearer auth when no secret is present.
    • +
  • The news backfill now requests article bodies explicitly. When Alpaca still omits full content, the service emits an escaped summary paragraph instead of a blank story body.
    • +
  • The native worker scope now treats ingest-news as a first-class worker everywhere the repo previously only handled options and equities.
    • +
  • The API now persists each consumed news story into ClickHouse before live fanout, which restores /news and history behavior without removing the live websocket path.
    • +
+
+ +
+

Relevant Diff Snippets

+ 
diff --git a/packages/config/src/alpaca.ts b/packages/config/src/alpaca.ts
++export const buildAlpacaAuthHeaders = (credentials) => ({
++  "APCA-API-KEY-ID": credentials.keyId,
++  "APCA-API-SECRET-KEY": credentials.secret
++})
++export const buildAlpacaWebSocketAuthMessage = (credentials) => ({
++  action: "auth",
++  key: credentials.keyId,
++  secret: credentials.secret
++})
+ 
diff --git a/services/ingest-news/src/index.ts b/services/ingest-news/src/index.ts
+-  ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(200).default(100),
++  ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(50).default(50),
++  url.searchParams.set("include_content", "true");
++  const contentHtml = item.content?.trim() || (summary ? `<p>${escapeHtml(summary)}</p>` : "");
+ 
diff --git a/services/api/src/index.ts b/services/api/src/index.ts
+   const payload = NewsStorySchema.parse(newsSubscription.decode(msg));
++  await insertNewsStory(clickhouse, payload);
+   await fanoutLive({ channel: "news" }, payload, "news");
+   msg.ack();
+

These snippets are included in a diff-style rendering format for fast review.

+
+ +
+

Expected Impact for End-Users

+

+ Native Islandflow deployments on the VPS now have a real Alpaca-backed news worker instead of a missing unit + and a crash loop. News stories populate with actual article body content in the feed more reliably, and the + API's /news path can serve persisted recent stories instead of only depending on live websocket + state. +

+
+ +
+

Validation

+
    +
  • Ran local targeted tests: bun test packages/config/tests packages/storage/tests/news.test.ts services/ingest-news/tests services/ingest-equities/tests and all passed.
    • +
  • Ran bun run check:docker-workspace and confirmed the Docker workspace snapshot stayed in sync.
    • +
  • Verified against current Alpaca docs that market-data auth uses key ID + secret and that the news endpoint limit is capped at 50.
    • +
  • On the VPS, confirmed the new islandflow-ingest-news.service unit is installed, enabled, and active under systemd --user.
    • +
  • Queried Alpaca directly from the VPS with the configured credentials and confirmed GET https://data.alpaca.markets/v1beta1/news?limit=1&sort=desc returned HTTP 200.
    • +
  • Restarted the VPS api and ingest-news services after the persistence fix so the API would store newly republished backfill stories.
    • +
  • Verified VPS API output: GET http://127.0.0.1:4000/news?limit=3 returned 3 recent real Alpaca stories with non-empty content_html payloads.
    • +
  • Verified ClickHouse persistence: SELECT count(), max(story_id), max(published_ts) FROM news returned 50 rows after the republished backfill.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • The server checkout still carries an unrelated untracked file, deployment/docker/signal-cli-0.14.3-Linux-native.tar.gz. It does not block the news fix, but it is repo hygiene debt on the VPS checkout.
    • +
  • The shared Alpaca helper keeps a legacy bearer fallback so older setups do not fail immediately, but the repo documentation now treats key ID + secret as the supported path.
    • +
  • Some Alpaca/Benzinga stories may still omit full content. The summary fallback prevents a blank drawer in those cases, but it cannot synthesize text Alpaca does not send.
    • +
+
+ +
+

Follow-up Work

+
    +
  • No new follow-up Beads issue was required to ship this repair.
    • +
  • If native Alpaca options or equities are re-enabled later, the shared credential changes in this turn already cover the same key ID + secret auth model.
    • +
  • If the team wants historical news beyond the startup backfill, the next logical extension is a scheduled catch-up cursor instead of only restart-time republishing.
    • +
+
+
+ + diff --git a/docs/turns/2026-05-19-harden-native-ssh-deploy-checks.html b/docs/turns/2026-05-19-harden-native-ssh-deploy-checks.html new file mode 100644 index 0000000..7cee829 --- /dev/null +++ b/docs/turns/2026-05-19-harden-native-ssh-deploy-checks.html @@ -0,0 +1,191 @@ + + + + + + 2026-05-19 Harden Native SSH Deploy Checks + + + +
+
+

Harden Native SSH Deploy Checks

+

+ Native deploys over SSH were failing for avoidable operator reasons: the remote shell did not inherit Bun's install path, and native verification assumed it was already running from the repository root before it called checked-in health scripts. This patch makes the SSH path more forgiving and fixes the verification working directory. +

+
Generated 2026-05-19 19:38 EDT
+
+ +
+

Summary

+

+ Updated scripts/deploy.ts so native SSH deploys prepend $HOME/.bun/bin when it exists, and native verification now explicitly cds into the remote repo before running the checked-in health helpers. +

+
+ +
+

Changes Made

+
    +
  • Prepended $HOME/.bun/bin during native remote precheck when available.
    • +
  • Prepended $HOME/.bun/bin during native remote rollout when available.
    • +
  • Changed native remote verification to run from /home/delta/islandflow before calling deployment/native/check-native-infra.sh.
    • +
+
+ +
+

Context

+

+ During a live native rollout, the deploy helper failed first because the non-login SSH shell could not find bun even though it was installed under the deploy user's home directory. After that was corrected on the host, worker rollout still reported failure because remote verification executed from the home directory and could not resolve the relative path to the checked-in infra check script. +

+
+ +
+

Important Implementation Details

+
    +
  • The fallback only adjusts PATH when $HOME/.bun/bin/bun exists, so it stays harmless on hosts that already expose Bun globally.
    • +
  • The repo-root cd keeps the existing relative helper calls intact instead of hardcoding every individual script path in multiple places.
    • +
  • This change improves SSH-based deploys without changing local-server deploy behavior.
    • +
+
+ +
+

Relevant Diff Snippets

+

Unified diff blocks below are formatted for diffs-compatible rendering.

+ 
diff --git a/scripts/deploy.ts b/scripts/deploy.ts
+@@ -754,6 +754,10 @@ set -euo pipefail
+ 
+ cd ${shellEscape(REMOTE_REPO)}
+ 
++if [[ -x "$HOME/.bun/bin/bun" ]]; then
++  export PATH="$HOME/.bun/bin:$PATH"
++fi
++
+ if ! command -v bun >/dev/null 2>&1; then
+
+@@ -855,6 +859,10 @@ set -euo pipefail
+ 
++if [[ -x "$HOME/.bun/bin/bun" ]]; then
++  export PATH="$HOME/.bun/bin:$PATH"
++fi
++
+ ${remoteGitUpdateScript(mode, remote, branch)}
+
+@@ -943,6 +951,12 @@ set -euo pipefail
+ 
++cd ${shellEscape(REMOTE_REPO)}
++
++if [[ -x "$HOME/.bun/bin/bun" ]]; then
++  export PATH="$HOME/.bun/bin:$PATH"
++fi
++
+ declare -a units=(${units})
+
+ +
+

Expected Impact for End-Users

+

+ End users should see fewer failed native deploy attempts and fewer partial restarts caused by tooling assumptions rather than application health. This lowers the odds of avoidable downtime during native rollouts. +

+
+ +
+

Validation

+
    +
  • Observed the original failures during live rollout: missing bun in SSH PATH and missing deployment/native/check-native-infra.sh during remote verification.
    • +
  • Used the patched operational path to complete native worker, API, and web rollouts successfully on the VPS.
    • +
  • Verified API health at http://127.0.0.1:4000/health and web health at both http://127.0.0.1:3000/ and https://flow.deltaisland.io.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • This patch does not solve the separate ingest-news credential problem. Full native deploys still need that unit and provider path to be made healthy before they are completely clean.
    • +
  • The VPS also needed a host-level Bun symlink during this recovery. The repo patch reduces dependence on that fix for future SSH deploys but does not remove it retroactively.
    • +
+
+ +
+

Follow-up Work

+
    +
  • islandflow-fmg: Keep the deploy helper aligned with the actual VPS runtime assumptions and add regression checks around native verification paths.
    • +
  • islandflow-wf5: Decide whether ingest-news and live options should stay provider-backed or remain intentionally synthetic until auth is hardened.
    • +
+
+
+ + diff --git a/docs/turns/2026-05-19-native-options-recovery-guardrails.html b/docs/turns/2026-05-19-native-options-recovery-guardrails.html new file mode 100644 index 0000000..441ade2 --- /dev/null +++ b/docs/turns/2026-05-19-native-options-recovery-guardrails.html @@ -0,0 +1,183 @@ + + + + + + 2026-05-19 Native Options Recovery Guardrails + + + +
+
+

Native Options Recovery Guardrails

+

+ The production outage turned out to be a native deployment config mismatch, not a data-pipeline code failure. I restored the VPS to the last known-good synthetic options adapter, then tightened the checked-in native deployment assets so they no longer imply a systemd override will beat the repo .env. +

+
Generated 2026-05-19 19:24 EDT
+
+ +
+

Summary

+

+ The repo-side change is small and targeted: remove the misleading Environment=OPTIONS_INGEST_ADAPTER=synthetic line from the checked-in native ingest-options unit, and document that Bun-launched services effectively take adapter selection from /home/delta/islandflow/.env. +

+
+ +
+

Changes Made

+
    +
  • Removed the checked-in systemd override from deployment/native/systemd/user/islandflow-ingest-options.service.
    • +
  • Added an explicit env-precedence warning to deployment/native/README.md.
    • +
  • Captured the live diagnosis that the native server had drifted to OPTIONS_INGEST_ADAPTER=alpaca while the prior Docker deployment was running synthetic options.
    • +
+
+ +
+

Context

+

+ On the VPS, islandflow-ingest-options.service was crash-looping with repeated 401 Unauthorized responses from Alpaca while the rest of the native stack stayed healthy. The previous Docker-owned islandflow-vps-ingest-options-1 container showed OPTIONS_INGEST_ADAPTER=synthetic, which explains why the UI had been healthy before the runtime transition. +

+
+ +
+

Important Implementation Details

+
    +
  • The checked-in unit already referenced /home/delta/islandflow/.env, and Bun's runtime env loading meant a conflicting adapter value there still won in practice.
    • +
  • The static key currently stored as ALPACA_API_KEY does not authenticate the failing market-data snapshot request as a Bearer token.
    • +
  • Because the real outage fix required a server-side .env correction, this repo patch focuses on preventing operator confusion during the next native cutover.
    • +
+
+ +
+

Relevant Diff Snippets

+

Unified diff blocks below are formatted for diffs-compatible rendering.

+ 
diff --git a/deployment/native/README.md b/deployment/native/README.md
+@@ -98,6 +98,8 @@ These are written for the current VPS layout:
+ - Bun binary: `/home/delta/.bun/bin/bun`
+ - env file: `/home/delta/islandflow/.env`
+ 
++Important: treat `/home/delta/islandflow/.env` as the effective source of truth for adapter selection. The Bun-launched services read that file directly at runtime, so a conflicting `OPTIONS_INGEST_ADAPTER` value in `.env` can still win over a systemd-only override and push `ingest-options` onto the wrong provider path.
++
+ ### Install the units
+
+diff --git a/deployment/native/systemd/user/islandflow-ingest-options.service b/deployment/native/systemd/user/islandflow-ingest-options.service
+@@ -7,7 +7,6 @@ Wants=network-online.target
+ Type=simple
+ WorkingDirectory=/home/delta/islandflow
+ EnvironmentFile=/home/delta/islandflow/.env
+-Environment=OPTIONS_INGEST_ADAPTER=synthetic
+ ExecStart=/home/delta/.bun/bin/bun services/ingest-options/src/index.ts
+
+ +
+

Expected Impact for End-Users

+

+ End users should not see the options tape stall the next time native units are installed or audited by following the checked-in assets. Operators now have a clearer paper trail that the actual runtime adapter comes from the deployment env file. +

+
+ +
+

Validation

+
    +
  • Verified the native outage mode on the VPS: islandflow-ingest-options.service crash-looped on Alpaca 401 responses.
    • +
  • Confirmed the previous Docker container had been running OPTIONS_INGEST_ADAPTER=synthetic.
    • +
  • After the server-side env fix, confirmed fresh rows in default.option_prints and new compute emissions in the native logs.
    • +
  • Ran git diff to verify the repo change stayed scoped to the deployment README and the checked-in user unit.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • The repo patch does not add new credential support for Alpaca. It only documents the current env-precedence behavior and removes a misleading override.
    • +
  • The live server is restored with synthetic options, which matches the last known-good Docker behavior, but it is not a true live Alpaca ingest path.
    • +
+
+ +
+

Follow-up Work

+
    +
  • islandflow-wf5: Decide whether production options should remain synthetic or move to a fully supported live provider configuration.
    • +
  • islandflow-wf5: If Alpaca live data is still desired, add a validated auth flow and a deploy-time smoke test that catches provider auth failures before cutover.
    • +
+
+
+ + diff --git a/docs/turns/2026-05-19-publish-docs-pages-index.html b/docs/turns/2026-05-19-publish-docs-pages-index.html new file mode 100644 index 0000000..9946b33 --- /dev/null +++ b/docs/turns/2026-05-19-publish-docs-pages-index.html @@ -0,0 +1,195 @@ + + + + + + Turn Report - Publish Docs to GitHub Pages + + + +
+

Publish docs/ to GitHub Pages with navigable index

+

Completed on May 19, 2026 at 9:38 AM ET.

+ +
+

Summary

+

+ Added an automated docs publishing flow to GitHub Pages and generated a new + docs/index.html browsing experience so docs are easy to navigate at + /islandflow/docs/. +

+
+ +
+

Changes Made

+
    +
  • Added scripts/generate-docs-index.mjs to build a browsable index of files under docs/.
    • +
  • Added .github/workflows/docs-pages.yml to publish docs to GitHub Pages on pushes to main.
    • +
  • Generated docs/index.html from current docs content.
    • +
  • Configured deployment artifact layout so docs are available at /docs/ under the project Pages site.
    • +
+
+ +
+

Context

+

+ The repository already stores operational and implementation documentation under + docs/, but there was no dedicated GitHub Pages pipeline and no curated + index page for discovery. This task focused on syncing that folder to Pages and + making it easy to browse by category and filename. +

+
+ +
+

Important Implementation Details

+
    +
  • The index generator excludes hidden files and avoids self-including docs/index.html.
    • +
  • Files are grouped by first path segment (turns, general, plans, and others) with quick category chips.
    • +
  • The index includes client-side filtering so users can search docs by path text in-browser.
    • +
  • Pages deployment packages a site/ payload where docs are copied into site/docs and root redirects to ./docs/.
    • +
+
+ +
+

Relevant Diff Snippets

+

+ Snippets are shown in a compact style aligned with diffs.com presentation patterns. +

+ 
+++ .github/workflows/docs-pages.yml
+name: Publish Docs
+on:
+  push:
+    branches: [main]
+    paths:
+      - "docs/**"
+      - "scripts/generate-docs-index.mjs"
+      - ".github/workflows/docs-pages.yml"
+  workflow_dispatch:
+
+jobs:
+  build:
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/configure-pages@v5
+      - run: node scripts/generate-docs-index.mjs
+      - run: cp -R docs/. site/docs/
+      - uses: actions/upload-pages-artifact@v3
+  deploy:
+    needs: build
+    steps:
+      - uses: actions/deploy-pages@v4
+ 
+++ scripts/generate-docs-index.mjs
+const files = await collectDocsFiles(docsDir);
+const html = renderDocument(files);
+await fs.writeFile(outputFile, html, "utf8");
+
+// Generated index features:
+// - grouped sections
+// - search filter
+// - file size and modified time metadata
+// - links preserving docs folder structure
+
+ +
+

Expected Impact for End-Users

+
    +
  • Docs are reachable via a stable Pages URL path: dirtydishes.github.io/islandflow/docs/.
    • +
  • Readers can quickly scan categories and search by filename instead of relying on raw directory browsing.
    • +
  • New docs added to the repository are published automatically on main pushes.
    • +
+
+ +
+

Validation

+
    +
  • Ran node scripts/generate-docs-index.mjs successfully.
    • +
  • Ran node --check scripts/generate-docs-index.mjs for syntax validation.
    • +
  • Confirmed generated index contains expected navigation/search markers and category anchors.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • GitHub Pages must be enabled for this repository and set to GitHub Actions deployment.
    • +
  • The index reflects files present at build time and does not include full-text search inside documents.
    • +
  • Markdown files are linked as-is; rendering behavior depends on GitHub Pages static hosting behavior.
    • +
+
+ +
+

Follow-up Work

+
    +
  • Add a docs landing page summary for key collections (turn docs, runbooks, daily notes).
    • +
  • Optionally add link-checking in CI for docs URLs and local references.
    • +
  • Consider tagging docs with metadata for richer filtering by date, topic, and type.
    • +
+
+
+ + diff --git a/docs/turns/2026-05-19-reconcile-pr-conflicts.html b/docs/turns/2026-05-19-reconcile-pr-conflicts.html new file mode 100644 index 0000000..2861d8b --- /dev/null +++ b/docs/turns/2026-05-19-reconcile-pr-conflicts.html @@ -0,0 +1,276 @@ + + + + + + Reconcile PR Conflicts + + + +
+
+
Turn document • 2026-05-19 18:56 ET
+

Reconcile PR Conflicts

+

+ Merged forgejo/main into nextjs-upgrade, resolved the checked-in Beads and README conflicts, kept the native deployment work from main, and updated the JetStream tests for the merged nanosecond retention behavior. +

+
+ +
+

Summary

+

+ The PR branch now incorporates the current mainline deployment changes while preserving the Next.js upgrade branch. The only hand-edited conflict resolution was in .beads/issues.jsonl and README.md; the rest of the mainline merge applied cleanly. +

+
+ +
+

Changes Made

+
    +
  • Resolved .beads/issues.jsonl by keeping issue records from both sides of the merge.
    • +
  • Resolved the README deployment workflow section by combining the branch’s command-oriented guidance with main’s newer worker-only, local-server, and native edge cutover notes.
    • +
  • Accepted mainline native deployment assets, Docker deployment refinements, API host binding support, deploy timing output, and worker-only deployment scope.
    • +
  • Adjusted packages/bus/tests/jetstream.test.ts so retention assertions expect NATS nanoseconds after the merged runtime change.
    • +
+
+ +
+

Context

+

+ The branch was clean before the merge, but Forgejo reported PR conflicts against main. Reproducing the merge locally showed conflicts in the Beads export file and the README deployment section. The automatic merge also brought in mainline native deployment work that touched deploy scripts, Docker deployment files, native systemd templates, public edge documentation, the API host setting, and JetStream retention units. +

+
+ +
+

Important Implementation Details

+
+
+ README resolution +

Kept Docker as the recommended VPS path, preserved explicit deploy commands, and added --workers-only, local server execution, and native worker iteration guidance.

+
+
+ Beads resolution +

Removed conflict markers without dropping either branch’s issue records, so Beads history remains complete.

+
+
+ Test repair +

Main now stores JetStream max_age in nanoseconds via NATS helpers. Tests now assert against nanos(...) instead of raw millisecond values.

+
+
+
+ +
+

Relevant Diff Snippets

+

+ Diff snippets are presented in the style of diffs.com, using structured additions and deletions for quick review. +

+ 
diff --git a/README.md b/README.md
+- Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, `--fast`, `--no-build`, and `--force-recreate`.
++ Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, `--workers-only`, `--fast`, `--no-build`, and `--force-recreate`.
++ When run from `/home/delta/islandflow` on the VPS itself, `./deploy` can execute locally instead of SSHing back into the same server.
+- Native deployment expects Bun, systemd units, host-reachable infra, and deliberate reverse-proxy changes. The open follow-up is to add native unit templates and rollback helpers.
++ Native deployment expects Bun, systemd units, host-reachable infra, and deliberate reverse-proxy changes. Native deploys are intended primarily for worker-only fast iteration until the public edge is cut over deliberately.
+ + 
diff --git a/packages/bus/tests/jetstream.test.ts b/packages/bus/tests/jetstream.test.ts
+- import type { JetStreamManager, StreamConfig } from "nats";
++ import { nanos, type JetStreamManager, type StreamConfig } from "nats";
+-       max_age: 3_600_000,
++       max_age: nanos(3_600_000),
+-       max_age: 43_200_000,
++       max_age: nanos(43_200_000),
+
+ +
+

Expected Impact for End-Users

+

+ The PR should no longer show merge conflicts against main. Users and operators get the Next.js upgrade branch plus the newer deployment safety work from main, including worker-only native deploy guidance and current Docker deployment notes. +

+
+ +
+

Validation

+
    +
  • git diff --check passed.
    • +
  • bun run scripts/deploy.ts --help passed.
    • +
  • bun run check:docker-workspace passed.
    • +
  • bun test services/api/tests packages/bus/tests passed with 45 tests.
    • +
  • bun --cwd=apps/web run build passed on Next.js 16.2.6.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
+

+ The first focused test run failed because the merged JetStream implementation correctly returned nanosecond retention values while the existing tests still expected milliseconds. The tests were updated to use the same NATS nanos helper as the runtime behavior, then the suite passed. +

+
+
+ +
+

Follow-up Work

+
    +
  • No new follow-up was created from this reconciliation.
    • +
  • Existing deployment follow-ups remain in Beads, including native public edge posture and cutover decisions.
    • +
+
+
+ + diff --git a/docs/turns/2026-05-19-upgrade-nextjs-16.html b/docs/turns/2026-05-19-upgrade-nextjs-16.html new file mode 100644 index 0000000..cdbb2f1 --- /dev/null +++ b/docs/turns/2026-05-19-upgrade-nextjs-16.html @@ -0,0 +1,229 @@ + + + + + + Upgrade apps/web to Next.js 16.2.6 + + + + + + +
+
+

Turn document · 2026-05-19

+

Upgrade apps/web to Next.js 16.2.6

+

The web app now builds and passes focused validation on Next.js 16.2.6 with React 19. The change keeps route behavior and synthetic admin proxy behavior intact while refreshing the root and Docker workspace Bun lockfiles.

+
+ +
+

Summary

+

Upgraded apps/web from the Next 14 / React 18 stack to Next 16.2.6 and React 19.2.x. The Bun lockfile was refreshed, the Docker workspace lock snapshot was synced, and a React 19 nullable ref type issue exposed by the Next 16 build was fixed.

+
+ +
+

Changes Made

+
    +
  • Updated apps/web/package.json to request next ^16.2.6, react ^19.2.0, and react-dom ^19.2.0.
    • +
  • Updated React type dependencies to @types/react ^19.2.7 and added @types/react-dom ^19.2.3.
    • +
  • Ran bun install, which resolved Next to 16.2.6 and React/React DOM to 19.2.6 in bun.lock.
    • +
  • Ran bun run sync:docker-workspace so deployment/docker/workspace-root/bun.lock matches the root lock snapshot.
    • +
  • Adjusted the terminal list ref types to accept HTMLDivElement | null, matching React 19's stricter ref object typing.
    • +
  • Allowed Next 16 to regenerate apps/web/next-env.d.ts with its updated TypeScript reference comment and generated route type import.
    • +
+
+ +
+

Context

+

The requested upgrade was intentionally dependency-focused. No routes, backend contracts, environment variable names, or shared package exports were changed. Before editing, the web build and the targeted route tests passed on the previous locked Next 14.2.35 stack.

+
+ +
+

Important Implementation Details

+

No broad codemod was run. The only source-code change was a targeted type correction in apps/web/app/terminal.tsx. Next 16's build now runs with Turbopack by default in this project and completed successfully after the ref typing was narrowed to the actual nullable runtime value.

+

The Docker workspace sync changed the mirrored lockfile, but did not need to rewrite the mirrored package manifest or TypeScript base config.

+
+ +
+

Relevant Diff Snippets

+ 
"next": "^16.2.6",
+"react": "^19.2.0",
+"react-dom": "^19.2.0",
+"@types/react": "^19.2.7",
+"@types/react-dom": "^19.2.3"
+ 
type ListScrollState = {
+  listRef: React.RefObject<HTMLDivElement | null>;
+  listNode: HTMLDivElement | null;
+  setListRef: (node: HTMLDivElement | null) => void;
+};
+
+ +
+

Expected Impact for End-Users

+

There should be no intentional user-facing behavior change. The expected visible behavior remains: /, /tape, and /news render the terminal app; /signals, /charts, and /replay redirect to /; synthetic admin API routes keep their gated proxy behavior.

+
+ +
+

Validation

+
    +
  • Baseline before edits: bun --cwd=apps/web run build passed on Next 14.2.35.
    • +
  • Baseline before edits: bun test apps/web/app/routes.test.ts passed, 3 tests.
    • +
  • Baseline before edits: bun test apps/web/app/terminal.test.ts passed, 70 tests.
    • +
  • Baseline before edits: bun test apps/web/app/api/admin/synthetic/routes.test.ts passed, 4 tests.
    • +
  • After upgrade: bun --cwd=apps/web run build passed on Next 16.2.6.
    • +
  • After upgrade: bun test apps/web/app/routes.test.ts passed, 3 tests.
    • +
  • After upgrade: bun test apps/web/app/terminal.test.ts passed, 70 tests.
    • +
  • After upgrade: bun test apps/web/app/api/admin/synthetic/routes.test.ts passed, 4 tests.
    • +
  • After upgrade: bun run check:docker-workspace passed.
    • +
  • Manual smoke: bun run dev:web served Next 16.2.6 on localhost:3000.
    • +
  • Manual smoke: browser checks confirmed /, /tape, and /news render with title Islandflow Terminal.
    • +
  • Manual smoke: /signals, /charts, and /replay returned 307 redirects to /.
    • +
  • Manual smoke: synthetic admin status and control routes returned gated 404 responses when the internal UI flag was off.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+

During dev:web smoke testing, the browser logged a live socket channel validation warning because only the web app was running, not the full backend service stack. Route rendering, redirect behavior, and gated synthetic admin proxy behavior were still verified. A full-stack live feed verification can be done separately with bun run dev if needed.

+

The upgrade did not include a full monorepo test run because the acceptance bar was intentionally web-focused.

+
+ +
+

Follow-up Work

+
    +
  • No required follow-up Beads issue was opened for this upgrade.
    • +
  • Optional: run a full-stack live feed smoke with infra and services running if you want runtime stream confidence beyond the web-focused acceptance checks.
    • +
  • Optional: run the full monorepo bun test suite before a larger release branch merge.
    • +
+
+ +
+

Helpful Links

+ +
+
+ + diff --git a/docs/turns/2026-05-20-fix-alert-flow-packet-history.html b/docs/turns/2026-05-20-fix-alert-flow-packet-history.html new file mode 100644 index 0000000..d7e2b30 --- /dev/null +++ b/docs/turns/2026-05-20-fix-alert-flow-packet-history.html @@ -0,0 +1,412 @@ + + + + + + Fix historical alert flow packet persistence in the web terminal + + + + + + +
+
+

Turn Document · 2026-05-20 02:56 EDT

+

Historical Alert Flow Packets Persist Again

+

Alert detail drawers now resolve persisted flow packets from ClickHouse-backed historical context instead of assuming the first evidence reference is the packet. This restores packet visibility for replayed and older alerts after their Redis hot-cache entries have aged out.

+
+ Beads: islandflow-yza + Surface: apps/web terminal + Validation: tests + prod build +
+
+ +
+
+

Summary

+

The web terminal was assuming alert.evidence_refs[0] always pointed at a flow packet. For compute-generated alerts, the first evidence ref is often the smart-money event id, with the actual packet id later in the list. That made persisted historical packets look missing even when ClickHouse context had already hydrated them successfully.

+
+ +
+

Changes Made

+
    +
  • Added shared alert helpers in apps/web/app/terminal.tsx to extract all flow-packet refs from an alert and resolve the first hydrated packet semantically.
    • +
  • Switched the alert drawer's selected packet lookup to use the shared resolver instead of the first evidence ref.
    • +
  • Updated alert-underlying inference, visible-alert prefetch, pinned-flow retention keys, and classifier-hit-to-alert matching to use the same alert packet semantics.
    • +
  • Added focused regression coverage in apps/web/app/terminal.test.ts for alerts whose packet ref is not the first evidence entry.
    • +
+
+ +
+

Context

+

Islandflow alert detail views combine live Redis retention with ClickHouse historical hydration. Once a packet leaves the hot cache, the UI must treat ClickHouse-loaded evidence as first-class persisted context, not as a degraded fallback. The bug was in the web client’s interpretation of alert evidence ordering, not in the persistence of the packet itself.

+
+ Historical packet context was already present. The terminal simply was not selecting it unless the packet id happened to be the first evidence ref. +
+
+ +
+

Important Implementation Details

+
    +
  • The fix is backward-compatible with already-persisted alerts because it tolerates existing evidence ordering instead of rewriting stored records.
    • +
  • The shared resolver centralizes the packet-selection rule so replay, pinning, and alert navigation do not drift apart again.
    • +
  • The classifier-hit alert matching path now finds alerts by any embedded packet ref, which improves consistency when opening related alert context from signal panes.
    • +
+
+ +
+

Relevant Diff Snippets

+
+
+

apps/web/app/terminal.tsx · alert packet resolution

+
+ 
-const packetId = selectedAlert.evidence_refs[0];
+-return packetId ? resolvedFlowPacketMap.get(packetId) ?? null : null;
++return resolveAlertFlowPacket(selectedAlert, resolvedFlowPacketMap);
+
+ +
+

apps/web/app/terminal.tsx · prefetch and alert matching

+
+ 
-const visiblePacketIds = visibleAlerts
+-  .map((alert) => alert.evidence_refs[0] ?? null)
+-  .filter((id): id is string => Boolean(id) && id.startsWith("flowpacket:"));
++const visiblePacketIds = visibleAlerts.flatMap((alert) => getAlertFlowPacketRefs(alert));
+
+-alertsFeed.items.find((item) => item.trace_id === desiredTrace || item.evidence_refs[0] === packetId)
++alertsFeed.items.find(
++  (item) => item.trace_id === desiredTrace || getAlertFlowPacketRefs(item).includes(packetId)
++)
+
+
+

These snippets are rendered client-side with Diffs using the same old/new code blocks shown in the fallback text if the library cannot load.

+
+ +
+

Expected Impact for End-Users

+

Older or replayed alerts should now show their persisted flow packet summary in the detail drawer even after the Redis hot cache no longer has that packet. Users investigating signal history should keep the same evidence continuity they get from live data: packet summary, print context, and related alert linkage stay intact.

+
+ +
+

Validation

+
    +
  • bun test apps/web/app/terminal.test.ts passed with 72 tests.
    • +
  • bun --cwd=apps/web run build passed on Next.js 16.2.6.
    • +
  • The new tests specifically cover alerts where a smart-money event id precedes the packet id in evidence_refs.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • This change does not alter how compute persists alert evidence ordering. Instead, it makes the terminal resilient to existing and future mixed evidence lists.
    • +
  • The Diffs rendering in this document loads from the published package at view time. A plain-text fallback is included directly in the HTML so the document remains readable offline.
    • +
  • No full monorepo test sweep was run because the change was isolated to the web terminal alert-context path.
    • +
+
+ +
+

Follow-up Work

+
    +
  • No additional Beads issue was required for this fix.
    • +
  • Optional: audit whether compute should emit packet ids before higher-level event ids in evidence_refs for simpler downstream consumers.
    • +
  • Optional: add a small integration test around alert drawer selection if the web app gains component-level interaction tests later.
    • +
+
+
+
+ + + + diff --git a/packages/bus/src/jetstream.ts b/packages/bus/src/jetstream.ts index 2eaf6a0..04bfa85 100644 --- a/packages/bus/src/jetstream.ts +++ b/packages/bus/src/jetstream.ts @@ -9,7 +9,9 @@ import { type StreamUpdateConfig, JSONCodec, type JsMsg, - createInbox + createInbox, + nanos, + millis } from "nats"; import { getKnownStreamDefinitions, getStreamDefinition, type StreamRetentionClass } from "./streams"; @@ -164,13 +166,13 @@ export const resolveStreamRetention = ( ): Pick => { if (streamClass === "raw") { return { - max_age: parseBoundedNumber(env.STREAM_RAW_MAX_AGE_MS, 3_600_000), + max_age: nanos(parseBoundedNumber(env.STREAM_RAW_MAX_AGE_MS, 3_600_000)), max_bytes: parseBoundedNumber(env.STREAM_RAW_MAX_BYTES, 536_870_912) }; } return { - max_age: parseBoundedNumber(env.STREAM_DERIVED_MAX_AGE_MS, 43_200_000), + max_age: nanos(parseBoundedNumber(env.STREAM_DERIVED_MAX_AGE_MS, 43_200_000)), max_bytes: parseBoundedNumber(env.STREAM_DERIVED_MAX_BYTES, 268_435_456) }; }; @@ -417,7 +419,7 @@ const formatBytes = (value: number): string => { }; const formatRetentionSummary = (config: StreamConfig): string => { - return `age=${formatDurationMs(Number(config.max_age))} bytes=${formatBytes(config.max_bytes)} replicas=${config.num_replicas} retention=${config.retention} discard=${config.discard}`; + return `age=${formatDurationMs(millis(Number(config.max_age)))} bytes=${formatBytes(config.max_bytes)} replicas=${config.num_replicas} retention=${config.retention} discard=${config.discard}`; }; const formatReportLine = ( @@ -442,12 +444,12 @@ const formatReportLine = ( const details = report.retentionDrift .map((delta) => { const desiredValue = delta.field === "max_age" - ? formatDurationMs(Number(delta.desired)) + ? formatDurationMs(millis(Number(delta.desired))) : delta.field === "max_bytes" ? formatBytes(Number(delta.desired)) : formatStructuredValue(delta.desired); const currentValue = delta.field === "max_age" - ? formatDurationMs(Number(delta.current)) + ? formatDurationMs(millis(Number(delta.current))) : delta.field === "max_bytes" ? formatBytes(Number(delta.current)) : formatStructuredValue(delta.current); diff --git a/packages/bus/src/streams.ts b/packages/bus/src/streams.ts index eeb8116..b23c125 100644 --- a/packages/bus/src/streams.ts +++ b/packages/bus/src/streams.ts @@ -7,6 +7,7 @@ import { STREAM_EQUITY_QUOTES, STREAM_FLOW_PACKETS, STREAM_INFERRED_DARK, + STREAM_NEWS, STREAM_OPTION_NBBO, STREAM_OPTION_PRINTS, STREAM_OPTION_SIGNAL_PRINTS, @@ -19,6 +20,7 @@ import { SUBJECT_EQUITY_QUOTES, SUBJECT_FLOW_PACKETS, SUBJECT_INFERRED_DARK, + SUBJECT_NEWS, SUBJECT_OPTION_NBBO, SUBJECT_OPTION_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, @@ -53,7 +55,8 @@ export const STREAM_CATALOG: readonly KnownStreamDefinition[] = [ retentionClass: "derived" }, { name: STREAM_CLASSIFIER_HITS, subject: SUBJECT_CLASSIFIER_HITS, retentionClass: "derived" }, - { name: STREAM_ALERTS, subject: SUBJECT_ALERTS, retentionClass: "derived" } + { name: STREAM_ALERTS, subject: SUBJECT_ALERTS, retentionClass: "derived" }, + { name: STREAM_NEWS, subject: SUBJECT_NEWS, retentionClass: "derived" } ]; const STREAM_CATALOG_BY_NAME = new Map(STREAM_CATALOG.map((definition) => [definition.name, definition])); diff --git a/packages/bus/src/subjects.ts b/packages/bus/src/subjects.ts index 6b21afd..956d357 100644 --- a/packages/bus/src/subjects.ts +++ b/packages/bus/src/subjects.ts @@ -22,3 +22,5 @@ export const STREAM_CLASSIFIER_HITS = "CLASSIFIER_HITS"; export const SUBJECT_CLASSIFIER_HITS = "flow.classifier_hits"; export const STREAM_ALERTS = "ALERTS"; export const SUBJECT_ALERTS = "flow.alerts"; +export const STREAM_NEWS = "NEWS"; +export const SUBJECT_NEWS = "flow.news"; diff --git a/packages/bus/tests/jetstream.test.ts b/packages/bus/tests/jetstream.test.ts index 8e25773..671632a 100644 --- a/packages/bus/tests/jetstream.test.ts +++ b/packages/bus/tests/jetstream.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "bun:test"; -import type { JetStreamManager, StreamConfig } from "nats"; +import { nanos, type JetStreamManager, type StreamConfig } from "nats"; import { auditStreamConfig, buildKnownStreamConfig, @@ -52,14 +52,14 @@ const buildAllKnownConfigs = (env: Record = {}) => { describe("jetstream retention defaults", () => { it("resolves raw defaults to 60m and 512 MiB", () => { expect(resolveStreamRetention("raw")).toEqual({ - max_age: 3_600_000, + max_age: nanos(3_600_000), max_bytes: 536_870_912 }); }); it("resolves derived defaults to 12h and 256 MiB", () => { expect(resolveStreamRetention("derived")).toEqual({ - max_age: 43_200_000, + max_age: nanos(43_200_000), max_bytes: 268_435_456 }); }); @@ -71,7 +71,7 @@ describe("jetstream retention defaults", () => { STREAM_RAW_MAX_BYTES: "5678" }) ).toEqual({ - max_age: 1234, + max_age: nanos(1234), max_bytes: 5678 }); }); diff --git a/packages/config/src/alpaca.ts b/packages/config/src/alpaca.ts new file mode 100644 index 0000000..697d65b --- /dev/null +++ b/packages/config/src/alpaca.ts @@ -0,0 +1,76 @@ +export type AlpacaCredentials = { + keyId: string; + secret: string; + legacyToken: string; + usesLegacyBearer: boolean; +}; + +type AlpacaCredentialEnv = { + ALPACA_API_KEY?: string; + ALPACA_API_KEY_ID?: string; + ALPACA_KEY_ID?: string; + ALPACA_API_SECRET_KEY?: string; + ALPACA_SECRET_KEY?: string; +}; + +const normalize = (value: string | undefined): string => value?.trim() ?? ""; + +export const resolveAlpacaCredentials = ( + env: AlpacaCredentialEnv +): AlpacaCredentials => { + const legacyToken = normalize(env.ALPACA_API_KEY); + const explicitKeyId = + normalize(env.ALPACA_API_KEY_ID) || normalize(env.ALPACA_KEY_ID); + const secret = + normalize(env.ALPACA_API_SECRET_KEY) || normalize(env.ALPACA_SECRET_KEY); + const keyId = explicitKeyId || legacyToken; + const usesLegacyBearer = !explicitKeyId && !secret && legacyToken.length > 0; + + return { + keyId, + secret, + legacyToken, + usesLegacyBearer + }; +}; + +export const hasAlpacaCredentials = (credentials: AlpacaCredentials): boolean => { + if (credentials.usesLegacyBearer) { + return credentials.legacyToken.length > 0; + } + + return credentials.keyId.length > 0 && credentials.secret.length > 0; +}; + +export const buildAlpacaAuthHeaders = ( + credentials: AlpacaCredentials +): Record => { + if (credentials.usesLegacyBearer) { + return { + Authorization: `Bearer ${credentials.legacyToken}` + }; + } + + return { + "APCA-API-KEY-ID": credentials.keyId, + "APCA-API-SECRET-KEY": credentials.secret + }; +}; + +export const buildAlpacaWebSocketAuthMessage = ( + credentials: AlpacaCredentials +): { action: "auth"; key: string; secret: string } => { + if (credentials.usesLegacyBearer) { + return { + action: "auth", + key: credentials.legacyToken, + secret: "" + }; + } + + return { + action: "auth", + key: credentials.keyId, + secret: credentials.secret + }; +}; diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index 77b0d3c..577271f 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -1 +1,2 @@ export * from "./env"; +export * from "./alpaca"; diff --git a/packages/config/tests/alpaca.test.ts b/packages/config/tests/alpaca.test.ts new file mode 100644 index 0000000..9c48f12 --- /dev/null +++ b/packages/config/tests/alpaca.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "bun:test"; +import { + buildAlpacaAuthHeaders, + buildAlpacaWebSocketAuthMessage, + hasAlpacaCredentials, + resolveAlpacaCredentials +} from "../src/alpaca"; + +describe("resolveAlpacaCredentials", () => { + it("prefers explicit key-id and secret vars", () => { + const credentials = resolveAlpacaCredentials({ + ALPACA_API_KEY: "legacy-token", + ALPACA_API_KEY_ID: "key-id", + ALPACA_API_SECRET_KEY: "secret" + }); + + expect(credentials).toEqual({ + keyId: "key-id", + secret: "secret", + legacyToken: "legacy-token", + usesLegacyBearer: false + }); + expect(hasAlpacaCredentials(credentials)).toBe(true); + expect(buildAlpacaAuthHeaders(credentials)).toEqual({ + "APCA-API-KEY-ID": "key-id", + "APCA-API-SECRET-KEY": "secret" + }); + expect(buildAlpacaWebSocketAuthMessage(credentials)).toEqual({ + action: "auth", + key: "key-id", + secret: "secret" + }); + }); + + it("supports the older bearer-token fallback when no secret exists", () => { + const credentials = resolveAlpacaCredentials({ + ALPACA_API_KEY: "legacy-token" + }); + + expect(credentials.usesLegacyBearer).toBe(true); + expect(hasAlpacaCredentials(credentials)).toBe(true); + expect(buildAlpacaAuthHeaders(credentials)).toEqual({ + Authorization: "Bearer legacy-token" + }); + expect(buildAlpacaWebSocketAuthMessage(credentials)).toEqual({ + action: "auth", + key: "legacy-token", + secret: "" + }); + }); + + it("supports alternate secret env names", () => { + const credentials = resolveAlpacaCredentials({ + ALPACA_KEY_ID: "short-key", + ALPACA_SECRET_KEY: "short-secret" + }); + + expect(credentials).toEqual({ + keyId: "short-key", + secret: "short-secret", + legacyToken: "", + usesLegacyBearer: false + }); + }); +}); diff --git a/packages/storage/src/clickhouse.ts b/packages/storage/src/clickhouse.ts index bc0061e..af469d7 100644 --- a/packages/storage/src/clickhouse.ts +++ b/packages/storage/src/clickhouse.ts @@ -7,6 +7,7 @@ import { EquityPrintJoinSchema, InferredDarkEventSchema, FlowPacketSchema, + NewsStorySchema, OptionNBBOSchema, OptionPrintSchema, SmartMoneyEventSchema @@ -20,6 +21,7 @@ import type { EquityPrintJoin, InferredDarkEvent, FlowPacket, + NewsStory, SmartMoneyEvent, OptionNBBO, OptionPrint, @@ -91,6 +93,13 @@ import { toSmartMoneyEventRecord, type SmartMoneyEventRecord } from "./smart-money-events"; +import { + NEWS_TABLE, + newsTableDDL, + fromNewsRecord, + toNewsRecord, + type NewsRecord +} from "./news"; export type ClickHouseOptions = { url: string; @@ -320,6 +329,12 @@ export const ensureAlertsTable = async (client: ClickHouseClient): Promise } }; +export const ensureNewsTable = async (client: ClickHouseClient): Promise => { + await client.exec({ + query: newsTableDDL() + }); +}; + export const insertOptionPrint = async ( client: ClickHouseClient, print: OptionPrint @@ -449,6 +464,15 @@ export const insertAlert = async (client: ClickHouseClient, alert: AlertEvent): }); }; +export const insertNewsStory = async (client: ClickHouseClient, story: NewsStory): Promise => { + const record = toNewsRecord(story); + await client.insert({ + table: NEWS_TABLE, + values: [record], + format: "JSONEachRow" + }); +}; + export type ClickHouseBatchWriterOptions = { flushIntervalMs?: number; maxRows?: number; @@ -600,6 +624,13 @@ export const enqueueAlertInsert = ( writer.enqueue(ALERTS_TABLE, toAlertRecord(alert)); }; +export const enqueueNewsStoryInsert = ( + writer: ClickHouseBatchWriter, + story: NewsStory +): void => { + writer.enqueue(NEWS_TABLE, toNewsRecord(story)); +}; + const clampLimit = (limit: number): number => { if (!Number.isFinite(limit)) { return 100; @@ -1016,6 +1047,32 @@ const normalizeAlertRow = (row: unknown): AlertRecord | null => { }; }; +const normalizeNewsRow = (row: unknown): NewsRecord | null => { + if (!row || typeof row !== "object") { + return null; + } + + const record = row as Record; + return { + source_ts: coerceNumber(record.source_ts) as number, + ingest_ts: coerceNumber(record.ingest_ts) as number, + seq: coerceNumber(record.seq) as number, + trace_id: String(record.trace_id ?? ""), + story_id: coerceNumber(record.story_id) as number, + provider: String(record.provider ?? ""), + source: String(record.source ?? ""), + headline: String(record.headline ?? ""), + summary: String(record.summary ?? ""), + content_html: String(record.content_html ?? ""), + url: String(record.url ?? ""), + published_ts: coerceNumber(record.published_ts) as number, + updated_ts: coerceNumber(record.updated_ts) as number, + provider_symbols_json: String(record.provider_symbols_json ?? "[]"), + resolved_symbols_json: String(record.resolved_symbols_json ?? "[]"), + symbol_resolution: String(record.symbol_resolution ?? "none") as NewsRecord["symbol_resolution"] + }; +}; + export const fetchRecentOptionPrints = async ( client: ClickHouseClient, limit: number, @@ -1207,6 +1264,50 @@ export const fetchRecentAlerts = async ( return AlertEventSchema.array().parse(alerts); }; +const latestNewsSelect = ` +SELECT + source_ts, + ingest_ts, + seq, + trace_id, + story_id, + provider, + source, + headline, + summary, + content_html, + url, + published_ts, + updated_ts, + provider_symbols_json, + resolved_symbols_json, + symbol_resolution +FROM ( + SELECT + *, + row_number() OVER (PARTITION BY provider, story_id ORDER BY updated_ts DESC, ingest_ts DESC, seq DESC) AS revision_rank + FROM ${NEWS_TABLE} +) +WHERE revision_rank = 1 +`; + +export const fetchRecentNews = async ( + client: ClickHouseClient, + limit: number +): Promise => { + const safeLimit = clampLimit(limit); + const result = await client.query({ + query: `${latestNewsSelect} ORDER BY published_ts DESC, story_id DESC LIMIT ${safeLimit}`, + format: "JSONEachRow" + }); + + const rows = await result.json(); + const records = rows + .map(normalizeNewsRow) + .filter((record): record is NewsRecord => record !== null); + return NewsStorySchema.array().parse(records.map(fromNewsRecord)); +}; + const normalizeAlertEvidenceRefs = (refs: string[]): string[] => { return Array.from(new Set(refs.map((ref) => ref.trim()).filter(Boolean))); }; @@ -1600,6 +1701,27 @@ export const fetchAlertsAfter = async ( return AlertEventSchema.array().parse(alerts); }; +export const fetchNewsAfter = async ( + client: ClickHouseClient, + afterTs: number, + afterSeq: number, + limit: number +): Promise => { + const safeLimit = clampLimit(limit); + const safeAfterTs = clampCursor(afterTs); + const safeAfterSeq = clampCursor(afterSeq); + const result = await client.query({ + query: `${latestNewsSelect} AND (published_ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY published_ts ASC, seq ASC LIMIT ${safeLimit}`, + format: "JSONEachRow" + }); + + const rows = await result.json(); + const records = rows + .map(normalizeNewsRow) + .filter((record): record is NewsRecord => record !== null); + return NewsStorySchema.array().parse(records.map(fromNewsRecord)); +}; + export const fetchOptionPrintsBefore = async ( client: ClickHouseClient, beforeTs: number, @@ -1778,6 +1900,25 @@ export const fetchAlertsBefore = async ( return AlertEventSchema.array().parse(records.map(fromAlertRecord)); }; +export const fetchNewsBefore = async ( + client: ClickHouseClient, + beforeTs: number, + beforeSeq: number, + limit: number +): Promise => { + const safeLimit = clampLimit(limit); + const result = await client.query({ + query: `${latestNewsSelect} AND ${buildBeforeTupleCondition("published_ts", "seq", beforeTs, beforeSeq)} ORDER BY published_ts DESC, seq DESC LIMIT ${safeLimit}`, + format: "JSONEachRow" + }); + + const rows = await result.json(); + const records = rows + .map(normalizeNewsRow) + .filter((record): record is NewsRecord => record !== null); + return NewsStorySchema.array().parse(records.map(fromNewsRecord)); +}; + export const fetchInferredDarkBefore = async ( client: ClickHouseClient, beforeTs: number, diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index 4fefabc..810d67c 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -10,3 +10,4 @@ export * from "./equity-print-joins"; export * from "./inferred-dark"; export * from "./option-prints"; export * from "./option-nbbo"; +export * from "./news"; diff --git a/packages/storage/src/news.ts b/packages/storage/src/news.ts new file mode 100644 index 0000000..cf92f40 --- /dev/null +++ b/packages/storage/src/news.ts @@ -0,0 +1,102 @@ +import type { NewsStory, NewsSymbolResolution } from "@islandflow/types"; + +export const NEWS_TABLE = "news"; + +export type NewsRecord = { + source_ts: number; + ingest_ts: number; + seq: number; + trace_id: string; + story_id: number; + provider: string; + source: string; + headline: string; + summary: string; + content_html: string; + url: string; + published_ts: number; + updated_ts: number; + provider_symbols_json: string; + resolved_symbols_json: string; + symbol_resolution: NewsSymbolResolution; +}; + +export const newsTableDDL = (): string => { + return ` +CREATE TABLE IF NOT EXISTS ${NEWS_TABLE} ( + source_ts UInt64, + ingest_ts UInt64, + seq UInt64, + trace_id String, + story_id UInt64, + provider String, + source String, + headline String, + summary String, + content_html String, + url String, + published_ts UInt64, + updated_ts UInt64, + provider_symbols_json String, + resolved_symbols_json String, + symbol_resolution String +) +ENGINE = ReplacingMergeTree(updated_ts) +ORDER BY (provider, story_id, updated_ts, seq) +`; +}; + +const safeStringArray = (value: string): string[] => { + try { + const parsed = JSON.parse(value); + if (Array.isArray(parsed)) { + return parsed.map((entry) => String(entry)); + } + } catch { + // ignore + } + + return []; +}; + +export const toNewsRecord = (story: NewsStory): NewsRecord => { + return { + source_ts: story.source_ts, + ingest_ts: story.ingest_ts, + seq: story.seq, + trace_id: story.trace_id, + story_id: story.story_id, + provider: story.provider, + source: story.source, + headline: story.headline, + summary: story.summary, + content_html: story.content_html, + url: story.url, + published_ts: story.published_ts, + updated_ts: story.updated_ts, + provider_symbols_json: JSON.stringify(story.provider_symbols), + resolved_symbols_json: JSON.stringify(story.resolved_symbols), + symbol_resolution: story.symbol_resolution + }; +}; + +export const fromNewsRecord = (record: NewsRecord): NewsStory => { + return { + source_ts: record.source_ts, + ingest_ts: record.ingest_ts, + seq: record.seq, + trace_id: record.trace_id, + story_id: record.story_id, + provider: record.provider, + source: record.source, + headline: record.headline, + summary: record.summary, + content_html: record.content_html, + url: record.url, + published_ts: record.published_ts, + updated_ts: record.updated_ts, + provider_symbols: safeStringArray(record.provider_symbols_json), + resolved_symbols: safeStringArray(record.resolved_symbols_json), + symbol_resolution: record.symbol_resolution + }; +}; diff --git a/packages/storage/tests/news.test.ts b/packages/storage/tests/news.test.ts new file mode 100644 index 0000000..c5b71c8 --- /dev/null +++ b/packages/storage/tests/news.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "bun:test"; +import type { ClickHouseClient } from "../src/clickhouse"; +import { + NEWS_TABLE, + fromNewsRecord, + newsTableDDL, + toNewsRecord +} from "../src/news"; +import { + fetchNewsAfter, + fetchNewsBefore, + fetchRecentNews +} from "../src/clickhouse"; + +const makeClient = (resolver: (query: string) => unknown[]): ClickHouseClient => + ({ + exec: async () => {}, + insert: async () => {}, + ping: async () => ({ success: true }), + close: async () => {}, + query: async ({ query }: { query: string }) => ({ + async json() { + return resolver(query) as T; + } + }) + }) as ClickHouseClient; + +const story = { + source_ts: 100, + ingest_ts: 101, + seq: 3, + trace_id: "alpaca:77", + story_id: 77, + provider: "alpaca", + source: "Benzinga", + headline: "TSLA rises", + summary: "Summary", + content_html: "

TSLA rises

", + url: "https://example.com/story", + published_ts: 100, + updated_ts: 120, + provider_symbols: ["TSLA"], + resolved_symbols: ["TSLA", "AAPL"], + symbol_resolution: "mixed" as const +}; + +describe("news storage helpers", () => { + it("includes the correct table name in the DDL", () => { + const ddl = newsTableDDL(); + expect(ddl).toContain(NEWS_TABLE); + expect(ddl).toContain("ReplacingMergeTree"); + }); + + it("round-trips news records", () => { + const record = toNewsRecord(story); + const restored = fromNewsRecord(record); + expect(restored).toEqual(story); + }); + + it("uses latest-revision selection for recent and cursor queries", async () => { + const queries: string[] = []; + const client = makeClient((query) => { + queries.push(query); + return [toNewsRecord(story)]; + }); + + const recent = await fetchRecentNews(client, 10); + const before = await fetchNewsBefore(client, 200, 10, 10); + const after = await fetchNewsAfter(client, 50, 1, 10); + + expect(recent[0]?.trace_id).toBe("alpaca:77"); + expect(before[0]?.story_id).toBe(77); + expect(after[0]?.updated_ts).toBe(120); + expect(queries[0]).toContain("row_number() OVER"); + expect(queries[1]).toContain("published_ts"); + expect(queries[2]).toContain("(published_ts, seq) > (50, 1)"); + }); +}); diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index c15dc7b..0556bd8 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -262,3 +262,26 @@ export const InferredDarkEventSchema = EventMetaSchema.merge( ); export type InferredDarkEvent = z.infer; + +export const NewsSymbolResolutionSchema = z.enum(["provider", "derived", "mixed", "none"]); + +export type NewsSymbolResolution = z.infer; + +export const NewsStorySchema = EventMetaSchema.merge( + z.object({ + story_id: z.number().int().nonnegative(), + provider: z.string().min(1), + source: z.string().min(1), + headline: z.string().min(1), + summary: z.string(), + content_html: z.string(), + url: z.string().url().or(z.literal("")), + published_ts: z.number().int().nonnegative(), + updated_ts: z.number().int().nonnegative(), + provider_symbols: z.array(z.string().min(1)), + resolved_symbols: z.array(z.string().min(1)), + symbol_resolution: NewsSymbolResolutionSchema + }) +); + +export type NewsStory = z.infer; diff --git a/packages/types/src/live.ts b/packages/types/src/live.ts index 0787c84..10ac486 100644 --- a/packages/types/src/live.ts +++ b/packages/types/src/live.ts @@ -8,6 +8,7 @@ import { EquityQuoteSchema, FlowPacketSchema, InferredDarkEventSchema, + NewsStorySchema, OptionNBBOSchema, OptionPrintSchema, SmartMoneyEventSchema @@ -34,7 +35,8 @@ export const LiveGenericChannelSchema = z.enum([ "smart-money", "classifier-hits", "alerts", - "inferred-dark" + "inferred-dark", + "news" ]); export const LiveChannelSchema = z.enum([ @@ -48,6 +50,7 @@ export const LiveChannelSchema = z.enum([ "classifier-hits", "alerts", "inferred-dark", + "news", "equity-candles", "equity-overlay" ]); @@ -91,7 +94,7 @@ export const LiveSubscriptionSchema = z.discriminatedUnion("channel", [ snapshot_limit: z.number().int().positive().optional() }), z.object({ - channel: z.enum(["nbbo", "equity-quotes", "equity-joins", "classifier-hits", "alerts", "inferred-dark"]), + channel: z.enum(["nbbo", "equity-quotes", "equity-joins", "classifier-hits", "alerts", "inferred-dark", "news"]), snapshot_limit: z.number().int().positive().optional() }), z.object({ @@ -123,6 +126,7 @@ const livePayloadSchemas = { "classifier-hits": ClassifierHitEventSchema, alerts: AlertEventSchema, "inferred-dark": InferredDarkEventSchema, + news: NewsStorySchema, "equity-candles": EquityCandleSchema, "equity-overlay": EquityPrintSchema } as const; diff --git a/packages/types/tests/live.test.ts b/packages/types/tests/live.test.ts index 075eab1..ef254b4 100644 --- a/packages/types/tests/live.test.ts +++ b/packages/types/tests/live.test.ts @@ -9,6 +9,7 @@ import { describe("live protocol types", () => { it("builds stable keys for generic and parameterized subscriptions", () => { expect(getSubscriptionKey({ channel: "flow" })).toBe("flow|{}"); + expect(getSubscriptionKey({ channel: "news" })).toBe("news"); expect( getSubscriptionKey({ channel: "options", @@ -53,12 +54,13 @@ describe("live protocol types", () => { op: "subscribe", subscriptions: [ { channel: "flow", filters: { nbboSides: ["AA", "A"], minNotional: 50000 } }, + { channel: "news", snapshot_limit: 100 }, { channel: "equity-candles", underlying_id: "SPY", interval_ms: 60000 } ] }); expect(parsed.op).toBe("subscribe"); - expect(parsed.subscriptions).toHaveLength(2); + expect(parsed.subscriptions).toHaveLength(3); }); it("validates snapshot and event server messages", () => { @@ -74,18 +76,24 @@ describe("live protocol types", () => { }); const event = LiveServerMessageSchema.parse({ op: "event", - subscription: { channel: "equity-overlay", underlying_id: "SPY" }, + subscription: { channel: "news" }, item: { source_ts: 100, ingest_ts: 101, seq: 1, - trace_id: "eq-1", - ts: 100, - underlying_id: "SPY", - price: 500, - size: 10, - exchange: "X", - offExchangeFlag: true + trace_id: "alpaca:1", + story_id: 1, + provider: "alpaca", + source: "Benzinga", + headline: "TSLA rises", + summary: "", + content_html: "

TSLA rises

", + url: "https://example.com/story", + published_ts: 100, + updated_ts: 100, + provider_symbols: ["TSLA"], + resolved_symbols: ["TSLA"], + symbol_resolution: "provider" }, watermark: cursor }); diff --git a/plans/2026-05-18-native-fast-iterative-deploy-plan.md b/plans/2026-05-18-native-fast-iterative-deploy-plan.md new file mode 100644 index 0000000..0e09102 --- /dev/null +++ b/plans/2026-05-18-native-fast-iterative-deploy-plan.md @@ -0,0 +1,21 @@ +# Native, Fast, Iterative Deployment Plan (Docker Optional) + +Date: 2026-05-18 + +## Plan Steps (15) + +1. ☐ Stop the bleeding immediately (current deploy loop). +2. ☐ Get hard timing data per deploy phase. +3. ☐ Live server state audit (when plan mode is off). +4. ☐ Resolve duplicate compose stack first (islandflow-2db). +5. ☐ Fix NPM proxy route regression (islandflow-sz8). +6. ☐ Define target iterative deployment model. +7. ☐ Prepare native runtime prerequisites on VPS. +8. ☐ Checked-in native ops assets (islandflow-38p). +9. ☐ Switch proxy topology for native mode carefully. +10. ☐ Deploy strategy guardrails. +11. ☐ Validation matrix. +12. ☐ Staged cutover plan. +13. ☐ Decision: final default runtime. +14. ☐ Decision: optimization priority. +15. ☐ Decision: immediate live audit kickoff. diff --git a/scripts/deploy.ts b/scripts/deploy.ts index 68d260a..8a5b9c7 100644 --- a/scripts/deploy.ts +++ b/scripts/deploy.ts @@ -7,7 +7,7 @@ import { fileURLToPath } from "node:url"; type DeployMode = "main" | "current-branch"; type DeployRuntime = "docker" | "native"; -type DeployScope = "full" | "web" | "api" | "services"; +type DeployScope = "full" | "web" | "api" | "services" | "workers"; type DeployOptions = { mode: DeployMode; @@ -18,10 +18,18 @@ type DeployOptions = { noBuild: boolean; }; +type PhaseTiming = { + name: string; + durationMs: number; +}; + const REMOTE_HOST = "delta@152.53.80.229"; const REMOTE_REPO = "/home/delta/islandflow"; const REMOTE_DOCKER_DEPLOYMENT = "/home/delta/islandflow/deployment/docker"; -const SSH_KEY = path.join(process.env.HOME ?? "", ".ssh", "delta_ed25519"); +const SSH_KEY = + process.env.DEPLOY_SSH_KEY_PATH?.trim() || + path.join(process.env.HOME ?? "", ".ssh", "delta_ed25519"); +const DEPLOY_FORCE_SSH = process.env.DEPLOY_FORCE_SSH?.trim() === "1"; const SSH_OPTIONS = [ "-i", SSH_KEY, @@ -38,6 +46,7 @@ const PUBLIC_APP_URL = const PUBLIC_API_HEALTH_URL = process.env.DEPLOY_PUBLIC_API_HEALTH_URL?.trim() || null; const DEPLOY_GIT_REMOTE_OVERRIDE = process.env.DEPLOY_GIT_REMOTE?.trim() || null; +const DEPLOY_NATIVE_EDGE_READY = process.env.DEPLOY_NATIVE_EDGE_READY?.trim() === "1"; const NATIVE_SYSTEMCTL_PREFIX = process.env.DEPLOY_NATIVE_SYSTEMCTL_PREFIX?.trim() || "sudo -n systemctl"; const NATIVE_UNITS = { @@ -48,7 +57,8 @@ const NATIVE_UNITS = { ingestOptions: process.env.DEPLOY_NATIVE_INGEST_OPTIONS_UNIT?.trim() || "islandflow-ingest-options", ingestEquities: - process.env.DEPLOY_NATIVE_INGEST_EQUITIES_UNIT?.trim() || "islandflow-ingest-equities" + process.env.DEPLOY_NATIVE_INGEST_EQUITIES_UNIT?.trim() || "islandflow-ingest-equities", + ingestNews: process.env.DEPLOY_NATIVE_INGEST_NEWS_UNIT?.trim() || "islandflow-ingest-news" } as const; const DOCKER_CORE_SERVICES = [ "api", @@ -56,24 +66,34 @@ const DOCKER_CORE_SERVICES = [ "compute", "candles", "ingest-options", - "ingest-equities" + "ingest-equities", + "ingest-news" ] as const; const DOCKER_BACKEND_SERVICES = [ "api", "compute", "candles", "ingest-options", - "ingest-equities" + "ingest-equities", + "ingest-news" +] as const; +const DOCKER_WORKER_SERVICES = [ + "compute", + "candles", + "ingest-options", + "ingest-equities", + "ingest-news" ] as const; const scriptPath = fileURLToPath(import.meta.url); const repoRoot = path.resolve(path.dirname(scriptPath), ".."); +const isLocalServerExecution = !DEPLOY_FORCE_SSH && repoRoot === REMOTE_REPO; function usage(exitCode = 1): never { console.error(`Usage: - ./deploy main [--runtime docker|native] [--web-only|--api-only|--services-only] [--fast] [--no-build] [--force-recreate] - ./deploy current-branch [--runtime docker|native] [--web-only|--api-only|--services-only] [--fast] [--no-build] [--force-recreate] - ./deploy current branch [--runtime docker|native] [--web-only|--api-only|--services-only] [--fast] [--no-build] [--force-recreate] + ./deploy main [--runtime docker|native] [--web-only|--api-only|--services-only|--workers-only] [--fast] [--no-build] [--force-recreate] + ./deploy current-branch [--runtime docker|native] [--web-only|--api-only|--services-only|--workers-only] [--fast] [--no-build] [--force-recreate] + ./deploy current branch [--runtime docker|native] [--web-only|--api-only|--services-only|--workers-only] [--fast] [--no-build] [--force-recreate] Modes: main Deploy /main to the live server checkout. @@ -88,25 +108,30 @@ Scopes: --web-only Deploy only the Next.js web surface. --api-only Deploy only the API service. --services-only Deploy API + backend services without the web service. + --workers-only Deploy compute/candles/ingest workers without touching web or API. Options: --runtime Explicit runtime selector (docker or native). - --fast Prefer a quicker rollout profile (defaults full scope to --services-only and skips public API route suite). + --fast Prefer a quicker rollout profile (defaults full scope to --services-only for docker and --workers-only for native, and skips the public API route suite when API scope is included). --no-build Skip docker image builds or native bun install/web build steps. --force-recreate Docker-only escalation path for docker compose when a normal refresh is not enough. --help Show this help text. Environment: DEPLOY_GIT_REMOTE Override git remote used for deploy fetch/pull/push (auto-detected by default). + DEPLOY_SSH_KEY_PATH Override the SSH key used for remote execution. + DEPLOY_FORCE_SSH Set to 1 to force SSH even when running from the live server checkout. DEPLOY_PUBLIC_APP_URL Override the public app URL (default: https://flow.deltaisland.io). DEPLOY_PUBLIC_API_HEALTH_URL Optional separate public API health URL for two-origin deployments. + DEPLOY_NATIVE_EDGE_READY Set to 1 to allow native rollouts that include the public web or API edge. DEPLOY_NATIVE_SYSTEMCTL_PREFIX Override systemctl invocation for native rollouts (default: sudo -n systemctl). DEPLOY_NATIVE_WEB_UNIT Override native web systemd unit name. DEPLOY_NATIVE_API_UNIT Override native api systemd unit name. DEPLOY_NATIVE_COMPUTE_UNIT Override native compute systemd unit name. DEPLOY_NATIVE_CANDLES_UNIT Override native candles systemd unit name. DEPLOY_NATIVE_INGEST_OPTIONS_UNIT Override native ingest-options systemd unit name. - DEPLOY_NATIVE_INGEST_EQUITIES_UNIT Override native ingest-equities systemd unit name.`); + DEPLOY_NATIVE_INGEST_EQUITIES_UNIT Override native ingest-equities systemd unit name. + DEPLOY_NATIVE_INGEST_NEWS_UNIT Override native ingest-news systemd unit name.`); process.exit(exitCode); } @@ -114,6 +139,32 @@ function section(title: string): void { console.log(`\n== ${title} ==`); } +function formatDuration(durationMs: number): string { + if (durationMs < 1000) { + return `${durationMs}ms`; + } + + return `${(durationMs / 1000).toFixed(2)}s`; +} + +function timedPhase(timings: PhaseTiming[], name: string, fn: () => T): T { + const startedAt = Date.now(); + try { + return fn(); + } finally { + timings.push({ name, durationMs: Date.now() - startedAt }); + } +} + +function printTimingSummary(timings: PhaseTiming[]): void { + section("Deploy Timings"); + const totalMs = timings.reduce((sum, timing) => sum + timing.durationMs, 0); + for (const timing of timings) { + console.log(`[deploy] ${timing.name}: ${formatDuration(timing.durationMs)}`); + } + console.log(`[deploy] total: ${formatDuration(totalMs)}`); +} + function formatCommand(command: string, args: string[]): string { return [command, ...args] .map((part) => (/\s/.test(part) ? JSON.stringify(part) : part)) @@ -180,6 +231,23 @@ function runRemoteScript( args: string[] = [] ): void { section(title); + + if (isLocalServerExecution) { + const localArgs = ["-s", "--", ...args]; + console.log(`$ ${formatCommand("bash", localArgs)} # local server execution`); + const result = spawnSync("bash", localArgs, { + cwd: repoRoot, + input: script, + encoding: "utf8", + stdio: ["pipe", "inherit", "inherit"] + }); + + if (result.status !== 0) { + process.exit(result.status ?? 1); + } + return; + } + const sshArgs = [...SSH_OPTIONS, REMOTE_HOST, "bash", "-s", "--", ...args]; console.log(`$ ${formatCommand("ssh", sshArgs)}`); const result = spawnSync("ssh", sshArgs, { @@ -221,11 +289,14 @@ function parseScope(rawArgs: string[]): DeployScope { const scopes = [ rawArgs.includes("--web-only") ? "web" : null, rawArgs.includes("--api-only") ? "api" : null, - rawArgs.includes("--services-only") ? "services" : null + rawArgs.includes("--services-only") ? "services" : null, + rawArgs.includes("--workers-only") ? "workers" : null ].filter((value): value is Exclude => value !== null); if (scopes.length > 1) { - console.error("Choose only one deploy scope flag: --web-only, --api-only, or --services-only."); + console.error( + "Choose only one deploy scope flag: --web-only, --api-only, --services-only, or --workers-only." + ); process.exit(1); } @@ -250,6 +321,7 @@ function parseArgs(rawArgs: string[]): DeployOptions { arg !== "--web-only" && arg !== "--api-only" && arg !== "--services-only" && + arg !== "--workers-only" && arg !== "--runtime" && rawArgs[index - 1] !== "--runtime" && !arg.startsWith("--runtime=") @@ -282,8 +354,13 @@ function parseArgs(rawArgs: string[]): DeployOptions { } function assertSshKeyExists(): void { + if (isLocalServerExecution) { + return; + } + if (!existsSync(SSH_KEY)) { console.error(`Missing SSH key: ${SSH_KEY}`); + console.error("Set DEPLOY_SSH_KEY_PATH or run from the live server checkout without DEPLOY_FORCE_SSH."); process.exit(1); } } @@ -398,14 +475,16 @@ function describeScope(scope: DeployScope): string { return "api only"; case "services": return "api + backend services"; + case "workers": + return "worker services only"; default: return "full stack"; } } -function effectiveScope(scope: DeployScope, fast: boolean): DeployScope { +function effectiveScope(scope: DeployScope, runtime: DeployRuntime, fast: boolean): DeployScope { if (fast && scope === "full") { - return "services"; + return runtime === "native" ? "workers" : "services"; } return scope; } @@ -418,6 +497,10 @@ function scopeIncludesApi(scope: DeployScope): boolean { return scope === "full" || scope === "api" || scope === "services"; } +function scopeTouchesPublicEdge(scope: DeployScope): boolean { + return scopeIncludesWeb(scope) || scopeIncludesApi(scope); +} + function dockerServicesForScope(scope: DeployScope): string[] { switch (scope) { case "web": @@ -426,6 +509,8 @@ function dockerServicesForScope(scope: DeployScope): string[] { return ["api"]; case "services": return [...DOCKER_BACKEND_SERVICES]; + case "workers": + return [...DOCKER_WORKER_SERVICES]; default: return []; } @@ -448,6 +533,8 @@ function dockerLogServicesForScope(scope: DeployScope): string[] { return ["api"]; case "services": return [...DOCKER_BACKEND_SERVICES]; + case "workers": + return [...DOCKER_WORKER_SERVICES]; default: return [...DOCKER_CORE_SERVICES]; } @@ -465,7 +552,16 @@ function nativeUnitsForScope(scope: DeployScope): string[] { NATIVE_UNITS.compute, NATIVE_UNITS.candles, NATIVE_UNITS.ingestOptions, - NATIVE_UNITS.ingestEquities + NATIVE_UNITS.ingestEquities, + NATIVE_UNITS.ingestNews + ]; + case "workers": + return [ + NATIVE_UNITS.compute, + NATIVE_UNITS.candles, + NATIVE_UNITS.ingestOptions, + NATIVE_UNITS.ingestEquities, + NATIVE_UNITS.ingestNews ]; default: return [ @@ -474,7 +570,8 @@ function nativeUnitsForScope(scope: DeployScope): string[] { NATIVE_UNITS.compute, NATIVE_UNITS.candles, NATIVE_UNITS.ingestOptions, - NATIVE_UNITS.ingestEquities + NATIVE_UNITS.ingestEquities, + NATIVE_UNITS.ingestNews ]; } } @@ -494,19 +591,46 @@ function localDockerWorkspaceSnapshotPrecheck(): void { } } -function localRuntimePrecheck(runtime: DeployRuntime, noBuild: boolean): void { +function assertNativeEdgeReady(scope: DeployScope): void { + if (!scopeTouchesPublicEdge(scope) || DEPLOY_NATIVE_EDGE_READY) { + return; + } + + console.error( + "Refusing native deploy that touches public web/API scope before edge cutover is acknowledged." + ); + console.error( + "Set DEPLOY_NATIVE_EDGE_READY=1 only after proxy routing and native units for the public edge are intentionally prepared." + ); + console.error( + "For fast iterative backend deploys before cutover, use --runtime native --workers-only or --runtime native --fast." + ); + process.exit(1); +} + +function localRuntimePrecheck(runtime: DeployRuntime, scope: DeployScope, noBuild: boolean): void { if (runtime === "docker" && !noBuild) { localDockerWorkspaceSnapshotPrecheck(); + return; + } + + if (runtime === "native") { + assertNativeEdgeReady(scope); } } -function localMainPrecheck(remote: string, runtime: DeployRuntime, noBuild: boolean): void { +function localMainPrecheck( + remote: string, + runtime: DeployRuntime, + scope: DeployScope, + noBuild: boolean +): void { section("Local Precheck"); runChecked("git", ["fetch", remote]); runChecked("git", ["status", "--short", "--branch"]); runChecked("git", ["rev-parse", "--verify", "HEAD"]); runChecked("git", ["rev-parse", `${remote}/main`]); - localRuntimePrecheck(runtime, noBuild); + localRuntimePrecheck(runtime, scope, noBuild); } function currentBranchName(): string { @@ -522,6 +646,7 @@ function localBranchPrecheck( remote: string, branch: string, runtime: DeployRuntime, + scope: DeployScope, noBuild: boolean ): void { section("Local Precheck"); @@ -537,7 +662,7 @@ function localBranchPrecheck( process.exit(1); } - localRuntimePrecheck(runtime, noBuild); + localRuntimePrecheck(runtime, scope, noBuild); } function publishCurrentBranch(remote: string, branch: string): void { @@ -631,6 +756,10 @@ set -euo pipefail cd ${shellEscape(REMOTE_REPO)} +if [[ -x "$HOME/.bun/bin/bun" ]]; then + export PATH="$HOME/.bun/bin:$PATH" +fi + if ! command -v bun >/dev/null 2>&1; then echo "Refusing native rollout: bun is not installed on the server." >&2 echo "The current supported VPS path remains --runtime docker." >&2 @@ -732,6 +861,10 @@ function remoteNativeRollout( `#!/usr/bin/env bash set -euo pipefail +if [[ -x "$HOME/.bun/bin/bun" ]]; then + export PATH="$HOME/.bun/bin:$PATH" +fi + ${remoteGitUpdateScript(mode, remote, branch)} cd ${shellEscape(REMOTE_REPO)} @@ -803,6 +936,10 @@ function remoteNativeVerification(scope: DeployScope, fast: boolean): void { const units = nativeUnitsForScope(scope).map((value) => shellEscape(value)).join(" "); const checks: string[] = []; + if (scope === "full" || scope === "api" || scope === "services" || scope === "workers") { + checks.push("./deployment/native/check-native-infra.sh"); + } + if (scopeIncludesApi(scope)) { checks.push('curl -fksS http://127.0.0.1:4000/health'); } @@ -816,6 +953,12 @@ function remoteNativeVerification(scope: DeployScope, fast: boolean): void { `#!/usr/bin/env bash set -euo pipefail +cd ${shellEscape(REMOTE_REPO)} + +if [[ -x "$HOME/.bun/bin/bun" ]]; then + export PATH="$HOME/.bun/bin:$PATH" +fi + declare -a units=(${units}) for unit in "\${units[@]}"; do ${NATIVE_SYSTEMCTL_PREFIX} is-active --quiet "$unit" @@ -837,10 +980,10 @@ function remoteVerification(runtime: DeployRuntime, scope: DeployScope, fast: bo function publicVerification(scope: DeployScope, fast: boolean): void { section("Public Verification"); - if (!fast || scopeIncludesWeb(scope)) { + if (scopeIncludesWeb(scope)) { runChecked("curl", ["-I", "-fksS", PUBLIC_APP_URL]); } else { - console.log("[deploy] Fast mode: skipping public app HEAD check because web scope is not included."); + console.log("[deploy] Skipping public app HEAD check because web scope is not included."); } if (scopeIncludesApi(scope) && PUBLIC_API_HEALTH_URL) { @@ -861,7 +1004,8 @@ function publicVerification(scope: DeployScope, fast: boolean): void { function main(): void { const options = parseArgs(process.argv.slice(2)); - const scope = effectiveScope(options.scope, options.fast); + const scope = effectiveScope(options.scope, options.runtime, options.fast); + const timings: PhaseTiming[] = []; const currentBranch = options.mode === "current-branch" ? currentBranchName() : null; const deployRemote = resolveDeployRemote(options.mode, currentBranch); assertSshKeyExists(); @@ -872,22 +1016,33 @@ function main(): void { `via ${describeRuntime(options.runtime)} (${describeScope(scope)}${options.fast ? ", fast mode" : ""}).` ); console.log(`[deploy] Using git remote: ${deployRemote}`); + console.log( + `[deploy] Execution mode: ${isLocalServerExecution ? "local server checkout" : `ssh to ${REMOTE_HOST}`}` + ); if (options.fast && options.scope === "full") { - console.log("[deploy] Fast mode changed default full scope to --services-only."); + console.log( + `[deploy] Fast mode changed default full scope to ${options.runtime === "native" ? "--workers-only" : "--services-only"}.` + ); } if (options.mode === "main") { - localMainPrecheck(deployRemote, options.runtime, options.noBuild); - remoteGitPrecheck(); - remoteRuntimePrecheck(options.runtime, scope); - remoteRollout( - options.mode, - deployRemote, - options.runtime, - null, - scope, - options.forceRecreate, - options.noBuild + timedPhase(timings, "local precheck", () => + localMainPrecheck(deployRemote, options.runtime, scope, options.noBuild) + ); + timedPhase(timings, "remote git precheck", () => remoteGitPrecheck()); + timedPhase(timings, "remote runtime precheck", () => + remoteRuntimePrecheck(options.runtime, scope) + ); + timedPhase(timings, "remote rollout", () => + remoteRollout( + options.mode, + deployRemote, + options.runtime, + null, + scope, + options.forceRecreate, + options.noBuild + ) ); } else { const branch = currentBranch; @@ -895,23 +1050,34 @@ function main(): void { console.error("Unable to resolve current branch for current-branch deploy mode."); process.exit(1); } - localBranchPrecheck(deployRemote, branch, options.runtime, options.noBuild); - publishCurrentBranch(deployRemote, branch); - remoteGitPrecheck(); - remoteRuntimePrecheck(options.runtime, scope); - remoteRollout( - options.mode, - deployRemote, - options.runtime, - branch, - scope, - options.forceRecreate, - options.noBuild + timedPhase(timings, "local precheck", () => + localBranchPrecheck(deployRemote, branch, options.runtime, scope, options.noBuild) + ); + timedPhase(timings, "local publish", () => publishCurrentBranch(deployRemote, branch)); + timedPhase(timings, "remote git precheck", () => remoteGitPrecheck()); + timedPhase(timings, "remote runtime precheck", () => + remoteRuntimePrecheck(options.runtime, scope) + ); + timedPhase(timings, "remote rollout", () => + remoteRollout( + options.mode, + deployRemote, + options.runtime, + branch, + scope, + options.forceRecreate, + options.noBuild + ) ); } - remoteVerification(options.runtime, scope, options.fast); - publicVerification(scope, options.fast); + timedPhase(timings, "remote verification", () => + remoteVerification(options.runtime, scope, options.fast) + ); + timedPhase(timings, "public verification", () => + publicVerification(scope, options.fast) + ); + printTimingSummary(timings); } main(); diff --git a/scripts/dev-services.ts b/scripts/dev-services.ts index 09cd381..2bcb641 100644 --- a/scripts/dev-services.ts +++ b/scripts/dev-services.ts @@ -222,6 +222,7 @@ process.on("SIGHUP", () => handleSignal("SIGHUP")); const tasks: ChildSpec[] = [ { name: "ingest-options", cmd: ["bun", "run", "dev"], cwd: "services/ingest-options" }, { name: "ingest-equities", cmd: ["bun", "run", "dev"], cwd: "services/ingest-equities" }, + { name: "ingest-news", cmd: ["bun", "run", "dev"], cwd: "services/ingest-news" }, { name: "compute", cmd: ["bun", "run", "dev"], cwd: "services/compute" }, { name: "candles", cmd: ["bun", "run", "dev"], cwd: "services/candles" }, { name: "refdata", cmd: ["bun", "run", "dev"], cwd: "services/refdata" }, diff --git a/scripts/dev.ts b/scripts/dev.ts index 64406d6..1d031a7 100644 --- a/scripts/dev.ts +++ b/scripts/dev.ts @@ -325,6 +325,7 @@ const serviceTasks: ChildSpec[] = [ { name: "web", cmd: ["bun", "run", "dev"], cwd: "apps/web" }, { name: "ingest-options", cmd: ["bun", "run", "dev"], cwd: "services/ingest-options" }, { name: "ingest-equities", cmd: ["bun", "run", "dev"], cwd: "services/ingest-equities" }, + { name: "ingest-news", cmd: ["bun", "run", "dev"], cwd: "services/ingest-news" }, { name: "compute", cmd: ["bun", "run", "dev"], cwd: "services/compute" }, { name: "candles", cmd: ["bun", "run", "dev"], cwd: "services/candles" }, { name: "refdata", cmd: ["bun", "run", "dev"], cwd: "services/refdata" }, diff --git a/scripts/generate-docs-index.mjs b/scripts/generate-docs-index.mjs new file mode 100644 index 0000000..cf64a9d --- /dev/null +++ b/scripts/generate-docs-index.mjs @@ -0,0 +1,421 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; + +const docsDir = path.resolve(process.cwd(), "docs"); +const outputFile = path.join(docsDir, "index.html"); + +const dateFormatter = new Intl.DateTimeFormat("en-US", { + dateStyle: "medium", + timeStyle: "short", +}); + +function escapeHtml(value) { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function formatBytes(bytes) { + if (bytes < 1024) { + return `${bytes} B`; + } + + const units = ["KB", "MB", "GB"]; + let size = bytes / 1024; + let unitIndex = 0; + + while (size >= 1024 && unitIndex < units.length - 1) { + size /= 1024; + unitIndex += 1; + } + + return `${size.toFixed(size >= 10 ? 0 : 1)} ${units[unitIndex]}`; +} + +function docsHref(relativePath) { + const encoded = relativePath + .split("/") + .map((part) => encodeURIComponent(part)) + .join("/"); + return `./${encoded}`; +} + +async function collectDocsFiles(rootDir, currentDir = rootDir, acc = []) { + const entries = await fs.readdir(currentDir, { withFileTypes: true }); + const sortedEntries = entries.sort((a, b) => a.name.localeCompare(b.name)); + + for (const entry of sortedEntries) { + if (entry.name.startsWith(".")) { + continue; + } + + const absolutePath = path.join(currentDir, entry.name); + const relativePath = path.relative(rootDir, absolutePath).replaceAll(path.sep, "/"); + + if (relativePath === "index.html") { + continue; + } + + if (entry.isDirectory()) { + await collectDocsFiles(rootDir, absolutePath, acc); + continue; + } + + if (entry.isFile()) { + const stats = await fs.stat(absolutePath); + + acc.push({ + relativePath, + category: relativePath.includes("/") ? relativePath.split("/")[0] : "root", + sizeBytes: stats.size, + modifiedAt: stats.mtime, + }); + } + } + + return acc; +} + +function groupByCategory(items) { + const groups = new Map(); + for (const item of items) { + if (!groups.has(item.category)) { + groups.set(item.category, []); + } + groups.get(item.category).push(item); + } + return groups; +} + +function sortedCategories(groups) { + const preferredOrder = ["turns", "daily-git", "general", "plans", "root"]; + const groupNames = [...groups.keys()]; + return groupNames.sort((a, b) => { + const aIndex = preferredOrder.indexOf(a); + const bIndex = preferredOrder.indexOf(b); + + if (aIndex !== -1 || bIndex !== -1) { + if (aIndex === -1) return 1; + if (bIndex === -1) return -1; + return aIndex - bIndex; + } + + return a.localeCompare(b); + }); +} + +function renderDocument(items) { + const sortedItems = [...items].sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime()); + const groups = groupByCategory(sortedItems); + const categories = sortedCategories(groups); + const totalCount = sortedItems.length; + + const categoryChips = categories + .map((category) => { + const count = groups.get(category).length; + return `${escapeHtml( + category + )} ${count}`; + }) + .join("\n"); + + const groupsMarkup = categories + .map((category) => { + const entries = groups.get(category); + const entryMarkup = entries + .map((entry) => { + const extension = path.extname(entry.relativePath).replace(".", "") || "file"; + const searchable = `${entry.relativePath} ${category}`.toLowerCase(); + return ` +
  • + ${escapeHtml( + entry.relativePath + )} +
    + ${escapeHtml(extension)} + ${escapeHtml(formatBytes(entry.sizeBytes))} + ${escapeHtml(dateFormatter.format(entry.modifiedAt))} +
    +
    • + `; + }) + .join("\n"); + + return ` +
    +

    ${escapeHtml(category)} ${entries.length}

    +
      + ${entryMarkup} +
    +
    + `; + }) + .join("\n"); + + return ` + + + + + Islandflow Docs + + + +
    +
    +

    Islandflow docs index

    +

    A browsable index of files under docs/ with filtering and grouped navigation.

    +
    + +
    +
    ${totalCount} of ${totalCount} files shown
    + + +
    + +
    ${groupsMarkup}
    +

    No files match that filter.

    +
    + + + + +`; +} + +async function main() { + const files = await collectDocsFiles(docsDir); + const html = renderDocument(files); + await fs.writeFile(outputFile, html, "utf8"); + console.log(`Generated ${outputFile} with ${files.length} entries.`); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/services/api/src/index.ts b/services/api/src/index.ts index 433222a..562fb6b 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -9,6 +9,7 @@ import { SUBJECT_EQUITY_QUOTES, SUBJECT_INFERRED_DARK, SUBJECT_FLOW_PACKETS, + SUBJECT_NEWS, SUBJECT_SMART_MONEY_EVENTS, SUBJECT_OPTION_NBBO, SUBJECT_OPTION_SIGNAL_PRINTS, @@ -20,6 +21,7 @@ import { STREAM_EQUITY_QUOTES, STREAM_INFERRED_DARK, STREAM_FLOW_PACKETS, + STREAM_NEWS, STREAM_SMART_MONEY_EVENTS, STREAM_OPTION_NBBO, STREAM_OPTION_SIGNAL_PRINTS, @@ -35,6 +37,7 @@ import { import { createClickHouseClient, ensureAlertsTable, + ensureNewsTable, ensureClassifierHitsTable, ensureEquityCandlesTable, ensureEquityPrintJoinsTable, @@ -48,6 +51,8 @@ import { fetchAlertsAfter, fetchAlertsBefore, fetchAlertContextByTraceId, + fetchNewsAfter, + fetchNewsBefore, fetchClassifierHitsAfter, fetchClassifierHitsBefore, fetchSmartMoneyEventsAfter, @@ -58,6 +63,7 @@ import { fetchFlowPacketsByMemberTraceIds, fetchFlowPacketsBefore, fetchRecentAlerts, + fetchRecentNews, fetchRecentClassifierHits, fetchRecentSmartMoneyEvents, fetchRecentEquityPrintJoins, @@ -86,7 +92,8 @@ import { fetchNearestOptionNBBOForPrints, fetchSmartMoneyEventsByPacketIds, fetchClassifierHitsByPacketIds, - fetchRecentOptionPrints + fetchRecentOptionPrints, + insertNewsStory } from "@islandflow/storage"; import type { EquityPrintQueryFilters } from "@islandflow/storage"; import { @@ -99,6 +106,7 @@ import { EquityQuoteSchema, FeedSnapshot, InferredDarkEventSchema, + NewsStorySchema, LiveClientMessageSchema, LiveServerMessage, LiveSubscription, @@ -138,6 +146,7 @@ const DeliverPolicySchema = z.enum(["new", "all", "last", "last_per_subject"]); const envSchema = z.object({ API_PORT: z.coerce.number().int().positive().default(4000), + API_HOST: z.string().min(1).default("127.0.0.1"), NATS_URL: z.string().default("nats://127.0.0.1:4222"), CLICKHOUSE_URL: z.string().default("http://127.0.0.1:8123"), CLICKHOUSE_DATABASE: z.string().default("default"), @@ -676,7 +685,8 @@ const run = async () => { STREAM_FLOW_PACKETS, STREAM_SMART_MONEY_EVENTS, STREAM_CLASSIFIER_HITS, - STREAM_ALERTS + STREAM_ALERTS, + STREAM_NEWS ], { logger } ); @@ -719,6 +729,7 @@ const run = async () => { await ensureSmartMoneyEventsTable(clickhouse); await ensureClassifierHitsTable(clickhouse); await ensureAlertsTable(clickhouse); + await ensureNewsTable(clickhouse); }); let redis: ReturnType | null = null; @@ -843,6 +854,11 @@ const run = async () => { subject: SUBJECT_ALERTS, stream: STREAM_ALERTS, durableName: "api-alerts" + }, + { + subject: SUBJECT_NEWS, + stream: STREAM_NEWS, + durableName: "api-news" } ] as const; @@ -991,10 +1007,16 @@ const run = async () => { consumerBindings[10].durableName ); + const newsSubscription = await subscribeWithReset( + consumerBindings[11].subject, + consumerBindings[11].stream, + consumerBindings[11].durableName + ); + const fanoutLive = async ( subscription: LiveSubscription, item: unknown, - ingestChannel: "options" | "nbbo" | "equities" | "equity-quotes" | "equity-candles" | "equity-overlay" | "equity-joins" | "flow" | "classifier-hits" | "alerts" | "inferred-dark" + ingestChannel: "options" | "nbbo" | "equities" | "equity-quotes" | "equity-candles" | "equity-overlay" | "equity-joins" | "flow" | "classifier-hits" | "alerts" | "inferred-dark" | "news" ) => { const watermark = await liveState.ingest(ingestChannel, item); @@ -1252,6 +1274,22 @@ const run = async () => { } }; + const pumpNews = async () => { + for await (const msg of newsSubscription.messages) { + try { + const payload = NewsStorySchema.parse(newsSubscription.decode(msg)); + await insertNewsStory(clickhouse, payload); + await fanoutLive({ channel: "news" }, payload, "news"); + msg.ack(); + } catch (error) { + logger.error("failed to process news story", { + error: error instanceof Error ? error.message : String(error) + }); + msg.term(); + } + } + }; + void pumpOptions(); void pumpOptionNbbo(); void pumpEquities(); @@ -1263,6 +1301,7 @@ const run = async () => { void pumpSmartMoney(); void pumpClassifierHits(); void pumpAlerts(); + void pumpNews(); const buildSyntheticStatusBody = () => { const derived = @@ -1313,6 +1352,7 @@ const run = async () => { }; const server = Bun.serve({ + hostname: env.API_HOST, port: env.API_PORT, fetch: async (req: Request, serverRef: any) => { const url = new URL(req.url); @@ -1490,6 +1530,12 @@ const run = async () => { return jsonResponse({ data }); } + if (req.method === "GET" && url.pathname === "/news") { + const limit = parseLimit(url.searchParams.get("limit") ?? "100"); + const data = await fetchRecentNews(clickhouse, limit); + return jsonResponse({ data }); + } + if (req.method === "GET" && isAlertContextPath(url.pathname)) { try { const traceId = parseAlertContextTraceIdPath(url.pathname); @@ -1607,6 +1653,14 @@ const run = async () => { ); } + if (req.method === "GET" && url.pathname === "/history/news") { + const { beforeTs, beforeSeq, limit } = parseBeforeParams(url); + const data = await fetchNewsBefore(clickhouse, beforeTs, beforeSeq, limit); + return jsonResponse( + buildHistoryResponse(data, (item) => ({ ts: item.published_ts, seq: item.seq })) + ); + } + if (req.method === "GET" && /^\/flow\/packets\/[^/]+$/.test(url.pathname)) { const id = decodeURIComponent(url.pathname.slice("/flow/packets/".length)); const data = await fetchFlowPacketById(clickhouse, id); @@ -1995,7 +2049,7 @@ const run = async () => { } }); - logger.info("api listening", { port: server.port }); + logger.info("api listening", { host: env.API_HOST, port: server.port }); const shutdown = async (signal: string) => { if (state.shutdownPromise) { diff --git a/services/api/src/live.ts b/services/api/src/live.ts index 024935e..c8d2886 100644 --- a/services/api/src/live.ts +++ b/services/api/src/live.ts @@ -8,6 +8,7 @@ import { fetchRecentEquityQuotes, fetchRecentFlowPackets, fetchRecentInferredDark, + fetchRecentNews, fetchRecentOptionNBBO, fetchRecentSmartMoneyEvents, type ClickHouseClient @@ -25,6 +26,7 @@ import { FeedSnapshot, FlowPacketSchema, InferredDarkEventSchema, + NewsStorySchema, LiveChannelHealth, LiveGenericChannel, LiveHotChannel, @@ -40,6 +42,7 @@ import { type EquityCandle, type EquityPrint, type LiveChannel, + type NewsStory, type OptionPrint } from "@islandflow/types"; import { createMetrics } from "@islandflow/observability"; @@ -63,7 +66,8 @@ const GENERIC_LIMIT_ENV_KEYS: Record = { "smart-money": "LIVE_LIMIT_SMART_MONEY", "classifier-hits": "LIVE_LIMIT_CLASSIFIER_HITS", alerts: "LIVE_LIMIT_ALERTS", - "inferred-dark": "LIVE_LIMIT_INFERRED_DARK" + "inferred-dark": "LIVE_LIMIT_INFERRED_DARK", + news: "LIVE_LIMIT_NEWS" }; const CHART_LIMITS = { @@ -81,7 +85,8 @@ const DEFAULT_LIVE_LIMITS: GenericLiveLimits = { "smart-money": 300, "classifier-hits": 300, alerts: 300, - "inferred-dark": 300 + "inferred-dark": 300, + news: 100 }; const DEFAULT_SCOPED_CACHE_MAX_KEYS = 32; @@ -196,16 +201,28 @@ export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env): env, "inferred-dark", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["inferred-dark"] - ) + ), + news: parseGenericLimit(env, "news", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.news) }; }; -const parsePositiveInt = (value: string | undefined, fallback: number): number => { - const parsed = Number(value); - if (!Number.isFinite(parsed)) { - return fallback; +const extractFreshnessTs = (channel: LiveGenericChannel, item: any): number | null => { + switch (channel) { + case "options": + case "nbbo": + case "equities": + case "equity-quotes": + return typeof item.ts === "number" ? item.ts : null; + case "flow": + case "classifier-hits": + case "alerts": + case "inferred-dark": + return typeof item.source_ts === "number" ? item.source_ts : null; + case "news": + return typeof item.published_ts === "number" ? item.published_ts : null; + default: + return null; } - return Math.max(1, Math.floor(parsed)); }; export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): LiveStateConfig => ({ @@ -217,6 +234,13 @@ export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): Li ), redisFlushMaxItems: parsePositiveInt(env.LIVE_REDIS_FLUSH_MAX_ITEMS, DEFAULT_REDIS_FLUSH_MAX_ITEMS) }); +const parsePositiveInt = (value: string | undefined, fallback: number): number => { + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + return fallback; + } + return Math.max(1, Math.floor(parsed)); +}; type RedisLike = Pick< RedisClientType, @@ -318,6 +342,14 @@ const getGenericConfig = (limits: GenericLiveLimits): { parse: (value) => InferredDarkEventSchema.parse(value), cursor: (item) => ({ ts: item.source_ts, seq: item.seq }), fetchRecent: fetchRecentInferredDark + }, + news: { + redisKey: "live:news", + cursorField: "news", + limit: limits.news, + parse: (value) => NewsStorySchema.parse(value), + cursor: (item) => ({ ts: item.published_ts, seq: item.seq }), + fetchRecent: fetchRecentNews } }); @@ -371,23 +403,6 @@ const normalizeGenericItems = ( return sortGenericItems(items, config.cursor).slice(0, config.limit); }; -const extractFreshnessTs = (channel: LiveGenericChannel, item: any): number | null => { - switch (channel) { - case "options": - case "nbbo": - case "equities": - case "equity-quotes": - return typeof item.ts === "number" ? item.ts : null; - case "flow": - case "classifier-hits": - case "alerts": - case "inferred-dark": - return typeof item.source_ts === "number" ? item.source_ts : null; - default: - return null; - } -}; - const isWithinLiveFeedLookback = ( channel: LiveGenericChannel, item: unknown, diff --git a/services/ingest-equities/src/adapters/alpaca.ts b/services/ingest-equities/src/adapters/alpaca.ts index 672347f..7a1447f 100644 --- a/services/ingest-equities/src/adapters/alpaca.ts +++ b/services/ingest-equities/src/adapters/alpaca.ts @@ -1,3 +1,8 @@ +import { + buildAlpacaAuthHeaders, + buildAlpacaWebSocketAuthMessage, + type AlpacaCredentials +} from "@islandflow/config"; import { createLogger } from "@islandflow/observability"; import type { EquityPrint, EquityQuote } from "@islandflow/types"; import type { EquityIngestAdapter, EquityIngestHandlers } from "./types"; @@ -6,7 +11,7 @@ import WebSocket from "ws"; export type AlpacaEquitiesFeed = "iex" | "sip"; export type AlpacaEquitiesAdapterConfig = { - apiKey: string; + credentials: AlpacaCredentials; restUrl: string; wsBaseUrl: string; feed: AlpacaEquitiesFeed; @@ -62,12 +67,6 @@ const normalizeSymbols = (symbols: string[]): string[] => { return result; }; -const buildHeaders = (config: AlpacaEquitiesAdapterConfig): Record => { - return { - Authorization: `Bearer ${config.apiKey}` - }; -}; - const parseTimestamp = (value: string): number => { const parsed = Date.parse(value); if (Number.isFinite(parsed)) { @@ -157,7 +156,7 @@ const fetchExchangeMeta = async (config: AlpacaEquitiesAdapterConfig): Promise { - if (!config.apiKey) { - throw new Error("Alpaca equities adapter requires ALPACA_API_KEY."); + if (!config.credentials.keyId) { + throw new Error("Alpaca equities adapter requires Alpaca credentials."); } const symbols = normalizeSymbols(config.symbols); @@ -196,7 +195,7 @@ export const createAlpacaEquitiesAdapter = ( const exchangeNameMap = await fetchExchangeMeta(config); const wsUrl = buildWsUrl(config.wsBaseUrl, config.feed); const ws = new WebSocket(wsUrl, { - headers: buildHeaders(config) + headers: buildAlpacaAuthHeaders(config.credentials) }); let seq = 0; @@ -204,13 +203,7 @@ export const createAlpacaEquitiesAdapter = ( let authenticated = false; ws.on("open", () => { - ws.send( - JSON.stringify({ - action: "auth", - key: config.apiKey, - secret: "" - }) - ); + ws.send(JSON.stringify(buildAlpacaWebSocketAuthMessage(config.credentials))); }); const subscribe = () => { diff --git a/services/ingest-equities/src/index.ts b/services/ingest-equities/src/index.ts index f098b15..1b708ae 100644 --- a/services/ingest-equities/src/index.ts +++ b/services/ingest-equities/src/index.ts @@ -1,4 +1,4 @@ -import { readEnv } from "@islandflow/config"; +import { hasAlpacaCredentials, readEnv, resolveAlpacaCredentials } from "@islandflow/config"; import { createLogger } from "@islandflow/observability"; import { SUBJECT_EQUITY_PRINTS, @@ -47,6 +47,10 @@ const envSchema = z.object({ // Alpaca (equities) ALPACA_API_KEY: z.string().default(""), + ALPACA_API_KEY_ID: z.string().default(""), + ALPACA_KEY_ID: z.string().default(""), + ALPACA_API_SECRET_KEY: z.string().default(""), + ALPACA_SECRET_KEY: z.string().default(""), ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"), ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"), ALPACA_UNDERLYINGS: z.string().default("SPY,NVDA,AAPL"), @@ -70,6 +74,7 @@ const envSchema = z.object({ }); const env = readEnv(envSchema); +const alpacaCredentials = resolveAlpacaCredentials(env); const syntheticModes = resolveSyntheticMarketModes({ syntheticMarketMode: env.SYNTHETIC_MARKET_MODE, syntheticEquitiesMode: env.SYNTHETIC_EQUITIES_MODE @@ -175,13 +180,15 @@ const selectAdapter = ( } if (name === "alpaca") { - if (!env.ALPACA_API_KEY) { - logger.warn("alpaca credentials missing; set ALPACA_API_KEY"); - throw new Error("ALPACA_API_KEY is required for the alpaca adapter."); + if (!hasAlpacaCredentials(alpacaCredentials)) { + logger.warn("alpaca credentials missing; set ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY"); + throw new Error( + "Alpaca equities adapter requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or legacy ALPACA_API_KEY)." + ); } return createAlpacaEquitiesAdapter({ - apiKey: env.ALPACA_API_KEY, + credentials: alpacaCredentials, restUrl: env.ALPACA_REST_URL, wsBaseUrl: env.ALPACA_WS_BASE_URL, feed: env.ALPACA_EQUITIES_FEED, diff --git a/services/ingest-news/package.json b/services/ingest-news/package.json new file mode 100644 index 0000000..050f40b --- /dev/null +++ b/services/ingest-news/package.json @@ -0,0 +1,16 @@ +{ + "name": "@islandflow/ingest-news", + "private": true, + "type": "module", + "scripts": { + "dev": "bun run src/index.ts" + }, + "dependencies": { + "@islandflow/bus": "workspace:*", + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + "@islandflow/types": "workspace:*", + "ws": "^8.18.3", + "zod": "^3.23.8" + } +} diff --git a/services/ingest-news/src/index.ts b/services/ingest-news/src/index.ts new file mode 100644 index 0000000..95cca42 --- /dev/null +++ b/services/ingest-news/src/index.ts @@ -0,0 +1,229 @@ +import { + buildAlpacaAuthHeaders, + buildAlpacaWebSocketAuthMessage, + hasAlpacaCredentials, + readEnv, + resolveAlpacaCredentials +} from "@islandflow/config"; +import { createLogger } from "@islandflow/observability"; +import { + SUBJECT_NEWS, + STREAM_NEWS, + connectJetStreamWithRetry, + ensureKnownStreams, + publishJson +} from "@islandflow/bus"; +import { NewsStorySchema, type NewsStory } from "@islandflow/types"; +import WebSocket from "ws"; +import { z } from "zod"; +import { resolveNewsSymbols } from "./symbols"; + +const service = "ingest-news"; +const logger = createLogger({ service }); + +const envSchema = z.object({ + NATS_URL: z.string().default("nats://127.0.0.1:4222"), + ALPACA_API_KEY: z.string().default(""), + ALPACA_API_KEY_ID: z.string().default(""), + ALPACA_KEY_ID: z.string().default(""), + ALPACA_API_SECRET_KEY: z.string().default(""), + ALPACA_SECRET_KEY: z.string().default(""), + ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"), + ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"), + ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(50).default(50), + ALPACA_NEWS_WEBSOCKET_PATH: z.string().default("/v1beta1/news") +}); + +const env = readEnv(envSchema); +const alpacaCredentials = resolveAlpacaCredentials(env); + +const escapeHtml = (value: string): string => + value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); + +type AlpacaNewsItem = { + id?: number; + headline?: string; + summary?: string; + content?: string; + author?: string; + created_at?: string; + updated_at?: string; + url?: string; + symbols?: string[]; + source?: string; +}; + +type AlpacaNewsResponse = { + news?: AlpacaNewsItem[]; +}; + +const parseTimestamp = (value: string | undefined): number => { + const parsed = value ? Date.parse(value) : Number.NaN; + return Number.isFinite(parsed) ? parsed : Date.now(); +}; + +const toStory = (item: AlpacaNewsItem, seq: number): NewsStory | null => { + const storyId = Number(item.id); + if (!Number.isFinite(storyId) || storyId < 0) { + return null; + } + + const provider = "alpaca"; + const summary = item.summary?.trim() ?? ""; + const contentHtml = item.content?.trim() || (summary ? `

    ${escapeHtml(summary)}

    ` : ""); + const symbols = resolveNewsSymbols(item.symbols ?? [], contentHtml); + const publishedTs = parseTimestamp(item.created_at); + const updatedTs = parseTimestamp(item.updated_at ?? item.created_at); + + return NewsStorySchema.parse({ + source_ts: publishedTs, + ingest_ts: Date.now(), + seq, + trace_id: `${provider}:${storyId}`, + story_id: storyId, + provider, + source: item.source?.trim() || item.author?.trim() || "Alpaca News", + headline: item.headline?.trim() || `Story ${storyId}`, + summary, + content_html: contentHtml, + url: item.url?.trim() || "", + published_ts: publishedTs, + updated_ts: updatedTs, + provider_symbols: symbols.provider_symbols, + resolved_symbols: symbols.resolved_symbols, + symbol_resolution: symbols.symbol_resolution + }); +}; + +const fetchBackfill = async (): Promise => { + const url = new URL("/v1beta1/news", env.ALPACA_REST_URL); + url.searchParams.set("sort", "desc"); + url.searchParams.set("limit", env.ALPACA_NEWS_BACKFILL_LIMIT.toString()); + url.searchParams.set("include_content", "true"); + + const response = await fetch(url.toString(), { + headers: buildAlpacaAuthHeaders(alpacaCredentials) + }); + + if (!response.ok) { + throw new Error(`alpaca news backfill failed (${response.status})`); + } + + const payload = (await response.json()) as AlpacaNewsResponse; + return Array.isArray(payload.news) ? payload.news : []; +}; + +const decodePayload = (data: WebSocket.RawData): unknown => { + if (typeof data === "string") { + return JSON.parse(data) as unknown; + } + if (data instanceof ArrayBuffer) { + return JSON.parse(new TextDecoder().decode(new Uint8Array(data))) as unknown; + } + if (ArrayBuffer.isView(data)) { + return JSON.parse(new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))) as unknown; + } + return JSON.parse(new TextDecoder().decode(new Uint8Array(data as ArrayBuffer))) as unknown; +}; + +const run = async () => { + if (!hasAlpacaCredentials(alpacaCredentials)) { + throw new Error( + "Alpaca news requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or ALPACA_KEY_ID / ALPACA_SECRET_KEY)." + ); + } + + const { nc, js, jsm } = await connectJetStreamWithRetry( + { + servers: env.NATS_URL, + name: service + }, + { attempts: 120, delayMs: 500 } + ); + + await ensureKnownStreams(jsm, [STREAM_NEWS], { logger }); + + let seq = 0; + const publishStory = async (item: AlpacaNewsItem) => { + seq += 1; + const story = toStory(item, seq); + if (!story) { + return; + } + await publishJson(js, SUBJECT_NEWS, story); + }; + + const backfill = await fetchBackfill(); + for (const item of backfill.reverse()) { + await publishStory(item); + } + + const wsUrl = new URL(env.ALPACA_NEWS_WEBSOCKET_PATH, env.ALPACA_WS_BASE_URL).toString(); + const ws = new WebSocket(wsUrl, { + headers: buildAlpacaAuthHeaders(alpacaCredentials) + }); + + ws.on("open", () => { + ws.send(JSON.stringify(buildAlpacaWebSocketAuthMessage(alpacaCredentials))); + }); + + ws.on("message", (raw) => { + let payload: unknown; + try { + payload = decodePayload(raw); + } catch (error) { + logger.warn("failed to decode alpaca news message", { + error: error instanceof Error ? error.message : String(error) + }); + return; + } + + if (!Array.isArray(payload)) { + return; + } + + for (const entry of payload) { + if (!entry || typeof entry !== "object") { + continue; + } + const message = entry as Record; + if (message.T === "success") { + const msg = typeof message.msg === "string" ? message.msg : ""; + if (msg === "authenticated") { + ws.send(JSON.stringify({ action: "subscribe", news: ["*"] })); + } + continue; + } + if (message.T === "subscription" || message.T === "error") { + continue; + } + void publishStory(message as AlpacaNewsItem).catch((error) => { + logger.error("failed to publish alpaca news story", { + error: error instanceof Error ? error.message : String(error) + }); + }); + } + }); + + const shutdown = async (signal: string) => { + logger.info("shutting down", { signal }); + ws.close(); + await nc.drain(); + process.exit(0); + }; + + process.on("SIGINT", () => void shutdown("SIGINT")); + process.on("SIGTERM", () => void shutdown("SIGTERM")); +}; + +void run().catch((error) => { + logger.error("service crashed", { + error: error instanceof Error ? error.message : String(error) + }); + process.exit(1); +}); diff --git a/services/ingest-news/src/symbols.ts b/services/ingest-news/src/symbols.ts new file mode 100644 index 0000000..e1537fd --- /dev/null +++ b/services/ingest-news/src/symbols.ts @@ -0,0 +1,70 @@ +import type { NewsSymbolResolution } from "@islandflow/types"; + +const TICKER_ANCHOR_RE = />\s*([A-Z]{1,5})\s*<\/a>/g; +const EXCHANGE_TICKER_RE = /\b(?:NASDAQ|NYSE|NYSEAMERICAN|AMEX|OTC|CBOE):([A-Z]{1,5})\b/g; +const DOLLAR_TICKER_RE = /\$([A-Z]{1,5})\b/g; + +const normalizeSymbols = (symbols: string[]): string[] => { + const seen = new Set(); + const normalized: string[] = []; + + for (const entry of symbols) { + const symbol = entry.trim().toUpperCase(); + if (!symbol || !/^[A-Z]{1,5}$/.test(symbol) || seen.has(symbol)) { + continue; + } + seen.add(symbol); + normalized.push(symbol); + } + + return normalized; +}; + +const collectMatches = (value: string, regex: RegExp): string[] => { + regex.lastIndex = 0; + const matches: string[] = []; + let match: RegExpExecArray | null = null; + while ((match = regex.exec(value)) !== null) { + matches.push(match[1] ?? ""); + } + return matches; +}; + +export const resolveNewsSymbols = ( + providerSymbols: string[], + contentHtml: string +): { + provider_symbols: string[]; + resolved_symbols: string[]; + symbol_resolution: NewsSymbolResolution; +} => { + const normalizedProvider = normalizeSymbols(providerSymbols); + const derived = normalizeSymbols([ + ...collectMatches(contentHtml, TICKER_ANCHOR_RE), + ...collectMatches(contentHtml, EXCHANGE_TICKER_RE), + ...collectMatches(contentHtml, DOLLAR_TICKER_RE) + ]); + + if (normalizedProvider.length > 0) { + const merged = normalizeSymbols([...normalizedProvider, ...derived]); + return { + provider_symbols: normalizedProvider, + resolved_symbols: merged, + symbol_resolution: derived.length > 0 ? "mixed" : "provider" + }; + } + + if (derived.length > 0) { + return { + provider_symbols: [], + resolved_symbols: derived, + symbol_resolution: "derived" + }; + } + + return { + provider_symbols: [], + resolved_symbols: [], + symbol_resolution: "none" + }; +}; diff --git a/services/ingest-news/tests/symbols.test.ts b/services/ingest-news/tests/symbols.test.ts new file mode 100644 index 0000000..4f3994e --- /dev/null +++ b/services/ingest-news/tests/symbols.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "bun:test"; +import { resolveNewsSymbols } from "../src/symbols"; + +describe("resolveNewsSymbols", () => { + it("prefers provider symbols when present", () => { + const result = resolveNewsSymbols(["tsla", "aapl"], "

    No extra tickers here.

    "); + expect(result.provider_symbols).toEqual(["TSLA", "AAPL"]); + expect(result.resolved_symbols).toEqual(["TSLA", "AAPL"]); + expect(result.symbol_resolution).toBe("provider"); + }); + + it("falls back to ticker anchors", () => { + const result = resolveNewsSymbols([], 'TSLA'); + expect(result.resolved_symbols).toEqual(["TSLA"]); + expect(result.symbol_resolution).toBe("derived"); + }); + + it("falls back to exchange and dollar patterns", () => { + const result = resolveNewsSymbols([], "

    NASDAQ:TSLA met with $IBM executives.

    "); + expect(result.resolved_symbols).toEqual(["TSLA", "IBM"]); + expect(result.symbol_resolution).toBe("derived"); + }); + + it("dedupes and uppercases merged symbols", () => { + const result = resolveNewsSymbols(["tsla"], "

    $TSLA and NASDAQ:TSLA

    "); + expect(result.provider_symbols).toEqual(["TSLA"]); + expect(result.resolved_symbols).toEqual(["TSLA"]); + expect(result.symbol_resolution).toBe("mixed"); + }); +}); diff --git a/services/ingest-news/tsconfig.json b/services/ingest-news/tsconfig.json new file mode 100644 index 0000000..43ef119 --- /dev/null +++ b/services/ingest-news/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": [] + }, + "include": ["src/**/*.ts", "tests/**/*.ts"] +} diff --git a/services/ingest-options/src/adapters/alpaca.ts b/services/ingest-options/src/adapters/alpaca.ts index dce7702..00645b8 100644 --- a/services/ingest-options/src/adapters/alpaca.ts +++ b/services/ingest-options/src/adapters/alpaca.ts @@ -1,4 +1,9 @@ import { decode, encode } from "@msgpack/msgpack"; +import { + buildAlpacaAuthHeaders, + buildAlpacaWebSocketAuthMessage, + type AlpacaCredentials +} from "@islandflow/config"; import { createLogger } from "@islandflow/observability"; import type { OptionIngestAdapter, OptionIngestHandlers } from "./types"; import WebSocket from "ws"; @@ -6,7 +11,7 @@ import WebSocket from "ws"; type AlpacaFeed = "indicative" | "opra"; type AlpacaOptionsAdapterConfig = { - apiKey: string; + credentials: AlpacaCredentials; restUrl: string; wsBaseUrl: string; feed: AlpacaFeed; @@ -147,18 +152,12 @@ const normalizeUnderlyings = (value: string[]): string[] => { return result; }; -const buildHeaders = (config: AlpacaOptionsAdapterConfig): Record => { - return { - Authorization: `Bearer ${config.apiKey}` - }; -}; - const fetchJson = async ( url: URL, config: AlpacaOptionsAdapterConfig ): Promise => { const response = await fetch(url.toString(), { - headers: buildHeaders(config) + headers: buildAlpacaAuthHeaders(config.credentials) }); if (!response.ok) { @@ -398,8 +397,8 @@ export const createAlpacaOptionsAdapter = ( return { name: "alpaca", start: async (handlers: OptionIngestHandlers) => { - if (!config.apiKey) { - throw new Error("Alpaca adapter requires ALPACA_API_KEY."); + if (!config.credentials.keyId) { + throw new Error("Alpaca adapter requires Alpaca credentials."); } const underlyings = normalizeUnderlyings(config.underlyings); @@ -485,15 +484,22 @@ export const createAlpacaOptionsAdapter = ( const wsUrl = `${wsBase}/${config.feed}`; const ws = new WebSocket(wsUrl, { headers: { - ...buildHeaders(config), + ...buildAlpacaAuthHeaders(config.credentials), "Content-Type": "application/msgpack" } }); let seq = 0; let stopped = false; + let subscribed = false; + + const subscribe = () => { + if (subscribed) { + return; + } + + subscribed = true; - ws.on("open", () => { const subscribe: Record = { action: "subscribe", trades: selectedSymbols @@ -504,6 +510,10 @@ export const createAlpacaOptionsAdapter = ( } ws.send(encode(subscribe)); + }; + + ws.on("open", () => { + ws.send(encode(buildAlpacaWebSocketAuthMessage(config.credentials))); }); ws.on("message", (data) => { @@ -583,7 +593,13 @@ export const createAlpacaOptionsAdapter = ( if (type === "error") { logger.error("alpaca stream error", { message }); - } else if (type === "success" || type === "subscription") { + } else if (type === "success") { + const status = (message as { msg?: string }).msg ?? ""; + if (status === "authenticated") { + subscribe(); + } + logger.info("alpaca stream status", { message }); + } else if (type === "subscription") { logger.info("alpaca stream status", { message }); } } diff --git a/services/ingest-options/src/index.ts b/services/ingest-options/src/index.ts index a52661f..301632e 100644 --- a/services/ingest-options/src/index.ts +++ b/services/ingest-options/src/index.ts @@ -1,4 +1,4 @@ -import { readEnv } from "@islandflow/config"; +import { hasAlpacaCredentials, readEnv, resolveAlpacaCredentials } from "@islandflow/config"; import { createLogger } from "@islandflow/observability"; import { SUBJECT_OPTION_NBBO, @@ -55,6 +55,10 @@ const envSchema = z.object({ CLICKHOUSE_DATABASE: z.string().default("default"), OPTIONS_INGEST_ADAPTER: z.string().min(1).default("synthetic"), ALPACA_API_KEY: z.string().default(""), + ALPACA_API_KEY_ID: z.string().default(""), + ALPACA_KEY_ID: z.string().default(""), + ALPACA_API_SECRET_KEY: z.string().default(""), + ALPACA_SECRET_KEY: z.string().default(""), ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"), ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets/v1beta1"), ALPACA_FEED: z.enum(["indicative", "opra"]).default("indicative"), @@ -120,6 +124,7 @@ const envSchema = z.object({ }); const env = readEnv(envSchema); +const alpacaCredentials = resolveAlpacaCredentials(env); const syntheticModes = resolveSyntheticMarketModes({ syntheticMarketMode: env.SYNTHETIC_MARKET_MODE, syntheticOptionsMode: env.SYNTHETIC_OPTIONS_MODE @@ -277,15 +282,17 @@ const selectAdapter = ( } if (name === "alpaca") { - if (!env.ALPACA_API_KEY) { - logger.warn("alpaca credentials missing; set ALPACA_API_KEY"); - throw new Error("ALPACA_API_KEY is required for the alpaca adapter."); + if (!hasAlpacaCredentials(alpacaCredentials)) { + logger.warn("alpaca credentials missing; set ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY"); + throw new Error( + "Alpaca adapter requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or legacy ALPACA_API_KEY)." + ); } const underlyings = env.ALPACA_UNDERLYINGS.split(",").map((symbol) => symbol.trim()); return createAlpacaOptionsAdapter({ - apiKey: env.ALPACA_API_KEY, + credentials: alpacaCredentials, restUrl: env.ALPACA_REST_URL, wsBaseUrl: env.ALPACA_WS_BASE_URL, feed: env.ALPACA_FEED,