alpaca-news #5
23 changed files with 535 additions and 85 deletions
|
|
@ -15,6 +15,7 @@
|
|||
{"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-laq","title":"fix native alpaca news deploy and auth","description":"Why this issue exists and what needs to be done:\\n\\nNative Islandflow rollout is incomplete because services/ingest-news is not healthy on the VPS. The checked-in native user units and helper scripts do not fully include ingest-news, and the current service uses bearer-style auth that returns 401 against Alpaca news endpoints.\\n\\nThis task should verify the current Alpaca news auth requirements against official docs, update the repo code and native deployment assets as needed, install and enable the missing VPS unit, verify news events flow end-to-end, and document the work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:47:07Z","created_by":"dirtydishes","updated_at":"2026-05-20T00:05:20Z","started_at":"2026-05-19T23:47:12Z","closed_at":"2026-05-20T00:05:20Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-fmg","title":"Fix native deploy SSH path and verification cwd assumptions","description":"Native deploys over SSH assumed bun was already on PATH and that remote verification would run from the repository root. On the live VPS, non-login SSH shells omitted /home/delta/.bun/bin and remote native verification could not find deployment/native/check-native-infra.sh because it ran from the home directory. Update the deploy helper to prepend /Users/kell/.bun/bin when present and cd into the repo before native verification checks run.","status":"closed","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:38:32Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:40:33Z","closed_at":"2026-05-19T23:40:33Z","close_reason":"Updated native SSH deploy flow to prepend Bun's home install path when present and run native verification from the repo root before health scripts.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-wf5","title":"Harden native options provider configuration after synthetic recovery","description":"Native production recovery restored OPTIONS_INGEST_ADAPTER=synthetic because the current Alpaca setup fails authentication and crash-loops ingest-options. Follow up by deciding whether production options should remain synthetic or move to a supported live provider auth path, then add a deploy-time smoke test or config validation that catches provider auth failures before native cutover.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:27:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:51Z","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-m83","title":"Restore options ingestion and print generation on native deployment","description":"After moving the production/VPS deployment from Docker-managed services to the native runtime, the options feed appears behind and fresh option prints are not reaching the UI. Investigate the native deployment path on the server, identify the ingestion or compute breakage, apply the required code and/or host configuration changes, validate that fresh option prints resume, and document any follow-up operational work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:20:01Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:52Z","started_at":"2026-05-19T23:20:10Z","closed_at":"2026-05-19T23:27:52Z","close_reason":"Restored native options ingest by switching the VPS back to the last known-good synthetic adapter, verified fresh option prints and compute output, and documented the native env precedence gotcha.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,10 @@ REDIS_URL=redis://127.0.0.1:6379
|
|||
# Options ingest
|
||||
OPTIONS_INGEST_ADAPTER=synthetic
|
||||
ALPACA_API_KEY=
|
||||
ALPACA_API_KEY_ID=
|
||||
ALPACA_KEY_ID=
|
||||
ALPACA_API_SECRET_KEY=
|
||||
ALPACA_SECRET_KEY=
|
||||
ALPACA_REST_URL=https://data.alpaca.markets
|
||||
ALPACA_WS_BASE_URL=wss://stream.data.alpaca.markets/v1beta1
|
||||
ALPACA_FEED=indicative
|
||||
|
|
|
|||
|
|
@ -255,7 +255,11 @@ All runtime configuration comes from `.env`.
|
|||
|
||||
| Variable | Default | What it controls |
|
||||
| --- | --- | --- |
|
||||
| `ALPACA_API_KEY` | empty | Single-token Alpaca API auth for options, equities, and news adapters. |
|
||||
| `ALPACA_API_KEY` | empty | Legacy single-token fallback kept for older Alpaca setups. Prefer explicit key ID + secret vars for current Alpaca auth. |
|
||||
| `ALPACA_API_KEY_ID` | empty | Preferred Alpaca key ID used for market-data REST and websocket auth. |
|
||||
| `ALPACA_KEY_ID` | empty | Alternate name accepted for the Alpaca key ID. |
|
||||
| `ALPACA_API_SECRET_KEY` | empty | Preferred Alpaca secret key paired with `ALPACA_API_KEY_ID`. |
|
||||
| `ALPACA_SECRET_KEY` | empty | Alternate name accepted for the Alpaca secret key. |
|
||||
| `ALPACA_REST_URL` | `https://data.alpaca.markets` | Alpaca REST base URL. |
|
||||
| `ALPACA_WS_BASE_URL` | `wss://stream.data.alpaca.markets/v1beta1` for options, `wss://stream.data.alpaca.markets` for equities/news | Alpaca websocket base URL. |
|
||||
| `ALPACA_FEED` | `indicative` | Options feed tier: `indicative` or `opra`. |
|
||||
|
|
@ -266,7 +270,7 @@ All runtime configuration comes from `.env`.
|
|||
| `ALPACA_MONEYNESS_FALLBACK_PCT` | `0.1` | Wider fallback moneyness filter if candidate set is too sparse. |
|
||||
| `ALPACA_MAX_QUOTES` | `200` | Upper bound on selected Alpaca options contracts/quotes per cycle. |
|
||||
| `ALPACA_EQUITIES_FEED` | `iex` | Alpaca equities feed: `iex` or `sip`. |
|
||||
| `ALPACA_NEWS_BACKFILL_LIMIT` | `100` | Alpaca news stories fetched on startup, capped at 200. |
|
||||
| `ALPACA_NEWS_BACKFILL_LIMIT` | `50` | Alpaca news stories fetched on startup, capped at 50 by the Alpaca News API. |
|
||||
| `ALPACA_NEWS_WEBSOCKET_PATH` | `/v1beta1/news` | Alpaca news websocket path. |
|
||||
|
||||
### Databento replay adapter configuration
|
||||
|
|
|
|||
|
|
@ -27,6 +27,10 @@ NEXT_PUBLIC_NBBO_MAX_AGE_MS=1000
|
|||
# Options ingest
|
||||
OPTIONS_INGEST_ADAPTER=synthetic
|
||||
ALPACA_API_KEY=
|
||||
ALPACA_API_KEY_ID=
|
||||
ALPACA_KEY_ID=
|
||||
ALPACA_API_SECRET_KEY=
|
||||
ALPACA_SECRET_KEY=
|
||||
ALPACA_REST_URL=https://data.alpaca.markets
|
||||
ALPACA_WS_BASE_URL=wss://stream.data.alpaca.markets/v1beta1
|
||||
ALPACA_FEED=indicative
|
||||
|
|
|
|||
|
|
@ -161,8 +161,10 @@ Set the adapter values and credentials in `.env`:
|
|||
|
||||
- `OPTIONS_INGEST_ADAPTER=alpaca`
|
||||
- `EQUITIES_INGEST_ADAPTER=alpaca`
|
||||
- `ALPACA_KEY_ID=...`
|
||||
- `ALPACA_SECRET_KEY=...`
|
||||
- `ALPACA_API_KEY_ID=...`
|
||||
- `ALPACA_API_SECRET_KEY=...`
|
||||
|
||||
The older single-variable `ALPACA_API_KEY` fallback is still accepted for legacy setups, but Alpaca's current market-data auth expects a key ID plus secret key pair.
|
||||
|
||||
### Databento mode
|
||||
|
||||
|
|
@ -284,7 +286,7 @@ Scoped Docker deploys now build only the selected image set and then restart onl
|
|||
- `--web-only`: `docker compose build web`, then `docker compose up -d web`
|
||||
- `--api-only`: `docker compose build api`, then `docker compose up -d api`
|
||||
- `--services-only`: builds and restarts `api`, `compute`, `candles`, `ingest-options`, and `ingest-equities`
|
||||
- `--workers-only`: builds and restarts `compute`, `candles`, `ingest-options`, and `ingest-equities` without touching `web` or `api`
|
||||
- `--workers-only`: builds and restarts `compute`, `candles`, `ingest-options`, `ingest-equities`, and `ingest-news` without touching `web` or `api`
|
||||
- `--fast`: when no explicit scope flag is given, treats the deploy as `--services-only` and skips the public API route suite for quicker completion. It still runs remote service health checks.
|
||||
|
||||
Use `--no-build` only when the image is already correct and you need Compose to recreate or restart containers, such as after changing server-side environment values that do not affect a Next.js build-time variable. Do not use `--no-build` for dependency changes, application source changes, or `NEXT_PUBLIC_*` changes.
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ Checked-in unit files live under:
|
|||
- `deployment/native/systemd/user/islandflow-candles.service`
|
||||
- `deployment/native/systemd/user/islandflow-ingest-options.service`
|
||||
- `deployment/native/systemd/user/islandflow-ingest-equities.service`
|
||||
- `deployment/native/systemd/user/islandflow-ingest-news.service`
|
||||
|
||||
These are written for the current VPS layout:
|
||||
|
||||
|
|
@ -175,6 +176,7 @@ Default unit names used by `scripts/deploy.ts`:
|
|||
- `islandflow-candles`
|
||||
- `islandflow-ingest-options`
|
||||
- `islandflow-ingest-equities`
|
||||
- `islandflow-ingest-news`
|
||||
|
||||
Override them from your local shell before running `./deploy` if the server uses different names:
|
||||
|
||||
|
|
@ -191,6 +193,7 @@ Available overrides:
|
|||
- `DEPLOY_NATIVE_CANDLES_UNIT`
|
||||
- `DEPLOY_NATIVE_INGEST_OPTIONS_UNIT`
|
||||
- `DEPLOY_NATIVE_INGEST_EQUITIES_UNIT`
|
||||
- `DEPLOY_NATIVE_INGEST_NEWS_UNIT`
|
||||
|
||||
## systemctl invocation
|
||||
|
||||
|
|
@ -220,7 +223,7 @@ Scope behavior:
|
|||
- `--web-only`: rebuild/restart only the web unit
|
||||
- `--api-only`: restart only the API unit
|
||||
- `--services-only`: restart API + worker units without touching the web unit
|
||||
- `--workers-only`: restart only `compute`, `candles`, `ingest-options`, and `ingest-equities`
|
||||
- `--workers-only`: restart only `compute`, `candles`, `ingest-options`, `ingest-equities`, and `ingest-news`
|
||||
- `--fast`: when no explicit scope flag is provided, native deploys now default to `--workers-only`
|
||||
- `--no-build`: skip `bun install --frozen-lockfile` and skip the web build step
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ units=()
|
|||
|
||||
case "$scope" in
|
||||
full)
|
||||
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
web)
|
||||
units=(islandflow-web.service)
|
||||
|
|
@ -16,10 +16,10 @@ case "$scope" in
|
|||
units=(islandflow-api.service)
|
||||
;;
|
||||
services)
|
||||
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
workers)
|
||||
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
*)
|
||||
echo "Unknown scope: $scope" >&2
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ esac
|
|||
echo "Stopping Docker-owned Islandflow app services before native ownership starts."
|
||||
(
|
||||
cd "$repo_root/deployment/docker"
|
||||
docker compose stop web api compute candles ingest-options ingest-equities
|
||||
docker compose stop web api compute candles ingest-options ingest-equities ingest-news
|
||||
)
|
||||
|
||||
if [[ "$scope" == "full" || "$scope" == "services" || "$scope" == "api" || "$scope" == "web" ]]; then
|
||||
|
|
@ -24,9 +24,9 @@ if [[ "$scope" == "full" || "$scope" == "services" || "$scope" == "api" || "$sco
|
|||
fi
|
||||
|
||||
systemctl --user restart $(case "$scope" in
|
||||
full) echo islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;;
|
||||
services) echo islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;;
|
||||
workers) echo islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;;
|
||||
full) echo islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service ;;
|
||||
services) echo islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service ;;
|
||||
workers) echo islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service ;;
|
||||
api) echo islandflow-api.service ;;
|
||||
web) echo islandflow-web.service ;;
|
||||
esac)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ set -euo pipefail
|
|||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
|
||||
echo "Stopping native app services."
|
||||
systemctl --user stop islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service || true
|
||||
systemctl --user stop islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service || true
|
||||
|
||||
echo "Stopping native infra before Docker reopens durable data."
|
||||
if [[ "${EUID}" -eq 0 ]]; then
|
||||
|
|
@ -19,7 +19,7 @@ echo "Switching NPM Islandflow upstreams back to Docker service names."
|
|||
echo "Restarting Docker Islandflow runtime."
|
||||
(
|
||||
cd "$repo_root/deployment/docker"
|
||||
docker compose up -d web api compute candles ingest-options ingest-equities
|
||||
docker compose up -d web api compute candles ingest-options ingest-equities ingest-news
|
||||
)
|
||||
|
||||
curl -I -fksS "${DEPLOY_PUBLIC_APP_URL:-https://flow.deltaisland.io}" >/dev/null
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ case "$scope" in
|
|||
none)
|
||||
;;
|
||||
full)
|
||||
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
web)
|
||||
units=(islandflow-web.service)
|
||||
|
|
@ -20,10 +20,10 @@ case "$scope" in
|
|||
units=(islandflow-api.service)
|
||||
;;
|
||||
services)
|
||||
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
workers)
|
||||
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
*)
|
||||
echo "Unknown scope: $scope" >&2
|
||||
|
|
@ -46,4 +46,4 @@ if [[ ${#units[@]} -gt 0 ]]; then
|
|||
echo "Enabled scope: $scope"
|
||||
else
|
||||
echo "No units enabled yet. Pass a scope such as workers when you are ready."
|
||||
fi
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ fi
|
|||
|
||||
case "$scope" in
|
||||
full)
|
||||
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
web)
|
||||
units=(islandflow-web.service)
|
||||
|
|
@ -39,10 +39,10 @@ case "$scope" in
|
|||
units=(islandflow-api.service)
|
||||
;;
|
||||
services)
|
||||
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
workers)
|
||||
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
*)
|
||||
echo "Unknown scope: $scope" >&2
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
[Unit]
|
||||
Description=Islandflow ingest-news
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/home/delta/islandflow
|
||||
EnvironmentFile=/home/delta/islandflow/.env
|
||||
ExecStart=/home/delta/.bun/bin/bun services/ingest-news/src/index.ts
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
KillSignal=SIGINT
|
||||
TimeoutStopSec=20
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
233
docs/turns/2026-05-19-fix-native-alpaca-news.html
Normal file
233
docs/turns/2026-05-19-fix-native-alpaca-news.html
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Turn Report: Fix Native Alpaca News</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: #0b0f14;
|
||||
--panel: #121821;
|
||||
--panel-2: #0f141b;
|
||||
--border: rgba(255, 255, 255, 0.08);
|
||||
--text: #e8eef5;
|
||||
--muted: #93a3b5;
|
||||
--accent: #7dd3fc;
|
||||
--accent-2: #a78bfa;
|
||||
--good: #86efac;
|
||||
--warn: #fbbf24;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0;
|
||||
background:
|
||||
radial-gradient(circle at top left, rgba(125, 211, 252, 0.12), transparent 28%),
|
||||
radial-gradient(circle at top right, rgba(167, 139, 250, 0.12), transparent 32%),
|
||||
linear-gradient(180deg, #080b10 0%, var(--bg) 100%);
|
||||
color: var(--text);
|
||||
font: 15px/1.65 "IBM Plex Sans", "Segoe UI", sans-serif;
|
||||
padding: 32px;
|
||||
}
|
||||
main {
|
||||
max-width: 1040px;
|
||||
margin: 0 auto;
|
||||
background: rgba(18, 24, 33, 0.92);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 20px;
|
||||
padding: 32px;
|
||||
box-shadow: 0 24px 80px rgba(0, 0, 0, 0.35);
|
||||
}
|
||||
h1, h2 {
|
||||
margin: 0 0 12px;
|
||||
font-family: "IBM Plex Mono", monospace;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
h1 { font-size: 1.85rem; }
|
||||
h2 { font-size: 1rem; margin-top: 28px; }
|
||||
p, li { margin: 0 0 12px; }
|
||||
.meta {
|
||||
color: var(--muted);
|
||||
font-size: 0.9rem;
|
||||
margin-bottom: 18px;
|
||||
}
|
||||
.summary {
|
||||
padding: 18px 20px;
|
||||
border-radius: 16px;
|
||||
border: 1px solid rgba(125, 211, 252, 0.24);
|
||||
background: linear-gradient(135deg, rgba(125, 211, 252, 0.10), rgba(167, 139, 250, 0.10));
|
||||
}
|
||||
section {
|
||||
margin-top: 28px;
|
||||
padding-top: 22px;
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
ul {
|
||||
margin: 0;
|
||||
padding-left: 18px;
|
||||
}
|
||||
code, pre {
|
||||
font-family: "IBM Plex Mono", monospace;
|
||||
}
|
||||
pre {
|
||||
margin: 0 0 14px;
|
||||
padding: 16px;
|
||||
overflow: auto;
|
||||
border-radius: 14px;
|
||||
background: var(--panel-2);
|
||||
border: 1px solid var(--border);
|
||||
}
|
||||
.pill-row {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
margin: 14px 0 0;
|
||||
}
|
||||
.pill {
|
||||
padding: 7px 10px;
|
||||
border-radius: 999px;
|
||||
border: 1px solid var(--border);
|
||||
background: rgba(255, 255, 255, 0.03);
|
||||
color: var(--muted);
|
||||
font-size: 0.84rem;
|
||||
}
|
||||
.good { color: var(--good); }
|
||||
.warn { color: var(--warn); }
|
||||
a { color: var(--accent); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<p class="meta">Created 2026-05-19 20:05 EDT · Branch: <code>alpaca-news</code> · Issue: <code>islandflow-laq</code></p>
|
||||
<h1>Fix Native Alpaca News</h1>
|
||||
<div class="summary">
|
||||
<p>
|
||||
Restored the native Alpaca news pipeline on the VPS by correcting Alpaca auth to use key ID + secret,
|
||||
adding the missing native <code>islandflow-ingest-news</code> unit and worker-scope wiring, fixing the
|
||||
Alpaca news backfill defaults to match the current API contract, requesting article content explicitly,
|
||||
and repairing API-side news persistence so the feed is both live and queryable.
|
||||
</p>
|
||||
<div class="pill-row">
|
||||
<span class="pill">VPS unit installed and enabled</span>
|
||||
<span class="pill">Alpaca auth aligned to current docs</span>
|
||||
<span class="pill">Live news confirmed</span>
|
||||
<span class="pill">ClickHouse news history confirmed</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section>
|
||||
<h2>Summary</h2>
|
||||
<p>
|
||||
The original native news rollout failed for two separate reasons: the repo never fully wired
|
||||
<code>ingest-news</code> into the native worker templates, and the service was still using bearer-style
|
||||
Alpaca auth plus an oversized backfill limit that Alpaca's current News API rejects. After the service
|
||||
started flowing again, one more pipeline gap appeared: the API fanned news out live but never persisted it
|
||||
to ClickHouse, so <code>/news</code> stayed empty even when headlines showed up in the UI.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Changes Made</h2>
|
||||
<ul>
|
||||
<li>Added shared Alpaca credential helpers in <code>packages/config</code> with support for official key ID + secret auth and a legacy bearer fallback.</li>
|
||||
<li>Rewired the Alpaca news, options, and equities adapters to use the shared auth model instead of hardcoded bearer headers and empty websocket secrets.</li>
|
||||
<li>Added the checked-in native user unit <code>deployment/native/systemd/user/islandflow-ingest-news.service</code>.</li>
|
||||
<li>Updated native install, health, cutover, rollback, and deploy-scope scripts so worker/native rollouts include <code>ingest-news</code>.</li>
|
||||
<li>Corrected the native and Docker env/docs story to advertise current Alpaca credential names.</li>
|
||||
<li>Lowered the default Alpaca news backfill limit from <code>100</code> to <code>50</code> to match the current endpoint contract.</li>
|
||||
<li>Requested <code>include_content=true</code> for Alpaca news backfill and added a safe summary fallback when article content is missing.</li>
|
||||
<li>Fixed API-side persistence by inserting each consumed news story into ClickHouse before live fanout.</li>
|
||||
<li>On the VPS, created a fresh <code>.env</code> backup, added <code>ALPACA_API_KEY_ID</code> and <code>ALPACA_API_SECRET_KEY</code>, set <code>ALPACA_NEWS_BACKFILL_LIMIT=50</code>, switched the server checkout to <code>alpaca-news</code>, installed the new user unit, and restarted <code>api</code> plus <code>ingest-news</code>.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Context</h2>
|
||||
<p>
|
||||
Alpaca's current official auth docs require the <code>APCA-API-KEY-ID</code> and
|
||||
<code>APCA-API-SECRET-KEY</code> header pair for market-data requests, and the current News endpoint
|
||||
documents a <code>limit</code> range of <code>1..50</code> plus optional
|
||||
<code>include_content</code>. This turn aligned Islandflow's native news path with those present-day
|
||||
contracts instead of relying on the older single-token assumption that had drifted into the repo.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Important Implementation Details</h2>
|
||||
<ul>
|
||||
<li>The shared helper prefers <code>ALPACA_API_KEY_ID</code> + <code>ALPACA_API_SECRET_KEY</code>, also accepts <code>ALPACA_KEY_ID</code> + <code>ALPACA_SECRET_KEY</code>, and only falls back to legacy bearer auth when no secret is present.</li>
|
||||
<li>The news backfill now requests article bodies explicitly. When Alpaca still omits full content, the service emits an escaped summary paragraph instead of a blank story body.</li>
|
||||
<li>The native worker scope now treats <code>ingest-news</code> as a first-class worker everywhere the repo previously only handled options and equities.</li>
|
||||
<li>The API now persists each consumed news story into ClickHouse before live fanout, which restores <code>/news</code> and history behavior without removing the live websocket path.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Relevant Diff Snippets</h2>
|
||||
<pre><code class="language-diff">diff --git a/packages/config/src/alpaca.ts b/packages/config/src/alpaca.ts
|
||||
+export const buildAlpacaAuthHeaders = (credentials) => ({
|
||||
+ "APCA-API-KEY-ID": credentials.keyId,
|
||||
+ "APCA-API-SECRET-KEY": credentials.secret
|
||||
+})
|
||||
+export const buildAlpacaWebSocketAuthMessage = (credentials) => ({
|
||||
+ action: "auth",
|
||||
+ key: credentials.keyId,
|
||||
+ secret: credentials.secret
|
||||
+})</code></pre>
|
||||
<pre><code class="language-diff">diff --git a/services/ingest-news/src/index.ts b/services/ingest-news/src/index.ts
|
||||
- ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(200).default(100),
|
||||
+ ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(50).default(50),
|
||||
+ url.searchParams.set("include_content", "true");
|
||||
+ const contentHtml = item.content?.trim() || (summary ? `<p>${escapeHtml(summary)}</p>` : "");</code></pre>
|
||||
<pre><code class="language-diff">diff --git a/services/api/src/index.ts b/services/api/src/index.ts
|
||||
const payload = NewsStorySchema.parse(newsSubscription.decode(msg));
|
||||
+ await insertNewsStory(clickhouse, payload);
|
||||
await fanoutLive({ channel: "news" }, payload, "news");
|
||||
msg.ack();</code></pre>
|
||||
<p class="meta">These snippets are included in a diff-style rendering format for fast review.</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Expected Impact for End-Users</h2>
|
||||
<p>
|
||||
Native Islandflow deployments on the VPS now have a real Alpaca-backed news worker instead of a missing unit
|
||||
and a crash loop. News stories populate with actual article body content in the feed more reliably, and the
|
||||
API's <code>/news</code> path can serve persisted recent stories instead of only depending on live websocket
|
||||
state.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Validation</h2>
|
||||
<ul>
|
||||
<li>Ran local targeted tests: <code>bun test packages/config/tests packages/storage/tests/news.test.ts services/ingest-news/tests services/ingest-equities/tests</code> and all passed.</li>
|
||||
<li>Ran <code>bun run check:docker-workspace</code> and confirmed the Docker workspace snapshot stayed in sync.</li>
|
||||
<li>Verified against current Alpaca docs that market-data auth uses key ID + secret and that the news endpoint limit is capped at 50.</li>
|
||||
<li>On the VPS, confirmed the new <code>islandflow-ingest-news.service</code> unit is installed, enabled, and active under <code>systemd --user</code>.</li>
|
||||
<li>Queried Alpaca directly from the VPS with the configured credentials and confirmed <code>GET https://data.alpaca.markets/v1beta1/news?limit=1&sort=desc</code> returned <span class="good">HTTP 200</span>.</li>
|
||||
<li>Restarted the VPS <code>api</code> and <code>ingest-news</code> services after the persistence fix so the API would store newly republished backfill stories.</li>
|
||||
<li>Verified VPS API output: <code>GET http://127.0.0.1:4000/news?limit=3</code> returned 3 recent real Alpaca stories with non-empty <code>content_html</code> payloads.</li>
|
||||
<li>Verified ClickHouse persistence: <code>SELECT count(), max(story_id), max(published_ts) FROM news</code> returned <code>50</code> rows after the republished backfill.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Issues, Limitations, and Mitigations</h2>
|
||||
<ul>
|
||||
<li>The server checkout still carries an unrelated untracked file, <code>deployment/docker/signal-cli-0.14.3-Linux-native.tar.gz</code>. It does not block the news fix, but it is repo hygiene debt on the VPS checkout.</li>
|
||||
<li>The shared Alpaca helper keeps a legacy bearer fallback so older setups do not fail immediately, but the repo documentation now treats key ID + secret as the supported path.</li>
|
||||
<li>Some Alpaca/Benzinga stories may still omit full content. The summary fallback prevents a blank drawer in those cases, but it cannot synthesize text Alpaca does not send.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Follow-up Work</h2>
|
||||
<ul>
|
||||
<li>No new follow-up Beads issue was required to ship this repair.</li>
|
||||
<li>If native Alpaca options or equities are re-enabled later, the shared credential changes in this turn already cover the same key ID + secret auth model.</li>
|
||||
<li>If the team wants historical news beyond the startup backfill, the next logical extension is a scheduled catch-up cursor instead of only restart-time republishing.</li>
|
||||
</ul>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
76
packages/config/src/alpaca.ts
Normal file
76
packages/config/src/alpaca.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
export type AlpacaCredentials = {
|
||||
keyId: string;
|
||||
secret: string;
|
||||
legacyToken: string;
|
||||
usesLegacyBearer: boolean;
|
||||
};
|
||||
|
||||
type AlpacaCredentialEnv = {
|
||||
ALPACA_API_KEY?: string;
|
||||
ALPACA_API_KEY_ID?: string;
|
||||
ALPACA_KEY_ID?: string;
|
||||
ALPACA_API_SECRET_KEY?: string;
|
||||
ALPACA_SECRET_KEY?: string;
|
||||
};
|
||||
|
||||
const normalize = (value: string | undefined): string => value?.trim() ?? "";
|
||||
|
||||
export const resolveAlpacaCredentials = (
|
||||
env: AlpacaCredentialEnv
|
||||
): AlpacaCredentials => {
|
||||
const legacyToken = normalize(env.ALPACA_API_KEY);
|
||||
const explicitKeyId =
|
||||
normalize(env.ALPACA_API_KEY_ID) || normalize(env.ALPACA_KEY_ID);
|
||||
const secret =
|
||||
normalize(env.ALPACA_API_SECRET_KEY) || normalize(env.ALPACA_SECRET_KEY);
|
||||
const keyId = explicitKeyId || legacyToken;
|
||||
const usesLegacyBearer = !explicitKeyId && !secret && legacyToken.length > 0;
|
||||
|
||||
return {
|
||||
keyId,
|
||||
secret,
|
||||
legacyToken,
|
||||
usesLegacyBearer
|
||||
};
|
||||
};
|
||||
|
||||
export const hasAlpacaCredentials = (credentials: AlpacaCredentials): boolean => {
|
||||
if (credentials.usesLegacyBearer) {
|
||||
return credentials.legacyToken.length > 0;
|
||||
}
|
||||
|
||||
return credentials.keyId.length > 0 && credentials.secret.length > 0;
|
||||
};
|
||||
|
||||
export const buildAlpacaAuthHeaders = (
|
||||
credentials: AlpacaCredentials
|
||||
): Record<string, string> => {
|
||||
if (credentials.usesLegacyBearer) {
|
||||
return {
|
||||
Authorization: `Bearer ${credentials.legacyToken}`
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
"APCA-API-KEY-ID": credentials.keyId,
|
||||
"APCA-API-SECRET-KEY": credentials.secret
|
||||
};
|
||||
};
|
||||
|
||||
export const buildAlpacaWebSocketAuthMessage = (
|
||||
credentials: AlpacaCredentials
|
||||
): { action: "auth"; key: string; secret: string } => {
|
||||
if (credentials.usesLegacyBearer) {
|
||||
return {
|
||||
action: "auth",
|
||||
key: credentials.legacyToken,
|
||||
secret: ""
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
action: "auth",
|
||||
key: credentials.keyId,
|
||||
secret: credentials.secret
|
||||
};
|
||||
};
|
||||
|
|
@ -1 +1,2 @@
|
|||
export * from "./env";
|
||||
export * from "./alpaca";
|
||||
|
|
|
|||
65
packages/config/tests/alpaca.test.ts
Normal file
65
packages/config/tests/alpaca.test.ts
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
buildAlpacaAuthHeaders,
|
||||
buildAlpacaWebSocketAuthMessage,
|
||||
hasAlpacaCredentials,
|
||||
resolveAlpacaCredentials
|
||||
} from "../src/alpaca";
|
||||
|
||||
describe("resolveAlpacaCredentials", () => {
|
||||
it("prefers explicit key-id and secret vars", () => {
|
||||
const credentials = resolveAlpacaCredentials({
|
||||
ALPACA_API_KEY: "legacy-token",
|
||||
ALPACA_API_KEY_ID: "key-id",
|
||||
ALPACA_API_SECRET_KEY: "secret"
|
||||
});
|
||||
|
||||
expect(credentials).toEqual({
|
||||
keyId: "key-id",
|
||||
secret: "secret",
|
||||
legacyToken: "legacy-token",
|
||||
usesLegacyBearer: false
|
||||
});
|
||||
expect(hasAlpacaCredentials(credentials)).toBe(true);
|
||||
expect(buildAlpacaAuthHeaders(credentials)).toEqual({
|
||||
"APCA-API-KEY-ID": "key-id",
|
||||
"APCA-API-SECRET-KEY": "secret"
|
||||
});
|
||||
expect(buildAlpacaWebSocketAuthMessage(credentials)).toEqual({
|
||||
action: "auth",
|
||||
key: "key-id",
|
||||
secret: "secret"
|
||||
});
|
||||
});
|
||||
|
||||
it("supports the older bearer-token fallback when no secret exists", () => {
|
||||
const credentials = resolveAlpacaCredentials({
|
||||
ALPACA_API_KEY: "legacy-token"
|
||||
});
|
||||
|
||||
expect(credentials.usesLegacyBearer).toBe(true);
|
||||
expect(hasAlpacaCredentials(credentials)).toBe(true);
|
||||
expect(buildAlpacaAuthHeaders(credentials)).toEqual({
|
||||
Authorization: "Bearer legacy-token"
|
||||
});
|
||||
expect(buildAlpacaWebSocketAuthMessage(credentials)).toEqual({
|
||||
action: "auth",
|
||||
key: "legacy-token",
|
||||
secret: ""
|
||||
});
|
||||
});
|
||||
|
||||
it("supports alternate secret env names", () => {
|
||||
const credentials = resolveAlpacaCredentials({
|
||||
ALPACA_KEY_ID: "short-key",
|
||||
ALPACA_SECRET_KEY: "short-secret"
|
||||
});
|
||||
|
||||
expect(credentials).toEqual({
|
||||
keyId: "short-key",
|
||||
secret: "short-secret",
|
||||
legacyToken: "",
|
||||
usesLegacyBearer: false
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -81,7 +81,8 @@ const DOCKER_WORKER_SERVICES = [
|
|||
"compute",
|
||||
"candles",
|
||||
"ingest-options",
|
||||
"ingest-equities"
|
||||
"ingest-equities",
|
||||
"ingest-news"
|
||||
] as const;
|
||||
|
||||
const scriptPath = fileURLToPath(import.meta.url);
|
||||
|
|
@ -559,7 +560,8 @@ function nativeUnitsForScope(scope: DeployScope): string[] {
|
|||
NATIVE_UNITS.compute,
|
||||
NATIVE_UNITS.candles,
|
||||
NATIVE_UNITS.ingestOptions,
|
||||
NATIVE_UNITS.ingestEquities
|
||||
NATIVE_UNITS.ingestEquities,
|
||||
NATIVE_UNITS.ingestNews
|
||||
];
|
||||
default:
|
||||
return [
|
||||
|
|
|
|||
|
|
@ -92,7 +92,8 @@ import {
|
|||
fetchNearestOptionNBBOForPrints,
|
||||
fetchSmartMoneyEventsByPacketIds,
|
||||
fetchClassifierHitsByPacketIds,
|
||||
fetchRecentOptionPrints
|
||||
fetchRecentOptionPrints,
|
||||
insertNewsStory
|
||||
} from "@islandflow/storage";
|
||||
import type { EquityPrintQueryFilters } from "@islandflow/storage";
|
||||
import {
|
||||
|
|
@ -1277,6 +1278,7 @@ const run = async () => {
|
|||
for await (const msg of newsSubscription.messages) {
|
||||
try {
|
||||
const payload = NewsStorySchema.parse(newsSubscription.decode(msg));
|
||||
await insertNewsStory(clickhouse, payload);
|
||||
await fanoutLive({ channel: "news" }, payload, "news");
|
||||
msg.ack();
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,8 @@
|
|||
import {
|
||||
buildAlpacaAuthHeaders,
|
||||
buildAlpacaWebSocketAuthMessage,
|
||||
type AlpacaCredentials
|
||||
} from "@islandflow/config";
|
||||
import { createLogger } from "@islandflow/observability";
|
||||
import type { EquityPrint, EquityQuote } from "@islandflow/types";
|
||||
import type { EquityIngestAdapter, EquityIngestHandlers } from "./types";
|
||||
|
|
@ -6,7 +11,7 @@ import WebSocket from "ws";
|
|||
export type AlpacaEquitiesFeed = "iex" | "sip";
|
||||
|
||||
export type AlpacaEquitiesAdapterConfig = {
|
||||
apiKey: string;
|
||||
credentials: AlpacaCredentials;
|
||||
restUrl: string;
|
||||
wsBaseUrl: string;
|
||||
feed: AlpacaEquitiesFeed;
|
||||
|
|
@ -62,12 +67,6 @@ const normalizeSymbols = (symbols: string[]): string[] => {
|
|||
return result;
|
||||
};
|
||||
|
||||
const buildHeaders = (config: AlpacaEquitiesAdapterConfig): Record<string, string> => {
|
||||
return {
|
||||
Authorization: `Bearer ${config.apiKey}`
|
||||
};
|
||||
};
|
||||
|
||||
const parseTimestamp = (value: string): number => {
|
||||
const parsed = Date.parse(value);
|
||||
if (Number.isFinite(parsed)) {
|
||||
|
|
@ -157,7 +156,7 @@ const fetchExchangeMeta = async (config: AlpacaEquitiesAdapterConfig): Promise<M
|
|||
|
||||
try {
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: buildHeaders(config)
|
||||
headers: buildAlpacaAuthHeaders(config.credentials)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
|
|
@ -184,8 +183,8 @@ export const createAlpacaEquitiesAdapter = (
|
|||
return {
|
||||
name: "alpaca",
|
||||
start: async (handlers: EquityIngestHandlers) => {
|
||||
if (!config.apiKey) {
|
||||
throw new Error("Alpaca equities adapter requires ALPACA_API_KEY.");
|
||||
if (!config.credentials.keyId) {
|
||||
throw new Error("Alpaca equities adapter requires Alpaca credentials.");
|
||||
}
|
||||
|
||||
const symbols = normalizeSymbols(config.symbols);
|
||||
|
|
@ -196,7 +195,7 @@ export const createAlpacaEquitiesAdapter = (
|
|||
const exchangeNameMap = await fetchExchangeMeta(config);
|
||||
const wsUrl = buildWsUrl(config.wsBaseUrl, config.feed);
|
||||
const ws = new WebSocket(wsUrl, {
|
||||
headers: buildHeaders(config)
|
||||
headers: buildAlpacaAuthHeaders(config.credentials)
|
||||
});
|
||||
|
||||
let seq = 0;
|
||||
|
|
@ -204,13 +203,7 @@ export const createAlpacaEquitiesAdapter = (
|
|||
let authenticated = false;
|
||||
|
||||
ws.on("open", () => {
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
action: "auth",
|
||||
key: config.apiKey,
|
||||
secret: ""
|
||||
})
|
||||
);
|
||||
ws.send(JSON.stringify(buildAlpacaWebSocketAuthMessage(config.credentials)));
|
||||
});
|
||||
|
||||
const subscribe = () => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { readEnv } from "@islandflow/config";
|
||||
import { hasAlpacaCredentials, readEnv, resolveAlpacaCredentials } from "@islandflow/config";
|
||||
import { createLogger } from "@islandflow/observability";
|
||||
import {
|
||||
SUBJECT_EQUITY_PRINTS,
|
||||
|
|
@ -47,6 +47,10 @@ const envSchema = z.object({
|
|||
|
||||
// Alpaca (equities)
|
||||
ALPACA_API_KEY: z.string().default(""),
|
||||
ALPACA_API_KEY_ID: z.string().default(""),
|
||||
ALPACA_KEY_ID: z.string().default(""),
|
||||
ALPACA_API_SECRET_KEY: z.string().default(""),
|
||||
ALPACA_SECRET_KEY: z.string().default(""),
|
||||
ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"),
|
||||
ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"),
|
||||
ALPACA_UNDERLYINGS: z.string().default("SPY,NVDA,AAPL"),
|
||||
|
|
@ -70,6 +74,7 @@ const envSchema = z.object({
|
|||
});
|
||||
|
||||
const env = readEnv(envSchema);
|
||||
const alpacaCredentials = resolveAlpacaCredentials(env);
|
||||
const syntheticModes = resolveSyntheticMarketModes({
|
||||
syntheticMarketMode: env.SYNTHETIC_MARKET_MODE,
|
||||
syntheticEquitiesMode: env.SYNTHETIC_EQUITIES_MODE
|
||||
|
|
@ -175,13 +180,15 @@ const selectAdapter = (
|
|||
}
|
||||
|
||||
if (name === "alpaca") {
|
||||
if (!env.ALPACA_API_KEY) {
|
||||
logger.warn("alpaca credentials missing; set ALPACA_API_KEY");
|
||||
throw new Error("ALPACA_API_KEY is required for the alpaca adapter.");
|
||||
if (!hasAlpacaCredentials(alpacaCredentials)) {
|
||||
logger.warn("alpaca credentials missing; set ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY");
|
||||
throw new Error(
|
||||
"Alpaca equities adapter requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or legacy ALPACA_API_KEY)."
|
||||
);
|
||||
}
|
||||
|
||||
return createAlpacaEquitiesAdapter({
|
||||
apiKey: env.ALPACA_API_KEY,
|
||||
credentials: alpacaCredentials,
|
||||
restUrl: env.ALPACA_REST_URL,
|
||||
wsBaseUrl: env.ALPACA_WS_BASE_URL,
|
||||
feed: env.ALPACA_EQUITIES_FEED,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,10 @@
|
|||
import { readEnv } from "@islandflow/config";
|
||||
import {
|
||||
buildAlpacaAuthHeaders,
|
||||
buildAlpacaWebSocketAuthMessage,
|
||||
hasAlpacaCredentials,
|
||||
readEnv,
|
||||
resolveAlpacaCredentials
|
||||
} from "@islandflow/config";
|
||||
import { createLogger } from "@islandflow/observability";
|
||||
import {
|
||||
SUBJECT_NEWS,
|
||||
|
|
@ -18,13 +24,26 @@ const logger = createLogger({ service });
|
|||
const envSchema = z.object({
|
||||
NATS_URL: z.string().default("nats://127.0.0.1:4222"),
|
||||
ALPACA_API_KEY: z.string().default(""),
|
||||
ALPACA_API_KEY_ID: z.string().default(""),
|
||||
ALPACA_KEY_ID: z.string().default(""),
|
||||
ALPACA_API_SECRET_KEY: z.string().default(""),
|
||||
ALPACA_SECRET_KEY: z.string().default(""),
|
||||
ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"),
|
||||
ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"),
|
||||
ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(200).default(100),
|
||||
ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(50).default(50),
|
||||
ALPACA_NEWS_WEBSOCKET_PATH: z.string().default("/v1beta1/news")
|
||||
});
|
||||
|
||||
const env = readEnv(envSchema);
|
||||
const alpacaCredentials = resolveAlpacaCredentials(env);
|
||||
|
||||
const escapeHtml = (value: string): string =>
|
||||
value
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """)
|
||||
.replaceAll("'", "'");
|
||||
|
||||
type AlpacaNewsItem = {
|
||||
id?: number;
|
||||
|
|
@ -43,10 +62,6 @@ type AlpacaNewsResponse = {
|
|||
news?: AlpacaNewsItem[];
|
||||
};
|
||||
|
||||
const buildHeaders = (): Record<string, string> => ({
|
||||
Authorization: `Bearer ${env.ALPACA_API_KEY}`
|
||||
});
|
||||
|
||||
const parseTimestamp = (value: string | undefined): number => {
|
||||
const parsed = value ? Date.parse(value) : Number.NaN;
|
||||
return Number.isFinite(parsed) ? parsed : Date.now();
|
||||
|
|
@ -59,7 +74,8 @@ const toStory = (item: AlpacaNewsItem, seq: number): NewsStory | null => {
|
|||
}
|
||||
|
||||
const provider = "alpaca";
|
||||
const contentHtml = item.content ?? "";
|
||||
const summary = item.summary?.trim() ?? "";
|
||||
const contentHtml = item.content?.trim() || (summary ? `<p>${escapeHtml(summary)}</p>` : "");
|
||||
const symbols = resolveNewsSymbols(item.symbols ?? [], contentHtml);
|
||||
const publishedTs = parseTimestamp(item.created_at);
|
||||
const updatedTs = parseTimestamp(item.updated_at ?? item.created_at);
|
||||
|
|
@ -73,7 +89,7 @@ const toStory = (item: AlpacaNewsItem, seq: number): NewsStory | null => {
|
|||
provider,
|
||||
source: item.source?.trim() || item.author?.trim() || "Alpaca News",
|
||||
headline: item.headline?.trim() || `Story ${storyId}`,
|
||||
summary: item.summary?.trim() || "",
|
||||
summary,
|
||||
content_html: contentHtml,
|
||||
url: item.url?.trim() || "",
|
||||
published_ts: publishedTs,
|
||||
|
|
@ -88,9 +104,10 @@ const fetchBackfill = async (): Promise<AlpacaNewsItem[]> => {
|
|||
const url = new URL("/v1beta1/news", env.ALPACA_REST_URL);
|
||||
url.searchParams.set("sort", "desc");
|
||||
url.searchParams.set("limit", env.ALPACA_NEWS_BACKFILL_LIMIT.toString());
|
||||
url.searchParams.set("include_content", "true");
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: buildHeaders()
|
||||
headers: buildAlpacaAuthHeaders(alpacaCredentials)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
|
|
@ -115,8 +132,10 @@ const decodePayload = (data: WebSocket.RawData): unknown => {
|
|||
};
|
||||
|
||||
const run = async () => {
|
||||
if (!env.ALPACA_API_KEY) {
|
||||
throw new Error("ALPACA_API_KEY is required for ingest-news.");
|
||||
if (!hasAlpacaCredentials(alpacaCredentials)) {
|
||||
throw new Error(
|
||||
"Alpaca news requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or ALPACA_KEY_ID / ALPACA_SECRET_KEY)."
|
||||
);
|
||||
}
|
||||
|
||||
const { nc, js, jsm } = await connectJetStreamWithRetry(
|
||||
|
|
@ -146,17 +165,11 @@ const run = async () => {
|
|||
|
||||
const wsUrl = new URL(env.ALPACA_NEWS_WEBSOCKET_PATH, env.ALPACA_WS_BASE_URL).toString();
|
||||
const ws = new WebSocket(wsUrl, {
|
||||
headers: buildHeaders()
|
||||
headers: buildAlpacaAuthHeaders(alpacaCredentials)
|
||||
});
|
||||
|
||||
ws.on("open", () => {
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
action: "auth",
|
||||
key: env.ALPACA_API_KEY,
|
||||
secret: ""
|
||||
})
|
||||
);
|
||||
ws.send(JSON.stringify(buildAlpacaWebSocketAuthMessage(alpacaCredentials)));
|
||||
});
|
||||
|
||||
ws.on("message", (raw) => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
import { decode, encode } from "@msgpack/msgpack";
|
||||
import {
|
||||
buildAlpacaAuthHeaders,
|
||||
buildAlpacaWebSocketAuthMessage,
|
||||
type AlpacaCredentials
|
||||
} from "@islandflow/config";
|
||||
import { createLogger } from "@islandflow/observability";
|
||||
import type { OptionIngestAdapter, OptionIngestHandlers } from "./types";
|
||||
import WebSocket from "ws";
|
||||
|
|
@ -6,7 +11,7 @@ import WebSocket from "ws";
|
|||
type AlpacaFeed = "indicative" | "opra";
|
||||
|
||||
type AlpacaOptionsAdapterConfig = {
|
||||
apiKey: string;
|
||||
credentials: AlpacaCredentials;
|
||||
restUrl: string;
|
||||
wsBaseUrl: string;
|
||||
feed: AlpacaFeed;
|
||||
|
|
@ -147,18 +152,12 @@ const normalizeUnderlyings = (value: string[]): string[] => {
|
|||
return result;
|
||||
};
|
||||
|
||||
const buildHeaders = (config: AlpacaOptionsAdapterConfig): Record<string, string> => {
|
||||
return {
|
||||
Authorization: `Bearer ${config.apiKey}`
|
||||
};
|
||||
};
|
||||
|
||||
const fetchJson = async <T>(
|
||||
url: URL,
|
||||
config: AlpacaOptionsAdapterConfig
|
||||
): Promise<T> => {
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: buildHeaders(config)
|
||||
headers: buildAlpacaAuthHeaders(config.credentials)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
|
|
@ -398,8 +397,8 @@ export const createAlpacaOptionsAdapter = (
|
|||
return {
|
||||
name: "alpaca",
|
||||
start: async (handlers: OptionIngestHandlers) => {
|
||||
if (!config.apiKey) {
|
||||
throw new Error("Alpaca adapter requires ALPACA_API_KEY.");
|
||||
if (!config.credentials.keyId) {
|
||||
throw new Error("Alpaca adapter requires Alpaca credentials.");
|
||||
}
|
||||
|
||||
const underlyings = normalizeUnderlyings(config.underlyings);
|
||||
|
|
@ -485,15 +484,22 @@ export const createAlpacaOptionsAdapter = (
|
|||
const wsUrl = `${wsBase}/${config.feed}`;
|
||||
const ws = new WebSocket(wsUrl, {
|
||||
headers: {
|
||||
...buildHeaders(config),
|
||||
...buildAlpacaAuthHeaders(config.credentials),
|
||||
"Content-Type": "application/msgpack"
|
||||
}
|
||||
});
|
||||
|
||||
let seq = 0;
|
||||
let stopped = false;
|
||||
let subscribed = false;
|
||||
|
||||
const subscribe = () => {
|
||||
if (subscribed) {
|
||||
return;
|
||||
}
|
||||
|
||||
subscribed = true;
|
||||
|
||||
ws.on("open", () => {
|
||||
const subscribe: Record<string, unknown> = {
|
||||
action: "subscribe",
|
||||
trades: selectedSymbols
|
||||
|
|
@ -504,6 +510,10 @@ export const createAlpacaOptionsAdapter = (
|
|||
}
|
||||
|
||||
ws.send(encode(subscribe));
|
||||
};
|
||||
|
||||
ws.on("open", () => {
|
||||
ws.send(encode(buildAlpacaWebSocketAuthMessage(config.credentials)));
|
||||
});
|
||||
|
||||
ws.on("message", (data) => {
|
||||
|
|
@ -583,7 +593,13 @@ export const createAlpacaOptionsAdapter = (
|
|||
|
||||
if (type === "error") {
|
||||
logger.error("alpaca stream error", { message });
|
||||
} else if (type === "success" || type === "subscription") {
|
||||
} else if (type === "success") {
|
||||
const status = (message as { msg?: string }).msg ?? "";
|
||||
if (status === "authenticated") {
|
||||
subscribe();
|
||||
}
|
||||
logger.info("alpaca stream status", { message });
|
||||
} else if (type === "subscription") {
|
||||
logger.info("alpaca stream status", { message });
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { readEnv } from "@islandflow/config";
|
||||
import { hasAlpacaCredentials, readEnv, resolveAlpacaCredentials } from "@islandflow/config";
|
||||
import { createLogger } from "@islandflow/observability";
|
||||
import {
|
||||
SUBJECT_OPTION_NBBO,
|
||||
|
|
@ -55,6 +55,10 @@ const envSchema = z.object({
|
|||
CLICKHOUSE_DATABASE: z.string().default("default"),
|
||||
OPTIONS_INGEST_ADAPTER: z.string().min(1).default("synthetic"),
|
||||
ALPACA_API_KEY: z.string().default(""),
|
||||
ALPACA_API_KEY_ID: z.string().default(""),
|
||||
ALPACA_KEY_ID: z.string().default(""),
|
||||
ALPACA_API_SECRET_KEY: z.string().default(""),
|
||||
ALPACA_SECRET_KEY: z.string().default(""),
|
||||
ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"),
|
||||
ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets/v1beta1"),
|
||||
ALPACA_FEED: z.enum(["indicative", "opra"]).default("indicative"),
|
||||
|
|
@ -120,6 +124,7 @@ const envSchema = z.object({
|
|||
});
|
||||
|
||||
const env = readEnv(envSchema);
|
||||
const alpacaCredentials = resolveAlpacaCredentials(env);
|
||||
const syntheticModes = resolveSyntheticMarketModes({
|
||||
syntheticMarketMode: env.SYNTHETIC_MARKET_MODE,
|
||||
syntheticOptionsMode: env.SYNTHETIC_OPTIONS_MODE
|
||||
|
|
@ -277,15 +282,17 @@ const selectAdapter = (
|
|||
}
|
||||
|
||||
if (name === "alpaca") {
|
||||
if (!env.ALPACA_API_KEY) {
|
||||
logger.warn("alpaca credentials missing; set ALPACA_API_KEY");
|
||||
throw new Error("ALPACA_API_KEY is required for the alpaca adapter.");
|
||||
if (!hasAlpacaCredentials(alpacaCredentials)) {
|
||||
logger.warn("alpaca credentials missing; set ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY");
|
||||
throw new Error(
|
||||
"Alpaca adapter requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or legacy ALPACA_API_KEY)."
|
||||
);
|
||||
}
|
||||
|
||||
const underlyings = env.ALPACA_UNDERLYINGS.split(",").map((symbol) => symbol.trim());
|
||||
|
||||
return createAlpacaOptionsAdapter({
|
||||
apiKey: env.ALPACA_API_KEY,
|
||||
credentials: alpacaCredentials,
|
||||
restUrl: env.ALPACA_REST_URL,
|
||||
wsBaseUrl: env.ALPACA_WS_BASE_URL,
|
||||
feed: env.ALPACA_FEED,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue