fix alpaca news auth and native worker wiring
This commit is contained in:
parent
e9739f5dc9
commit
7d25608b35
21 changed files with 285 additions and 80 deletions
|
|
@ -15,6 +15,7 @@
|
|||
{"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-laq","title":"fix native alpaca news deploy and auth","description":"Why this issue exists and what needs to be done:\\n\\nNative Islandflow rollout is incomplete because services/ingest-news is not healthy on the VPS. The checked-in native user units and helper scripts do not fully include ingest-news, and the current service uses bearer-style auth that returns 401 against Alpaca news endpoints.\\n\\nThis task should verify the current Alpaca news auth requirements against official docs, update the repo code and native deployment assets as needed, install and enable the missing VPS unit, verify news events flow end-to-end, and document the work.","status":"in_progress","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:47:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:47:12Z","started_at":"2026-05-19T23:47:12Z","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-fmg","title":"Fix native deploy SSH path and verification cwd assumptions","description":"Native deploys over SSH assumed bun was already on PATH and that remote verification would run from the repository root. On the live VPS, non-login SSH shells omitted /home/delta/.bun/bin and remote native verification could not find deployment/native/check-native-infra.sh because it ran from the home directory. Update the deploy helper to prepend /Users/kell/.bun/bin when present and cd into the repo before native verification checks run.","status":"closed","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:38:32Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:40:33Z","closed_at":"2026-05-19T23:40:33Z","close_reason":"Updated native SSH deploy flow to prepend Bun's home install path when present and run native verification from the repo root before health scripts.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-wf5","title":"Harden native options provider configuration after synthetic recovery","description":"Native production recovery restored OPTIONS_INGEST_ADAPTER=synthetic because the current Alpaca setup fails authentication and crash-loops ingest-options. Follow up by deciding whether production options should remain synthetic or move to a supported live provider auth path, then add a deploy-time smoke test or config validation that catches provider auth failures before native cutover.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:27:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:51Z","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
{"_type":"issue","id":"islandflow-m83","title":"Restore options ingestion and print generation on native deployment","description":"After moving the production/VPS deployment from Docker-managed services to the native runtime, the options feed appears behind and fresh option prints are not reaching the UI. Investigate the native deployment path on the server, identify the ingestion or compute breakage, apply the required code and/or host configuration changes, validate that fresh option prints resume, and document any follow-up operational work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:20:01Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:52Z","started_at":"2026-05-19T23:20:10Z","closed_at":"2026-05-19T23:27:52Z","close_reason":"Restored native options ingest by switching the VPS back to the last known-good synthetic adapter, verified fresh option prints and compute output, and documented the native env precedence gotcha.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,10 @@ REDIS_URL=redis://127.0.0.1:6379
|
|||
# Options ingest
|
||||
OPTIONS_INGEST_ADAPTER=synthetic
|
||||
ALPACA_API_KEY=
|
||||
ALPACA_API_KEY_ID=
|
||||
ALPACA_KEY_ID=
|
||||
ALPACA_API_SECRET_KEY=
|
||||
ALPACA_SECRET_KEY=
|
||||
ALPACA_REST_URL=https://data.alpaca.markets
|
||||
ALPACA_WS_BASE_URL=wss://stream.data.alpaca.markets/v1beta1
|
||||
ALPACA_FEED=indicative
|
||||
|
|
|
|||
|
|
@ -255,7 +255,11 @@ All runtime configuration comes from `.env`.
|
|||
|
||||
| Variable | Default | What it controls |
|
||||
| --- | --- | --- |
|
||||
| `ALPACA_API_KEY` | empty | Single-token Alpaca API auth for options, equities, and news adapters. |
|
||||
| `ALPACA_API_KEY` | empty | Legacy single-token fallback kept for older Alpaca setups. Prefer explicit key ID + secret vars for current Alpaca auth. |
|
||||
| `ALPACA_API_KEY_ID` | empty | Preferred Alpaca key ID used for market-data REST and websocket auth. |
|
||||
| `ALPACA_KEY_ID` | empty | Alternate name accepted for the Alpaca key ID. |
|
||||
| `ALPACA_API_SECRET_KEY` | empty | Preferred Alpaca secret key paired with `ALPACA_API_KEY_ID`. |
|
||||
| `ALPACA_SECRET_KEY` | empty | Alternate name accepted for the Alpaca secret key. |
|
||||
| `ALPACA_REST_URL` | `https://data.alpaca.markets` | Alpaca REST base URL. |
|
||||
| `ALPACA_WS_BASE_URL` | `wss://stream.data.alpaca.markets/v1beta1` for options, `wss://stream.data.alpaca.markets` for equities/news | Alpaca websocket base URL. |
|
||||
| `ALPACA_FEED` | `indicative` | Options feed tier: `indicative` or `opra`. |
|
||||
|
|
|
|||
|
|
@ -27,6 +27,10 @@ NEXT_PUBLIC_NBBO_MAX_AGE_MS=1000
|
|||
# Options ingest
|
||||
OPTIONS_INGEST_ADAPTER=synthetic
|
||||
ALPACA_API_KEY=
|
||||
ALPACA_API_KEY_ID=
|
||||
ALPACA_KEY_ID=
|
||||
ALPACA_API_SECRET_KEY=
|
||||
ALPACA_SECRET_KEY=
|
||||
ALPACA_REST_URL=https://data.alpaca.markets
|
||||
ALPACA_WS_BASE_URL=wss://stream.data.alpaca.markets/v1beta1
|
||||
ALPACA_FEED=indicative
|
||||
|
|
|
|||
|
|
@ -161,8 +161,10 @@ Set the adapter values and credentials in `.env`:
|
|||
|
||||
- `OPTIONS_INGEST_ADAPTER=alpaca`
|
||||
- `EQUITIES_INGEST_ADAPTER=alpaca`
|
||||
- `ALPACA_KEY_ID=...`
|
||||
- `ALPACA_SECRET_KEY=...`
|
||||
- `ALPACA_API_KEY_ID=...`
|
||||
- `ALPACA_API_SECRET_KEY=...`
|
||||
|
||||
The older single-variable `ALPACA_API_KEY` fallback is still accepted for legacy setups, but Alpaca's current market-data auth expects a key ID plus secret key pair.
|
||||
|
||||
### Databento mode
|
||||
|
||||
|
|
@ -284,7 +286,7 @@ Scoped Docker deploys now build only the selected image set and then restart onl
|
|||
- `--web-only`: `docker compose build web`, then `docker compose up -d web`
|
||||
- `--api-only`: `docker compose build api`, then `docker compose up -d api`
|
||||
- `--services-only`: builds and restarts `api`, `compute`, `candles`, `ingest-options`, and `ingest-equities`
|
||||
- `--workers-only`: builds and restarts `compute`, `candles`, `ingest-options`, and `ingest-equities` without touching `web` or `api`
|
||||
- `--workers-only`: builds and restarts `compute`, `candles`, `ingest-options`, `ingest-equities`, and `ingest-news` without touching `web` or `api`
|
||||
- `--fast`: when no explicit scope flag is given, treats the deploy as `--services-only` and skips the public API route suite for quicker completion. It still runs remote service health checks.
|
||||
|
||||
Use `--no-build` only when the image is already correct and you need Compose to recreate or restart containers, such as after changing server-side environment values that do not affect a Next.js build-time variable. Do not use `--no-build` for dependency changes, application source changes, or `NEXT_PUBLIC_*` changes.
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ Checked-in unit files live under:
|
|||
- `deployment/native/systemd/user/islandflow-candles.service`
|
||||
- `deployment/native/systemd/user/islandflow-ingest-options.service`
|
||||
- `deployment/native/systemd/user/islandflow-ingest-equities.service`
|
||||
- `deployment/native/systemd/user/islandflow-ingest-news.service`
|
||||
|
||||
These are written for the current VPS layout:
|
||||
|
||||
|
|
@ -175,6 +176,7 @@ Default unit names used by `scripts/deploy.ts`:
|
|||
- `islandflow-candles`
|
||||
- `islandflow-ingest-options`
|
||||
- `islandflow-ingest-equities`
|
||||
- `islandflow-ingest-news`
|
||||
|
||||
Override them from your local shell before running `./deploy` if the server uses different names:
|
||||
|
||||
|
|
@ -191,6 +193,7 @@ Available overrides:
|
|||
- `DEPLOY_NATIVE_CANDLES_UNIT`
|
||||
- `DEPLOY_NATIVE_INGEST_OPTIONS_UNIT`
|
||||
- `DEPLOY_NATIVE_INGEST_EQUITIES_UNIT`
|
||||
- `DEPLOY_NATIVE_INGEST_NEWS_UNIT`
|
||||
|
||||
## systemctl invocation
|
||||
|
||||
|
|
@ -220,7 +223,7 @@ Scope behavior:
|
|||
- `--web-only`: rebuild/restart only the web unit
|
||||
- `--api-only`: restart only the API unit
|
||||
- `--services-only`: restart API + worker units without touching the web unit
|
||||
- `--workers-only`: restart only `compute`, `candles`, `ingest-options`, and `ingest-equities`
|
||||
- `--workers-only`: restart only `compute`, `candles`, `ingest-options`, `ingest-equities`, and `ingest-news`
|
||||
- `--fast`: when no explicit scope flag is provided, native deploys now default to `--workers-only`
|
||||
- `--no-build`: skip `bun install --frozen-lockfile` and skip the web build step
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ units=()
|
|||
|
||||
case "$scope" in
|
||||
full)
|
||||
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
web)
|
||||
units=(islandflow-web.service)
|
||||
|
|
@ -16,10 +16,10 @@ case "$scope" in
|
|||
units=(islandflow-api.service)
|
||||
;;
|
||||
services)
|
||||
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
workers)
|
||||
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
*)
|
||||
echo "Unknown scope: $scope" >&2
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ esac
|
|||
echo "Stopping Docker-owned Islandflow app services before native ownership starts."
|
||||
(
|
||||
cd "$repo_root/deployment/docker"
|
||||
docker compose stop web api compute candles ingest-options ingest-equities
|
||||
docker compose stop web api compute candles ingest-options ingest-equities ingest-news
|
||||
)
|
||||
|
||||
if [[ "$scope" == "full" || "$scope" == "services" || "$scope" == "api" || "$scope" == "web" ]]; then
|
||||
|
|
@ -24,9 +24,9 @@ if [[ "$scope" == "full" || "$scope" == "services" || "$scope" == "api" || "$sco
|
|||
fi
|
||||
|
||||
systemctl --user restart $(case "$scope" in
|
||||
full) echo islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;;
|
||||
services) echo islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;;
|
||||
workers) echo islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;;
|
||||
full) echo islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service ;;
|
||||
services) echo islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service ;;
|
||||
workers) echo islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service ;;
|
||||
api) echo islandflow-api.service ;;
|
||||
web) echo islandflow-web.service ;;
|
||||
esac)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ set -euo pipefail
|
|||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
|
||||
echo "Stopping native app services."
|
||||
systemctl --user stop islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service || true
|
||||
systemctl --user stop islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service || true
|
||||
|
||||
echo "Stopping native infra before Docker reopens durable data."
|
||||
if [[ "${EUID}" -eq 0 ]]; then
|
||||
|
|
@ -19,7 +19,7 @@ echo "Switching NPM Islandflow upstreams back to Docker service names."
|
|||
echo "Restarting Docker Islandflow runtime."
|
||||
(
|
||||
cd "$repo_root/deployment/docker"
|
||||
docker compose up -d web api compute candles ingest-options ingest-equities
|
||||
docker compose up -d web api compute candles ingest-options ingest-equities ingest-news
|
||||
)
|
||||
|
||||
curl -I -fksS "${DEPLOY_PUBLIC_APP_URL:-https://flow.deltaisland.io}" >/dev/null
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ case "$scope" in
|
|||
none)
|
||||
;;
|
||||
full)
|
||||
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
web)
|
||||
units=(islandflow-web.service)
|
||||
|
|
@ -20,10 +20,10 @@ case "$scope" in
|
|||
units=(islandflow-api.service)
|
||||
;;
|
||||
services)
|
||||
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
workers)
|
||||
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
*)
|
||||
echo "Unknown scope: $scope" >&2
|
||||
|
|
@ -46,4 +46,4 @@ if [[ ${#units[@]} -gt 0 ]]; then
|
|||
echo "Enabled scope: $scope"
|
||||
else
|
||||
echo "No units enabled yet. Pass a scope such as workers when you are ready."
|
||||
fi
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ fi
|
|||
|
||||
case "$scope" in
|
||||
full)
|
||||
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
web)
|
||||
units=(islandflow-web.service)
|
||||
|
|
@ -39,10 +39,10 @@ case "$scope" in
|
|||
units=(islandflow-api.service)
|
||||
;;
|
||||
services)
|
||||
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
workers)
|
||||
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service)
|
||||
;;
|
||||
*)
|
||||
echo "Unknown scope: $scope" >&2
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
[Unit]
|
||||
Description=Islandflow ingest-news
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/home/delta/islandflow
|
||||
EnvironmentFile=/home/delta/islandflow/.env
|
||||
ExecStart=/home/delta/.bun/bin/bun services/ingest-news/src/index.ts
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
KillSignal=SIGINT
|
||||
TimeoutStopSec=20
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
76
packages/config/src/alpaca.ts
Normal file
76
packages/config/src/alpaca.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
export type AlpacaCredentials = {
|
||||
keyId: string;
|
||||
secret: string;
|
||||
legacyToken: string;
|
||||
usesLegacyBearer: boolean;
|
||||
};
|
||||
|
||||
type AlpacaCredentialEnv = {
|
||||
ALPACA_API_KEY?: string;
|
||||
ALPACA_API_KEY_ID?: string;
|
||||
ALPACA_KEY_ID?: string;
|
||||
ALPACA_API_SECRET_KEY?: string;
|
||||
ALPACA_SECRET_KEY?: string;
|
||||
};
|
||||
|
||||
const normalize = (value: string | undefined): string => value?.trim() ?? "";
|
||||
|
||||
export const resolveAlpacaCredentials = (
|
||||
env: AlpacaCredentialEnv
|
||||
): AlpacaCredentials => {
|
||||
const legacyToken = normalize(env.ALPACA_API_KEY);
|
||||
const explicitKeyId =
|
||||
normalize(env.ALPACA_API_KEY_ID) || normalize(env.ALPACA_KEY_ID);
|
||||
const secret =
|
||||
normalize(env.ALPACA_API_SECRET_KEY) || normalize(env.ALPACA_SECRET_KEY);
|
||||
const keyId = explicitKeyId || legacyToken;
|
||||
const usesLegacyBearer = !explicitKeyId && !secret && legacyToken.length > 0;
|
||||
|
||||
return {
|
||||
keyId,
|
||||
secret,
|
||||
legacyToken,
|
||||
usesLegacyBearer
|
||||
};
|
||||
};
|
||||
|
||||
export const hasAlpacaCredentials = (credentials: AlpacaCredentials): boolean => {
|
||||
if (credentials.usesLegacyBearer) {
|
||||
return credentials.legacyToken.length > 0;
|
||||
}
|
||||
|
||||
return credentials.keyId.length > 0 && credentials.secret.length > 0;
|
||||
};
|
||||
|
||||
export const buildAlpacaAuthHeaders = (
|
||||
credentials: AlpacaCredentials
|
||||
): Record<string, string> => {
|
||||
if (credentials.usesLegacyBearer) {
|
||||
return {
|
||||
Authorization: `Bearer ${credentials.legacyToken}`
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
"APCA-API-KEY-ID": credentials.keyId,
|
||||
"APCA-API-SECRET-KEY": credentials.secret
|
||||
};
|
||||
};
|
||||
|
||||
export const buildAlpacaWebSocketAuthMessage = (
|
||||
credentials: AlpacaCredentials
|
||||
): { action: "auth"; key: string; secret: string } => {
|
||||
if (credentials.usesLegacyBearer) {
|
||||
return {
|
||||
action: "auth",
|
||||
key: credentials.legacyToken,
|
||||
secret: ""
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
action: "auth",
|
||||
key: credentials.keyId,
|
||||
secret: credentials.secret
|
||||
};
|
||||
};
|
||||
|
|
@ -1 +1,2 @@
|
|||
export * from "./env";
|
||||
export * from "./alpaca";
|
||||
|
|
|
|||
65
packages/config/tests/alpaca.test.ts
Normal file
65
packages/config/tests/alpaca.test.ts
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
buildAlpacaAuthHeaders,
|
||||
buildAlpacaWebSocketAuthMessage,
|
||||
hasAlpacaCredentials,
|
||||
resolveAlpacaCredentials
|
||||
} from "../src/alpaca";
|
||||
|
||||
describe("resolveAlpacaCredentials", () => {
|
||||
it("prefers explicit key-id and secret vars", () => {
|
||||
const credentials = resolveAlpacaCredentials({
|
||||
ALPACA_API_KEY: "legacy-token",
|
||||
ALPACA_API_KEY_ID: "key-id",
|
||||
ALPACA_API_SECRET_KEY: "secret"
|
||||
});
|
||||
|
||||
expect(credentials).toEqual({
|
||||
keyId: "key-id",
|
||||
secret: "secret",
|
||||
legacyToken: "legacy-token",
|
||||
usesLegacyBearer: false
|
||||
});
|
||||
expect(hasAlpacaCredentials(credentials)).toBe(true);
|
||||
expect(buildAlpacaAuthHeaders(credentials)).toEqual({
|
||||
"APCA-API-KEY-ID": "key-id",
|
||||
"APCA-API-SECRET-KEY": "secret"
|
||||
});
|
||||
expect(buildAlpacaWebSocketAuthMessage(credentials)).toEqual({
|
||||
action: "auth",
|
||||
key: "key-id",
|
||||
secret: "secret"
|
||||
});
|
||||
});
|
||||
|
||||
it("supports the older bearer-token fallback when no secret exists", () => {
|
||||
const credentials = resolveAlpacaCredentials({
|
||||
ALPACA_API_KEY: "legacy-token"
|
||||
});
|
||||
|
||||
expect(credentials.usesLegacyBearer).toBe(true);
|
||||
expect(hasAlpacaCredentials(credentials)).toBe(true);
|
||||
expect(buildAlpacaAuthHeaders(credentials)).toEqual({
|
||||
Authorization: "Bearer legacy-token"
|
||||
});
|
||||
expect(buildAlpacaWebSocketAuthMessage(credentials)).toEqual({
|
||||
action: "auth",
|
||||
key: "legacy-token",
|
||||
secret: ""
|
||||
});
|
||||
});
|
||||
|
||||
it("supports alternate secret env names", () => {
|
||||
const credentials = resolveAlpacaCredentials({
|
||||
ALPACA_KEY_ID: "short-key",
|
||||
ALPACA_SECRET_KEY: "short-secret"
|
||||
});
|
||||
|
||||
expect(credentials).toEqual({
|
||||
keyId: "short-key",
|
||||
secret: "short-secret",
|
||||
legacyToken: "",
|
||||
usesLegacyBearer: false
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -81,7 +81,8 @@ const DOCKER_WORKER_SERVICES = [
|
|||
"compute",
|
||||
"candles",
|
||||
"ingest-options",
|
||||
"ingest-equities"
|
||||
"ingest-equities",
|
||||
"ingest-news"
|
||||
] as const;
|
||||
|
||||
const scriptPath = fileURLToPath(import.meta.url);
|
||||
|
|
@ -559,7 +560,8 @@ function nativeUnitsForScope(scope: DeployScope): string[] {
|
|||
NATIVE_UNITS.compute,
|
||||
NATIVE_UNITS.candles,
|
||||
NATIVE_UNITS.ingestOptions,
|
||||
NATIVE_UNITS.ingestEquities
|
||||
NATIVE_UNITS.ingestEquities,
|
||||
NATIVE_UNITS.ingestNews
|
||||
];
|
||||
default:
|
||||
return [
|
||||
|
|
|
|||
|
|
@ -1,3 +1,8 @@
|
|||
import {
|
||||
buildAlpacaAuthHeaders,
|
||||
buildAlpacaWebSocketAuthMessage,
|
||||
type AlpacaCredentials
|
||||
} from "@islandflow/config";
|
||||
import { createLogger } from "@islandflow/observability";
|
||||
import type { EquityPrint, EquityQuote } from "@islandflow/types";
|
||||
import type { EquityIngestAdapter, EquityIngestHandlers } from "./types";
|
||||
|
|
@ -6,7 +11,7 @@ import WebSocket from "ws";
|
|||
export type AlpacaEquitiesFeed = "iex" | "sip";
|
||||
|
||||
export type AlpacaEquitiesAdapterConfig = {
|
||||
apiKey: string;
|
||||
credentials: AlpacaCredentials;
|
||||
restUrl: string;
|
||||
wsBaseUrl: string;
|
||||
feed: AlpacaEquitiesFeed;
|
||||
|
|
@ -62,12 +67,6 @@ const normalizeSymbols = (symbols: string[]): string[] => {
|
|||
return result;
|
||||
};
|
||||
|
||||
const buildHeaders = (config: AlpacaEquitiesAdapterConfig): Record<string, string> => {
|
||||
return {
|
||||
Authorization: `Bearer ${config.apiKey}`
|
||||
};
|
||||
};
|
||||
|
||||
const parseTimestamp = (value: string): number => {
|
||||
const parsed = Date.parse(value);
|
||||
if (Number.isFinite(parsed)) {
|
||||
|
|
@ -157,7 +156,7 @@ const fetchExchangeMeta = async (config: AlpacaEquitiesAdapterConfig): Promise<M
|
|||
|
||||
try {
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: buildHeaders(config)
|
||||
headers: buildAlpacaAuthHeaders(config.credentials)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
|
|
@ -184,8 +183,8 @@ export const createAlpacaEquitiesAdapter = (
|
|||
return {
|
||||
name: "alpaca",
|
||||
start: async (handlers: EquityIngestHandlers) => {
|
||||
if (!config.apiKey) {
|
||||
throw new Error("Alpaca equities adapter requires ALPACA_API_KEY.");
|
||||
if (!config.credentials.keyId) {
|
||||
throw new Error("Alpaca equities adapter requires Alpaca credentials.");
|
||||
}
|
||||
|
||||
const symbols = normalizeSymbols(config.symbols);
|
||||
|
|
@ -196,7 +195,7 @@ export const createAlpacaEquitiesAdapter = (
|
|||
const exchangeNameMap = await fetchExchangeMeta(config);
|
||||
const wsUrl = buildWsUrl(config.wsBaseUrl, config.feed);
|
||||
const ws = new WebSocket(wsUrl, {
|
||||
headers: buildHeaders(config)
|
||||
headers: buildAlpacaAuthHeaders(config.credentials)
|
||||
});
|
||||
|
||||
let seq = 0;
|
||||
|
|
@ -204,13 +203,7 @@ export const createAlpacaEquitiesAdapter = (
|
|||
let authenticated = false;
|
||||
|
||||
ws.on("open", () => {
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
action: "auth",
|
||||
key: config.apiKey,
|
||||
secret: ""
|
||||
})
|
||||
);
|
||||
ws.send(JSON.stringify(buildAlpacaWebSocketAuthMessage(config.credentials)));
|
||||
});
|
||||
|
||||
const subscribe = () => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { readEnv } from "@islandflow/config";
|
||||
import { hasAlpacaCredentials, readEnv, resolveAlpacaCredentials } from "@islandflow/config";
|
||||
import { createLogger } from "@islandflow/observability";
|
||||
import {
|
||||
SUBJECT_EQUITY_PRINTS,
|
||||
|
|
@ -47,6 +47,10 @@ const envSchema = z.object({
|
|||
|
||||
// Alpaca (equities)
|
||||
ALPACA_API_KEY: z.string().default(""),
|
||||
ALPACA_API_KEY_ID: z.string().default(""),
|
||||
ALPACA_KEY_ID: z.string().default(""),
|
||||
ALPACA_API_SECRET_KEY: z.string().default(""),
|
||||
ALPACA_SECRET_KEY: z.string().default(""),
|
||||
ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"),
|
||||
ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"),
|
||||
ALPACA_UNDERLYINGS: z.string().default("SPY,NVDA,AAPL"),
|
||||
|
|
@ -70,6 +74,7 @@ const envSchema = z.object({
|
|||
});
|
||||
|
||||
const env = readEnv(envSchema);
|
||||
const alpacaCredentials = resolveAlpacaCredentials(env);
|
||||
const syntheticModes = resolveSyntheticMarketModes({
|
||||
syntheticMarketMode: env.SYNTHETIC_MARKET_MODE,
|
||||
syntheticEquitiesMode: env.SYNTHETIC_EQUITIES_MODE
|
||||
|
|
@ -175,13 +180,15 @@ const selectAdapter = (
|
|||
}
|
||||
|
||||
if (name === "alpaca") {
|
||||
if (!env.ALPACA_API_KEY) {
|
||||
logger.warn("alpaca credentials missing; set ALPACA_API_KEY");
|
||||
throw new Error("ALPACA_API_KEY is required for the alpaca adapter.");
|
||||
if (!hasAlpacaCredentials(alpacaCredentials)) {
|
||||
logger.warn("alpaca credentials missing; set ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY");
|
||||
throw new Error(
|
||||
"Alpaca equities adapter requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or legacy ALPACA_API_KEY)."
|
||||
);
|
||||
}
|
||||
|
||||
return createAlpacaEquitiesAdapter({
|
||||
apiKey: env.ALPACA_API_KEY,
|
||||
credentials: alpacaCredentials,
|
||||
restUrl: env.ALPACA_REST_URL,
|
||||
wsBaseUrl: env.ALPACA_WS_BASE_URL,
|
||||
feed: env.ALPACA_EQUITIES_FEED,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,10 @@
|
|||
import { readEnv } from "@islandflow/config";
|
||||
import {
|
||||
buildAlpacaAuthHeaders,
|
||||
buildAlpacaWebSocketAuthMessage,
|
||||
hasAlpacaCredentials,
|
||||
readEnv,
|
||||
resolveAlpacaCredentials
|
||||
} from "@islandflow/config";
|
||||
import { createLogger } from "@islandflow/observability";
|
||||
import {
|
||||
SUBJECT_NEWS,
|
||||
|
|
@ -18,6 +24,10 @@ const logger = createLogger({ service });
|
|||
const envSchema = z.object({
|
||||
NATS_URL: z.string().default("nats://127.0.0.1:4222"),
|
||||
ALPACA_API_KEY: z.string().default(""),
|
||||
ALPACA_API_KEY_ID: z.string().default(""),
|
||||
ALPACA_KEY_ID: z.string().default(""),
|
||||
ALPACA_API_SECRET_KEY: z.string().default(""),
|
||||
ALPACA_SECRET_KEY: z.string().default(""),
|
||||
ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"),
|
||||
ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"),
|
||||
ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(200).default(100),
|
||||
|
|
@ -25,6 +35,7 @@ const envSchema = z.object({
|
|||
});
|
||||
|
||||
const env = readEnv(envSchema);
|
||||
const alpacaCredentials = resolveAlpacaCredentials(env);
|
||||
|
||||
type AlpacaNewsItem = {
|
||||
id?: number;
|
||||
|
|
@ -43,10 +54,6 @@ type AlpacaNewsResponse = {
|
|||
news?: AlpacaNewsItem[];
|
||||
};
|
||||
|
||||
const buildHeaders = (): Record<string, string> => ({
|
||||
Authorization: `Bearer ${env.ALPACA_API_KEY}`
|
||||
});
|
||||
|
||||
const parseTimestamp = (value: string | undefined): number => {
|
||||
const parsed = value ? Date.parse(value) : Number.NaN;
|
||||
return Number.isFinite(parsed) ? parsed : Date.now();
|
||||
|
|
@ -90,7 +97,7 @@ const fetchBackfill = async (): Promise<AlpacaNewsItem[]> => {
|
|||
url.searchParams.set("limit", env.ALPACA_NEWS_BACKFILL_LIMIT.toString());
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: buildHeaders()
|
||||
headers: buildAlpacaAuthHeaders(alpacaCredentials)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
|
|
@ -115,8 +122,10 @@ const decodePayload = (data: WebSocket.RawData): unknown => {
|
|||
};
|
||||
|
||||
const run = async () => {
|
||||
if (!env.ALPACA_API_KEY) {
|
||||
throw new Error("ALPACA_API_KEY is required for ingest-news.");
|
||||
if (!hasAlpacaCredentials(alpacaCredentials)) {
|
||||
throw new Error(
|
||||
"Alpaca news requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or ALPACA_KEY_ID / ALPACA_SECRET_KEY)."
|
||||
);
|
||||
}
|
||||
|
||||
const { nc, js, jsm } = await connectJetStreamWithRetry(
|
||||
|
|
@ -146,17 +155,11 @@ const run = async () => {
|
|||
|
||||
const wsUrl = new URL(env.ALPACA_NEWS_WEBSOCKET_PATH, env.ALPACA_WS_BASE_URL).toString();
|
||||
const ws = new WebSocket(wsUrl, {
|
||||
headers: buildHeaders()
|
||||
headers: buildAlpacaAuthHeaders(alpacaCredentials)
|
||||
});
|
||||
|
||||
ws.on("open", () => {
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
action: "auth",
|
||||
key: env.ALPACA_API_KEY,
|
||||
secret: ""
|
||||
})
|
||||
);
|
||||
ws.send(JSON.stringify(buildAlpacaWebSocketAuthMessage(alpacaCredentials)));
|
||||
});
|
||||
|
||||
ws.on("message", (raw) => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
import { decode, encode } from "@msgpack/msgpack";
|
||||
import {
|
||||
buildAlpacaAuthHeaders,
|
||||
buildAlpacaWebSocketAuthMessage,
|
||||
type AlpacaCredentials
|
||||
} from "@islandflow/config";
|
||||
import { createLogger } from "@islandflow/observability";
|
||||
import type { OptionIngestAdapter, OptionIngestHandlers } from "./types";
|
||||
import WebSocket from "ws";
|
||||
|
|
@ -6,7 +11,7 @@ import WebSocket from "ws";
|
|||
type AlpacaFeed = "indicative" | "opra";
|
||||
|
||||
type AlpacaOptionsAdapterConfig = {
|
||||
apiKey: string;
|
||||
credentials: AlpacaCredentials;
|
||||
restUrl: string;
|
||||
wsBaseUrl: string;
|
||||
feed: AlpacaFeed;
|
||||
|
|
@ -147,18 +152,12 @@ const normalizeUnderlyings = (value: string[]): string[] => {
|
|||
return result;
|
||||
};
|
||||
|
||||
const buildHeaders = (config: AlpacaOptionsAdapterConfig): Record<string, string> => {
|
||||
return {
|
||||
Authorization: `Bearer ${config.apiKey}`
|
||||
};
|
||||
};
|
||||
|
||||
const fetchJson = async <T>(
|
||||
url: URL,
|
||||
config: AlpacaOptionsAdapterConfig
|
||||
): Promise<T> => {
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: buildHeaders(config)
|
||||
headers: buildAlpacaAuthHeaders(config.credentials)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
|
|
@ -398,8 +397,8 @@ export const createAlpacaOptionsAdapter = (
|
|||
return {
|
||||
name: "alpaca",
|
||||
start: async (handlers: OptionIngestHandlers) => {
|
||||
if (!config.apiKey) {
|
||||
throw new Error("Alpaca adapter requires ALPACA_API_KEY.");
|
||||
if (!config.credentials.keyId) {
|
||||
throw new Error("Alpaca adapter requires Alpaca credentials.");
|
||||
}
|
||||
|
||||
const underlyings = normalizeUnderlyings(config.underlyings);
|
||||
|
|
@ -485,15 +484,22 @@ export const createAlpacaOptionsAdapter = (
|
|||
const wsUrl = `${wsBase}/${config.feed}`;
|
||||
const ws = new WebSocket(wsUrl, {
|
||||
headers: {
|
||||
...buildHeaders(config),
|
||||
...buildAlpacaAuthHeaders(config.credentials),
|
||||
"Content-Type": "application/msgpack"
|
||||
}
|
||||
});
|
||||
|
||||
let seq = 0;
|
||||
let stopped = false;
|
||||
let subscribed = false;
|
||||
|
||||
const subscribe = () => {
|
||||
if (subscribed) {
|
||||
return;
|
||||
}
|
||||
|
||||
subscribed = true;
|
||||
|
||||
ws.on("open", () => {
|
||||
const subscribe: Record<string, unknown> = {
|
||||
action: "subscribe",
|
||||
trades: selectedSymbols
|
||||
|
|
@ -504,6 +510,10 @@ export const createAlpacaOptionsAdapter = (
|
|||
}
|
||||
|
||||
ws.send(encode(subscribe));
|
||||
};
|
||||
|
||||
ws.on("open", () => {
|
||||
ws.send(encode(buildAlpacaWebSocketAuthMessage(config.credentials)));
|
||||
});
|
||||
|
||||
ws.on("message", (data) => {
|
||||
|
|
@ -583,7 +593,13 @@ export const createAlpacaOptionsAdapter = (
|
|||
|
||||
if (type === "error") {
|
||||
logger.error("alpaca stream error", { message });
|
||||
} else if (type === "success" || type === "subscription") {
|
||||
} else if (type === "success") {
|
||||
const status = (message as { msg?: string }).msg ?? "";
|
||||
if (status === "authenticated") {
|
||||
subscribe();
|
||||
}
|
||||
logger.info("alpaca stream status", { message });
|
||||
} else if (type === "subscription") {
|
||||
logger.info("alpaca stream status", { message });
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { readEnv } from "@islandflow/config";
|
||||
import { hasAlpacaCredentials, readEnv, resolveAlpacaCredentials } from "@islandflow/config";
|
||||
import { createLogger } from "@islandflow/observability";
|
||||
import {
|
||||
SUBJECT_OPTION_NBBO,
|
||||
|
|
@ -55,6 +55,10 @@ const envSchema = z.object({
|
|||
CLICKHOUSE_DATABASE: z.string().default("default"),
|
||||
OPTIONS_INGEST_ADAPTER: z.string().min(1).default("synthetic"),
|
||||
ALPACA_API_KEY: z.string().default(""),
|
||||
ALPACA_API_KEY_ID: z.string().default(""),
|
||||
ALPACA_KEY_ID: z.string().default(""),
|
||||
ALPACA_API_SECRET_KEY: z.string().default(""),
|
||||
ALPACA_SECRET_KEY: z.string().default(""),
|
||||
ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"),
|
||||
ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets/v1beta1"),
|
||||
ALPACA_FEED: z.enum(["indicative", "opra"]).default("indicative"),
|
||||
|
|
@ -120,6 +124,7 @@ const envSchema = z.object({
|
|||
});
|
||||
|
||||
const env = readEnv(envSchema);
|
||||
const alpacaCredentials = resolveAlpacaCredentials(env);
|
||||
const syntheticModes = resolveSyntheticMarketModes({
|
||||
syntheticMarketMode: env.SYNTHETIC_MARKET_MODE,
|
||||
syntheticOptionsMode: env.SYNTHETIC_OPTIONS_MODE
|
||||
|
|
@ -277,15 +282,17 @@ const selectAdapter = (
|
|||
}
|
||||
|
||||
if (name === "alpaca") {
|
||||
if (!env.ALPACA_API_KEY) {
|
||||
logger.warn("alpaca credentials missing; set ALPACA_API_KEY");
|
||||
throw new Error("ALPACA_API_KEY is required for the alpaca adapter.");
|
||||
if (!hasAlpacaCredentials(alpacaCredentials)) {
|
||||
logger.warn("alpaca credentials missing; set ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY");
|
||||
throw new Error(
|
||||
"Alpaca adapter requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or legacy ALPACA_API_KEY)."
|
||||
);
|
||||
}
|
||||
|
||||
const underlyings = env.ALPACA_UNDERLYINGS.split(",").map((symbol) => symbol.trim());
|
||||
|
||||
return createAlpacaOptionsAdapter({
|
||||
apiKey: env.ALPACA_API_KEY,
|
||||
credentials: alpacaCredentials,
|
||||
restUrl: env.ALPACA_REST_URL,
|
||||
wsBaseUrl: env.ALPACA_WS_BASE_URL,
|
||||
feed: env.ALPACA_FEED,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue