dirtydishes/islandflow
1
0
Fork 0
Code Issues 2 Pull requests 4 Projects Releases Packages Wiki Activity Actions 1

fix alpaca news auth and native worker wiring

Browse source
This commit is contained in:
dirtydishes 2026-05-19 19:57:56 -04:00
parent e9739f5dc9
commit 7d25608b35
21 changed files with 285 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

29
services/ingest-equities/src/adapters/alpaca.ts
View file

@ -1,3 +1,8 @@
import {
buildAlpacaAuthHeaders,
buildAlpacaWebSocketAuthMessage,
type AlpacaCredentials
} from "@islandflow/config";
import { createLogger } from "@islandflow/observability";
import type { EquityPrint, EquityQuote } from "@islandflow/types";
import type { EquityIngestAdapter, EquityIngestHandlers } from "./types";
@ -6,7 +11,7 @@ import WebSocket from "ws";
export type AlpacaEquitiesFeed = "iex" | "sip";
export type AlpacaEquitiesAdapterConfig = {
apiKey: string;
credentials: AlpacaCredentials;
restUrl: string;
wsBaseUrl: string;
feed: AlpacaEquitiesFeed;
@ -62,12 +67,6 @@ const normalizeSymbols = (symbols: string[]): string[] => {
return result;
};
const buildHeaders = (config: AlpacaEquitiesAdapterConfig): Record<string, string> => {
return {
Authorization: `Bearer ${config.apiKey}`
};
};
const parseTimestamp = (value: string): number => {
const parsed = Date.parse(value);
if (Number.isFinite(parsed)) {
@ -157,7 +156,7 @@ const fetchExchangeMeta = async (config: AlpacaEquitiesAdapterConfig): Promise<M
try {
const response = await fetch(url.toString(), {
headers: buildHeaders(config)
headers: buildAlpacaAuthHeaders(config.credentials)
});
if (!response.ok) {
@ -184,8 +183,8 @@ export const createAlpacaEquitiesAdapter = (
return {
name: "alpaca",
start: async (handlers: EquityIngestHandlers) => {
if (!config.apiKey) {
throw new Error("Alpaca equities adapter requires ALPACA_API_KEY.");
if (!config.credentials.keyId) {
throw new Error("Alpaca equities adapter requires Alpaca credentials.");
}
const symbols = normalizeSymbols(config.symbols);
@ -196,7 +195,7 @@ export const createAlpacaEquitiesAdapter = (
const exchangeNameMap = await fetchExchangeMeta(config);
const wsUrl = buildWsUrl(config.wsBaseUrl, config.feed);
const ws = new WebSocket(wsUrl, {
headers: buildHeaders(config)
headers: buildAlpacaAuthHeaders(config.credentials)
});
let seq = 0;
@ -204,13 +203,7 @@ export const createAlpacaEquitiesAdapter = (
let authenticated = false;
ws.on("open", () => {
ws.send(
JSON.stringify({
action: "auth",
key: config.apiKey,
secret: ""
})
);
ws.send(JSON.stringify(buildAlpacaWebSocketAuthMessage(config.credentials)));
});
const subscribe = () => {

17
services/ingest-equities/src/index.ts
View file

@ -1,4 +1,4 @@
import { readEnv } from "@islandflow/config";
import { hasAlpacaCredentials, readEnv, resolveAlpacaCredentials } from "@islandflow/config";
import { createLogger } from "@islandflow/observability";
import {
SUBJECT_EQUITY_PRINTS,
@ -47,6 +47,10 @@ const envSchema = z.object({
// Alpaca (equities)
ALPACA_API_KEY: z.string().default(""),
ALPACA_API_KEY_ID: z.string().default(""),
ALPACA_KEY_ID: z.string().default(""),
ALPACA_API_SECRET_KEY: z.string().default(""),
ALPACA_SECRET_KEY: z.string().default(""),
ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"),
ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"),
ALPACA_UNDERLYINGS: z.string().default("SPY,NVDA,AAPL"),
@ -70,6 +74,7 @@ const envSchema = z.object({
});
const env = readEnv(envSchema);
const alpacaCredentials = resolveAlpacaCredentials(env);
const syntheticModes = resolveSyntheticMarketModes({
syntheticMarketMode: env.SYNTHETIC_MARKET_MODE,
syntheticEquitiesMode: env.SYNTHETIC_EQUITIES_MODE
@ -175,13 +180,15 @@ const selectAdapter = (
}
if (name === "alpaca") {
if (!env.ALPACA_API_KEY) {
logger.warn("alpaca credentials missing; set ALPACA_API_KEY");
throw new Error("ALPACA_API_KEY is required for the alpaca adapter.");
if (!hasAlpacaCredentials(alpacaCredentials)) {
logger.warn("alpaca credentials missing; set ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY");
throw new Error(
"Alpaca equities adapter requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or legacy ALPACA_API_KEY)."
);
}
return createAlpacaEquitiesAdapter({
apiKey: env.ALPACA_API_KEY,
credentials: alpacaCredentials,
restUrl: env.ALPACA_REST_URL,
wsBaseUrl: env.ALPACA_WS_BASE_URL,
feed: env.ALPACA_EQUITIES_FEED,

35
services/ingest-news/src/index.ts
View file

@ -1,4 +1,10 @@
import { readEnv } from "@islandflow/config";
import {
buildAlpacaAuthHeaders,
buildAlpacaWebSocketAuthMessage,
hasAlpacaCredentials,
readEnv,
resolveAlpacaCredentials
} from "@islandflow/config";
import { createLogger } from "@islandflow/observability";
import {
SUBJECT_NEWS,
@ -18,6 +24,10 @@ const logger = createLogger({ service });
const envSchema = z.object({
NATS_URL: z.string().default("nats://127.0.0.1:4222"),
ALPACA_API_KEY: z.string().default(""),
ALPACA_API_KEY_ID: z.string().default(""),
ALPACA_KEY_ID: z.string().default(""),
ALPACA_API_SECRET_KEY: z.string().default(""),
ALPACA_SECRET_KEY: z.string().default(""),
ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"),
ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"),
ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(200).default(100),
@ -25,6 +35,7 @@ const envSchema = z.object({
});
const env = readEnv(envSchema);
const alpacaCredentials = resolveAlpacaCredentials(env);
type AlpacaNewsItem = {
id?: number;
@ -43,10 +54,6 @@ type AlpacaNewsResponse = {
news?: AlpacaNewsItem[];
};
const buildHeaders = (): Record<string, string> => ({
Authorization: `Bearer ${env.ALPACA_API_KEY}`
});
const parseTimestamp = (value: string | undefined): number => {
const parsed = value ? Date.parse(value) : Number.NaN;
return Number.isFinite(parsed) ? parsed : Date.now();
@ -90,7 +97,7 @@ const fetchBackfill = async (): Promise<AlpacaNewsItem[]> => {
url.searchParams.set("limit", env.ALPACA_NEWS_BACKFILL_LIMIT.toString());
const response = await fetch(url.toString(), {
headers: buildHeaders()
headers: buildAlpacaAuthHeaders(alpacaCredentials)
});
if (!response.ok) {
@ -115,8 +122,10 @@ const decodePayload = (data: WebSocket.RawData): unknown => {
};
const run = async () => {
if (!env.ALPACA_API_KEY) {
throw new Error("ALPACA_API_KEY is required for ingest-news.");
if (!hasAlpacaCredentials(alpacaCredentials)) {
throw new Error(
"Alpaca news requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or ALPACA_KEY_ID / ALPACA_SECRET_KEY)."
);
}
const { nc, js, jsm } = await connectJetStreamWithRetry(
@ -146,17 +155,11 @@ const run = async () => {
const wsUrl = new URL(env.ALPACA_NEWS_WEBSOCKET_PATH, env.ALPACA_WS_BASE_URL).toString();
const ws = new WebSocket(wsUrl, {
headers: buildHeaders()
headers: buildAlpacaAuthHeaders(alpacaCredentials)
});
ws.on("open", () => {
ws.send(
JSON.stringify({
action: "auth",
key: env.ALPACA_API_KEY,
secret: ""
})
);
ws.send(JSON.stringify(buildAlpacaWebSocketAuthMessage(alpacaCredentials)));
});
ws.on("message", (raw) => {

42
services/ingest-options/src/adapters/alpaca.ts
View file

@ -1,4 +1,9 @@
import { decode, encode } from "@msgpack/msgpack";
import {
buildAlpacaAuthHeaders,
buildAlpacaWebSocketAuthMessage,
type AlpacaCredentials
} from "@islandflow/config";
import { createLogger } from "@islandflow/observability";
import type { OptionIngestAdapter, OptionIngestHandlers } from "./types";
import WebSocket from "ws";
@ -6,7 +11,7 @@ import WebSocket from "ws";
type AlpacaFeed = "indicative" | "opra";
type AlpacaOptionsAdapterConfig = {
apiKey: string;
credentials: AlpacaCredentials;
restUrl: string;
wsBaseUrl: string;
feed: AlpacaFeed;
@ -147,18 +152,12 @@ const normalizeUnderlyings = (value: string[]): string[] => {
return result;
};
const buildHeaders = (config: AlpacaOptionsAdapterConfig): Record<string, string> => {
return {
Authorization: `Bearer ${config.apiKey}`
};
};
const fetchJson = async <T>(
url: URL,
config: AlpacaOptionsAdapterConfig
): Promise<T> => {
const response = await fetch(url.toString(), {
headers: buildHeaders(config)
headers: buildAlpacaAuthHeaders(config.credentials)
});
if (!response.ok) {
@ -398,8 +397,8 @@ export const createAlpacaOptionsAdapter = (
return {
name: "alpaca",
start: async (handlers: OptionIngestHandlers) => {
if (!config.apiKey) {
throw new Error("Alpaca adapter requires ALPACA_API_KEY.");
if (!config.credentials.keyId) {
throw new Error("Alpaca adapter requires Alpaca credentials.");
}
const underlyings = normalizeUnderlyings(config.underlyings);
@ -485,15 +484,22 @@ export const createAlpacaOptionsAdapter = (
const wsUrl = `${wsBase}/${config.feed}`;
const ws = new WebSocket(wsUrl, {
headers: {
...buildHeaders(config),
...buildAlpacaAuthHeaders(config.credentials),
"Content-Type": "application/msgpack"
}
});
let seq = 0;
let stopped = false;
let subscribed = false;
const subscribe = () => {
if (subscribed) {
return;
}
subscribed = true;
ws.on("open", () => {
const subscribe: Record<string, unknown> = {
action: "subscribe",
trades: selectedSymbols
@ -504,6 +510,10 @@ export const createAlpacaOptionsAdapter = (
}
ws.send(encode(subscribe));
};
ws.on("open", () => {
ws.send(encode(buildAlpacaWebSocketAuthMessage(config.credentials)));
});
ws.on("message", (data) => {
@ -583,7 +593,13 @@ export const createAlpacaOptionsAdapter = (
if (type === "error") {
logger.error("alpaca stream error", { message });
} else if (type === "success" || type === "subscription") {
} else if (type === "success") {
const status = (message as { msg?: string }).msg ?? "";
if (status === "authenticated") {
subscribe();
}
logger.info("alpaca stream status", { message });
} else if (type === "subscription") {
logger.info("alpaca stream status", { message });
}
}

17
services/ingest-options/src/index.ts
View file

@ -1,4 +1,4 @@
import { readEnv } from "@islandflow/config";
import { hasAlpacaCredentials, readEnv, resolveAlpacaCredentials } from "@islandflow/config";
import { createLogger } from "@islandflow/observability";
import {
SUBJECT_OPTION_NBBO,
@ -55,6 +55,10 @@ const envSchema = z.object({
CLICKHOUSE_DATABASE: z.string().default("default"),
OPTIONS_INGEST_ADAPTER: z.string().min(1).default("synthetic"),
ALPACA_API_KEY: z.string().default(""),
ALPACA_API_KEY_ID: z.string().default(""),
ALPACA_KEY_ID: z.string().default(""),
ALPACA_API_SECRET_KEY: z.string().default(""),
ALPACA_SECRET_KEY: z.string().default(""),
ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"),
ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets/v1beta1"),
ALPACA_FEED: z.enum(["indicative", "opra"]).default("indicative"),
@ -120,6 +124,7 @@ const envSchema = z.object({
});
const env = readEnv(envSchema);
const alpacaCredentials = resolveAlpacaCredentials(env);
const syntheticModes = resolveSyntheticMarketModes({
syntheticMarketMode: env.SYNTHETIC_MARKET_MODE,
syntheticOptionsMode: env.SYNTHETIC_OPTIONS_MODE
@ -277,15 +282,17 @@ const selectAdapter = (
}
if (name === "alpaca") {
if (!env.ALPACA_API_KEY) {
logger.warn("alpaca credentials missing; set ALPACA_API_KEY");
throw new Error("ALPACA_API_KEY is required for the alpaca adapter.");
if (!hasAlpacaCredentials(alpacaCredentials)) {
logger.warn("alpaca credentials missing; set ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY");
throw new Error(
"Alpaca adapter requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or legacy ALPACA_API_KEY)."
);
}
const underlyings = env.ALPACA_UNDERLYINGS.split(",").map((symbol) => symbol.trim());
return createAlpacaOptionsAdapter({
apiKey: env.ALPACA_API_KEY,
credentials: alpacaCredentials,
restUrl: env.ALPACA_REST_URL,
wsBaseUrl: env.ALPACA_WS_BASE_URL,
feed: env.ALPACA_FEED,