dirtydishes/islandflow
1
0
Fork 0
Code Issues 2 Pull requests 4 Projects Releases Packages Wiki Activity Actions 1

add alpaca news wire across ingest api and web

Browse source
This commit is contained in:
dirtydishes 2026-05-18 16:55:31 -04:00
parent 62aae70878
commit 906fe411c9
31 changed files with 1407 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

54
services/api/src/index.ts
View file

@ -9,6 +9,7 @@ import {
SUBJECT_EQUITY_QUOTES,
SUBJECT_INFERRED_DARK,
SUBJECT_FLOW_PACKETS,
SUBJECT_NEWS,
SUBJECT_SMART_MONEY_EVENTS,
SUBJECT_OPTION_NBBO,
SUBJECT_OPTION_SIGNAL_PRINTS,
@ -20,6 +21,7 @@ import {
STREAM_EQUITY_QUOTES,
STREAM_INFERRED_DARK,
STREAM_FLOW_PACKETS,
STREAM_NEWS,
STREAM_SMART_MONEY_EVENTS,
STREAM_OPTION_NBBO,
STREAM_OPTION_SIGNAL_PRINTS,
@ -35,6 +37,7 @@ import {
import {
createClickHouseClient,
ensureAlertsTable,
ensureNewsTable,
ensureClassifierHitsTable,
ensureEquityCandlesTable,
ensureEquityPrintJoinsTable,
@ -48,6 +51,8 @@ import {
fetchAlertsAfter,
fetchAlertsBefore,
fetchAlertContextByTraceId,
fetchNewsAfter,
fetchNewsBefore,
fetchClassifierHitsAfter,
fetchClassifierHitsBefore,
fetchSmartMoneyEventsAfter,
@ -58,6 +63,7 @@ import {
fetchFlowPacketsByMemberTraceIds,
fetchFlowPacketsBefore,
fetchRecentAlerts,
fetchRecentNews,
fetchRecentClassifierHits,
fetchRecentSmartMoneyEvents,
fetchRecentEquityPrintJoins,
@ -99,6 +105,7 @@ import {
EquityQuoteSchema,
FeedSnapshot,
InferredDarkEventSchema,
NewsStorySchema,
LiveClientMessageSchema,
LiveServerMessage,
LiveSubscription,
@ -676,7 +683,8 @@ const run = async () => {
STREAM_FLOW_PACKETS,
STREAM_SMART_MONEY_EVENTS,
STREAM_CLASSIFIER_HITS,
STREAM_ALERTS
STREAM_ALERTS,
STREAM_NEWS
],
{ logger }
);
@ -719,6 +727,7 @@ const run = async () => {
await ensureSmartMoneyEventsTable(clickhouse);
await ensureClassifierHitsTable(clickhouse);
await ensureAlertsTable(clickhouse);
await ensureNewsTable(clickhouse);
});
let redis: ReturnType<typeof createClient> | null = null;
@ -843,6 +852,11 @@ const run = async () => {
subject: SUBJECT_ALERTS,
stream: STREAM_ALERTS,
durableName: "api-alerts"
},
{
subject: SUBJECT_NEWS,
stream: STREAM_NEWS,
durableName: "api-news"
}
] as const;
@ -991,10 +1005,16 @@ const run = async () => {
consumerBindings[10].durableName
);
const newsSubscription = await subscribeWithReset(
consumerBindings[11].subject,
consumerBindings[11].stream,
consumerBindings[11].durableName
);
const fanoutLive = async (
subscription: LiveSubscription,
item: unknown,
ingestChannel: "options" | "nbbo" | "equities" | "equity-quotes" | "equity-candles" | "equity-overlay" | "equity-joins" | "flow" | "classifier-hits" | "alerts" | "inferred-dark"
ingestChannel: "options" | "nbbo" | "equities" | "equity-quotes" | "equity-candles" | "equity-overlay" | "equity-joins" | "flow" | "classifier-hits" | "alerts" | "inferred-dark" | "news"
) => {
const watermark = await liveState.ingest(ingestChannel, item);
@ -1252,6 +1272,21 @@ const run = async () => {
}
};
const pumpNews = async () => {
for await (const msg of newsSubscription.messages) {
try {
const payload = NewsStorySchema.parse(newsSubscription.decode(msg));
await fanoutLive({ channel: "news" }, payload, "news");
msg.ack();
} catch (error) {
logger.error("failed to process news story", {
error: error instanceof Error ? error.message : String(error)
});
msg.term();
}
}
};
void pumpOptions();
void pumpOptionNbbo();
void pumpEquities();
@ -1263,6 +1298,7 @@ const run = async () => {
void pumpSmartMoney();
void pumpClassifierHits();
void pumpAlerts();
void pumpNews();
const buildSyntheticStatusBody = () => {
const derived =
@ -1490,6 +1526,12 @@ const run = async () => {
return jsonResponse({ data });
}
if (req.method === "GET" && url.pathname === "/news") {
const limit = parseLimit(url.searchParams.get("limit") ?? "100");
const data = await fetchRecentNews(clickhouse, limit);
return jsonResponse({ data });
}
if (req.method === "GET" && isAlertContextPath(url.pathname)) {
try {
const traceId = parseAlertContextTraceIdPath(url.pathname);
@ -1607,6 +1649,14 @@ const run = async () => {
);
}
if (req.method === "GET" && url.pathname === "/history/news") {
const { beforeTs, beforeSeq, limit } = parseBeforeParams(url);
const data = await fetchNewsBefore(clickhouse, beforeTs, beforeSeq, limit);
return jsonResponse(
buildHistoryResponse(data, (item) => ({ ts: item.published_ts, seq: item.seq }))
);
}
if (req.method === "GET" && /^\/flow\/packets\/[^/]+$/.test(url.pathname)) {
const id = decodeURIComponent(url.pathname.slice("/flow/packets/".length));
const data = await fetchFlowPacketById(clickhouse, id);

65
services/api/src/live.ts
View file

@ -8,6 +8,7 @@ import {
fetchRecentEquityQuotes,
fetchRecentFlowPackets,
fetchRecentInferredDark,
fetchRecentNews,
fetchRecentOptionNBBO,
fetchRecentSmartMoneyEvents,
type ClickHouseClient
@ -25,6 +26,7 @@ import {
FeedSnapshot,
FlowPacketSchema,
InferredDarkEventSchema,
NewsStorySchema,
LiveChannelHealth,
LiveGenericChannel,
LiveHotChannel,
@ -40,6 +42,7 @@ import {
type EquityCandle,
type EquityPrint,
type LiveChannel,
type NewsStory,
type OptionPrint
} from "@islandflow/types";
import { createMetrics } from "@islandflow/observability";
@ -63,7 +66,8 @@ const GENERIC_LIMIT_ENV_KEYS: Record<LiveGenericChannel, string> = {
"smart-money": "LIVE_LIMIT_SMART_MONEY",
"classifier-hits": "LIVE_LIMIT_CLASSIFIER_HITS",
alerts: "LIVE_LIMIT_ALERTS",
"inferred-dark": "LIVE_LIMIT_INFERRED_DARK"
"inferred-dark": "LIVE_LIMIT_INFERRED_DARK",
news: "LIVE_LIMIT_NEWS"
};
const CHART_LIMITS = {
@ -81,7 +85,8 @@ const DEFAULT_LIVE_LIMITS: GenericLiveLimits = {
"smart-money": 300,
"classifier-hits": 300,
alerts: 300,
"inferred-dark": 300
"inferred-dark": 300,
news: 100
};
const DEFAULT_SCOPED_CACHE_MAX_KEYS = 32;
@ -196,16 +201,28 @@ export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env):
env,
"inferred-dark",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["inferred-dark"]
)
),
news: parseGenericLimit(env, "news", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.news)
};
};
const parsePositiveInt = (value: string | undefined, fallback: number): number => {
const parsed = Number(value);
if (!Number.isFinite(parsed)) {
return fallback;
const extractFreshnessTs = (channel: LiveGenericChannel, item: any): number | null => {
switch (channel) {
case "options":
case "nbbo":
case "equities":
case "equity-quotes":
return typeof item.ts === "number" ? item.ts : null;
case "flow":
case "classifier-hits":
case "alerts":
case "inferred-dark":
return typeof item.source_ts === "number" ? item.source_ts : null;
case "news":
return typeof item.published_ts === "number" ? item.published_ts : null;
default:
return null;
}
return Math.max(1, Math.floor(parsed));
};
export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): LiveStateConfig => ({
@ -217,6 +234,13 @@ export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): Li
),
redisFlushMaxItems: parsePositiveInt(env.LIVE_REDIS_FLUSH_MAX_ITEMS, DEFAULT_REDIS_FLUSH_MAX_ITEMS)
});
const parsePositiveInt = (value: string | undefined, fallback: number): number => {
const parsed = Number(value);
if (!Number.isFinite(parsed)) {
return fallback;
}
return Math.max(1, Math.floor(parsed));
};
type RedisLike = Pick<
RedisClientType,
@ -318,6 +342,14 @@ const getGenericConfig = (limits: GenericLiveLimits): {
parse: (value) => InferredDarkEventSchema.parse(value),
cursor: (item) => ({ ts: item.source_ts, seq: item.seq }),
fetchRecent: fetchRecentInferredDark
},
news: {
redisKey: "live:news",
cursorField: "news",
limit: limits.news,
parse: (value) => NewsStorySchema.parse(value),
cursor: (item) => ({ ts: item.published_ts, seq: item.seq }),
fetchRecent: fetchRecentNews
}
});
@ -371,23 +403,6 @@ const normalizeGenericItems = <T>(
return sortGenericItems(items, config.cursor).slice(0, config.limit);
};
const extractFreshnessTs = (channel: LiveGenericChannel, item: any): number | null => {
switch (channel) {
case "options":
case "nbbo":
case "equities":
case "equity-quotes":
return typeof item.ts === "number" ? item.ts : null;
case "flow":
case "classifier-hits":
case "alerts":
case "inferred-dark":
return typeof item.source_ts === "number" ? item.source_ts : null;
default:
return null;
}
};
const isWithinLiveFeedLookback = (
channel: LiveGenericChannel,
item: unknown,