diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 465b525..1176d76 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -4,6 +4,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-neu","title":"Add Alpha Vantage event calendar provider","description":"Add an Alpha Vantage earnings-calendar provider to services/refdata that fetches CSV, normalizes entries, writes the JSON cache consumed by compute, and documents the required env variables.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:00:31Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:02:30Z","started_at":"2026-05-05T07:00:37Z","closed_at":"2026-05-05T07:02:30Z","close_reason":"Added Alpha Vantage event-calendar provider","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-b6d","title":"Finish smart-money event-calendar enrichment","description":"Finish the smart-money event-calendar provider layer in services/refdata and connect days-to-event / expiry-after-event enrichment into compute using timestamp-available data only.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:26Z","created_by":"dirtydishes","updated_at":"2026-05-04T23:21:09Z","started_at":"2026-05-04T23:18:29Z","closed_at":"2026-05-04T23:21:09Z","close_reason":"Completed event-calendar provider and compute enrichment","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e60","title":"Add smart-money replay evaluation harness","description":"Add replay-style live-vs-batch consistency tests plus evaluation utilities for parent-event precision/recall, calibration, abstention rate, and economic sanity checks.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:25Z","created_by":"dirtydishes","updated_at":"2026-05-05T06:08:08Z","started_at":"2026-05-05T06:07:22Z","closed_at":"2026-05-05T06:08:08Z","close_reason":"Completed smart-money replay consistency harness and evaluation utilities.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-020","title":"Rebuild synthetic smart-money scenarios","description":"Rework services/ingest-options synthetic generation around labeled parent-event templates for the six core smart-money profiles plus neutral background noise, with deterministic test/demo modes and hidden labels for tests.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:24Z","created_by":"dirtydishes","updated_at":"2026-05-05T05:29:27Z","started_at":"2026-05-05T05:25:39Z","closed_at":"2026-05-05T05:29:27Z","close_reason":"Completed Phase 5 synthetic smart-money scenario rebuild","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/.env.example b/.env.example index 4ee3924..d7d7f6c 100644 --- a/.env.example +++ b/.env.example @@ -80,12 +80,16 @@ CLASSIFIER_0DTE_MIN_PREMIUM=20000 CLASSIFIER_0DTE_MIN_SIZE=400 # Smart money refdata -# Optional JSON event-calendar file used by compute for event-driven profile enrichment. -# Example row: -# [{"symbol":"AAPL","event_date":"2025-01-31T21:00:00Z","event_kind":"earnings","announced_ts":"2024-12-20T21:00:00Z"}] -SMART_MONEY_EVENT_CALENDAR_PATH= +# Optional JSON event-calendar cache used by compute for event-driven profile enrichment. +SMART_MONEY_EVENT_CALENDAR_PATH=data/event-calendar.json # Refdata service also accepts REFDATA_EVENT_CALENDAR_PATH; if unset it falls back to SMART_MONEY_EVENT_CALENDAR_PATH. REFDATA_EVENT_CALENDAR_PATH= +# Set to alpha_vantage to refresh the JSON cache from Alpha Vantage's EARNINGS_CALENDAR CSV endpoint. +REFDATA_EVENT_CALENDAR_PROVIDER= +ALPHA_VANTAGE_API_KEY= +ALPHA_VANTAGE_EARNINGS_HORIZON=3month +ALPHA_VANTAGE_EARNINGS_SYMBOL= +REFDATA_EVENT_CALENDAR_REFRESH_MS=86400000 # Replay service REPLAY_ENABLED=false diff --git a/README.md b/README.md index 3372006..986f562 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,11 @@ Default `smart-money` policy rejects lower-information prints and keeps high-con | `CLASSIFIER_0DTE_MIN_SIZE` | `400` | Minimum size for 0DTE classifier events. | | `SMART_MONEY_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file used by compute to enrich event-driven smart-money profile features. | | `REFDATA_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file for refdata service startup validation; falls back to `SMART_MONEY_EVENT_CALENDAR_PATH` when unset. | +| `REFDATA_EVENT_CALENDAR_PROVIDER` | empty | Set to `alpha_vantage` to have refdata refresh the calendar cache from Alpha Vantage. | +| `ALPHA_VANTAGE_API_KEY` | empty | Alpha Vantage key used when `REFDATA_EVENT_CALENDAR_PROVIDER=alpha_vantage`. | +| `ALPHA_VANTAGE_EARNINGS_HORIZON` | `3month` | Alpha Vantage earnings horizon: `3month`, `6month`, or `12month`. | +| `ALPHA_VANTAGE_EARNINGS_SYMBOL` | empty | Optional single-symbol Alpha Vantage earnings query; empty fetches the full scheduled earnings list. | +| `REFDATA_EVENT_CALENDAR_REFRESH_MS` | `86400000` | Refdata refresh cadence for provider-backed event-calendar cache writes. | Event-calendar rows may use `symbol`, `underlying`, or `underlying_id`; `event_date`, `event_time`, or `event_ts`; and `announced_ts`, `available_ts`, `as_of_ts`, or `created_ts`. Compute only uses events already available at the packet timestamp, so missing or unavailable rows leave event-alignment features as neutral `null` values. diff --git a/deployment/docker/.env.example b/deployment/docker/.env.example index 2dd4b0e..ea6ba5c 100644 --- a/deployment/docker/.env.example +++ b/deployment/docker/.env.example @@ -97,8 +97,13 @@ CLASSIFIER_0DTE_MIN_PREMIUM=20000 CLASSIFIER_0DTE_MIN_SIZE=400 # Smart money refdata -SMART_MONEY_EVENT_CALENDAR_PATH= +SMART_MONEY_EVENT_CALENDAR_PATH=data/event-calendar.json REFDATA_EVENT_CALENDAR_PATH= +REFDATA_EVENT_CALENDAR_PROVIDER= +ALPHA_VANTAGE_API_KEY= +ALPHA_VANTAGE_EARNINGS_HORIZON=3month +ALPHA_VANTAGE_EARNINGS_SYMBOL= +REFDATA_EVENT_CALENDAR_REFRESH_MS=86400000 # Candles CANDLE_INTERVALS_MS=60000,300000 diff --git a/deployment/npm/.env.example b/deployment/npm/.env.example index d123359..f8123eb 100644 --- a/deployment/npm/.env.example +++ b/deployment/npm/.env.example @@ -4,5 +4,10 @@ NPM_EDGE_NETWORK=nextcloud_edge NPM_SHARED_NETWORK=npm-shared # Smart money refdata -SMART_MONEY_EVENT_CALENDAR_PATH= +SMART_MONEY_EVENT_CALENDAR_PATH=data/event-calendar.json REFDATA_EVENT_CALENDAR_PATH= +REFDATA_EVENT_CALENDAR_PROVIDER= +ALPHA_VANTAGE_API_KEY= +ALPHA_VANTAGE_EARNINGS_HORIZON=3month +ALPHA_VANTAGE_EARNINGS_SYMBOL= +REFDATA_EVENT_CALENDAR_REFRESH_MS=86400000 diff --git a/services/refdata/src/event-calendar.ts b/services/refdata/src/event-calendar.ts index 3ac603c..ba32599 100644 --- a/services/refdata/src/event-calendar.ts +++ b/services/refdata/src/event-calendar.ts @@ -1,3 +1,5 @@ +import { mkdir } from "node:fs/promises"; + export type EventCalendarKind = "earnings" | "dividend" | "corporate_action" | "m_and_a" | "news" | "other"; export type EventCalendarEntry = { @@ -17,7 +19,16 @@ export type EventCalendarProvider = { findNextEvent(underlyingId: string, asOfTs: number): EventCalendarMatch | null; }; +export type AlphaVantageEarningsCalendarOptions = { + apiKey: string; + horizon?: "3month" | "6month" | "12month"; + symbol?: string; + nowTs?: number; + fetchFn?: typeof fetch; +}; + const MS_PER_DAY = 86_400_000; +const ALPHA_VANTAGE_URL = "https://www.alphavantage.co/query"; const EVENT_KINDS = new Set([ "earnings", @@ -47,6 +58,77 @@ const asNumber = (value: unknown): number | null => { const asString = (value: unknown): string | null => (typeof value === "string" && value.trim() ? value.trim() : null); +const parseCsvLine = (line: string): string[] => { + const values: string[] = []; + let current = ""; + let quoted = false; + + for (let index = 0; index < line.length; index += 1) { + const char = line[index]; + const next = line[index + 1]; + if (char === '"' && quoted && next === '"') { + current += '"'; + index += 1; + } else if (char === '"') { + quoted = !quoted; + } else if (char === "," && !quoted) { + values.push(current); + current = ""; + } else { + current += char; + } + } + + values.push(current); + return values.map((value) => value.trim()); +}; + +const parseCsv = (csv: string): Record[] => { + const lines = csv + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + const [headerLine, ...dataLines] = lines; + if (!headerLine) { + return []; + } + + const headers = parseCsvLine(headerLine); + return dataLines.map((line) => { + const values = parseCsvLine(line); + return Object.fromEntries(headers.map((header, index) => [header, values[index] ?? ""])); + }); +}; + +export const parseAlphaVantageEarningsCalendar = ( + csv: string, + announcedTs: number = Date.now() +): EventCalendarEntry[] => { + return parseCsv(csv).flatMap((row): EventCalendarEntry[] => { + const symbol = asString(row.symbol); + const reportDate = asString(row.reportDate); + if (!symbol || !reportDate) { + return []; + } + + const eventTs = Date.parse(`${reportDate}T21:00:00Z`); + if (!Number.isFinite(eventTs)) { + return []; + } + + return [ + { + underlying_id: normalizeUnderlying(symbol), + event_ts: eventTs, + event_kind: "earnings", + announced_ts: Math.trunc(announcedTs), + source: "alpha_vantage", + source_event_id: `${normalizeUnderlying(symbol)}:${reportDate}:earnings` + } + ]; + }); +}; + export const parseEventCalendarEntries = (value: unknown): EventCalendarEntry[] => { const rows = Array.isArray(value) ? value : []; return rows.flatMap((row): EventCalendarEntry[] => { @@ -114,3 +196,36 @@ export const loadEventCalendarProviderFromFile = async (path: string): Promise => { + const horizon = options.horizon ?? "3month"; + const url = new URL(ALPHA_VANTAGE_URL); + url.searchParams.set("function", "EARNINGS_CALENDAR"); + url.searchParams.set("horizon", horizon); + url.searchParams.set("apikey", options.apiKey); + if (options.symbol) { + url.searchParams.set("symbol", normalizeUnderlying(options.symbol)); + } + + const response = await (options.fetchFn ?? fetch)(url); + const text = await response.text(); + if (!response.ok) { + throw new Error(`Alpha Vantage earnings calendar request failed: ${response.status} ${text.slice(0, 160)}`); + } + if (/^(?:\s*\{|\s*Thank you for using Alpha Vantage)/i.test(text)) { + throw new Error(`Alpha Vantage returned a non-calendar response: ${text.slice(0, 200)}`); + } + + return parseAlphaVantageEarningsCalendar(text, options.nowTs ?? Date.now()); +}; + +export const writeEventCalendarEntries = async (path: string, entries: EventCalendarEntry[]): Promise => { + const directory = path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : ""; + if (directory) { + await mkdir(directory, { recursive: true }); + } + const file = Bun.file(path); + await Bun.write(file, `${JSON.stringify(entries, null, 2)}\n`); +}; diff --git a/services/refdata/src/index.ts b/services/refdata/src/index.ts index 0ab68d1..c836adb 100644 --- a/services/refdata/src/index.ts +++ b/services/refdata/src/index.ts @@ -1,5 +1,11 @@ import { createLogger } from "@islandflow/observability"; -import { createEmptyEventCalendarProvider, loadEventCalendarProviderFromFile } from "./event-calendar"; +import { + createEmptyEventCalendarProvider, + fetchAlphaVantageEarningsCalendar, + loadEventCalendarProviderFromFile, + writeEventCalendarEntries, + type AlphaVantageEarningsCalendarOptions +} from "./event-calendar"; const service = "refdata"; const logger = createLogger({ service }); @@ -7,6 +13,70 @@ const logger = createLogger({ service }); logger.info("service starting"); const eventCalendarPath = process.env.REFDATA_EVENT_CALENDAR_PATH ?? process.env.SMART_MONEY_EVENT_CALENDAR_PATH; +const eventCalendarProvider = process.env.REFDATA_EVENT_CALENDAR_PROVIDER ?? process.env.EVENT_CALENDAR_PROVIDER; +const refreshMs = Math.max(0, Number(process.env.REFDATA_EVENT_CALENDAR_REFRESH_MS ?? 86_400_000) || 0); + +const getAlphaVantageOptions = (): AlphaVantageEarningsCalendarOptions | null => { + const apiKey = process.env.ALPHA_VANTAGE_API_KEY; + if (!apiKey) { + logger.warn("alpha vantage event calendar disabled; missing ALPHA_VANTAGE_API_KEY"); + return null; + } + + const horizon = process.env.ALPHA_VANTAGE_EARNINGS_HORIZON; + return { + apiKey, + horizon: horizon === "6month" || horizon === "12month" ? horizon : "3month", + symbol: process.env.ALPHA_VANTAGE_EARNINGS_SYMBOL || undefined + }; +}; + +const refreshEventCalendar = async (): Promise => { + if (!eventCalendarPath) { + logger.warn("event calendar refresh disabled; missing SMART_MONEY_EVENT_CALENDAR_PATH or REFDATA_EVENT_CALENDAR_PATH"); + return; + } + if (eventCalendarProvider !== "alpha_vantage") { + return; + } + + const options = getAlphaVantageOptions(); + if (!options) { + return; + } + + const entries = await fetchAlphaVantageEarningsCalendar(options); + await writeEventCalendarEntries(eventCalendarPath, entries); + logger.info("event calendar refreshed", { + provider: "alpha_vantage", + path: eventCalendarPath, + count: entries.length, + horizon: options.horizon, + symbol: options.symbol ?? "ALL" + }); +}; + +if (eventCalendarProvider === "alpha_vantage") { + try { + await refreshEventCalendar(); + } catch (error) { + logger.warn("event calendar refresh failed", { + provider: "alpha_vantage", + error: error instanceof Error ? error.message : String(error) + }); + } + + if (refreshMs > 0) { + setInterval(() => { + refreshEventCalendar().catch((error) => { + logger.warn("event calendar refresh failed", { + provider: "alpha_vantage", + error: error instanceof Error ? error.message : String(error) + }); + }); + }, refreshMs); + } +} if (eventCalendarPath) { try { diff --git a/services/refdata/tests/event-calendar.test.ts b/services/refdata/tests/event-calendar.test.ts index 28978c2..04cdba1 100644 --- a/services/refdata/tests/event-calendar.test.ts +++ b/services/refdata/tests/event-calendar.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "bun:test"; -import { createStaticEventCalendarProvider, parseEventCalendarEntries } from "../src/event-calendar"; +import { + createStaticEventCalendarProvider, + parseAlphaVantageEarningsCalendar, + parseEventCalendarEntries +} from "../src/event-calendar"; describe("event calendar refdata", () => { it("parses provider rows and filters by timestamp availability", () => { @@ -28,4 +32,24 @@ describe("event calendar refdata", () => { expect(afterAnnouncement?.underlying_id).toBe("AAPL"); expect(afterAnnouncement?.days_to_event).toBeGreaterThan(0); }); + + it("normalizes Alpha Vantage earnings CSV rows", () => { + const entries = parseAlphaVantageEarningsCalendar( + [ + "symbol,name,reportDate,fiscalDateEnding,estimate,currency", + "aapl,Apple Inc,2025-01-31,2024-12-31,2.11,USD", + "MSFT,Microsoft Corp,2025-02-05,2024-12-31,3.04,USD" + ].join("\n"), + Date.parse("2025-01-15T12:00:00Z") + ); + + expect(entries).toHaveLength(2); + expect(entries[0]).toMatchObject({ + underlying_id: "AAPL", + event_kind: "earnings", + announced_ts: Date.parse("2025-01-15T12:00:00Z"), + source: "alpha_vantage", + source_event_id: "AAPL:2025-01-31:earnings" + }); + }); });