dirtydishes/islandflow
1
0
Fork 0
Code Issues 2 Pull requests 4 Projects Releases Packages Wiki Activity Actions 1

Add Alpha Vantage event calendar provider

Browse source
This commit is contained in:
dirtydishes 2026-05-05 03:02:51 -04:00
parent 9bace6932e
commit dd32be7717
8 changed files with 237 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.beads/issues.jsonl
View file

@ -4,6 +4,7 @@
{"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-neu","title":"Add Alpha Vantage event calendar provider","description":"Add an Alpha Vantage earnings-calendar provider to services/refdata that fetches CSV, normalizes entries, writes the JSON cache consumed by compute, and documents the required env variables.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:00:31Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:02:30Z","started_at":"2026-05-05T07:00:37Z","closed_at":"2026-05-05T07:02:30Z","close_reason":"Added Alpha Vantage event-calendar provider","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-b6d","title":"Finish smart-money event-calendar enrichment","description":"Finish the smart-money event-calendar provider layer in services/refdata and connect days-to-event / expiry-after-event enrichment into compute using timestamp-available data only.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:26Z","created_by":"dirtydishes","updated_at":"2026-05-04T23:21:09Z","started_at":"2026-05-04T23:18:29Z","closed_at":"2026-05-04T23:21:09Z","close_reason":"Completed event-calendar provider and compute enrichment","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-e60","title":"Add smart-money replay evaluation harness","description":"Add replay-style live-vs-batch consistency tests plus evaluation utilities for parent-event precision/recall, calibration, abstention rate, and economic sanity checks.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:25Z","created_by":"dirtydishes","updated_at":"2026-05-05T06:08:08Z","started_at":"2026-05-05T06:07:22Z","closed_at":"2026-05-05T06:08:08Z","close_reason":"Completed smart-money replay consistency harness and evaluation utilities.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-020","title":"Rebuild synthetic smart-money scenarios","description":"Rework services/ingest-options synthetic generation around labeled parent-event templates for the six core smart-money profiles plus neutral background noise, with deterministic test/demo modes and hidden labels for tests.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:24Z","created_by":"dirtydishes","updated_at":"2026-05-05T05:29:27Z","started_at":"2026-05-05T05:25:39Z","closed_at":"2026-05-05T05:29:27Z","close_reason":"Completed Phase 5 synthetic smart-money scenario rebuild","dependency_count":0,"dependent_count":0,"comment_count":0}

12
.env.example
View file

@ -80,12 +80,16 @@ CLASSIFIER_0DTE_MIN_PREMIUM=20000
CLASSIFIER_0DTE_MIN_SIZE=400
# Smart money refdata
# Optional JSON event-calendar file used by compute for event-driven profile enrichment.
# Example row:
# [{"symbol":"AAPL","event_date":"2025-01-31T21:00:00Z","event_kind":"earnings","announced_ts":"2024-12-20T21:00:00Z"}]
SMART_MONEY_EVENT_CALENDAR_PATH=
# Optional JSON event-calendar cache used by compute for event-driven profile enrichment.
SMART_MONEY_EVENT_CALENDAR_PATH=data/event-calendar.json
# Refdata service also accepts REFDATA_EVENT_CALENDAR_PATH; if unset it falls back to SMART_MONEY_EVENT_CALENDAR_PATH.
REFDATA_EVENT_CALENDAR_PATH=
# Set to alpha_vantage to refresh the JSON cache from Alpha Vantage's EARNINGS_CALENDAR CSV endpoint.
REFDATA_EVENT_CALENDAR_PROVIDER=
ALPHA_VANTAGE_API_KEY=
ALPHA_VANTAGE_EARNINGS_HORIZON=3month
ALPHA_VANTAGE_EARNINGS_SYMBOL=
REFDATA_EVENT_CALENDAR_REFRESH_MS=86400000
# Replay service
REPLAY_ENABLED=false

5
README.md
View file

@ -241,6 +241,11 @@ Default `smart-money` policy rejects lower-information prints and keeps high-con
| `CLASSIFIER_0DTE_MIN_SIZE` | `400` | Minimum size for 0DTE classifier events. |
| `SMART_MONEY_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file used by compute to enrich event-driven smart-money profile features. |
| `REFDATA_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file for refdata service startup validation; falls back to `SMART_MONEY_EVENT_CALENDAR_PATH` when unset. |
| `REFDATA_EVENT_CALENDAR_PROVIDER` | empty | Set to `alpha_vantage` to have refdata refresh the calendar cache from Alpha Vantage. |
| `ALPHA_VANTAGE_API_KEY` | empty | Alpha Vantage key used when `REFDATA_EVENT_CALENDAR_PROVIDER=alpha_vantage`. |
| `ALPHA_VANTAGE_EARNINGS_HORIZON` | `3month` | Alpha Vantage earnings horizon: `3month`, `6month`, or `12month`. |
| `ALPHA_VANTAGE_EARNINGS_SYMBOL` | empty | Optional single-symbol Alpha Vantage earnings query; empty fetches the full scheduled earnings list. |
| `REFDATA_EVENT_CALENDAR_REFRESH_MS` | `86400000` | Refdata refresh cadence for provider-backed event-calendar cache writes. |
Event-calendar rows may use `symbol`, `underlying`, or `underlying_id`; `event_date`, `event_time`, or `event_ts`; and `announced_ts`, `available_ts`, `as_of_ts`, or `created_ts`. Compute only uses events already available at the packet timestamp, so missing or unavailable rows leave event-alignment features as neutral `null` values.

7
deployment/docker/.env.example
View file

@ -97,8 +97,13 @@ CLASSIFIER_0DTE_MIN_PREMIUM=20000
CLASSIFIER_0DTE_MIN_SIZE=400
# Smart money refdata
SMART_MONEY_EVENT_CALENDAR_PATH=
SMART_MONEY_EVENT_CALENDAR_PATH=data/event-calendar.json
REFDATA_EVENT_CALENDAR_PATH=
REFDATA_EVENT_CALENDAR_PROVIDER=
ALPHA_VANTAGE_API_KEY=
ALPHA_VANTAGE_EARNINGS_HORIZON=3month
ALPHA_VANTAGE_EARNINGS_SYMBOL=
REFDATA_EVENT_CALENDAR_REFRESH_MS=86400000
# Candles
CANDLE_INTERVALS_MS=60000,300000

7
deployment/npm/.env.example
View file

@ -4,5 +4,10 @@ NPM_EDGE_NETWORK=nextcloud_edge
NPM_SHARED_NETWORK=npm-shared
# Smart money refdata
SMART_MONEY_EVENT_CALENDAR_PATH=
SMART_MONEY_EVENT_CALENDAR_PATH=data/event-calendar.json
REFDATA_EVENT_CALENDAR_PATH=
REFDATA_EVENT_CALENDAR_PROVIDER=
ALPHA_VANTAGE_API_KEY=
ALPHA_VANTAGE_EARNINGS_HORIZON=3month
ALPHA_VANTAGE_EARNINGS_SYMBOL=
REFDATA_EVENT_CALENDAR_REFRESH_MS=86400000

115
services/refdata/src/event-calendar.ts
View file

@ -1,3 +1,5 @@
import { mkdir } from "node:fs/promises";
export type EventCalendarKind = "earnings" | "dividend" | "corporate_action" | "m_and_a" | "news" | "other";
export type EventCalendarEntry = {
@ -17,7 +19,16 @@ export type EventCalendarProvider = {
findNextEvent(underlyingId: string, asOfTs: number): EventCalendarMatch | null;
};
export type AlphaVantageEarningsCalendarOptions = {
apiKey: string;
horizon?: "3month" | "6month" | "12month";
symbol?: string;
nowTs?: number;
fetchFn?: typeof fetch;
};
const MS_PER_DAY = 86_400_000;
const ALPHA_VANTAGE_URL = "https://www.alphavantage.co/query";
const EVENT_KINDS = new Set<EventCalendarKind>([
"earnings",
@ -47,6 +58,77 @@ const asNumber = (value: unknown): number | null => {
const asString = (value: unknown): string | null => (typeof value === "string" && value.trim() ? value.trim() : null);
const parseCsvLine = (line: string): string[] => {
const values: string[] = [];
let current = "";
let quoted = false;
for (let index = 0; index < line.length; index += 1) {
const char = line[index];
const next = line[index + 1];
if (char === '"' && quoted && next === '"') {
current += '"';
index += 1;
} else if (char === '"') {
quoted = !quoted;
} else if (char === "," && !quoted) {
values.push(current);
current = "";
} else {
current += char;
}
}
values.push(current);
return values.map((value) => value.trim());
};
const parseCsv = (csv: string): Record<string, string>[] => {
const lines = csv
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean);
const [headerLine, ...dataLines] = lines;
if (!headerLine) {
return [];
}
const headers = parseCsvLine(headerLine);
return dataLines.map((line) => {
const values = parseCsvLine(line);
return Object.fromEntries(headers.map((header, index) => [header, values[index] ?? ""]));
});
};
export const parseAlphaVantageEarningsCalendar = (
csv: string,
announcedTs: number = Date.now()
): EventCalendarEntry[] => {
return parseCsv(csv).flatMap((row): EventCalendarEntry[] => {
const symbol = asString(row.symbol);
const reportDate = asString(row.reportDate);
if (!symbol || !reportDate) {
return [];
}
const eventTs = Date.parse(`${reportDate}T21:00:00Z`);
if (!Number.isFinite(eventTs)) {
return [];
}
return [
{
underlying_id: normalizeUnderlying(symbol),
event_ts: eventTs,
event_kind: "earnings",
announced_ts: Math.trunc(announcedTs),
source: "alpha_vantage",
source_event_id: `${normalizeUnderlying(symbol)}:${reportDate}:earnings`
}
];
});
};
export const parseEventCalendarEntries = (value: unknown): EventCalendarEntry[] => {
const rows = Array.isArray(value) ? value : [];
return rows.flatMap((row): EventCalendarEntry[] => {
@ -114,3 +196,36 @@ export const loadEventCalendarProviderFromFile = async (path: string): Promise<E
const text = await Bun.file(path).text();
return createStaticEventCalendarProvider(parseEventCalendarEntries(JSON.parse(text)));
};
export const fetchAlphaVantageEarningsCalendar = async (
options: AlphaVantageEarningsCalendarOptions
): Promise<EventCalendarEntry[]> => {
const horizon = options.horizon ?? "3month";
const url = new URL(ALPHA_VANTAGE_URL);
url.searchParams.set("function", "EARNINGS_CALENDAR");
url.searchParams.set("horizon", horizon);
url.searchParams.set("apikey", options.apiKey);
if (options.symbol) {
url.searchParams.set("symbol", normalizeUnderlying(options.symbol));
}
const response = await (options.fetchFn ?? fetch)(url);
const text = await response.text();
if (!response.ok) {
throw new Error(`Alpha Vantage earnings calendar request failed: ${response.status} ${text.slice(0, 160)}`);
}
if (/^(?:\s*\{|\s*Thank you for using Alpha Vantage)/i.test(text)) {
throw new Error(`Alpha Vantage returned a non-calendar response: ${text.slice(0, 200)}`);
}
return parseAlphaVantageEarningsCalendar(text, options.nowTs ?? Date.now());
};
export const writeEventCalendarEntries = async (path: string, entries: EventCalendarEntry[]): Promise<void> => {
const directory = path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : "";
if (directory) {
await mkdir(directory, { recursive: true });
}
const file = Bun.file(path);
await Bun.write(file, `${JSON.stringify(entries, null, 2)}\n`);
};

72
services/refdata/src/index.ts
View file

@ -1,5 +1,11 @@
import { createLogger } from "@islandflow/observability";
import { createEmptyEventCalendarProvider, loadEventCalendarProviderFromFile } from "./event-calendar";
import {
createEmptyEventCalendarProvider,
fetchAlphaVantageEarningsCalendar,
loadEventCalendarProviderFromFile,
writeEventCalendarEntries,
type AlphaVantageEarningsCalendarOptions
} from "./event-calendar";
const service = "refdata";
const logger = createLogger({ service });
@ -7,6 +13,70 @@ const logger = createLogger({ service });
logger.info("service starting");
const eventCalendarPath = process.env.REFDATA_EVENT_CALENDAR_PATH ?? process.env.SMART_MONEY_EVENT_CALENDAR_PATH;
const eventCalendarProvider = process.env.REFDATA_EVENT_CALENDAR_PROVIDER ?? process.env.EVENT_CALENDAR_PROVIDER;
const refreshMs = Math.max(0, Number(process.env.REFDATA_EVENT_CALENDAR_REFRESH_MS ?? 86_400_000) || 0);
const getAlphaVantageOptions = (): AlphaVantageEarningsCalendarOptions | null => {
const apiKey = process.env.ALPHA_VANTAGE_API_KEY;
if (!apiKey) {
logger.warn("alpha vantage event calendar disabled; missing ALPHA_VANTAGE_API_KEY");
return null;
}
const horizon = process.env.ALPHA_VANTAGE_EARNINGS_HORIZON;
return {
apiKey,
horizon: horizon === "6month" || horizon === "12month" ? horizon : "3month",
symbol: process.env.ALPHA_VANTAGE_EARNINGS_SYMBOL || undefined
};
};
const refreshEventCalendar = async (): Promise<void> => {
if (!eventCalendarPath) {
logger.warn("event calendar refresh disabled; missing SMART_MONEY_EVENT_CALENDAR_PATH or REFDATA_EVENT_CALENDAR_PATH");
return;
}
if (eventCalendarProvider !== "alpha_vantage") {
return;
}
const options = getAlphaVantageOptions();
if (!options) {
return;
}
const entries = await fetchAlphaVantageEarningsCalendar(options);
await writeEventCalendarEntries(eventCalendarPath, entries);
logger.info("event calendar refreshed", {
provider: "alpha_vantage",
path: eventCalendarPath,
count: entries.length,
horizon: options.horizon,
symbol: options.symbol ?? "ALL"
});
};
if (eventCalendarProvider === "alpha_vantage") {
try {
await refreshEventCalendar();
} catch (error) {
logger.warn("event calendar refresh failed", {
provider: "alpha_vantage",
error: error instanceof Error ? error.message : String(error)
});
}
if (refreshMs > 0) {
setInterval(() => {
refreshEventCalendar().catch((error) => {
logger.warn("event calendar refresh failed", {
provider: "alpha_vantage",
error: error instanceof Error ? error.message : String(error)
});
});
}, refreshMs);
}
}
if (eventCalendarPath) {
try {

26
services/refdata/tests/event-calendar.test.ts
View file

@ -1,5 +1,9 @@
import { describe, expect, it } from "bun:test";
import { createStaticEventCalendarProvider, parseEventCalendarEntries } from "../src/event-calendar";
import {
createStaticEventCalendarProvider,
parseAlphaVantageEarningsCalendar,
parseEventCalendarEntries
} from "../src/event-calendar";
describe("event calendar refdata", () => {
it("parses provider rows and filters by timestamp availability", () => {
@ -28,4 +32,24 @@ describe("event calendar refdata", () => {
expect(afterAnnouncement?.underlying_id).toBe("AAPL");
expect(afterAnnouncement?.days_to_event).toBeGreaterThan(0);
});
it("normalizes Alpha Vantage earnings CSV rows", () => {
const entries = parseAlphaVantageEarningsCalendar(
[
"symbol,name,reportDate,fiscalDateEnding,estimate,currency",
"aapl,Apple Inc,2025-01-31,2024-12-31,2.11,USD",
"MSFT,Microsoft Corp,2025-02-05,2024-12-31,3.04,USD"
].join("\n"),
Date.parse("2025-01-15T12:00:00Z")
);
expect(entries).toHaveLength(2);
expect(entries[0]).toMatchObject({
underlying_id: "AAPL",
event_kind: "earnings",
announced_ts: Date.parse("2025-01-15T12:00:00Z"),
source: "alpha_vantage",
source_event_id: "AAPL:2025-01-31:earnings"
});
});
});