Add equity candle aggregation pipeline
This commit is contained in:
parent
f889a2597b
commit
a87df21baa
13 changed files with 1188 additions and 10 deletions
6
bun.lock
6
bun.lock
|
|
@ -55,14 +55,20 @@
|
||||||
"@islandflow/observability": "workspace:*",
|
"@islandflow/observability": "workspace:*",
|
||||||
"@islandflow/storage": "workspace:*",
|
"@islandflow/storage": "workspace:*",
|
||||||
"@islandflow/types": "workspace:*",
|
"@islandflow/types": "workspace:*",
|
||||||
|
"redis": "^5.10.0",
|
||||||
"zod": "^3.23.8",
|
"zod": "^3.23.8",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"services/candles": {
|
"services/candles": {
|
||||||
"name": "@islandflow/candles",
|
"name": "@islandflow/candles",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@islandflow/bus": "workspace:*",
|
||||||
"@islandflow/config": "workspace:*",
|
"@islandflow/config": "workspace:*",
|
||||||
"@islandflow/observability": "workspace:*",
|
"@islandflow/observability": "workspace:*",
|
||||||
|
"@islandflow/storage": "workspace:*",
|
||||||
|
"@islandflow/types": "workspace:*",
|
||||||
|
"redis": "^5.10.0",
|
||||||
|
"zod": "^3.23.8",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"services/compute": {
|
"services/compute": {
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,8 @@ export const STREAM_EQUITY_PRINTS = "EQUITY_PRINTS";
|
||||||
export const SUBJECT_EQUITY_PRINTS = "equities.prints";
|
export const SUBJECT_EQUITY_PRINTS = "equities.prints";
|
||||||
export const STREAM_EQUITY_QUOTES = "EQUITY_QUOTES";
|
export const STREAM_EQUITY_QUOTES = "EQUITY_QUOTES";
|
||||||
export const SUBJECT_EQUITY_QUOTES = "equities.quotes";
|
export const SUBJECT_EQUITY_QUOTES = "equities.quotes";
|
||||||
|
export const STREAM_EQUITY_CANDLES = "EQUITY_CANDLES";
|
||||||
|
export const SUBJECT_EQUITY_CANDLES = "equities.candles";
|
||||||
export const STREAM_EQUITY_JOINS = "EQUITY_JOINS";
|
export const STREAM_EQUITY_JOINS = "EQUITY_JOINS";
|
||||||
export const SUBJECT_EQUITY_JOINS = "equities.joins";
|
export const SUBJECT_EQUITY_JOINS = "equities.joins";
|
||||||
export const STREAM_INFERRED_DARK = "INFERRED_DARK";
|
export const STREAM_INFERRED_DARK = "INFERRED_DARK";
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import { createClient, type ClickHouseClient } from "@clickhouse/client";
|
||||||
import {
|
import {
|
||||||
AlertEventSchema,
|
AlertEventSchema,
|
||||||
ClassifierHitEventSchema,
|
ClassifierHitEventSchema,
|
||||||
|
EquityCandleSchema,
|
||||||
EquityPrintSchema,
|
EquityPrintSchema,
|
||||||
EquityQuoteSchema,
|
EquityQuoteSchema,
|
||||||
EquityPrintJoinSchema,
|
EquityPrintJoinSchema,
|
||||||
|
|
@ -13,6 +14,7 @@ import {
|
||||||
import type {
|
import type {
|
||||||
AlertEvent,
|
AlertEvent,
|
||||||
ClassifierHitEvent,
|
ClassifierHitEvent,
|
||||||
|
EquityCandle,
|
||||||
EquityPrint,
|
EquityPrint,
|
||||||
EquityQuote,
|
EquityQuote,
|
||||||
EquityPrintJoin,
|
EquityPrintJoin,
|
||||||
|
|
@ -37,6 +39,11 @@ import {
|
||||||
EQUITY_QUOTES_TABLE,
|
EQUITY_QUOTES_TABLE,
|
||||||
normalizeEquityQuote
|
normalizeEquityQuote
|
||||||
} from "./equity-quotes";
|
} from "./equity-quotes";
|
||||||
|
import {
|
||||||
|
equityCandlesTableDDL,
|
||||||
|
EQUITY_CANDLES_TABLE,
|
||||||
|
normalizeEquityCandle
|
||||||
|
} from "./equity-candles";
|
||||||
import {
|
import {
|
||||||
equityPrintJoinsTableDDL,
|
equityPrintJoinsTableDDL,
|
||||||
EQUITY_PRINT_JOINS_TABLE,
|
EQUITY_PRINT_JOINS_TABLE,
|
||||||
|
|
@ -121,6 +128,14 @@ export const ensureEquityQuotesTable = async (
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const ensureEquityCandlesTable = async (
|
||||||
|
client: ClickHouseClient
|
||||||
|
): Promise<void> => {
|
||||||
|
await client.exec({
|
||||||
|
query: equityCandlesTableDDL()
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
export const ensureEquityPrintJoinsTable = async (
|
export const ensureEquityPrintJoinsTable = async (
|
||||||
client: ClickHouseClient
|
client: ClickHouseClient
|
||||||
): Promise<void> => {
|
): Promise<void> => {
|
||||||
|
|
@ -207,6 +222,18 @@ export const insertEquityQuote = async (
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const insertEquityCandle = async (
|
||||||
|
client: ClickHouseClient,
|
||||||
|
candle: EquityCandle
|
||||||
|
): Promise<void> => {
|
||||||
|
const record = normalizeEquityCandle(candle);
|
||||||
|
await client.insert({
|
||||||
|
table: EQUITY_CANDLES_TABLE,
|
||||||
|
values: [record],
|
||||||
|
format: "JSONEachRow"
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
export const insertEquityPrintJoin = async (
|
export const insertEquityPrintJoin = async (
|
||||||
client: ClickHouseClient,
|
client: ClickHouseClient,
|
||||||
join: EquityPrintJoin
|
join: EquityPrintJoin
|
||||||
|
|
@ -272,6 +299,14 @@ const clampLimit = (limit: number): number => {
|
||||||
return Math.max(1, Math.min(1000, Math.floor(limit)));
|
return Math.max(1, Math.min(1000, Math.floor(limit)));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const clampPositiveInt = (value: number, fallback = 1): number => {
|
||||||
|
if (!Number.isFinite(value)) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Math.max(1, Math.floor(value));
|
||||||
|
};
|
||||||
|
|
||||||
const clampCursor = (value: number): number => {
|
const clampCursor = (value: number): number => {
|
||||||
if (!Number.isFinite(value)) {
|
if (!Number.isFinite(value)) {
|
||||||
return 0;
|
return 0;
|
||||||
|
|
@ -291,6 +326,10 @@ const coerceNumber = (value: unknown): unknown => {
|
||||||
return value;
|
return value;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const quoteString = (value: string): string => {
|
||||||
|
return JSON.stringify(value);
|
||||||
|
};
|
||||||
|
|
||||||
const normalizeNumericFields = (
|
const normalizeNumericFields = (
|
||||||
row: Record<string, unknown>,
|
row: Record<string, unknown>,
|
||||||
fields: string[]
|
fields: string[]
|
||||||
|
|
@ -353,6 +392,26 @@ const normalizeEquityQuoteRow = (row: unknown): unknown => {
|
||||||
return row;
|
return row;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const normalizeEquityCandleRow = (row: unknown): unknown => {
|
||||||
|
if (row && typeof row === "object") {
|
||||||
|
return normalizeNumericFields(row as Record<string, unknown>, [
|
||||||
|
"source_ts",
|
||||||
|
"ingest_ts",
|
||||||
|
"seq",
|
||||||
|
"ts",
|
||||||
|
"interval_ms",
|
||||||
|
"open",
|
||||||
|
"high",
|
||||||
|
"low",
|
||||||
|
"close",
|
||||||
|
"volume",
|
||||||
|
"trade_count"
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return row;
|
||||||
|
};
|
||||||
|
|
||||||
const normalizeEquityRow = (row: unknown): unknown => {
|
const normalizeEquityRow = (row: unknown): unknown => {
|
||||||
if (row && typeof row === "object") {
|
if (row && typeof row === "object") {
|
||||||
const record = normalizeNumericFields(row as Record<string, unknown>, [
|
const record = normalizeNumericFields(row as Record<string, unknown>, [
|
||||||
|
|
@ -525,6 +584,24 @@ export const fetchRecentEquityQuotes = async (
|
||||||
return EquityQuoteSchema.array().parse(rows.map(normalizeEquityQuoteRow));
|
return EquityQuoteSchema.array().parse(rows.map(normalizeEquityQuoteRow));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const fetchRecentEquityCandles = async (
|
||||||
|
client: ClickHouseClient,
|
||||||
|
underlyingId: string,
|
||||||
|
intervalMs: number,
|
||||||
|
limit: number
|
||||||
|
): Promise<EquityCandle[]> => {
|
||||||
|
const safeLimit = clampLimit(limit);
|
||||||
|
const safeInterval = clampPositiveInt(intervalMs, 1000);
|
||||||
|
const safeUnderlying = quoteString(underlyingId);
|
||||||
|
const result = await client.query({
|
||||||
|
query: `SELECT * FROM ${EQUITY_CANDLES_TABLE} WHERE underlying_id = ${safeUnderlying} AND interval_ms = ${safeInterval} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
|
||||||
|
format: "JSONEachRow"
|
||||||
|
});
|
||||||
|
|
||||||
|
const rows = await result.json<unknown[]>();
|
||||||
|
return EquityCandleSchema.array().parse(rows.map(normalizeEquityCandleRow));
|
||||||
|
};
|
||||||
|
|
||||||
export const fetchRecentEquityPrintJoins = async (
|
export const fetchRecentEquityPrintJoins = async (
|
||||||
client: ClickHouseClient,
|
client: ClickHouseClient,
|
||||||
limit: number
|
limit: number
|
||||||
|
|
@ -691,6 +768,52 @@ export const fetchEquityQuotesAfter = async (
|
||||||
return EquityQuoteSchema.array().parse(rows.map(normalizeEquityQuoteRow));
|
return EquityQuoteSchema.array().parse(rows.map(normalizeEquityQuoteRow));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const fetchEquityCandlesAfter = async (
|
||||||
|
client: ClickHouseClient,
|
||||||
|
underlyingId: string,
|
||||||
|
intervalMs: number,
|
||||||
|
afterTs: number,
|
||||||
|
afterSeq: number,
|
||||||
|
limit: number
|
||||||
|
): Promise<EquityCandle[]> => {
|
||||||
|
const safeLimit = clampLimit(limit);
|
||||||
|
const safeAfterTs = clampCursor(afterTs);
|
||||||
|
const safeAfterSeq = clampCursor(afterSeq);
|
||||||
|
const safeInterval = clampPositiveInt(intervalMs, 1000);
|
||||||
|
const safeUnderlying = quoteString(underlyingId);
|
||||||
|
|
||||||
|
const result = await client.query({
|
||||||
|
query: `SELECT * FROM ${EQUITY_CANDLES_TABLE} WHERE underlying_id = ${safeUnderlying} AND interval_ms = ${safeInterval} AND (ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
|
||||||
|
format: "JSONEachRow"
|
||||||
|
});
|
||||||
|
|
||||||
|
const rows = await result.json<unknown[]>();
|
||||||
|
return EquityCandleSchema.array().parse(rows.map(normalizeEquityCandleRow));
|
||||||
|
};
|
||||||
|
|
||||||
|
export const fetchEquityCandlesRange = async (
|
||||||
|
client: ClickHouseClient,
|
||||||
|
underlyingId: string,
|
||||||
|
intervalMs: number,
|
||||||
|
startTs: number,
|
||||||
|
endTs: number,
|
||||||
|
limit: number
|
||||||
|
): Promise<EquityCandle[]> => {
|
||||||
|
const safeLimit = clampLimit(limit);
|
||||||
|
const safeStart = clampCursor(startTs);
|
||||||
|
const safeEnd = clampCursor(endTs);
|
||||||
|
const safeInterval = clampPositiveInt(intervalMs, 1000);
|
||||||
|
const safeUnderlying = quoteString(underlyingId);
|
||||||
|
|
||||||
|
const result = await client.query({
|
||||||
|
query: `SELECT * FROM ${EQUITY_CANDLES_TABLE} WHERE underlying_id = ${safeUnderlying} AND interval_ms = ${safeInterval} AND ts >= ${safeStart} AND ts <= ${safeEnd} ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
|
||||||
|
format: "JSONEachRow"
|
||||||
|
});
|
||||||
|
|
||||||
|
const rows = await result.json<unknown[]>();
|
||||||
|
return EquityCandleSchema.array().parse(rows.map(normalizeEquityCandleRow));
|
||||||
|
};
|
||||||
|
|
||||||
export const fetchEquityPrintJoinsAfter = async (
|
export const fetchEquityPrintJoinsAfter = async (
|
||||||
client: ClickHouseClient,
|
client: ClickHouseClient,
|
||||||
afterTs: number,
|
afterTs: number,
|
||||||
|
|
|
||||||
29
packages/storage/src/equity-candles.ts
Normal file
29
packages/storage/src/equity-candles.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
import type { EquityCandle } from "@islandflow/types";
|
||||||
|
|
||||||
|
export const EQUITY_CANDLES_TABLE = "equity_candles";
|
||||||
|
|
||||||
|
export const equityCandlesTableDDL = (): string => {
|
||||||
|
return `
|
||||||
|
CREATE TABLE IF NOT EXISTS ${EQUITY_CANDLES_TABLE} (
|
||||||
|
source_ts UInt64,
|
||||||
|
ingest_ts UInt64,
|
||||||
|
seq UInt64,
|
||||||
|
trace_id String,
|
||||||
|
ts UInt64,
|
||||||
|
interval_ms UInt32,
|
||||||
|
underlying_id String,
|
||||||
|
open Float64,
|
||||||
|
high Float64,
|
||||||
|
low Float64,
|
||||||
|
close Float64,
|
||||||
|
volume UInt64,
|
||||||
|
trade_count UInt32
|
||||||
|
)
|
||||||
|
ENGINE = MergeTree
|
||||||
|
ORDER BY (underlying_id, interval_ms, ts)
|
||||||
|
`;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const normalizeEquityCandle = (candle: EquityCandle): EquityCandle => {
|
||||||
|
return candle;
|
||||||
|
};
|
||||||
|
|
@ -4,6 +4,7 @@ export * from "./alerts";
|
||||||
export * from "./flow-packets";
|
export * from "./flow-packets";
|
||||||
export * from "./equity-prints";
|
export * from "./equity-prints";
|
||||||
export * from "./equity-quotes";
|
export * from "./equity-quotes";
|
||||||
|
export * from "./equity-candles";
|
||||||
export * from "./equity-print-joins";
|
export * from "./equity-print-joins";
|
||||||
export * from "./inferred-dark";
|
export * from "./inferred-dark";
|
||||||
export * from "./option-prints";
|
export * from "./option-prints";
|
||||||
|
|
|
||||||
35
packages/storage/tests/equity-candles.test.ts
Normal file
35
packages/storage/tests/equity-candles.test.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
import { describe, expect, it } from "bun:test";
|
||||||
|
import {
|
||||||
|
equityCandlesTableDDL,
|
||||||
|
EQUITY_CANDLES_TABLE,
|
||||||
|
normalizeEquityCandle
|
||||||
|
} from "../src/equity-candles";
|
||||||
|
|
||||||
|
const baseCandle = {
|
||||||
|
source_ts: 100,
|
||||||
|
ingest_ts: 200,
|
||||||
|
seq: 3,
|
||||||
|
trace_id: "candle:SPY:1000:0",
|
||||||
|
ts: 0,
|
||||||
|
interval_ms: 1000,
|
||||||
|
underlying_id: "SPY",
|
||||||
|
open: 450,
|
||||||
|
high: 451.5,
|
||||||
|
low: 449.25,
|
||||||
|
close: 450.75,
|
||||||
|
volume: 1200,
|
||||||
|
trade_count: 15
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("equity-candles storage helpers", () => {
|
||||||
|
it("keeps required fields intact", () => {
|
||||||
|
const normalized = normalizeEquityCandle(baseCandle);
|
||||||
|
expect(normalized).toEqual(baseCandle);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("includes the correct table name in the DDL", () => {
|
||||||
|
const ddl = equityCandlesTableDDL();
|
||||||
|
expect(ddl).toContain(EQUITY_CANDLES_TABLE);
|
||||||
|
expect(ddl).toContain("CREATE TABLE IF NOT EXISTS");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -59,6 +59,22 @@ export const EquityQuoteSchema = EventMetaSchema.merge(
|
||||||
|
|
||||||
export type EquityQuote = z.infer<typeof EquityQuoteSchema>;
|
export type EquityQuote = z.infer<typeof EquityQuoteSchema>;
|
||||||
|
|
||||||
|
export const EquityCandleSchema = EventMetaSchema.merge(
|
||||||
|
z.object({
|
||||||
|
ts: z.number().int().nonnegative(),
|
||||||
|
interval_ms: z.number().int().positive(),
|
||||||
|
underlying_id: z.string().min(1),
|
||||||
|
open: z.number().nonnegative(),
|
||||||
|
high: z.number().nonnegative(),
|
||||||
|
low: z.number().nonnegative(),
|
||||||
|
close: z.number().nonnegative(),
|
||||||
|
volume: z.number().int().nonnegative(),
|
||||||
|
trade_count: z.number().int().nonnegative()
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
export type EquityCandle = z.infer<typeof EquityCandleSchema>;
|
||||||
|
|
||||||
export const EquityPrintJoinSchema = EventMetaSchema.merge(
|
export const EquityPrintJoinSchema = EventMetaSchema.merge(
|
||||||
z.object({
|
z.object({
|
||||||
id: z.string().min(1),
|
id: z.string().min(1),
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@
|
||||||
"@islandflow/observability": "workspace:*",
|
"@islandflow/observability": "workspace:*",
|
||||||
"@islandflow/storage": "workspace:*",
|
"@islandflow/storage": "workspace:*",
|
||||||
"@islandflow/types": "workspace:*",
|
"@islandflow/types": "workspace:*",
|
||||||
|
"redis": "^5.10.0",
|
||||||
"zod": "^3.23.8"
|
"zod": "^3.23.8"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ import { createLogger } from "@islandflow/observability";
|
||||||
import {
|
import {
|
||||||
SUBJECT_ALERTS,
|
SUBJECT_ALERTS,
|
||||||
SUBJECT_CLASSIFIER_HITS,
|
SUBJECT_CLASSIFIER_HITS,
|
||||||
|
SUBJECT_EQUITY_CANDLES,
|
||||||
SUBJECT_EQUITY_JOINS,
|
SUBJECT_EQUITY_JOINS,
|
||||||
SUBJECT_EQUITY_PRINTS,
|
SUBJECT_EQUITY_PRINTS,
|
||||||
SUBJECT_EQUITY_QUOTES,
|
SUBJECT_EQUITY_QUOTES,
|
||||||
|
|
@ -12,6 +13,7 @@ import {
|
||||||
SUBJECT_OPTION_PRINTS,
|
SUBJECT_OPTION_PRINTS,
|
||||||
STREAM_ALERTS,
|
STREAM_ALERTS,
|
||||||
STREAM_CLASSIFIER_HITS,
|
STREAM_CLASSIFIER_HITS,
|
||||||
|
STREAM_EQUITY_CANDLES,
|
||||||
STREAM_EQUITY_JOINS,
|
STREAM_EQUITY_JOINS,
|
||||||
STREAM_EQUITY_PRINTS,
|
STREAM_EQUITY_PRINTS,
|
||||||
STREAM_EQUITY_QUOTES,
|
STREAM_EQUITY_QUOTES,
|
||||||
|
|
@ -28,6 +30,7 @@ import {
|
||||||
createClickHouseClient,
|
createClickHouseClient,
|
||||||
ensureAlertsTable,
|
ensureAlertsTable,
|
||||||
ensureClassifierHitsTable,
|
ensureClassifierHitsTable,
|
||||||
|
ensureEquityCandlesTable,
|
||||||
ensureEquityPrintJoinsTable,
|
ensureEquityPrintJoinsTable,
|
||||||
ensureEquityPrintsTable,
|
ensureEquityPrintsTable,
|
||||||
ensureEquityQuotesTable,
|
ensureEquityQuotesTable,
|
||||||
|
|
@ -41,6 +44,8 @@ import {
|
||||||
fetchRecentFlowPackets,
|
fetchRecentFlowPackets,
|
||||||
fetchRecentInferredDark,
|
fetchRecentInferredDark,
|
||||||
fetchRecentEquityQuotes,
|
fetchRecentEquityQuotes,
|
||||||
|
fetchEquityCandlesAfter,
|
||||||
|
fetchEquityCandlesRange,
|
||||||
fetchRecentOptionNBBO,
|
fetchRecentOptionNBBO,
|
||||||
fetchEquityPrintsAfter,
|
fetchEquityPrintsAfter,
|
||||||
fetchEquityPrintJoinsAfter,
|
fetchEquityPrintJoinsAfter,
|
||||||
|
|
@ -54,6 +59,7 @@ import {
|
||||||
import {
|
import {
|
||||||
AlertEventSchema,
|
AlertEventSchema,
|
||||||
ClassifierHitEventSchema,
|
ClassifierHitEventSchema,
|
||||||
|
EquityCandleSchema,
|
||||||
EquityPrintSchema,
|
EquityPrintSchema,
|
||||||
EquityPrintJoinSchema,
|
EquityPrintJoinSchema,
|
||||||
EquityQuoteSchema,
|
EquityQuoteSchema,
|
||||||
|
|
@ -62,6 +68,7 @@ import {
|
||||||
OptionNBBOSchema,
|
OptionNBBOSchema,
|
||||||
OptionPrintSchema
|
OptionPrintSchema
|
||||||
} from "@islandflow/types";
|
} from "@islandflow/types";
|
||||||
|
import { createClient } from "redis";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
|
||||||
const service = "api";
|
const service = "api";
|
||||||
|
|
@ -72,6 +79,7 @@ const envSchema = z.object({
|
||||||
NATS_URL: z.string().default("nats://localhost:4222"),
|
NATS_URL: z.string().default("nats://localhost:4222"),
|
||||||
CLICKHOUSE_URL: z.string().default("http://localhost:8123"),
|
CLICKHOUSE_URL: z.string().default("http://localhost:8123"),
|
||||||
CLICKHOUSE_DATABASE: z.string().default("default"),
|
CLICKHOUSE_DATABASE: z.string().default("default"),
|
||||||
|
REDIS_URL: z.string().default("redis://localhost:6379"),
|
||||||
REST_DEFAULT_LIMIT: z.coerce.number().int().positive().default(200)
|
REST_DEFAULT_LIMIT: z.coerce.number().int().positive().default(200)
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -105,16 +113,30 @@ const retry = async <T>(
|
||||||
};
|
};
|
||||||
|
|
||||||
const limitSchema = z.coerce.number().int().positive().max(1000);
|
const limitSchema = z.coerce.number().int().positive().max(1000);
|
||||||
|
const candleLimitSchema = z.coerce.number().int().positive().max(5000);
|
||||||
const replayParamsSchema = z.object({
|
const replayParamsSchema = z.object({
|
||||||
after_ts: z.coerce.number().int().nonnegative().default(0),
|
after_ts: z.coerce.number().int().nonnegative().default(0),
|
||||||
after_seq: z.coerce.number().int().nonnegative().default(0),
|
after_seq: z.coerce.number().int().nonnegative().default(0),
|
||||||
limit: z.coerce.number().int().positive().max(1000).default(200)
|
limit: z.coerce.number().int().positive().max(1000).default(200)
|
||||||
});
|
});
|
||||||
|
const candleQuerySchema = z.object({
|
||||||
|
underlying_id: z.string().min(1),
|
||||||
|
interval_ms: z.coerce.number().int().positive(),
|
||||||
|
start_ts: z.coerce.number().int().nonnegative().optional(),
|
||||||
|
end_ts: z.coerce.number().int().nonnegative().optional(),
|
||||||
|
limit: candleLimitSchema.optional(),
|
||||||
|
cache: z.string().optional()
|
||||||
|
});
|
||||||
|
const candleReplaySchema = replayParamsSchema.extend({
|
||||||
|
underlying_id: z.string().min(1),
|
||||||
|
interval_ms: z.coerce.number().int().positive()
|
||||||
|
});
|
||||||
|
|
||||||
type Channel =
|
type Channel =
|
||||||
| "options"
|
| "options"
|
||||||
| "options-nbbo"
|
| "options-nbbo"
|
||||||
| "equities"
|
| "equities"
|
||||||
|
| "equity-candles"
|
||||||
| "equity-quotes"
|
| "equity-quotes"
|
||||||
| "equity-joins"
|
| "equity-joins"
|
||||||
| "inferred-dark"
|
| "inferred-dark"
|
||||||
|
|
@ -129,6 +151,7 @@ type WsData = {
|
||||||
const optionSockets = new Set<WebSocket<WsData>>();
|
const optionSockets = new Set<WebSocket<WsData>>();
|
||||||
const optionNbboSockets = new Set<WebSocket<WsData>>();
|
const optionNbboSockets = new Set<WebSocket<WsData>>();
|
||||||
const equitySockets = new Set<WebSocket<WsData>>();
|
const equitySockets = new Set<WebSocket<WsData>>();
|
||||||
|
const equityCandleSockets = new Set<WebSocket<WsData>>();
|
||||||
const equityQuoteSockets = new Set<WebSocket<WsData>>();
|
const equityQuoteSockets = new Set<WebSocket<WsData>>();
|
||||||
const equityJoinSockets = new Set<WebSocket<WsData>>();
|
const equityJoinSockets = new Set<WebSocket<WsData>>();
|
||||||
const inferredDarkSockets = new Set<WebSocket<WsData>>();
|
const inferredDarkSockets = new Set<WebSocket<WsData>>();
|
||||||
|
|
@ -167,6 +190,70 @@ const parseReplayParams = (url: URL): { afterTs: number; afterSeq: number; limit
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const parseBooleanParam = (value: string | null | undefined): boolean => {
|
||||||
|
if (!value) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const normalized = value.trim().toLowerCase();
|
||||||
|
return ["1", "true", "yes", "on"].includes(normalized);
|
||||||
|
};
|
||||||
|
|
||||||
|
const parseCandleParams = (
|
||||||
|
url: URL
|
||||||
|
): {
|
||||||
|
underlyingId: string;
|
||||||
|
intervalMs: number;
|
||||||
|
startTs: number;
|
||||||
|
endTs: number;
|
||||||
|
limit: number;
|
||||||
|
useCache: boolean;
|
||||||
|
} => {
|
||||||
|
const params = candleQuerySchema.parse({
|
||||||
|
underlying_id: url.searchParams.get("underlying_id") ?? undefined,
|
||||||
|
interval_ms: url.searchParams.get("interval_ms") ?? undefined,
|
||||||
|
start_ts: url.searchParams.get("start_ts") ?? undefined,
|
||||||
|
end_ts: url.searchParams.get("end_ts") ?? undefined,
|
||||||
|
limit: url.searchParams.get("limit") ?? undefined,
|
||||||
|
cache: url.searchParams.get("cache") ?? undefined
|
||||||
|
});
|
||||||
|
|
||||||
|
const endTs = params.end_ts ?? Date.now();
|
||||||
|
const limit = params.limit ?? env.REST_DEFAULT_LIMIT;
|
||||||
|
const startTs =
|
||||||
|
params.start_ts ?? Math.max(0, Math.floor(endTs - params.interval_ms * limit));
|
||||||
|
const rangeStart = Math.min(startTs, endTs);
|
||||||
|
const rangeEnd = Math.max(startTs, endTs);
|
||||||
|
|
||||||
|
return {
|
||||||
|
underlyingId: params.underlying_id,
|
||||||
|
intervalMs: params.interval_ms,
|
||||||
|
startTs: rangeStart,
|
||||||
|
endTs: rangeEnd,
|
||||||
|
limit,
|
||||||
|
useCache: parseBooleanParam(params.cache)
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const parseCandleReplayParams = (
|
||||||
|
url: URL
|
||||||
|
): { underlyingId: string; intervalMs: number; afterTs: number; afterSeq: number; limit: number } => {
|
||||||
|
const params = candleReplaySchema.parse({
|
||||||
|
underlying_id: url.searchParams.get("underlying_id") ?? undefined,
|
||||||
|
interval_ms: url.searchParams.get("interval_ms") ?? undefined,
|
||||||
|
after_ts: url.searchParams.get("after_ts") ?? undefined,
|
||||||
|
after_seq: url.searchParams.get("after_seq") ?? undefined,
|
||||||
|
limit: url.searchParams.get("limit") ?? undefined
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
underlyingId: params.underlying_id,
|
||||||
|
intervalMs: params.interval_ms,
|
||||||
|
afterTs: params.after_ts,
|
||||||
|
afterSeq: params.after_seq,
|
||||||
|
limit: params.limit
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
const broadcast = (sockets: Set<WebSocket<WsData>>, payload: unknown): void => {
|
const broadcast = (sockets: Set<WebSocket<WsData>>, payload: unknown): void => {
|
||||||
const message = JSON.stringify(payload);
|
const message = JSON.stringify(payload);
|
||||||
|
|
||||||
|
|
@ -182,6 +269,40 @@ const broadcast = (sockets: Set<WebSocket<WsData>>, payload: unknown): void => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const buildCandleCacheKey = (underlyingId: string, intervalMs: number): string => {
|
||||||
|
return `candles:equity:${intervalMs}:${underlyingId}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
const fetchEquityCandlesFromCache = async (
|
||||||
|
client: ReturnType<typeof createClient>,
|
||||||
|
underlyingId: string,
|
||||||
|
intervalMs: number,
|
||||||
|
startTs: number,
|
||||||
|
endTs: number
|
||||||
|
): Promise<unknown[]> => {
|
||||||
|
const key = buildCandleCacheKey(underlyingId, intervalMs);
|
||||||
|
const payloads = await client.zRangeByScore(key, startTs, endTs);
|
||||||
|
const parsed = payloads
|
||||||
|
.map((payload) => {
|
||||||
|
try {
|
||||||
|
return JSON.parse(payload) as unknown;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.filter((value): value is unknown => value !== null);
|
||||||
|
|
||||||
|
const validated: unknown[] = [];
|
||||||
|
for (const entry of parsed) {
|
||||||
|
const result = EquityCandleSchema.safeParse(entry);
|
||||||
|
if (result.success) {
|
||||||
|
validated.push(result.data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return validated;
|
||||||
|
};
|
||||||
|
|
||||||
const run = async () => {
|
const run = async () => {
|
||||||
logger.info("service starting");
|
logger.info("service starting");
|
||||||
|
|
||||||
|
|
@ -245,6 +366,19 @@ const run = async () => {
|
||||||
num_replicas: 1
|
num_replicas: 1
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await ensureStream(jsm, {
|
||||||
|
name: STREAM_EQUITY_CANDLES,
|
||||||
|
subjects: [SUBJECT_EQUITY_CANDLES],
|
||||||
|
retention: "limits",
|
||||||
|
storage: "file",
|
||||||
|
discard: "old",
|
||||||
|
max_msgs_per_subject: -1,
|
||||||
|
max_msgs: -1,
|
||||||
|
max_bytes: -1,
|
||||||
|
max_age: 0,
|
||||||
|
num_replicas: 1
|
||||||
|
});
|
||||||
|
|
||||||
await ensureStream(jsm, {
|
await ensureStream(jsm, {
|
||||||
name: STREAM_EQUITY_JOINS,
|
name: STREAM_EQUITY_JOINS,
|
||||||
subjects: [SUBJECT_EQUITY_JOINS],
|
subjects: [SUBJECT_EQUITY_JOINS],
|
||||||
|
|
@ -320,6 +454,7 @@ const run = async () => {
|
||||||
await ensureOptionNBBOTable(clickhouse);
|
await ensureOptionNBBOTable(clickhouse);
|
||||||
await ensureEquityPrintsTable(clickhouse);
|
await ensureEquityPrintsTable(clickhouse);
|
||||||
await ensureEquityQuotesTable(clickhouse);
|
await ensureEquityQuotesTable(clickhouse);
|
||||||
|
await ensureEquityCandlesTable(clickhouse);
|
||||||
await ensureEquityPrintJoinsTable(clickhouse);
|
await ensureEquityPrintJoinsTable(clickhouse);
|
||||||
await ensureInferredDarkTable(clickhouse);
|
await ensureInferredDarkTable(clickhouse);
|
||||||
await ensureFlowPacketsTable(clickhouse);
|
await ensureFlowPacketsTable(clickhouse);
|
||||||
|
|
@ -327,6 +462,27 @@ const run = async () => {
|
||||||
await ensureAlertsTable(clickhouse);
|
await ensureAlertsTable(clickhouse);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let redis: ReturnType<typeof createClient> | null = null;
|
||||||
|
try {
|
||||||
|
redis = createClient({ url: env.REDIS_URL });
|
||||||
|
redis.on("error", (error) => {
|
||||||
|
logger.warn("redis client error", {
|
||||||
|
error: error instanceof Error ? error.message : String(error)
|
||||||
|
});
|
||||||
|
});
|
||||||
|
await retry("redis connect", 5, 500, async () => {
|
||||||
|
if (!redis) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await redis.connect();
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn("redis unavailable, skipping candle cache", {
|
||||||
|
error: error instanceof Error ? error.message : String(error)
|
||||||
|
});
|
||||||
|
redis = null;
|
||||||
|
}
|
||||||
|
|
||||||
const subscribeWithReset = async <T>(
|
const subscribeWithReset = async <T>(
|
||||||
subject: string,
|
subject: string,
|
||||||
stream: string,
|
stream: string,
|
||||||
|
|
@ -389,6 +545,12 @@ const run = async () => {
|
||||||
"api-equity-quotes"
|
"api-equity-quotes"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const equityCandleSubscription = await subscribeWithReset(
|
||||||
|
SUBJECT_EQUITY_CANDLES,
|
||||||
|
STREAM_EQUITY_CANDLES,
|
||||||
|
"api-equity-candles"
|
||||||
|
);
|
||||||
|
|
||||||
const equityJoinSubscription = await subscribeWithReset(
|
const equityJoinSubscription = await subscribeWithReset(
|
||||||
SUBJECT_EQUITY_JOINS,
|
SUBJECT_EQUITY_JOINS,
|
||||||
STREAM_EQUITY_JOINS,
|
STREAM_EQUITY_JOINS,
|
||||||
|
|
@ -479,6 +641,21 @@ const run = async () => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const pumpEquityCandles = async () => {
|
||||||
|
for await (const msg of equityCandleSubscription.messages) {
|
||||||
|
try {
|
||||||
|
const payload = EquityCandleSchema.parse(equityCandleSubscription.decode(msg));
|
||||||
|
broadcast(equityCandleSockets, { type: "equity-candle", payload });
|
||||||
|
msg.ack();
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("failed to process equity candle", {
|
||||||
|
error: error instanceof Error ? error.message : String(error)
|
||||||
|
});
|
||||||
|
msg.term();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const pumpEquityJoins = async () => {
|
const pumpEquityJoins = async () => {
|
||||||
for await (const msg of equityJoinSubscription.messages) {
|
for await (const msg of equityJoinSubscription.messages) {
|
||||||
try {
|
try {
|
||||||
|
|
@ -558,6 +735,7 @@ const run = async () => {
|
||||||
void pumpOptionNbbo();
|
void pumpOptionNbbo();
|
||||||
void pumpEquities();
|
void pumpEquities();
|
||||||
void pumpEquityQuotes();
|
void pumpEquityQuotes();
|
||||||
|
void pumpEquityCandles();
|
||||||
void pumpEquityJoins();
|
void pumpEquityJoins();
|
||||||
void pumpInferredDark();
|
void pumpInferredDark();
|
||||||
void pumpFlow();
|
void pumpFlow();
|
||||||
|
|
@ -597,6 +775,43 @@ const run = async () => {
|
||||||
return jsonResponse({ data });
|
return jsonResponse({ data });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname === "/candles/equities") {
|
||||||
|
try {
|
||||||
|
const { underlyingId, intervalMs, startTs, endTs, limit, useCache } =
|
||||||
|
parseCandleParams(url);
|
||||||
|
if (useCache && redis && redis.isOpen) {
|
||||||
|
const cached = await fetchEquityCandlesFromCache(
|
||||||
|
redis,
|
||||||
|
underlyingId,
|
||||||
|
intervalMs,
|
||||||
|
startTs,
|
||||||
|
endTs
|
||||||
|
);
|
||||||
|
if (cached.length > 0) {
|
||||||
|
return jsonResponse({ data: cached });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await fetchEquityCandlesRange(
|
||||||
|
clickhouse,
|
||||||
|
underlyingId,
|
||||||
|
intervalMs,
|
||||||
|
startTs,
|
||||||
|
endTs,
|
||||||
|
limit
|
||||||
|
);
|
||||||
|
return jsonResponse({ data });
|
||||||
|
} catch (error) {
|
||||||
|
return jsonResponse(
|
||||||
|
{
|
||||||
|
error: "invalid candle query",
|
||||||
|
detail: error instanceof Error ? error.message : String(error)
|
||||||
|
},
|
||||||
|
400
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (req.method === "GET" && url.pathname === "/joins/equities") {
|
if (req.method === "GET" && url.pathname === "/joins/equities") {
|
||||||
const limit = parseLimit(url.searchParams.get("limit"));
|
const limit = parseLimit(url.searchParams.get("limit"));
|
||||||
const data = await fetchRecentEquityPrintJoins(clickhouse, limit);
|
const data = await fetchRecentEquityPrintJoins(clickhouse, limit);
|
||||||
|
|
@ -659,6 +874,32 @@ const run = async () => {
|
||||||
return jsonResponse({ data, next });
|
return jsonResponse({ data, next });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname === "/replay/equity-candles") {
|
||||||
|
try {
|
||||||
|
const { underlyingId, intervalMs, afterTs, afterSeq, limit } =
|
||||||
|
parseCandleReplayParams(url);
|
||||||
|
const data = await fetchEquityCandlesAfter(
|
||||||
|
clickhouse,
|
||||||
|
underlyingId,
|
||||||
|
intervalMs,
|
||||||
|
afterTs,
|
||||||
|
afterSeq,
|
||||||
|
limit
|
||||||
|
);
|
||||||
|
const last = data.at(-1);
|
||||||
|
const next = last ? { ts: last.ts, seq: last.seq } : null;
|
||||||
|
return jsonResponse({ data, next });
|
||||||
|
} catch (error) {
|
||||||
|
return jsonResponse(
|
||||||
|
{
|
||||||
|
error: "invalid candle replay query",
|
||||||
|
detail: error instanceof Error ? error.message : String(error)
|
||||||
|
},
|
||||||
|
400
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (req.method === "GET" && url.pathname === "/replay/equity-joins") {
|
if (req.method === "GET" && url.pathname === "/replay/equity-joins") {
|
||||||
const { afterTs, afterSeq, limit } = parseReplayParams(url);
|
const { afterTs, afterSeq, limit } = parseReplayParams(url);
|
||||||
const data = await fetchEquityPrintJoinsAfter(clickhouse, afterTs, afterSeq, limit);
|
const data = await fetchEquityPrintJoinsAfter(clickhouse, afterTs, afterSeq, limit);
|
||||||
|
|
@ -699,6 +940,14 @@ const run = async () => {
|
||||||
return jsonResponse({ error: "websocket upgrade failed" }, 400);
|
return jsonResponse({ error: "websocket upgrade failed" }, 400);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname === "/ws/equity-candles") {
|
||||||
|
if (serverRef.upgrade(req, { data: { channel: "equity-candles" } })) {
|
||||||
|
return new Response(null, { status: 101 });
|
||||||
|
}
|
||||||
|
|
||||||
|
return jsonResponse({ error: "websocket upgrade failed" }, 400);
|
||||||
|
}
|
||||||
|
|
||||||
if (req.method === "GET" && url.pathname === "/ws/equity-quotes") {
|
if (req.method === "GET" && url.pathname === "/ws/equity-quotes") {
|
||||||
if (serverRef.upgrade(req, { data: { channel: "equity-quotes" } })) {
|
if (serverRef.upgrade(req, { data: { channel: "equity-quotes" } })) {
|
||||||
return new Response(null, { status: 101 });
|
return new Response(null, { status: 101 });
|
||||||
|
|
@ -757,6 +1006,8 @@ const run = async () => {
|
||||||
optionNbboSockets.add(socket);
|
optionNbboSockets.add(socket);
|
||||||
} else if (socket.data.channel === "equities") {
|
} else if (socket.data.channel === "equities") {
|
||||||
equitySockets.add(socket);
|
equitySockets.add(socket);
|
||||||
|
} else if (socket.data.channel === "equity-candles") {
|
||||||
|
equityCandleSockets.add(socket);
|
||||||
} else if (socket.data.channel === "equity-quotes") {
|
} else if (socket.data.channel === "equity-quotes") {
|
||||||
equityQuoteSockets.add(socket);
|
equityQuoteSockets.add(socket);
|
||||||
} else if (socket.data.channel === "equity-joins") {
|
} else if (socket.data.channel === "equity-joins") {
|
||||||
|
|
@ -780,6 +1031,8 @@ const run = async () => {
|
||||||
optionNbboSockets.delete(socket);
|
optionNbboSockets.delete(socket);
|
||||||
} else if (socket.data.channel === "equities") {
|
} else if (socket.data.channel === "equities") {
|
||||||
equitySockets.delete(socket);
|
equitySockets.delete(socket);
|
||||||
|
} else if (socket.data.channel === "equity-candles") {
|
||||||
|
equityCandleSockets.delete(socket);
|
||||||
} else if (socket.data.channel === "equity-quotes") {
|
} else if (socket.data.channel === "equity-quotes") {
|
||||||
equityQuoteSockets.delete(socket);
|
equityQuoteSockets.delete(socket);
|
||||||
} else if (socket.data.channel === "equity-joins") {
|
} else if (socket.data.channel === "equity-joins") {
|
||||||
|
|
@ -804,6 +1057,9 @@ const run = async () => {
|
||||||
const shutdown = async (signal: string) => {
|
const shutdown = async (signal: string) => {
|
||||||
logger.info("service stopping", { signal });
|
logger.info("service stopping", { signal });
|
||||||
server.stop();
|
server.stop();
|
||||||
|
if (redis && redis.isOpen) {
|
||||||
|
await redis.quit();
|
||||||
|
}
|
||||||
await nc.drain();
|
await nc.drain();
|
||||||
await clickhouse.close();
|
await clickhouse.close();
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,12 @@
|
||||||
"dev": "bun run src/index.ts"
|
"dev": "bun run src/index.ts"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@islandflow/bus": "workspace:*",
|
||||||
"@islandflow/config": "workspace:*",
|
"@islandflow/config": "workspace:*",
|
||||||
"@islandflow/observability": "workspace:*"
|
"@islandflow/observability": "workspace:*",
|
||||||
|
"@islandflow/storage": "workspace:*",
|
||||||
|
"@islandflow/types": "workspace:*",
|
||||||
|
"redis": "^5.10.0",
|
||||||
|
"zod": "^3.23.8"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
253
services/candles/src/aggregator.ts
Normal file
253
services/candles/src/aggregator.ts
Normal file
|
|
@ -0,0 +1,253 @@
|
||||||
|
import type { EquityCandle, EquityPrint } from "@islandflow/types";
|
||||||
|
|
||||||
|
export type CandleAggregationConfig = {
|
||||||
|
intervalsMs: number[];
|
||||||
|
maxLateMs: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type CandleAggregationResult = {
|
||||||
|
emitted: EquityCandle[];
|
||||||
|
droppedLate: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
type CandleBuilder = {
|
||||||
|
windowStart: number;
|
||||||
|
intervalMs: number;
|
||||||
|
underlyingId: string;
|
||||||
|
open: number;
|
||||||
|
high: number;
|
||||||
|
low: number;
|
||||||
|
close: number;
|
||||||
|
volume: number;
|
||||||
|
tradeCount: number;
|
||||||
|
openTs: number;
|
||||||
|
openSeq: number;
|
||||||
|
openSourceTs: number;
|
||||||
|
closeTs: number;
|
||||||
|
closeSeq: number;
|
||||||
|
closeIngestTs: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
type IntervalState = {
|
||||||
|
intervalMs: number;
|
||||||
|
underlyingId: string;
|
||||||
|
lastTsSeen: number;
|
||||||
|
windows: Map<number, CandleBuilder>;
|
||||||
|
};
|
||||||
|
|
||||||
|
const toPositiveInt = (value: number): number | null => {
|
||||||
|
if (!Number.isFinite(value)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const normalized = Math.floor(value);
|
||||||
|
if (normalized <= 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const normalizeIntervals = (intervals: number[]): number[] => {
|
||||||
|
const unique = new Set<number>();
|
||||||
|
for (const interval of intervals) {
|
||||||
|
const normalized = toPositiveInt(interval);
|
||||||
|
if (normalized) {
|
||||||
|
unique.add(normalized);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Array.from(unique).sort((a, b) => a - b);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const parseIntervals = (value: string | undefined, fallback: number[]): number[] => {
|
||||||
|
if (!value) {
|
||||||
|
return normalizeIntervals(fallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = value
|
||||||
|
.split(",")
|
||||||
|
.map((entry) => Number(entry.trim()))
|
||||||
|
.filter((entry) => Number.isFinite(entry));
|
||||||
|
|
||||||
|
const normalized = normalizeIntervals(parsed);
|
||||||
|
return normalized.length > 0 ? normalized : normalizeIntervals(fallback);
|
||||||
|
};
|
||||||
|
|
||||||
|
const buildStateKey = (underlyingId: string, intervalMs: number): string => {
|
||||||
|
return `${underlyingId}:${intervalMs}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getWindowStart = (ts: number, intervalMs: number): number => {
|
||||||
|
return Math.floor(ts / intervalMs) * intervalMs;
|
||||||
|
};
|
||||||
|
|
||||||
|
const isEarlier = (ts: number, seq: number, otherTs: number, otherSeq: number): boolean => {
|
||||||
|
if (ts !== otherTs) {
|
||||||
|
return ts < otherTs;
|
||||||
|
}
|
||||||
|
return seq < otherSeq;
|
||||||
|
};
|
||||||
|
|
||||||
|
const isLater = (ts: number, seq: number, otherTs: number, otherSeq: number): boolean => {
|
||||||
|
if (ts !== otherTs) {
|
||||||
|
return ts > otherTs;
|
||||||
|
}
|
||||||
|
return seq > otherSeq;
|
||||||
|
};
|
||||||
|
|
||||||
|
const createBuilder = (
|
||||||
|
print: EquityPrint,
|
||||||
|
intervalMs: number,
|
||||||
|
windowStart: number
|
||||||
|
): CandleBuilder => {
|
||||||
|
return {
|
||||||
|
windowStart,
|
||||||
|
intervalMs,
|
||||||
|
underlyingId: print.underlying_id,
|
||||||
|
open: print.price,
|
||||||
|
high: print.price,
|
||||||
|
low: print.price,
|
||||||
|
close: print.price,
|
||||||
|
volume: print.size,
|
||||||
|
tradeCount: 1,
|
||||||
|
openTs: print.ts,
|
||||||
|
openSeq: print.seq,
|
||||||
|
openSourceTs: print.source_ts,
|
||||||
|
closeTs: print.ts,
|
||||||
|
closeSeq: print.seq,
|
||||||
|
closeIngestTs: print.ingest_ts
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const updateBuilder = (builder: CandleBuilder, print: EquityPrint): CandleBuilder => {
|
||||||
|
builder.volume += print.size;
|
||||||
|
builder.tradeCount += 1;
|
||||||
|
builder.high = Math.max(builder.high, print.price);
|
||||||
|
builder.low = Math.min(builder.low, print.price);
|
||||||
|
|
||||||
|
if (isEarlier(print.ts, print.seq, builder.openTs, builder.openSeq)) {
|
||||||
|
builder.open = print.price;
|
||||||
|
builder.openTs = print.ts;
|
||||||
|
builder.openSeq = print.seq;
|
||||||
|
builder.openSourceTs = print.source_ts;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isLater(print.ts, print.seq, builder.closeTs, builder.closeSeq)) {
|
||||||
|
builder.close = print.price;
|
||||||
|
builder.closeTs = print.ts;
|
||||||
|
builder.closeSeq = print.seq;
|
||||||
|
builder.closeIngestTs = print.ingest_ts;
|
||||||
|
}
|
||||||
|
|
||||||
|
return builder;
|
||||||
|
};
|
||||||
|
|
||||||
|
const toEquityCandle = (builder: CandleBuilder): EquityCandle => {
|
||||||
|
return {
|
||||||
|
source_ts: builder.openSourceTs,
|
||||||
|
ingest_ts: builder.closeIngestTs,
|
||||||
|
seq: builder.closeSeq,
|
||||||
|
trace_id: `candle:${builder.underlyingId}:${builder.intervalMs}:${builder.windowStart}`,
|
||||||
|
ts: builder.windowStart,
|
||||||
|
interval_ms: builder.intervalMs,
|
||||||
|
underlying_id: builder.underlyingId,
|
||||||
|
open: builder.open,
|
||||||
|
high: builder.high,
|
||||||
|
low: builder.low,
|
||||||
|
close: builder.close,
|
||||||
|
volume: builder.volume,
|
||||||
|
trade_count: builder.tradeCount
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const flushState = (state: IntervalState, watermark: number): EquityCandle[] => {
|
||||||
|
const eligibleStarts: number[] = [];
|
||||||
|
for (const start of state.windows.keys()) {
|
||||||
|
if (start + state.intervalMs <= watermark) {
|
||||||
|
eligibleStarts.push(start);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (eligibleStarts.length === 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
eligibleStarts.sort((a, b) => a - b);
|
||||||
|
const emitted: EquityCandle[] = [];
|
||||||
|
for (const start of eligibleStarts) {
|
||||||
|
const builder = state.windows.get(start);
|
||||||
|
if (!builder) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
state.windows.delete(start);
|
||||||
|
emitted.push(toEquityCandle(builder));
|
||||||
|
}
|
||||||
|
|
||||||
|
return emitted;
|
||||||
|
};
|
||||||
|
|
||||||
|
export class CandleAggregator {
|
||||||
|
private readonly intervalsMs: number[];
|
||||||
|
private readonly maxLateMs: number;
|
||||||
|
private readonly stateByKey = new Map<string, IntervalState>();
|
||||||
|
|
||||||
|
constructor(config: CandleAggregationConfig) {
|
||||||
|
this.intervalsMs = normalizeIntervals(config.intervalsMs);
|
||||||
|
this.maxLateMs = Math.max(0, Math.floor(config.maxLateMs));
|
||||||
|
}
|
||||||
|
|
||||||
|
ingest(print: EquityPrint): CandleAggregationResult {
|
||||||
|
const emitted: EquityCandle[] = [];
|
||||||
|
let droppedLate = 0;
|
||||||
|
|
||||||
|
for (const intervalMs of this.intervalsMs) {
|
||||||
|
const key = buildStateKey(print.underlying_id, intervalMs);
|
||||||
|
const state =
|
||||||
|
this.stateByKey.get(key) ??
|
||||||
|
({
|
||||||
|
intervalMs,
|
||||||
|
underlyingId: print.underlying_id,
|
||||||
|
lastTsSeen: 0,
|
||||||
|
windows: new Map()
|
||||||
|
} satisfies IntervalState);
|
||||||
|
|
||||||
|
state.lastTsSeen = Math.max(state.lastTsSeen, print.ts);
|
||||||
|
this.stateByKey.set(key, state);
|
||||||
|
|
||||||
|
const windowStart = getWindowStart(print.ts, intervalMs);
|
||||||
|
const windowEnd = windowStart + intervalMs;
|
||||||
|
const watermark = Math.max(0, state.lastTsSeen - this.maxLateMs);
|
||||||
|
|
||||||
|
if (windowEnd <= watermark && !state.windows.has(windowStart)) {
|
||||||
|
droppedLate += 1;
|
||||||
|
} else {
|
||||||
|
const existing = state.windows.get(windowStart);
|
||||||
|
if (existing) {
|
||||||
|
updateBuilder(existing, print);
|
||||||
|
} else {
|
||||||
|
state.windows.set(windowStart, createBuilder(print, intervalMs, windowStart));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
emitted.push(...flushState(state, watermark));
|
||||||
|
}
|
||||||
|
|
||||||
|
return { emitted, droppedLate };
|
||||||
|
}
|
||||||
|
|
||||||
|
drain(): EquityCandle[] {
|
||||||
|
const emitted: EquityCandle[] = [];
|
||||||
|
|
||||||
|
for (const state of this.stateByKey.values()) {
|
||||||
|
const starts = Array.from(state.windows.keys()).sort((a, b) => a - b);
|
||||||
|
for (const start of starts) {
|
||||||
|
const builder = state.windows.get(start);
|
||||||
|
if (!builder) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
state.windows.delete(start);
|
||||||
|
emitted.push(toEquityCandle(builder));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return emitted;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,17 +1,387 @@
|
||||||
import { createLogger } from "@islandflow/observability";
|
import { readEnv } from "@islandflow/config";
|
||||||
|
import { createLogger, createMetrics } from "@islandflow/observability";
|
||||||
|
import {
|
||||||
|
SUBJECT_EQUITY_CANDLES,
|
||||||
|
SUBJECT_EQUITY_PRINTS,
|
||||||
|
STREAM_EQUITY_CANDLES,
|
||||||
|
STREAM_EQUITY_PRINTS,
|
||||||
|
buildDurableConsumer,
|
||||||
|
connectJetStreamWithRetry,
|
||||||
|
ensureStream,
|
||||||
|
publishJson,
|
||||||
|
subscribeJson
|
||||||
|
} from "@islandflow/bus";
|
||||||
|
import {
|
||||||
|
createClickHouseClient,
|
||||||
|
ensureEquityCandlesTable,
|
||||||
|
insertEquityCandle
|
||||||
|
} from "@islandflow/storage";
|
||||||
|
import { EquityCandleSchema, EquityPrintSchema, type EquityCandle } from "@islandflow/types";
|
||||||
|
import { createClient } from "redis";
|
||||||
|
import { z } from "zod";
|
||||||
|
import { CandleAggregator, parseIntervals } from "./aggregator";
|
||||||
|
|
||||||
const service = "candles";
|
const service = "candles";
|
||||||
const logger = createLogger({ service });
|
const logger = createLogger({ service });
|
||||||
|
const metrics = createMetrics({ service });
|
||||||
|
|
||||||
|
const envSchema = z.object({
|
||||||
|
NATS_URL: z.string().default("nats://localhost:4222"),
|
||||||
|
CLICKHOUSE_URL: z.string().default("http://localhost:8123"),
|
||||||
|
CLICKHOUSE_DATABASE: z.string().default("default"),
|
||||||
|
REDIS_URL: z.string().default("redis://localhost:6379"),
|
||||||
|
CANDLE_INTERVALS_MS: z.string().default("1000,5000,60000"),
|
||||||
|
CANDLE_MAX_LATE_MS: z.coerce.number().int().nonnegative().default(0),
|
||||||
|
CANDLE_CACHE_LIMIT: z.coerce.number().int().nonnegative().default(2000),
|
||||||
|
CANDLE_DELIVER_POLICY: z
|
||||||
|
.enum(["new", "all", "last", "last_per_subject"])
|
||||||
|
.default("new"),
|
||||||
|
CANDLE_CONSUMER_RESET: z
|
||||||
|
.preprocess((value) => {
|
||||||
|
if (typeof value === "string") {
|
||||||
|
const normalized = value.trim().toLowerCase();
|
||||||
|
if (["1", "true", "yes", "on"].includes(normalized)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (["0", "false", "no", "off"].includes(normalized)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}, z.boolean())
|
||||||
|
.default(false)
|
||||||
|
});
|
||||||
|
|
||||||
|
const env = readEnv(envSchema);
|
||||||
|
|
||||||
|
const retry = async <T>(
|
||||||
|
label: string,
|
||||||
|
attempts: number,
|
||||||
|
delayMs: number,
|
||||||
|
task: () => Promise<T>
|
||||||
|
): Promise<T> => {
|
||||||
|
let lastError: unknown;
|
||||||
|
|
||||||
|
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
||||||
|
try {
|
||||||
|
return await task();
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error;
|
||||||
|
logger.warn(`${label} attempt failed`, {
|
||||||
|
attempt,
|
||||||
|
error: error instanceof Error ? error.message : String(error)
|
||||||
|
});
|
||||||
|
|
||||||
|
if (attempt < attempts) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw lastError ?? new Error(`${label} failed after retries`);
|
||||||
|
};
|
||||||
|
|
||||||
|
const applyDeliverPolicy = (
|
||||||
|
opts: ReturnType<typeof buildDurableConsumer>,
|
||||||
|
policy: typeof env.CANDLE_DELIVER_POLICY
|
||||||
|
) => {
|
||||||
|
switch (policy) {
|
||||||
|
case "all":
|
||||||
|
opts.deliverAll();
|
||||||
|
break;
|
||||||
|
case "last":
|
||||||
|
opts.deliverLast();
|
||||||
|
break;
|
||||||
|
case "last_per_subject":
|
||||||
|
opts.deliverLastPerSubject();
|
||||||
|
break;
|
||||||
|
case "new":
|
||||||
|
default:
|
||||||
|
opts.deliverNew();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const createRedisClient = (url: string) => {
|
||||||
|
return createClient({ url });
|
||||||
|
};
|
||||||
|
|
||||||
|
const buildCacheKey = (underlyingId: string, intervalMs: number): string => {
|
||||||
|
return `candles:equity:${intervalMs}:${underlyingId}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
const cacheCandle = async (
|
||||||
|
client: ReturnType<typeof createClient>,
|
||||||
|
candle: EquityCandle,
|
||||||
|
cacheLimit: number
|
||||||
|
): Promise<void> => {
|
||||||
|
if (cacheLimit <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const key = buildCacheKey(candle.underlying_id, candle.interval_ms);
|
||||||
|
const payload = JSON.stringify(candle);
|
||||||
|
const maxAgeMs = candle.interval_ms * cacheLimit;
|
||||||
|
const trimBefore = Math.max(0, candle.ts - maxAgeMs);
|
||||||
|
const multi = client.multi();
|
||||||
|
multi.zAdd(key, { score: candle.ts, value: payload });
|
||||||
|
if (trimBefore > 0) {
|
||||||
|
multi.zRemRangeByScore(key, 0, trimBefore);
|
||||||
|
}
|
||||||
|
await multi.exec();
|
||||||
|
};
|
||||||
|
|
||||||
|
const emitCandle = async (
|
||||||
|
clickhouse: ReturnType<typeof createClickHouseClient>,
|
||||||
|
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
|
||||||
|
redis: ReturnType<typeof createClient> | null,
|
||||||
|
candle: EquityCandle,
|
||||||
|
cacheLimit: number
|
||||||
|
): Promise<void> => {
|
||||||
|
try {
|
||||||
|
await insertEquityCandle(clickhouse, candle);
|
||||||
|
} catch (error) {
|
||||||
|
metrics.count("candles.persist_failed", 1);
|
||||||
|
logger.error("failed to persist candle", {
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
trace_id: candle.trace_id,
|
||||||
|
underlying_id: candle.underlying_id,
|
||||||
|
interval_ms: candle.interval_ms
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
metrics.count("candles.emitted", 1, {
|
||||||
|
interval_ms: String(candle.interval_ms)
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await publishJson(js, SUBJECT_EQUITY_CANDLES, candle);
|
||||||
|
} catch (error) {
|
||||||
|
metrics.count("candles.publish_failed", 1);
|
||||||
|
logger.error("failed to publish candle", {
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
trace_id: candle.trace_id,
|
||||||
|
underlying_id: candle.underlying_id,
|
||||||
|
interval_ms: candle.interval_ms
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (redis && redis.isOpen) {
|
||||||
|
try {
|
||||||
|
await cacheCandle(redis, candle, cacheLimit);
|
||||||
|
} catch (error) {
|
||||||
|
metrics.count("candles.cache_failed", 1);
|
||||||
|
logger.warn("failed to cache candle", {
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
trace_id: candle.trace_id,
|
||||||
|
underlying_id: candle.underlying_id,
|
||||||
|
interval_ms: candle.interval_ms
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
logger.info("service starting");
|
logger.info("service starting");
|
||||||
|
|
||||||
const shutdown = (signal: string) => {
|
const intervalsMs = parseIntervals(env.CANDLE_INTERVALS_MS, [1000, 5000, 60000]);
|
||||||
|
if (intervalsMs.length === 0) {
|
||||||
|
throw new Error("CANDLE_INTERVALS_MS produced no valid intervals");
|
||||||
|
}
|
||||||
|
|
||||||
|
const aggregator = new CandleAggregator({
|
||||||
|
intervalsMs,
|
||||||
|
maxLateMs: env.CANDLE_MAX_LATE_MS
|
||||||
|
});
|
||||||
|
|
||||||
|
const { nc, js, jsm } = await connectJetStreamWithRetry(
|
||||||
|
{
|
||||||
|
servers: env.NATS_URL,
|
||||||
|
name: service
|
||||||
|
},
|
||||||
|
{ attempts: 20, delayMs: 500 }
|
||||||
|
);
|
||||||
|
|
||||||
|
await ensureStream(jsm, {
|
||||||
|
name: STREAM_EQUITY_PRINTS,
|
||||||
|
subjects: [SUBJECT_EQUITY_PRINTS],
|
||||||
|
retention: "limits",
|
||||||
|
storage: "file",
|
||||||
|
discard: "old",
|
||||||
|
max_msgs_per_subject: -1,
|
||||||
|
max_msgs: -1,
|
||||||
|
max_bytes: -1,
|
||||||
|
max_age: 0,
|
||||||
|
num_replicas: 1
|
||||||
|
});
|
||||||
|
|
||||||
|
await ensureStream(jsm, {
|
||||||
|
name: STREAM_EQUITY_CANDLES,
|
||||||
|
subjects: [SUBJECT_EQUITY_CANDLES],
|
||||||
|
retention: "limits",
|
||||||
|
storage: "file",
|
||||||
|
discard: "old",
|
||||||
|
max_msgs_per_subject: -1,
|
||||||
|
max_msgs: -1,
|
||||||
|
max_bytes: -1,
|
||||||
|
max_age: 0,
|
||||||
|
num_replicas: 1
|
||||||
|
});
|
||||||
|
|
||||||
|
const clickhouse = createClickHouseClient({
|
||||||
|
url: env.CLICKHOUSE_URL,
|
||||||
|
database: env.CLICKHOUSE_DATABASE
|
||||||
|
});
|
||||||
|
|
||||||
|
await retry("clickhouse table init", 20, 500, async () => {
|
||||||
|
await ensureEquityCandlesTable(clickhouse);
|
||||||
|
});
|
||||||
|
|
||||||
|
let redis: ReturnType<typeof createClient> | null = null;
|
||||||
|
try {
|
||||||
|
redis = createRedisClient(env.REDIS_URL);
|
||||||
|
redis.on("error", (error) => {
|
||||||
|
logger.warn("redis client error", {
|
||||||
|
error: error instanceof Error ? error.message : String(error)
|
||||||
|
});
|
||||||
|
});
|
||||||
|
await retry("redis connect", 20, 500, async () => {
|
||||||
|
if (!redis) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await redis.connect();
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn("redis unavailable, skipping hot cache", {
|
||||||
|
error: error instanceof Error ? error.message : String(error)
|
||||||
|
});
|
||||||
|
redis = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const durableName = "candles-equity-prints";
|
||||||
|
if (env.CANDLE_CONSUMER_RESET) {
|
||||||
|
try {
|
||||||
|
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, durableName);
|
||||||
|
logger.warn("reset jetstream consumer", { durable: durableName });
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
if (!message.includes("not found")) {
|
||||||
|
logger.warn("failed to reset jetstream consumer", { durable: durableName, error: message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
const info = await jsm.consumers.info(STREAM_EQUITY_PRINTS, durableName);
|
||||||
|
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.CANDLE_DELIVER_POLICY) {
|
||||||
|
logger.warn("resetting consumer due to deliver policy change", {
|
||||||
|
durable: durableName,
|
||||||
|
current: info.config.deliver_policy,
|
||||||
|
desired: env.CANDLE_DELIVER_POLICY
|
||||||
|
});
|
||||||
|
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, durableName);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
if (!message.includes("not found")) {
|
||||||
|
logger.warn("failed to inspect jetstream consumer", { durable: durableName, error: message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const subscribeWithReset = async () => {
|
||||||
|
const opts = buildDurableConsumer(durableName);
|
||||||
|
applyDeliverPolicy(opts, env.CANDLE_DELIVER_POLICY);
|
||||||
|
try {
|
||||||
|
return await subscribeJson(js, SUBJECT_EQUITY_PRINTS, opts);
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
const shouldReset =
|
||||||
|
message.includes("duplicate subscription") ||
|
||||||
|
message.includes("durable requires") ||
|
||||||
|
message.includes("subject does not match consumer");
|
||||||
|
|
||||||
|
if (!shouldReset) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.warn("resetting jetstream consumer", { durable: durableName, error: message });
|
||||||
|
|
||||||
|
try {
|
||||||
|
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, durableName);
|
||||||
|
} catch (deleteError) {
|
||||||
|
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError);
|
||||||
|
if (!deleteMessage.includes("not found")) {
|
||||||
|
logger.warn("failed to delete jetstream consumer", {
|
||||||
|
durable: durableName,
|
||||||
|
error: deleteMessage
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const resetOpts = buildDurableConsumer(durableName);
|
||||||
|
applyDeliverPolicy(resetOpts, env.CANDLE_DELIVER_POLICY);
|
||||||
|
return await subscribeJson(js, SUBJECT_EQUITY_PRINTS, resetOpts);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const subscription = await subscribeWithReset();
|
||||||
|
let droppedLate = 0;
|
||||||
|
let lastLateLog = Date.now();
|
||||||
|
|
||||||
|
const loop = async () => {
|
||||||
|
for await (const msg of subscription.messages) {
|
||||||
|
try {
|
||||||
|
const print = EquityPrintSchema.parse(subscription.decode(msg));
|
||||||
|
metrics.count("candles.prints", 1);
|
||||||
|
|
||||||
|
const result = aggregator.ingest(print);
|
||||||
|
if (result.droppedLate > 0) {
|
||||||
|
droppedLate += result.droppedLate;
|
||||||
|
metrics.count("candles.prints_late", result.droppedLate);
|
||||||
|
const now = Date.now();
|
||||||
|
if (now - lastLateLog > 5000) {
|
||||||
|
logger.warn("late equity prints dropped", { dropped: droppedLate });
|
||||||
|
droppedLate = 0;
|
||||||
|
lastLateLog = now;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const candle of result.emitted) {
|
||||||
|
const validated = EquityCandleSchema.parse(candle);
|
||||||
|
await emitCandle(clickhouse, js, redis, validated, env.CANDLE_CACHE_LIMIT);
|
||||||
|
}
|
||||||
|
|
||||||
|
msg.ack();
|
||||||
|
} catch (error) {
|
||||||
|
metrics.count("candles.prints_failed", 1);
|
||||||
|
logger.error("failed to process equity print", {
|
||||||
|
error: error instanceof Error ? error.message : String(error)
|
||||||
|
});
|
||||||
|
msg.term();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const shutdown = async (signal: string) => {
|
||||||
logger.info("service stopping", { signal });
|
logger.info("service stopping", { signal });
|
||||||
|
const remaining = aggregator.drain();
|
||||||
|
for (const candle of remaining) {
|
||||||
|
const validated = EquityCandleSchema.parse(candle);
|
||||||
|
await emitCandle(clickhouse, js, redis, validated, env.CANDLE_CACHE_LIMIT);
|
||||||
|
}
|
||||||
|
if (redis && redis.isOpen) {
|
||||||
|
await redis.quit();
|
||||||
|
}
|
||||||
|
await nc.drain();
|
||||||
|
await clickhouse.close();
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
};
|
};
|
||||||
|
|
||||||
process.on("SIGINT", () => shutdown("SIGINT"));
|
process.on("SIGINT", () => void shutdown("SIGINT"));
|
||||||
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
process.on("SIGTERM", () => void shutdown("SIGTERM"));
|
||||||
|
|
||||||
// Keep the process alive until real listeners are wired.
|
void loop();
|
||||||
setInterval(() => {}, 60_000);
|
};
|
||||||
|
|
||||||
|
await run();
|
||||||
|
|
|
||||||
81
services/candles/tests/aggregator.test.ts
Normal file
81
services/candles/tests/aggregator.test.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
||||||
|
import { describe, expect, test } from "bun:test";
|
||||||
|
import type { EquityPrint } from "@islandflow/types";
|
||||||
|
import { CandleAggregator } from "../src/aggregator";
|
||||||
|
|
||||||
|
const buildPrint = (overrides: Partial<EquityPrint> = {}): EquityPrint => {
|
||||||
|
const ts = overrides.ts ?? 0;
|
||||||
|
return {
|
||||||
|
source_ts: overrides.source_ts ?? ts,
|
||||||
|
ingest_ts: overrides.ingest_ts ?? ts,
|
||||||
|
seq: overrides.seq ?? 0,
|
||||||
|
trace_id: overrides.trace_id ?? `print:${overrides.seq ?? 0}`,
|
||||||
|
ts,
|
||||||
|
underlying_id: overrides.underlying_id ?? "AAPL",
|
||||||
|
price: overrides.price ?? 0,
|
||||||
|
size: overrides.size ?? 1,
|
||||||
|
exchange: overrides.exchange ?? "TEST",
|
||||||
|
offExchangeFlag: overrides.offExchangeFlag ?? false
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("CandleAggregator", () => {
|
||||||
|
test("emits candle with correct OHLC and volume", () => {
|
||||||
|
const aggregator = new CandleAggregator({ intervalsMs: [1000], maxLateMs: 0 });
|
||||||
|
|
||||||
|
const first = buildPrint({ ts: 1000, price: 10, size: 100, seq: 1 });
|
||||||
|
const second = buildPrint({ ts: 1500, price: 12, size: 50, seq: 2 });
|
||||||
|
const third = buildPrint({ ts: 2500, price: 11, size: 10, seq: 3 });
|
||||||
|
|
||||||
|
expect(aggregator.ingest(first).emitted).toHaveLength(0);
|
||||||
|
expect(aggregator.ingest(second).emitted).toHaveLength(0);
|
||||||
|
|
||||||
|
const result = aggregator.ingest(third);
|
||||||
|
expect(result.emitted).toHaveLength(1);
|
||||||
|
|
||||||
|
const candle = result.emitted[0];
|
||||||
|
expect(candle.ts).toBe(1000);
|
||||||
|
expect(candle.open).toBe(10);
|
||||||
|
expect(candle.high).toBe(12);
|
||||||
|
expect(candle.low).toBe(10);
|
||||||
|
expect(candle.close).toBe(12);
|
||||||
|
expect(candle.volume).toBe(150);
|
||||||
|
expect(candle.trade_count).toBe(2);
|
||||||
|
expect(candle.seq).toBe(2);
|
||||||
|
expect(candle.source_ts).toBe(1000);
|
||||||
|
expect(candle.ingest_ts).toBe(1500);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("respects open and close order with out-of-order prints", () => {
|
||||||
|
const aggregator = new CandleAggregator({ intervalsMs: [1000], maxLateMs: 2000 });
|
||||||
|
|
||||||
|
const late = buildPrint({ ts: 1500, price: 15, size: 10, seq: 2 });
|
||||||
|
const early = buildPrint({ ts: 1200, price: 11, size: 20, seq: 1 });
|
||||||
|
|
||||||
|
aggregator.ingest(late);
|
||||||
|
aggregator.ingest(early);
|
||||||
|
|
||||||
|
const [candle] = aggregator.drain();
|
||||||
|
expect(candle.open).toBe(11);
|
||||||
|
expect(candle.close).toBe(15);
|
||||||
|
expect(candle.trade_count).toBe(2);
|
||||||
|
expect(candle.seq).toBe(2);
|
||||||
|
expect(candle.source_ts).toBe(1200);
|
||||||
|
expect(candle.ingest_ts).toBe(1500);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("drops late prints once window is closed", () => {
|
||||||
|
const aggregator = new CandleAggregator({ intervalsMs: [1000], maxLateMs: 0 });
|
||||||
|
|
||||||
|
const first = buildPrint({ ts: 1000, price: 10, size: 100, seq: 1 });
|
||||||
|
const next = buildPrint({ ts: 3000, price: 14, size: 50, seq: 2 });
|
||||||
|
const late = buildPrint({ ts: 1500, price: 9, size: 25, seq: 3 });
|
||||||
|
|
||||||
|
aggregator.ingest(first);
|
||||||
|
const flush = aggregator.ingest(next);
|
||||||
|
expect(flush.emitted).toHaveLength(1);
|
||||||
|
|
||||||
|
const lateResult = aggregator.ingest(late);
|
||||||
|
expect(lateResult.emitted).toHaveLength(0);
|
||||||
|
expect(lateResult.droppedLate).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
Loading…
Add table
Add a link
Reference in a new issue