Add equity candle aggregation pipeline

packages/storage/src/clickhouse.ts

@@ -2,6 +2,7 @@ import { createClient, type ClickHouseClient } from "@clickhouse/client";
 import {
   AlertEventSchema,
   ClassifierHitEventSchema,
+  EquityCandleSchema,
   EquityPrintSchema,
   EquityQuoteSchema,
   EquityPrintJoinSchema,
@@ -13,6 +14,7 @@ import {
 import type {
   AlertEvent,
   ClassifierHitEvent,
+  EquityCandle,
   EquityPrint,
   EquityQuote,
   EquityPrintJoin,
@@ -37,6 +39,11 @@ import {
   EQUITY_QUOTES_TABLE,
   normalizeEquityQuote
 } from "./equity-quotes";
+import {
+  equityCandlesTableDDL,
+  EQUITY_CANDLES_TABLE,
+  normalizeEquityCandle
+} from "./equity-candles";
 import {
   equityPrintJoinsTableDDL,
   EQUITY_PRINT_JOINS_TABLE,
@@ -121,6 +128,14 @@ export const ensureEquityQuotesTable = async (
   });
 };
 
+export const ensureEquityCandlesTable = async (
+  client: ClickHouseClient
+): Promise<void> => {
+  await client.exec({
+    query: equityCandlesTableDDL()
+  });
+};
+
 export const ensureEquityPrintJoinsTable = async (
   client: ClickHouseClient
 ): Promise<void> => {
@@ -207,6 +222,18 @@ export const insertEquityQuote = async (
   });
 };
 
+export const insertEquityCandle = async (
+  client: ClickHouseClient,
+  candle: EquityCandle
+): Promise<void> => {
+  const record = normalizeEquityCandle(candle);
+  await client.insert({
+    table: EQUITY_CANDLES_TABLE,
+    values: [record],
+    format: "JSONEachRow"
+  });
+};
+
 export const insertEquityPrintJoin = async (
   client: ClickHouseClient,
   join: EquityPrintJoin
@@ -272,6 +299,14 @@ const clampLimit = (limit: number): number => {
   return Math.max(1, Math.min(1000, Math.floor(limit)));
 };
 
+const clampPositiveInt = (value: number, fallback = 1): number => {
+  if (!Number.isFinite(value)) {
+    return fallback;
+  }
+
+  return Math.max(1, Math.floor(value));
+};
+
 const clampCursor = (value: number): number => {
   if (!Number.isFinite(value)) {
     return 0;
@@ -291,6 +326,10 @@ const coerceNumber = (value: unknown): unknown => {
   return value;
 };
 
+const quoteString = (value: string): string => {
+  return JSON.stringify(value);
+};
+
 const normalizeNumericFields = (
   row: Record<string, unknown>,
   fields: string[]
@@ -353,6 +392,26 @@ const normalizeEquityQuoteRow = (row: unknown): unknown => {
   return row;
 };
 
+const normalizeEquityCandleRow = (row: unknown): unknown => {
+  if (row && typeof row === "object") {
+    return normalizeNumericFields(row as Record<string, unknown>, [
+      "source_ts",
+      "ingest_ts",
+      "seq",
+      "ts",
+      "interval_ms",
+      "open",
+      "high",
+      "low",
+      "close",
+      "volume",
+      "trade_count"
+    ]);
+  }
+
+  return row;
+};
+
 const normalizeEquityRow = (row: unknown): unknown => {
   if (row && typeof row === "object") {
     const record = normalizeNumericFields(row as Record<string, unknown>, [
@@ -525,6 +584,24 @@ export const fetchRecentEquityQuotes = async (
   return EquityQuoteSchema.array().parse(rows.map(normalizeEquityQuoteRow));
 };
 
+export const fetchRecentEquityCandles = async (
+  client: ClickHouseClient,
+  underlyingId: string,
+  intervalMs: number,
+  limit: number
+): Promise<EquityCandle[]> => {
+  const safeLimit = clampLimit(limit);
+  const safeInterval = clampPositiveInt(intervalMs, 1000);
+  const safeUnderlying = quoteString(underlyingId);
+  const result = await client.query({
+    query: `SELECT * FROM ${EQUITY_CANDLES_TABLE} WHERE underlying_id = ${safeUnderlying} AND interval_ms = ${safeInterval} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
+    format: "JSONEachRow"
+  });
+
+  const rows = await result.json<unknown[]>();
+  return EquityCandleSchema.array().parse(rows.map(normalizeEquityCandleRow));
+};
+
 export const fetchRecentEquityPrintJoins = async (
   client: ClickHouseClient,
   limit: number
@@ -691,6 +768,52 @@ export const fetchEquityQuotesAfter = async (
   return EquityQuoteSchema.array().parse(rows.map(normalizeEquityQuoteRow));
 };
 
+export const fetchEquityCandlesAfter = async (
+  client: ClickHouseClient,
+  underlyingId: string,
+  intervalMs: number,
+  afterTs: number,
+  afterSeq: number,
+  limit: number
+): Promise<EquityCandle[]> => {
+  const safeLimit = clampLimit(limit);
+  const safeAfterTs = clampCursor(afterTs);
+  const safeAfterSeq = clampCursor(afterSeq);
+  const safeInterval = clampPositiveInt(intervalMs, 1000);
+  const safeUnderlying = quoteString(underlyingId);
+
+  const result = await client.query({
+    query: `SELECT * FROM ${EQUITY_CANDLES_TABLE} WHERE underlying_id = ${safeUnderlying} AND interval_ms = ${safeInterval} AND (ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
+    format: "JSONEachRow"
+  });
+
+  const rows = await result.json<unknown[]>();
+  return EquityCandleSchema.array().parse(rows.map(normalizeEquityCandleRow));
+};
+
+export const fetchEquityCandlesRange = async (
+  client: ClickHouseClient,
+  underlyingId: string,
+  intervalMs: number,
+  startTs: number,
+  endTs: number,
+  limit: number
+): Promise<EquityCandle[]> => {
+  const safeLimit = clampLimit(limit);
+  const safeStart = clampCursor(startTs);
+  const safeEnd = clampCursor(endTs);
+  const safeInterval = clampPositiveInt(intervalMs, 1000);
+  const safeUnderlying = quoteString(underlyingId);
+
+  const result = await client.query({
+    query: `SELECT * FROM ${EQUITY_CANDLES_TABLE} WHERE underlying_id = ${safeUnderlying} AND interval_ms = ${safeInterval} AND ts >= ${safeStart} AND ts <= ${safeEnd} ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
+    format: "JSONEachRow"
+  });
+
+  const rows = await result.json<unknown[]>();
+  return EquityCandleSchema.array().parse(rows.map(normalizeEquityCandleRow));
+};
+
 export const fetchEquityPrintJoinsAfter = async (
   client: ClickHouseClient,
   afterTs: number,

packages/storage/src/equity-candles.ts

@@ -0,0 +1,29 @@
+import type { EquityCandle } from "@islandflow/types";
+
+export const EQUITY_CANDLES_TABLE = "equity_candles";
+
+export const equityCandlesTableDDL = (): string => {
+  return `
+CREATE TABLE IF NOT EXISTS ${EQUITY_CANDLES_TABLE} (
+  source_ts UInt64,
+  ingest_ts UInt64,
+  seq UInt64,
+  trace_id String,
+  ts UInt64,
+  interval_ms UInt32,
+  underlying_id String,
+  open Float64,
+  high Float64,
+  low Float64,
+  close Float64,
+  volume UInt64,
+  trade_count UInt32
+)
+ENGINE = MergeTree
+ORDER BY (underlying_id, interval_ms, ts)
+`;
+};
+
+export const normalizeEquityCandle = (candle: EquityCandle): EquityCandle => {
+  return candle;
+};

packages/storage/src/index.ts

@@ -4,6 +4,7 @@ export * from "./alerts";
 export * from "./flow-packets";
 export * from "./equity-prints";
 export * from "./equity-quotes";
+export * from "./equity-candles";
 export * from "./equity-print-joins";
 export * from "./inferred-dark";
 export * from "./option-prints";

packages/storage/tests/equity-candles.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from "bun:test";
+import {
+  equityCandlesTableDDL,
+  EQUITY_CANDLES_TABLE,
+  normalizeEquityCandle
+} from "../src/equity-candles";
+
+const baseCandle = {
+  source_ts: 100,
+  ingest_ts: 200,
+  seq: 3,
+  trace_id: "candle:SPY:1000:0",
+  ts: 0,
+  interval_ms: 1000,
+  underlying_id: "SPY",
+  open: 450,
+  high: 451.5,
+  low: 449.25,
+  close: 450.75,
+  volume: 1200,
+  trade_count: 15
+};
+
+describe("equity-candles storage helpers", () => {
+  it("keeps required fields intact", () => {
+    const normalized = normalizeEquityCandle(baseCandle);
+    expect(normalized).toEqual(baseCandle);
+  });
+
+  it("includes the correct table name in the DDL", () => {
+    const ddl = equityCandlesTableDDL();
+    expect(ddl).toContain(EQUITY_CANDLES_TABLE);
+    expect(ddl).toContain("CREATE TABLE IF NOT EXISTS");
+  });
+});