Add equity prints ingestion

This commit is contained in:
dirtydishes 2025-12-27 19:21:01 -05:00
parent 488ae82ed6
commit 6a1f457028
9 changed files with 252 additions and 12 deletions

View file

@ -1,2 +1,4 @@
export const STREAM_OPTION_PRINTS = "OPTIONS_PRINTS";
export const SUBJECT_OPTION_PRINTS = "options.prints";
export const STREAM_EQUITY_PRINTS = "EQUITY_PRINTS";
export const SUBJECT_EQUITY_PRINTS = "equities.prints";

View file

@ -1,6 +1,15 @@
import { createClient, type ClickHouseClient } from "@clickhouse/client";
import type { OptionPrint } from "@islandflow/types";
import { normalizeOptionPrint, optionPrintsTableDDL, OPTION_PRINTS_TABLE } from "./option-prints";
import type { EquityPrint, OptionPrint } from "@islandflow/types";
import {
normalizeOptionPrint,
optionPrintsTableDDL,
OPTION_PRINTS_TABLE
} from "./option-prints";
import {
equityPrintsTableDDL,
EQUITY_PRINTS_TABLE,
normalizeEquityPrint
} from "./equity-prints";
export type ClickHouseOptions = {
url: string;
@ -26,6 +35,14 @@ export const ensureOptionPrintsTable = async (
});
};
export const ensureEquityPrintsTable = async (
client: ClickHouseClient
): Promise<void> => {
await client.exec({
query: equityPrintsTableDDL()
});
};
export const insertOptionPrint = async (
client: ClickHouseClient,
print: OptionPrint
@ -37,3 +54,15 @@ export const insertOptionPrint = async (
format: "JSONEachRow"
});
};
export const insertEquityPrint = async (
client: ClickHouseClient,
print: EquityPrint
): Promise<void> => {
const record = normalizeEquityPrint(print);
await client.insert({
table: EQUITY_PRINTS_TABLE,
values: [record],
format: "JSONEachRow"
});
};

View file

@ -0,0 +1,26 @@
import type { EquityPrint } from "@islandflow/types";
export const EQUITY_PRINTS_TABLE = "equity_prints";
export const equityPrintsTableDDL = (): string => {
return `
CREATE TABLE IF NOT EXISTS ${EQUITY_PRINTS_TABLE} (
source_ts UInt64,
ingest_ts UInt64,
seq UInt64,
trace_id String,
ts UInt64,
underlying_id String,
price Float64,
size UInt32,
exchange String,
offExchangeFlag Bool
)
ENGINE = MergeTree
ORDER BY (ts, underlying_id)
`;
};
export const normalizeEquityPrint = (print: EquityPrint): EquityPrint => {
return print;
};

View file

@ -1,2 +1,3 @@
export * from "./clickhouse";
export * from "./equity-prints";
export * from "./option-prints";

View file

@ -0,0 +1,27 @@
import { describe, expect, it } from "bun:test";
import { equityPrintsTableDDL, EQUITY_PRINTS_TABLE } from "../src/equity-prints";
const basePrint = {
source_ts: 100,
ingest_ts: 200,
seq: 1,
trace_id: "trace-1",
ts: 100,
underlying_id: "SPY",
price: 450.1,
size: 100,
exchange: "TEST",
offExchangeFlag: false
};
describe("equity-prints storage helpers", () => {
it("keeps required fields intact", () => {
expect(basePrint.offExchangeFlag).toBe(false);
});
it("includes the correct table name in the DDL", () => {
const ddl = equityPrintsTableDDL();
expect(ddl).toContain(EQUITY_PRINTS_TABLE);
expect(ddl).toContain("CREATE TABLE IF NOT EXISTS");
});
});