dirtydishes/islandflow
1
0
Fork 0
Code Issues 2 Pull requests 4 Projects Releases Packages Wiki Activity Actions 1

Implement scoped live 24h feed visibility

Browse source
This commit is contained in:
dirtydishes 2026-05-04 05:52:38 -04:00
parent f28c8e641f
commit 48b0d980a6
11 changed files with 547 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

60
packages/storage/src/clickhouse.ts
View file

@ -552,6 +552,14 @@ export type OptionPrintQueryFilters = {
security?: "stock" | "etf" | "all";
optionTypes?: string[];
nbboSides?: string[];
underlyingIds?: string[];
optionContractId?: string;
sinceTs?: number;
};
export type EquityPrintQueryFilters = {
underlyingIds?: string[];
sinceTs?: number;
};
const buildOptionPrintFilterConditions = (
@ -590,6 +598,32 @@ const buildOptionPrintFilterConditions = (
conditions.push(`nbbo_side IN (${buildStringList(filters.nbboSides)})`);
}
if (filters.underlyingIds && filters.underlyingIds.length > 0) {
conditions.push(`underlying_id IN (${buildStringList(filters.underlyingIds)})`);
}
if (filters.optionContractId) {
conditions.push(`option_contract_id = ${quoteString(filters.optionContractId)}`);
}
if (typeof filters.sinceTs === "number" && Number.isFinite(filters.sinceTs)) {
conditions.push(`ts >= ${clampCursor(filters.sinceTs)}`);
}
return conditions;
};
const buildEquityPrintFilterConditions = (filters?: EquityPrintQueryFilters): string[] => {
const conditions: string[] = [];
if (!filters) {
return conditions;
}
if (filters.underlyingIds && filters.underlyingIds.length > 0) {
conditions.push(`underlying_id IN (${buildStringList(filters.underlyingIds)})`);
}
if (typeof filters.sinceTs === "number" && Number.isFinite(filters.sinceTs)) {
conditions.push(`ts >= ${clampCursor(filters.sinceTs)}`);
}
return conditions;
};
@ -798,11 +832,14 @@ export const fetchRecentOptionNBBO = async (
export const fetchRecentEquityPrints = async (
client: ClickHouseClient,
limit: number
limit: number,
filters?: EquityPrintQueryFilters
): Promise<EquityPrint[]> => {
const safeLimit = clampLimit(limit);
const conditions = buildEquityPrintFilterConditions(filters);
const whereClause = conditions.length > 0 ? ` WHERE ${conditions.join(" AND ")}` : "";
const result = await client.query({
query: `SELECT * FROM ${EQUITY_PRINTS_TABLE} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
query: `SELECT * FROM ${EQUITY_PRINTS_TABLE}${whereClause} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
format: "JSONEachRow"
});
@ -983,14 +1020,20 @@ export const fetchEquityPrintsAfter = async (
client: ClickHouseClient,
afterTs: number,
afterSeq: number,
limit: number
limit: number,
filters?: EquityPrintQueryFilters
): Promise<EquityPrint[]> => {
const safeLimit = clampLimit(limit);
const safeAfterTs = clampCursor(afterTs);
const safeAfterSeq = clampCursor(afterSeq);
const conditions = [
`((ts, seq) > (${safeAfterTs}, ${safeAfterSeq}))`,
...buildEquityPrintFilterConditions(filters)
];
const result = await client.query({
query: `SELECT * FROM ${EQUITY_PRINTS_TABLE} WHERE (ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
query: `SELECT * FROM ${EQUITY_PRINTS_TABLE} WHERE ${conditions.join(" AND ")} ORDER BY ts ASC, seq ASC LIMIT ${safeLimit}`,
format: "JSONEachRow"
});
@ -1252,11 +1295,16 @@ export const fetchEquityPrintsBefore = async (
client: ClickHouseClient,
beforeTs: number,
beforeSeq: number,
limit: number
limit: number,
filters?: EquityPrintQueryFilters
): Promise<EquityPrint[]> => {
const safeLimit = clampLimit(limit);
const conditions = [
buildBeforeTupleCondition("ts", "seq", beforeTs, beforeSeq),
...buildEquityPrintFilterConditions(filters)
];
const result = await client.query({
query: `SELECT * FROM ${EQUITY_PRINTS_TABLE} WHERE ${buildBeforeTupleCondition("ts", "seq", beforeTs, beforeSeq)} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
query: `SELECT * FROM ${EQUITY_PRINTS_TABLE} WHERE ${conditions.join(" AND ")} ORDER BY ts DESC, seq DESC LIMIT ${safeLimit}`,
format: "JSONEachRow"
});

41
packages/storage/tests/equity-prints.test.ts
View file

@ -1,4 +1,10 @@
import { describe, expect, it } from "bun:test";
import {
createClickHouseClient,
fetchEquityPrintsAfter,
fetchEquityPrintsBefore,
fetchRecentEquityPrints
} from "../src/clickhouse";
import { equityPrintsTableDDL, EQUITY_PRINTS_TABLE } from "../src/equity-prints";
const basePrint = {
@ -24,4 +30,39 @@ describe("equity-prints storage helpers", () => {
expect(ddl).toContain(EQUITY_PRINTS_TABLE);
expect(ddl).toContain("CREATE TABLE IF NOT EXISTS");
});
it("builds scoped recent, before, and after queries", async () => {
const queries: string[] = [];
const client = createClickHouseClient({ url: "http://127.0.0.1:8123" });
client.query = async ({ query }) => {
queries.push(query);
return {
async json<T>() {
return [] as T;
}
};
};
await fetchRecentEquityPrints(client, 25, {
underlyingIds: ["AAPL", "NVDA"],
sinceTs: 123
});
await fetchEquityPrintsBefore(client, 100, 5, 20, {
underlyingIds: ["AAPL"],
sinceTs: 50
});
await fetchEquityPrintsAfter(client, 100, 5, 20, {
underlyingIds: ["NVDA"],
sinceTs: 50
});
expect(queries[0]).toContain("underlying_id IN ('AAPL', 'NVDA')");
expect(queries[0]).toContain("ts >= 123");
expect(queries[1]).toContain("(ts, seq) < (100, 5)");
expect(queries[1]).toContain("underlying_id IN ('AAPL')");
expect(queries[1]).toContain("ts >= 50");
expect(queries[2]).toContain("((ts, seq) > (100, 5))");
expect(queries[2]).toContain("underlying_id IN ('NVDA')");
expect(queries[2]).toContain("ts >= 50");
});
});

8
packages/storage/tests/option-prints.test.ts
View file

@ -58,7 +58,10 @@ describe("option-prints storage helpers", () => {
security: "stock",
nbboSides: ["AA", "A"],
optionTypes: ["call"],
minNotional: 25_000
minNotional: 25_000,
underlyingIds: ["AAPL", "NVDA"],
optionContractId: "AAPL-2025-01-17-200-C",
sinceTs: 123
});
await fetchOptionPrintsBefore(client, 100, 5, 20, "alpaca");
await fetchOptionPrintsByTraceIds(client, ["trace-1", "trace-2"]);
@ -68,6 +71,9 @@ describe("option-prints storage helpers", () => {
expect(queries[0]).toContain("nbbo_side IN ('AA', 'A')");
expect(queries[0]).toContain("option_type IN ('call')");
expect(queries[0]).toContain("notional >= 25000");
expect(queries[0]).toContain("underlying_id IN ('AAPL', 'NVDA')");
expect(queries[0]).toContain("option_contract_id = 'AAPL-2025-01-17-200-C'");
expect(queries[0]).toContain("ts >= 123");
expect(queries[1]).toContain("(ts, seq) < (100, 5)");
expect(queries[1]).toContain("startsWith(trace_id, 'alpaca')");
expect(queries[1]).toContain("ORDER BY ts DESC, seq DESC LIMIT 20");