Implement smart money event bridge
This commit is contained in:
parent
a8cc2e3875
commit
6822fa1ba4
16 changed files with 1047 additions and 15 deletions
|
|
@ -1,4 +1,4 @@
|
|||
import type { AlertEvent, ClassifierHit } from "@islandflow/types";
|
||||
import type { AlertEvent, ClassifierHit, SmartMoneyProfileScore } from "@islandflow/types";
|
||||
|
||||
export const ALERTS_TABLE = "alerts";
|
||||
|
||||
|
|
@ -11,6 +11,8 @@ export type AlertRecord = {
|
|||
severity: string;
|
||||
hits_json: string;
|
||||
evidence_refs_json: string;
|
||||
primary_profile_id: string;
|
||||
profile_scores_json: string;
|
||||
};
|
||||
|
||||
export const alertsTableDDL = (): string => {
|
||||
|
|
@ -23,13 +25,20 @@ CREATE TABLE IF NOT EXISTS ${ALERTS_TABLE} (
|
|||
score Float64,
|
||||
severity String,
|
||||
hits_json String,
|
||||
evidence_refs_json String
|
||||
evidence_refs_json String,
|
||||
primary_profile_id String DEFAULT '',
|
||||
profile_scores_json String DEFAULT '[]'
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (source_ts, seq)
|
||||
`;
|
||||
};
|
||||
|
||||
export const alertsTableMigrations = (): string[] => [
|
||||
`ALTER TABLE ${ALERTS_TABLE} ADD COLUMN IF NOT EXISTS primary_profile_id String DEFAULT ''`,
|
||||
`ALTER TABLE ${ALERTS_TABLE} ADD COLUMN IF NOT EXISTS profile_scores_json String DEFAULT '[]'`
|
||||
];
|
||||
|
||||
export const toAlertRecord = (alert: AlertEvent): AlertRecord => {
|
||||
return {
|
||||
source_ts: alert.source_ts,
|
||||
|
|
@ -39,7 +48,9 @@ export const toAlertRecord = (alert: AlertEvent): AlertRecord => {
|
|||
score: alert.score,
|
||||
severity: alert.severity,
|
||||
hits_json: JSON.stringify(alert.hits),
|
||||
evidence_refs_json: JSON.stringify(alert.evidence_refs)
|
||||
evidence_refs_json: JSON.stringify(alert.evidence_refs),
|
||||
primary_profile_id: alert.primary_profile_id ?? "",
|
||||
profile_scores_json: JSON.stringify(alert.profile_scores ?? [])
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -79,6 +90,28 @@ const safeStringArray = (value: string): string[] => {
|
|||
return [];
|
||||
};
|
||||
|
||||
const safeProfileScoreArray = (value: string): SmartMoneyProfileScore[] => {
|
||||
try {
|
||||
const parsed = JSON.parse(value);
|
||||
if (Array.isArray(parsed)) {
|
||||
return parsed.map((entry) => {
|
||||
const record = entry as Partial<SmartMoneyProfileScore>;
|
||||
return {
|
||||
profile_id: String(record.profile_id ?? "") as SmartMoneyProfileScore["profile_id"],
|
||||
probability: Number(record.probability ?? 0),
|
||||
confidence_band: String(record.confidence_band ?? "low") as SmartMoneyProfileScore["confidence_band"],
|
||||
direction: String(record.direction ?? "unknown") as SmartMoneyProfileScore["direction"],
|
||||
reasons: Array.isArray(record.reasons) ? record.reasons.map((item) => String(item)) : []
|
||||
};
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
return [];
|
||||
};
|
||||
|
||||
export const fromAlertRecord = (record: AlertRecord): AlertEvent => {
|
||||
return {
|
||||
source_ts: record.source_ts,
|
||||
|
|
@ -88,6 +121,8 @@ export const fromAlertRecord = (record: AlertRecord): AlertEvent => {
|
|||
score: record.score,
|
||||
severity: record.severity,
|
||||
hits: safeHitArray(record.hits_json),
|
||||
evidence_refs: safeStringArray(record.evidence_refs_json)
|
||||
evidence_refs: safeStringArray(record.evidence_refs_json),
|
||||
...(record.primary_profile_id ? { primary_profile_id: record.primary_profile_id as AlertEvent["primary_profile_id"] } : {}),
|
||||
profile_scores: safeProfileScoreArray(record.profile_scores_json)
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ import {
|
|||
InferredDarkEventSchema,
|
||||
FlowPacketSchema,
|
||||
OptionNBBOSchema,
|
||||
OptionPrintSchema
|
||||
OptionPrintSchema,
|
||||
SmartMoneyEventSchema
|
||||
} from "@islandflow/types";
|
||||
import type {
|
||||
AlertEvent,
|
||||
|
|
@ -19,6 +20,7 @@ import type {
|
|||
EquityPrintJoin,
|
||||
InferredDarkEvent,
|
||||
FlowPacket,
|
||||
SmartMoneyEvent,
|
||||
OptionNBBO,
|
||||
OptionPrint,
|
||||
OptionFlowFilters,
|
||||
|
|
@ -76,11 +78,19 @@ import {
|
|||
} from "./classifier-hits";
|
||||
import {
|
||||
ALERTS_TABLE,
|
||||
alertsTableMigrations,
|
||||
alertsTableDDL,
|
||||
fromAlertRecord,
|
||||
toAlertRecord,
|
||||
type AlertRecord
|
||||
} from "./alerts";
|
||||
import {
|
||||
SMART_MONEY_EVENTS_TABLE,
|
||||
smartMoneyEventsTableDDL,
|
||||
fromSmartMoneyEventRecord,
|
||||
toSmartMoneyEventRecord,
|
||||
type SmartMoneyEventRecord
|
||||
} from "./smart-money-events";
|
||||
|
||||
export type ClickHouseOptions = {
|
||||
url: string;
|
||||
|
|
@ -285,6 +295,14 @@ export const ensureFlowPacketsTable = async (
|
|||
});
|
||||
};
|
||||
|
||||
export const ensureSmartMoneyEventsTable = async (
|
||||
client: ClickHouseClient
|
||||
): Promise<void> => {
|
||||
await client.exec({
|
||||
query: smartMoneyEventsTableDDL()
|
||||
});
|
||||
};
|
||||
|
||||
export const ensureClassifierHitsTable = async (
|
||||
client: ClickHouseClient
|
||||
): Promise<void> => {
|
||||
|
|
@ -297,6 +315,9 @@ export const ensureAlertsTable = async (client: ClickHouseClient): Promise<void>
|
|||
await client.exec({
|
||||
query: alertsTableDDL()
|
||||
});
|
||||
for (const query of alertsTableMigrations()) {
|
||||
await client.exec({ query });
|
||||
}
|
||||
};
|
||||
|
||||
export const insertOptionPrint = async (
|
||||
|
|
@ -395,6 +416,18 @@ export const insertFlowPacket = async (
|
|||
});
|
||||
};
|
||||
|
||||
export const insertSmartMoneyEvent = async (
|
||||
client: ClickHouseClient,
|
||||
event: SmartMoneyEvent
|
||||
): Promise<void> => {
|
||||
const record = toSmartMoneyEventRecord(event);
|
||||
await client.insert({
|
||||
table: SMART_MONEY_EVENTS_TABLE,
|
||||
values: [record],
|
||||
format: "JSONEachRow"
|
||||
});
|
||||
};
|
||||
|
||||
export const insertClassifierHit = async (
|
||||
client: ClickHouseClient,
|
||||
hit: ClassifierHitEvent
|
||||
|
|
@ -777,6 +810,34 @@ const normalizeClassifierHitRow = (row: unknown): ClassifierHitRecord | null =>
|
|||
};
|
||||
};
|
||||
|
||||
const normalizeSmartMoneyEventRow = (row: unknown): SmartMoneyEventRecord | null => {
|
||||
if (!row || typeof row !== "object") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const record = row as Record<string, unknown>;
|
||||
return {
|
||||
source_ts: coerceNumber(record.source_ts) as number,
|
||||
ingest_ts: coerceNumber(record.ingest_ts) as number,
|
||||
seq: coerceNumber(record.seq) as number,
|
||||
trace_id: String(record.trace_id ?? ""),
|
||||
event_id: String(record.event_id ?? ""),
|
||||
packet_ids: Array.isArray(record.packet_ids) ? record.packet_ids.map((value) => String(value)) : [],
|
||||
member_print_ids: Array.isArray(record.member_print_ids)
|
||||
? record.member_print_ids.map((value) => String(value))
|
||||
: [],
|
||||
underlying_id: String(record.underlying_id ?? ""),
|
||||
event_kind: String(record.event_kind ?? ""),
|
||||
event_window_ms: coerceNumber(record.event_window_ms) as number,
|
||||
features_json: String(record.features_json ?? "{}"),
|
||||
profile_scores_json: String(record.profile_scores_json ?? "[]"),
|
||||
primary_profile_id: String(record.primary_profile_id ?? ""),
|
||||
primary_direction: String(record.primary_direction ?? "unknown"),
|
||||
abstained: Boolean(record.abstained),
|
||||
suppressed_reasons_json: String(record.suppressed_reasons_json ?? "[]")
|
||||
};
|
||||
};
|
||||
|
||||
const normalizeAlertRow = (row: unknown): AlertRecord | null => {
|
||||
if (!row || typeof row !== "object") {
|
||||
return null;
|
||||
|
|
@ -791,7 +852,9 @@ const normalizeAlertRow = (row: unknown): AlertRecord | null => {
|
|||
score: Number(coerceNumber(record.score) ?? 0),
|
||||
severity: String(record.severity ?? ""),
|
||||
hits_json: String(record.hits_json ?? "[]"),
|
||||
evidence_refs_json: String(record.evidence_refs_json ?? "[]")
|
||||
evidence_refs_json: String(record.evidence_refs_json ?? "[]"),
|
||||
primary_profile_id: String(record.primary_profile_id ?? ""),
|
||||
profile_scores_json: String(record.profile_scores_json ?? "[]")
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -951,6 +1014,23 @@ export const fetchRecentClassifierHits = async (
|
|||
return ClassifierHitEventSchema.array().parse(hits);
|
||||
};
|
||||
|
||||
export const fetchRecentSmartMoneyEvents = async (
|
||||
client: ClickHouseClient,
|
||||
limit: number
|
||||
): Promise<SmartMoneyEvent[]> => {
|
||||
const safeLimit = clampLimit(limit);
|
||||
const result = await client.query({
|
||||
query: `SELECT * FROM ${SMART_MONEY_EVENTS_TABLE} ORDER BY source_ts DESC, seq DESC LIMIT ${safeLimit}`,
|
||||
format: "JSONEachRow"
|
||||
});
|
||||
|
||||
const rows = await result.json<unknown[]>();
|
||||
const records = rows
|
||||
.map(normalizeSmartMoneyEventRow)
|
||||
.filter((record): record is SmartMoneyEventRecord => record !== null);
|
||||
return SmartMoneyEventSchema.array().parse(records.map(fromSmartMoneyEventRecord));
|
||||
};
|
||||
|
||||
export const fetchRecentAlerts = async (
|
||||
client: ClickHouseClient,
|
||||
limit: number
|
||||
|
|
@ -1222,6 +1302,28 @@ export const fetchClassifierHitsAfter = async (
|
|||
return ClassifierHitEventSchema.array().parse(hits);
|
||||
};
|
||||
|
||||
export const fetchSmartMoneyEventsAfter = async (
|
||||
client: ClickHouseClient,
|
||||
afterTs: number,
|
||||
afterSeq: number,
|
||||
limit: number
|
||||
): Promise<SmartMoneyEvent[]> => {
|
||||
const safeLimit = clampLimit(limit);
|
||||
const safeAfterTs = clampCursor(afterTs);
|
||||
const safeAfterSeq = clampCursor(afterSeq);
|
||||
|
||||
const result = await client.query({
|
||||
query: `SELECT * FROM ${SMART_MONEY_EVENTS_TABLE} WHERE (source_ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY source_ts ASC, seq ASC LIMIT ${safeLimit}`,
|
||||
format: "JSONEachRow"
|
||||
});
|
||||
|
||||
const rows = await result.json<unknown[]>();
|
||||
const records = rows
|
||||
.map(normalizeSmartMoneyEventRow)
|
||||
.filter((record): record is SmartMoneyEventRecord => record !== null);
|
||||
return SmartMoneyEventSchema.array().parse(records.map(fromSmartMoneyEventRecord));
|
||||
};
|
||||
|
||||
export const fetchAlertsAfter = async (
|
||||
client: ClickHouseClient,
|
||||
afterTs: number,
|
||||
|
|
@ -1385,6 +1487,25 @@ export const fetchClassifierHitsBefore = async (
|
|||
return ClassifierHitEventSchema.array().parse(records.map(fromClassifierHitRecord));
|
||||
};
|
||||
|
||||
export const fetchSmartMoneyEventsBefore = async (
|
||||
client: ClickHouseClient,
|
||||
beforeTs: number,
|
||||
beforeSeq: number,
|
||||
limit: number
|
||||
): Promise<SmartMoneyEvent[]> => {
|
||||
const safeLimit = clampLimit(limit);
|
||||
const result = await client.query({
|
||||
query: `SELECT * FROM ${SMART_MONEY_EVENTS_TABLE} WHERE ${buildBeforeTupleCondition("source_ts", "seq", beforeTs, beforeSeq)} ORDER BY source_ts DESC, seq DESC LIMIT ${safeLimit}`,
|
||||
format: "JSONEachRow"
|
||||
});
|
||||
|
||||
const rows = await result.json<unknown[]>();
|
||||
const records = rows
|
||||
.map(normalizeSmartMoneyEventRow)
|
||||
.filter((record): record is SmartMoneyEventRecord => record !== null);
|
||||
return SmartMoneyEventSchema.array().parse(records.map(fromSmartMoneyEventRecord));
|
||||
};
|
||||
|
||||
export const fetchAlertsBefore = async (
|
||||
client: ClickHouseClient,
|
||||
beforeTs: number,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ export * from "./clickhouse";
|
|||
export * from "./classifier-hits";
|
||||
export * from "./alerts";
|
||||
export * from "./flow-packets";
|
||||
export * from "./smart-money-events";
|
||||
export * from "./equity-prints";
|
||||
export * from "./equity-quotes";
|
||||
export * from "./equity-candles";
|
||||
|
|
|
|||
100
packages/storage/src/smart-money-events.ts
Normal file
100
packages/storage/src/smart-money-events.ts
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
import type { SmartMoneyEvent } from "@islandflow/types";
|
||||
|
||||
export const SMART_MONEY_EVENTS_TABLE = "smart_money_events";
|
||||
|
||||
export type SmartMoneyEventRecord = {
|
||||
source_ts: number;
|
||||
ingest_ts: number;
|
||||
seq: number;
|
||||
trace_id: string;
|
||||
event_id: string;
|
||||
packet_ids: string[];
|
||||
member_print_ids: string[];
|
||||
underlying_id: string;
|
||||
event_kind: string;
|
||||
event_window_ms: number;
|
||||
features_json: string;
|
||||
profile_scores_json: string;
|
||||
primary_profile_id: string;
|
||||
primary_direction: string;
|
||||
abstained: boolean;
|
||||
suppressed_reasons_json: string;
|
||||
};
|
||||
|
||||
export const smartMoneyEventsTableDDL = (): string => {
|
||||
return `
|
||||
CREATE TABLE IF NOT EXISTS ${SMART_MONEY_EVENTS_TABLE} (
|
||||
source_ts UInt64,
|
||||
ingest_ts UInt64,
|
||||
seq UInt64,
|
||||
trace_id String,
|
||||
event_id String,
|
||||
packet_ids Array(String),
|
||||
member_print_ids Array(String),
|
||||
underlying_id String,
|
||||
event_kind String,
|
||||
event_window_ms UInt64,
|
||||
features_json String,
|
||||
profile_scores_json String,
|
||||
primary_profile_id String,
|
||||
primary_direction String,
|
||||
abstained Bool,
|
||||
suppressed_reasons_json String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (source_ts, seq)
|
||||
`;
|
||||
};
|
||||
|
||||
export const toSmartMoneyEventRecord = (event: SmartMoneyEvent): SmartMoneyEventRecord => {
|
||||
return {
|
||||
source_ts: event.source_ts,
|
||||
ingest_ts: event.ingest_ts,
|
||||
seq: event.seq,
|
||||
trace_id: event.trace_id,
|
||||
event_id: event.event_id,
|
||||
packet_ids: event.packet_ids,
|
||||
member_print_ids: event.member_print_ids,
|
||||
underlying_id: event.underlying_id,
|
||||
event_kind: event.event_kind,
|
||||
event_window_ms: event.event_window_ms,
|
||||
features_json: JSON.stringify(event.features),
|
||||
profile_scores_json: JSON.stringify(event.profile_scores),
|
||||
primary_profile_id: event.primary_profile_id ?? "",
|
||||
primary_direction: event.primary_direction,
|
||||
abstained: event.abstained,
|
||||
suppressed_reasons_json: JSON.stringify(event.suppressed_reasons)
|
||||
};
|
||||
};
|
||||
|
||||
const safeJson = <T>(value: string, fallback: T): T => {
|
||||
try {
|
||||
return JSON.parse(value) as T;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
};
|
||||
|
||||
export const fromSmartMoneyEventRecord = (record: SmartMoneyEventRecord): SmartMoneyEvent => {
|
||||
const primaryProfileId = record.primary_profile_id.trim();
|
||||
return {
|
||||
source_ts: record.source_ts,
|
||||
ingest_ts: record.ingest_ts,
|
||||
seq: record.seq,
|
||||
trace_id: record.trace_id,
|
||||
event_id: record.event_id,
|
||||
packet_ids: record.packet_ids,
|
||||
member_print_ids: record.member_print_ids,
|
||||
underlying_id: record.underlying_id,
|
||||
event_kind: record.event_kind as SmartMoneyEvent["event_kind"],
|
||||
event_window_ms: record.event_window_ms,
|
||||
features: safeJson(record.features_json, {} as SmartMoneyEvent["features"]),
|
||||
profile_scores: safeJson(record.profile_scores_json, [] as SmartMoneyEvent["profile_scores"]),
|
||||
primary_profile_id: primaryProfileId
|
||||
? (primaryProfileId as SmartMoneyEvent["primary_profile_id"])
|
||||
: null,
|
||||
primary_direction: record.primary_direction as SmartMoneyEvent["primary_direction"],
|
||||
abstained: Boolean(record.abstained),
|
||||
suppressed_reasons: safeJson(record.suppressed_reasons_json, [] as string[])
|
||||
};
|
||||
};
|
||||
Loading…
Add table
Add a link
Reference in a new issue