Add classifier hits and alerts pipeline
Add NATS subjects + ClickHouse tables for classifier hits/alerts, evaluate sweep/spike rules in compute, expose API/WS endpoints, and cover storage helpers with tests.
This commit is contained in:
parent
ad58c62c37
commit
58485b4d97
11 changed files with 861 additions and 8 deletions
|
|
@ -48,3 +48,7 @@ EMIT_INTERVAL_MS=1000
|
||||||
# Compute consumer behavior
|
# Compute consumer behavior
|
||||||
COMPUTE_DELIVER_POLICY=new
|
COMPUTE_DELIVER_POLICY=new
|
||||||
COMPUTE_CONSUMER_RESET=false
|
COMPUTE_CONSUMER_RESET=false
|
||||||
|
CLASSIFIER_SWEEP_MIN_PREMIUM=50000
|
||||||
|
CLASSIFIER_SWEEP_MIN_COUNT=3
|
||||||
|
CLASSIFIER_SPIKE_MIN_PREMIUM=25000
|
||||||
|
CLASSIFIER_SPIKE_MIN_SIZE=500
|
||||||
|
|
|
||||||
|
|
@ -4,3 +4,7 @@ export const STREAM_EQUITY_PRINTS = "EQUITY_PRINTS";
|
||||||
export const SUBJECT_EQUITY_PRINTS = "equities.prints";
|
export const SUBJECT_EQUITY_PRINTS = "equities.prints";
|
||||||
export const STREAM_FLOW_PACKETS = "FLOW_PACKETS";
|
export const STREAM_FLOW_PACKETS = "FLOW_PACKETS";
|
||||||
export const SUBJECT_FLOW_PACKETS = "flow.packets";
|
export const SUBJECT_FLOW_PACKETS = "flow.packets";
|
||||||
|
export const STREAM_CLASSIFIER_HITS = "CLASSIFIER_HITS";
|
||||||
|
export const SUBJECT_CLASSIFIER_HITS = "flow.classifier_hits";
|
||||||
|
export const STREAM_ALERTS = "ALERTS";
|
||||||
|
export const SUBJECT_ALERTS = "flow.alerts";
|
||||||
|
|
|
||||||
93
packages/storage/src/alerts.ts
Normal file
93
packages/storage/src/alerts.ts
Normal file
|
|
@ -0,0 +1,93 @@
|
||||||
|
import type { AlertEvent, ClassifierHit } from "@islandflow/types";
|
||||||
|
|
||||||
|
export const ALERTS_TABLE = "alerts";
|
||||||
|
|
||||||
|
export type AlertRecord = {
|
||||||
|
source_ts: number;
|
||||||
|
ingest_ts: number;
|
||||||
|
seq: number;
|
||||||
|
trace_id: string;
|
||||||
|
score: number;
|
||||||
|
severity: string;
|
||||||
|
hits_json: string;
|
||||||
|
evidence_refs_json: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const alertsTableDDL = (): string => {
|
||||||
|
return `
|
||||||
|
CREATE TABLE IF NOT EXISTS ${ALERTS_TABLE} (
|
||||||
|
source_ts UInt64,
|
||||||
|
ingest_ts UInt64,
|
||||||
|
seq UInt64,
|
||||||
|
trace_id String,
|
||||||
|
score Float64,
|
||||||
|
severity String,
|
||||||
|
hits_json String,
|
||||||
|
evidence_refs_json String
|
||||||
|
)
|
||||||
|
ENGINE = MergeTree
|
||||||
|
ORDER BY (source_ts, seq)
|
||||||
|
`;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const toAlertRecord = (alert: AlertEvent): AlertRecord => {
|
||||||
|
return {
|
||||||
|
source_ts: alert.source_ts,
|
||||||
|
ingest_ts: alert.ingest_ts,
|
||||||
|
seq: alert.seq,
|
||||||
|
trace_id: alert.trace_id,
|
||||||
|
score: alert.score,
|
||||||
|
severity: alert.severity,
|
||||||
|
hits_json: JSON.stringify(alert.hits),
|
||||||
|
evidence_refs_json: JSON.stringify(alert.evidence_refs)
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const safeHitArray = (value: string): ClassifierHit[] => {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(value);
|
||||||
|
if (Array.isArray(parsed)) {
|
||||||
|
return parsed.map((entry) => {
|
||||||
|
const record = entry as Partial<ClassifierHit>;
|
||||||
|
return {
|
||||||
|
classifier_id: String(record.classifier_id ?? ""),
|
||||||
|
confidence: Number(record.confidence ?? 0),
|
||||||
|
direction: String(record.direction ?? ""),
|
||||||
|
explanations: Array.isArray(record.explanations)
|
||||||
|
? record.explanations.map((item) => String(item))
|
||||||
|
: []
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
|
||||||
|
return [];
|
||||||
|
};
|
||||||
|
|
||||||
|
const safeStringArray = (value: string): string[] => {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(value);
|
||||||
|
if (Array.isArray(parsed)) {
|
||||||
|
return parsed.map((entry) => String(entry));
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
|
||||||
|
return [];
|
||||||
|
};
|
||||||
|
|
||||||
|
export const fromAlertRecord = (record: AlertRecord): AlertEvent => {
|
||||||
|
return {
|
||||||
|
source_ts: record.source_ts,
|
||||||
|
ingest_ts: record.ingest_ts,
|
||||||
|
seq: record.seq,
|
||||||
|
trace_id: record.trace_id,
|
||||||
|
score: record.score,
|
||||||
|
severity: record.severity,
|
||||||
|
hits: safeHitArray(record.hits_json),
|
||||||
|
evidence_refs: safeStringArray(record.evidence_refs_json)
|
||||||
|
};
|
||||||
|
};
|
||||||
70
packages/storage/src/classifier-hits.ts
Normal file
70
packages/storage/src/classifier-hits.ts
Normal file
|
|
@ -0,0 +1,70 @@
|
||||||
|
import type { ClassifierHitEvent } from "@islandflow/types";
|
||||||
|
|
||||||
|
export const CLASSIFIER_HITS_TABLE = "classifier_hits";
|
||||||
|
|
||||||
|
export type ClassifierHitRecord = {
|
||||||
|
source_ts: number;
|
||||||
|
ingest_ts: number;
|
||||||
|
seq: number;
|
||||||
|
trace_id: string;
|
||||||
|
classifier_id: string;
|
||||||
|
confidence: number;
|
||||||
|
direction: string;
|
||||||
|
explanations_json: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const classifierHitsTableDDL = (): string => {
|
||||||
|
return `
|
||||||
|
CREATE TABLE IF NOT EXISTS ${CLASSIFIER_HITS_TABLE} (
|
||||||
|
source_ts UInt64,
|
||||||
|
ingest_ts UInt64,
|
||||||
|
seq UInt64,
|
||||||
|
trace_id String,
|
||||||
|
classifier_id String,
|
||||||
|
confidence Float64,
|
||||||
|
direction String,
|
||||||
|
explanations_json String
|
||||||
|
)
|
||||||
|
ENGINE = MergeTree
|
||||||
|
ORDER BY (source_ts, seq)
|
||||||
|
`;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const toClassifierHitRecord = (hit: ClassifierHitEvent): ClassifierHitRecord => {
|
||||||
|
return {
|
||||||
|
source_ts: hit.source_ts,
|
||||||
|
ingest_ts: hit.ingest_ts,
|
||||||
|
seq: hit.seq,
|
||||||
|
trace_id: hit.trace_id,
|
||||||
|
classifier_id: hit.classifier_id,
|
||||||
|
confidence: hit.confidence,
|
||||||
|
direction: hit.direction,
|
||||||
|
explanations_json: JSON.stringify(hit.explanations)
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const safeJsonArray = (value: string): string[] => {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(value);
|
||||||
|
if (Array.isArray(parsed)) {
|
||||||
|
return parsed.map((entry) => String(entry));
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
|
||||||
|
return [];
|
||||||
|
};
|
||||||
|
|
||||||
|
export const fromClassifierHitRecord = (record: ClassifierHitRecord): ClassifierHitEvent => {
|
||||||
|
return {
|
||||||
|
source_ts: record.source_ts,
|
||||||
|
ingest_ts: record.ingest_ts,
|
||||||
|
seq: record.seq,
|
||||||
|
trace_id: record.trace_id,
|
||||||
|
classifier_id: record.classifier_id,
|
||||||
|
confidence: record.confidence,
|
||||||
|
direction: record.direction,
|
||||||
|
explanations: safeJsonArray(record.explanations_json)
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
@ -1,6 +1,18 @@
|
||||||
import { createClient, type ClickHouseClient } from "@clickhouse/client";
|
import { createClient, type ClickHouseClient } from "@clickhouse/client";
|
||||||
import { EquityPrintSchema, FlowPacketSchema, OptionPrintSchema } from "@islandflow/types";
|
import {
|
||||||
import type { EquityPrint, FlowPacket, OptionPrint } from "@islandflow/types";
|
AlertEventSchema,
|
||||||
|
ClassifierHitEventSchema,
|
||||||
|
EquityPrintSchema,
|
||||||
|
FlowPacketSchema,
|
||||||
|
OptionPrintSchema
|
||||||
|
} from "@islandflow/types";
|
||||||
|
import type {
|
||||||
|
AlertEvent,
|
||||||
|
ClassifierHitEvent,
|
||||||
|
EquityPrint,
|
||||||
|
FlowPacket,
|
||||||
|
OptionPrint
|
||||||
|
} from "@islandflow/types";
|
||||||
import {
|
import {
|
||||||
normalizeOptionPrint,
|
normalizeOptionPrint,
|
||||||
optionPrintsTableDDL,
|
optionPrintsTableDDL,
|
||||||
|
|
@ -18,6 +30,20 @@ import {
|
||||||
toFlowPacketRecord,
|
toFlowPacketRecord,
|
||||||
type FlowPacketRecord
|
type FlowPacketRecord
|
||||||
} from "./flow-packets";
|
} from "./flow-packets";
|
||||||
|
import {
|
||||||
|
CLASSIFIER_HITS_TABLE,
|
||||||
|
classifierHitsTableDDL,
|
||||||
|
fromClassifierHitRecord,
|
||||||
|
toClassifierHitRecord,
|
||||||
|
type ClassifierHitRecord
|
||||||
|
} from "./classifier-hits";
|
||||||
|
import {
|
||||||
|
ALERTS_TABLE,
|
||||||
|
alertsTableDDL,
|
||||||
|
fromAlertRecord,
|
||||||
|
toAlertRecord,
|
||||||
|
type AlertRecord
|
||||||
|
} from "./alerts";
|
||||||
|
|
||||||
export type ClickHouseOptions = {
|
export type ClickHouseOptions = {
|
||||||
url: string;
|
url: string;
|
||||||
|
|
@ -59,6 +85,20 @@ export const ensureFlowPacketsTable = async (
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const ensureClassifierHitsTable = async (
|
||||||
|
client: ClickHouseClient
|
||||||
|
): Promise<void> => {
|
||||||
|
await client.exec({
|
||||||
|
query: classifierHitsTableDDL()
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
export const ensureAlertsTable = async (client: ClickHouseClient): Promise<void> => {
|
||||||
|
await client.exec({
|
||||||
|
query: alertsTableDDL()
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
export const insertOptionPrint = async (
|
export const insertOptionPrint = async (
|
||||||
client: ClickHouseClient,
|
client: ClickHouseClient,
|
||||||
print: OptionPrint
|
print: OptionPrint
|
||||||
|
|
@ -95,6 +135,27 @@ export const insertFlowPacket = async (
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const insertClassifierHit = async (
|
||||||
|
client: ClickHouseClient,
|
||||||
|
hit: ClassifierHitEvent
|
||||||
|
): Promise<void> => {
|
||||||
|
const record = toClassifierHitRecord(hit);
|
||||||
|
await client.insert({
|
||||||
|
table: CLASSIFIER_HITS_TABLE,
|
||||||
|
values: [record],
|
||||||
|
format: "JSONEachRow"
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
export const insertAlert = async (client: ClickHouseClient, alert: AlertEvent): Promise<void> => {
|
||||||
|
const record = toAlertRecord(alert);
|
||||||
|
await client.insert({
|
||||||
|
table: ALERTS_TABLE,
|
||||||
|
values: [record],
|
||||||
|
format: "JSONEachRow"
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
const clampLimit = (limit: number): number => {
|
const clampLimit = (limit: number): number => {
|
||||||
if (!Number.isFinite(limit)) {
|
if (!Number.isFinite(limit)) {
|
||||||
return 100;
|
return 100;
|
||||||
|
|
@ -196,6 +257,42 @@ const normalizeFlowPacketRow = (row: unknown): FlowPacketRecord | null => {
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const normalizeClassifierHitRow = (row: unknown): ClassifierHitRecord | null => {
|
||||||
|
if (!row || typeof row !== "object") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const record = row as Record<string, unknown>;
|
||||||
|
return {
|
||||||
|
source_ts: coerceNumber(record.source_ts) as number,
|
||||||
|
ingest_ts: coerceNumber(record.ingest_ts) as number,
|
||||||
|
seq: coerceNumber(record.seq) as number,
|
||||||
|
trace_id: String(record.trace_id ?? ""),
|
||||||
|
classifier_id: String(record.classifier_id ?? ""),
|
||||||
|
confidence: Number(coerceNumber(record.confidence) ?? 0),
|
||||||
|
direction: String(record.direction ?? ""),
|
||||||
|
explanations_json: String(record.explanations_json ?? "[]")
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const normalizeAlertRow = (row: unknown): AlertRecord | null => {
|
||||||
|
if (!row || typeof row !== "object") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const record = row as Record<string, unknown>;
|
||||||
|
return {
|
||||||
|
source_ts: coerceNumber(record.source_ts) as number,
|
||||||
|
ingest_ts: coerceNumber(record.ingest_ts) as number,
|
||||||
|
seq: coerceNumber(record.seq) as number,
|
||||||
|
trace_id: String(record.trace_id ?? ""),
|
||||||
|
score: Number(coerceNumber(record.score) ?? 0),
|
||||||
|
severity: String(record.severity ?? ""),
|
||||||
|
hits_json: String(record.hits_json ?? "[]"),
|
||||||
|
evidence_refs_json: String(record.evidence_refs_json ?? "[]")
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
export const fetchRecentOptionPrints = async (
|
export const fetchRecentOptionPrints = async (
|
||||||
client: ClickHouseClient,
|
client: ClickHouseClient,
|
||||||
limit: number
|
limit: number
|
||||||
|
|
@ -242,6 +339,42 @@ export const fetchRecentFlowPackets = async (
|
||||||
return FlowPacketSchema.array().parse(packets);
|
return FlowPacketSchema.array().parse(packets);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const fetchRecentClassifierHits = async (
|
||||||
|
client: ClickHouseClient,
|
||||||
|
limit: number
|
||||||
|
): Promise<ClassifierHitEvent[]> => {
|
||||||
|
const safeLimit = clampLimit(limit);
|
||||||
|
const result = await client.query({
|
||||||
|
query: `SELECT * FROM ${CLASSIFIER_HITS_TABLE} ORDER BY source_ts DESC, seq DESC LIMIT ${safeLimit}`,
|
||||||
|
format: "JSONEachRow"
|
||||||
|
});
|
||||||
|
|
||||||
|
const rows = await result.json<unknown[]>();
|
||||||
|
const records = rows
|
||||||
|
.map(normalizeClassifierHitRow)
|
||||||
|
.filter((record): record is ClassifierHitRecord => record !== null);
|
||||||
|
const hits = records.map(fromClassifierHitRecord);
|
||||||
|
return ClassifierHitEventSchema.array().parse(hits);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const fetchRecentAlerts = async (
|
||||||
|
client: ClickHouseClient,
|
||||||
|
limit: number
|
||||||
|
): Promise<AlertEvent[]> => {
|
||||||
|
const safeLimit = clampLimit(limit);
|
||||||
|
const result = await client.query({
|
||||||
|
query: `SELECT * FROM ${ALERTS_TABLE} ORDER BY source_ts DESC, seq DESC LIMIT ${safeLimit}`,
|
||||||
|
format: "JSONEachRow"
|
||||||
|
});
|
||||||
|
|
||||||
|
const rows = await result.json<unknown[]>();
|
||||||
|
const records = rows
|
||||||
|
.map(normalizeAlertRow)
|
||||||
|
.filter((record): record is AlertRecord => record !== null);
|
||||||
|
const alerts = records.map(fromAlertRecord);
|
||||||
|
return AlertEventSchema.array().parse(alerts);
|
||||||
|
};
|
||||||
|
|
||||||
export const fetchOptionPrintsAfter = async (
|
export const fetchOptionPrintsAfter = async (
|
||||||
client: ClickHouseClient,
|
client: ClickHouseClient,
|
||||||
afterTs: number,
|
afterTs: number,
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
export * from "./clickhouse";
|
export * from "./clickhouse";
|
||||||
|
export * from "./classifier-hits";
|
||||||
|
export * from "./alerts";
|
||||||
export * from "./flow-packets";
|
export * from "./flow-packets";
|
||||||
export * from "./equity-prints";
|
export * from "./equity-prints";
|
||||||
export * from "./option-prints";
|
export * from "./option-prints";
|
||||||
|
|
|
||||||
36
packages/storage/tests/alerts.test.ts
Normal file
36
packages/storage/tests/alerts.test.ts
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
import { describe, expect, it } from "bun:test";
|
||||||
|
import { alertsTableDDL, ALERTS_TABLE, fromAlertRecord, toAlertRecord } from "../src/alerts";
|
||||||
|
|
||||||
|
const alert = {
|
||||||
|
source_ts: 10,
|
||||||
|
ingest_ts: 20,
|
||||||
|
seq: 1,
|
||||||
|
trace_id: "alert:fp-1",
|
||||||
|
score: 78,
|
||||||
|
severity: "medium",
|
||||||
|
hits: [
|
||||||
|
{
|
||||||
|
classifier_id: "large_bullish_call_sweep",
|
||||||
|
confidence: 0.72,
|
||||||
|
direction: "bullish",
|
||||||
|
explanations: ["Likely call sweep.", "Premium $50000."]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
evidence_refs: ["flowpacket:1", "print:1"]
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("alerts storage helpers", () => {
|
||||||
|
it("includes the correct table name in the DDL", () => {
|
||||||
|
const ddl = alertsTableDDL();
|
||||||
|
expect(ddl).toContain(ALERTS_TABLE);
|
||||||
|
expect(ddl).toContain("CREATE TABLE IF NOT EXISTS");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("round-trips alert records", () => {
|
||||||
|
const record = toAlertRecord(alert);
|
||||||
|
const restored = fromAlertRecord(record);
|
||||||
|
expect(restored.hits).toEqual(alert.hits);
|
||||||
|
expect(restored.evidence_refs).toEqual(alert.evidence_refs);
|
||||||
|
expect(restored.severity).toBe(alert.severity);
|
||||||
|
});
|
||||||
|
});
|
||||||
34
packages/storage/tests/classifier-hits.test.ts
Normal file
34
packages/storage/tests/classifier-hits.test.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
import { describe, expect, it } from "bun:test";
|
||||||
|
import {
|
||||||
|
classifierHitsTableDDL,
|
||||||
|
CLASSIFIER_HITS_TABLE,
|
||||||
|
fromClassifierHitRecord,
|
||||||
|
toClassifierHitRecord
|
||||||
|
} from "../src/classifier-hits";
|
||||||
|
|
||||||
|
const hit = {
|
||||||
|
source_ts: 10,
|
||||||
|
ingest_ts: 20,
|
||||||
|
seq: 1,
|
||||||
|
trace_id: "classifier:large_bullish_call_sweep:fp-1",
|
||||||
|
classifier_id: "large_bullish_call_sweep",
|
||||||
|
confidence: 0.72,
|
||||||
|
direction: "bullish",
|
||||||
|
explanations: ["Likely call sweep.", "Premium $50000."]
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("classifier hits storage helpers", () => {
|
||||||
|
it("includes the correct table name in the DDL", () => {
|
||||||
|
const ddl = classifierHitsTableDDL();
|
||||||
|
expect(ddl).toContain(CLASSIFIER_HITS_TABLE);
|
||||||
|
expect(ddl).toContain("CREATE TABLE IF NOT EXISTS");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("round-trips classifier hit records", () => {
|
||||||
|
const record = toClassifierHitRecord(hit);
|
||||||
|
const restored = fromClassifierHitRecord(record);
|
||||||
|
expect(restored.explanations).toEqual(hit.explanations);
|
||||||
|
expect(restored.classifier_id).toBe(hit.classifier_id);
|
||||||
|
expect(restored.direction).toBe(hit.direction);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -1,9 +1,13 @@
|
||||||
import { readEnv } from "@islandflow/config";
|
import { readEnv } from "@islandflow/config";
|
||||||
import { createLogger } from "@islandflow/observability";
|
import { createLogger } from "@islandflow/observability";
|
||||||
import {
|
import {
|
||||||
|
SUBJECT_ALERTS,
|
||||||
|
SUBJECT_CLASSIFIER_HITS,
|
||||||
SUBJECT_EQUITY_PRINTS,
|
SUBJECT_EQUITY_PRINTS,
|
||||||
SUBJECT_FLOW_PACKETS,
|
SUBJECT_FLOW_PACKETS,
|
||||||
SUBJECT_OPTION_PRINTS,
|
SUBJECT_OPTION_PRINTS,
|
||||||
|
STREAM_ALERTS,
|
||||||
|
STREAM_CLASSIFIER_HITS,
|
||||||
STREAM_EQUITY_PRINTS,
|
STREAM_EQUITY_PRINTS,
|
||||||
STREAM_FLOW_PACKETS,
|
STREAM_FLOW_PACKETS,
|
||||||
STREAM_OPTION_PRINTS,
|
STREAM_OPTION_PRINTS,
|
||||||
|
|
@ -14,16 +18,26 @@ import {
|
||||||
} from "@islandflow/bus";
|
} from "@islandflow/bus";
|
||||||
import {
|
import {
|
||||||
createClickHouseClient,
|
createClickHouseClient,
|
||||||
|
ensureAlertsTable,
|
||||||
|
ensureClassifierHitsTable,
|
||||||
ensureEquityPrintsTable,
|
ensureEquityPrintsTable,
|
||||||
ensureFlowPacketsTable,
|
ensureFlowPacketsTable,
|
||||||
ensureOptionPrintsTable,
|
ensureOptionPrintsTable,
|
||||||
|
fetchRecentAlerts,
|
||||||
|
fetchRecentClassifierHits,
|
||||||
fetchRecentFlowPackets,
|
fetchRecentFlowPackets,
|
||||||
fetchEquityPrintsAfter,
|
fetchEquityPrintsAfter,
|
||||||
fetchRecentEquityPrints,
|
fetchRecentEquityPrints,
|
||||||
fetchOptionPrintsAfter,
|
fetchOptionPrintsAfter,
|
||||||
fetchRecentOptionPrints
|
fetchRecentOptionPrints
|
||||||
} from "@islandflow/storage";
|
} from "@islandflow/storage";
|
||||||
import { EquityPrintSchema, FlowPacketSchema, OptionPrintSchema } from "@islandflow/types";
|
import {
|
||||||
|
AlertEventSchema,
|
||||||
|
ClassifierHitEventSchema,
|
||||||
|
EquityPrintSchema,
|
||||||
|
FlowPacketSchema,
|
||||||
|
OptionPrintSchema
|
||||||
|
} from "@islandflow/types";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
|
||||||
const service = "api";
|
const service = "api";
|
||||||
|
|
@ -73,7 +87,7 @@ const replayParamsSchema = z.object({
|
||||||
limit: z.coerce.number().int().positive().max(1000).default(200)
|
limit: z.coerce.number().int().positive().max(1000).default(200)
|
||||||
});
|
});
|
||||||
|
|
||||||
type Channel = "options" | "equities" | "flow";
|
type Channel = "options" | "equities" | "flow" | "classifier-hits" | "alerts";
|
||||||
|
|
||||||
type WsData = {
|
type WsData = {
|
||||||
channel: Channel;
|
channel: Channel;
|
||||||
|
|
@ -82,6 +96,8 @@ type WsData = {
|
||||||
const optionSockets = new Set<WebSocket<WsData>>();
|
const optionSockets = new Set<WebSocket<WsData>>();
|
||||||
const equitySockets = new Set<WebSocket<WsData>>();
|
const equitySockets = new Set<WebSocket<WsData>>();
|
||||||
const flowSockets = new Set<WebSocket<WsData>>();
|
const flowSockets = new Set<WebSocket<WsData>>();
|
||||||
|
const classifierHitSockets = new Set<WebSocket<WsData>>();
|
||||||
|
const alertSockets = new Set<WebSocket<WsData>>();
|
||||||
|
|
||||||
const jsonResponse = (body: unknown, status = 200): Response => {
|
const jsonResponse = (body: unknown, status = 200): Response => {
|
||||||
return new Response(JSON.stringify(body), {
|
return new Response(JSON.stringify(body), {
|
||||||
|
|
@ -179,6 +195,32 @@ const run = async () => {
|
||||||
num_replicas: 1
|
num_replicas: 1
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await ensureStream(jsm, {
|
||||||
|
name: STREAM_CLASSIFIER_HITS,
|
||||||
|
subjects: [SUBJECT_CLASSIFIER_HITS],
|
||||||
|
retention: "limits",
|
||||||
|
storage: "file",
|
||||||
|
discard: "old",
|
||||||
|
max_msgs_per_subject: -1,
|
||||||
|
max_msgs: -1,
|
||||||
|
max_bytes: -1,
|
||||||
|
max_age: 0,
|
||||||
|
num_replicas: 1
|
||||||
|
});
|
||||||
|
|
||||||
|
await ensureStream(jsm, {
|
||||||
|
name: STREAM_ALERTS,
|
||||||
|
subjects: [SUBJECT_ALERTS],
|
||||||
|
retention: "limits",
|
||||||
|
storage: "file",
|
||||||
|
discard: "old",
|
||||||
|
max_msgs_per_subject: -1,
|
||||||
|
max_msgs: -1,
|
||||||
|
max_bytes: -1,
|
||||||
|
max_age: 0,
|
||||||
|
num_replicas: 1
|
||||||
|
});
|
||||||
|
|
||||||
const clickhouse = createClickHouseClient({
|
const clickhouse = createClickHouseClient({
|
||||||
url: env.CLICKHOUSE_URL,
|
url: env.CLICKHOUSE_URL,
|
||||||
database: env.CLICKHOUSE_DATABASE
|
database: env.CLICKHOUSE_DATABASE
|
||||||
|
|
@ -188,6 +230,8 @@ const run = async () => {
|
||||||
await ensureOptionPrintsTable(clickhouse);
|
await ensureOptionPrintsTable(clickhouse);
|
||||||
await ensureEquityPrintsTable(clickhouse);
|
await ensureEquityPrintsTable(clickhouse);
|
||||||
await ensureFlowPacketsTable(clickhouse);
|
await ensureFlowPacketsTable(clickhouse);
|
||||||
|
await ensureClassifierHitsTable(clickhouse);
|
||||||
|
await ensureAlertsTable(clickhouse);
|
||||||
});
|
});
|
||||||
|
|
||||||
const optionSubscription = await subscribeJson(
|
const optionSubscription = await subscribeJson(
|
||||||
|
|
@ -208,6 +252,18 @@ const run = async () => {
|
||||||
buildDurableConsumer("api-flow-packets")
|
buildDurableConsumer("api-flow-packets")
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const classifierHitSubscription = await subscribeJson(
|
||||||
|
js,
|
||||||
|
SUBJECT_CLASSIFIER_HITS,
|
||||||
|
buildDurableConsumer("api-classifier-hits")
|
||||||
|
);
|
||||||
|
|
||||||
|
const alertSubscription = await subscribeJson(
|
||||||
|
js,
|
||||||
|
SUBJECT_ALERTS,
|
||||||
|
buildDurableConsumer("api-alerts")
|
||||||
|
);
|
||||||
|
|
||||||
const pumpOptions = async () => {
|
const pumpOptions = async () => {
|
||||||
for await (const msg of optionSubscription.messages) {
|
for await (const msg of optionSubscription.messages) {
|
||||||
try {
|
try {
|
||||||
|
|
@ -253,9 +309,41 @@ const run = async () => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const pumpClassifierHits = async () => {
|
||||||
|
for await (const msg of classifierHitSubscription.messages) {
|
||||||
|
try {
|
||||||
|
const payload = ClassifierHitEventSchema.parse(classifierHitSubscription.decode(msg));
|
||||||
|
broadcast(classifierHitSockets, { type: "classifier-hit", payload });
|
||||||
|
msg.ack();
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("failed to process classifier hit", {
|
||||||
|
error: error instanceof Error ? error.message : String(error)
|
||||||
|
});
|
||||||
|
msg.term();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const pumpAlerts = async () => {
|
||||||
|
for await (const msg of alertSubscription.messages) {
|
||||||
|
try {
|
||||||
|
const payload = AlertEventSchema.parse(alertSubscription.decode(msg));
|
||||||
|
broadcast(alertSockets, { type: "alert", payload });
|
||||||
|
msg.ack();
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("failed to process alert", {
|
||||||
|
error: error instanceof Error ? error.message : String(error)
|
||||||
|
});
|
||||||
|
msg.term();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
void pumpOptions();
|
void pumpOptions();
|
||||||
void pumpEquities();
|
void pumpEquities();
|
||||||
void pumpFlow();
|
void pumpFlow();
|
||||||
|
void pumpClassifierHits();
|
||||||
|
void pumpAlerts();
|
||||||
|
|
||||||
const server = Bun.serve<WsData>({
|
const server = Bun.serve<WsData>({
|
||||||
port: env.API_PORT,
|
port: env.API_PORT,
|
||||||
|
|
@ -284,6 +372,18 @@ const run = async () => {
|
||||||
return jsonResponse({ data });
|
return jsonResponse({ data });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname === "/flow/classifier-hits") {
|
||||||
|
const limit = parseLimit(url.searchParams.get("limit"));
|
||||||
|
const data = await fetchRecentClassifierHits(clickhouse, limit);
|
||||||
|
return jsonResponse({ data });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname === "/flow/alerts") {
|
||||||
|
const limit = parseLimit(url.searchParams.get("limit"));
|
||||||
|
const data = await fetchRecentAlerts(clickhouse, limit);
|
||||||
|
return jsonResponse({ data });
|
||||||
|
}
|
||||||
|
|
||||||
if (req.method === "GET" && url.pathname === "/replay/options") {
|
if (req.method === "GET" && url.pathname === "/replay/options") {
|
||||||
const { afterTs, afterSeq, limit } = parseReplayParams(url);
|
const { afterTs, afterSeq, limit } = parseReplayParams(url);
|
||||||
const data = await fetchOptionPrintsAfter(clickhouse, afterTs, afterSeq, limit);
|
const data = await fetchOptionPrintsAfter(clickhouse, afterTs, afterSeq, limit);
|
||||||
|
|
@ -324,6 +424,22 @@ const run = async () => {
|
||||||
return jsonResponse({ error: "websocket upgrade failed" }, 400);
|
return jsonResponse({ error: "websocket upgrade failed" }, 400);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname === "/ws/classifier-hits") {
|
||||||
|
if (serverRef.upgrade(req, { data: { channel: "classifier-hits" } })) {
|
||||||
|
return new Response(null, { status: 101 });
|
||||||
|
}
|
||||||
|
|
||||||
|
return jsonResponse({ error: "websocket upgrade failed" }, 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && url.pathname === "/ws/alerts") {
|
||||||
|
if (serverRef.upgrade(req, { data: { channel: "alerts" } })) {
|
||||||
|
return new Response(null, { status: 101 });
|
||||||
|
}
|
||||||
|
|
||||||
|
return jsonResponse({ error: "websocket upgrade failed" }, 400);
|
||||||
|
}
|
||||||
|
|
||||||
return jsonResponse({ error: "not found" }, 404);
|
return jsonResponse({ error: "not found" }, 404);
|
||||||
},
|
},
|
||||||
websocket: {
|
websocket: {
|
||||||
|
|
@ -332,8 +448,12 @@ const run = async () => {
|
||||||
optionSockets.add(socket);
|
optionSockets.add(socket);
|
||||||
} else if (socket.data.channel === "equities") {
|
} else if (socket.data.channel === "equities") {
|
||||||
equitySockets.add(socket);
|
equitySockets.add(socket);
|
||||||
} else {
|
} else if (socket.data.channel === "flow") {
|
||||||
flowSockets.add(socket);
|
flowSockets.add(socket);
|
||||||
|
} else if (socket.data.channel === "classifier-hits") {
|
||||||
|
classifierHitSockets.add(socket);
|
||||||
|
} else {
|
||||||
|
alertSockets.add(socket);
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info("websocket connected", { channel: socket.data.channel });
|
logger.info("websocket connected", { channel: socket.data.channel });
|
||||||
|
|
@ -343,8 +463,12 @@ const run = async () => {
|
||||||
optionSockets.delete(socket);
|
optionSockets.delete(socket);
|
||||||
} else if (socket.data.channel === "equities") {
|
} else if (socket.data.channel === "equities") {
|
||||||
equitySockets.delete(socket);
|
equitySockets.delete(socket);
|
||||||
} else {
|
} else if (socket.data.channel === "flow") {
|
||||||
flowSockets.delete(socket);
|
flowSockets.delete(socket);
|
||||||
|
} else if (socket.data.channel === "classifier-hits") {
|
||||||
|
classifierHitSockets.delete(socket);
|
||||||
|
} else {
|
||||||
|
alertSockets.delete(socket);
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info("websocket disconnected", { channel: socket.data.channel });
|
logger.info("websocket disconnected", { channel: socket.data.channel });
|
||||||
|
|
|
||||||
221
services/compute/src/classifiers.ts
Normal file
221
services/compute/src/classifiers.ts
Normal file
|
|
@ -0,0 +1,221 @@
|
||||||
|
import type { ClassifierHit, FlowPacket } from "@islandflow/types";
|
||||||
|
|
||||||
|
type ParsedContract = {
|
||||||
|
root: string;
|
||||||
|
expiry: string;
|
||||||
|
strike: number;
|
||||||
|
right: "C" | "P";
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ClassifierConfig = {
|
||||||
|
sweepMinPremium: number;
|
||||||
|
sweepMinCount: number;
|
||||||
|
spikeMinPremium: number;
|
||||||
|
spikeMinSize: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
const clamp = (value: number, min = 0, max = 1): number => {
|
||||||
|
if (!Number.isFinite(value)) {
|
||||||
|
return min;
|
||||||
|
}
|
||||||
|
return Math.max(min, Math.min(max, value));
|
||||||
|
};
|
||||||
|
|
||||||
|
const formatUsd = (value: number): string => {
|
||||||
|
if (!Number.isFinite(value)) {
|
||||||
|
return "$0";
|
||||||
|
}
|
||||||
|
return `$${value.toFixed(2)}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
const parseDashedContract = (value: string): ParsedContract | null => {
|
||||||
|
const parts = value.split("-");
|
||||||
|
if (parts.length < 6) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const rightRaw = parts.at(-1) ?? "";
|
||||||
|
if (rightRaw !== "C" && rightRaw !== "P") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const strikeRaw = parts.at(-2) ?? "";
|
||||||
|
const strike = Number(strikeRaw);
|
||||||
|
const expiryParts = parts.slice(-5, -2);
|
||||||
|
const expiry = expiryParts.join("-");
|
||||||
|
const root = parts.slice(0, -5).join("-");
|
||||||
|
|
||||||
|
if (!root || !expiry || !Number.isFinite(strike)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
root,
|
||||||
|
expiry,
|
||||||
|
strike,
|
||||||
|
right: rightRaw
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const parseOccContract = (value: string): ParsedContract | null => {
|
||||||
|
if (value.length < 15) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const tail = value.slice(-15);
|
||||||
|
const root = value.slice(0, -15).trim();
|
||||||
|
const expiryRaw = tail.slice(0, 6);
|
||||||
|
const right = tail.slice(6, 7);
|
||||||
|
const strikeRaw = tail.slice(7);
|
||||||
|
|
||||||
|
if (!/^\d{6}$/.test(expiryRaw) || !/^\d{8}$/.test(strikeRaw)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (right !== "C" && right !== "P") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const year = 2000 + Number(expiryRaw.slice(0, 2));
|
||||||
|
const month = Number(expiryRaw.slice(2, 4)) - 1;
|
||||||
|
const day = Number(expiryRaw.slice(4, 6));
|
||||||
|
const expiryDate = new Date(Date.UTC(year, month, day));
|
||||||
|
const expiry = expiryDate.toISOString().slice(0, 10);
|
||||||
|
const strike = Number(strikeRaw) / 1000;
|
||||||
|
|
||||||
|
if (!root || !Number.isFinite(strike)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
root,
|
||||||
|
expiry,
|
||||||
|
strike,
|
||||||
|
right
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const parseContractId = (value: string | undefined): ParsedContract | null => {
|
||||||
|
if (!value) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return parseDashedContract(value) ?? parseOccContract(value);
|
||||||
|
};
|
||||||
|
|
||||||
|
const getNumberFeature = (packet: FlowPacket, key: string): number => {
|
||||||
|
const value = packet.features[key];
|
||||||
|
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
const buildSweepHit = (
|
||||||
|
packet: FlowPacket,
|
||||||
|
contract: ParsedContract,
|
||||||
|
direction: "bullish" | "bearish",
|
||||||
|
config: ClassifierConfig
|
||||||
|
): ClassifierHit | null => {
|
||||||
|
const count = getNumberFeature(packet, "count");
|
||||||
|
const totalPremium = getNumberFeature(packet, "total_premium");
|
||||||
|
const totalSize = getNumberFeature(packet, "total_size");
|
||||||
|
const firstPrice = getNumberFeature(packet, "first_price");
|
||||||
|
const lastPrice = getNumberFeature(packet, "last_price");
|
||||||
|
const windowMs = getNumberFeature(packet, "window_ms");
|
||||||
|
|
||||||
|
if (count < config.sweepMinCount || totalPremium < config.sweepMinPremium) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const priceDelta = lastPrice - firstPrice;
|
||||||
|
const priceTrend = priceDelta >= 0 ? "up" : "down";
|
||||||
|
|
||||||
|
let confidence = 0.55;
|
||||||
|
if (priceDelta >= 0) {
|
||||||
|
confidence += 0.1;
|
||||||
|
}
|
||||||
|
if (count >= config.sweepMinCount + 2) {
|
||||||
|
confidence += 0.1;
|
||||||
|
}
|
||||||
|
if (totalPremium >= config.sweepMinPremium * 2) {
|
||||||
|
confidence += 0.15;
|
||||||
|
}
|
||||||
|
|
||||||
|
confidence = clamp(confidence, 0, 0.95);
|
||||||
|
|
||||||
|
return {
|
||||||
|
classifier_id: direction === "bullish" ? "large_bullish_call_sweep" : "large_bearish_put_sweep",
|
||||||
|
confidence,
|
||||||
|
direction,
|
||||||
|
explanations: [
|
||||||
|
`Likely ${direction === "bullish" ? "call" : "put"} sweep: ${count} prints in ${Math.round(windowMs)}ms for ${packet.features.option_contract_id ?? packet.id}.`,
|
||||||
|
`Premium ${formatUsd(totalPremium)} across ${Math.round(totalSize)} contracts; price ${priceTrend}.`,
|
||||||
|
`Thresholds: >=${config.sweepMinCount} prints and >=${formatUsd(config.sweepMinPremium)} premium.`
|
||||||
|
]
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const buildSpikeHit = (packet: FlowPacket, config: ClassifierConfig): ClassifierHit | null => {
|
||||||
|
const count = getNumberFeature(packet, "count");
|
||||||
|
const totalPremium = getNumberFeature(packet, "total_premium");
|
||||||
|
const totalSize = getNumberFeature(packet, "total_size");
|
||||||
|
const windowMs = getNumberFeature(packet, "window_ms");
|
||||||
|
|
||||||
|
if (totalSize < config.spikeMinSize || totalPremium < config.spikeMinPremium) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
let confidence = 0.5;
|
||||||
|
if (totalSize >= config.spikeMinSize * 2) {
|
||||||
|
confidence += 0.15;
|
||||||
|
}
|
||||||
|
if (totalPremium >= config.spikeMinPremium * 2) {
|
||||||
|
confidence += 0.15;
|
||||||
|
}
|
||||||
|
if (count >= 3) {
|
||||||
|
confidence += 0.1;
|
||||||
|
}
|
||||||
|
|
||||||
|
confidence = clamp(confidence, 0, 0.9);
|
||||||
|
|
||||||
|
return {
|
||||||
|
classifier_id: "unusual_contract_spike",
|
||||||
|
confidence,
|
||||||
|
direction: "neutral",
|
||||||
|
explanations: [
|
||||||
|
`Unusual contract spike: ${count} prints in ${Math.round(windowMs)}ms for ${packet.features.option_contract_id ?? packet.id}.`,
|
||||||
|
`Premium ${formatUsd(totalPremium)} across ${Math.round(totalSize)} contracts.`,
|
||||||
|
`Thresholds: >=${config.spikeMinSize} contracts and >=${formatUsd(config.spikeMinPremium)} premium.`
|
||||||
|
]
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const evaluateClassifiers = (
|
||||||
|
packet: FlowPacket,
|
||||||
|
config: ClassifierConfig
|
||||||
|
): ClassifierHit[] => {
|
||||||
|
const contractId = typeof packet.features.option_contract_id === "string"
|
||||||
|
? packet.features.option_contract_id
|
||||||
|
: "";
|
||||||
|
const contract = parseContractId(contractId);
|
||||||
|
const hits: ClassifierHit[] = [];
|
||||||
|
|
||||||
|
if (contract?.right === "C") {
|
||||||
|
const hit = buildSweepHit(packet, contract, "bullish", config);
|
||||||
|
if (hit) {
|
||||||
|
hits.push(hit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (contract?.right === "P") {
|
||||||
|
const hit = buildSweepHit(packet, contract, "bearish", config);
|
||||||
|
if (hit) {
|
||||||
|
hits.push(hit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const spikeHit = buildSpikeHit(packet, config);
|
||||||
|
if (spikeHit) {
|
||||||
|
hits.push(spikeHit);
|
||||||
|
}
|
||||||
|
|
||||||
|
return hits;
|
||||||
|
};
|
||||||
|
|
@ -1,8 +1,12 @@
|
||||||
import { readEnv } from "@islandflow/config";
|
import { readEnv } from "@islandflow/config";
|
||||||
import { createLogger } from "@islandflow/observability";
|
import { createLogger } from "@islandflow/observability";
|
||||||
import {
|
import {
|
||||||
|
SUBJECT_ALERTS,
|
||||||
|
SUBJECT_CLASSIFIER_HITS,
|
||||||
SUBJECT_FLOW_PACKETS,
|
SUBJECT_FLOW_PACKETS,
|
||||||
SUBJECT_OPTION_PRINTS,
|
SUBJECT_OPTION_PRINTS,
|
||||||
|
STREAM_ALERTS,
|
||||||
|
STREAM_CLASSIFIER_HITS,
|
||||||
STREAM_FLOW_PACKETS,
|
STREAM_FLOW_PACKETS,
|
||||||
STREAM_OPTION_PRINTS,
|
STREAM_OPTION_PRINTS,
|
||||||
buildDurableConsumer,
|
buildDurableConsumer,
|
||||||
|
|
@ -13,11 +17,25 @@ import {
|
||||||
} from "@islandflow/bus";
|
} from "@islandflow/bus";
|
||||||
import {
|
import {
|
||||||
createClickHouseClient,
|
createClickHouseClient,
|
||||||
|
ensureAlertsTable,
|
||||||
|
ensureClassifierHitsTable,
|
||||||
ensureFlowPacketsTable,
|
ensureFlowPacketsTable,
|
||||||
|
insertAlert,
|
||||||
|
insertClassifierHit,
|
||||||
insertFlowPacket
|
insertFlowPacket
|
||||||
} from "@islandflow/storage";
|
} from "@islandflow/storage";
|
||||||
import { FlowPacketSchema, OptionPrintSchema, type FlowPacket, type OptionPrint } from "@islandflow/types";
|
import {
|
||||||
|
AlertEventSchema,
|
||||||
|
ClassifierHitEventSchema,
|
||||||
|
FlowPacketSchema,
|
||||||
|
OptionPrintSchema,
|
||||||
|
type AlertEvent,
|
||||||
|
type ClassifierHitEvent,
|
||||||
|
type FlowPacket,
|
||||||
|
type OptionPrint
|
||||||
|
} from "@islandflow/types";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
import { evaluateClassifiers, type ClassifierConfig } from "./classifiers";
|
||||||
|
|
||||||
const service = "compute";
|
const service = "compute";
|
||||||
const logger = createLogger({ service });
|
const logger = createLogger({ service });
|
||||||
|
|
@ -41,11 +59,22 @@ const envSchema = z.object({
|
||||||
}
|
}
|
||||||
return value;
|
return value;
|
||||||
}, z.boolean())
|
}, z.boolean())
|
||||||
.default(false)
|
.default(false),
|
||||||
|
CLASSIFIER_SWEEP_MIN_PREMIUM: z.coerce.number().positive().default(50_000),
|
||||||
|
CLASSIFIER_SWEEP_MIN_COUNT: z.coerce.number().int().positive().default(3),
|
||||||
|
CLASSIFIER_SPIKE_MIN_PREMIUM: z.coerce.number().positive().default(25_000),
|
||||||
|
CLASSIFIER_SPIKE_MIN_SIZE: z.coerce.number().int().positive().default(500)
|
||||||
});
|
});
|
||||||
|
|
||||||
const env = readEnv(envSchema);
|
const env = readEnv(envSchema);
|
||||||
|
|
||||||
|
const classifierConfig: ClassifierConfig = {
|
||||||
|
sweepMinPremium: env.CLASSIFIER_SWEEP_MIN_PREMIUM,
|
||||||
|
sweepMinCount: env.CLASSIFIER_SWEEP_MIN_COUNT,
|
||||||
|
spikeMinPremium: env.CLASSIFIER_SPIKE_MIN_PREMIUM,
|
||||||
|
spikeMinSize: env.CLASSIFIER_SPIKE_MIN_SIZE
|
||||||
|
};
|
||||||
|
|
||||||
const retry = async <T>(
|
const retry = async <T>(
|
||||||
label: string,
|
label: string,
|
||||||
attempts: number,
|
attempts: number,
|
||||||
|
|
@ -170,6 +199,8 @@ const flushCluster = async (
|
||||||
await insertFlowPacket(clickhouse, validated);
|
await insertFlowPacket(clickhouse, validated);
|
||||||
await publishJson(js, SUBJECT_FLOW_PACKETS, validated);
|
await publishJson(js, SUBJECT_FLOW_PACKETS, validated);
|
||||||
|
|
||||||
|
await emitClassifiers(clickhouse, js, validated);
|
||||||
|
|
||||||
logger.info("emitted flow packet", {
|
logger.info("emitted flow packet", {
|
||||||
id: validated.id,
|
id: validated.id,
|
||||||
contract: cluster.contractId,
|
contract: cluster.contractId,
|
||||||
|
|
@ -177,6 +208,79 @@ const flushCluster = async (
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const scoreAlert = (packet: FlowPacket, hits: ClassifierHitEvent[]): { score: number; severity: string } => {
|
||||||
|
const premium =
|
||||||
|
typeof packet.features.total_premium === "number" ? packet.features.total_premium : 0;
|
||||||
|
const premiumScore = Math.min(70, Math.round(premium / 1000));
|
||||||
|
const maxConfidence = hits.reduce((max, hit) => Math.max(max, hit.confidence), 0);
|
||||||
|
const confidenceScore = Math.round(maxConfidence * 20);
|
||||||
|
const hitScore = Math.min(20, hits.length * 5);
|
||||||
|
const score = Math.max(0, Math.min(100, premiumScore + confidenceScore + hitScore));
|
||||||
|
const severity = score >= 80 ? "high" : score >= 45 ? "medium" : "low";
|
||||||
|
return { score, severity };
|
||||||
|
};
|
||||||
|
|
||||||
|
const emitClassifiers = async (
|
||||||
|
clickhouse: ReturnType<typeof createClickHouseClient>,
|
||||||
|
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
|
||||||
|
packet: FlowPacket
|
||||||
|
): Promise<void> => {
|
||||||
|
const hits = evaluateClassifiers(packet, classifierConfig);
|
||||||
|
if (hits.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const hitEvents: ClassifierHitEvent[] = hits.map((hit) =>
|
||||||
|
ClassifierHitEventSchema.parse({
|
||||||
|
source_ts: packet.source_ts,
|
||||||
|
ingest_ts: packet.ingest_ts,
|
||||||
|
seq: packet.seq,
|
||||||
|
trace_id: `classifier:${hit.classifier_id}:${packet.id}`,
|
||||||
|
...hit
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const hit of hitEvents) {
|
||||||
|
try {
|
||||||
|
await insertClassifierHit(clickhouse, hit);
|
||||||
|
await publishJson(js, SUBJECT_CLASSIFIER_HITS, hit);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("failed to emit classifier hit", {
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
classifier_id: hit.classifier_id,
|
||||||
|
packet_id: packet.id
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const { score, severity } = scoreAlert(packet, hitEvents);
|
||||||
|
const alert: AlertEvent = AlertEventSchema.parse({
|
||||||
|
source_ts: packet.source_ts,
|
||||||
|
ingest_ts: packet.ingest_ts,
|
||||||
|
seq: packet.seq,
|
||||||
|
trace_id: `alert:${packet.id}`,
|
||||||
|
score,
|
||||||
|
severity,
|
||||||
|
hits: hitEvents.map((hit) => ({
|
||||||
|
classifier_id: hit.classifier_id,
|
||||||
|
confidence: hit.confidence,
|
||||||
|
direction: hit.direction,
|
||||||
|
explanations: hit.explanations
|
||||||
|
})),
|
||||||
|
evidence_refs: [packet.id, ...packet.members]
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await insertAlert(clickhouse, alert);
|
||||||
|
await publishJson(js, SUBJECT_ALERTS, alert);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("failed to emit alert", {
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
packet_id: packet.id
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const flushEligibleClusters = async (
|
const flushEligibleClusters = async (
|
||||||
clickhouse: ReturnType<typeof createClickHouseClient>,
|
clickhouse: ReturnType<typeof createClickHouseClient>,
|
||||||
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
|
js: Awaited<ReturnType<typeof connectJetStreamWithRetry>>["js"],
|
||||||
|
|
@ -232,6 +336,32 @@ const run = async () => {
|
||||||
num_replicas: 1
|
num_replicas: 1
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await ensureStream(jsm, {
|
||||||
|
name: STREAM_CLASSIFIER_HITS,
|
||||||
|
subjects: [SUBJECT_CLASSIFIER_HITS],
|
||||||
|
retention: "limits",
|
||||||
|
storage: "file",
|
||||||
|
discard: "old",
|
||||||
|
max_msgs_per_subject: -1,
|
||||||
|
max_msgs: -1,
|
||||||
|
max_bytes: -1,
|
||||||
|
max_age: 0,
|
||||||
|
num_replicas: 1
|
||||||
|
});
|
||||||
|
|
||||||
|
await ensureStream(jsm, {
|
||||||
|
name: STREAM_ALERTS,
|
||||||
|
subjects: [SUBJECT_ALERTS],
|
||||||
|
retention: "limits",
|
||||||
|
storage: "file",
|
||||||
|
discard: "old",
|
||||||
|
max_msgs_per_subject: -1,
|
||||||
|
max_msgs: -1,
|
||||||
|
max_bytes: -1,
|
||||||
|
max_age: 0,
|
||||||
|
num_replicas: 1
|
||||||
|
});
|
||||||
|
|
||||||
const clickhouse = createClickHouseClient({
|
const clickhouse = createClickHouseClient({
|
||||||
url: env.CLICKHOUSE_URL,
|
url: env.CLICKHOUSE_URL,
|
||||||
database: env.CLICKHOUSE_DATABASE
|
database: env.CLICKHOUSE_DATABASE
|
||||||
|
|
@ -239,6 +369,8 @@ const run = async () => {
|
||||||
|
|
||||||
await retry("clickhouse table init", 20, 500, async () => {
|
await retry("clickhouse table init", 20, 500, async () => {
|
||||||
await ensureFlowPacketsTable(clickhouse);
|
await ensureFlowPacketsTable(clickhouse);
|
||||||
|
await ensureClassifierHitsTable(clickhouse);
|
||||||
|
await ensureAlertsTable(clickhouse);
|
||||||
});
|
});
|
||||||
|
|
||||||
const durableName = "compute-option-prints";
|
const durableName = "compute-option-prints";
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue