expand ci quality gates
All checks were successful
CI / Validate (push) Successful in 1m13s

This commit is contained in:
dirtydishes 2026-05-30 02:34:28 -04:00
parent 65139bf8d0
commit 44431c4e66
71 changed files with 2262 additions and 1173 deletions

View file

@ -24,6 +24,7 @@
{"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-cig","title":"Expand CI quality gates","description":"Add a more robust CI workflow for the Bun/TypeScript monorepo, including formatting, linting, type checking, builds, and tests where appropriate.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-30T06:29:33Z","created_by":"dirtydishes","updated_at":"2026-05-30T06:34:11Z","started_at":"2026-05-30T06:29:41Z","closed_at":"2026-05-30T06:34:11Z","close_reason":"Expanded CI quality gates with Biome formatting/linting, public API route checks, Docker snapshot validation, tests, typecheck, and web build validation.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-3l6","title":"fix ci typecheck bun path resolution","description":"Forgejo CI fails in scripts/typecheck.ts because the script shells out to bunx, which expects bun on PATH. The runner installs Bun by absolute path, so the typecheck helper should use the current Bun executable instead of PATH lookup.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-30T05:34:55Z","created_by":"dirtydishes","updated_at":"2026-05-30T06:00:31Z","started_at":"2026-05-30T05:35:02Z","closed_at":"2026-05-30T06:00:31Z","close_reason":"Fixed the Forgejo CI terminal import mismatch by switching the terminal client component to a namespace import; verified locally and on Forgejo run #56.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-3l6","title":"fix ci typecheck bun path resolution","description":"Forgejo CI fails in scripts/typecheck.ts because the script shells out to bunx, which expects bun on PATH. The runner installs Bun by absolute path, so the typecheck helper should use the current Bun executable instead of PATH lookup.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-30T05:34:55Z","created_by":"dirtydishes","updated_at":"2026-05-30T06:00:31Z","started_at":"2026-05-30T05:35:02Z","closed_at":"2026-05-30T06:00:31Z","close_reason":"Fixed the Forgejo CI terminal import mismatch by switching the terminal client component to a namespace import; verified locally and on Forgejo run #56.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-wtg","title":"Harden drawer dialog focus behavior","description":"Fix terminal drawers so they expose modal dialog semantics, trap keyboard focus while open, and restore focus to the invoking control after close.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-29T22:55:25Z","created_by":"dirtydishes","updated_at":"2026-05-29T23:09:45Z","started_at":"2026-05-29T22:56:22Z","closed_at":"2026-05-29T23:09:45Z","close_reason":"Implemented modal dialog semantics, focus trapping, Escape dismissal, focus restoration, validation, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-wtg","title":"Harden drawer dialog focus behavior","description":"Fix terminal drawers so they expose modal dialog semantics, trap keyboard focus while open, and restore focus to the invoking control after close.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-29T22:55:25Z","created_by":"dirtydishes","updated_at":"2026-05-29T23:09:45Z","started_at":"2026-05-29T22:56:22Z","closed_at":"2026-05-29T23:09:45Z","close_reason":"Implemented modal dialog semantics, focus trapping, Escape dismissal, focus restoration, validation, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-833","title":"Improve narrow options table responsiveness","description":"Adapt the Options route for narrow screens so dense tape tables remain contained in their panes, preserve row identity while horizontally panning, and keep the mobile ticker/filter controls readable.","acceptance_criteria":"Options tape panes have bounded heights on narrow screens; table body scrolls internally; first table column remains visible while panning; mobile topbar and filter controls have adequate spacing; web production build passes.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-29T22:34:05Z","created_by":"dirtydishes","updated_at":"2026-05-29T22:36:20Z","started_at":"2026-05-29T22:34:24Z","closed_at":"2026-05-29T22:36:20Z","close_reason":"Implemented narrow-screen options pane containment, sticky row context, touch-scroll affordances, and mobile control spacing. Validated with web build and in-browser narrow viewport checks.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-833","title":"Improve narrow options table responsiveness","description":"Adapt the Options route for narrow screens so dense tape tables remain contained in their panes, preserve row identity while horizontally panning, and keep the mobile ticker/filter controls readable.","acceptance_criteria":"Options tape panes have bounded heights on narrow screens; table body scrolls internally; first table column remains visible while panning; mobile topbar and filter controls have adequate spacing; web production build passes.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-29T22:34:05Z","created_by":"dirtydishes","updated_at":"2026-05-29T22:36:20Z","started_at":"2026-05-29T22:34:24Z","closed_at":"2026-05-29T22:36:20Z","close_reason":"Implemented narrow-screen options pane containment, sticky row context, touch-scroll affordances, and mobile control spacing. Validated with web build and in-browser narrow viewport checks.","dependency_count":0,"dependent_count":0,"comment_count":0}

View file

@ -36,12 +36,21 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: ~/.bun/bin/bun install --frozen-lockfile run: ~/.bun/bin/bun install --frozen-lockfile
- name: Check formatting
run: ~/.bun/bin/bun run fmt:check
- name: Run lint
run: ~/.bun/bin/bun run lint
- name: Run typecheck - name: Run typecheck
run: ~/.bun/bin/bun run typecheck run: ~/.bun/bin/bun run typecheck
- name: Run tests - name: Run tests
run: ~/.bun/bin/bun test run: ~/.bun/bin/bun test
- name: Check public API routes
run: ~/.bun/bin/bun run check:public-api-routes
- name: Check Docker workspace snapshot - name: Check Docker workspace snapshot
run: ~/.bun/bin/bun run check:docker-workspace run: ~/.bun/bin/bun run check:docker-workspace

View file

@ -9,11 +9,8 @@ export async function GET(): Promise<Response> {
} }
export async function PUT(req: Request): Promise<Response> { export async function PUT(req: Request): Promise<Response> {
return proxySyntheticAdminRequest( return proxySyntheticAdminRequest("/admin/synthetic/control", {
"/admin/synthetic/control",
{
method: "PUT", method: "PUT",
body: await req.text() body: await req.text()
} });
);
} }

View file

@ -1,8 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test"; import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
import { import { getSyntheticAdminProxyConfig, isSyntheticAdminFeatureEnabled } from "./shared";
getSyntheticAdminProxyConfig,
isSyntheticAdminFeatureEnabled
} from "./shared";
const originalFetch = globalThis.fetch; const originalFetch = globalThis.fetch;

View file

@ -18,25 +18,29 @@ const variants: Record<
> = { > = {
mock1: { mock1: {
title: "Command Deck", title: "Command Deck",
premise: "Closest to the reference: left navigation, ticker ribbon, dense evidence panes, replay rail.", premise:
"Closest to the reference: left navigation, ticker ribbon, dense evidence panes, replay rail.",
mode: "Dense ops", mode: "Dense ops",
layout: "classic" layout: "classic"
}, },
mock2: { mock2: {
title: "Investigation Stack", title: "Investigation Stack",
premise: "A calmer analyst layout with the selected symbol story in the center and context wrapped around it.", premise:
"A calmer analyst layout with the selected symbol story in the center and context wrapped around it.",
mode: "Forensic", mode: "Forensic",
layout: "focus" layout: "focus"
}, },
mock3: { mock3: {
title: "Signal Wall", title: "Signal Wall",
premise: "Prioritizes alert triage and cross-symbol scanning before a user drills into price action.", premise:
"Prioritizes alert triage and cross-symbol scanning before a user drills into price action.",
mode: "Triage", mode: "Triage",
layout: "signals" layout: "signals"
}, },
mock4: { mock4: {
title: "Replay Lab", title: "Replay Lab",
premise: "A replay-first structure with timeline, event tape, and causality context always visible.", premise:
"A replay-first structure with timeline, event tape, and causality context always visible.",
mode: "Replay", mode: "Replay",
layout: "replay" layout: "replay"
} }
@ -93,7 +97,10 @@ export function DashboardMock({ variant }: DashboardMockProps) {
const config = variants[variant]; const config = variants[variant];
return ( return (
<section className={`mock-terminal mock-terminal-${config.layout}`} aria-labelledby="mock-title"> <section
className={`mock-terminal mock-terminal-${config.layout}`}
aria-labelledby="mock-title"
>
<MockHeader config={config} active={variant} /> <MockHeader config={config} active={variant} />
<TickerRail /> <TickerRail />
{variant === "mock1" ? <ClassicLayout /> : null} {variant === "mock1" ? <ClassicLayout /> : null}
@ -277,7 +284,11 @@ function OptionTape({ condensed = false }: { condensed?: boolean }) {
function ChartPanel({ compact = false }: { compact?: boolean }) { function ChartPanel({ compact = false }: { compact?: boolean }) {
return ( return (
<Panel title="AAPL | Price & Flow" meta="1m / 5m / 15m" className={compact ? "mock-chart is-compact" : "mock-chart"}> <Panel
title="AAPL | Price & Flow"
meta="1m / 5m / 15m"
className={compact ? "mock-chart is-compact" : "mock-chart"}
>
<div className="mock-chart-meta"> <div className="mock-chart-meta">
<strong>194.88</strong> <strong>194.88</strong>
<span className="mock-move is-up">+2.34 (+1.22%)</span> <span className="mock-move is-up">+2.34 (+1.22%)</span>
@ -306,16 +317,24 @@ function ChartPanel({ compact = false }: { compact?: boolean }) {
function SignalPanel({ hero = false }: { hero?: boolean }) { function SignalPanel({ hero = false }: { hero?: boolean }) {
return ( return (
<Panel title="Signals & Alerts" meta="All / Signals / System" className={hero ? "mock-signals is-hero" : "mock-signals"}> <Panel
title="Signals & Alerts"
meta="All / Signals / System"
className={hero ? "mock-signals is-hero" : "mock-signals"}
>
<div className="mock-signal-list"> <div className="mock-signal-list">
{signals.map(([time, title, symbol, value, tag]) => ( {signals.map(([time, title, symbol, value, tag]) => (
<article className="mock-signal-item" key={`${time}-${title}`}> <article className="mock-signal-item" key={`${time}-${title}`}>
<time>{time}</time> <time>{time}</time>
<div> <div>
<strong>{title}</strong> <strong>{title}</strong>
<span>{symbol} / {value}</span> <span>
{symbol} / {value}
</span>
</div> </div>
<span className={`mock-pill ${tag === "Bearish" ? "is-bearish" : tag === "News" ? "is-news" : "is-bullish"}`}> <span
className={`mock-pill ${tag === "Bearish" ? "is-bearish" : tag === "News" ? "is-news" : "is-bullish"}`}
>
{tag} {tag}
</span> </span>
</article> </article>
@ -332,7 +351,9 @@ function FeedHealth() {
{feedHealth.map(([feed, status, lag, rate]) => ( {feedHealth.map(([feed, status, lag, rate]) => (
<div className="mock-table-row" key={feed}> <div className="mock-table-row" key={feed}>
<span>{feed}</span> <span>{feed}</span>
<span className={`mock-pill ${status === "Degraded" ? "is-warning" : "is-bullish"}`}>{status}</span> <span className={`mock-pill ${status === "Degraded" ? "is-warning" : "is-bullish"}`}>
{status}
</span>
<span>{lag}</span> <span>{lag}</span>
<span>{rate}/s</span> <span>{rate}/s</span>
</div> </div>
@ -350,7 +371,9 @@ function DarkFlow() {
<div className="mock-table-row" key={`${time}-${side}-${size}`}> <div className="mock-table-row" key={`${time}-${side}-${size}`}>
<span>{time}</span> <span>{time}</span>
<strong>{symbol}</strong> <strong>{symbol}</strong>
<span className={`mock-pill ${side === "Sell" ? "is-bearish" : "is-bullish"}`}>{side}</span> <span className={`mock-pill ${side === "Sell" ? "is-bearish" : "is-bullish"}`}>
{side}
</span>
<span>{size}</span> <span>{size}</span>
<span>{notional}</span> <span>{notional}</span>
<span>{type}</span> <span>{type}</span>
@ -402,7 +425,11 @@ function EventContext() {
function ReplayRail({ compact = false }: { compact?: boolean }) { function ReplayRail({ compact = false }: { compact?: boolean }) {
return ( return (
<Panel title="Replay" meta="May 16, 2024" className={compact ? "mock-replay is-compact" : "mock-replay"}> <Panel
title="Replay"
meta="May 16, 2024"
className={compact ? "mock-replay is-compact" : "mock-replay"}
>
<div className="mock-replay-controls"> <div className="mock-replay-controls">
<button type="button">Prev</button> <button type="button">Prev</button>
<button type="button">Pause</button> <button type="button">Pause</button>
@ -430,8 +457,9 @@ function SymbolBrief() {
<span className="mock-move is-up">+1.22%</span> <span className="mock-move is-up">+1.22%</span>
</div> </div>
<p> <p>
Dark sweep pressure aligns with short-window momentum and a fresh news catalyst. Context confidence is high, but Dark sweep pressure aligns with short-window momentum and a fresh news catalyst. Context
the largest block remains off-exchange and should be checked against next print behavior. confidence is high, but the largest block remains off-exchange and should be checked against
next print behavior.
</p> </p>
<div className="mock-brief-tags"> <div className="mock-brief-tags">
<span className="mock-pill is-bullish">Bullish</span> <span className="mock-pill is-bullish">Bullish</span>
@ -444,7 +472,12 @@ function SymbolBrief() {
function Sparkline({ direction }: { direction: string }) { function Sparkline({ direction }: { direction: string }) {
return ( return (
<svg className="mock-sparkline" viewBox="0 0 96 28" role="img" aria-label={`${direction} sparkline`}> <svg
className="mock-sparkline"
viewBox="0 0 96 28"
role="img"
aria-label={`${direction} sparkline`}
>
<polyline <polyline
fill="none" fill="none"
points={ points={

View file

@ -161,7 +161,10 @@ input {
text-transform: uppercase; text-transform: uppercase;
letter-spacing: 0.14em; letter-spacing: 0.14em;
font-size: 0.76rem; font-size: 0.76rem;
transition: border-color 0.15s ease, background-color 0.15s ease, color 0.15s ease; transition:
border-color 0.15s ease,
background-color 0.15s ease,
color 0.15s ease;
} }
.terminal-nav-link:hover { .terminal-nav-link:hover {
@ -800,8 +803,7 @@ h3 {
border: 1px solid var(--border-strong); border: 1px solid var(--border-strong);
border-radius: 8px; border-radius: 8px;
background: background:
linear-gradient(135deg, oklch(0.78 0.12 74 / 0.7), oklch(0.28 0.035 250)), linear-gradient(135deg, oklch(0.78 0.12 74 / 0.7), oklch(0.28 0.035 250)), var(--accent-soft);
var(--accent-soft);
} }
.command-deck-kicker, .command-deck-kicker,
@ -1608,19 +1610,31 @@ h3 {
.data-table-row-classified { .data-table-row-classified {
background: background:
linear-gradient(90deg, rgba(var(--classifier-rgb, 192, 200, 210), calc(0.012 + var(--classifier-intensity, 0) * 0.06)), transparent 62%), linear-gradient(
90deg,
rgba(
var(--classifier-rgb, 192, 200, 210),
calc(0.012 + var(--classifier-intensity, 0) * 0.06)
),
transparent 62%
),
oklch(0.98 0.008 250 / 0.008); oklch(0.98 0.008 250 / 0.008);
} }
.data-table-row-classified:hover, .data-table-row-classified:hover,
.data-table-row-classified:focus-visible { .data-table-row-classified:focus-visible {
background: background:
linear-gradient(90deg, rgba(var(--classifier-rgb, 192, 200, 210), calc(0.02 + var(--classifier-intensity, 0) * 0.1)), transparent 68%), linear-gradient(
90deg,
rgba(var(--classifier-rgb, 192, 200, 210), calc(0.02 + var(--classifier-intensity, 0) * 0.1)),
transparent 68%
),
oklch(0.78 0.12 74 / 0.035); oklch(0.78 0.12 74 / 0.035);
} }
.data-table-row-classified.is-classified { .data-table-row-classified.is-classified {
box-shadow: inset 0 0 0 1px rgba(var(--classifier-rgb), calc(0.16 + var(--classifier-intensity) * 0.12)); box-shadow: inset 0 0 0 1px
rgba(var(--classifier-rgb), calc(0.16 + var(--classifier-intensity) * 0.12));
} }
.data-table-row-warn, .data-table-row-warn,
@ -1641,32 +1655,56 @@ h3 {
.data-table-options .data-table-head, .data-table-options .data-table-head,
.data-table-options .data-table-row { .data-table-options .data-table-row {
grid-template-columns: minmax(72px, 0.8fr) minmax(50px, 0.55fr) minmax(64px, 0.7fr) minmax(58px, 0.6fr) minmax(34px, 0.35fr) minmax(62px, 0.65fr) minmax(104px, 1fr) minmax(54px, 0.55fr) minmax(66px, 0.7fr) minmax(48px, 0.5fr) minmax(42px, 0.45fr) minmax(92px, 0.9fr); grid-template-columns: minmax(72px, 0.8fr) minmax(50px, 0.55fr) minmax(64px, 0.7fr) minmax(
58px,
0.6fr
) minmax(34px, 0.35fr) minmax(62px, 0.65fr) minmax(104px, 1fr) minmax(54px, 0.55fr) minmax(
66px,
0.7fr
) minmax(48px, 0.5fr) minmax(42px, 0.45fr) minmax(92px, 0.9fr);
} }
.data-table-equities .data-table-head, .data-table-equities .data-table-head,
.data-table-equities .data-table-row { .data-table-equities .data-table-row {
grid-template-columns: minmax(76px, 0.9fr) minmax(70px, 0.8fr) minmax(76px, 0.8fr) minmax(70px, 0.75fr) minmax(80px, 0.8fr) minmax(76px, 0.75fr); grid-template-columns: minmax(76px, 0.9fr) minmax(70px, 0.8fr) minmax(76px, 0.8fr) minmax(
70px,
0.75fr
) minmax(80px, 0.8fr) minmax(76px, 0.75fr);
} }
.data-table-flow .data-table-head, .data-table-flow .data-table-head,
.data-table-flow .data-table-row { .data-table-flow .data-table-row {
grid-template-columns: minmax(148px, 1.1fr) minmax(180px, 1.4fr) minmax(62px, 0.45fr) minmax(70px, 0.5fr) minmax(88px, 0.7fr) minmax(74px, 0.55fr) minmax(132px, 1fr) minmax(110px, 0.8fr) minmax(210px, 1.6fr); grid-template-columns: minmax(148px, 1.1fr) minmax(180px, 1.4fr) minmax(62px, 0.45fr) minmax(
70px,
0.5fr
) minmax(88px, 0.7fr) minmax(74px, 0.55fr) minmax(132px, 1fr) minmax(110px, 0.8fr) minmax(
210px,
1.6fr
);
} }
.data-table-alerts .data-table-head, .data-table-alerts .data-table-head,
.data-table-alerts .data-table-row { .data-table-alerts .data-table-row {
grid-template-columns: minmax(76px, 0.75fr) minmax(170px, 1.4fr) minmax(52px, 0.45fr) minmax(58px, 0.45fr) minmax(52px, 0.4fr) minmax(66px, 0.55fr) minmax(260px, 2fr); grid-template-columns: minmax(76px, 0.75fr) minmax(170px, 1.4fr) minmax(52px, 0.45fr) minmax(
58px,
0.45fr
) minmax(52px, 0.4fr) minmax(66px, 0.55fr) minmax(260px, 2fr);
} }
.data-table-classifier .data-table-head, .data-table-classifier .data-table-head,
.data-table-classifier .data-table-row { .data-table-classifier .data-table-row {
grid-template-columns: minmax(76px, 0.75fr) minmax(180px, 1.45fr) minmax(70px, 0.6fr) minmax(74px, 0.65fr) minmax(300px, 2.2fr); grid-template-columns: minmax(76px, 0.75fr) minmax(180px, 1.45fr) minmax(70px, 0.6fr) minmax(
74px,
0.65fr
) minmax(300px, 2.2fr);
} }
.data-table-dark .data-table-head, .data-table-dark .data-table-head,
.data-table-dark .data-table-row { .data-table-dark .data-table-row {
grid-template-columns: minmax(76px, 0.75fr) minmax(170px, 1.35fr) minmax(76px, 0.65fr) minmax(74px, 0.65fr) minmax(74px, 0.65fr) minmax(260px, 2fr); grid-template-columns: minmax(76px, 0.75fr) minmax(170px, 1.35fr) minmax(76px, 0.65fr) minmax(
74px,
0.65fr
) minmax(74px, 0.65fr) minmax(260px, 2fr);
} }
.data-table-cell { .data-table-cell {
@ -1698,7 +1736,13 @@ h3 {
.options-table-head, .options-table-head,
.options-table-row { .options-table-row {
display: grid; display: grid;
grid-template-columns: minmax(72px, 0.8fr) minmax(50px, 0.55fr) minmax(64px, 0.7fr) minmax(58px, 0.6fr) minmax(34px, 0.35fr) minmax(62px, 0.65fr) minmax(104px, 1fr) minmax(54px, 0.55fr) minmax(66px, 0.7fr) minmax(48px, 0.5fr) minmax(42px, 0.45fr) minmax(92px, 0.9fr); grid-template-columns: minmax(72px, 0.8fr) minmax(50px, 0.55fr) minmax(64px, 0.7fr) minmax(
58px,
0.6fr
) minmax(34px, 0.35fr) minmax(62px, 0.65fr) minmax(104px, 1fr) minmax(54px, 0.55fr) minmax(
66px,
0.7fr
) minmax(48px, 0.5fr) minmax(42px, 0.45fr) minmax(92px, 0.9fr);
align-items: center; align-items: center;
column-gap: 8px; column-gap: 8px;
} }
@ -1729,7 +1773,14 @@ h3 {
border: 0; border: 0;
border-bottom: 1px solid oklch(0.72 0.012 250 / 0.08); border-bottom: 1px solid oklch(0.72 0.012 250 / 0.08);
background: background:
linear-gradient(90deg, rgba(var(--classifier-rgb, 192, 200, 210), calc(0.012 + var(--classifier-intensity, 0) * 0.06)), transparent 62%), linear-gradient(
90deg,
rgba(
var(--classifier-rgb, 192, 200, 210),
calc(0.012 + var(--classifier-intensity, 0) * 0.06)
),
transparent 62%
),
oklch(0.98 0.008 250 / 0.012); oklch(0.98 0.008 250 / 0.012);
color: inherit; color: inherit;
font: inherit; font: inherit;
@ -1740,13 +1791,18 @@ h3 {
.options-table-row:focus-visible { .options-table-row:focus-visible {
outline: none; outline: none;
background: background:
linear-gradient(90deg, rgba(var(--classifier-rgb, 192, 200, 210), calc(0.02 + var(--classifier-intensity, 0) * 0.1)), transparent 68%), linear-gradient(
90deg,
rgba(var(--classifier-rgb, 192, 200, 210), calc(0.02 + var(--classifier-intensity, 0) * 0.1)),
transparent 68%
),
oklch(0.78 0.12 74 / 0.03); oklch(0.78 0.12 74 / 0.03);
} }
.options-table-row.is-classified { .options-table-row.is-classified {
cursor: pointer; cursor: pointer;
box-shadow: inset 0 0 0 1px rgba(var(--classifier-rgb), calc(0.16 + var(--classifier-intensity) * 0.12)); box-shadow: inset 0 0 0 1px
rgba(var(--classifier-rgb), calc(0.16 + var(--classifier-intensity) * 0.12));
} }
.options-table-row > span { .options-table-row > span {
@ -1761,17 +1817,39 @@ h3 {
font-variant-numeric: tabular-nums; font-variant-numeric: tabular-nums;
} }
.classifier-green { --classifier-rgb: 37, 193, 122; } .classifier-green {
.classifier-red { --classifier-rgb: 255, 107, 95; } --classifier-rgb: 37, 193, 122;
.classifier-amber { --classifier-rgb: 245, 166, 35; } }
.classifier-copper { --classifier-rgb: 198, 122, 75; } .classifier-red {
.classifier-blue { --classifier-rgb: 77, 163, 255; } --classifier-rgb: 255, 107, 95;
.classifier-teal { --classifier-rgb: 64, 210, 190; } }
.classifier-yellowgreen { --classifier-rgb: 174, 210, 78; } .classifier-amber {
.classifier-violet { --classifier-rgb: 170, 130, 255; } --classifier-rgb: 245, 166, 35;
.classifier-cyan { --classifier-rgb: 94, 214, 255; } }
.classifier-magenta { --classifier-rgb: 255, 92, 205; } .classifier-copper {
.classifier-neutral { --classifier-rgb: 192, 200, 210; } --classifier-rgb: 198, 122, 75;
}
.classifier-blue {
--classifier-rgb: 77, 163, 255;
}
.classifier-teal {
--classifier-rgb: 64, 210, 190;
}
.classifier-yellowgreen {
--classifier-rgb: 174, 210, 78;
}
.classifier-violet {
--classifier-rgb: 170, 130, 255;
}
.classifier-cyan {
--classifier-rgb: 94, 214, 255;
}
.classifier-magenta {
--classifier-rgb: 255, 92, 205;
}
.classifier-neutral {
--classifier-rgb: 192, 200, 210;
}
.contract, .contract,
.drawer-row-title { .drawer-row-title {
@ -1921,7 +1999,9 @@ h3 {
opacity: 0; opacity: 0;
pointer-events: none; pointer-events: none;
transform: translateY(8px); transform: translateY(8px);
transition: opacity 0.15s ease, transform 0.15s ease; transition:
opacity 0.15s ease,
transform 0.15s ease;
z-index: 5; z-index: 5;
} }
@ -2047,7 +2127,10 @@ h3 {
color: var(--text-dim); color: var(--text-dim);
box-shadow: 0 10px 28px rgba(0, 0, 0, 0.28); box-shadow: 0 10px 28px rgba(0, 0, 0, 0.28);
z-index: 45; z-index: 45;
transition: border-color 0.16s ease, background-color 0.16s ease, color 0.16s ease; transition:
border-color 0.16s ease,
background-color 0.16s ease,
color 0.16s ease;
} }
.synthetic-control-gear:hover, .synthetic-control-gear:hover,
@ -2213,7 +2296,9 @@ h3 {
background: oklch(0.18 0.012 250 / 0.6); background: oklch(0.18 0.012 250 / 0.6);
color: var(--text); color: var(--text);
text-align: left; text-align: left;
transition: border-color 150ms ease, background 150ms ease; transition:
border-color 150ms ease,
background 150ms ease;
} }
.news-row:hover { .news-row:hover {
@ -2520,7 +2605,11 @@ h3 {
@media (max-width: 720px) { @media (max-width: 720px) {
.terminal-shell { .terminal-shell {
background-size: 24px 24px, 24px 24px, 100% 100%, auto; background-size:
24px 24px,
24px 24px,
100% 100%,
auto;
} }
.terminal-nav-drawer { .terminal-nav-drawer {
@ -2877,9 +2966,7 @@ h3 {
width: 34px; width: 34px;
height: 34px; height: 34px;
border-radius: 9px; border-radius: 9px;
background: background: linear-gradient(135deg, oklch(0.68 0.14 246), oklch(0.68 0.12 164)), var(--blue-soft);
linear-gradient(135deg, oklch(0.68 0.14 246), oklch(0.68 0.12 164)),
var(--blue-soft);
box-shadow: inset 0 0 0 1px oklch(0.94 0.02 240 / 0.24); box-shadow: inset 0 0 0 1px oklch(0.94 0.02 240 / 0.24);
} }

View file

@ -311,12 +311,16 @@ describe("live manifest", () => {
}); });
it("includes news subscriptions on home and /news", () => { it("includes news subscriptions on home and /news", () => {
expect(getLiveManifest("/", "SPY", 60000, buildDefaultFlowFilters()).map((subscription) => subscription.channel)).toContain( expect(
"news" getLiveManifest("/", "SPY", 60000, buildDefaultFlowFilters()).map(
); (subscription) => subscription.channel
expect(getLiveManifest("/news", "SPY", 60000, buildDefaultFlowFilters()).map((subscription) => subscription.channel)).toEqual([ )
"news" ).toContain("news");
]); expect(
getLiveManifest("/news", "SPY", 60000, buildDefaultFlowFilters()).map(
(subscription) => subscription.channel
)
).toEqual(["news"]);
}); });
it("scopes /charts subscriptions to chart channels only", () => { it("scopes /charts subscriptions to chart channels only", () => {
@ -520,12 +524,36 @@ describe("route feature map", () => {
describe("fixed tape virtualization config", () => { describe("fixed tape virtualization config", () => {
it("uses expected fixed row heights and overscan by table", () => { it("uses expected fixed row heights and overscan by table", () => {
expect(getTapeVirtualConfig("options")).toEqual({ rowHeight: 36, overscan: 44, debugLabel: "options" }); expect(getTapeVirtualConfig("options")).toEqual({
expect(getTapeVirtualConfig("equities")).toEqual({ rowHeight: 36, overscan: 36, debugLabel: "equities" }); rowHeight: 36,
expect(getTapeVirtualConfig("flow")).toEqual({ rowHeight: 44, overscan: 24, debugLabel: "flow" }); overscan: 44,
expect(getTapeVirtualConfig("alerts")).toEqual({ rowHeight: 44, overscan: 24, debugLabel: "alerts" }); debugLabel: "options"
expect(getTapeVirtualConfig("classifier")).toEqual({ rowHeight: 44, overscan: 24, debugLabel: "classifier" }); });
expect(getTapeVirtualConfig("dark")).toEqual({ rowHeight: 44, overscan: 24, debugLabel: "dark" }); expect(getTapeVirtualConfig("equities")).toEqual({
rowHeight: 36,
overscan: 36,
debugLabel: "equities"
});
expect(getTapeVirtualConfig("flow")).toEqual({
rowHeight: 44,
overscan: 24,
debugLabel: "flow"
});
expect(getTapeVirtualConfig("alerts")).toEqual({
rowHeight: 44,
overscan: 24,
debugLabel: "alerts"
});
expect(getTapeVirtualConfig("classifier")).toEqual({
rowHeight: 44,
overscan: 24,
debugLabel: "classifier"
});
expect(getTapeVirtualConfig("dark")).toEqual({
rowHeight: 44,
overscan: 24,
debugLabel: "dark"
});
}); });
}); });
@ -712,7 +740,11 @@ describe("live tape history helpers", () => {
}); });
it("promotes hot-window overflow into the history tail", () => { it("promotes hot-window overflow into the history tail", () => {
const currentHot = [makeItem("hot-3", 3, 300), makeItem("hot-2", 2, 200), makeItem("hot-1", 1, 100)]; const currentHot = [
makeItem("hot-3", 3, 300),
makeItem("hot-2", 2, 200),
makeItem("hot-1", 1, 100)
];
const incoming = [makeItem("hot-4", 4, 400)]; const incoming = [makeItem("hot-4", 4, 400)];
const { kept, evicted } = mergeNewestWithOverflow(incoming, currentHot, 3); const { kept, evicted } = mergeNewestWithOverflow(incoming, currentHot, 3);
@ -727,7 +759,11 @@ describe("live tape history helpers", () => {
let history: Array<ReturnType<typeof makeItem>> = []; let history: Array<ReturnType<typeof makeItem>> = [];
for (let seq = 1; seq <= 5; seq += 1) { for (let seq = 1; seq <= 5; seq += 1) {
const { kept, evicted } = mergeNewestWithOverflow([makeItem(`row-${seq}`, seq, seq * 100)], hot, 2); const { kept, evicted } = mergeNewestWithOverflow(
[makeItem(`row-${seq}`, seq, seq * 100)],
hot,
2
);
hot = kept; hot = kept;
history = appendHistoryTail(history, evicted, hot, 5000); history = appendHistoryTail(history, evicted, hot, 5000);
} }
@ -762,13 +798,24 @@ describe("live tape history helpers", () => {
}); });
it("dedupes the seam between promoted overflow and fetched history", () => { it("dedupes the seam between promoted overflow and fetched history", () => {
const currentHot = [makeItem("hot-3", 3, 300), makeItem("hot-2", 2, 200), makeItem("hot-1", 1, 100)]; const currentHot = [
makeItem("hot-3", 3, 300),
makeItem("hot-2", 2, 200),
makeItem("hot-1", 1, 100)
];
const { kept, evicted } = mergeNewestWithOverflow([makeItem("hot-4", 4, 400)], currentHot, 3); const { kept, evicted } = mergeNewestWithOverflow([makeItem("hot-4", 4, 400)], currentHot, 3);
const promoted = appendHistoryTail([], evicted, kept, 5000); const promoted = appendHistoryTail([], evicted, kept, 5000);
const merged = appendHistoryTail(promoted, [makeItem("hot-1", 1, 100), makeItem("older", 0, 50)], kept, 5000); const merged = appendHistoryTail(
promoted,
[makeItem("hot-1", 1, 100), makeItem("older", 0, 50)],
kept,
5000
);
expect(merged.map((item) => item.trace_id)).toEqual(["hot-1", "older"]); expect(merged.map((item) => item.trace_id)).toEqual(["hot-1", "older"]);
expect(new Set([...kept, ...merged].map((item) => item.trace_id)).size).toBe(kept.length + merged.length); expect(new Set([...kept, ...merged].map((item) => item.trace_id)).size).toBe(
kept.length + merged.length
);
}); });
it("trims the history tail to the soft cap", () => { it("trims the history tail to the soft cap", () => {
@ -821,10 +868,9 @@ describe("live tape history helpers", () => {
makeItem("hist-2", 2, 200) makeItem("hist-2", 2, 200)
]; ];
expect(mergeHeldTapeHistory(displayed, incoming, frozenLive).map((item) => item.trace_id)).toEqual([ expect(
"hist-3", mergeHeldTapeHistory(displayed, incoming, frozenLive).map((item) => item.trace_id)
"hist-2" ).toEqual(["hist-3", "hist-2"]);
]);
}); });
it("appends truly older lazy-loaded rows to the held history tail", () => { it("appends truly older lazy-loaded rows to the held history tail", () => {
@ -837,12 +883,9 @@ describe("live tape history helpers", () => {
makeItem("older-0", 0, 50) makeItem("older-0", 0, 50)
]; ];
expect(mergeHeldTapeHistory(displayed, incoming, frozenLive).map((item) => item.trace_id)).toEqual([ expect(
"hist-3", mergeHeldTapeHistory(displayed, incoming, frozenLive).map((item) => item.trace_id)
"hist-2", ).toEqual(["hist-3", "hist-2", "older-1", "older-0"]);
"older-1",
"older-0"
]);
}); });
it("resyncs buffered live history by replacing the held segment after resume", () => { it("resyncs buffered live history by replacing the held segment after resume", () => {
@ -855,7 +898,12 @@ describe("live tape history helpers", () => {
const resynced = appendHistoryTail([], [makeItem("overflow-newer", 6, 600), ...held], [], 0); const resynced = appendHistoryTail([], [makeItem("overflow-newer", 6, 600), ...held], [], 0);
expect(held.map((item) => item.trace_id)).toEqual(["hist-3", "hist-2", "older-1"]); expect(held.map((item) => item.trace_id)).toEqual(["hist-3", "hist-2", "older-1"]);
expect(resynced.map((item) => item.trace_id)).toEqual(["overflow-newer", "hist-3", "hist-2", "older-1"]); expect(resynced.map((item) => item.trace_id)).toEqual([
"overflow-newer",
"hist-3",
"hist-2",
"older-1"
]);
}); });
}); });
@ -935,9 +983,21 @@ describe("classifier row decoration helpers", () => {
it("selects primary hits by confidence, source timestamp, then seq", () => { it("selects primary hits by confidence, source timestamp, then seq", () => {
const hit = selectPrimaryClassifierHit([ const hit = selectPrimaryClassifierHit([
{ ...makeAlert({ classifier_id: "old", confidence: 0.9, source_ts: 1_000, seq: 1 }), direction: "bullish", explanations: [] }, {
{ ...makeAlert({ classifier_id: "new", confidence: 0.9, source_ts: 2_000, seq: 1 }), direction: "bullish", explanations: [] }, ...makeAlert({ classifier_id: "old", confidence: 0.9, source_ts: 1_000, seq: 1 }),
{ ...makeAlert({ classifier_id: "low", confidence: 0.5, source_ts: 3_000, seq: 9 }), direction: "bullish", explanations: [] } direction: "bullish",
explanations: []
},
{
...makeAlert({ classifier_id: "new", confidence: 0.9, source_ts: 2_000, seq: 1 }),
direction: "bullish",
explanations: []
},
{
...makeAlert({ classifier_id: "low", confidence: 0.5, source_ts: 3_000, seq: 9 }),
direction: "bullish",
explanations: []
}
]); ]);
expect(hit?.classifier_id).toBe("new"); expect(hit?.classifier_id).toBe("new");
@ -1010,9 +1070,9 @@ describe("signals helpers", () => {
) )
).toBe("bearish"); ).toBe("bearish");
expect(deriveAlertDirection(makeAlert({ hits: [{ direction: "weird", confidence: 0.4 }] }))).toBe( expect(
"neutral" deriveAlertDirection(makeAlert({ hits: [{ direction: "weird", confidence: 0.4 }] }))
); ).toBe("neutral");
expect(deriveAlertDirection(makeAlert({ hits: [] }))).toBe("neutral"); expect(deriveAlertDirection(makeAlert({ hits: [] }))).toBe("neutral");
}); });

File diff suppressed because it is too large Load diff

View file

@ -2,11 +2,7 @@
"extends": "../../tsconfig.base.json", "extends": "../../tsconfig.base.json",
"compilerOptions": { "compilerOptions": {
"jsx": "preserve", "jsx": "preserve",
"lib": [ "lib": ["DOM", "DOM.Iterable", "ES2022"],
"DOM",
"DOM.Iterable",
"ES2022"
],
"incremental": true, "incremental": true,
"noEmit": true, "noEmit": true,
"allowJs": true, "allowJs": true,
@ -24,8 +20,5 @@
".next/types/**/*.ts", ".next/types/**/*.ts",
".next-dev/types/**/*.ts" ".next-dev/types/**/*.ts"
], ],
"exclude": [ "exclude": ["node_modules", "scripts"]
"node_modules",
"scripts"
]
} }

93
biome.json Normal file
View file

@ -0,0 +1,93 @@
{
"$schema": "https://biomejs.dev/schemas/2.4.16/schema.json",
"vcs": {
"enabled": true,
"clientKind": "git",
"useIgnoreFile": true
},
"files": {
"includes": [
"*.json",
"*.ts",
".forgejo/workflows/*.yml",
"apps/**",
"deployment/docker/workspace-root/package.json",
"packages/**",
"scripts/**",
"services/**",
"!**/node_modules",
"!**/.next",
"!**/dist",
"!**/out",
"!**/coverage",
"!apps/web/tsconfig.tsbuildinfo"
]
},
"formatter": {
"enabled": true,
"indentStyle": "space",
"indentWidth": 2,
"lineWidth": 100
},
"linter": {
"enabled": true,
"rules": {
"recommended": true,
"a11y": {
"useAriaPropsSupportedByRole": "off",
"useFocusableInteractive": "off",
"useSemanticElements": "off"
},
"complexity": {
"noImportantStyles": "off",
"noUselessContinue": "off",
"noUselessSwitchCase": "off",
"noUselessUndefinedInitialization": "off",
"useOptionalChain": "off"
},
"correctness": {
"useExhaustiveDependencies": "off",
"noUnusedFunctionParameters": "off",
"noUnusedImports": "off",
"noUnusedVariables": "off"
},
"suspicious": {
"noArrayIndexKey": "off",
"noControlCharactersInRegex": "off",
"noExplicitAny": "off",
"noAssignInExpressions": "off",
"noShorthandPropertyOverrides": "off"
},
"security": {
"noDangerouslySetInnerHtml": "off"
},
"style": {
"noDescendingSpecificity": "off",
"noNonNullAssertion": "off",
"useExponentiationOperator": "off",
"useImportType": "off",
"useTemplate": "off"
}
}
},
"javascript": {
"formatter": {
"quoteStyle": "double",
"semicolons": "always",
"trailingCommas": "none"
}
},
"json": {
"parser": {
"allowComments": true,
"allowTrailingCommas": true
}
},
"assist": {
"actions": {
"source": {
"organizeImports": "on"
}
}
}
}

View file

@ -8,6 +8,7 @@
"@pierre/diffs": "^1.2.2", "@pierre/diffs": "^1.2.2",
}, },
"devDependencies": { "devDependencies": {
"@biomejs/biome": "^2.4.16",
"@types/bun": "^1.3.3", "@types/bun": "^1.3.3",
"@types/ws": "^8.18.1", "@types/ws": "^8.18.1",
"typescript": "^5.9.3", "typescript": "^5.9.3",
@ -178,6 +179,24 @@
"tmp": "^0.2.5", "tmp": "^0.2.5",
}, },
"packages": { "packages": {
"@biomejs/biome": ["@biomejs/biome@2.4.16", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.16", "@biomejs/cli-darwin-x64": "2.4.16", "@biomejs/cli-linux-arm64": "2.4.16", "@biomejs/cli-linux-arm64-musl": "2.4.16", "@biomejs/cli-linux-x64": "2.4.16", "@biomejs/cli-linux-x64-musl": "2.4.16", "@biomejs/cli-win32-arm64": "2.4.16", "@biomejs/cli-win32-x64": "2.4.16" }, "bin": { "biome": "bin/biome" } }, "sha512-x9ajFh1zChVybCiM3TN6OD4phAqLgtPZjFrZF+aTMYCPjwBO+k529TX7PPsAqtGNLeV4UgzwQnowEgS7bGmzcA=="],
"@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.4.16", "", { "os": "darwin", "cpu": "arm64" }, "sha512-wxPvu4XOA85YJk9ixSWUmq/QBHbid85BISbOAqqBM/5xQpPk9ayjk5375tOlSC0BeCwNSbPFafQBm+vBumXq0A=="],
"@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.4.16", "", { "os": "darwin", "cpu": "x64" }, "sha512-xFCqGPwYusQJp4N4NJLi1XJiZqjwFdjhT+KqtNy+Ug3qgfczqnTa6MSDvxJF6TkuDLoYJItMapz6tAf7kCekFw=="],
"@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.4.16", "", { "os": "linux", "cpu": "arm64" }, "sha512-2kFb4//jxfZaP6D+Rj5VkHkxgyD9EoRAVBEQb8PKRv+s4NO2zYNJKXFaJmK1CmhufJOWEfpHKaRbOja7qjmdhQ=="],
"@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.4.16", "", { "os": "linux", "cpu": "arm64" }, "sha512-oYxnW0ARfJkr72ezzF2OR8N/rtkgLUQeYtF8cFhVswbknHxtTcmzSsanVJP8yQKnGpGpc2ck6c5zLvHahL6Cbg=="],
"@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.4.16", "", { "os": "linux", "cpu": "x64" }, "sha512-NbcBbi/nJqn5baae6wqRXdS7Gadf2uRpehSh6vMSYpG8OhkXl/Xg8aorWrJ+9VWqAT5ml90alLvorkpMW0nBwQ=="],
"@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.4.16", "", { "os": "linux", "cpu": "x64" }, "sha512-iHDS+MCM65DPqWGu+ECC3uoALyj2H7F4nVUPxIPjz/PIl94EUu+EDfGZDzFP+NY1EOPVt9NQvwFqq7HdMmowdg=="],
"@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.4.16", "", { "os": "win32", "cpu": "arm64" }, "sha512-0rgImMsNb5v/chhkIFe3wu7PEFClS6RBAYUijGL9UsYN3PanSaoK24HSSuSJb1pYbYYVjzAyZTl3gtjJ84BM8A=="],
"@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.16", "", { "os": "win32", "cpu": "x64" }, "sha512-Kp85jgoBHa05gix6UIRjfCDiUV3w/8VIdZ247VyyO2gEjaw12WEVhdIjlxp/AMzXxqxQwbxNTDVZ3Mwd2RG5rw=="],
"@clickhouse/client": ["@clickhouse/client@0.2.10", "", { "dependencies": { "@clickhouse/client-common": "0.2.10" } }, "sha512-ZwBgzjEAFN/ogS0ym5KHVbR7Hx/oYCX01qGp2baEyfN2HM73kf/7Vp3GvMHWRy+zUXISONEtFv7UTViOXnmFrg=="], "@clickhouse/client": ["@clickhouse/client@0.2.10", "", { "dependencies": { "@clickhouse/client-common": "0.2.10" } }, "sha512-ZwBgzjEAFN/ogS0ym5KHVbR7Hx/oYCX01qGp2baEyfN2HM73kf/7Vp3GvMHWRy+zUXISONEtFv7UTViOXnmFrg=="],
"@clickhouse/client-common": ["@clickhouse/client-common@0.2.10", "", {}, "sha512-BvTY0IXS96y9RUeNCpKL4HUzHmY80L0lDcGN0lmUD6zjOqYMn78+xyHYJ/AIAX7JQsc+/KwFt2soZutQTKxoGQ=="], "@clickhouse/client-common": ["@clickhouse/client-common@0.2.10", "", {}, "sha512-BvTY0IXS96y9RUeNCpKL4HUzHmY80L0lDcGN0lmUD6zjOqYMn78+xyHYJ/AIAX7JQsc+/KwFt2soZutQTKxoGQ=="],

View file

@ -8,6 +8,7 @@
"@pierre/diffs": "^1.2.2", "@pierre/diffs": "^1.2.2",
}, },
"devDependencies": { "devDependencies": {
"@biomejs/biome": "^2.4.16",
"@types/bun": "^1.3.3", "@types/bun": "^1.3.3",
"@types/ws": "^8.18.1", "@types/ws": "^8.18.1",
"typescript": "^5.9.3", "typescript": "^5.9.3",
@ -178,6 +179,24 @@
"tmp": "^0.2.5", "tmp": "^0.2.5",
}, },
"packages": { "packages": {
"@biomejs/biome": ["@biomejs/biome@2.4.16", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.16", "@biomejs/cli-darwin-x64": "2.4.16", "@biomejs/cli-linux-arm64": "2.4.16", "@biomejs/cli-linux-arm64-musl": "2.4.16", "@biomejs/cli-linux-x64": "2.4.16", "@biomejs/cli-linux-x64-musl": "2.4.16", "@biomejs/cli-win32-arm64": "2.4.16", "@biomejs/cli-win32-x64": "2.4.16" }, "bin": { "biome": "bin/biome" } }, "sha512-x9ajFh1zChVybCiM3TN6OD4phAqLgtPZjFrZF+aTMYCPjwBO+k529TX7PPsAqtGNLeV4UgzwQnowEgS7bGmzcA=="],
"@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.4.16", "", { "os": "darwin", "cpu": "arm64" }, "sha512-wxPvu4XOA85YJk9ixSWUmq/QBHbid85BISbOAqqBM/5xQpPk9ayjk5375tOlSC0BeCwNSbPFafQBm+vBumXq0A=="],
"@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.4.16", "", { "os": "darwin", "cpu": "x64" }, "sha512-xFCqGPwYusQJp4N4NJLi1XJiZqjwFdjhT+KqtNy+Ug3qgfczqnTa6MSDvxJF6TkuDLoYJItMapz6tAf7kCekFw=="],
"@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.4.16", "", { "os": "linux", "cpu": "arm64" }, "sha512-2kFb4//jxfZaP6D+Rj5VkHkxgyD9EoRAVBEQb8PKRv+s4NO2zYNJKXFaJmK1CmhufJOWEfpHKaRbOja7qjmdhQ=="],
"@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.4.16", "", { "os": "linux", "cpu": "arm64" }, "sha512-oYxnW0ARfJkr72ezzF2OR8N/rtkgLUQeYtF8cFhVswbknHxtTcmzSsanVJP8yQKnGpGpc2ck6c5zLvHahL6Cbg=="],
"@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.4.16", "", { "os": "linux", "cpu": "x64" }, "sha512-NbcBbi/nJqn5baae6wqRXdS7Gadf2uRpehSh6vMSYpG8OhkXl/Xg8aorWrJ+9VWqAT5ml90alLvorkpMW0nBwQ=="],
"@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.4.16", "", { "os": "linux", "cpu": "x64" }, "sha512-iHDS+MCM65DPqWGu+ECC3uoALyj2H7F4nVUPxIPjz/PIl94EUu+EDfGZDzFP+NY1EOPVt9NQvwFqq7HdMmowdg=="],
"@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.4.16", "", { "os": "win32", "cpu": "arm64" }, "sha512-0rgImMsNb5v/chhkIFe3wu7PEFClS6RBAYUijGL9UsYN3PanSaoK24HSSuSJb1pYbYYVjzAyZTl3gtjJ84BM8A=="],
"@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.16", "", { "os": "win32", "cpu": "x64" }, "sha512-Kp85jgoBHa05gix6UIRjfCDiUV3w/8VIdZ247VyyO2gEjaw12WEVhdIjlxp/AMzXxqxQwbxNTDVZ3Mwd2RG5rw=="],
"@clickhouse/client": ["@clickhouse/client@0.2.10", "", { "dependencies": { "@clickhouse/client-common": "0.2.10" } }, "sha512-ZwBgzjEAFN/ogS0ym5KHVbR7Hx/oYCX01qGp2baEyfN2HM73kf/7Vp3GvMHWRy+zUXISONEtFv7UTViOXnmFrg=="], "@clickhouse/client": ["@clickhouse/client@0.2.10", "", { "dependencies": { "@clickhouse/client-common": "0.2.10" } }, "sha512-ZwBgzjEAFN/ogS0ym5KHVbR7Hx/oYCX01qGp2baEyfN2HM73kf/7Vp3GvMHWRy+zUXISONEtFv7UTViOXnmFrg=="],
"@clickhouse/client-common": ["@clickhouse/client-common@0.2.10", "", {}, "sha512-BvTY0IXS96y9RUeNCpKL4HUzHmY80L0lDcGN0lmUD6zjOqYMn78+xyHYJ/AIAX7JQsc+/KwFt2soZutQTKxoGQ=="], "@clickhouse/client-common": ["@clickhouse/client-common@0.2.10", "", {}, "sha512-BvTY0IXS96y9RUeNCpKL4HUzHmY80L0lDcGN0lmUD6zjOqYMn78+xyHYJ/AIAX7JQsc+/KwFt2soZutQTKxoGQ=="],

View file

@ -15,6 +15,10 @@
"dev:desktop:remote": "bun run scripts/dev-desktop.ts --remote", "dev:desktop:remote": "bun run scripts/dev-desktop.ts --remote",
"dev:web": "bun --cwd=apps/web run dev", "dev:web": "bun --cwd=apps/web run dev",
"dev:services": "bun run scripts/dev-services.ts", "dev:services": "bun run scripts/dev-services.ts",
"fmt": "biome format --write .",
"fmt:check": "biome format .",
"lint": "biome lint .",
"check": "biome check .",
"package:desktop": "bun --cwd=apps/desktop run package", "package:desktop": "bun --cwd=apps/desktop run package",
"make:desktop": "bun --cwd=apps/desktop run make", "make:desktop": "bun --cwd=apps/desktop run make",
"deploy": "bun run scripts/deploy.ts", "deploy": "bun run scripts/deploy.ts",
@ -26,6 +30,7 @@
"check:docker-workspace": "bun run scripts/check-docker-workspace.ts" "check:docker-workspace": "bun run scripts/check-docker-workspace.ts"
}, },
"devDependencies": { "devDependencies": {
"@biomejs/biome": "^2.4.16",
"@types/bun": "^1.3.3", "@types/bun": "^1.3.3",
"@types/ws": "^8.18.1", "@types/ws": "^8.18.1",
"typescript": "^5.9.3", "typescript": "^5.9.3",

View file

@ -8,6 +8,6 @@
"isolatedModules": true, "isolatedModules": true,
"resolveJsonModule": true, "resolveJsonModule": true,
"skipLibCheck": true, "skipLibCheck": true,
"noEmit": true, "noEmit": true
}, }
} }

File diff suppressed because one or more lines are too long

View file

@ -15,6 +15,10 @@
"dev:desktop:remote": "bun run scripts/dev-desktop.ts --remote", "dev:desktop:remote": "bun run scripts/dev-desktop.ts --remote",
"dev:web": "bun --cwd=apps/web run dev", "dev:web": "bun --cwd=apps/web run dev",
"dev:services": "bun run scripts/dev-services.ts", "dev:services": "bun run scripts/dev-services.ts",
"fmt": "biome format --write .",
"fmt:check": "biome format .",
"lint": "biome lint .",
"check": "biome check .",
"package:desktop": "bun --cwd=apps/desktop run package", "package:desktop": "bun --cwd=apps/desktop run package",
"make:desktop": "bun --cwd=apps/desktop run make", "make:desktop": "bun --cwd=apps/desktop run make",
"deploy": "bun run scripts/deploy.ts", "deploy": "bun run scripts/deploy.ts",
@ -26,6 +30,7 @@
"check:docker-workspace": "bun run scripts/check-docker-workspace.ts" "check:docker-workspace": "bun run scripts/check-docker-workspace.ts"
}, },
"devDependencies": { "devDependencies": {
"@biomejs/biome": "^2.4.16",
"@types/bun": "^1.3.3", "@types/bun": "^1.3.3",
"@types/ws": "^8.18.1", "@types/ws": "^8.18.1",
"typescript": "^5.9.3", "typescript": "^5.9.3",

View file

@ -16,7 +16,11 @@ import {
nanos, nanos,
millis millis
} from "nats"; } from "nats";
import { getKnownStreamDefinitions, getStreamDefinition, type StreamRetentionClass } from "./streams"; import {
getKnownStreamDefinitions,
getStreamDefinition,
type StreamRetentionClass
} from "./streams";
export type NatsConnectionOptions = { export type NatsConnectionOptions = {
servers: string | string[]; servers: string | string[];
@ -251,7 +255,8 @@ const diffConfigFields = (
for (const field of fields) { for (const field of fields) {
const currentValue = getFieldValue(current, field); const currentValue = getFieldValue(current, field);
const desiredValue = getFieldValue(desired, field); const desiredValue = getFieldValue(desired, field);
const matches = Array.isArray(currentValue) && Array.isArray(desiredValue) const matches =
Array.isArray(currentValue) && Array.isArray(desiredValue)
? arraysEqual(currentValue, desiredValue) ? arraysEqual(currentValue, desiredValue)
: currentValue === desiredValue; : currentValue === desiredValue;
@ -391,7 +396,10 @@ const formatStructuredValue = (value: unknown): string => {
const formatStructuralMismatchMessage = (audit: StreamAuditReport): string => { const formatStructuralMismatchMessage = (audit: StreamAuditReport): string => {
const details = audit.structuralMismatch const details = audit.structuralMismatch
.map((delta) => `${delta.field} current=${formatStructuredValue(delta.current)} desired=${formatStructuredValue(delta.desired)}`) .map(
(delta) =>
`${delta.field} current=${formatStructuredValue(delta.current)} desired=${formatStructuredValue(delta.desired)}`
)
.join("; "); .join("; ");
return `Refusing to reconcile stream ${audit.name}: structural mismatch (${details})`; return `Refusing to reconcile stream ${audit.name}: structural mismatch (${details})`;
}; };
@ -447,12 +455,14 @@ const formatReportLine = (
case "retention_drift": { case "retention_drift": {
const details = report.retentionDrift const details = report.retentionDrift
.map((delta) => { .map((delta) => {
const desiredValue = delta.field === "max_age" const desiredValue =
delta.field === "max_age"
? formatDurationMs(millis(Number(delta.desired))) ? formatDurationMs(millis(Number(delta.desired)))
: delta.field === "max_bytes" : delta.field === "max_bytes"
? formatBytes(Number(delta.desired)) ? formatBytes(Number(delta.desired))
: formatStructuredValue(delta.desired); : formatStructuredValue(delta.desired);
const currentValue = delta.field === "max_age" const currentValue =
delta.field === "max_age"
? formatDurationMs(millis(Number(delta.current))) ? formatDurationMs(millis(Number(delta.current)))
: delta.field === "max_bytes" : delta.field === "max_bytes"
? formatBytes(Number(delta.current)) ? formatBytes(Number(delta.current))
@ -464,7 +474,10 @@ const formatReportLine = (
} }
case "structural_mismatch": { case "structural_mismatch": {
const details = report.structuralMismatch const details = report.structuralMismatch
.map((delta) => `${delta.field}:${formatStructuredValue(delta.current)}->${formatStructuredValue(delta.desired)}`) .map(
(delta) =>
`${delta.field}:${formatStructuredValue(delta.current)}->${formatStructuredValue(delta.desired)}`
)
.join(" "); .join(" ");
return `${report.name} structural-mismatch ${details}`; return `${report.name} structural-mismatch ${details}`;
} }

View file

@ -59,7 +59,9 @@ export const STREAM_CATALOG: readonly KnownStreamDefinition[] = [
{ name: STREAM_NEWS, subject: SUBJECT_NEWS, retentionClass: "derived" } { name: STREAM_NEWS, subject: SUBJECT_NEWS, retentionClass: "derived" }
]; ];
const STREAM_CATALOG_BY_NAME = new Map(STREAM_CATALOG.map((definition) => [definition.name, definition])); const STREAM_CATALOG_BY_NAME = new Map(
STREAM_CATALOG.map((definition) => [definition.name, definition])
);
export const getKnownStreamDefinitions = (): readonly KnownStreamDefinition[] => { export const getKnownStreamDefinitions = (): readonly KnownStreamDefinition[] => {
return STREAM_CATALOG; return STREAM_CATALOG;

View file

@ -11,44 +11,31 @@ export const SYNTHETIC_CONTROL_GLOBAL_KEY = "global";
const codec = JSONCodec<SyntheticControlState>(); const codec = JSONCodec<SyntheticControlState>();
const decodeSyntheticControlEntry = ( const decodeSyntheticControlEntry = (entry: KvEntry | null | undefined): SyntheticControlState => {
entry: KvEntry | null | undefined
): SyntheticControlState => {
if (!entry || entry.operation !== "PUT") { if (!entry || entry.operation !== "PUT") {
return DEFAULT_SYNTHETIC_CONTROL_STATE; return DEFAULT_SYNTHETIC_CONTROL_STATE;
} }
return SyntheticControlStateSchema.parse(entry.json()); return SyntheticControlStateSchema.parse(entry.json());
}; };
export const openSyntheticControlKv = async ( export const openSyntheticControlKv = async (js: JetStreamClient): Promise<KV> => {
js: JetStreamClient
): Promise<KV> => {
return js.views.kv(SYNTHETIC_CONTROL_BUCKET, { return js.views.kv(SYNTHETIC_CONTROL_BUCKET, {
description: "Hosted synthetic market internal control state", description: "Hosted synthetic market internal control state",
history: 8 history: 8
}); });
}; };
export const readSyntheticControlState = async ( export const readSyntheticControlState = async (kv: KV): Promise<SyntheticControlState> => {
kv: KV return decodeSyntheticControlEntry(await kv.get(SYNTHETIC_CONTROL_GLOBAL_KEY));
): Promise<SyntheticControlState> => {
return decodeSyntheticControlEntry(
await kv.get(SYNTHETIC_CONTROL_GLOBAL_KEY)
);
}; };
export const ensureSyntheticControlState = async ( export const ensureSyntheticControlState = async (kv: KV): Promise<SyntheticControlState> => {
kv: KV
): Promise<SyntheticControlState> => {
const current = await kv.get(SYNTHETIC_CONTROL_GLOBAL_KEY); const current = await kv.get(SYNTHETIC_CONTROL_GLOBAL_KEY);
if (current && current.operation === "PUT") { if (current && current.operation === "PUT") {
return SyntheticControlStateSchema.parse(current.json()); return SyntheticControlStateSchema.parse(current.json());
} }
await kv.put( await kv.put(SYNTHETIC_CONTROL_GLOBAL_KEY, codec.encode(DEFAULT_SYNTHETIC_CONTROL_STATE));
SYNTHETIC_CONTROL_GLOBAL_KEY,
codec.encode(DEFAULT_SYNTHETIC_CONTROL_STATE)
);
return DEFAULT_SYNTHETIC_CONTROL_STATE; return DEFAULT_SYNTHETIC_CONTROL_STATE;
}; };
@ -57,10 +44,7 @@ export const writeSyntheticControlState = async (
control: Partial<SyntheticControlState> control: Partial<SyntheticControlState>
): Promise<SyntheticControlState> => { ): Promise<SyntheticControlState> => {
const normalized = normalizeSyntheticControlState(control); const normalized = normalizeSyntheticControlState(control);
await kv.put( await kv.put(SYNTHETIC_CONTROL_GLOBAL_KEY, codec.encode(normalized));
SYNTHETIC_CONTROL_GLOBAL_KEY,
codec.encode(normalized)
);
return normalized; return normalized;
}; };

View file

@ -43,10 +43,9 @@ const buildMockStreamManager = (configs: Record<string, StreamConfig | null>) =>
}; };
const buildAllKnownConfigs = (env: Record<string, string | undefined> = {}) => { const buildAllKnownConfigs = (env: Record<string, string | undefined> = {}) => {
return Object.fromEntries(STREAMS.map((name) => [name, buildKnownStreamConfig(name, env)])) as Record< return Object.fromEntries(
string, STREAMS.map((name) => [name, buildKnownStreamConfig(name, env)])
StreamConfig ) as Record<string, StreamConfig>;
>;
}; };
describe("jetstream retention defaults", () => { describe("jetstream retention defaults", () => {
@ -194,7 +193,9 @@ describe("runReconcileStreamsCommand", () => {
}); });
expect(exitCode).toBe(1); expect(exitCode).toBe(1);
expect(outputs.some((line) => line.includes("OPTIONS_PRINTS") && line.includes("drift"))).toBe(true); expect(outputs.some((line) => line.includes("OPTIONS_PRINTS") && line.includes("drift"))).toBe(
true
);
}); });
it("updates drift in --apply mode and reports actions", async () => { it("updates drift in --apply mode and reports actions", async () => {
@ -240,7 +241,11 @@ describe("runReconcileStreamsCommand", () => {
}); });
expect(exitCode).toBe(1); expect(exitCode).toBe(1);
expect(outputs.some((line) => line.includes("OPTIONS_PRINTS") && line.includes("structural-mismatch"))).toBe(true); expect(
outputs.some(
(line) => line.includes("OPTIONS_PRINTS") && line.includes("structural-mismatch")
)
).toBe(true);
expect(errors.some((line) => line.includes("OPTIONS_PRINTS"))).toBe(true); expect(errors.some((line) => line.includes("OPTIONS_PRINTS"))).toBe(true);
}); });
}); });

View file

@ -15,14 +15,10 @@ type AlpacaCredentialEnv = {
const normalize = (value: string | undefined): string => value?.trim() ?? ""; const normalize = (value: string | undefined): string => value?.trim() ?? "";
export const resolveAlpacaCredentials = ( export const resolveAlpacaCredentials = (env: AlpacaCredentialEnv): AlpacaCredentials => {
env: AlpacaCredentialEnv
): AlpacaCredentials => {
const legacyToken = normalize(env.ALPACA_API_KEY); const legacyToken = normalize(env.ALPACA_API_KEY);
const explicitKeyId = const explicitKeyId = normalize(env.ALPACA_API_KEY_ID) || normalize(env.ALPACA_KEY_ID);
normalize(env.ALPACA_API_KEY_ID) || normalize(env.ALPACA_KEY_ID); const secret = normalize(env.ALPACA_API_SECRET_KEY) || normalize(env.ALPACA_SECRET_KEY);
const secret =
normalize(env.ALPACA_API_SECRET_KEY) || normalize(env.ALPACA_SECRET_KEY);
const keyId = explicitKeyId || legacyToken; const keyId = explicitKeyId || legacyToken;
const usesLegacyBearer = !explicitKeyId && !secret && legacyToken.length > 0; const usesLegacyBearer = !explicitKeyId && !secret && legacyToken.length > 0;
@ -42,9 +38,7 @@ export const hasAlpacaCredentials = (credentials: AlpacaCredentials): boolean =>
return credentials.keyId.length > 0 && credentials.secret.length > 0; return credentials.keyId.length > 0 && credentials.secret.length > 0;
}; };
export const buildAlpacaAuthHeaders = ( export const buildAlpacaAuthHeaders = (credentials: AlpacaCredentials): Record<string, string> => {
credentials: AlpacaCredentials
): Record<string, string> => {
if (credentials.usesLegacyBearer) { if (credentials.usesLegacyBearer) {
return { return {
Authorization: `Bearer ${credentials.legacyToken}` Authorization: `Bearer ${credentials.legacyToken}`

View file

@ -99,7 +99,9 @@ const safeProfileScoreArray = (value: string): SmartMoneyProfileScore[] => {
return { return {
profile_id: String(record.profile_id ?? "") as SmartMoneyProfileScore["profile_id"], profile_id: String(record.profile_id ?? "") as SmartMoneyProfileScore["profile_id"],
probability: Number(record.probability ?? 0), probability: Number(record.probability ?? 0),
confidence_band: String(record.confidence_band ?? "low") as SmartMoneyProfileScore["confidence_band"], confidence_band: String(
record.confidence_band ?? "low"
) as SmartMoneyProfileScore["confidence_band"],
direction: String(record.direction ?? "unknown") as SmartMoneyProfileScore["direction"], direction: String(record.direction ?? "unknown") as SmartMoneyProfileScore["direction"],
reasons: Array.isArray(record.reasons) ? record.reasons.map((item) => String(item)) : [] reasons: Array.isArray(record.reasons) ? record.reasons.map((item) => String(item)) : []
}; };
@ -122,7 +124,9 @@ export const fromAlertRecord = (record: AlertRecord): AlertEvent => {
severity: record.severity, severity: record.severity,
hits: safeHitArray(record.hits_json), hits: safeHitArray(record.hits_json),
evidence_refs: safeStringArray(record.evidence_refs_json), evidence_refs: safeStringArray(record.evidence_refs_json),
...(record.primary_profile_id ? { primary_profile_id: record.primary_profile_id as AlertEvent["primary_profile_id"] } : {}), ...(record.primary_profile_id
? { primary_profile_id: record.primary_profile_id as AlertEvent["primary_profile_id"] }
: {}),
profile_scores: safeProfileScoreArray(record.profile_scores_json) profile_scores: safeProfileScoreArray(record.profile_scores_json)
}; };
}; };

View file

@ -35,16 +35,8 @@ import {
OPTION_PRINTS_TABLE OPTION_PRINTS_TABLE
} from "./option-prints"; } from "./option-prints";
import { normalizeOptionNBBO, optionNBBOTableDDL, OPTION_NBBO_TABLE } from "./option-nbbo"; import { normalizeOptionNBBO, optionNBBOTableDDL, OPTION_NBBO_TABLE } from "./option-nbbo";
import { import { equityPrintsTableDDL, EQUITY_PRINTS_TABLE, normalizeEquityPrint } from "./equity-prints";
equityPrintsTableDDL, import { equityQuotesTableDDL, EQUITY_QUOTES_TABLE, normalizeEquityQuote } from "./equity-quotes";
EQUITY_PRINTS_TABLE,
normalizeEquityPrint
} from "./equity-prints";
import {
equityQuotesTableDDL,
EQUITY_QUOTES_TABLE,
normalizeEquityQuote
} from "./equity-quotes";
import { import {
equityCandlesTableDDL, equityCandlesTableDDL,
EQUITY_CANDLES_TABLE, EQUITY_CANDLES_TABLE,
@ -93,13 +85,7 @@ import {
toSmartMoneyEventRecord, toSmartMoneyEventRecord,
type SmartMoneyEventRecord type SmartMoneyEventRecord
} from "./smart-money-events"; } from "./smart-money-events";
import { import { NEWS_TABLE, newsTableDDL, fromNewsRecord, toNewsRecord, type NewsRecord } from "./news";
NEWS_TABLE,
newsTableDDL,
fromNewsRecord,
toNewsRecord,
type NewsRecord
} from "./news";
export type ClickHouseOptions = { export type ClickHouseOptions = {
url: string; url: string;
@ -116,7 +102,11 @@ type ClickHouseQueryResult = {
export type ClickHouseClient = { export type ClickHouseClient = {
exec(params: { query: string }): Promise<void>; exec(params: { query: string }): Promise<void>;
insert(params: { table: string; values: unknown[]; format: ClickHouseQueryFormat }): Promise<void>; insert(params: {
table: string;
values: unknown[];
format: ClickHouseQueryFormat;
}): Promise<void>;
query(params: { query: string; format: ClickHouseQueryFormat }): Promise<ClickHouseQueryResult>; query(params: { query: string; format: ClickHouseQueryFormat }): Promise<ClickHouseQueryResult>;
ping(): Promise<{ success: boolean; error?: Error }>; ping(): Promise<{ success: boolean; error?: Error }>;
close(): Promise<void>; close(): Promise<void>;
@ -140,7 +130,9 @@ const buildHeaders = (options: ClickHouseOptions, hasBody: boolean): Headers =>
} }
if (options.username || options.password) { if (options.username || options.password) {
const auth = Buffer.from(`${options.username ?? "default"}:${options.password ?? ""}`).toString("base64"); const auth = Buffer.from(`${options.username ?? "default"}:${options.password ?? ""}`).toString(
"base64"
);
headers.set("authorization", `Basic ${auth}`); headers.set("authorization", `Basic ${auth}`);
} }
@ -217,7 +209,8 @@ export const createClickHouseClient = (options: ClickHouseOptions): ClickHouseCl
}); });
if (!response.ok) { if (!response.ok) {
const message = (await response.text()).trim() || `${response.status} ${response.statusText}`; const message =
(await response.text()).trim() || `${response.status} ${response.statusText}`;
return { success: false, error: new Error(message) }; return { success: false, error: new Error(message) };
} }
@ -237,9 +230,7 @@ export const createClickHouseClient = (options: ClickHouseOptions): ClickHouseCl
}; };
}; };
export const ensureOptionPrintsTable = async ( export const ensureOptionPrintsTable = async (client: ClickHouseClient): Promise<void> => {
client: ClickHouseClient
): Promise<void> => {
await client.exec({ await client.exec({
query: optionPrintsTableDDL() query: optionPrintsTableDDL()
}); });
@ -248,73 +239,55 @@ export const ensureOptionPrintsTable = async (
} }
}; };
export const ensureOptionNBBOTable = async ( export const ensureOptionNBBOTable = async (client: ClickHouseClient): Promise<void> => {
client: ClickHouseClient
): Promise<void> => {
await client.exec({ await client.exec({
query: optionNBBOTableDDL() query: optionNBBOTableDDL()
}); });
}; };
export const ensureEquityPrintsTable = async ( export const ensureEquityPrintsTable = async (client: ClickHouseClient): Promise<void> => {
client: ClickHouseClient
): Promise<void> => {
await client.exec({ await client.exec({
query: equityPrintsTableDDL() query: equityPrintsTableDDL()
}); });
}; };
export const ensureEquityQuotesTable = async ( export const ensureEquityQuotesTable = async (client: ClickHouseClient): Promise<void> => {
client: ClickHouseClient
): Promise<void> => {
await client.exec({ await client.exec({
query: equityQuotesTableDDL() query: equityQuotesTableDDL()
}); });
}; };
export const ensureEquityCandlesTable = async ( export const ensureEquityCandlesTable = async (client: ClickHouseClient): Promise<void> => {
client: ClickHouseClient
): Promise<void> => {
await client.exec({ await client.exec({
query: equityCandlesTableDDL() query: equityCandlesTableDDL()
}); });
}; };
export const ensureEquityPrintJoinsTable = async ( export const ensureEquityPrintJoinsTable = async (client: ClickHouseClient): Promise<void> => {
client: ClickHouseClient
): Promise<void> => {
await client.exec({ await client.exec({
query: equityPrintJoinsTableDDL() query: equityPrintJoinsTableDDL()
}); });
}; };
export const ensureInferredDarkTable = async ( export const ensureInferredDarkTable = async (client: ClickHouseClient): Promise<void> => {
client: ClickHouseClient
): Promise<void> => {
await client.exec({ await client.exec({
query: inferredDarkTableDDL() query: inferredDarkTableDDL()
}); });
}; };
export const ensureFlowPacketsTable = async ( export const ensureFlowPacketsTable = async (client: ClickHouseClient): Promise<void> => {
client: ClickHouseClient
): Promise<void> => {
await client.exec({ await client.exec({
query: flowPacketsTableDDL() query: flowPacketsTableDDL()
}); });
}; };
export const ensureSmartMoneyEventsTable = async ( export const ensureSmartMoneyEventsTable = async (client: ClickHouseClient): Promise<void> => {
client: ClickHouseClient
): Promise<void> => {
await client.exec({ await client.exec({
query: smartMoneyEventsTableDDL() query: smartMoneyEventsTableDDL()
}); });
}; };
export const ensureClassifierHitsTable = async ( export const ensureClassifierHitsTable = async (client: ClickHouseClient): Promise<void> => {
client: ClickHouseClient
): Promise<void> => {
await client.exec({ await client.exec({
query: classifierHitsTableDDL() query: classifierHitsTableDDL()
}); });
@ -464,7 +437,10 @@ export const insertAlert = async (client: ClickHouseClient, alert: AlertEvent):
}); });
}; };
export const insertNewsStory = async (client: ClickHouseClient, story: NewsStory): Promise<void> => { export const insertNewsStory = async (
client: ClickHouseClient,
story: NewsStory
): Promise<void> => {
const record = toNewsRecord(story); const record = toNewsRecord(story);
await client.insert({ await client.insert({
table: NEWS_TABLE, table: NEWS_TABLE,
@ -617,17 +593,11 @@ export const enqueueClassifierHitInsert = (
writer.enqueue(CLASSIFIER_HITS_TABLE, toClassifierHitRecord(hit)); writer.enqueue(CLASSIFIER_HITS_TABLE, toClassifierHitRecord(hit));
}; };
export const enqueueAlertInsert = ( export const enqueueAlertInsert = (writer: ClickHouseBatchWriter, alert: AlertEvent): void => {
writer: ClickHouseBatchWriter,
alert: AlertEvent
): void => {
writer.enqueue(ALERTS_TABLE, toAlertRecord(alert)); writer.enqueue(ALERTS_TABLE, toAlertRecord(alert));
}; };
export const enqueueNewsStoryInsert = ( export const enqueueNewsStoryInsert = (writer: ClickHouseBatchWriter, story: NewsStory): void => {
writer: ClickHouseBatchWriter,
story: NewsStory
): void => {
writer.enqueue(NEWS_TABLE, toNewsRecord(story)); writer.enqueue(NEWS_TABLE, toNewsRecord(story));
}; };
@ -973,9 +943,7 @@ const normalizeFlowPacketRow = (row: unknown): FlowPacketRecord | null => {
seq: coerceNumber(record.seq) as number, seq: coerceNumber(record.seq) as number,
trace_id: String(record.trace_id ?? ""), trace_id: String(record.trace_id ?? ""),
id: String(record.id ?? ""), id: String(record.id ?? ""),
members: Array.isArray(record.members) members: Array.isArray(record.members) ? record.members.map((value) => String(value)) : [],
? record.members.map((value) => String(value))
: [],
features_json: String(record.features_json ?? "{}"), features_json: String(record.features_json ?? "{}"),
join_quality_json: String(record.join_quality_json ?? "{}") join_quality_json: String(record.join_quality_json ?? "{}")
}; };
@ -1011,7 +979,9 @@ const normalizeSmartMoneyEventRow = (row: unknown): SmartMoneyEventRecord | null
seq: coerceNumber(record.seq) as number, seq: coerceNumber(record.seq) as number,
trace_id: String(record.trace_id ?? ""), trace_id: String(record.trace_id ?? ""),
event_id: String(record.event_id ?? ""), event_id: String(record.event_id ?? ""),
packet_ids: Array.isArray(record.packet_ids) ? record.packet_ids.map((value) => String(value)) : [], packet_ids: Array.isArray(record.packet_ids)
? record.packet_ids.map((value) => String(value))
: [],
member_print_ids: Array.isArray(record.member_print_ids) member_print_ids: Array.isArray(record.member_print_ids)
? record.member_print_ids.map((value) => String(value)) ? record.member_print_ids.map((value) => String(value))
: [], : [],
@ -1390,8 +1360,12 @@ export const fetchAlertContextByTraceId = async (
const packetIds = new Set(flowPackets.flatMap((packet) => [packet.id, packet.trace_id])); const packetIds = new Set(flowPackets.flatMap((packet) => [packet.id, packet.trace_id]));
const printIds = new Set(optionPrints.map((print) => print.trace_id)); const printIds = new Set(optionPrints.map((print) => print.trace_id));
const missingRefs = refs.filter((ref) => { const missingRefs = refs.filter((ref) => {
const packetResolved = flowPacketCandidatesFromRef(ref).some((candidate) => packetIds.has(candidate)); const packetResolved = flowPacketCandidatesFromRef(ref).some((candidate) =>
const printResolved = optionPrintCandidatesFromRef(ref).some((candidate) => printIds.has(candidate)); packetIds.has(candidate)
);
const printResolved = optionPrintCandidatesFromRef(ref).some((candidate) =>
printIds.has(candidate)
);
return !packetResolved && !printResolved; return !packetResolved && !printResolved;
}); });

View file

@ -130,7 +130,10 @@ describe("alerts storage helpers", () => {
}); });
it("returns an empty context when the alert is missing", async () => { it("returns an empty context when the alert is missing", async () => {
const bundle = await fetchAlertContextByTraceId(makeClient(() => []), "alert:missing"); const bundle = await fetchAlertContextByTraceId(
makeClient(() => []),
"alert:missing"
);
expect(bundle).toEqual({ expect(bundle).toEqual({
alert: null, alert: null,

View file

@ -1,5 +1,9 @@
import { describe, expect, it } from "bun:test"; import { describe, expect, it } from "bun:test";
import { createClickHouseClient, fetchFlowPacketById, fetchFlowPacketsBefore } from "../src/clickhouse"; import {
createClickHouseClient,
fetchFlowPacketById,
fetchFlowPacketsBefore
} from "../src/clickhouse";
import { import {
flowPacketsTableDDL, flowPacketsTableDDL,
FLOW_PACKETS_TABLE, FLOW_PACKETS_TABLE,

View file

@ -1,16 +1,7 @@
import { describe, expect, it } from "bun:test"; import { describe, expect, it } from "bun:test";
import type { ClickHouseClient } from "../src/clickhouse"; import type { ClickHouseClient } from "../src/clickhouse";
import { import { NEWS_TABLE, fromNewsRecord, newsTableDDL, toNewsRecord } from "../src/news";
NEWS_TABLE, import { fetchNewsAfter, fetchNewsBefore, fetchRecentNews } from "../src/clickhouse";
fromNewsRecord,
newsTableDDL,
toNewsRecord
} from "../src/news";
import {
fetchNewsAfter,
fetchNewsBefore,
fetchRecentNews
} from "../src/clickhouse";
const makeClient = (resolver: (query: string) => unknown[]): ClickHouseClient => const makeClient = (resolver: (query: string) => unknown[]): ClickHouseClient =>
({ ({

View file

@ -5,7 +5,11 @@ import {
fetchOptionPrintsByTraceIds, fetchOptionPrintsByTraceIds,
fetchRecentOptionPrints fetchRecentOptionPrints
} from "../src/clickhouse"; } from "../src/clickhouse";
import { normalizeOptionPrint, optionPrintsTableDDL, OPTION_PRINTS_TABLE } from "../src/option-prints"; import {
normalizeOptionPrint,
optionPrintsTableDDL,
OPTION_PRINTS_TABLE
} from "../src/option-prints";
const basePrint = { const basePrint = {
source_ts: 100, source_ts: 100,

View file

@ -18,37 +18,103 @@ export const OptionPrintSchema = EventMetaSchema.merge(
size: z.number().int().positive(), size: z.number().int().positive(),
exchange: z.string().min(1), exchange: z.string().min(1),
conditions: z.array(z.string().min(1)).optional(), conditions: z.array(z.string().min(1)).optional(),
underlying_id: z.preprocess((value) => (value === null ? undefined : value), z.string().min(1).optional()), underlying_id: z.preprocess(
option_type: z.preprocess((value) => (value === null ? undefined : value), OptionTypeSchema.optional()), (value) => (value === null ? undefined : value),
notional: z.preprocess((value) => (value === null ? undefined : value), z.number().nonnegative().optional()), z.string().min(1).optional()
nbbo_side: z.preprocess((value) => (value === null ? undefined : value), OptionNbboSideSchema.optional()), ),
execution_nbbo_bid: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), option_type: z.preprocess(
execution_nbbo_ask: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), (value) => (value === null ? undefined : value),
execution_nbbo_mid: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), OptionTypeSchema.optional()
execution_nbbo_spread: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), ),
execution_nbbo_bid_size: z.preprocess((value) => (value === null ? undefined : value), z.number().int().nonnegative().optional()), notional: z.preprocess(
execution_nbbo_ask_size: z.preprocess((value) => (value === null ? undefined : value), z.number().int().nonnegative().optional()), (value) => (value === null ? undefined : value),
execution_nbbo_ts: z.preprocess((value) => (value === null ? undefined : value), z.number().int().nonnegative().optional()), z.number().nonnegative().optional()
execution_nbbo_age_ms: z.preprocess((value) => (value === null ? undefined : value), z.number().nonnegative().optional()), ),
execution_nbbo_side: z.preprocess((value) => (value === null ? undefined : value), OptionNbboSideSchema.optional()), nbbo_side: z.preprocess(
execution_underlying_spot: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), (value) => (value === null ? undefined : value),
execution_underlying_bid: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), OptionNbboSideSchema.optional()
execution_underlying_ask: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), ),
execution_underlying_mid: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), execution_nbbo_bid: z.preprocess(
execution_underlying_spread: z.preprocess((value) => (value === null ? undefined : value), z.number().optional()), (value) => (value === null ? undefined : value),
execution_underlying_ts: z.preprocess((value) => (value === null ? undefined : value), z.number().int().nonnegative().optional()), z.number().optional()
execution_underlying_age_ms: z.preprocess((value) => (value === null ? undefined : value), z.number().nonnegative().optional()), ),
execution_nbbo_ask: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().optional()
),
execution_nbbo_mid: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().optional()
),
execution_nbbo_spread: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().optional()
),
execution_nbbo_bid_size: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().int().nonnegative().optional()
),
execution_nbbo_ask_size: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().int().nonnegative().optional()
),
execution_nbbo_ts: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().int().nonnegative().optional()
),
execution_nbbo_age_ms: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().nonnegative().optional()
),
execution_nbbo_side: z.preprocess(
(value) => (value === null ? undefined : value),
OptionNbboSideSchema.optional()
),
execution_underlying_spot: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().optional()
),
execution_underlying_bid: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().optional()
),
execution_underlying_ask: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().optional()
),
execution_underlying_mid: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().optional()
),
execution_underlying_spread: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().optional()
),
execution_underlying_ts: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().int().nonnegative().optional()
),
execution_underlying_age_ms: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().nonnegative().optional()
),
execution_underlying_source: z.preprocess( execution_underlying_source: z.preprocess(
(value) => (value === null ? undefined : value), (value) => (value === null ? undefined : value),
z.literal("equity_quote_mid").optional() z.literal("equity_quote_mid").optional()
), ),
execution_iv: z.preprocess((value) => (value === null ? undefined : value), z.number().nonnegative().optional()), execution_iv: z.preprocess(
(value) => (value === null ? undefined : value),
z.number().nonnegative().optional()
),
execution_iv_source: z.preprocess( execution_iv_source: z.preprocess(
(value) => (value === null ? undefined : value), (value) => (value === null ? undefined : value),
z.enum(["provider", "synthetic_pressure_model"]).optional() z.enum(["provider", "synthetic_pressure_model"]).optional()
), ),
is_etf: z.preprocess((value) => (value === null ? undefined : value), z.boolean().optional()), is_etf: z.preprocess((value) => (value === null ? undefined : value), z.boolean().optional()),
signal_pass: z.preprocess((value) => (value === null ? undefined : value), z.boolean().optional()), signal_pass: z.preprocess(
(value) => (value === null ? undefined : value),
z.boolean().optional()
),
signal_reasons: z.array(z.string().min(1)).optional(), signal_reasons: z.array(z.string().min(1)).optional(),
signal_profile: z.preprocess( signal_profile: z.preprocess(
(value) => (value === null ? undefined : value), (value) => (value === null ? undefined : value),
@ -146,7 +212,13 @@ export const SmartMoneyProfileIdSchema = z.enum([
export type SmartMoneyProfileId = z.infer<typeof SmartMoneyProfileIdSchema>; export type SmartMoneyProfileId = z.infer<typeof SmartMoneyProfileIdSchema>;
export const SmartMoneyDirectionSchema = z.enum(["bullish", "bearish", "neutral", "mixed", "unknown"]); export const SmartMoneyDirectionSchema = z.enum([
"bullish",
"bearish",
"neutral",
"mixed",
"unknown"
]);
export type SmartMoneyDirection = z.infer<typeof SmartMoneyDirectionSchema>; export type SmartMoneyDirection = z.infer<typeof SmartMoneyDirectionSchema>;

View file

@ -13,10 +13,7 @@ import {
OptionPrintSchema, OptionPrintSchema,
SmartMoneyEventSchema SmartMoneyEventSchema
} from "./events"; } from "./events";
import { import { OptionFlowFiltersSchema, optionFlowFilterKey } from "./options-flow";
OptionFlowFiltersSchema,
optionFlowFilterKey
} from "./options-flow";
export const CursorSchema = z.object({ export const CursorSchema = z.object({
ts: z.number().int().nonnegative(), ts: z.number().int().nonnegative(),
@ -94,7 +91,15 @@ export const LiveSubscriptionSchema = z.discriminatedUnion("channel", [
snapshot_limit: z.number().int().positive().optional() snapshot_limit: z.number().int().positive().optional()
}), }),
z.object({ z.object({
channel: z.enum(["nbbo", "equity-quotes", "equity-joins", "classifier-hits", "alerts", "inferred-dark", "news"]), channel: z.enum([
"nbbo",
"equity-quotes",
"equity-joins",
"classifier-hits",
"alerts",
"inferred-dark",
"news"
]),
snapshot_limit: z.number().int().positive().optional() snapshot_limit: z.number().int().positive().optional()
}), }),
z.object({ z.object({

View file

@ -212,7 +212,8 @@ export const deriveOptionPrintMetadata = (
const parsed = parseOptionContractId(print.option_contract_id); const parsed = parseOptionContractId(print.option_contract_id);
const underlying = parsed?.root?.toUpperCase(); const underlying = parsed?.root?.toUpperCase();
const optionType = parsed?.right === "C" ? "call" : parsed?.right === "P" ? "put" : undefined; const optionType = parsed?.right === "C" ? "call" : parsed?.right === "P" ? "put" : undefined;
const notional = Number.isFinite(print.price) && Number.isFinite(print.size) const notional =
Number.isFinite(print.price) && Number.isFinite(print.size)
? Number((print.price * print.size * 100).toFixed(2)) ? Number((print.price * print.size * 100).toFixed(2))
: undefined; : undefined;
@ -243,7 +244,14 @@ const balancedThresholds = (config: OptionsSignalConfig): OptionsSignalConfig =>
export const evaluateOptionSignal = ( export const evaluateOptionSignal = (
print: Pick< print: Pick<
OptionPrint, OptionPrint,
"size" | "conditions" | "signal_profile" | "underlying_id" | "option_type" | "notional" | "nbbo_side" | "is_etf" | "size"
| "conditions"
| "signal_profile"
| "underlying_id"
| "option_type"
| "notional"
| "nbbo_side"
| "is_etf"
>, >,
baseConfig: OptionsSignalConfig baseConfig: OptionsSignalConfig
): OptionSignalDecision => { ): OptionSignalDecision => {
@ -260,7 +268,8 @@ export const evaluateOptionSignal = (
const reasons: string[] = []; const reasons: string[] = [];
const notional = print.notional ?? 0; const notional = print.notional ?? 0;
const side = print.nbbo_side ?? "MISSING"; const side = print.nbbo_side ?? "MISSING";
const isSweepOrIso = hasCondition(print.conditions, "SWEEP") || hasCondition(print.conditions, "ISO"); const isSweepOrIso =
hasCondition(print.conditions, "SWEEP") || hasCondition(print.conditions, "ISO");
if (notional < config.minNotional) { if (notional < config.minNotional) {
return { return {
@ -413,8 +422,14 @@ export const matchesFlowPacketFilters = (
} }
const features = packet.features ?? {}; const features = packet.features ?? {};
const totalNotional = typeof features.total_notional === "number" ? features.total_notional : Number(features.total_notional ?? 0); const totalNotional =
if (typeof filters.minNotional === "number" && (!Number.isFinite(totalNotional) || totalNotional < filters.minNotional)) { typeof features.total_notional === "number"
? features.total_notional
: Number(features.total_notional ?? 0);
if (
typeof filters.minNotional === "number" &&
(!Number.isFinite(totalNotional) || totalNotional < filters.minNotional)
) {
return false; return false;
} }
@ -433,10 +448,7 @@ export const matchesFlowPacketFilters = (
: typeof features.structure_rights === "string" : typeof features.structure_rights === "string"
? features.structure_rights.toLowerCase() ? features.structure_rights.toLowerCase()
: null; : null;
if ( if (!optionType || !filters.optionTypes.some((selected) => optionType.includes(selected))) {
!optionType ||
!filters.optionTypes.some((selected) => optionType.includes(selected))
) {
return false; return false;
} }
} }

View file

@ -501,7 +501,7 @@ export const SP500_SYMBOLS = [
"YUM", "YUM",
"ZBRA", "ZBRA",
"ZBH", "ZBH",
"ZTS", "ZTS"
] as const; ] as const;
export type Sp500Symbol = typeof SP500_SYMBOLS[number]; export type Sp500Symbol = (typeof SP500_SYMBOLS)[number];

View file

@ -26,10 +26,7 @@ const SMART_MONEY_PROFILE_IDS = [
"arbitrage", "arbitrage",
"hedge_reactive" "hedge_reactive"
] as const satisfies readonly SmartMoneyProfileId[]; ] as const satisfies readonly SmartMoneyProfileId[];
const SYNTHETIC_SCENARIO_FAMILY_IDS = [ const SYNTHETIC_SCENARIO_FAMILY_IDS = [...SMART_MONEY_PROFILE_IDS, "neutral_noise"] as const;
...SMART_MONEY_PROFILE_IDS,
"neutral_noise"
] as const;
const REGIME_IDS = [ const REGIME_IDS = [
"trend_up", "trend_up",
"trend_down", "trend_down",
@ -54,18 +51,14 @@ export const SyntheticCoverageWindowMinutesSchema = z.union([
z.literal(20), z.literal(20),
z.literal(30) z.literal(30)
]); ]);
export type SyntheticCoverageWindowMinutes = z.infer< export type SyntheticCoverageWindowMinutes = z.infer<typeof SyntheticCoverageWindowMinutesSchema>;
typeof SyntheticCoverageWindowMinutesSchema
>;
export const SyntheticProfileWeightValueSchema = z.union([ export const SyntheticProfileWeightValueSchema = z.union([
z.literal(0.6), z.literal(0.6),
z.literal(1.0), z.literal(1.0),
z.literal(1.6) z.literal(1.6)
]); ]);
export type SyntheticProfileWeightValue = z.infer< export type SyntheticProfileWeightValue = z.infer<typeof SyntheticProfileWeightValueSchema>;
typeof SyntheticProfileWeightValueSchema
>;
export const SyntheticProfileWeightMapSchema = z export const SyntheticProfileWeightMapSchema = z
.object({ .object({
@ -77,9 +70,7 @@ export const SyntheticProfileWeightMapSchema = z
hedge_reactive: SyntheticProfileWeightValueSchema hedge_reactive: SyntheticProfileWeightValueSchema
}) })
.strict(); .strict();
export type SyntheticProfileWeightMap = z.infer< export type SyntheticProfileWeightMap = z.infer<typeof SyntheticProfileWeightMapSchema>;
typeof SyntheticProfileWeightMapSchema
>;
export const SyntheticControlStateSchema = z export const SyntheticControlStateSchema = z
.object({ .object({
@ -94,23 +85,14 @@ export const SyntheticControlStateSchema = z
.strict(); .strict();
export type SyntheticControlState = z.infer<typeof SyntheticControlStateSchema>; export type SyntheticControlState = z.infer<typeof SyntheticControlStateSchema>;
export const SyntheticSessionPhaseSchema = z.enum([ export const SyntheticSessionPhaseSchema = z.enum(["open", "midday", "power_hour", "after_event"]);
"open",
"midday",
"power_hour",
"after_event"
]);
export type SyntheticSessionPhase = z.infer<typeof SyntheticSessionPhaseSchema>; export type SyntheticSessionPhase = z.infer<typeof SyntheticSessionPhaseSchema>;
export const SyntheticRegimeSchema = z.enum(REGIME_IDS); export const SyntheticRegimeSchema = z.enum(REGIME_IDS);
export type SyntheticRegime = z.infer<typeof SyntheticRegimeSchema>; export type SyntheticRegime = z.infer<typeof SyntheticRegimeSchema>;
export const SyntheticScenarioFamilyIdSchema = z.enum( export const SyntheticScenarioFamilyIdSchema = z.enum(SYNTHETIC_SCENARIO_FAMILY_IDS);
SYNTHETIC_SCENARIO_FAMILY_IDS export type SyntheticScenarioFamilyId = z.infer<typeof SyntheticScenarioFamilyIdSchema>;
);
export type SyntheticScenarioFamilyId = z.infer<
typeof SyntheticScenarioFamilyIdSchema
>;
export const SyntheticCoverageConfigSchema = z export const SyntheticCoverageConfigSchema = z
.object({ .object({
@ -118,9 +100,7 @@ export const SyntheticCoverageConfigSchema = z
coverage_window_minutes: SyntheticCoverageWindowMinutesSchema coverage_window_minutes: SyntheticCoverageWindowMinutesSchema
}) })
.strict(); .strict();
export type SyntheticCoverageConfig = z.infer< export type SyntheticCoverageConfig = z.infer<typeof SyntheticCoverageConfigSchema>;
typeof SyntheticCoverageConfigSchema
>;
export const SyntheticDerivedStatusSchema = z export const SyntheticDerivedStatusSchema = z
.object({ .object({
@ -131,9 +111,7 @@ export const SyntheticDerivedStatusSchema = z
coverage_window_minutes: SyntheticCoverageWindowMinutesSchema coverage_window_minutes: SyntheticCoverageWindowMinutesSchema
}) })
.strict(); .strict();
export type SyntheticDerivedStatus = z.infer< export type SyntheticDerivedStatus = z.infer<typeof SyntheticDerivedStatusSchema>;
typeof SyntheticDerivedStatusSchema
>;
export type SyntheticSessionState = { export type SyntheticSessionState = {
session_phase: SyntheticSessionPhase; session_phase: SyntheticSessionPhase;
@ -160,10 +138,7 @@ export type SyntheticUnderlyingState = {
offExchangeBias: number; offExchangeBias: number;
}; };
export type SyntheticScenarioWeightMap = Record< export type SyntheticScenarioWeightMap = Record<SyntheticScenarioFamilyId, number>;
SyntheticScenarioFamilyId,
number
>;
export type SyntheticCoverageState = { export type SyntheticCoverageState = {
profile_hit_counts: Record<SmartMoneyProfileId, number>; profile_hit_counts: Record<SmartMoneyProfileId, number>;
@ -195,10 +170,7 @@ export const DEFAULT_SYNTHETIC_CONTROL_STATE: SyntheticControlState = {
updated_by: "system" updated_by: "system"
}; };
const PRESET_REGIME_BIAS: Record< const PRESET_REGIME_BIAS: Record<SyntheticControlPresetId, Record<SyntheticRegime, number>> = {
SyntheticControlPresetId,
Record<SyntheticRegime, number>
> = {
balanced_demo: { balanced_demo: {
trend_up: 1.0, trend_up: 1.0,
trend_down: 0.95, trend_down: 0.95,
@ -257,10 +229,7 @@ const PRESET_ACTIVITY_BIAS: Record<
quiet_range: { focusCount: 2, eventCount: 1, amplitude: 0.72 } quiet_range: { focusCount: 2, eventCount: 1, amplitude: 0.72 }
}; };
const REGIME_PROFILE_BIAS: Record< const REGIME_PROFILE_BIAS: Record<SyntheticRegime, SyntheticScenarioWeightMap> = {
SyntheticRegime,
SyntheticScenarioWeightMap
> = {
trend_up: { trend_up: {
institutional_directional: 1.35, institutional_directional: 1.35,
retail_whale: 1.05, retail_whale: 1.05,
@ -411,16 +380,12 @@ const mixSeed = (...parts: number[]): number => {
return seed >>> 0; return seed >>> 0;
}; };
const pick = <T,>(items: readonly T[], seed: number): T => { const pick = <T>(items: readonly T[], seed: number): T => {
const index = Math.abs(seed) % items.length; const index = Math.abs(seed) % items.length;
return items[index]!; return items[index]!;
}; };
const pickManyUnique = <T,>( const pickManyUnique = <T>(items: readonly T[], count: number, seed: number): T[] => {
items: readonly T[],
count: number,
seed: number
): T[] => {
const pool = [...items]; const pool = [...items];
const output: T[] = []; const output: T[] = [];
let cursor = seed; let cursor = seed;
@ -432,10 +397,7 @@ const pickManyUnique = <T,>(
return output; return output;
}; };
const weightedPick = <T extends string>( const weightedPick = <T extends string>(weights: Record<T, number>, seed: number): T => {
weights: Record<T, number>,
seed: number
): T => {
const entries = Object.entries(weights) as Array<[T, number]>; const entries = Object.entries(weights) as Array<[T, number]>;
const total = entries.reduce((sum, [, weight]) => sum + Math.max(0.0001, weight), 0); const total = entries.reduce((sum, [, weight]) => sum + Math.max(0.0001, weight), 0);
let target = positiveNoise(seed) * total; let target = positiveNoise(seed) * total;
@ -461,10 +423,7 @@ export const hashSyntheticSymbol = (value: string): number => {
return hash; return hash;
}; };
export const buildEmptySyntheticProfileHitCounts = (): Record< export const buildEmptySyntheticProfileHitCounts = (): Record<SmartMoneyProfileId, number> => ({
SmartMoneyProfileId,
number
> => ({
institutional_directional: 0, institutional_directional: 0,
retail_whale: 0, retail_whale: 0,
event_driven: 0, event_driven: 0,
@ -487,10 +446,7 @@ export const normalizeSyntheticControlState = (
return SyntheticControlStateSchema.parse(merged); return SyntheticControlStateSchema.parse(merged);
}; };
const resolvePhaseBias = ( const resolvePhaseBias = (phase: SyntheticSessionPhase, regime: SyntheticRegime): number => {
phase: SyntheticSessionPhase,
regime: SyntheticRegime
): number => {
if (phase === "open") { if (phase === "open") {
return regime === "event_ramp" ? 1.08 : 1.02; return regime === "event_ramp" ? 1.08 : 1.02;
} }
@ -566,10 +522,7 @@ export const getSyntheticSessionState = (
mixSeed(activitySeed, 211) mixSeed(activitySeed, 211)
); );
const focus_symbols: string[] = pickManyUnique( const focus_symbols: string[] = pickManyUnique(
[ [...event_symbols, ...SYNTHETIC_SYMBOLS.filter((symbol) => !event_symbols.includes(symbol))],
...event_symbols,
...SYNTHETIC_SYMBOLS.filter((symbol) => !event_symbols.includes(symbol))
],
focusCount, focusCount,
mixSeed(activitySeed, 389) mixSeed(activitySeed, 389)
); );
@ -579,11 +532,7 @@ export const getSyntheticSessionState = (
session_phase: phase, session_phase: phase,
regime, regime,
volatility_level: roundTo( volatility_level: roundTo(
clamp( clamp(stateBase.volatility * amplitude + signedNoise(activitySeed + 3) * 0.08, 0.18, 1.2)
stateBase.volatility * amplitude + signedNoise(activitySeed + 3) * 0.08,
0.18,
1.2
)
), ),
liquidity_level: roundTo( liquidity_level: roundTo(
clamp( clamp(
@ -656,9 +605,7 @@ export const getSyntheticUnderlyingState = (
? -meanRevertWave * (12 + session.liquidity_level * 10) ? -meanRevertWave * (12 + session.liquidity_level * 10)
: meanRevertWave * 6; : meanRevertWave * 6;
const gammaChop = const gammaChop =
session.regime === "dealer_gamma" session.regime === "dealer_gamma" ? Math.sin((minuteOfSession + (hash % 11)) / 2.8) * 16 : 0;
? Math.sin((minuteOfSession + (hash % 11)) / 2.8) * 16
: 0;
const noiseBps = const noiseBps =
signedNoise(mixSeed(hash, session.seed_bucket, control.shared_seed)) * signedNoise(mixSeed(hash, session.seed_bucket, control.shared_seed)) *
(6 + session.volatility_level * 18); (6 + session.volatility_level * 18);
@ -731,10 +678,7 @@ export const getSyntheticScenarioWeights = (
}; };
for (const profileId of SMART_MONEY_PROFILE_IDS) { for (const profileId of SMART_MONEY_PROFILE_IDS) {
weights[profileId] = roundTo( weights[profileId] = roundTo(weights[profileId] * normalized.profile_weights[profileId], 4);
weights[profileId] * normalized.profile_weights[profileId],
4
);
} }
if (isFocus) { if (isFocus) {
@ -745,10 +689,7 @@ export const getSyntheticScenarioWeights = (
} }
if (isEvent) { if (isEvent) {
weights.event_driven = roundTo(weights.event_driven * 1.36, 4); weights.event_driven = roundTo(weights.event_driven * 1.36, 4);
weights.institutional_directional = roundTo( weights.institutional_directional = roundTo(weights.institutional_directional * 1.04, 4);
weights.institutional_directional * 1.04,
4
);
weights.neutral_noise = roundTo(weights.neutral_noise * 0.8, 4); weights.neutral_noise = roundTo(weights.neutral_noise * 0.8, 4);
} }
if (isPower) { if (isPower) {
@ -765,10 +706,7 @@ export const getSyntheticScenarioWeights = (
export const getSyntheticCoverageBoost = ( export const getSyntheticCoverageBoost = (
profileId: SmartMoneyProfileId, profileId: SmartMoneyProfileId,
coverageState: SyntheticCoverageState, coverageState: SyntheticCoverageState,
control: Pick< control: Pick<SyntheticControlState, "coverage_assist" | "coverage_window_minutes">
SyntheticControlState,
"coverage_assist" | "coverage_window_minutes"
>
): number => { ): number => {
if (!control.coverage_assist) { if (!control.coverage_assist) {
return 1; return 1;

View file

@ -31,9 +31,7 @@ describe("live protocol types", () => {
underlying_ids: ["NVDA", "AAPL"], underlying_ids: ["NVDA", "AAPL"],
option_contract_id: "AAPL-2025-01-17-200-C" option_contract_id: "AAPL-2025-01-17-200-C"
}) })
).toBe( ).toBe('options|{"view":"signal"}|underlyings:AAPL,NVDA|contract:AAPL-2025-01-17-200-C');
'options|{"view":"signal"}|underlyings:AAPL,NVDA|contract:AAPL-2025-01-17-200-C'
);
expect(getSubscriptionKey({ channel: "equities", underlying_ids: ["NVDA", "AAPL"] })).toBe( expect(getSubscriptionKey({ channel: "equities", underlying_ids: ["NVDA", "AAPL"] })).toBe(
"equities|underlyings:AAPL,NVDA" "equities|underlyings:AAPL,NVDA"
); );

View file

@ -68,7 +68,9 @@ const listWorkspacePaths = async (workspacePatterns: string[]): Promise<string[]
const paths = new Set<string>(); const paths = new Set<string>();
for (const pattern of workspacePatterns) { for (const pattern of workspacePatterns) {
const globPattern = pattern.endsWith("/") ? `${pattern}package.json` : `${pattern}/package.json`; const globPattern = pattern.endsWith("/")
? `${pattern}package.json`
: `${pattern}/package.json`;
const glob = new Bun.Glob(globPattern); const glob = new Bun.Glob(globPattern);
for await (const match of glob.scan({ cwd: repoRoot })) { for await (const match of glob.scan({ cwd: repoRoot })) {
const normalized = match.replaceAll("\\", "/"); const normalized = match.replaceAll("\\", "/");
@ -124,8 +126,14 @@ const formatDependencyDiff = (
const check = async (): Promise<number> => { const check = async (): Promise<number> => {
const issues: string[] = []; const issues: string[] = [];
const [rootPackage, deploymentPackage, rootTsconfig, deploymentTsconfig, rootLock, deploymentLock] = const [
await Promise.all([ rootPackage,
deploymentPackage,
rootTsconfig,
deploymentTsconfig,
rootLock,
deploymentLock
] = await Promise.all([
parseObjectLiteral<RootPackageManifest>(rootPackagePath), parseObjectLiteral<RootPackageManifest>(rootPackagePath),
parseObjectLiteral(deploymentPackagePath), parseObjectLiteral(deploymentPackagePath),
parseObjectLiteral(rootTsconfigPath), parseObjectLiteral(rootTsconfigPath),
@ -172,7 +180,9 @@ const check = async (): Promise<number> => {
"peerDependencies" "peerDependencies"
]; ];
for (const section of sections) { for (const section of sections) {
const expectedMap = normalizedDependencyMap(workspacePackage[section] as DependencyMap | undefined); const expectedMap = normalizedDependencyMap(
workspacePackage[section] as DependencyMap | undefined
);
const actualMap = normalizedDependencyMap( const actualMap = normalizedDependencyMap(
deploymentWorkspace[section] as DependencyMap | undefined deploymentWorkspace[section] as DependencyMap | undefined
); );
@ -212,7 +222,9 @@ const check = async (): Promise<number> => {
"peerDependencies" "peerDependencies"
]; ];
for (const section of sections) { for (const section of sections) {
const expectedMap = normalizedDependencyMap(rootWorkspace[section] as DependencyMap | undefined); const expectedMap = normalizedDependencyMap(
rootWorkspace[section] as DependencyMap | undefined
);
const actualMap = normalizedDependencyMap( const actualMap = normalizedDependencyMap(
deploymentWorkspace[section] as DependencyMap | undefined deploymentWorkspace[section] as DependencyMap | undefined
); );

View file

@ -7,7 +7,10 @@ type RouteCheck = {
const routeChecks: RouteCheck[] = [ const routeChecks: RouteCheck[] = [
{ path: "/prints/options?view=signal&limit=1", expectJson: true }, { path: "/prints/options?view=signal&limit=1", expectJson: true },
{ path: "/history/options?view=signal&before_ts=4102444800000&before_seq=999999999&limit=1", expectJson: true }, {
path: "/history/options?view=signal&before_ts=4102444800000&before_seq=999999999&limit=1",
expectJson: true
},
{ path: "/replay/options?view=signal&after_ts=0&after_seq=0&limit=1", expectJson: true }, { path: "/replay/options?view=signal&after_ts=0&after_seq=0&limit=1", expectJson: true },
{ path: "/nbbo/options?limit=1", expectJson: true }, { path: "/nbbo/options?limit=1", expectJson: true },
{ path: "/ws/live", expectJson: true } { path: "/ws/live", expectJson: true }
@ -31,7 +34,9 @@ const assertPublicApiRoute = async ({ path, expectJson }: RouteCheck): Promise<v
if (expectJson && !isJsonResponse(response)) { if (expectJson && !isJsonResponse(response)) {
const sample = responseText.replace(/\s+/g, " ").slice(0, 120); const sample = responseText.replace(/\s+/g, " ").slice(0, 120);
throw new Error(`${url.pathname} returned non-JSON content (${response.headers.get("content-type") ?? "none"}): ${sample}`); throw new Error(
`${url.pathname} returned non-JSON content (${response.headers.get("content-type") ?? "none"}): ${sample}`
);
} }
}; };

View file

@ -30,21 +30,12 @@ const SSH_KEY =
process.env.DEPLOY_SSH_KEY_PATH?.trim() || process.env.DEPLOY_SSH_KEY_PATH?.trim() ||
path.join(process.env.HOME ?? "", ".ssh", "delta_ed25519"); path.join(process.env.HOME ?? "", ".ssh", "delta_ed25519");
const DEPLOY_FORCE_SSH = process.env.DEPLOY_FORCE_SSH?.trim() === "1"; const DEPLOY_FORCE_SSH = process.env.DEPLOY_FORCE_SSH?.trim() === "1";
const SSH_OPTIONS = [ const SSH_OPTIONS = ["-i", SSH_KEY, "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes"];
"-i",
SSH_KEY,
"-o",
"IdentitiesOnly=yes",
"-o",
"BatchMode=yes"
];
const ALLOWED_REMOTE_UNTRACKED = new Set([ const ALLOWED_REMOTE_UNTRACKED = new Set([
"deployment/docker/signal-cli-0.14.3-Linux-native.tar.gz" "deployment/docker/signal-cli-0.14.3-Linux-native.tar.gz"
]); ]);
const PUBLIC_APP_URL = const PUBLIC_APP_URL = process.env.DEPLOY_PUBLIC_APP_URL?.trim() || "https://flow.deltaisland.io";
process.env.DEPLOY_PUBLIC_APP_URL?.trim() || "https://flow.deltaisland.io"; const PUBLIC_API_HEALTH_URL = process.env.DEPLOY_PUBLIC_API_HEALTH_URL?.trim() || null;
const PUBLIC_API_HEALTH_URL =
process.env.DEPLOY_PUBLIC_API_HEALTH_URL?.trim() || null;
const DEPLOY_GIT_REMOTE_OVERRIDE = process.env.DEPLOY_GIT_REMOTE?.trim() || null; const DEPLOY_GIT_REMOTE_OVERRIDE = process.env.DEPLOY_GIT_REMOTE?.trim() || null;
const DEPLOY_NATIVE_EDGE_READY = process.env.DEPLOY_NATIVE_EDGE_READY?.trim() === "1"; const DEPLOY_NATIVE_EDGE_READY = process.env.DEPLOY_NATIVE_EDGE_READY?.trim() === "1";
const NATIVE_SYSTEMCTL_PREFIX = const NATIVE_SYSTEMCTL_PREFIX =
@ -171,11 +162,7 @@ function formatCommand(command: string, args: string[]): string {
.join(" "); .join(" ");
} }
function runChecked( function runChecked(command: string, args: string[], options: SpawnSyncOptions = {}): void {
command: string,
args: string[],
options: SpawnSyncOptions = {}
): void {
console.log(`$ ${formatCommand(command, args)}`); console.log(`$ ${formatCommand(command, args)}`);
const result = spawnSync(command, args, { const result = spawnSync(command, args, {
cwd: repoRoot, cwd: repoRoot,
@ -188,11 +175,7 @@ function runChecked(
} }
} }
function captureChecked( function captureChecked(command: string, args: string[], options: SpawnSyncOptions = {}): string {
command: string,
args: string[],
options: SpawnSyncOptions = {}
): string {
const result = spawnSync(command, args, { const result = spawnSync(command, args, {
cwd: repoRoot, cwd: repoRoot,
encoding: "utf8", encoding: "utf8",
@ -225,11 +208,7 @@ function tryCapture(
return result.stdout ?? ""; return result.stdout ?? "";
} }
function runRemoteScript( function runRemoteScript(title: string, script: string, args: string[] = []): void {
title: string,
script: string,
args: string[] = []
): void {
section(title); section(title);
if (isLocalServerExecution) { if (isLocalServerExecution) {
@ -360,7 +339,9 @@ function assertSshKeyExists(): void {
if (!existsSync(SSH_KEY)) { if (!existsSync(SSH_KEY)) {
console.error(`Missing SSH key: ${SSH_KEY}`); console.error(`Missing SSH key: ${SSH_KEY}`);
console.error("Set DEPLOY_SSH_KEY_PATH or run from the live server checkout without DEPLOY_FORCE_SSH."); console.error(
"Set DEPLOY_SSH_KEY_PATH or run from the live server checkout without DEPLOY_FORCE_SSH."
);
process.exit(1); process.exit(1);
} }
} }
@ -399,10 +380,12 @@ function localGitRemotes(): string[] {
} }
function localHasRemote(name: string): boolean { function localHasRemote(name: string): boolean {
return spawnSync("git", ["remote", "get-url", name], { return (
spawnSync("git", ["remote", "get-url", name], {
cwd: repoRoot, cwd: repoRoot,
stdio: "ignore" stdio: "ignore"
}).status === 0; }).status === 0
);
} }
function resolveDeployRemote(mode: DeployMode, branch: string | null): string { function resolveDeployRemote(mode: DeployMode, branch: string | null): string {
@ -444,12 +427,8 @@ function resolveDeployRemote(mode: DeployMode, branch: string | null): string {
return selected; return selected;
} }
console.error( console.error(`Unable to resolve a deploy git remote. Checked candidates: ${deduped.join(", ")}`);
`Unable to resolve a deploy git remote. Checked candidates: ${deduped.join(", ")}` console.error("Set DEPLOY_GIT_REMOTE to a valid remote name or configure branch.<name>.remote.");
);
console.error(
"Set DEPLOY_GIT_REMOTE to a valid remote name or configure branch.<name>.remote."
);
process.exit(1); process.exit(1);
} }
@ -748,7 +727,9 @@ fi
return; return;
} }
const units = nativeUnitsForScope(scope).map((value) => shellEscape(value)).join(" "); const units = nativeUnitsForScope(scope)
.map((value) => shellEscape(value))
.join(" ");
runRemoteScript( runRemoteScript(
"Remote Runtime Precheck", "Remote Runtime Precheck",
`#!/usr/bin/env bash `#!/usr/bin/env bash
@ -819,9 +800,7 @@ function remoteDockerRollout(
upArgs.push("--force-recreate"); upArgs.push("--force-recreate");
} }
const buildServices = dockerBuildServicesForScope(scope); const buildServices = dockerBuildServicesForScope(scope);
const buildCommand = noBuild const buildCommand = noBuild ? null : `docker compose build ${buildServices.join(" ")}`;
? null
: `docker compose build ${buildServices.join(" ")}`;
const upCommand = `docker compose ${[...upArgs, ...rolloutServices].join(" ")}`; const upCommand = `docker compose ${[...upArgs, ...rolloutServices].join(" ")}`;
runRemoteScript( runRemoteScript(
@ -844,7 +823,9 @@ function remoteNativeRollout(
scope: DeployScope, scope: DeployScope,
noBuild: boolean noBuild: boolean
): void { ): void {
const units = nativeUnitsForScope(scope).map((value) => shellEscape(value)).join(" "); const units = nativeUnitsForScope(scope)
.map((value) => shellEscape(value))
.join(" ");
const buildSteps: string[] = []; const buildSteps: string[] = [];
if (!noBuild) { if (!noBuild) {
@ -854,7 +835,11 @@ function remoteNativeRollout(
} }
} }
buildSteps.push(`${NATIVE_SYSTEMCTL_PREFIX} restart ${nativeUnitsForScope(scope).map((value) => shellEscape(value)).join(" ")}`); buildSteps.push(
`${NATIVE_SYSTEMCTL_PREFIX} restart ${nativeUnitsForScope(scope)
.map((value) => shellEscape(value))
.join(" ")}`
);
runRemoteScript( runRemoteScript(
"Remote Rollout", "Remote Rollout",
@ -899,9 +884,7 @@ function remoteDockerVerification(scope: DeployScope, fast: boolean): void {
const psServices = dockerServicesForScope(scope); const psServices = dockerServicesForScope(scope);
const logServices = dockerLogServicesForScope(scope); const logServices = dockerLogServicesForScope(scope);
const psCommand = const psCommand =
psServices.length > 0 psServices.length > 0 ? `docker compose ps ${psServices.join(" ")}` : "docker compose ps";
? `docker compose ps ${psServices.join(" ")}`
: "docker compose ps";
const logCommand = fast const logCommand = fast
? `echo '[deploy] Fast mode: skipping docker compose logs tail for quicker feedback.'` ? `echo '[deploy] Fast mode: skipping docker compose logs tail for quicker feedback.'`
: `docker compose logs --tail=100 ${logServices.join(" ")}`; : `docker compose logs --tail=100 ${logServices.join(" ")}`;
@ -933,7 +916,9 @@ ${checks.join("\n")}
} }
function remoteNativeVerification(scope: DeployScope, fast: boolean): void { function remoteNativeVerification(scope: DeployScope, fast: boolean): void {
const units = nativeUnitsForScope(scope).map((value) => shellEscape(value)).join(" "); const units = nativeUnitsForScope(scope)
.map((value) => shellEscape(value))
.join(" ");
const checks: string[] = []; const checks: string[] = [];
if (scope === "full" || scope === "api" || scope === "services" || scope === "workers") { if (scope === "full" || scope === "api" || scope === "services" || scope === "workers") {
@ -941,11 +926,11 @@ function remoteNativeVerification(scope: DeployScope, fast: boolean): void {
} }
if (scopeIncludesApi(scope)) { if (scopeIncludesApi(scope)) {
checks.push('curl -fksS http://127.0.0.1:4000/health'); checks.push("curl -fksS http://127.0.0.1:4000/health");
} }
if (scopeIncludesWeb(scope)) { if (scopeIncludesWeb(scope)) {
checks.push('curl -I -fksS http://127.0.0.1:3000/'); checks.push("curl -I -fksS http://127.0.0.1:3000/");
} }
runRemoteScript( runRemoteScript(
@ -962,7 +947,7 @@ fi
declare -a units=(${units}) declare -a units=(${units})
for unit in "\${units[@]}"; do for unit in "\${units[@]}"; do
${NATIVE_SYSTEMCTL_PREFIX} is-active --quiet "$unit" ${NATIVE_SYSTEMCTL_PREFIX} is-active --quiet "$unit"
${fast ? "echo \"[deploy] Fast mode: skipping unit status and recent journal dump for $unit.\"": `${NATIVE_SYSTEMCTL_PREFIX} status --no-pager "$unit" || true\n journalctl -u "$unit" -n 50 --no-pager || true`} ${fast ? 'echo "[deploy] Fast mode: skipping unit status and recent journal dump for $unit."' : `${NATIVE_SYSTEMCTL_PREFIX} status --no-pager "$unit" || true\n journalctl -u "$unit" -n 50 --no-pager || true`}
done done
${checks.join("\n")} ${checks.join("\n")}
` `
@ -1074,9 +1059,7 @@ function main(): void {
timedPhase(timings, "remote verification", () => timedPhase(timings, "remote verification", () =>
remoteVerification(options.runtime, scope, options.fast) remoteVerification(options.runtime, scope, options.fast)
); );
timedPhase(timings, "public verification", () => timedPhase(timings, "public verification", () => publicVerification(scope, options.fast));
publicVerification(scope, options.fast)
);
printTimingSummary(timings); printTimingSummary(timings);
} }

View file

@ -6,7 +6,7 @@ const outputFile = path.join(docsDir, "index.html");
const dateFormatter = new Intl.DateTimeFormat("en-US", { const dateFormatter = new Intl.DateTimeFormat("en-US", {
dateStyle: "medium", dateStyle: "medium",
timeStyle: "short", timeStyle: "short"
}); });
function escapeHtml(value) { function escapeHtml(value) {
@ -71,7 +71,7 @@ async function collectDocsFiles(rootDir, currentDir = rootDir, acc = []) {
relativePath, relativePath,
category: relativePath.includes("/") ? relativePath.split("/")[0] : "root", category: relativePath.includes("/") ? relativePath.split("/")[0] : "root",
sizeBytes: stats.size, sizeBytes: stats.size,
modifiedAt: stats.mtime, modifiedAt: stats.mtime
}); });
} }
} }

View file

@ -4,11 +4,7 @@ import path from "node:path";
const repoRoot = path.resolve(import.meta.dir, ".."); const repoRoot = path.resolve(import.meta.dir, "..");
const deploymentRoot = path.join(repoRoot, "deployment/docker/workspace-root"); const deploymentRoot = path.join(repoRoot, "deployment/docker/workspace-root");
const filesToSync = [ const filesToSync = ["package.json", "bun.lock", "tsconfig.base.json"] as const;
"package.json",
"bun.lock",
"tsconfig.base.json"
] as const;
for (const fileName of filesToSync) { for (const fileName of filesToSync) {
const source = path.join(repoRoot, fileName); const source = path.join(repoRoot, fileName);
@ -16,4 +12,3 @@ for (const fileName of filesToSync) {
await copyFile(source, destination); await copyFile(source, destination);
console.log(`synced ${fileName}`); console.log(`synced ${fileName}`);
} }

View file

@ -39,10 +39,24 @@ for (const tsconfig of tsconfigs) {
const label = relative(process.cwd(), tsconfig); const label = relative(process.cwd(), tsconfig);
console.log(`\nTypechecking ${label}`); console.log(`\nTypechecking ${label}`);
const result = Bun.spawnSync([bunExecutable, "x", "tsc", "-p", tsconfig, "--noEmit", "--incremental", "false", "--pretty", "false"], { const result = Bun.spawnSync(
[
bunExecutable,
"x",
"tsc",
"-p",
tsconfig,
"--noEmit",
"--incremental",
"false",
"--pretty",
"false"
],
{
stdout: "inherit", stdout: "inherit",
stderr: "inherit" stderr: "inherit"
}); }
);
if (result.exitCode !== 0) { if (result.exitCode !== 0) {
failed = true; failed = true;

View file

@ -465,8 +465,7 @@ const parseCandleParams = (
const endTs = params.end_ts ?? Date.now(); const endTs = params.end_ts ?? Date.now();
const limit = params.limit ?? env.REST_DEFAULT_LIMIT; const limit = params.limit ?? env.REST_DEFAULT_LIMIT;
const startTs = const startTs = params.start_ts ?? Math.max(0, Math.floor(endTs - params.interval_ms * limit));
params.start_ts ?? Math.max(0, Math.floor(endTs - params.interval_ms * limit));
const rangeStart = Math.min(startTs, endTs); const rangeStart = Math.min(startTs, endTs);
const rangeEnd = Math.max(startTs, endTs); const rangeEnd = Math.max(startTs, endTs);
@ -482,7 +481,13 @@ const parseCandleParams = (
const parseCandleReplayParams = ( const parseCandleReplayParams = (
url: URL url: URL
): { underlyingId: string; intervalMs: number; afterTs: number; afterSeq: number; limit: number } => { ): {
underlyingId: string;
intervalMs: number;
afterTs: number;
afterSeq: number;
limit: number;
} => {
const params = candleReplaySchema.parse({ const params = candleReplaySchema.parse({
underlying_id: url.searchParams.get("underlying_id") ?? undefined, underlying_id: url.searchParams.get("underlying_id") ?? undefined,
interval_ms: url.searchParams.get("interval_ms") ?? undefined, interval_ms: url.searchParams.get("interval_ms") ?? undefined,
@ -601,7 +606,10 @@ const matchesScopedOptionSubscription = (
print: { underlying_id?: string; option_contract_id: string }, print: { underlying_id?: string; option_contract_id: string },
subscription: Extract<LiveSubscription, { channel: "options" }> subscription: Extract<LiveSubscription, { channel: "options" }>
): boolean => { ): boolean => {
if (subscription.option_contract_id && subscription.option_contract_id !== print.option_contract_id) { if (
subscription.option_contract_id &&
subscription.option_contract_id !== print.option_contract_id
) {
return false; return false;
} }
if (subscription.underlying_ids?.length) { if (subscription.underlying_ids?.length) {
@ -693,8 +701,7 @@ const run = async () => {
env.OPTIONS_INGEST_ADAPTER, env.OPTIONS_INGEST_ADAPTER,
env.EQUITIES_INGEST_ADAPTER env.EQUITIES_INGEST_ADAPTER
); );
const syntheticBackendDisabledReason = const syntheticBackendDisabledReason = getSyntheticBackendDisabledReason(syntheticBackendMode);
getSyntheticBackendDisabledReason(syntheticBackendMode);
const syntheticControlKv = await openSyntheticControlKv(js); const syntheticControlKv = await openSyntheticControlKv(js);
let syntheticControl = await ensureSyntheticControlState(syntheticControlKv); let syntheticControl = await ensureSyntheticControlState(syntheticControlKv);
const syntheticProfileHits = createRollingSyntheticProfileHits(); const syntheticProfileHits = createRollingSyntheticProfileHits();
@ -899,11 +906,7 @@ const run = async () => {
} }
} }
const subscribeWithReset = async <T>( const subscribeWithReset = async <T>(subject: string, stream: string, durableName: string) => {
subject: string,
stream: string,
durableName: string
) => {
const opts = buildDurableConsumer(durableName); const opts = buildDurableConsumer(durableName);
applyDeliverPolicy(opts, env.API_DELIVER_POLICY); applyDeliverPolicy(opts, env.API_DELIVER_POLICY);
try { try {
@ -924,7 +927,8 @@ const run = async () => {
try { try {
await jsm.consumers.delete(stream, durableName); await jsm.consumers.delete(stream, durableName);
} catch (deleteError) { } catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError); const deleteMessage =
deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) { if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", { logger.warn("failed to delete jetstream consumer", {
durable: durableName, durable: durableName,
@ -1023,8 +1027,12 @@ const run = async () => {
} }
const matchingSubscriptions = const matchingSubscriptions =
subscription.channel === "options" || subscription.channel === "flow" || subscription.channel === "equities" subscription.channel === "options" ||
? [...subscriptionDefinitions.entries()].filter(([, candidate]) => candidate.channel === subscription.channel) subscription.channel === "flow" ||
subscription.channel === "equities"
? [...subscriptionDefinitions.entries()].filter(
([, candidate]) => candidate.channel === subscription.channel
)
: [[getSubscriptionKey(subscription), subscription] as const]; : [[getSubscriptionKey(subscription), subscription] as const];
if (matchingSubscriptions.length === 0) { if (matchingSubscriptions.length === 0) {
@ -1032,8 +1040,12 @@ const run = async () => {
} }
const optionItem = ingestChannel === "options" ? (item as OptionPrint) : null; const optionItem = ingestChannel === "options" ? (item as OptionPrint) : null;
const equityItem = ingestChannel === "equities" ? (item as Parameters<typeof matchesScopedEquitySubscription>[0]) : null; const equityItem =
const flowItem = ingestChannel === "flow" ? (item as Parameters<typeof matchesFlowPacketFilters>[0]) : null; ingestChannel === "equities"
? (item as Parameters<typeof matchesScopedEquitySubscription>[0])
: null;
const flowItem =
ingestChannel === "flow" ? (item as Parameters<typeof matchesFlowPacketFilters>[0]) : null;
let matchedSubscriptions = 0; let matchedSubscriptions = 0;
for (const [key, candidate] of matchingSubscriptions) { for (const [key, candidate] of matchingSubscriptions) {
@ -1315,9 +1327,7 @@ const run = async () => {
}, },
control: syntheticBackendMode === "synthetic" ? syntheticControl : null, control: syntheticBackendMode === "synthetic" ? syntheticControl : null,
derived, derived,
...(syntheticBackendDisabledReason ...(syntheticBackendDisabledReason ? { disabled_reason: syntheticBackendDisabledReason } : {})
? { disabled_reason: syntheticBackendDisabledReason }
: {})
}; };
}; };
@ -1385,11 +1395,7 @@ const run = async () => {
syntheticControl = await writeSyntheticControlState(syntheticControlKv, payload); syntheticControl = await writeSyntheticControlState(syntheticControlKv, payload);
return jsonResponse({ return jsonResponse({
control: syntheticControl, control: syntheticControl,
derived: buildSyntheticDerivedStatus( derived: buildSyntheticDerivedStatus(Date.now(), syntheticControl, syntheticProfileHits)
Date.now(),
syntheticControl,
syntheticProfileHits
)
}); });
} catch (error) { } catch (error) {
return jsonResponse( return jsonResponse(
@ -1436,7 +1442,13 @@ const run = async () => {
if (req.method === "GET" && url.pathname === "/prints/equities/range") { if (req.method === "GET" && url.pathname === "/prints/equities/range") {
try { try {
const { underlyingId, startTs, endTs, limit } = parseEquityPrintRangeParams(url); const { underlyingId, startTs, endTs, limit } = parseEquityPrintRangeParams(url);
const data = await fetchEquityPrintsRange(clickhouse, underlyingId, startTs, endTs, limit); const data = await fetchEquityPrintsRange(
clickhouse,
underlyingId,
startTs,
endTs,
limit
);
return jsonResponse({ data }); return jsonResponse({ data });
} catch (error) { } catch (error) {
return jsonResponse( return jsonResponse(
@ -1566,7 +1578,9 @@ const run = async () => {
source, source,
storageFilters storageFilters
); );
return jsonResponse(buildHistoryResponse(data, (item) => ({ ts: item.ts, seq: item.seq }))); return jsonResponse(
buildHistoryResponse(data, (item) => ({ ts: item.ts, seq: item.seq }))
);
} catch (error) { } catch (error) {
return jsonResponse( return jsonResponse(
{ {
@ -1986,7 +2000,9 @@ const run = async () => {
const payload = const payload =
typeof message === "string" typeof message === "string"
? message ? message
: new TextDecoder().decode(message instanceof Uint8Array ? message : new Uint8Array(message)); : new TextDecoder().decode(
message instanceof Uint8Array ? message : new Uint8Array(message)
);
const parsed = LiveClientMessageSchema.parse(JSON.parse(payload)); const parsed = LiveClientMessageSchema.parse(JSON.parse(payload));
if (parsed.op === "ping") { if (parsed.op === "ping") {
sendLiveMessage(socket, { sendLiveMessage(socket, {

View file

@ -165,11 +165,21 @@ const parseGenericLimitFallback = (env: NodeJS.ProcessEnv, fallback: number): nu
return Math.max(MIN_GENERIC_LIMIT, Math.min(MAX_GENERIC_LIMIT, Math.floor(parsed))); return Math.max(MIN_GENERIC_LIMIT, Math.min(MAX_GENERIC_LIMIT, Math.floor(parsed)));
}; };
export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env): GenericLiveLimits => { export const resolveGenericLiveLimits = (
env: NodeJS.ProcessEnv = process.env
): GenericLiveLimits => {
const liveLimitDefault = parseGenericLimitFallback(env, DEFAULT_GENERIC_LIMIT); const liveLimitDefault = parseGenericLimitFallback(env, DEFAULT_GENERIC_LIMIT);
return { return {
options: parseGenericLimit(env, "options", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.options), options: parseGenericLimit(
nbbo: parseGenericLimit(env, "nbbo", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.nbbo), env,
"options",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.options
),
nbbo: parseGenericLimit(
env,
"nbbo",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.nbbo
),
equities: parseGenericLimit( equities: parseGenericLimit(
env, env,
"equities", "equities",
@ -185,7 +195,11 @@ export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env):
"equity-joins", "equity-joins",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["equity-joins"] env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["equity-joins"]
), ),
flow: parseGenericLimit(env, "flow", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.flow), flow: parseGenericLimit(
env,
"flow",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.flow
),
"smart-money": parseGenericLimit( "smart-money": parseGenericLimit(
env, env,
"smart-money", "smart-money",
@ -196,13 +210,21 @@ export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env):
"classifier-hits", "classifier-hits",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["classifier-hits"] env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["classifier-hits"]
), ),
alerts: parseGenericLimit(env, "alerts", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.alerts), alerts: parseGenericLimit(
env,
"alerts",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.alerts
),
"inferred-dark": parseGenericLimit( "inferred-dark": parseGenericLimit(
env, env,
"inferred-dark", "inferred-dark",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["inferred-dark"] env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["inferred-dark"]
), ),
news: parseGenericLimit(env, "news", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.news) news: parseGenericLimit(
env,
"news",
env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.news
)
}; };
}; };
@ -227,12 +249,18 @@ const extractFreshnessTs = (channel: LiveGenericChannel, item: any): number | nu
export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): LiveStateConfig => ({ export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): LiveStateConfig => ({
limits: resolveGenericLiveLimits(env), limits: resolveGenericLiveLimits(env),
scopedCacheMaxKeys: parsePositiveInt(env.LIVE_SCOPED_CACHE_MAX_KEYS, DEFAULT_SCOPED_CACHE_MAX_KEYS), scopedCacheMaxKeys: parsePositiveInt(
env.LIVE_SCOPED_CACHE_MAX_KEYS,
DEFAULT_SCOPED_CACHE_MAX_KEYS
),
redisFlushIntervalMs: parsePositiveInt( redisFlushIntervalMs: parsePositiveInt(
env.LIVE_REDIS_FLUSH_INTERVAL_MS, env.LIVE_REDIS_FLUSH_INTERVAL_MS,
DEFAULT_REDIS_FLUSH_INTERVAL_MS DEFAULT_REDIS_FLUSH_INTERVAL_MS
), ),
redisFlushMaxItems: parsePositiveInt(env.LIVE_REDIS_FLUSH_MAX_ITEMS, DEFAULT_REDIS_FLUSH_MAX_ITEMS) redisFlushMaxItems: parsePositiveInt(
env.LIVE_REDIS_FLUSH_MAX_ITEMS,
DEFAULT_REDIS_FLUSH_MAX_ITEMS
)
}); });
const parsePositiveInt = (value: string | undefined, fallback: number): number => { const parsePositiveInt = (value: string | undefined, fallback: number): number => {
const parsed = Number(value); const parsed = Number(value);
@ -242,10 +270,7 @@ const parsePositiveInt = (value: string | undefined, fallback: number): number =
return Math.max(1, Math.floor(parsed)); return Math.max(1, Math.floor(parsed));
}; };
type RedisLike = Pick< type RedisLike = Pick<RedisClientType, "isOpen" | "lRange" | "lPush" | "lTrim" | "hGet" | "hSet">;
RedisClientType,
"isOpen" | "lRange" | "lPush" | "lTrim" | "hGet" | "hSet"
>;
const parseCursor = (value: string | null): Cursor | null => { const parseCursor = (value: string | null): Cursor | null => {
if (!value) { if (!value) {
@ -259,7 +284,9 @@ const parseCursor = (value: string | null): Cursor | null => {
} }
}; };
const getGenericConfig = (limits: GenericLiveLimits): { const getGenericConfig = (
limits: GenericLiveLimits
): {
[K in LiveGenericChannel]: GenericFeedConfig; [K in LiveGenericChannel]: GenericFeedConfig;
} => ({ } => ({
options: { options: {
@ -365,7 +392,7 @@ const parseJsonList = <T>(payloads: string[], parse: (value: unknown) => T): T[]
return items; return items;
}; };
const compareCursors = (a: Cursor, b: Cursor): number => (b.ts - a.ts) || (b.seq - a.seq); const compareCursors = (a: Cursor, b: Cursor): number => b.ts - a.ts || b.seq - a.seq;
const sortGenericItems = <T>(items: T[], cursorOf: (item: T) => Cursor): T[] => const sortGenericItems = <T>(items: T[], cursorOf: (item: T) => Cursor): T[] =>
[...items].sort((a, b) => compareCursors(cursorOf(a), cursorOf(b))); [...items].sort((a, b) => compareCursors(cursorOf(a), cursorOf(b)));
@ -480,7 +507,10 @@ const matchesScopedOptionSnapshot = (
return false; return false;
} }
if (subscription.option_contract_id && item.option_contract_id !== subscription.option_contract_id) { if (
subscription.option_contract_id &&
item.option_contract_id !== subscription.option_contract_id
) {
return false; return false;
} }
@ -529,11 +559,8 @@ const candleCursorField = (underlyingId: string, intervalMs: number): string =>
const overlayRedisKey = (underlyingId: string): string => `live:equity-overlay:${underlyingId}`; const overlayRedisKey = (underlyingId: string): string => `live:equity-overlay:${underlyingId}`;
const overlayCursorField = (underlyingId: string): string => `equities:${underlyingId}`; const overlayCursorField = (underlyingId: string): string => `equities:${underlyingId}`;
const dropMatchingCursor = <T>( const dropMatchingCursor = <T>(items: T[], target: Cursor, cursorOf: (item: T) => Cursor): T[] =>
items: T[], items.filter((item) => compareCursors(cursorOf(item), target) !== 0);
target: Cursor,
cursorOf: (item: T) => Cursor
): T[] => items.filter((item) => compareCursors(cursorOf(item), target) !== 0);
const insertNewestFirst = <T>( const insertNewestFirst = <T>(
items: T[], items: T[],
@ -676,7 +703,13 @@ export class LiveStateManager {
this.pendingRedisWrites.clear(); this.pendingRedisWrites.clear();
for (const write of writes) { for (const write of writes) {
await this.persistList(write.listKey, write.cursorField, write.items, write.limit, write.cursor); await this.persistList(
write.listKey,
write.cursorField,
write.items,
write.limit,
write.cursor
);
this.stats.redisFlushCount += 1; this.stats.redisFlushCount += 1;
this.stats.redisFlushItems += write.items.length; this.stats.redisFlushItems += write.items.length;
metrics.count("api.live.redis_flush_count", 1); metrics.count("api.live.redis_flush_count", 1);
@ -726,7 +759,12 @@ export class LiveStateManager {
} }
} }
private updateFreshnessMetric(listKey: string, channel: LiveChannel, item: unknown, now = Date.now()): void { private updateFreshnessMetric(
listKey: string,
channel: LiveChannel,
item: unknown,
now = Date.now()
): void {
const ts = const ts =
channel === "equity-candles" || channel === "equity-overlay" channel === "equity-candles" || channel === "equity-overlay"
? typeof (item as { ts?: unknown })?.ts === "number" ? typeof (item as { ts?: unknown })?.ts === "number"
@ -784,12 +822,22 @@ export class LiveStateManager {
config.cursorField, config.cursorField,
parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, config.cursorField)) parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, config.cursorField))
); );
await this.persistList(config.redisKey, config.cursorField, cached, config.limit, this.genericCursors.get(config.cursorField) ?? null); await this.persistList(
config.redisKey,
config.cursorField,
cached,
config.limit,
this.genericCursors.get(config.cursorField) ?? null
);
return; return;
} }
} }
const fresh = normalizeGenericItems(channel, await config.fetchRecent(this.clickhouse, config.limit), config); const fresh = normalizeGenericItems(
channel,
await config.fetchRecent(this.clickhouse, config.limit),
config
);
this.stats.genericHydrateFromClickHouse += 1; this.stats.genericHydrateFromClickHouse += 1;
this.stats.cacheDepthByKey.set(config.redisKey, fresh.length); this.stats.cacheDepthByKey.set(config.redisKey, fresh.length);
this.genericItems.set(channel, fresh); this.genericItems.set(channel, fresh);
@ -806,7 +854,8 @@ export class LiveStateManager {
case "options": { case "options": {
const config = this.generic.options; const config = this.generic.options;
const limit = snapshotLimitFor(subscription, config.limit); const limit = snapshotLimitFor(subscription, config.limit);
const scoped = Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id); const scoped =
Boolean(subscription.underlying_ids?.length) || Boolean(subscription.option_contract_id);
if (subscription.filters?.view === "raw" || scoped) { if (subscription.filters?.view === "raw" || scoped) {
const cached = (this.genericItems.get("options") ?? []) const cached = (this.genericItems.get("options") ?? [])
.filter((entry) => matchesScopedOptionSnapshot(entry, subscription)) .filter((entry) => matchesScopedOptionSnapshot(entry, subscription))
@ -815,8 +864,16 @@ export class LiveStateManager {
if (cached.length < limit) { if (cached.length < limit) {
this.stats.scopedClickHouseSnapshots += 1; this.stats.scopedClickHouseSnapshots += 1;
const storageFilters = buildOptionSnapshotFilters(subscription); const storageFilters = buildOptionSnapshotFilters(subscription);
const backfill = await fetchRecentOptionPrints(this.clickhouse, limit, undefined, storageFilters); const backfill = await fetchRecentOptionPrints(
items = mergeSnapshotBackfill(cached, backfill, limit, (entry) => ({ ts: entry.ts, seq: entry.seq })); this.clickhouse,
limit,
undefined,
storageFilters
);
items = mergeSnapshotBackfill(cached, backfill, limit, (entry) => ({
ts: entry.ts,
seq: entry.seq
}));
} }
return { return {
subscription, subscription,
@ -942,7 +999,11 @@ export class LiveStateManager {
this.candleItems.set(key, nextState.items); this.candleItems.set(key, nextState.items);
this.candleCursors.set(cursorField, cursor); this.candleCursors.set(cursorField, cursor);
this.touchAccess(this.candleAccess, key); this.touchAccess(this.candleAccess, key);
this.evictScopedCachesIfNeeded(this.candleItems as Map<string, unknown[]>, this.candleCursors, this.candleAccess); this.evictScopedCachesIfNeeded(
this.candleItems as Map<string, unknown[]>,
this.candleCursors,
this.candleAccess
);
if (nextState.outOfOrder) { if (nextState.outOfOrder) {
this.stats.outOfOrderEvents += 1; this.stats.outOfOrderEvents += 1;
metrics.count("api.live.out_of_order_events", 1); metrics.count("api.live.out_of_order_events", 1);
@ -968,7 +1029,11 @@ export class LiveStateManager {
this.overlayItems.set(key, nextState.items); this.overlayItems.set(key, nextState.items);
this.overlayCursors.set(cursorField, cursor); this.overlayCursors.set(cursorField, cursor);
this.touchAccess(this.overlayAccess, key); this.touchAccess(this.overlayAccess, key);
this.evictScopedCachesIfNeeded(this.overlayItems as Map<string, unknown[]>, this.overlayCursors, this.overlayAccess); this.evictScopedCachesIfNeeded(
this.overlayItems as Map<string, unknown[]>,
this.overlayCursors,
this.overlayAccess
);
if (nextState.outOfOrder) { if (nextState.outOfOrder) {
this.stats.outOfOrderEvents += 1; this.stats.outOfOrderEvents += 1;
metrics.count("api.live.out_of_order_events", 1); metrics.count("api.live.out_of_order_events", 1);
@ -991,10 +1056,19 @@ export class LiveStateManager {
const nextState = const nextState =
channel === "nbbo" channel === "nbbo"
? { ? {
items: normalizeGenericItems(channel, [parsed, ...(this.genericItems.get(channel) ?? [])], config), items: normalizeGenericItems(
channel,
[parsed, ...(this.genericItems.get(channel) ?? [])],
config
),
outOfOrder: false outOfOrder: false
} }
: insertNewestFirst(this.genericItems.get(channel) ?? [], parsed, config.cursor, config.limit); : insertNewestFirst(
this.genericItems.get(channel) ?? [],
parsed,
config.cursor,
config.limit
);
if (nextState.outOfOrder) { if (nextState.outOfOrder) {
this.stats.outOfOrderEvents += 1; this.stats.outOfOrderEvents += 1;
@ -1007,7 +1081,13 @@ export class LiveStateManager {
if (nextState.items.length > 0) { if (nextState.items.length > 0) {
this.updateFreshnessMetric(config.redisKey, channel, nextState.items[0]); this.updateFreshnessMetric(config.redisKey, channel, nextState.items[0]);
} }
this.queueRedisWrite(config.redisKey, config.cursorField, nextState.items, config.limit, cursor); this.queueRedisWrite(
config.redisKey,
config.cursorField,
nextState.items,
config.limit,
cursor
);
return cursor; return cursor;
} }
} }
@ -1022,18 +1102,34 @@ export class LiveStateManager {
if (cached.length > 0) { if (cached.length > 0) {
this.candleItems.set(key, cached); this.candleItems.set(key, cached);
this.touchAccess(this.candleAccess, key); this.touchAccess(this.candleAccess, key);
this.evictScopedCachesIfNeeded(this.candleItems as Map<string, unknown[]>, this.candleCursors, this.candleAccess); this.evictScopedCachesIfNeeded(
this.candleItems as Map<string, unknown[]>,
this.candleCursors,
this.candleAccess
);
this.stats.cacheDepthByKey.set(key, cached.length); this.stats.cacheDepthByKey.set(key, cached.length);
this.updateFreshnessMetric(key, "equity-candles", cached[0]); this.updateFreshnessMetric(key, "equity-candles", cached[0]);
this.candleCursors.set(cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField))); this.candleCursors.set(
cursorField,
parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField))
);
return; return;
} }
} }
const fresh = await fetchRecentEquityCandles(this.clickhouse, underlyingId, intervalMs, CHART_LIMITS.candles); const fresh = await fetchRecentEquityCandles(
this.clickhouse,
underlyingId,
intervalMs,
CHART_LIMITS.candles
);
this.candleItems.set(key, fresh); this.candleItems.set(key, fresh);
this.touchAccess(this.candleAccess, key); this.touchAccess(this.candleAccess, key);
this.evictScopedCachesIfNeeded(this.candleItems as Map<string, unknown[]>, this.candleCursors, this.candleAccess); this.evictScopedCachesIfNeeded(
this.candleItems as Map<string, unknown[]>,
this.candleCursors,
this.candleAccess
);
this.stats.cacheDepthByKey.set(key, fresh.length); this.stats.cacheDepthByKey.set(key, fresh.length);
if (fresh.length > 0) { if (fresh.length > 0) {
this.updateFreshnessMetric(key, "equity-candles", fresh[0]); this.updateFreshnessMetric(key, "equity-candles", fresh[0]);
@ -1052,10 +1148,17 @@ export class LiveStateManager {
if (cached.length > 0) { if (cached.length > 0) {
this.overlayItems.set(key, cached); this.overlayItems.set(key, cached);
this.touchAccess(this.overlayAccess, key); this.touchAccess(this.overlayAccess, key);
this.evictScopedCachesIfNeeded(this.overlayItems as Map<string, unknown[]>, this.overlayCursors, this.overlayAccess); this.evictScopedCachesIfNeeded(
this.overlayItems as Map<string, unknown[]>,
this.overlayCursors,
this.overlayAccess
);
this.stats.cacheDepthByKey.set(key, cached.length); this.stats.cacheDepthByKey.set(key, cached.length);
this.updateFreshnessMetric(key, "equity-overlay", cached[0]); this.updateFreshnessMetric(key, "equity-overlay", cached[0]);
this.overlayCursors.set(cursorField, parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField))); this.overlayCursors.set(
cursorField,
parseCursor(await this.redis.hGet(CURSOR_HASH_KEY, cursorField))
);
return; return;
} }
} }
@ -1065,7 +1168,11 @@ export class LiveStateManager {
); );
this.overlayItems.set(key, fresh); this.overlayItems.set(key, fresh);
this.touchAccess(this.overlayAccess, key); this.touchAccess(this.overlayAccess, key);
this.evictScopedCachesIfNeeded(this.overlayItems as Map<string, unknown[]>, this.overlayCursors, this.overlayAccess); this.evictScopedCachesIfNeeded(
this.overlayItems as Map<string, unknown[]>,
this.overlayCursors,
this.overlayAccess
);
this.stats.cacheDepthByKey.set(key, fresh.length); this.stats.cacheDepthByKey.set(key, fresh.length);
if (fresh.length > 0) { if (fresh.length > 0) {
this.updateFreshnessMetric(key, "equity-overlay", fresh[0]); this.updateFreshnessMetric(key, "equity-overlay", fresh[0]);

View file

@ -83,11 +83,7 @@ export const buildSyntheticDerivedStatus = (
session_phase: session.session_phase, session_phase: session.session_phase,
regime: session.regime, regime: session.regime,
focus_symbols: session.focus_symbols, focus_symbols: session.focus_symbols,
profile_hit_counts: getSyntheticProfileHitCounts( profile_hit_counts: getSyntheticProfileHitCounts(state, now, control.coverage_window_minutes),
state,
now,
control.coverage_window_minutes
),
coverage_window_minutes: control.coverage_window_minutes coverage_window_minutes: control.coverage_window_minutes
}); });
}; };

View file

@ -3,7 +3,9 @@ import { isAlertContextPath, parseAlertContextTraceIdPath } from "../src/alert-c
describe("alert context route helpers", () => { describe("alert context route helpers", () => {
it("extracts a valid alert trace id from the context endpoint path", () => { it("extracts a valid alert trace id from the context endpoint path", () => {
expect(parseAlertContextTraceIdPath("/flow/alerts/alert%3Actx%2Fone/context")).toBe("alert:ctx/one"); expect(parseAlertContextTraceIdPath("/flow/alerts/alert%3Actx%2Fone/context")).toBe(
"alert:ctx/one"
);
}); });
it("returns null for unrelated alert paths", () => { it("returns null for unrelated alert paths", () => {

View file

@ -9,9 +9,7 @@ import {
shouldFanoutLiveEvent shouldFanoutLiveEvent
} from "../src/live"; } from "../src/live";
const makeClickHouse = ( const makeClickHouse = (queryResolver?: (query: string) => unknown[]): ClickHouseClient =>
queryResolver?: (query: string) => unknown[]
): ClickHouseClient =>
({ ({
exec: async () => {}, exec: async () => {},
insert: async () => {}, insert: async () => {},
@ -149,10 +147,7 @@ describe("LiveStateManager", () => {
it("trims generic windows to configured per-channel limits", async () => { it("trims generic windows to configured per-channel limits", async () => {
const redis = makeRedis(); const redis = makeRedis();
const now = Date.now(); const now = Date.now();
const manager = new LiveStateManager( const manager = new LiveStateManager(makeClickHouse(), redis as never, {
makeClickHouse(),
redis as never,
{
options: 10000, options: 10000,
nbbo: 10000, nbbo: 10000,
equities: 10000, equities: 10000,
@ -163,8 +158,7 @@ describe("LiveStateManager", () => {
"classifier-hits": 10000, "classifier-hits": 10000,
alerts: 10000, alerts: 10000,
"inferred-dark": 10000 "inferred-dark": 10000
} });
);
await manager.ingest("flow", { await manager.ingest("flow", {
source_ts: now, source_ts: now,
@ -503,18 +497,15 @@ describe("LiveStateManager", () => {
manager.getSnapshot({ channel: "flow" }) manager.getSnapshot({ channel: "flow" })
]); ]);
expect((optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ expect(
"opt-fresh", (optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
"opt-stale" ).toEqual(["opt-fresh", "opt-stale"]);
]); expect(
expect((nbboSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ (nbboSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
"nbbo-fresh", ).toEqual(["nbbo-fresh", "nbbo-stale"]);
"nbbo-stale" expect(
]); (equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
expect((equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ ).toEqual(["eq-fresh", "eq-stale"]);
"eq-fresh",
"eq-stale"
]);
expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([ expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([
"flow-fresh", "flow-fresh",
"flow-stale" "flow-stale"
@ -699,10 +690,9 @@ describe("LiveStateManager", () => {
option_contract_id: "AAPL-2025-01-17-200-C" option_contract_id: "AAPL-2025-01-17-200-C"
}); });
expect((snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id).slice(0, 2)).toEqual([ expect(
"opt-hot", (snapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id).slice(0, 2)
"opt-backfill" ).toEqual(["opt-hot", "opt-backfill"]);
]);
}); });
it("seeds scoped equity snapshots from clickhouse rows older than 24h", async () => { it("seeds scoped equity snapshots from clickhouse rows older than 24h", async () => {
@ -806,12 +796,12 @@ describe("LiveStateManager", () => {
manager.getSnapshot({ channel: "flow" }) manager.getSnapshot({ channel: "flow" })
]); ]);
expect((optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ expect(
"opt-retained" (optionsSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
]); ).toEqual(["opt-retained"]);
expect((equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)).toEqual([ expect(
"eq-retained" (equitiesSnapshot.items as Array<{ trace_id: string }>).map((item) => item.trace_id)
]); ).toEqual(["eq-retained"]);
expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([ expect((flowSnapshot.items as Array<{ id: string }>).map((item) => item.id)).toEqual([
"flow-retained" "flow-retained"
]); ]);
@ -1047,7 +1037,10 @@ describe("LiveStateManager", () => {
}); });
it("tracks generic cache and scoped clickhouse snapshot sources separately", async () => { it("tracks generic cache and scoped clickhouse snapshot sources separately", async () => {
const manager = new LiveStateManager(makeClickHouse(() => []), null); const manager = new LiveStateManager(
makeClickHouse(() => []),
null
);
const now = Date.now(); const now = Date.now();
await manager.ingest("options", { await manager.ingest("options", {
@ -1075,7 +1068,10 @@ describe("LiveStateManager", () => {
}); });
it("keeps backend channel health healthy when a scoped query is quiet", async () => { it("keeps backend channel health healthy when a scoped query is quiet", async () => {
const manager = new LiveStateManager(makeClickHouse(() => []), null); const manager = new LiveStateManager(
makeClickHouse(() => []),
null
);
const now = Date.now(); const now = Date.now();
await manager.ingest("options", { await manager.ingest("options", {
@ -1098,7 +1094,9 @@ describe("LiveStateManager", () => {
expect(quietSnapshot.items).toEqual([]); expect(quietSnapshot.items).toEqual([]);
expect(manager.getHotChannelHealth().options.healthy).toBe(true); expect(manager.getHotChannelHealth().options.healthy).toBe(true);
expect(manager.getStatsSnapshot().freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.options]).toBeLessThanOrEqual(50); expect(
manager.getStatsSnapshot().freshnessAgeMsByKey[HOT_LIVE_REDIS_KEYS.options]
).toBeLessThanOrEqual(50);
}); });
it("exposes freshness helper for feed status", () => { it("exposes freshness helper for feed status", () => {

View file

@ -33,9 +33,7 @@ const envSchema = z.object({
CANDLE_INTERVALS_MS: z.string().default("60000,300000"), CANDLE_INTERVALS_MS: z.string().default("60000,300000"),
CANDLE_MAX_LATE_MS: z.coerce.number().int().nonnegative().default(0), CANDLE_MAX_LATE_MS: z.coerce.number().int().nonnegative().default(0),
CANDLE_CACHE_LIMIT: z.coerce.number().int().nonnegative().default(2000), CANDLE_CACHE_LIMIT: z.coerce.number().int().nonnegative().default(2000),
CANDLE_DELIVER_POLICY: z CANDLE_DELIVER_POLICY: z.enum(["new", "all", "last", "last_per_subject"]).default("new"),
.enum(["new", "all", "last", "last_per_subject"])
.default("new"),
CANDLE_CONSUMER_RESET: z CANDLE_CONSUMER_RESET: z
.preprocess((value) => { .preprocess((value) => {
if (typeof value === "string") { if (typeof value === "string") {
@ -290,7 +288,10 @@ const run = async () => {
} else { } else {
try { try {
const info = await jsm.consumers.info(STREAM_EQUITY_PRINTS, durableName); const info = await jsm.consumers.info(STREAM_EQUITY_PRINTS, durableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.CANDLE_DELIVER_POLICY) { if (
info?.config?.deliver_policy &&
info.config.deliver_policy !== env.CANDLE_DELIVER_POLICY
) {
logger.warn("resetting consumer due to deliver policy change", { logger.warn("resetting consumer due to deliver policy change", {
durable: durableName, durable: durableName,
current: info.config.deliver_policy, current: info.config.deliver_policy,
@ -301,7 +302,10 @@ const run = async () => {
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : String(error); const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) { if (!message.includes("not found")) {
logger.warn("failed to inspect jetstream consumer", { durable: durableName, error: message }); logger.warn("failed to inspect jetstream consumer", {
durable: durableName,
error: message
});
} }
} }
} }
@ -327,7 +331,8 @@ const run = async () => {
try { try {
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, durableName); await jsm.consumers.delete(STREAM_EQUITY_PRINTS, durableName);
} catch (deleteError) { } catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError); const deleteMessage =
deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) { if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", { logger.warn("failed to delete jetstream consumer", {
durable: durableName, durable: durableName,

View file

@ -14,4 +14,3 @@ export const scoreAlert = (
const severity = score >= 80 ? "high" : score >= 45 ? "medium" : "low"; const severity = score >= 80 ? "high" : score >= 45 ? "medium" : "low";
return { score, severity }; return { score, severity };
}; };

View file

@ -573,10 +573,7 @@ const buildVerticalSpreadHit = (
}; };
}; };
const buildLadderHit = ( const buildLadderHit = (packet: FlowPacket, config: ClassifierConfig): ClassifierHit | null => {
packet: FlowPacket,
config: ClassifierConfig
): ClassifierHit | null => {
const structureType = getStringFeature(packet, "structure_type"); const structureType = getStringFeature(packet, "structure_type");
if (structureType !== "ladder") { if (structureType !== "ladder") {
return null; return null;
@ -648,7 +645,8 @@ const buildRollHit = (packet: FlowPacket, config: ClassifierConfig): ClassifierH
} }
const activity = getLargeActivity(packet, config); const activity = getLargeActivity(packet, config);
const qualifies = activity.totalPremium >= config.spikeMinPremium || activity.totalSize >= config.spikeMinSize; const qualifies =
activity.totalPremium >= config.spikeMinPremium || activity.totalSize >= config.spikeMinSize;
if (!qualifies) { if (!qualifies) {
return null; return null;
} }
@ -708,7 +706,9 @@ const buildRollHit = (packet: FlowPacket, config: ClassifierConfig): ClassifierH
const expiryNote = hasExpiryPair const expiryNote = hasExpiryPair
? `Expiries: ${fromExpiry} -> ${toExpiry}${ ? `Expiries: ${fromExpiry} -> ${toExpiry}${
expiryDaysDelta !== null && expiryDaysDelta !== 0 ? ` (${Math.round(expiryDaysDelta)}d)` : "" expiryDaysDelta !== null && expiryDaysDelta !== 0
? ` (${Math.round(expiryDaysDelta)}d)`
: ""
}.` }.`
: "Expiry pairing unavailable."; : "Expiry pairing unavailable.";
const strikeNote = hasStrikePair const strikeNote = hasStrikePair
@ -850,7 +850,8 @@ export const evaluateClassifiers = (
const packetKind = getStringFeature(packet, "packet_kind"); const packetKind = getStringFeature(packet, "packet_kind");
const structureOnly = packetKind === "structure"; const structureOnly = packetKind === "structure";
const contractId = typeof packet.features.option_contract_id === "string" const contractId =
typeof packet.features.option_contract_id === "string"
? packet.features.option_contract_id ? packet.features.option_contract_id
: ""; : "";
const contract = structureOnly ? null : parseContractId(contractId); const contract = structureOnly ? null : parseContractId(contractId);

View file

@ -15,10 +15,7 @@ const roundTo = (value: number, digits = 4): number => {
return Number(value.toFixed(digits)); return Number(value.toFixed(digits));
}; };
export const classifyQuotePlacement = ( export const classifyQuotePlacement = (price: number, join: EquityQuoteJoin): QuotePlacement => {
price: number,
join: EquityQuoteJoin
): QuotePlacement => {
if (!Number.isFinite(price)) { if (!Number.isFinite(price)) {
return "MISSING"; return "MISSING";
} }

View file

@ -46,7 +46,7 @@ import {
enqueueEquityPrintJoinInsert, enqueueEquityPrintJoinInsert,
enqueueFlowPacketInsert, enqueueFlowPacketInsert,
enqueueInferredDarkInsert, enqueueInferredDarkInsert,
enqueueSmartMoneyEventInsert, enqueueSmartMoneyEventInsert
} from "@islandflow/storage"; } from "@islandflow/storage";
import { import {
AlertEventSchema, AlertEventSchema,
@ -324,7 +324,9 @@ const buildPacketId = (cluster: ClusterState): string => {
const isExpectedShutdownNatsError = (error: unknown): boolean => { const isExpectedShutdownNatsError = (error: unknown): boolean => {
const code = getErrorCode(error); const code = getErrorCode(error);
return runtimeState.shuttingDown && (code === "CONNECTION_DRAINING" || code === "CONNECTION_CLOSED"); return (
runtimeState.shuttingDown && (code === "CONNECTION_DRAINING" || code === "CONNECTION_CLOSED")
);
}; };
const createPlacementCounts = (): NbboPlacementCounts => ({ const createPlacementCounts = (): NbboPlacementCounts => ({
@ -337,7 +339,14 @@ const createPlacementCounts = (): NbboPlacementCounts => ({
stale: 0 stale: 0
}); });
const SPECIAL_PRINT_CONDITIONS = new Set(["AUCTION", "CROSS", "OPENING", "CLOSING", "COMPLEX", "SPREAD"]); const SPECIAL_PRINT_CONDITIONS = new Set([
"AUCTION",
"CROSS",
"OPENING",
"CLOSING",
"COMPLEX",
"SPREAD"
]);
const SYNTHETIC_EVENT_CONDITION_RE = /^EVENT_(\d+)D$/i; const SYNTHETIC_EVENT_CONDITION_RE = /^EVENT_(\d+)D$/i;
const normalizeConditions = (conditions: readonly string[] | undefined): string[] => const normalizeConditions = (conditions: readonly string[] | undefined): string[] =>
@ -460,11 +469,7 @@ const storeRecentRootLeg = (leg: LegEvidence, anchorTs: number): void => {
recentLegsByRoot.set(key, next); recentLegsByRoot.set(key, next);
}; };
const collectActiveLegs = ( const collectActiveLegs = (key: string, anchorTs: number, excludeId: string): LegEvidence[] => {
key: string,
anchorTs: number,
excludeId: string
): LegEvidence[] => {
const legs: LegEvidence[] = []; const legs: LegEvidence[] = [];
for (const [contractId, cluster] of clusters) { for (const [contractId, cluster] of clusters) {
if (contractId === excludeId) { if (contractId === excludeId) {
@ -485,11 +490,7 @@ const collectActiveLegs = (
return legs; return legs;
}; };
const collectActiveRootLegs = ( const collectActiveRootLegs = (key: string, anchorTs: number, excludeId: string): LegEvidence[] => {
key: string,
anchorTs: number,
excludeId: string
): LegEvidence[] => {
const legs: LegEvidence[] = []; const legs: LegEvidence[] = [];
for (const [contractId, cluster] of clusters) { for (const [contractId, cluster] of clusters) {
if (contractId === excludeId) { if (contractId === excludeId) {
@ -601,12 +602,19 @@ const applyDeliverPolicy = (
const buildCluster = (print: OptionPrint): ClusterState => { const buildCluster = (print: OptionPrint): ClusterState => {
const placements = createPlacementCounts(); const placements = createPlacementCounts();
const normalizedConditions = normalizeConditions(print.conditions); const normalizedConditions = normalizeConditions(print.conditions);
const executionIv = typeof print.execution_iv === "number" && Number.isFinite(print.execution_iv) ? print.execution_iv : null; const executionIv =
typeof print.execution_iv === "number" && Number.isFinite(print.execution_iv)
? print.execution_iv
: null;
const executionUnderlyingMid = const executionUnderlyingMid =
typeof print.execution_underlying_mid === "number" && Number.isFinite(print.execution_underlying_mid) typeof print.execution_underlying_mid === "number" &&
Number.isFinite(print.execution_underlying_mid)
? print.execution_underlying_mid ? print.execution_underlying_mid
: null; : null;
recordPlacement(placements, classifyPlacement(print.price, selectNbbo(print.option_contract_id, print.ts))); recordPlacement(
placements,
classifyPlacement(print.price, selectNbbo(print.option_contract_id, print.ts))
);
return { return {
contractId: print.option_contract_id, contractId: print.option_contract_id,
underlyingId: print.underlying_id ?? null, underlyingId: print.underlying_id ?? null,
@ -661,11 +669,18 @@ const updateCluster = (cluster: ClusterState, print: OptionPrint): ClusterState
if (typeof print.execution_iv === "number" && Number.isFinite(print.execution_iv)) { if (typeof print.execution_iv === "number" && Number.isFinite(print.execution_iv)) {
cluster.lastExecutionIv = print.execution_iv; cluster.lastExecutionIv = print.execution_iv;
cluster.minExecutionIv = cluster.minExecutionIv =
cluster.minExecutionIv === null ? print.execution_iv : Math.min(cluster.minExecutionIv, print.execution_iv); cluster.minExecutionIv === null
? print.execution_iv
: Math.min(cluster.minExecutionIv, print.execution_iv);
cluster.maxExecutionIv = cluster.maxExecutionIv =
cluster.maxExecutionIv === null ? print.execution_iv : Math.max(cluster.maxExecutionIv, print.execution_iv); cluster.maxExecutionIv === null
? print.execution_iv
: Math.max(cluster.maxExecutionIv, print.execution_iv);
} }
if (typeof print.execution_underlying_mid === "number" && Number.isFinite(print.execution_underlying_mid)) { if (
typeof print.execution_underlying_mid === "number" &&
Number.isFinite(print.execution_underlying_mid)
) {
if (cluster.firstUnderlyingMid === null) { if (cluster.firstUnderlyingMid === null) {
cluster.firstUnderlyingMid = print.execution_underlying_mid; cluster.firstUnderlyingMid = print.execution_underlying_mid;
} }
@ -686,11 +701,7 @@ type NbboJoin = {
const updateNbboCache = (nbbo: OptionNBBO): void => { const updateNbboCache = (nbbo: OptionNBBO): void => {
const existing = nbboCache.get(nbbo.option_contract_id); const existing = nbboCache.get(nbbo.option_contract_id);
if ( if (!existing || nbbo.ts > existing.ts || (nbbo.ts === existing.ts && nbbo.seq >= existing.seq)) {
!existing ||
nbbo.ts > existing.ts ||
(nbbo.ts === existing.ts && nbbo.seq >= existing.seq)
) {
nbboCache.set(nbbo.option_contract_id, nbbo); nbboCache.set(nbbo.option_contract_id, nbbo);
nbboCacheTouchedAt.set(nbbo.option_contract_id, Date.now()); nbboCacheTouchedAt.set(nbbo.option_contract_id, Date.now());
} }
@ -907,14 +918,18 @@ const flushCluster = async (
features.special_print_count = cluster.specialPrintCount; features.special_print_count = cluster.specialPrintCount;
} }
if (cluster.minExecutionIv !== null && cluster.maxExecutionIv !== null) { if (cluster.minExecutionIv !== null && cluster.maxExecutionIv !== null) {
features.execution_iv_shock = roundTo(Math.max(0, cluster.maxExecutionIv - cluster.minExecutionIv)); features.execution_iv_shock = roundTo(
Math.max(0, cluster.maxExecutionIv - cluster.minExecutionIv)
);
} }
if ( if (
cluster.firstUnderlyingMid !== null && cluster.firstUnderlyingMid !== null &&
cluster.lastUnderlyingMid !== null && cluster.lastUnderlyingMid !== null &&
cluster.firstUnderlyingMid > 0 cluster.firstUnderlyingMid > 0
) { ) {
const moveBps = ((cluster.lastUnderlyingMid - cluster.firstUnderlyingMid) / cluster.firstUnderlyingMid) * 10_000; const moveBps =
((cluster.lastUnderlyingMid - cluster.firstUnderlyingMid) / cluster.firstUnderlyingMid) *
10_000;
features.underlying_move_bps = roundTo(moveBps); features.underlying_move_bps = roundTo(moveBps);
} }
const syntheticEventOffsetDays = parseSyntheticEventOffsetDays(cluster.conditions); const syntheticEventOffsetDays = parseSyntheticEventOffsetDays(cluster.conditions);
@ -1004,7 +1019,13 @@ const flushCluster = async (
const rollLegs = [currentLeg, ...rootCandidates]; const rollLegs = [currentLeg, ...rootCandidates];
const rollSummary = summarizeStructure(rollLegs); const rollSummary = summarizeStructure(rollLegs);
if (rollSummary?.type === "roll") { if (rollSummary?.type === "roll") {
await emitStructurePacketIfNeeded(js, batchWriter, rollLegs, rollSummary, currentLeg.contractId); await emitStructurePacketIfNeeded(
js,
batchWriter,
rollLegs,
rollSummary,
currentLeg.contractId
);
} }
storeRecentLeg(currentLeg, anchorTs); storeRecentLeg(currentLeg, anchorTs);
@ -1072,13 +1093,21 @@ const emitClassifiers = async (
const underlyingId = const underlyingId =
typeof packet.features.underlying_id === "string" typeof packet.features.underlying_id === "string"
? packet.features.underlying_id ? packet.features.underlying_id
: parseContractId(typeof packet.features.option_contract_id === "string" ? packet.features.option_contract_id : "")?.root; : parseContractId(
typeof packet.features.option_contract_id === "string"
? packet.features.option_contract_id
: ""
)?.root;
const referenceTs = const referenceTs =
typeof packet.features.end_ts === "number" && Number.isFinite(packet.features.end_ts) typeof packet.features.end_ts === "number" && Number.isFinite(packet.features.end_ts)
? packet.features.end_ts ? packet.features.end_ts
: packet.source_ts; : packet.source_ts;
const eventCalendarMatch = underlyingId ? eventCalendarProvider.findNextEvent(underlyingId, referenceTs) : null; const eventCalendarMatch = underlyingId
smartMoneyEvent = SmartMoneyEventSchema.parse(buildSmartMoneyEventFromPacket(packet, { eventCalendarMatch })); ? eventCalendarProvider.findNextEvent(underlyingId, referenceTs)
: null;
smartMoneyEvent = SmartMoneyEventSchema.parse(
buildSmartMoneyEventFromPacket(packet, { eventCalendarMatch })
);
enqueueSmartMoneyEventInsert(batchWriter, smartMoneyEvent); enqueueSmartMoneyEventInsert(batchWriter, smartMoneyEvent);
await publishJson(js, SUBJECT_SMART_MONEY_EVENTS, smartMoneyEvent); await publishJson(js, SUBJECT_SMART_MONEY_EVENTS, smartMoneyEvent);
emitCounters.smartMoneyEvents += 1; emitCounters.smartMoneyEvents += 1;
@ -1282,20 +1311,29 @@ const run = async () => {
if (env.SMART_MONEY_EVENT_CALENDAR_PATH) { if (env.SMART_MONEY_EVENT_CALENDAR_PATH) {
try { try {
eventCalendarProvider = await loadEventCalendarProviderFromFile(env.SMART_MONEY_EVENT_CALENDAR_PATH); eventCalendarProvider = await loadEventCalendarProviderFromFile(
logger.info("smart money event calendar loaded", { path: env.SMART_MONEY_EVENT_CALENDAR_PATH }); env.SMART_MONEY_EVENT_CALENDAR_PATH
);
logger.info("smart money event calendar loaded", {
path: env.SMART_MONEY_EVENT_CALENDAR_PATH
});
} catch (error) { } catch (error) {
eventCalendarProvider = createEmptyEventCalendarProvider(); eventCalendarProvider = createEmptyEventCalendarProvider();
logger.warn("smart money event calendar unavailable; scoring will use neutral event features", { logger.warn(
"smart money event calendar unavailable; scoring will use neutral event features",
{
path: env.SMART_MONEY_EVENT_CALENDAR_PATH, path: env.SMART_MONEY_EVENT_CALENDAR_PATH,
error: error instanceof Error ? error.message : String(error) error: error instanceof Error ? error.message : String(error)
}); }
);
} }
} }
const redis = createRedisClient(env.REDIS_URL); const redis = createRedisClient(env.REDIS_URL);
redis.on("error", (error) => { redis.on("error", (error) => {
logger.warn("redis client error", { error: error instanceof Error ? error.message : String(error) }); logger.warn("redis client error", {
error: error instanceof Error ? error.message : String(error)
});
}); });
await retry("redis connect", 120, 500, async () => { await retry("redis connect", 120, 500, async () => {
@ -1379,7 +1417,10 @@ const run = async () => {
} else { } else {
try { try {
const info = await jsm.consumers.info(STREAM_OPTION_SIGNAL_PRINTS, durableName); const info = await jsm.consumers.info(STREAM_OPTION_SIGNAL_PRINTS, durableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) { if (
info?.config?.deliver_policy &&
info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY
) {
logger.warn("resetting consumer due to deliver policy change", { logger.warn("resetting consumer due to deliver policy change", {
durable: durableName, durable: durableName,
current: info.config.deliver_policy, current: info.config.deliver_policy,
@ -1390,7 +1431,10 @@ const run = async () => {
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : String(error); const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) { if (!message.includes("not found")) {
logger.warn("failed to inspect jetstream consumer", { durable: durableName, error: message }); logger.warn("failed to inspect jetstream consumer", {
durable: durableName,
error: message
});
} }
} }
} }
@ -1402,13 +1446,19 @@ const run = async () => {
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : String(error); const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) { if (!message.includes("not found")) {
logger.warn("failed to reset jetstream consumer", { durable: nbboDurableName, error: message }); logger.warn("failed to reset jetstream consumer", {
durable: nbboDurableName,
error: message
});
} }
} }
} else { } else {
try { try {
const info = await jsm.consumers.info(STREAM_OPTION_NBBO, nbboDurableName); const info = await jsm.consumers.info(STREAM_OPTION_NBBO, nbboDurableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) { if (
info?.config?.deliver_policy &&
info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY
) {
logger.warn("resetting consumer due to deliver policy change", { logger.warn("resetting consumer due to deliver policy change", {
durable: nbboDurableName, durable: nbboDurableName,
current: info.config.deliver_policy, current: info.config.deliver_policy,
@ -1419,7 +1469,10 @@ const run = async () => {
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : String(error); const message = error instanceof Error ? error.message : String(error);
if (!message.includes("not found")) { if (!message.includes("not found")) {
logger.warn("failed to inspect jetstream consumer", { durable: nbboDurableName, error: message }); logger.warn("failed to inspect jetstream consumer", {
durable: nbboDurableName,
error: message
});
} }
} }
} }
@ -1440,7 +1493,10 @@ const run = async () => {
} else { } else {
try { try {
const info = await jsm.consumers.info(STREAM_EQUITY_PRINTS, equityPrintDurableName); const info = await jsm.consumers.info(STREAM_EQUITY_PRINTS, equityPrintDurableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) { if (
info?.config?.deliver_policy &&
info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY
) {
logger.warn("resetting consumer due to deliver policy change", { logger.warn("resetting consumer due to deliver policy change", {
durable: equityPrintDurableName, durable: equityPrintDurableName,
current: info.config.deliver_policy, current: info.config.deliver_policy,
@ -1475,7 +1531,10 @@ const run = async () => {
} else { } else {
try { try {
const info = await jsm.consumers.info(STREAM_EQUITY_QUOTES, equityQuoteDurableName); const info = await jsm.consumers.info(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
if (info?.config?.deliver_policy && info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY) { if (
info?.config?.deliver_policy &&
info.config.deliver_policy !== env.COMPUTE_DELIVER_POLICY
) {
logger.warn("resetting consumer due to deliver policy change", { logger.warn("resetting consumer due to deliver policy change", {
durable: equityQuoteDurableName, durable: equityQuoteDurableName,
current: info.config.deliver_policy, current: info.config.deliver_policy,
@ -1515,7 +1574,8 @@ const run = async () => {
try { try {
await jsm.consumers.delete(STREAM_OPTION_SIGNAL_PRINTS, durableName); await jsm.consumers.delete(STREAM_OPTION_SIGNAL_PRINTS, durableName);
} catch (deleteError) { } catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError); const deleteMessage =
deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) { if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", { logger.warn("failed to delete jetstream consumer", {
durable: durableName, durable: durableName,
@ -1551,7 +1611,8 @@ const run = async () => {
try { try {
await jsm.consumers.delete(STREAM_OPTION_NBBO, nbboDurableName); await jsm.consumers.delete(STREAM_OPTION_NBBO, nbboDurableName);
} catch (deleteError) { } catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError); const deleteMessage =
deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) { if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", { logger.warn("failed to delete jetstream consumer", {
durable: nbboDurableName, durable: nbboDurableName,
@ -1582,12 +1643,16 @@ const run = async () => {
throw error; throw error;
} }
logger.warn("resetting jetstream consumer", { durable: equityPrintDurableName, error: message }); logger.warn("resetting jetstream consumer", {
durable: equityPrintDurableName,
error: message
});
try { try {
await jsm.consumers.delete(STREAM_EQUITY_PRINTS, equityPrintDurableName); await jsm.consumers.delete(STREAM_EQUITY_PRINTS, equityPrintDurableName);
} catch (deleteError) { } catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError); const deleteMessage =
deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) { if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", { logger.warn("failed to delete jetstream consumer", {
durable: equityPrintDurableName, durable: equityPrintDurableName,
@ -1626,7 +1691,8 @@ const run = async () => {
try { try {
await jsm.consumers.delete(STREAM_EQUITY_QUOTES, equityQuoteDurableName); await jsm.consumers.delete(STREAM_EQUITY_QUOTES, equityQuoteDurableName);
} catch (deleteError) { } catch (deleteError) {
const deleteMessage = deleteError instanceof Error ? deleteError.message : String(deleteError); const deleteMessage =
deleteError instanceof Error ? deleteError.message : String(deleteError);
if (!deleteMessage.includes("not found")) { if (!deleteMessage.includes("not found")) {
logger.warn("failed to delete jetstream consumer", { logger.warn("failed to delete jetstream consumer", {
durable: equityQuoteDurableName, durable: equityQuoteDurableName,

View file

@ -78,7 +78,9 @@ const getDteDays = (packet: FlowPacket): number | null => {
const inferDirection = (packet: FlowPacket): SmartMoneyDirection => { const inferDirection = (packet: FlowPacket): SmartMoneyDirection => {
const structureRights = stringFeature(packet, "structure_rights"); const structureRights = stringFeature(packet, "structure_rights");
const optionType = stringFeature(packet, "option_type") || parseContractId(stringFeature(packet, "option_contract_id"))?.right; const optionType =
stringFeature(packet, "option_type") ||
parseContractId(stringFeature(packet, "option_contract_id"))?.right;
const buy = numberFeature(packet, "nbbo_aggressive_buy_ratio"); const buy = numberFeature(packet, "nbbo_aggressive_buy_ratio");
const sell = numberFeature(packet, "nbbo_aggressive_sell_ratio"); const sell = numberFeature(packet, "nbbo_aggressive_sell_ratio");
const sellDominant = sell >= buy + 0.12; const sellDominant = sell >= buy + 0.12;
@ -102,16 +104,26 @@ export type SmartMoneyParentEventOptions = {
eventCalendarMatch?: EventCalendarMatch | null; eventCalendarMatch?: EventCalendarMatch | null;
}; };
const buildFeatures = (packet: FlowPacket, options: SmartMoneyParentEventOptions = {}): SmartMoneyFeatures => { const buildFeatures = (
packet: FlowPacket,
options: SmartMoneyParentEventOptions = {}
): SmartMoneyFeatures => {
const contractId = stringFeature(packet, "option_contract_id"); const contractId = stringFeature(packet, "option_contract_id");
const contract = parseContractId(contractId); const contract = parseContractId(contractId);
const underlyingMid = numberFeature(packet, "underlying_mid"); const underlyingMid = numberFeature(packet, "underlying_mid");
const quoteAge = numberFeature(packet, "nbbo_age_ms") || numberFeature(packet, "underlying_quote_age_ms"); const quoteAge =
const printCount = Math.max(0, Math.round(numberFeature(packet, "count") || packet.members.length)); numberFeature(packet, "nbbo_age_ms") || numberFeature(packet, "underlying_quote_age_ms");
const printCount = Math.max(
0,
Math.round(numberFeature(packet, "count") || packet.members.length)
);
const staleCount = numberFeature(packet, "nbbo_stale_count"); const staleCount = numberFeature(packet, "nbbo_stale_count");
const missingCount = numberFeature(packet, "nbbo_missing_count"); const missingCount = numberFeature(packet, "nbbo_missing_count");
const structureLegs = Math.max(0, Math.round(numberFeature(packet, "structure_legs"))); const structureLegs = Math.max(0, Math.round(numberFeature(packet, "structure_legs")));
const strikeCount = Math.max(1, Math.round(numberFeature(packet, "structure_strikes") || (contract ? 1 : 0))); const strikeCount = Math.max(
1,
Math.round(numberFeature(packet, "structure_strikes") || (contract ? 1 : 0))
);
const specialCount = numberFeature(packet, "special_print_count"); const specialCount = numberFeature(packet, "special_print_count");
const calendarEventTs = options.eventCalendarMatch?.event_ts ?? null; const calendarEventTs = options.eventCalendarMatch?.event_ts ?? null;
const eventTs = calendarEventTs ?? numberFeature(packet, "corporate_event_ts"); const eventTs = calendarEventTs ?? numberFeature(packet, "corporate_event_ts");
@ -119,7 +131,9 @@ const buildFeatures = (packet: FlowPacket, options: SmartMoneyParentEventOptions
const expiryTs = contract ? Date.parse(`${contract.expiry}T00:00:00Z`) : Number.NaN; const expiryTs = contract ? Date.parse(`${contract.expiry}T00:00:00Z`) : Number.NaN;
const atmProximity = const atmProximity =
contract && underlyingMid > 0 ? Math.abs(contract.strike - underlyingMid) / underlyingMid : null; contract && underlyingMid > 0
? Math.abs(contract.strike - underlyingMid) / underlyingMid
: null;
return { return {
contract_count: Math.max(1, structureLegs || 1), contract_count: Math.max(1, structureLegs || 1),
@ -143,14 +157,18 @@ const buildFeatures = (packet: FlowPacket, options: SmartMoneyParentEventOptions
nbbo_stale_ratio: printCount > 0 ? clamp((staleCount + missingCount) / printCount) : 0, nbbo_stale_ratio: printCount > 0 ? clamp((staleCount + missingCount) / printCount) : 0,
quote_age_ms: quoteAge > 0 ? quoteAge : null, quote_age_ms: quoteAge > 0 ? quoteAge : null,
venue_count: Math.max(1, Math.round(numberFeature(packet, "venue_count") || 1)), venue_count: Math.max(1, Math.round(numberFeature(packet, "venue_count") || 1)),
inter_fill_ms_mean: printCount > 1 ? numberFeature(packet, "window_ms") / Math.max(1, printCount - 1) : null, inter_fill_ms_mean:
printCount > 1 ? numberFeature(packet, "window_ms") / Math.max(1, printCount - 1) : null,
strike_count: strikeCount, strike_count: strikeCount,
strike_concentration: strikeCount > 0 ? clamp(1 / strikeCount) : 0, strike_concentration: strikeCount > 0 ? clamp(1 / strikeCount) : 0,
...(stringFeature(packet, "structure_type") ? { structure_type: stringFeature(packet, "structure_type") } : {}), ...(stringFeature(packet, "structure_type")
? { structure_type: stringFeature(packet, "structure_type") }
: {}),
structure_legs: structureLegs, structure_legs: structureLegs,
same_size_leg_symmetry: clamp(numberFeature(packet, "same_size_leg_symmetry")), same_size_leg_symmetry: clamp(numberFeature(packet, "same_size_leg_symmetry")),
net_directional_bias: clamp( net_directional_bias: clamp(
numberFeature(packet, "nbbo_aggressive_buy_ratio") - numberFeature(packet, "nbbo_aggressive_sell_ratio"), numberFeature(packet, "nbbo_aggressive_buy_ratio") -
numberFeature(packet, "nbbo_aggressive_sell_ratio"),
-1, -1,
1 1
), ),
@ -159,7 +177,10 @@ const buildFeatures = (packet: FlowPacket, options: SmartMoneyParentEventOptions
underlying_move_bps: numberFeature(packet, "underlying_move_bps") || null, underlying_move_bps: numberFeature(packet, "underlying_move_bps") || null,
days_to_event: eventTs > 0 ? (eventTs - referenceTs) / MS_PER_DAY : null, days_to_event: eventTs > 0 ? (eventTs - referenceTs) / MS_PER_DAY : null,
expiry_after_event: eventTs > 0 && Number.isFinite(expiryTs) ? expiryTs >= eventTs : null, expiry_after_event: eventTs > 0 && Number.isFinite(expiryTs) ? expiryTs >= eventTs : null,
pre_event_concentration: eventTs > 0 && eventTs >= referenceTs ? clamp(1 - (eventTs - referenceTs) / (21 * MS_PER_DAY)) : null, pre_event_concentration:
eventTs > 0 && eventTs >= referenceTs
? clamp(1 - (eventTs - referenceTs) / (21 * MS_PER_DAY))
: null,
special_print_ratio: printCount > 0 ? clamp(specialCount / printCount) : 0 special_print_ratio: printCount > 0 ? clamp(specialCount / printCount) : 0
}; };
}; };
@ -170,7 +191,10 @@ const detectSuppression = (packet: FlowPacket, features: SmartMoneyFeatures): st
.split(",") .split(",")
.map((item) => item.trim().toUpperCase()) .map((item) => item.trim().toUpperCase())
.filter(Boolean); .filter(Boolean);
if (conditions.some((condition) => SPECIAL_CONDITIONS.has(condition)) || features.special_print_ratio >= 0.34) { if (
conditions.some((condition) => SPECIAL_CONDITIONS.has(condition)) ||
features.special_print_ratio >= 0.34
) {
reasons.push("special_print_or_complex_context"); reasons.push("special_print_or_complex_context");
} }
if (features.nbbo_coverage_ratio < 0.35 || features.nbbo_stale_ratio >= 0.5) { if (features.nbbo_coverage_ratio < 0.35 || features.nbbo_stale_ratio >= 0.5) {
@ -198,7 +222,10 @@ const evaluateProfiles = (
const burstFactor = clamp(features.print_count / 8); const burstFactor = clamp(features.print_count / 8);
const quality = clamp(features.nbbo_coverage_ratio - features.nbbo_stale_ratio); const quality = clamp(features.nbbo_coverage_ratio - features.nbbo_stale_ratio);
const shortDatedOtm = const shortDatedOtm =
dte <= 7 && features.atm_proximity !== null && features.atm_proximity >= 0.05 && features.option_type === "C"; dte <= 7 &&
features.atm_proximity !== null &&
features.atm_proximity >= 0.05 &&
features.option_type === "C";
const nearAtm = features.atm_proximity !== null && features.atm_proximity <= 0.015; const nearAtm = features.atm_proximity !== null && features.atm_proximity <= 0.015;
const preEvent = const preEvent =
features.days_to_event !== null && features.days_to_event !== null &&
@ -211,7 +238,11 @@ const evaluateProfiles = (
"institutional_directional", "institutional_directional",
suppressed.length > 0 || shortDatedOtm suppressed.length > 0 || shortDatedOtm
? 0.18 ? 0.18
: 0.2 + premiumFactor * 0.25 + burstFactor * 0.18 + quality * 0.16 + (buy >= 0.58 || sell >= 0.58 ? 0.12 : 0), : 0.2 +
premiumFactor * 0.25 +
burstFactor * 0.18 +
quality * 0.16 +
(buy >= 0.58 || sell >= 0.58 ? 0.12 : 0),
direction, direction,
[ [
"large_parent_event", "large_parent_event",
@ -232,13 +263,19 @@ const evaluateProfiles = (
), ),
score( score(
"event_driven", "event_driven",
0.12 + (preEvent ? 0.32 : 0) + premiumFactor * 0.14 + clamp(features.spread_widening ?? 0, 0, 0.16), 0.12 +
(preEvent ? 0.32 : 0) +
premiumFactor * 0.14 +
clamp(features.spread_widening ?? 0, 0, 0.16),
direction === "unknown" ? "neutral" : direction, direction === "unknown" ? "neutral" : direction,
["event_calendar_alignment", "expiry_after_event", "pre_event_concentration"] ["event_calendar_alignment", "expiry_after_event", "pre_event_concentration"]
), ),
score( score(
"vol_seller", "vol_seller",
0.12 + (sell >= 0.58 ? 0.24 : 0) + (structure === "straddle" || structure === "strangle" ? 0.2 : 0) + premiumFactor * 0.14, 0.12 +
(sell >= 0.58 ? 0.24 : 0) +
(structure === "straddle" || structure === "strangle" ? 0.2 : 0) +
premiumFactor * 0.14,
"neutral", "neutral",
["sell_side_premium", "short_vol_structure_evidence"] ["sell_side_premium", "short_vol_structure_evidence"]
), ),
@ -273,9 +310,14 @@ export const buildSmartMoneyEventFromPacket = (
const suppressed = detectSuppression(packet, features); const suppressed = detectSuppression(packet, features);
const profileScores = evaluateProfiles(packet, features, suppressed); const profileScores = evaluateProfiles(packet, features, suppressed);
const primary = profileScores[0] ?? null; const primary = profileScores[0] ?? null;
const abstained = !primary || primary.probability < 0.42 || suppressed.includes("stale_or_missing_quote_context"); const abstained =
const underlying = stringFeature(packet, "underlying_id") || parseContractId(features.option_contract_id ?? "")?.root || "UNKNOWN"; !primary || primary.probability < 0.42 || suppressed.includes("stale_or_missing_quote_context");
const eventKind = features.structure_legs >= 2 || stringFeature(packet, "packet_kind") === "structure" const underlying =
stringFeature(packet, "underlying_id") ||
parseContractId(features.option_contract_id ?? "")?.root ||
"UNKNOWN";
const eventKind =
features.structure_legs >= 2 || stringFeature(packet, "packet_kind") === "structure"
? "multi_leg_event" ? "multi_leg_event"
: "single_leg_event"; : "single_leg_event";
@ -292,8 +334,8 @@ export const buildSmartMoneyEventFromPacket = (
event_window_ms: features.window_ms, event_window_ms: features.window_ms,
features, features,
profile_scores: profileScores, profile_scores: profileScores,
primary_profile_id: abstained ? null : primary?.profile_id ?? null, primary_profile_id: abstained ? null : (primary?.profile_id ?? null),
primary_direction: abstained ? "unknown" : primary?.direction ?? "unknown", primary_direction: abstained ? "unknown" : (primary?.direction ?? "unknown"),
abstained, abstained,
suppressed_reasons: suppressed suppressed_reasons: suppressed
}); });
@ -308,7 +350,9 @@ const LEGACY_PROFILE_MAP: Record<SmartMoneyProfileId, string> = {
hedge_reactive: "smart_money_hedge_reactive" hedge_reactive: "smart_money_hedge_reactive"
}; };
export const deriveClassifierHitsFromSmartMoneyEvent = (event: SmartMoneyEvent): ClassifierHit[] => { export const deriveClassifierHitsFromSmartMoneyEvent = (
event: SmartMoneyEvent
): ClassifierHit[] => {
if (event.abstained || !event.primary_profile_id) { if (event.abstained || !event.primary_profile_id) {
return []; return [];
} }

View file

@ -24,9 +24,7 @@ type RollingWindowEntry = {
}; };
const toNumbers = (values: string[]): number[] => { const toNumbers = (values: string[]): number[] => {
return values return values.map((value) => Number(value)).filter((value) => Number.isFinite(value));
.map((value) => Number(value))
.filter((value) => Number.isFinite(value));
}; };
export const computeStats = (values: number[]): { mean: number; stddev: number; count: number } => { export const computeStats = (values: number[]): { mean: number; stddev: number; count: number } => {

View file

@ -1,4 +1,9 @@
import type { FlowPacket, SmartMoneyDirection, SmartMoneyEvent, SmartMoneyProfileId } from "@islandflow/types"; import type {
FlowPacket,
SmartMoneyDirection,
SmartMoneyEvent,
SmartMoneyProfileId
} from "@islandflow/types";
import { buildSmartMoneyEventFromPacket, type SmartMoneyParentEventOptions } from "./parent-events"; import { buildSmartMoneyEventFromPacket, type SmartMoneyParentEventOptions } from "./parent-events";
export type SmartMoneyLabel = { export type SmartMoneyLabel = {
@ -115,8 +120,12 @@ export const compareSmartMoneyReplayOutputs = (
liveEvents: SmartMoneyEvent[], liveEvents: SmartMoneyEvent[],
batchEvents: SmartMoneyEvent[] batchEvents: SmartMoneyEvent[]
): ReplayConsistencyReport => { ): ReplayConsistencyReport => {
const liveById = new Map(liveEvents.map((event) => [event.event_id, smartMoneyEventSignature(event)])); const liveById = new Map(
const batchById = new Map(batchEvents.map((event) => [event.event_id, smartMoneyEventSignature(event)])); liveEvents.map((event) => [event.event_id, smartMoneyEventSignature(event)])
);
const batchById = new Map(
batchEvents.map((event) => [event.event_id, smartMoneyEventSignature(event)])
);
const ids = [...new Set([...liveById.keys(), ...batchById.keys()])].sort(); const ids = [...new Set([...liveById.keys(), ...batchById.keys()])].sort();
const mismatches: ReplayConsistencyMismatch[] = []; const mismatches: ReplayConsistencyMismatch[] = [];
@ -153,7 +162,9 @@ export const evaluateSmartMoneyEvents = (
const labelsById = new Map(labels.map((label) => [label.event_id, label])); const labelsById = new Map(labels.map((label) => [label.event_id, label]));
const labeledEvents = events const labeledEvents = events
.map((event) => ({ event, label: labelsById.get(event.event_id) })) .map((event) => ({ event, label: labelsById.get(event.event_id) }))
.filter((entry): entry is { event: SmartMoneyEvent; label: SmartMoneyLabel } => Boolean(entry.label)); .filter((entry): entry is { event: SmartMoneyEvent; label: SmartMoneyLabel } =>
Boolean(entry.label)
);
const emitted = events.filter((event) => !event.abstained && event.primary_profile_id); const emitted = events.filter((event) => !event.abstained && event.primary_profile_id);
const profilePrecision: SmartMoneyEvaluationReport["profile_precision"] = {}; const profilePrecision: SmartMoneyEvaluationReport["profile_precision"] = {};
@ -163,7 +174,8 @@ export const evaluateSmartMoneyEvents = (
const predicted = labeledEvents.filter((entry) => entry.event.primary_profile_id === profile); const predicted = labeledEvents.filter((entry) => entry.event.primary_profile_id === profile);
const actual = labeledEvents.filter((entry) => entry.label.profile_id === profile); const actual = labeledEvents.filter((entry) => entry.label.profile_id === profile);
const truePositive = predicted.filter((entry) => entry.label.profile_id === profile).length; const truePositive = predicted.filter((entry) => entry.label.profile_id === profile).length;
profilePrecision[profile] = predicted.length > 0 ? round(truePositive / predicted.length) : null; profilePrecision[profile] =
predicted.length > 0 ? round(truePositive / predicted.length) : null;
profileRecall[profile] = actual.length > 0 ? round(truePositive / actual.length) : null; profileRecall[profile] = actual.length > 0 ? round(truePositive / actual.length) : null;
} }
@ -175,7 +187,10 @@ export const evaluateSmartMoneyEvents = (
labeled_count: labeledEvents.length, labeled_count: labeledEvents.length,
emitted_count: emitted.length, emitted_count: emitted.length,
abstained_count: events.filter((event) => event.abstained).length, abstained_count: events.filter((event) => event.abstained).length,
abstention_rate: events.length > 0 ? round(events.filter((event) => event.abstained).length / events.length) : 0, abstention_rate:
events.length > 0
? round(events.filter((event) => event.abstained).length / events.length)
: 0,
profile_precision: profilePrecision, profile_precision: profilePrecision,
profile_recall: profileRecall, profile_recall: profileRecall,
calibration, calibration,
@ -195,7 +210,9 @@ const buildCalibration = (
})); }));
for (const { event, label } of entries) { for (const { event, label } of entries) {
const probability = event.profile_scores.find((entry) => entry.profile_id === event.primary_profile_id)?.probability ?? 0; const probability =
event.profile_scores.find((entry) => entry.profile_id === event.primary_profile_id)
?.probability ?? 0;
const index = Math.min(bucketCount - 1, Math.floor(probability * bucketCount)); const index = Math.min(bucketCount - 1, Math.floor(probability * bucketCount));
buckets[index].probabilities.push(probability); buckets[index].probabilities.push(probability);
if (!event.abstained && event.primary_profile_id === label.profile_id) { if (!event.abstained && event.primary_profile_id === label.profile_id) {
@ -209,9 +226,13 @@ const buildCalibration = (
count: bucket.probabilities.length, count: bucket.probabilities.length,
average_probability: average_probability:
bucket.probabilities.length > 0 bucket.probabilities.length > 0
? round(bucket.probabilities.reduce((sum, value) => sum + value, 0) / bucket.probabilities.length) ? round(
bucket.probabilities.reduce((sum, value) => sum + value, 0) /
bucket.probabilities.length
)
: 0, : 0,
accuracy: bucket.probabilities.length > 0 ? round(bucket.correct / bucket.probabilities.length) : null accuracy:
bucket.probabilities.length > 0 ? round(bucket.correct / bucket.probabilities.length) : null
})); }));
}; };
@ -223,7 +244,10 @@ const buildEconomicSanity = (
sign: directionalSign(event.primary_direction), sign: directionalSign(event.primary_direction),
realized: label.realized_return_bps realized: label.realized_return_bps
})) }))
.filter((entry): entry is { sign: number; realized: number } => entry.sign !== 0 && Number.isFinite(entry.realized)); .filter(
(entry): entry is { sign: number; realized: number } =>
entry.sign !== 0 && Number.isFinite(entry.realized)
);
if (directional.length === 0) { if (directional.length === 0) {
return { return {
@ -236,7 +260,12 @@ const buildEconomicSanity = (
const signedReturns = directional.map((entry) => entry.sign * entry.realized); const signedReturns = directional.map((entry) => entry.sign * entry.realized);
return { return {
directional_count: directional.length, directional_count: directional.length,
direction_hit_rate: round(signedReturns.filter((value) => value > 0).length / directional.length), direction_hit_rate: round(
average_signed_return_bps: round(signedReturns.reduce((sum, value) => sum + value, 0) / signedReturns.length, 2) signedReturns.filter((value) => value > 0).length / directional.length
),
average_signed_return_bps: round(
signedReturns.reduce((sum, value) => sum + value, 0) / signedReturns.length,
2
)
}; };
}; };

View file

@ -134,7 +134,9 @@ const dayDiff = (from: string | null, to: string | null): number | null => {
}; };
const sameSizeLegSymmetry = (legs: LegEvidence[]): number => { const sameSizeLegSymmetry = (legs: LegEvidence[]): number => {
const sizes = legs.map((leg) => leg.totalSize).filter((value) => Number.isFinite(value) && value > 0); const sizes = legs
.map((leg) => leg.totalSize)
.filter((value) => Number.isFinite(value) && value > 0);
if (sizes.length < 2) { if (sizes.length < 2) {
return 0; return 0;
} }
@ -146,7 +148,10 @@ const sameSizeLegSymmetry = (legs: LegEvidence[]): number => {
return min / max; return min / max;
}; };
export const shouldEmitStructurePacket = (legs: LegEvidence[], currentLegContractId: string): boolean => { export const shouldEmitStructurePacket = (
legs: LegEvidence[],
currentLegContractId: string
): boolean => {
if (legs.length < 2) { if (legs.length < 2) {
return false; return false;
} }
@ -226,7 +231,8 @@ export const planStructurePacket = (
const totalSize = legs.reduce((sum, leg) => sum + leg.totalSize, 0); const totalSize = legs.reduce((sum, leg) => sum + leg.totalSize, 0);
const count = legs.reduce((sum, leg) => sum + leg.members.length, 0); const count = legs.reduce((sum, leg) => sum + leg.members.length, 0);
const placements = mergePlacements(legs); const placements = mergePlacements(legs);
const placementTotal = placements.aa + placements.a + placements.b + placements.bb + placements.mid; const placementTotal =
placements.aa + placements.a + placements.b + placements.bb + placements.mid;
const aggressiveTotal = placements.aa + placements.a + placements.b + placements.bb; const aggressiveTotal = placements.aa + placements.a + placements.b + placements.bb;
const aggressiveBuy = placements.aa + placements.a; const aggressiveBuy = placements.aa + placements.a;
const aggressiveSell = placements.bb + placements.b; const aggressiveSell = placements.bb + placements.b;
@ -235,7 +241,10 @@ export const planStructurePacket = (
const nbboAggressiveSellRatio = aggressiveTotal > 0 ? aggressiveSell / aggressiveTotal : 0; const nbboAggressiveSellRatio = aggressiveTotal > 0 ? aggressiveSell / aggressiveTotal : 0;
const nbboAggressiveRatio = placementTotal > 0 ? aggressiveTotal / placementTotal : 0; const nbboAggressiveRatio = placementTotal > 0 ? aggressiveTotal / placementTotal : 0;
const source_ts = legs.reduce((min, leg) => Math.min(min, leg.source_ts), Number.POSITIVE_INFINITY); const source_ts = legs.reduce(
(min, leg) => Math.min(min, leg.source_ts),
Number.POSITIVE_INFINITY
);
const ingest_ts = legs.reduce((max, leg) => Math.max(max, leg.ingest_ts), 0); const ingest_ts = legs.reduce((max, leg) => Math.max(max, leg.ingest_ts), 0);
const seq = legs.reduce((max, leg) => Math.max(max, leg.seq), 0); const seq = legs.reduce((max, leg) => Math.max(max, leg.seq), 0);

View file

@ -47,7 +47,10 @@ export const summarizeStructure = (legs: ContractLeg[]): StructureSummary | null
legs: legs.length, legs: legs.length,
strikes: strikes.length, strikes: strikes.length,
strikeSpan, strikeSpan,
rights: rights.size === 2 ? "C/P" : Array.from(rights)[0] ?? "", rights: rights.size === 2 ? "C/P" : (Array.from(rights)[0] ?? ""),
contractIds: legs.map((leg) => leg.contractId).slice().sort() contractIds: legs
.map((leg) => leg.contractId)
.slice()
.sort()
}; };
}; };

View file

@ -293,4 +293,3 @@ describe("compute classifiers", () => {
expect(hit!.explanations[0]).toMatch(/Consistent with/i); expect(hit!.explanations[0]).toMatch(/Consistent with/i);
}); });
}); });

View file

@ -17,7 +17,8 @@ export const TEST_CLASSIFIER_CONFIG: ClassifierConfig = {
zeroDteMinSize: 400 zeroDteMinSize: 400
}; };
export const buildFlowPacket = (opts: { export const buildFlowPacket = (
opts: {
id?: string; id?: string;
source_ts?: number; source_ts?: number;
ingest_ts?: number; ingest_ts?: number;
@ -26,7 +27,8 @@ export const buildFlowPacket = (opts: {
members?: string[]; members?: string[];
features?: FlowPacket["features"]; features?: FlowPacket["features"];
join_quality?: FlowPacket["join_quality"]; join_quality?: FlowPacket["join_quality"];
} = {}): FlowPacket => { } = {}
): FlowPacket => {
const id = opts.id ?? "flowpacket:test"; const id = opts.id ?? "flowpacket:test";
const source_ts = opts.source_ts ?? Date.parse("2025-01-01T14:30:00Z"); const source_ts = opts.source_ts ?? Date.parse("2025-01-01T14:30:00Z");
const ingest_ts = opts.ingest_ts ?? source_ts; const ingest_ts = opts.ingest_ts ?? source_ts;
@ -66,4 +68,3 @@ export const buildFlowPacket = (opts: {
export const getHit = (hits: ClassifierHit[], id: string): ClassifierHit | null => { export const getHit = (hits: ClassifierHit[], id: string): ClassifierHit | null => {
return hits.find((hit) => hit.classifier_id === id) ?? null; return hits.find((hit) => hit.classifier_id === id) ?? null;
}; };

View file

@ -18,7 +18,9 @@ const placements = (overrides?: Partial<LegEvidence["placements"]>): LegEvidence
...overrides ...overrides
}); });
const leg = (input: Partial<LegEvidence> & Pick<LegEvidence, "contractId" | "right" | "strike">): LegEvidence => { const leg = (
input: Partial<LegEvidence> & Pick<LegEvidence, "contractId" | "right" | "strike">
): LegEvidence => {
return { return {
contractId: input.contractId, contractId: input.contractId,
root: "SPY", root: "SPY",

View file

@ -85,10 +85,14 @@ const decodePayload = (data: WebSocket.RawData): unknown => {
} }
if (ArrayBuffer.isView(data)) { if (ArrayBuffer.isView(data)) {
return JSON.parse(new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))) as unknown; return JSON.parse(
new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))
) as unknown;
} }
return JSON.parse(new TextDecoder().decode(new Uint8Array(data as unknown as ArrayBuffer))) as unknown; return JSON.parse(
new TextDecoder().decode(new Uint8Array(data as unknown as ArrayBuffer))
) as unknown;
}; };
const extractExchangeMeta = (payload: unknown): AlpacaExchangeMetaEntry[] => { const extractExchangeMeta = (payload: unknown): AlpacaExchangeMetaEntry[] => {
@ -103,8 +107,18 @@ const extractExchangeMeta = (payload: unknown): AlpacaExchangeMetaEntry[] => {
continue; continue;
} }
const candidate = entry as Record<string, unknown>; const candidate = entry as Record<string, unknown>;
const code = typeof candidate.code === "string" ? candidate.code : typeof candidate.exchange === "string" ? candidate.exchange : null; const code =
const name = typeof candidate.name === "string" ? candidate.name : typeof candidate.description === "string" ? candidate.description : null; typeof candidate.code === "string"
? candidate.code
: typeof candidate.exchange === "string"
? candidate.exchange
: null;
const name =
typeof candidate.name === "string"
? candidate.name
: typeof candidate.description === "string"
? candidate.description
: null;
if (!code || !name) { if (!code || !name) {
continue; continue;
} }
@ -128,9 +142,19 @@ const buildExchangeNameMap = (entries: AlpacaExchangeMetaEntry[]): Map<string, s
return map; return map;
}; };
const OFF_EXCHANGE_HINTS = ["FINRA", "TRF", "ADF", "OTC", "Trade Reporting Facility", "Alternative Display Facility"]; const OFF_EXCHANGE_HINTS = [
"FINRA",
"TRF",
"ADF",
"OTC",
"Trade Reporting Facility",
"Alternative Display Facility"
];
export const inferOffExchangeFlag = (exchangeCode: string | undefined, exchangeNameMap: Map<string, string>): boolean => { export const inferOffExchangeFlag = (
exchangeCode: string | undefined,
exchangeNameMap: Map<string, string>
): boolean => {
if (!exchangeCode) { if (!exchangeCode) {
return false; return false;
} }
@ -151,7 +175,9 @@ const buildWsUrl = (wsBaseUrl: string, feed: AlpacaEquitiesFeed): string => {
return `${parsed.origin}/v2/${feed}`; return `${parsed.origin}/v2/${feed}`;
}; };
const fetchExchangeMeta = async (config: AlpacaEquitiesAdapterConfig): Promise<Map<string, string>> => { const fetchExchangeMeta = async (
config: AlpacaEquitiesAdapterConfig
): Promise<Map<string, string>> => {
const url = new URL("/v2/stocks/meta/exchanges", config.restUrl); const url = new URL("/v2/stocks/meta/exchanges", config.restUrl);
try { try {
@ -243,7 +269,10 @@ export const createAlpacaEquitiesAdapter = (
continue; continue;
} }
const message = entry as (AlpacaTradeMessage | AlpacaQuoteMessage | { T?: string; msg?: string }); const message = entry as
| AlpacaTradeMessage
| AlpacaQuoteMessage
| { T?: string; msg?: string };
const type = message.T; const type = message.T;
if (type === "success") { if (type === "success") {

View file

@ -89,11 +89,7 @@ const priceForPlacement = (
return formatPrice(Math.max(0.01, price)); return formatPrice(Math.max(0.01, price));
}; };
const buildQuoteContext = ( const buildQuoteContext = (symbol: string, now: number, control: SyntheticControlState) => {
symbol: string,
now: number,
control: SyntheticControlState
) => {
const session = getSyntheticSessionState(now, control); const session = getSyntheticSessionState(now, control);
const state = getSyntheticUnderlyingState(symbol, now, control, session); const state = getSyntheticUnderlyingState(symbol, now, control, session);
return { return {
@ -184,7 +180,9 @@ export const createSyntheticEquitiesAdapter = (
session.regime === "retail_chase"; session.regime === "retail_chase";
if (allowDark) { if (allowDark) {
const darkSymbol = focusSymbols[seq % focusSymbols.length] ?? SYNTHETIC_SYMBOLS[symbolCursor % SYNTHETIC_SYMBOLS.length]!; const darkSymbol =
focusSymbols[seq % focusSymbols.length] ??
SYNTHETIC_SYMBOLS[symbolCursor % SYNTHETIC_SYMBOLS.length]!;
const darkQuote = buildQuoteContext(darkSymbol, now, control); const darkQuote = buildQuoteContext(darkSymbol, now, control);
const darkPlacement = pickDarkPlacement( const darkPlacement = pickDarkPlacement(
darkQuote.state.driftBps, darkQuote.state.driftBps,
@ -203,13 +201,7 @@ export const createSyntheticEquitiesAdapter = (
if (handlers.onQuote) { if (handlers.onQuote) {
quoteSeq += 1; quoteSeq += 1;
void handlers.onQuote( void handlers.onQuote(
buildSyntheticQuote( buildSyntheticQuote(quoteSeq, now - 2, darkSymbol, darkQuote.bid, darkQuote.ask)
quoteSeq,
now - 2,
darkSymbol,
darkQuote.bid,
darkQuote.ask
)
); );
} }
@ -236,11 +228,7 @@ export const createSyntheticEquitiesAdapter = (
const eventTs = now + i * 4; const eventTs = now + i * 4;
const quote = buildQuoteContext(symbol, eventTs, control); const quote = buildQuoteContext(symbol, eventTs, control);
const clustered = focusSet.has(symbol); const clustered = focusSet.has(symbol);
const placement = pickPrimaryPlacement( const placement = pickPrimaryPlacement(quote.state.driftBps, session.regime, seq + i);
quote.state.driftBps,
session.regime,
seq + i
);
const exchange = EXCHANGES[(seq + symbol.charCodeAt(0) + i) % EXCHANGES.length]!; const exchange = EXCHANGES[(seq + symbol.charCodeAt(0) + i) % EXCHANGES.length]!;
const baseSize = const baseSize =
throughput.litSizeBase + throughput.litSizeBase +
@ -255,13 +243,7 @@ export const createSyntheticEquitiesAdapter = (
if (handlers.onQuote) { if (handlers.onQuote) {
quoteSeq += 1; quoteSeq += 1;
void handlers.onQuote( void handlers.onQuote(
buildSyntheticQuote( buildSyntheticQuote(quoteSeq, eventTs - 2, symbol, quote.bid, quote.ask)
quoteSeq,
eventTs - 2,
symbol,
quote.bid,
quote.ask
)
); );
} }

View file

@ -240,10 +240,7 @@ const run = async () => {
await ensureEquityQuotesTable(clickhouse); await ensureEquityQuotesTable(clickhouse);
}); });
const adapter = selectAdapter( const adapter = selectAdapter(env.EQUITIES_INGEST_ADAPTER, () => syntheticControl);
env.EQUITIES_INGEST_ADAPTER,
() => syntheticControl
);
logger.info("ingest adapter selected", { adapter: adapter.name }); logger.info("ingest adapter selected", { adapter: adapter.name });
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS); const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
const allowQuotePublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS); const allowQuotePublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);

View file

@ -126,9 +126,13 @@ const decodePayload = (data: WebSocket.RawData): unknown => {
return JSON.parse(new TextDecoder().decode(new Uint8Array(data))) as unknown; return JSON.parse(new TextDecoder().decode(new Uint8Array(data))) as unknown;
} }
if (ArrayBuffer.isView(data)) { if (ArrayBuffer.isView(data)) {
return JSON.parse(new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))) as unknown; return JSON.parse(
new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))
) as unknown;
} }
return JSON.parse(new TextDecoder().decode(new Uint8Array(data as unknown as ArrayBuffer))) as unknown; return JSON.parse(
new TextDecoder().decode(new Uint8Array(data as unknown as ArrayBuffer))
) as unknown;
}; };
const run = async () => { const run = async () => {

View file

@ -152,10 +152,7 @@ const normalizeUnderlyings = (value: string[]): string[] => {
return result; return result;
}; };
const fetchJson = async <T>( const fetchJson = async <T>(url: URL, config: AlpacaOptionsAdapterConfig): Promise<T> => {
url: URL,
config: AlpacaOptionsAdapterConfig
): Promise<T> => {
const response = await fetch(url.toString(), { const response = await fetch(url.toString(), {
headers: buildAlpacaAuthHeaders(config.credentials) headers: buildAlpacaAuthHeaders(config.credentials)
}); });
@ -235,10 +232,7 @@ const fetchOptionSnapshots = async (
return contracts; return contracts;
}; };
const selectExpiries = ( const selectExpiries = (contracts: OptionContract[], maxDteDays: number): ExpiryInfo[] => {
contracts: OptionContract[],
maxDteDays: number
): ExpiryInfo[] => {
const today = new Date(); const today = new Date();
const expiryMap = new Map<string, ExpiryInfo>(); const expiryMap = new Map<string, ExpiryInfo>();
@ -332,7 +326,9 @@ const selectContractsForUnderlying = (
const minStrike = price * (1 - config.moneynessPct); const minStrike = price * (1 - config.moneynessPct);
const maxStrike = price * (1 + config.moneynessPct); const maxStrike = price * (1 + config.moneynessPct);
const strikePairs = Array.from(strikeMap.entries()) const strikePairs = Array.from(strikeMap.entries())
.filter(([strike, pair]) => pair.call && pair.put && strike >= minStrike && strike <= maxStrike) .filter(
([strike, pair]) => pair.call && pair.put && strike >= minStrike && strike <= maxStrike
)
.map(([strike, pair]) => ({ .map(([strike, pair]) => ({
strike, strike,
call: pair.call as string, call: pair.call as string,
@ -540,7 +536,10 @@ export const createAlpacaOptionsAdapter = (
continue; continue;
} }
const message = entry as AlpacaTradeMessage | AlpacaQuoteMessage | { T?: string; msg?: string }; const message = entry as
| AlpacaTradeMessage
| AlpacaQuoteMessage
| { T?: string; msg?: string };
const type = message.T; const type = message.T;
if (type === "t") { if (type === "t") {

View file

@ -235,8 +235,7 @@ export const createDatabentoOptionsAdapter = (
return; return;
} }
const scaledPrice = const scaledPrice = config.priceScale === 1 ? price : price / config.priceScale;
config.priceScale === 1 ? price : price / config.priceScale;
const conditions = Array.isArray(payload.conditions) const conditions = Array.isArray(payload.conditions)
? payload.conditions.map((entry) => String(entry)) ? payload.conditions.map((entry) => String(entry))

View file

@ -59,9 +59,7 @@ const readLines = async (
} }
}; };
export const createIbkrOptionsAdapter = ( export const createIbkrOptionsAdapter = (config: IbkrOptionsAdapterConfig): OptionIngestAdapter => {
config: IbkrOptionsAdapterConfig
): OptionIngestAdapter => {
return { return {
name: "ibkr", name: "ibkr",
start: (handlers: OptionIngestHandlers) => { start: (handlers: OptionIngestHandlers) => {

View file

@ -715,10 +715,7 @@ const SYNTHETIC_PROFILES: Record<SyntheticMarketMode, SyntheticOptionsProfile> =
...scenario, ...scenario,
countRange: [scenario.countRange[0], scenario.countRange[1]], countRange: [scenario.countRange[0], scenario.countRange[1]],
sizeRange: [scenario.sizeRange[0], scenario.sizeRange[1]], sizeRange: [scenario.sizeRange[0], scenario.sizeRange[1]],
targetNotionalRange: [ targetNotionalRange: [scenario.targetNotionalRange[0], scenario.targetNotionalRange[1]]
scenario.targetNotionalRange[0],
scenario.targetNotionalRange[1]
]
})), })),
pricePlacements: PLACEMENTS pricePlacements: PLACEMENTS
}, },
@ -743,10 +740,7 @@ const SYNTHETIC_PROFILES: Record<SyntheticMarketMode, SyntheticOptionsProfile> =
scenarios: SCENARIO_LIBRARY.map((scenario) => ({ scenarios: SCENARIO_LIBRARY.map((scenario) => ({
...scenario, ...scenario,
countRange: [scenario.countRange[0] + 2, scenario.countRange[1] + 4], countRange: [scenario.countRange[0] + 2, scenario.countRange[1] + 4],
sizeRange: [ sizeRange: [Math.round(scenario.sizeRange[0] * 1.8), Math.round(scenario.sizeRange[1] * 2.1)],
Math.round(scenario.sizeRange[0] * 1.8),
Math.round(scenario.sizeRange[1] * 2.1)
],
targetNotionalRange: [ targetNotionalRange: [
Math.round(scenario.targetNotionalRange[0] * 1.7), Math.round(scenario.targetNotionalRange[0] * 1.7),
Math.round(scenario.targetNotionalRange[1] * 2.0) Math.round(scenario.targetNotionalRange[1] * 2.0)
@ -768,7 +762,7 @@ const SMART_MONEY_TEMPLATE_SCENARIOS: Record<
hedge_reactive: "reactive_put_wall" hedge_reactive: "reactive_put_wall"
}; };
const pick = <T,>(items: readonly T[], seed: number): T => { const pick = <T>(items: readonly T[], seed: number): T => {
return items[Math.abs(seed) % items.length]!; return items[Math.abs(seed) % items.length]!;
}; };
@ -850,9 +844,7 @@ export const updateSyntheticIvForTest = (
const sizeImpact = Math.log10(Math.max(10, input.size)) * 0.012; const sizeImpact = Math.log10(Math.max(10, input.size)) * 0.012;
const notionalImpact = Math.log10(Math.max(1_000, input.notional)) * 0.01; const notionalImpact = Math.log10(Math.max(1_000, input.notional)) * 0.01;
pressure += pressure +=
input.placement === "AA" input.placement === "AA" ? sizeImpact + notionalImpact : (sizeImpact + notionalImpact) * 0.65;
? sizeImpact + notionalImpact
: (sizeImpact + notionalImpact) * 0.65;
} else if (input.placement === "MID") { } else if (input.placement === "MID") {
pressure += 0.001; pressure += 0.001;
} else { } else {
@ -879,8 +871,7 @@ const estimateSyntheticOptionMid = (input: {
: Math.max(0, input.strike - input.underlying); : Math.max(0, input.strike - input.underlying);
const timeYears = Math.max(1, input.dteDays + 1) / 365; const timeYears = Math.max(1, input.dteDays + 1) / 365;
const baselineIv = initializeSyntheticIv(input.dteDays, input.moneyness); const baselineIv = initializeSyntheticIv(input.dteDays, input.moneyness);
const modeBoost = const modeBoost = input.mode === "firehose" ? 1.18 : input.mode === "active" ? 1.08 : 0.96;
input.mode === "firehose" ? 1.18 : input.mode === "active" ? 1.08 : 0.96;
const distance = Math.abs(input.moneyness - 1); const distance = Math.abs(input.moneyness - 1);
const extrinsic = const extrinsic =
input.underlying * input.underlying *
@ -939,12 +930,7 @@ const chooseScenario = (
): Scenario => { ): Scenario => {
const session = getSyntheticSessionState(now, control); const session = getSyntheticSessionState(now, control);
const focusSymbol = session.focus_symbols[0] ?? SYNTHETIC_SYMBOLS[0]!; const focusSymbol = session.focus_symbols[0] ?? SYNTHETIC_SYMBOLS[0]!;
const familyWeights = getSyntheticScenarioWeights( const familyWeights = getSyntheticScenarioWeights(focusSymbol, now, control, session);
focusSymbol,
now,
control,
session
);
const coverageCounts = getCoverageCounts(coverageState, now, control); const coverageCounts = getCoverageCounts(coverageState, now, control);
const weightedScenarios = profile.scenarios.map((scenario, index) => { const weightedScenarios = profile.scenarios.map((scenario, index) => {
const familyWeight = familyWeights[scenario.label]; const familyWeight = familyWeights[scenario.label];
@ -964,7 +950,10 @@ const chooseScenario = (
: 1; : 1;
return { return {
...scenario, ...scenario,
weight: Math.max(1, Math.round(scenario.weight * familyWeight * coverageBoost * quietBias * 100)) weight: Math.max(
1,
Math.round(scenario.weight * familyWeight * coverageBoost * quietBias * 100)
)
}; };
}); });
return pickWeighted(weightedScenarios, now + control.shared_seed * 31); return pickWeighted(weightedScenarios, now + control.shared_seed * 31);
@ -977,7 +966,8 @@ const pickScenarioSymbol = (
): string => { ): string => {
const session = getSyntheticSessionState(now, control); const session = getSyntheticSessionState(now, control);
const symbolPool = const symbolPool =
scenario.preferredSymbols?.length && (scenario.label === "event_driven" || Math.abs(now) % 4 === 0) scenario.preferredSymbols?.length &&
(scenario.label === "event_driven" || Math.abs(now) % 4 === 0)
? [...scenario.preferredSymbols] ? [...scenario.preferredSymbols]
: session.focus_symbols.length > 0 : session.focus_symbols.length > 0
? [...session.focus_symbols, ...SYNTHETIC_SYMBOLS] ? [...session.focus_symbols, ...SYNTHETIC_SYMBOLS]
@ -1033,7 +1023,8 @@ const buildDynamicFlowFeatures = (
0, 0,
0.26 0.26
), ),
underlying_move_bps: Math.round( underlying_move_bps:
Math.round(
(Number(scenario.flowFeatures.underlying_move_bps ?? underlying.driftBps) + (Number(scenario.flowFeatures.underlying_move_bps ?? underlying.driftBps) +
underlying.shockBps * 0.35) * underlying.shockBps * 0.35) *
100 100
@ -1059,18 +1050,14 @@ const buildBurst = (
coverageState: CoverageWindowState, coverageState: CoverageWindowState,
scenarioOverride?: Scenario scenarioOverride?: Scenario
): Burst => { ): Burst => {
const scenario = const scenario = scenarioOverride ?? chooseScenario(profile, now, control, coverageState);
scenarioOverride ?? chooseScenario(profile, now, control, coverageState);
const symbol = pickScenarioSymbol(scenario, now, control); const symbol = pickScenarioSymbol(scenario, now, control);
const symbolHash = hashSyntheticSymbol(symbol); const symbolHash = hashSyntheticSymbol(symbol);
const seed = symbolHash + burstIndex * 7; const seed = symbolHash + burstIndex * 7;
const session = getSyntheticSessionState(now, control); const session = getSyntheticSessionState(now, control);
const underlyingState = getSyntheticUnderlyingState(symbol, now, control, session); const underlyingState = getSyntheticUnderlyingState(symbol, now, control, session);
const baseUnderlying = underlyingState.mid; const baseUnderlying = underlyingState.mid;
const expiryOffset = pick( const expiryOffset = pick(scenario.expiryOffsets ?? EXPIRY_OFFSETS, symbolHash + burstIndex);
scenario.expiryOffsets ?? EXPIRY_OFFSETS,
symbolHash + burstIndex
);
const strikeStep = baseUnderlying >= 200 ? 10 : baseUnderlying >= 100 ? 5 : 2.5; const strikeStep = baseUnderlying >= 200 ? 10 : baseUnderlying >= 100 ? 5 : 2.5;
const right = const right =
scenario.right === "either" scenario.right === "either"
@ -1099,8 +1086,7 @@ const buildBurst = (
const priceStep = const priceStep =
scenario.priceTrend === "up" ? 0.01 : scenario.priceTrend === "down" ? -0.01 : 0; scenario.priceTrend === "up" ? 0.01 : scenario.priceTrend === "down" ? -0.01 : 0;
const flowFeatures = buildDynamicFlowFeatures(scenario, symbol, now, control); const flowFeatures = buildDynamicFlowFeatures(scenario, symbol, now, control);
const legTemplates = const legTemplates = scenario.legs?.length
scenario.legs?.length
? scenario.legs ? scenario.legs
: [ : [
{ {
@ -1127,8 +1113,7 @@ const buildBurst = (
const strike = Math.max( const strike = Math.max(
1, 1,
templateStrike ?? templateStrike ??
Math.round(baseUnderlying / strikeStep) * strikeStep + Math.round(baseUnderlying / strikeStep) * strikeStep + strikeOffset * strikeStep
strikeOffset * strikeStep
); );
const legSize = Math.max(1, Math.round(baseSize * (template.sizeMultiplier ?? 1))); const legSize = Math.max(1, Math.round(baseSize * (template.sizeMultiplier ?? 1)));
const legMoneyness = strike / baseUnderlying; const legMoneyness = strike / baseUnderlying;
@ -1141,13 +1126,13 @@ const buildBurst = (
mode mode
}); });
const targetMid = const targetMid =
targetNotionalPerLeg / targetNotionalPerLeg / Math.max(1, legSize * cycles * OPTION_CONTRACT_MULTIPLIER);
Math.max(1, legSize * cycles * OPTION_CONTRACT_MULTIPLIER);
const cappedTheoreticalMid = Math.min( const cappedTheoreticalMid = Math.min(
theoreticalMid, theoreticalMid,
Math.max(0.35, targetMid * (scenario.label === "institutional_directional" ? 2.2 : 2.6)) Math.max(0.35, targetMid * (scenario.label === "institutional_directional" ? 2.2 : 2.6))
); );
const blendedMid = cappedTheoreticalMid * 0.45 + targetMid * 0.55 * (template.priceMultiplier ?? 1); const blendedMid =
cappedTheoreticalMid * 0.45 + targetMid * 0.55 * (template.priceMultiplier ?? 1);
return { return {
contractId: `${symbol}-${expiry}-${formatStrike(strike)}-${template.right}`, contractId: `${symbol}-${expiry}-${formatStrike(strike)}-${template.right}`,
right: template.right, right: template.right,
@ -1184,8 +1169,7 @@ const buildBurst = (
scenario.missingQuoteProbability ?? scenario.missingQuoteProbability ??
clampValue((1 - session.quote_cleanliness) * 0.16, 0, 0.18), clampValue((1 - session.quote_cleanliness) * 0.16, 0, 0.18),
staleQuoteProbability: staleQuoteProbability:
scenario.staleQuoteProbability ?? scenario.staleQuoteProbability ?? clampValue((1 - session.quote_cleanliness) * 0.3, 0, 0.42)
clampValue((1 - session.quote_cleanliness) * 0.3, 0, 0.42)
}; };
}; };
@ -1202,7 +1186,9 @@ export const listSyntheticSmartMoneyScenariosForTest = (): SyntheticSmartMoneySc
hiddenLabel: hiddenLabel:
id === "neutral_noise" id === "neutral_noise"
? "single_print_mid" ? "single_print_mid"
: SMART_MONEY_TEMPLATE_SCENARIOS[id as Exclude<(typeof SMART_MONEY_SCENARIO_IDS)[number], "neutral_noise">] : SMART_MONEY_TEMPLATE_SCENARIOS[
id as Exclude<(typeof SMART_MONEY_SCENARIO_IDS)[number], "neutral_noise">
]
})); }));
export const buildSyntheticSmartMoneyBurstForTest = ( export const buildSyntheticSmartMoneyBurstForTest = (
@ -1233,16 +1219,16 @@ export const buildSyntheticSmartMoneyBurstForTest = (
updated_by: "system" updated_by: "system"
} satisfies SyntheticControlState; } satisfies SyntheticControlState;
const mode: SyntheticMarketMode = const mode: SyntheticMarketMode =
scenarioId === "retail_whale" || scenarioId === "neutral_noise" scenarioId === "retail_whale" || scenarioId === "neutral_noise" ? "realistic" : "active";
? "realistic"
: "active";
const profile = SYNTHETIC_PROFILES[mode]; const profile = SYNTHETIC_PROFILES[mode];
const coverageState = createCoverageWindowState(); const coverageState = createCoverageWindowState();
const scenario = const scenario =
scenarioId === "neutral_noise" scenarioId === "neutral_noise"
? profile.scenarios.find((candidate) => candidate.id === "single_print_mid")! ? profile.scenarios.find((candidate) => candidate.id === "single_print_mid")!
: profile.scenarios.find( : profile.scenarios.find(
(candidate) => candidate.id === SMART_MONEY_TEMPLATE_SCENARIOS[ (candidate) =>
candidate.id ===
SMART_MONEY_TEMPLATE_SCENARIOS[
scenarioId as Exclude<(typeof SMART_MONEY_SCENARIO_IDS)[number], "neutral_noise"> scenarioId as Exclude<(typeof SMART_MONEY_SCENARIO_IDS)[number], "neutral_noise">
] ]
)!; )!;
@ -1255,13 +1241,10 @@ export const buildSyntheticFlowPacketForTest = (
): { packet: FlowPacket; hiddenLabel: string } => { ): { packet: FlowPacket; hiddenLabel: string } => {
const burst = buildSyntheticSmartMoneyBurstForTest(scenarioId, now); const burst = buildSyntheticSmartMoneyBurstForTest(scenarioId, now);
const primaryLeg = burst.legs[0]!; const primaryLeg = burst.legs[0]!;
const corporateEventOffset = Number( const corporateEventOffset = Number(burst.flowFeatures.corporate_event_ts_offset_days ?? 0);
burst.flowFeatures.corporate_event_ts_offset_days ?? 0
);
const totalSize = burst.legs.reduce((sum, leg) => sum + leg.baseSize * burst.cycles, 0); const totalSize = burst.legs.reduce((sum, leg) => sum + leg.baseSize * burst.cycles, 0);
const totalPremium = burst.legs.reduce( const totalPremium = burst.legs.reduce(
(sum, leg) => (sum, leg) => sum + leg.basePrice * leg.baseSize * burst.cycles * OPTION_CONTRACT_MULTIPLIER,
sum + leg.basePrice * leg.baseSize * burst.cycles * OPTION_CONTRACT_MULTIPLIER,
0 0
); );
const flowFeatures: FlowPacket["features"] = { const flowFeatures: FlowPacket["features"] = {
@ -1272,15 +1255,10 @@ export const buildSyntheticFlowPacketForTest = (
window_ms: Math.max(0, (burst.printCount - 1) * 45), window_ms: Math.max(0, (burst.printCount - 1) * 45),
total_size: totalSize, total_size: totalSize,
total_premium: Number(totalPremium.toFixed(2)), total_premium: Number(totalPremium.toFixed(2)),
total_notional: Number( total_notional: Number((burst.underlying * totalSize * OPTION_CONTRACT_MULTIPLIER).toFixed(2)),
(burst.underlying * totalSize * OPTION_CONTRACT_MULTIPLIER).toFixed(2)
),
first_price: primaryLeg.basePrice, first_price: primaryLeg.basePrice,
last_price: Number( last_price: Number(
( (primaryLeg.basePrice * (1 + burst.priceStep * Math.max(0, burst.cycles - 1))).toFixed(2)
primaryLeg.basePrice *
(1 + burst.priceStep * Math.max(0, burst.cycles - 1))
).toFixed(2)
), ),
nbbo_missing_count: 0, nbbo_missing_count: 0,
nbbo_stale_count: 0, nbbo_stale_count: 0,
@ -1300,10 +1278,7 @@ export const buildSyntheticFlowPacketForTest = (
Number(flowFeatures.total_premium ?? totalPremium), Number(flowFeatures.total_premium ?? totalPremium),
72_000 72_000
); );
flowFeatures.execution_iv_shock = Math.max( flowFeatures.execution_iv_shock = Math.max(Number(flowFeatures.execution_iv_shock ?? 0), 0.22);
Number(flowFeatures.execution_iv_shock ?? 0),
0.22
);
} }
if (scenarioId === "event_driven") { if (scenarioId === "event_driven") {
flowFeatures.count = 2; flowFeatures.count = 2;
@ -1411,14 +1386,7 @@ export const buildSyntheticBurstForTest = (
return cached[burstIndex - 1]!; return cached[burstIndex - 1]!;
} }
for (let index = cached.length + 1; index <= burstIndex; index += 1) { for (let index = cached.length + 1; index <= burstIndex; index += 1) {
const current = buildBurst( const current = buildBurst(index, now + index * 1_000, mode, profile, control, coverageState);
index,
now + index * 1_000,
mode,
profile,
control,
coverageState
);
recordCoverageHit(coverageState, current.label, now + index * 1_000); recordCoverageHit(coverageState, current.label, now + index * 1_000);
cached.push(current); cached.push(current);
} }
@ -1466,14 +1434,7 @@ export const createSyntheticOptionsAdapter = (
}; };
if (!currentBurst || remainingRuns <= 0) { if (!currentBurst || remainingRuns <= 0) {
burstIndex += 1; burstIndex += 1;
currentBurst = buildBurst( currentBurst = buildBurst(burstIndex, now, config.mode, profile, control, coverageState);
burstIndex,
now,
config.mode,
profile,
control,
coverageState
);
recordCoverageHit(coverageState, currentBurst.label, now); recordCoverageHit(coverageState, currentBurst.label, now);
remainingRuns = pickInt( remainingRuns = pickInt(
profile.burstRunRange[0], profile.burstRunRange[0],
@ -1565,8 +1526,7 @@ export const createSyntheticOptionsAdapter = (
const quoteSeed = Math.abs(burst.seed + i * 17) % 1000; const quoteSeed = Math.abs(burst.seed + i * 17) % 1000;
const missingQuote = quoteSeed / 1000 < burst.missingQuoteProbability; const missingQuote = quoteSeed / 1000 < burst.missingQuoteProbability;
const staleQuote = const staleQuote =
!missingQuote && !missingQuote && ((quoteSeed + 233) % 1000) / 1000 < burst.staleQuoteProbability;
((quoteSeed + 233) % 1000) / 1000 < burst.staleQuoteProbability;
if (handlers.onNBBO && !missingQuote) { if (handlers.onNBBO && !missingQuote) {
nbboSeq += 1; nbboSeq += 1;

View file

@ -48,7 +48,11 @@ export const selectAtOrBefore = <T extends { ts: number; seq: number }>(
if (item.ts > ts) { if (item.ts > ts) {
continue; continue;
} }
if (!selected || item.ts > selected.ts || (item.ts === selected.ts && item.seq >= selected.seq)) { if (
!selected ||
item.ts > selected.ts ||
(item.ts === selected.ts && item.seq >= selected.seq)
) {
selected = item; selected = item;
} }
} }

View file

@ -43,7 +43,12 @@ import { createDatabentoOptionsAdapter } from "./adapters/databento";
import { createIbkrOptionsAdapter } from "./adapters/ibkr"; import { createIbkrOptionsAdapter } from "./adapters/ibkr";
import { createSyntheticOptionsAdapter } from "./adapters/synthetic"; import { createSyntheticOptionsAdapter } from "./adapters/synthetic";
import type { OptionIngestAdapter, StopHandler } from "./adapters/types"; import type { OptionIngestAdapter, StopHandler } from "./adapters/types";
import { enrichOptionPrint, rememberContext, selectAtOrBefore, type ContextHistory } from "./enrichment"; import {
enrichOptionPrint,
rememberContext,
selectAtOrBefore,
type ContextHistory
} from "./enrichment";
import { z } from "zod"; import { z } from "zod";
const service = "ingest-options"; const service = "ingest-options";
@ -87,7 +92,10 @@ const envSchema = z.object({
IBKR_EXPIRY: z.string().min(1).default("20250117"), IBKR_EXPIRY: z.string().min(1).default("20250117"),
IBKR_STRIKE: z.coerce.number().positive().default(450), IBKR_STRIKE: z.coerce.number().positive().default(450),
IBKR_RIGHT: z IBKR_RIGHT: z
.preprocess((value) => (typeof value === "string" ? value.toUpperCase() : value), z.enum(["C", "P"])) .preprocess(
(value) => (typeof value === "string" ? value.toUpperCase() : value),
z.enum(["C", "P"])
)
.default("C"), .default("C"),
IBKR_EXCHANGE: z.string().min(1).default("SMART"), IBKR_EXCHANGE: z.string().min(1).default("SMART"),
IBKR_CURRENCY: z.string().min(1).default("USD"), IBKR_CURRENCY: z.string().min(1).default("USD"),
@ -395,10 +403,7 @@ const run = async () => {
await ensureOptionNBBOTable(clickhouse); await ensureOptionNBBOTable(clickhouse);
}); });
const adapter = selectAdapter( const adapter = selectAdapter(env.OPTIONS_INGEST_ADAPTER, () => syntheticControl);
env.OPTIONS_INGEST_ADAPTER,
() => syntheticControl
);
logger.info("ingest adapter selected", { adapter: adapter.name }); logger.info("ingest adapter selected", { adapter: adapter.name });
const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS); const allowPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
const allowNbboPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS); const allowNbboPublish = buildThrottle(env.TESTING_MODE, env.TESTING_THROTTLE_MS);
@ -421,7 +426,10 @@ const run = async () => {
rawPrint.ts rawPrint.ts
); );
const equityQuote = parsedMetadata.underlying_id const equityQuote = parsedMetadata.underlying_id
? selectAtOrBefore(equityQuoteHistoryByUnderlying.get(parsedMetadata.underlying_id), rawPrint.ts) ? selectAtOrBefore(
equityQuoteHistoryByUnderlying.get(parsedMetadata.underlying_id),
rawPrint.ts
)
: null; : null;
const print = enrichOptionPrint(rawPrint, optionQuote, equityQuote, optionsSignalConfig); const print = enrichOptionPrint(rawPrint, optionQuote, equityQuote, optionsSignalConfig);
@ -500,8 +508,16 @@ const run = async () => {
const pruneTimer = setInterval(() => { const pruneTimer = setInterval(() => {
const removed = const removed =
pruneContextHistory(nbboHistoryByContract, env.OPTION_CONTEXT_MAX_KEYS, env.OPTION_CONTEXT_TTL_MS) + pruneContextHistory(
pruneContextHistory(equityQuoteHistoryByUnderlying, env.OPTION_CONTEXT_MAX_KEYS, env.OPTION_CONTEXT_TTL_MS); nbboHistoryByContract,
env.OPTION_CONTEXT_MAX_KEYS,
env.OPTION_CONTEXT_TTL_MS
) +
pruneContextHistory(
equityQuoteHistoryByUnderlying,
env.OPTION_CONTEXT_MAX_KEYS,
env.OPTION_CONTEXT_TTL_MS
);
logger.info("option context cache summary", { logger.info("option context cache summary", {
nbbo_context_keys: nbboHistoryByContract.size, nbbo_context_keys: nbboHistoryByContract.size,
equity_quote_context_keys: equityQuoteHistoryByUnderlying.size, equity_quote_context_keys: equityQuoteHistoryByUnderlying.size,

View file

@ -1,6 +1,12 @@
import { mkdir } from "node:fs/promises"; import { mkdir } from "node:fs/promises";
export type EventCalendarKind = "earnings" | "dividend" | "corporate_action" | "m_and_a" | "news" | "other"; export type EventCalendarKind =
| "earnings"
| "dividend"
| "corporate_action"
| "m_and_a"
| "news"
| "other";
export type EventCalendarEntry = { export type EventCalendarEntry = {
underlying_id: string; underlying_id: string;
@ -56,7 +62,8 @@ const asNumber = (value: unknown): number | null => {
return null; return null;
}; };
const asString = (value: unknown): string | null => (typeof value === "string" && value.trim() ? value.trim() : null); const asString = (value: unknown): string | null =>
typeof value === "string" && value.trim() ? value.trim() : null;
const parseCsvLine = (line: string): string[] => { const parseCsvLine = (line: string): string[] => {
const values: string[] = []; const values: string[] = [];
@ -139,9 +146,14 @@ export const parseEventCalendarEntries = (value: unknown): EventCalendarEntry[]
const record = row as Record<string, unknown>; const record = row as Record<string, unknown>;
const underlying = asString(record.underlying_id ?? record.underlying ?? record.symbol); const underlying = asString(record.underlying_id ?? record.underlying ?? record.symbol);
const eventTs = asNumber(record.event_ts ?? record.event_time ?? record.event_date); const eventTs = asNumber(record.event_ts ?? record.event_time ?? record.event_date);
const announcedTs = asNumber(record.announced_ts ?? record.available_ts ?? record.as_of_ts ?? record.created_ts) ?? 0; const announcedTs =
asNumber(
record.announced_ts ?? record.available_ts ?? record.as_of_ts ?? record.created_ts
) ?? 0;
const rawKind = asString(record.event_kind ?? record.kind ?? record.type) ?? "other"; const rawKind = asString(record.event_kind ?? record.kind ?? record.type) ?? "other";
const eventKind = EVENT_KINDS.has(rawKind as EventCalendarKind) ? (rawKind as EventCalendarKind) : "other"; const eventKind = EVENT_KINDS.has(rawKind as EventCalendarKind)
? (rawKind as EventCalendarKind)
: "other";
if (!underlying || eventTs === null || eventTs < 0 || announcedTs < 0) { if (!underlying || eventTs === null || eventTs < 0 || announcedTs < 0) {
return []; return [];
@ -162,7 +174,9 @@ export const parseEventCalendarEntries = (value: unknown): EventCalendarEntry[]
}); });
}; };
export const createStaticEventCalendarProvider = (entries: EventCalendarEntry[]): EventCalendarProvider => { export const createStaticEventCalendarProvider = (
entries: EventCalendarEntry[]
): EventCalendarProvider => {
const byUnderlying = new Map<string, EventCalendarEntry[]>(); const byUnderlying = new Map<string, EventCalendarEntry[]>();
for (const entry of entries) { for (const entry of entries) {
const key = normalizeUnderlying(entry.underlying_id); const key = normalizeUnderlying(entry.underlying_id);
@ -184,15 +198,20 @@ export const createStaticEventCalendarProvider = (entries: EventCalendarEntry[])
} }
const bucket = byUnderlying.get(key) ?? []; const bucket = byUnderlying.get(key) ?? [];
const entry = bucket.find((candidate) => candidate.announced_ts <= asOfTs && candidate.event_ts >= asOfTs); const entry = bucket.find(
(candidate) => candidate.announced_ts <= asOfTs && candidate.event_ts >= asOfTs
);
return entry ? { ...entry, days_to_event: (entry.event_ts - asOfTs) / MS_PER_DAY } : null; return entry ? { ...entry, days_to_event: (entry.event_ts - asOfTs) / MS_PER_DAY } : null;
} }
}; };
}; };
export const createEmptyEventCalendarProvider = (): EventCalendarProvider => createStaticEventCalendarProvider([]); export const createEmptyEventCalendarProvider = (): EventCalendarProvider =>
createStaticEventCalendarProvider([]);
export const loadEventCalendarProviderFromFile = async (path: string): Promise<EventCalendarProvider> => { export const loadEventCalendarProviderFromFile = async (
path: string
): Promise<EventCalendarProvider> => {
const text = await Bun.file(path).text(); const text = await Bun.file(path).text();
return createStaticEventCalendarProvider(parseEventCalendarEntries(JSON.parse(text))); return createStaticEventCalendarProvider(parseEventCalendarEntries(JSON.parse(text)));
}; };
@ -212,7 +231,9 @@ export const fetchAlphaVantageEarningsCalendar = async (
const response = await (options.fetchFn ?? fetch)(url); const response = await (options.fetchFn ?? fetch)(url);
const text = await response.text(); const text = await response.text();
if (!response.ok) { if (!response.ok) {
throw new Error(`Alpha Vantage earnings calendar request failed: ${response.status} ${text.slice(0, 160)}`); throw new Error(
`Alpha Vantage earnings calendar request failed: ${response.status} ${text.slice(0, 160)}`
);
} }
if (/^(?:\s*\{|\s*Thank you for using Alpha Vantage)/i.test(text)) { if (/^(?:\s*\{|\s*Thank you for using Alpha Vantage)/i.test(text)) {
throw new Error(`Alpha Vantage returned a non-calendar response: ${text.slice(0, 200)}`); throw new Error(`Alpha Vantage returned a non-calendar response: ${text.slice(0, 200)}`);
@ -221,7 +242,10 @@ export const fetchAlphaVantageEarningsCalendar = async (
return parseAlphaVantageEarningsCalendar(text, options.nowTs ?? Date.now()); return parseAlphaVantageEarningsCalendar(text, options.nowTs ?? Date.now());
}; };
export const writeEventCalendarEntries = async (path: string, entries: EventCalendarEntry[]): Promise<void> => { export const writeEventCalendarEntries = async (
path: string,
entries: EventCalendarEntry[]
): Promise<void> => {
const directory = path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : ""; const directory = path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : "";
if (directory) { if (directory) {
await mkdir(directory, { recursive: true }); await mkdir(directory, { recursive: true });

View file

@ -12,9 +12,14 @@ const logger = createLogger({ service });
logger.info("service starting"); logger.info("service starting");
const eventCalendarPath = process.env.REFDATA_EVENT_CALENDAR_PATH ?? process.env.SMART_MONEY_EVENT_CALENDAR_PATH; const eventCalendarPath =
const eventCalendarProvider = process.env.REFDATA_EVENT_CALENDAR_PROVIDER ?? process.env.EVENT_CALENDAR_PROVIDER; process.env.REFDATA_EVENT_CALENDAR_PATH ?? process.env.SMART_MONEY_EVENT_CALENDAR_PATH;
const refreshMs = Math.max(0, Number(process.env.REFDATA_EVENT_CALENDAR_REFRESH_MS ?? 86_400_000) || 0); const eventCalendarProvider =
process.env.REFDATA_EVENT_CALENDAR_PROVIDER ?? process.env.EVENT_CALENDAR_PROVIDER;
const refreshMs = Math.max(
0,
Number(process.env.REFDATA_EVENT_CALENDAR_REFRESH_MS ?? 86_400_000) || 0
);
const getAlphaVantageOptions = (): AlphaVantageEarningsCalendarOptions | null => { const getAlphaVantageOptions = (): AlphaVantageEarningsCalendarOptions | null => {
const apiKey = process.env.ALPHA_VANTAGE_API_KEY; const apiKey = process.env.ALPHA_VANTAGE_API_KEY;
@ -33,7 +38,9 @@ const getAlphaVantageOptions = (): AlphaVantageEarningsCalendarOptions | null =>
const refreshEventCalendar = async (): Promise<void> => { const refreshEventCalendar = async (): Promise<void> => {
if (!eventCalendarPath) { if (!eventCalendarPath) {
logger.warn("event calendar refresh disabled; missing SMART_MONEY_EVENT_CALENDAR_PATH or REFDATA_EVENT_CALENDAR_PATH"); logger.warn(
"event calendar refresh disabled; missing SMART_MONEY_EVENT_CALENDAR_PATH or REFDATA_EVENT_CALENDAR_PATH"
);
return; return;
} }
if (eventCalendarProvider !== "alpha_vantage") { if (eventCalendarProvider !== "alpha_vantage") {

View file

@ -52,11 +52,7 @@ type ReplayStreamKind = "options" | "nbbo" | "equities" | "equity-quotes";
type ReplayEvent = OptionPrint | OptionNBBO | EquityPrint | EquityQuote; type ReplayEvent = OptionPrint | OptionNBBO | EquityPrint | EquityQuote;
type FetchAfter = ( type FetchAfter = (afterTs: number, afterSeq: number, limit: number) => Promise<ReplayEvent[]>;
afterTs: number,
afterSeq: number,
limit: number
) => Promise<ReplayEvent[]>;
type ReplayStream = { type ReplayStream = {
kind: ReplayStreamKind; kind: ReplayStreamKind;
@ -79,7 +75,12 @@ const STREAM_DEFS: Record<
subject: string; subject: string;
streamName: string; streamName: string;
rank: number; rank: number;
fetchAfter: (client: ReturnType<typeof createClickHouseClient>, afterTs: number, afterSeq: number, limit: number) => Promise<ReplayEvent[]>; fetchAfter: (
client: ReturnType<typeof createClickHouseClient>,
afterTs: number,
afterSeq: number,
limit: number
) => Promise<ReplayEvent[]>;
} }
> = { > = {
options: { options: {
@ -196,7 +197,9 @@ const getEventIngestTs = (event: ReplayEvent): number =>
const getEventSeq = (event: ReplayEvent): number => (Number.isFinite(event.seq) ? event.seq : 0); const getEventSeq = (event: ReplayEvent): number => (Number.isFinite(event.seq) ? event.seq : 0);
const pickNextEvent = (streams: ReplayStream[]): { stream: ReplayStream; event: ReplayEvent } | null => { const pickNextEvent = (
streams: ReplayStream[]
): { stream: ReplayStream; event: ReplayEvent } | null => {
let choice: { stream: ReplayStream; event: ReplayEvent } | null = null; let choice: { stream: ReplayStream; event: ReplayEvent } | null = null;
for (const stream of streams) { for (const stream of streams) {
@ -313,7 +316,8 @@ const run = async () => {
kind, kind,
subject: def.subject, subject: def.subject,
streamName: def.streamName, streamName: def.streamName,
fetchAfter: (afterTs, afterSeq, limit) => def.fetchAfter(clickhouse, afterTs, afterSeq, limit), fetchAfter: (afterTs, afterSeq, limit) =>
def.fetchAfter(clickhouse, afterTs, afterSeq, limit),
buffer: [], buffer: [],
cursor: { ...startCursor }, cursor: { ...startCursor },
done: false, done: false,

View file

@ -8,6 +8,6 @@
"isolatedModules": true, "isolatedModules": true,
"resolveJsonModule": true, "resolveJsonModule": true,
"skipLibCheck": true, "skipLibCheck": true,
"noEmit": true, "noEmit": true
}, }
} }