dirtydishes/islandflow
1
0
Fork 0
Code Issues 2 Pull requests 4 Projects Releases Packages Wiki Activity Actions 1

Reconcile JetStream retention caps

Browse source
This commit is contained in:
dirtydishes 2026-05-08 15:52:32 -04:00
parent 26e69bf98d
commit f7aed36591
15 changed files with 837 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

31
services/api/src/index.ts
View file

@ -23,10 +23,9 @@ import {
STREAM_SMART_MONEY_EVENTS,
STREAM_OPTION_NBBO,
STREAM_OPTION_SIGNAL_PRINTS,
buildStreamConfig,
buildDurableConsumer,
connectJetStreamWithRetry,
ensureStream,
ensureKnownStreams,
subscribeJson
} from "@islandflow/bus";
import {
@ -624,17 +623,23 @@ const run = async () => {
{ attempts: 120, delayMs: 500 }
);
await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_CANDLES, SUBJECT_EQUITY_CANDLES, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_JOINS, SUBJECT_EQUITY_JOINS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_INFERRED_DARK, SUBJECT_INFERRED_DARK, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_FLOW_PACKETS, SUBJECT_FLOW_PACKETS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_SMART_MONEY_EVENTS, SUBJECT_SMART_MONEY_EVENTS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_CLASSIFIER_HITS, SUBJECT_CLASSIFIER_HITS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_ALERTS, SUBJECT_ALERTS, "derived"));
await ensureKnownStreams(
jsm,
[
STREAM_OPTION_SIGNAL_PRINTS,
STREAM_OPTION_NBBO,
STREAM_EQUITY_PRINTS,
STREAM_EQUITY_QUOTES,
STREAM_EQUITY_CANDLES,
STREAM_EQUITY_JOINS,
STREAM_INFERRED_DARK,
STREAM_FLOW_PACKETS,
STREAM_SMART_MONEY_EVENTS,
STREAM_CLASSIFIER_HITS,
STREAM_ALERTS
],
{ logger }
);
const clickhouse = createClickHouseClient({
url: env.CLICKHOUSE_URL,

6
services/candles/src/index.ts
View file

@ -5,10 +5,9 @@ import {
SUBJECT_EQUITY_PRINTS,
STREAM_EQUITY_CANDLES,
STREAM_EQUITY_PRINTS,
buildStreamConfig,
buildDurableConsumer,
connectJetStreamWithRetry,
ensureStream,
ensureKnownStreams,
publishJson,
subscribeJson
} from "@islandflow/bus";
@ -241,8 +240,7 @@ const run = async () => {
{ attempts: 120, delayMs: 500 }
);
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_CANDLES, SUBJECT_EQUITY_CANDLES, "derived"));
await ensureKnownStreams(jsm, [STREAM_EQUITY_PRINTS, STREAM_EQUITY_CANDLES], { logger });
const clickhouse = createClickHouseClient({
url: env.CLICKHOUSE_URL,

29
services/compute/src/index.ts
View file

@ -26,10 +26,9 @@ import {
STREAM_SMART_MONEY_EVENTS,
STREAM_OPTION_NBBO,
STREAM_OPTION_SIGNAL_PRINTS,
buildStreamConfig,
buildDurableConsumer,
connectJetStreamWithRetry,
ensureStream,
ensureKnownStreams,
publishJson,
subscribeJson
} from "@islandflow/bus";
@ -1174,16 +1173,22 @@ const run = async () => {
{ attempts: 120, delayMs: 500 }
);
await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_FLOW_PACKETS, SUBJECT_FLOW_PACKETS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_SMART_MONEY_EVENTS, SUBJECT_SMART_MONEY_EVENTS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_JOINS, SUBJECT_EQUITY_JOINS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_INFERRED_DARK, SUBJECT_INFERRED_DARK, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_CLASSIFIER_HITS, SUBJECT_CLASSIFIER_HITS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_ALERTS, SUBJECT_ALERTS, "derived"));
await ensureKnownStreams(
jsm,
[
STREAM_OPTION_SIGNAL_PRINTS,
STREAM_OPTION_NBBO,
STREAM_EQUITY_PRINTS,
STREAM_EQUITY_QUOTES,
STREAM_FLOW_PACKETS,
STREAM_SMART_MONEY_EVENTS,
STREAM_EQUITY_JOINS,
STREAM_INFERRED_DARK,
STREAM_CLASSIFIER_HITS,
STREAM_ALERTS
],
{ logger }
);
const clickhouse = createClickHouseClient({
url: env.CLICKHOUSE_URL,

6
services/ingest-equities/src/index.ts
View file

@ -5,9 +5,8 @@ import {
SUBJECT_EQUITY_QUOTES,
STREAM_EQUITY_PRINTS,
STREAM_EQUITY_QUOTES,
buildStreamConfig,
connectJetStreamWithRetry,
ensureStream,
ensureKnownStreams,
publishJson
} from "@islandflow/bus";
import {
@ -195,8 +194,7 @@ const run = async () => {
{ attempts: 120, delayMs: 500 }
);
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw"));
await ensureKnownStreams(jsm, [STREAM_EQUITY_PRINTS, STREAM_EQUITY_QUOTES], { logger });
const clickhouse = createClickHouseClient({
url: env.CLICKHOUSE_URL,

12
services/ingest-options/src/index.ts
View file

@ -9,10 +9,9 @@ import {
STREAM_OPTION_NBBO,
STREAM_OPTION_PRINTS,
STREAM_OPTION_SIGNAL_PRINTS,
buildStreamConfig,
buildDurableConsumer,
connectJetStreamWithRetry,
ensureStream,
ensureKnownStreams,
publishJson,
subscribeJson
} from "@islandflow/bus";
@ -346,10 +345,11 @@ const run = async () => {
{ attempts: 120, delayMs: 500 }
);
await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_PRINTS, SUBJECT_OPTION_PRINTS, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_NBBO, SUBJECT_OPTION_NBBO, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw"));
await ensureKnownStreams(
jsm,
[STREAM_OPTION_PRINTS, STREAM_OPTION_NBBO, STREAM_OPTION_SIGNAL_PRINTS, STREAM_EQUITY_QUOTES],
{ logger }
);
const clickhouse = createClickHouseClient({
url: env.CLICKHOUSE_URL,

7
services/replay/src/index.ts
View file

@ -11,9 +11,8 @@ import {
STREAM_OPTION_NBBO,
STREAM_OPTION_PRINTS,
STREAM_OPTION_SIGNAL_PRINTS,
buildStreamConfig,
connectJetStreamWithRetry,
ensureStream,
ensureKnownStreams,
publishJson
} from "@islandflow/bus";
import {
@ -292,10 +291,10 @@ const run = async () => {
for (const kind of streamKinds) {
const def = STREAM_DEFS[kind];
await ensureStream(jsm, buildStreamConfig(def.streamName, def.subject, "raw"));
await ensureKnownStreams(jsm, [def.streamName], { logger });
}
if (streamKinds.includes("options")) {
await ensureStream(jsm, buildStreamConfig(STREAM_OPTION_SIGNAL_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, "derived"));
await ensureKnownStreams(jsm, [STREAM_OPTION_SIGNAL_PRINTS], { logger });
}
const clickhouse = createClickHouseClient({