dirtydishes/islandflow
1
0
Fork 0
Code Issues 2 Pull requests 4 Projects Releases Packages Wiki Activity Actions 1

Implement first-pass load reduction controls

Browse source
This commit is contained in:
dirtydishes 2026-05-08 02:46:41 -04:00
parent 5d488fd7f5
commit e7f4805ccc
17 changed files with 1191 additions and 608 deletions
Download patch file Download diff file Expand all files Collapse all files

33
services/ingest-equities/src/index.ts
View file

@ -5,6 +5,7 @@ import {
SUBJECT_EQUITY_QUOTES,
STREAM_EQUITY_PRINTS,
STREAM_EQUITY_QUOTES,
buildStreamConfig,
connectJetStreamWithRetry,
ensureStream,
publishJson
@ -194,31 +195,8 @@ const run = async () => {
{ attempts: 120, delayMs: 500 }
);
await ensureStream(jsm, {
name: STREAM_EQUITY_PRINTS,
subjects: [SUBJECT_EQUITY_PRINTS],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, {
name: STREAM_EQUITY_QUOTES,
subjects: [SUBJECT_EQUITY_QUOTES],
retention: "limits",
storage: "file",
discard: "old",
max_msgs_per_subject: -1,
max_msgs: -1,
max_bytes: -1,
max_age: 0,
num_replicas: 1
});
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_PRINTS, SUBJECT_EQUITY_PRINTS, "raw"));
await ensureStream(jsm, buildStreamConfig(STREAM_EQUITY_QUOTES, SUBJECT_EQUITY_QUOTES, "raw"));
const clickhouse = createClickHouseClient({
url: env.CLICKHOUSE_URL,
@ -251,11 +229,6 @@ const run = async () => {
try {
await insertEquityPrint(clickhouse, print);
await publishJson(js, SUBJECT_EQUITY_PRINTS, print);
logger.info("published equity print", {
trace_id: print.trace_id,
seq: print.seq,
underlying_id: print.underlying_id
});
} catch (error) {
if (isExpectedShutdownError(error)) {
return;