dirtydishes/islandflow
1
0
Fork 0
Code Issues 2 Pull requests 4 Projects Releases Packages Wiki Activity Actions 1

total ui refactor

Browse source 
total ui refactor, deslopification, tuning, mildly bbg-inspired if u squint
This commit is contained in:
dirtydishes 2026-03-29 00:05:48 -04:00 • committed by GitHub
commit b3c63a19d0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 5722 additions and 4655 deletions
Download patch file Download diff file Expand all files Collapse all files

76
services/api/src/index.ts
View file

@ -89,6 +89,31 @@ const envSchema = z.object({
const env = readEnv(envSchema);
const state = {
shuttingDown: false,
shutdownPromise: null as Promise<void> | null
};
const getErrorMessage = (error: unknown): string => {
return error instanceof Error ? error.message : String(error);
};
const isExpectedShutdownError = (error: unknown): boolean => {
if (!state.shuttingDown) {
return false;
}
const message = getErrorMessage(error).toUpperCase();
return [
"SOCKET CONNECTION WAS CLOSED UNEXPECTEDLY",
"SOCKET CLOSED UNEXPECTEDLY",
"ECONNREFUSED",
"CONNECTION_CLOSED",
"CONNECTION_DRAINING",
"TIMEOUT"
].some((token) => message.includes(token));
};
const retry = async <T>(
label: string,
attempts: number,
@ -517,8 +542,12 @@ const run = async () => {
try {
redis = createClient({ url: env.REDIS_URL });
redis.on("error", (error) => {
if (isExpectedShutdownError(error)) {
return;
}
logger.warn("redis client error", {
error: error instanceof Error ? error.message : String(error)
error: getErrorMessage(error)
});
});
await retry("redis connect", 5, 500, async () => {
@ -1150,14 +1179,45 @@ const run = async () => {
logger.info("api listening", { port: server.port });
const shutdown = async (signal: string) => {
logger.info("service stopping", { signal });
server.stop();
if (redis && redis.isOpen) {
await redis.quit();
if (state.shutdownPromise) {
return state.shutdownPromise;
}
await nc.drain();
await clickhouse.close();
process.exit(0);
state.shuttingDown = true;
state.shutdownPromise = (async () => {
logger.info("service stopping", { signal });
server.stop();
if (redis && redis.isOpen) {
try {
await redis.quit();
} catch (error) {
if (!isExpectedShutdownError(error)) {
throw error;
}
}
}
try {
await nc.drain();
} catch (error) {
if (!isExpectedShutdownError(error)) {
throw error;
}
}
try {
await clickhouse.close();
} catch (error) {
if (!isExpectedShutdownError(error)) {
throw error;
}
}
process.exit(0);
})();
return state.shutdownPromise;
};
process.on("SIGINT", () => void shutdown("SIGINT"));

106
services/candles/src/index.ts
View file

@ -54,6 +54,31 @@ const envSchema = z.object({
const env = readEnv(envSchema);
const state = {
shuttingDown: false,
shutdownPromise: null as Promise<void> | null
};
const getErrorMessage = (error: unknown): string => {
return error instanceof Error ? error.message : String(error);
};
const isExpectedShutdownError = (error: unknown): boolean => {
if (!state.shuttingDown) {
return false;
}
const message = getErrorMessage(error).toUpperCase();
return [
"SOCKET CONNECTION WAS CLOSED UNEXPECTEDLY",
"SOCKET CLOSED UNEXPECTEDLY",
"ECONNREFUSED",
"CONNECTION_CLOSED",
"CONNECTION_DRAINING",
"TIMEOUT"
].some((token) => message.includes(token));
};
const retry = async <T>(
label: string,
attempts: number,
@ -141,9 +166,13 @@ const emitCandle = async (
try {
await insertEquityCandle(clickhouse, candle);
} catch (error) {
if (isExpectedShutdownError(error)) {
return;
}
metrics.count("candles.persist_failed", 1);
logger.error("failed to persist candle", {
error: error instanceof Error ? error.message : String(error),
error: getErrorMessage(error),
trace_id: candle.trace_id,
underlying_id: candle.underlying_id,
interval_ms: candle.interval_ms
@ -158,9 +187,13 @@ const emitCandle = async (
try {
await publishJson(js, SUBJECT_EQUITY_CANDLES, candle);
} catch (error) {
if (isExpectedShutdownError(error)) {
return;
}
metrics.count("candles.publish_failed", 1);
logger.error("failed to publish candle", {
error: error instanceof Error ? error.message : String(error),
error: getErrorMessage(error),
trace_id: candle.trace_id,
underlying_id: candle.underlying_id,
interval_ms: candle.interval_ms
@ -171,9 +204,13 @@ const emitCandle = async (
try {
await cacheCandle(redis, candle, cacheLimit);
} catch (error) {
if (isExpectedShutdownError(error)) {
return;
}
metrics.count("candles.cache_failed", 1);
logger.warn("failed to cache candle", {
error: error instanceof Error ? error.message : String(error),
error: getErrorMessage(error),
trace_id: candle.trace_id,
underlying_id: candle.underlying_id,
interval_ms: candle.interval_ms
@ -242,8 +279,12 @@ const run = async () => {
try {
redis = createRedisClient(env.REDIS_URL);
redis.on("error", (error) => {
if (isExpectedShutdownError(error)) {
return;
}
logger.warn("redis client error", {
error: error instanceof Error ? error.message : String(error)
error: getErrorMessage(error)
});
});
await retry("redis connect", 20, 500, async () => {
@ -376,20 +417,51 @@ const run = async () => {
};
const shutdown = async (signal: string) => {
logger.info("service stopping", { signal });
clearInterval(flushTimer);
await flushExpired();
const remaining = aggregator.drain();
for (const candle of remaining) {
const validated = EquityCandleSchema.parse(candle);
await emitCandle(clickhouse, js, redis, validated, env.CANDLE_CACHE_LIMIT);
if (state.shutdownPromise) {
return state.shutdownPromise;
}
if (redis && redis.isOpen) {
await redis.quit();
}
await nc.drain();
await clickhouse.close();
process.exit(0);
state.shuttingDown = true;
state.shutdownPromise = (async () => {
logger.info("service stopping", { signal });
clearInterval(flushTimer);
await flushExpired();
const remaining = aggregator.drain();
for (const candle of remaining) {
const validated = EquityCandleSchema.parse(candle);
await emitCandle(clickhouse, js, redis, validated, env.CANDLE_CACHE_LIMIT);
}
if (redis && redis.isOpen) {
try {
await redis.quit();
} catch (error) {
if (!isExpectedShutdownError(error)) {
throw error;
}
}
}
try {
await nc.drain();
} catch (error) {
if (!isExpectedShutdownError(error)) {
throw error;
}
}
try {
await clickhouse.close();
} catch (error) {
if (!isExpectedShutdownError(error)) {
throw error;
}
}
process.exit(0);
})();
return state.shutdownPromise;
};
process.on("SIGINT", () => void shutdown("SIGINT"));

168
services/compute/src/index.ts
View file

@ -192,6 +192,31 @@ const roundTo = (value: number, digits = 4): number => {
return Number(value.toFixed(digits));
};
const getErrorCode = (error: unknown): string | null => {
if (error && typeof error === "object" && "code" in error) {
const code = (error as { code?: unknown }).code;
if (typeof code === "string" && code.length > 0) {
return code;
}
}
if (error instanceof Error) {
const match = error.message.match(/\bCONNECTION_(?:DRAINING|CLOSED)\b/);
if (match?.[0]) {
return match[0];
}
}
if (typeof error === "string") {
const match = error.match(/\bCONNECTION_(?:DRAINING|CLOSED)\b/);
if (match?.[0]) {
return match[0];
}
}
return null;
};
type NbboPlacement = "AA" | "A" | "B" | "BB" | "MID" | "MISSING" | "STALE";
type NbboPlacementCounts = {
@ -217,6 +242,7 @@ type ClusterState = {
firstPrice: number;
lastPrice: number;
placements: NbboPlacementCounts;
flushed: boolean;
};
const clusters = new Map<string, ClusterState>();
@ -226,6 +252,10 @@ const darkInferenceState = createDarkInferenceState();
const recentLegsByKey = new Map<string, LegEvidence[]>();
const recentLegsByRoot = new Map<string, LegEvidence[]>();
const recentStructureEmits = new Map<string, number>();
const runtimeState = {
shuttingDown: false,
shutdownPromise: null as Promise<void> | null
};
const MAX_RECENT_LEGS = 20;
@ -233,6 +263,15 @@ const rollingKey = (metric: string, contractId: string): string => {
return `rolling:${metric}:${contractId}`;
};
const buildPacketId = (cluster: ClusterState): string => {
return `flowpacket:${cluster.contractId}:${cluster.startTs}:${cluster.endTs}`;
};
const isExpectedShutdownNatsError = (error: unknown): boolean => {
const code = getErrorCode(error);
return runtimeState.shuttingDown && (code === "CONNECTION_DRAINING" || code === "CONNECTION_CLOSED");
};
const createPlacementCounts = (): NbboPlacementCounts => ({
aa: 0,
a: 0,
@ -501,7 +540,8 @@ const buildCluster = (print: OptionPrint): ClusterState => {
totalPremium: print.price * print.size,
firstPrice: print.price,
lastPrice: print.price,
placements
placements,
flushed: false
};
};
@ -613,8 +653,14 @@ const flushCluster = async (
rollingConfig: RollingStatsConfig,
cluster: ClusterState
): Promise<void> => {
if (cluster.flushed) {
return;
}
cluster.flushed = true;
const joinQuality: Record<string, number> = {};
const nbboJoin = selectNbbo(cluster.contractId, cluster.endTs);
const packetId = buildPacketId(cluster);
const totalPremium = roundTo(cluster.totalPremium);
const totalNotional = roundTo(totalPremium * 100, 2);
@ -777,25 +823,38 @@ const flushCluster = async (
source_ts: cluster.startSourceTs,
ingest_ts: cluster.endIngestTs,
seq: cluster.endSeq,
trace_id: `flowpacket:${cluster.contractId}:${cluster.startTs}:${cluster.endTs}`,
id: `flowpacket:${cluster.contractId}:${cluster.startTs}:${cluster.endTs}`,
trace_id: packetId,
id: packetId,
members: cluster.members,
features,
join_quality: joinQuality
};
const validated = FlowPacketSchema.parse(packet);
try {
await insertFlowPacket(clickhouse, validated);
await publishJson(js, SUBJECT_FLOW_PACKETS, validated);
await insertFlowPacket(clickhouse, validated);
await publishJson(js, SUBJECT_FLOW_PACKETS, validated);
await emitClassifiers(clickhouse, js, validated);
await emitClassifiers(clickhouse, js, validated);
logger.info("emitted flow packet", {
id: validated.id,
contract: cluster.contractId,
count: cluster.members.length
});
} catch (error) {
if (isExpectedShutdownNatsError(error)) {
logger.info("skipped flow packet publish during shutdown", {
id: packetId,
contract: cluster.contractId,
error: getErrorCode(error) ?? (error instanceof Error ? error.message : String(error))
});
return;
}
logger.info("emitted flow packet", {
id: validated.id,
contract: cluster.contractId,
count: cluster.members.length
});
cluster.flushed = false;
throw error;
}
};
const emitClassifiers = async (
@ -823,6 +882,9 @@ const emitClassifiers = async (
await insertClassifierHit(clickhouse, hit);
await publishJson(js, SUBJECT_CLASSIFIER_HITS, hit);
} catch (error) {
if (isExpectedShutdownNatsError(error)) {
continue;
}
logger.error("failed to emit classifier hit", {
error: error instanceof Error ? error.message : String(error),
classifier_id: hit.classifier_id,
@ -852,6 +914,9 @@ const emitClassifiers = async (
await insertAlert(clickhouse, alert);
await publishJson(js, SUBJECT_ALERTS, alert);
} catch (error) {
if (isExpectedShutdownNatsError(error)) {
return;
}
logger.error("failed to emit alert", {
error: error instanceof Error ? error.message : String(error),
packet_id: packet.id
@ -880,6 +945,9 @@ const emitEquityJoin = async (
try {
await publishJson(js, SUBJECT_EQUITY_JOINS, payload);
} catch (error) {
if (isExpectedShutdownNatsError(error)) {
return;
}
logger.error("failed to publish equity print join", {
error: error instanceof Error ? error.message : String(error),
trace_id: payload.trace_id
@ -901,6 +969,9 @@ const emitDarkInferences = async (
await insertInferredDark(clickhouse, validated);
await publishJson(js, SUBJECT_INFERRED_DARK, validated);
} catch (error) {
if (isExpectedShutdownNatsError(error)) {
continue;
}
logger.error("failed to emit inferred dark event", {
error: error instanceof Error ? error.message : String(error),
trace_id: validated.trace_id
@ -1366,6 +1437,10 @@ const run = async () => {
const nbboLoop = async () => {
for await (const msg of nbboSubscription.messages) {
if (runtimeState.shuttingDown) {
break;
}
try {
const nbbo = OptionNBBOSchema.parse(nbboSubscription.decode(msg));
updateNbboCache(nbbo);
@ -1381,6 +1456,10 @@ const run = async () => {
const equityQuoteLoop = async () => {
for await (const msg of equityQuoteSubscription.messages) {
if (runtimeState.shuttingDown) {
break;
}
try {
const quote = EquityQuoteSchema.parse(equityQuoteSubscription.decode(msg));
updateEquityQuoteCache(quote);
@ -1396,6 +1475,10 @@ const run = async () => {
const equityPrintLoop = async () => {
for await (const msg of equitySubscription.messages) {
if (runtimeState.shuttingDown) {
break;
}
try {
const print = EquityPrintSchema.parse(equitySubscription.decode(msg));
await emitEquityJoin(clickhouse, js, print);
@ -1414,23 +1497,64 @@ const run = async () => {
void equityPrintLoop();
const shutdown = async (signal: string) => {
logger.info("service stopping", { signal });
for (const cluster of clusters.values()) {
await flushCluster(clickhouse, js, redis, rollingConfig, cluster);
if (runtimeState.shutdownPromise) {
await runtimeState.shutdownPromise;
return;
}
clusters.clear();
await nc.drain();
await clickhouse.close();
await redis.quit();
process.exit(0);
runtimeState.shuttingDown = true;
runtimeState.shutdownPromise = (async () => {
logger.info("service stopping", { signal });
for (const cluster of [...clusters.values()]) {
await flushCluster(clickhouse, js, redis, rollingConfig, cluster);
}
clusters.clear();
try {
await nc.drain();
} catch (error) {
if (!isExpectedShutdownNatsError(error)) {
throw error;
}
}
await clickhouse.close();
if (redis.isOpen) {
await redis.quit();
}
})();
try {
await runtimeState.shutdownPromise;
process.exit(0);
} catch (error) {
logger.error("service shutdown failed", {
error: error instanceof Error ? error.message : String(error)
});
try {
await clickhouse.close();
} catch {}
try {
if (redis.isOpen) {
await redis.quit();
}
} catch {}
process.exit(1);
}
};
process.on("SIGINT", () => void shutdown("SIGINT"));
process.on("SIGTERM", () => void shutdown("SIGTERM"));
for await (const msg of subscription.messages) {
if (runtimeState.shuttingDown) {
break;
}
try {
const print = OptionPrintSchema.parse(subscription.decode(msg));
await flushEligibleClusters(
@ -1442,6 +1566,10 @@ const run = async () => {
print.option_contract_id
);
if (runtimeState.shuttingDown) {
break;
}
const existing = clusters.get(print.option_contract_id);
if (!existing) {
clusters.set(print.option_contract_id, buildCluster(print));

66
services/ingest-equities/src/index.ts
View file

@ -65,7 +65,28 @@ const envSchema = z.object({
const env = readEnv(envSchema);
const state = {
shuttingDown: false
shuttingDown: false,
shutdownPromise: null as Promise<void> | null
};
const getErrorMessage = (error: unknown): string => {
return error instanceof Error ? error.message : String(error);
};
const isExpectedShutdownError = (error: unknown): boolean => {
if (!state.shuttingDown) {
return false;
}
const message = getErrorMessage(error).toUpperCase();
return [
"SOCKET CONNECTION WAS CLOSED UNEXPECTEDLY",
"SOCKET CLOSED UNEXPECTEDLY",
"ECONNREFUSED",
"CONNECTION_CLOSED",
"CONNECTION_DRAINING",
"TIMEOUT"
].some((token) => message.includes(token));
};
const buildThrottle = (enabled: boolean, throttleMs: number) => {
@ -223,8 +244,12 @@ const run = async () => {
underlying_id: print.underlying_id
});
} catch (error) {
if (isExpectedShutdownError(error)) {
return;
}
logger.error("failed to publish equity print", {
error: error instanceof Error ? error.message : String(error),
error: getErrorMessage(error),
trace_id: print.trace_id
});
}
@ -245,8 +270,12 @@ const run = async () => {
await insertEquityQuote(clickhouse, quote);
await publishJson(js, SUBJECT_EQUITY_QUOTES, quote);
} catch (error) {
if (isExpectedShutdownError(error)) {
return;
}
logger.error("failed to publish equity quote", {
error: error instanceof Error ? error.message : String(error),
error: getErrorMessage(error),
trace_id: quote.trace_id
});
}
@ -254,18 +283,35 @@ const run = async () => {
});
const shutdown = async (signal: string) => {
if (state.shuttingDown) {
return;
if (state.shutdownPromise) {
return state.shutdownPromise;
}
state.shuttingDown = true;
await stopAdapter();
state.shutdownPromise = (async () => {
logger.info("service stopping", { signal });
await stopAdapter();
logger.info("service stopping", { signal });
try {
await nc.drain();
} catch (error) {
if (!isExpectedShutdownError(error)) {
throw error;
}
}
await nc.drain();
await clickhouse.close();
process.exit(0);
try {
await clickhouse.close();
} catch (error) {
if (!isExpectedShutdownError(error)) {
throw error;
}
}
process.exit(0);
})();
return state.shutdownPromise;
};
process.on("SIGINT", () => void shutdown("SIGINT"));

66
services/ingest-options/src/index.ts
View file

@ -88,7 +88,28 @@ const envSchema = z.object({
const env = readEnv(envSchema);
const state = {
shuttingDown: false
shuttingDown: false,
shutdownPromise: null as Promise<void> | null
};
const getErrorMessage = (error: unknown): string => {
return error instanceof Error ? error.message : String(error);
};
const isExpectedShutdownError = (error: unknown): boolean => {
if (!state.shuttingDown) {
return false;
}
const message = getErrorMessage(error).toUpperCase();
return [
"SOCKET CONNECTION WAS CLOSED UNEXPECTEDLY",
"SOCKET CLOSED UNEXPECTEDLY",
"ECONNREFUSED",
"CONNECTION_CLOSED",
"CONNECTION_DRAINING",
"TIMEOUT"
].some((token) => message.includes(token));
};
const buildThrottle = (enabled: boolean, throttleMs: number) => {
@ -293,8 +314,12 @@ const run = async () => {
option_contract_id: print.option_contract_id
});
} catch (error) {
if (isExpectedShutdownError(error)) {
return;
}
logger.error("failed to publish option print", {
error: error instanceof Error ? error.message : String(error),
error: getErrorMessage(error),
trace_id: print.trace_id
});
}
@ -315,8 +340,12 @@ const run = async () => {
await insertOptionNBBO(clickhouse, nbbo);
await publishJson(js, SUBJECT_OPTION_NBBO, nbbo);
} catch (error) {
if (isExpectedShutdownError(error)) {
return;
}
logger.error("failed to publish option nbbo", {
error: error instanceof Error ? error.message : String(error),
error: getErrorMessage(error),
trace_id: nbbo.trace_id
});
}
@ -324,18 +353,35 @@ const run = async () => {
});
const shutdown = async (signal: string) => {
if (state.shuttingDown) {
return;
if (state.shutdownPromise) {
return state.shutdownPromise;
}
state.shuttingDown = true;
await stopAdapter();
state.shutdownPromise = (async () => {
logger.info("service stopping", { signal });
await stopAdapter();
logger.info("service stopping", { signal });
try {
await nc.drain();
} catch (error) {
if (!isExpectedShutdownError(error)) {
throw error;
}
}
await nc.drain();
await clickhouse.close();
process.exit(0);
try {
await clickhouse.close();
} catch (error) {
if (!isExpectedShutdownError(error)) {
throw error;
}
}
process.exit(0);
})();
return state.shutdownPromise;
};
process.on("SIGINT", () => void shutdown("SIGINT"));