Implement first-pass load reduction controls

This commit is contained in:
dirtydishes 2026-05-08 02:46:41 -04:00
parent 5d488fd7f5
commit e7f4805ccc
17 changed files with 1191 additions and 608 deletions

View file

@ -91,6 +91,7 @@ ALPHA_VANTAGE_EARNINGS_SYMBOL=
REFDATA_EVENT_CALENDAR_REFRESH_MS=86400000
# Replay service
LOG_LEVEL=info
REPLAY_ENABLED=false
REPLAY_STREAMS=options,nbbo,equities,equity-quotes
REPLAY_START_TS=0
@ -100,12 +101,33 @@ REPLAY_BATCH_SIZE=200
REPLAY_LOG_EVERY=1000
# API live retention (generic channels)
LIVE_LIMIT_OPTIONS=2000
LIVE_LIMIT_NBBO=10000
LIVE_LIMIT_EQUITIES=2000
LIVE_LIMIT_EQUITY_QUOTES=10000
LIVE_LIMIT_EQUITY_JOINS=10000
LIVE_LIMIT_FLOW=2000
LIVE_LIMIT_CLASSIFIER_HITS=10000
LIVE_LIMIT_ALERTS=10000
LIVE_LIMIT_INFERRED_DARK=10000
LIVE_LIMIT_DEFAULT=1000
LIVE_LIMIT_OPTIONS=1000
LIVE_LIMIT_NBBO=1000
LIVE_LIMIT_EQUITIES=1000
LIVE_LIMIT_EQUITY_QUOTES=500
LIVE_LIMIT_EQUITY_JOINS=500
LIVE_LIMIT_FLOW=500
LIVE_LIMIT_SMART_MONEY=300
LIVE_LIMIT_CLASSIFIER_HITS=300
LIVE_LIMIT_ALERTS=300
LIVE_LIMIT_INFERRED_DARK=300
LIVE_SCOPED_CACHE_MAX_KEYS=32
LIVE_REDIS_FLUSH_INTERVAL_MS=250
LIVE_REDIS_FLUSH_MAX_ITEMS=100
# Compute rolling/cache retention
ROLLING_CACHE_FLUSH_INTERVAL_MS=30000
ROLLING_CACHE_MAX_KEYS=20000
COMPUTE_NBBO_CACHE_MAX_KEYS=20000
COMPUTE_NBBO_CACHE_TTL_MS=900000
# Ingest context retention
OPTION_CONTEXT_MAX_KEYS=20000
OPTION_CONTEXT_TTL_MS=900000
# JetStream retention
STREAM_RAW_MAX_AGE_MS=7200000
STREAM_RAW_MAX_BYTES=1073741824
STREAM_DERIVED_MAX_AGE_MS=86400000
STREAM_DERIVED_MAX_BYTES=536870912