From 53eeb9e72f9e4e2f33941f28b167a10ef0a95dc7 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Wed, 6 May 2026 22:14:11 -0400 Subject: [PATCH] Gate live feed staleness and isolate Next dev artifacts - delay stale status for paused live feeds before surfacing disconnects - keep `next dev` output separate from production build artifacts - add coverage for the new live-feed stale threshold --- .beads/issues.jsonl | 1 + .gitignore | 1 + apps/web/app/terminal.test.ts | 5 +++ apps/web/app/terminal.tsx | 23 +++++++--- apps/web/next.config.mjs | 16 +++++++ apps/web/scripts/dev.ts | 8 ++++ apps/web/tsconfig.json | 3 +- plans/nextjs-upgrade-plan.md | 79 +++++++++++++++++++++++++++++++++++ 8 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 apps/web/next.config.mjs create mode 100644 plans/nextjs-upgrade-plan.md diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 91007f6..a281161 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-84s","title":"Implement seamless /tape live-to-history scroll gate","description":"Implement seamless live-to-ClickHouse scroll-gated history for /tape panes, including split live/history buffers in the web client, snapshot_limit support on live subscriptions, a bundled options support lookup endpoint, ClickHouse helpers for parity hydration, and test coverage for live head retention, background history loading, scoped options deep-hydration, and historical options decor restoration.\n","status":"in_progress","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T02:10:43Z","created_by":"dirtydishes","updated_at":"2026-05-07T02:10:47Z","started_at":"2026-05-07T02:10:47Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-sh1","title":"Fix live websocket stale lag and reconnect loop","description":"Investigate and fix API live consumer lag causing stale timestamps, feed-behind status, and reconnect loops. Optimize live cache persistence path, add lag telemetry/alerts, and validate in runtime.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T17:04:34Z","created_by":"dirtydishes","updated_at":"2026-05-04T17:09:44Z","started_at":"2026-05-04T17:04:38Z","closed_at":"2026-05-04T17:09:44Z","close_reason":"Completed: optimized live cache persistence path, added lag telemetry, deployed api via docker compose on di, verified ws freshness and low hotFeedLagMs","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-b3o","title":"Implement options tape table with execution spot","description":"Redesign OptionsPane into a dense classifier-colored table and preserve execution-time underlying spot on option prints from equity quote mid.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:41:59Z","created_by":"dirtydishes","updated_at":"2026-05-04T05:14:26Z","started_at":"2026-05-04T04:42:08Z","closed_at":"2026-05-04T05:14:26Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-ug1","title":"Fix false NBBO-missing badges in live Options tape","description":"Investigate and fix client-side cases where Options rows show NBBO missing/stale even when a fresh NBBO quote exists in the live nbbo map. Update rendering logic to prefer fresh quote-derived status and add regression tests.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-29T15:58:31Z","created_by":"dirtydishes","updated_at":"2026-04-29T16:01:28Z","started_at":"2026-04-29T15:58:35Z","closed_at":"2026-04-29T16:01:28Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/.gitignore b/.gitignore index 000f48c..1ee09a8 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ coverage/ logs/ .tmp/ apps/web/.next/ +apps/web/.next-dev/ # Local assistant artifacts session-ses_*.md diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index 48703d8..2071762 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -185,6 +185,11 @@ describe("live tape pausable helpers", () => { expect(getLiveFeedStatus("disconnected", 1000, 500, 1601)).toBe("disconnected"); }); + it("waits for an additional behind-delay before surfacing stale", () => { + expect(getLiveFeedStatus("connected", 1000, 500, 2000, 15_000)).toBe("connected"); + expect(getLiveFeedStatus("connected", 1000, 500, 16_501, 15_000)).toBe("stale"); + }); + it("keeps visible history even when live status is stale", () => { const projected = projectPausableTapeState([makeItem("stale", 7, 1000)], "stale", 2000); expect(projected.items.map((item) => item.trace_id)).toEqual(["stale"]); diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 87b5776..0a4bb56 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -72,6 +72,7 @@ const LIVE_HOT_WINDOW_OPTIONS = parseBoundedInt( const LIVE_OPTIONS_STALE_MS = 15_000; const LIVE_NBBO_STALE_MS = 15_000; const LIVE_EQUITIES_STALE_MS = 15_000; +const LIVE_FEED_BEHIND_DELAY_MS = 15_000; const LIVE_EQUITIES_SILENT_WARNING_MS = parseBoundedInt( process.env.NEXT_PUBLIC_LIVE_EQUITIES_SILENT_WARNING_MS, 25_000, @@ -491,7 +492,8 @@ export const getLiveFeedStatus = ( sourceStatus: WsStatus, freshestTs: number | null, thresholdMs: number, - now = Date.now() + now = Date.now(), + behindDelayMs = 0 ): WsStatus => { if (sourceStatus !== "connected") { return sourceStatus; @@ -499,7 +501,14 @@ export const getLiveFeedStatus = ( if (freshestTs === null) { return "connected"; } - return isFreshLiveItem(freshestTs, thresholdMs, now) ? "connected" : "stale"; + + const ageMs = now - freshestTs; + if (ageMs <= thresholdMs) { + return "connected"; + } + + const behindMs = ageMs - thresholdMs; + return behindMs > behindDelayMs ? "stale" : "connected"; }; type TapeState = { @@ -945,8 +954,6 @@ export const countActiveFlowFilterGroups = (filters: OptionFlowFilters): number return count; }; -const isFreshLiveItem = (ts: number, thresholdMs: number, now = Date.now()): boolean => now - ts <= thresholdMs; - export const toggleFilterValue = ( values: T[] | undefined, value: T, @@ -1995,7 +2002,13 @@ const usePausableTapeView = ( }, [config.sourceItems, getItemTs]); const status = config.enabled - ? getLiveFeedStatus(config.sourceStatus, freshestTs, config.freshnessMs, clock) + ? getLiveFeedStatus( + config.sourceStatus, + freshestTs, + config.freshnessMs, + clock, + LIVE_FEED_BEHIND_DELAY_MS + ) : "disconnected"; const projected = projectPausableTapeState(data.visible, status, config.lastUpdate); diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs new file mode 100644 index 0000000..ae6d971 --- /dev/null +++ b/apps/web/next.config.mjs @@ -0,0 +1,16 @@ +import { PHASE_DEVELOPMENT_SERVER } from "next/constants.js"; + +/** + * Keep dev and production build artifacts separate to avoid chunk/runtime + * mismatches when `next dev` and `next build` are run in overlapping sessions. + * + * @param {string} phase + * @returns {import("next").NextConfig} + */ +export default function nextConfig(phase) { + const isDev = phase === PHASE_DEVELOPMENT_SERVER; + + return { + distDir: isDev ? ".next-dev" : ".next" + }; +} diff --git a/apps/web/scripts/dev.ts b/apps/web/scripts/dev.ts index f194182..985f6e6 100644 --- a/apps/web/scripts/dev.ts +++ b/apps/web/scripts/dev.ts @@ -1,9 +1,17 @@ +import { rm } from "node:fs/promises"; + const run = async () => { const port = 3000; + const distDir = ".next-dev"; console.log(`[web] starting Next.js dev server on port ${port}`); const path = Bun.env.PATH ?? ""; const cwd = `${import.meta.dir}/..`; + const distPath = `${cwd}/${distDir}`; + + // Clear potentially stale dev artifacts from interrupted prior runs. + await rm(distPath, { recursive: true, force: true }); + console.log(`[web] cleared stale Next.js dev artifacts at ${distDir}`); const child = Bun.spawn(["next", "dev", "-p", String(port)], { cwd, diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index bdf5e1a..819bfbe 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -21,7 +21,8 @@ "next-env.d.ts", "**/*.ts", "**/*.tsx", - ".next/types/**/*.ts" + ".next/types/**/*.ts", + ".next-dev/types/**/*.ts" ], "exclude": [ "node_modules", diff --git a/plans/nextjs-upgrade-plan.md b/plans/nextjs-upgrade-plan.md new file mode 100644 index 0000000..f6e4a20 --- /dev/null +++ b/plans/nextjs-upgrade-plan.md @@ -0,0 +1,79 @@ +# Next.js Upgrade Plan: `14.2.35` -> `16.2.4` (via v15 compatibility pass) + +## Summary + +As of **May 5, 2026**, the web app is on Next `14.2.35` (locked) with React `18.3.1`, while npm `latest` for `next` is `16.2.4`. + +Based on repo inspection, risk is **moderate-low** for this app because it uses App Router with minimal server-only APIs and no custom webpack/middleware. The main required migration is the React 19 + Next 15/16 compatibility surface. + +## Current-State Findings + +- Declared deps: [apps/web/package.json](/Users/kell/Cloud/dev/islandflow/apps/web/package.json:13) +- Locked Next version: [bun.lock](/Users/kell/Cloud/dev/islandflow/bun.lock:241) +- Next config is simple (`distDir` only), no webpack/turbopack overrides: [apps/web/next.config.mjs](/Users/kell/Cloud/dev/islandflow/apps/web/next.config.mjs:1) +- App Router only (`app/*`), no route handlers or middleware in `apps/web` +- No `cookies()`, `headers()`, `draftMode()`, or `params/searchParams` async-migration hotspots found in `apps/web/app` +- Baseline build passes on current branch (`bun --cwd=apps/web run build`) + +## Public APIs / Interfaces / Types Impact + +- External API contracts for the project: **no intentional changes** +- Dependency interface upgrades required: +- `next` -> `16.2.4` +- `react` / `react-dom` -> `19.x` +- `@types/react` / `@types/react-dom` -> latest 19-compatible +- If future server components introduce request APIs, they must follow async forms from v15+ (`await cookies()`, etc.) + +## Implementation Plan + +1. Create an upgrade branch and snapshot baseline: + - Record current `bun --cwd=apps/web run build` and `bun test` status. +2. Upgrade deps in `apps/web`: + - Bump `next`, `react`, `react-dom`, and React type packages to latest compatible. + - Run install and refresh lockfile. +3. Run codemod-assisted checks: + - Use Next codemod guidance for v15/v16 migration candidates. + - Verify no required transforms are missed, especially async request APIs and config migrations. +4. Validate Next 16 runtime/build behavior: + - `bun --cwd=apps/web run build` + - `bun --cwd=apps/web run dev` smoke test for `/`, `/tape`, and redirect routes. +5. Validate tests: + - `bun test apps/web/app/routes.test.ts` + - `bun test apps/web/app/terminal.test.ts` + - `bun test` repo-wide if CI parity is expected. +6. Fix issues discovered in validation: + - Resolve React 19 typing/hook warnings if any appear. + - Confirm no changed behavior in navigation/replay/tape flows. +7. Final verification: + - Re-run build and relevant tests. + - Capture upgrade notes, what changed, what was checked, and residual risk. + +## Test Cases and Scenarios + +- Build: + - Production build succeeds (`next build`) with Next 16 + React 19. +- Routing: + - `/` and `/tape` render correctly. + - `/signals`, `/charts`, `/replay` still redirect to `/`. +- Client navigation/cache behavior: + - `` navigation between Home/Tape remains correct under updated client cache semantics. +- Live/replay terminal UI: + - No regressions in fetch-driven panels and websocket-driven status behavior. +- Type safety: + - No TypeScript errors from React 19 types in terminal-heavy UI code. +- Regression check: + - Existing Bun tests continue to pass. + +## Assumptions and Defaults Chosen + +- Target selected: **Next 16 latest** +- Default strategy: **single upgrade stream with v15 compatibility checks included**, not a prolonged 14 -> 15 -> 16 rollout, because the code has low exposure to v15 breaking server APIs +- No custom webpack migration required unless hidden plugin behavior introduces it +- No expected changes to backend service contracts or shared `@islandflow/types` interfaces + +## Official References + +- Next 15 upgrade guide: https://nextjs.org/docs/app/guides/upgrading/version-15 +- Next 16 upgrade guide: https://nextjs.org/docs/app/guides/upgrading/version-16 +- Next 15 release notes: https://nextjs.org/blog/next-15 +- npm package page: https://www.npmjs.com/package/next