From dc0aeaa7d2309ce3be63b70d3405da54170c3e3f Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Thu, 7 May 2026 02:08:02 -0400 Subject: [PATCH] Fix Docker workspace lockfile drift and add sync guard --- .beads/issues.jsonl | 1 + deployment/docker/README.md | 17 ++ deployment/docker/workspace-root/bun.lock | 6 + deployment/docker/workspace-root/package.json | 4 +- .../docker/workspace-root/tsconfig.base.json | 4 +- package.json | 4 +- scripts/check-docker-workspace.ts | 244 ++++++++++++++++++ scripts/sync-docker-workspace.ts | 19 ++ 8 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 scripts/check-docker-workspace.ts create mode 100644 scripts/sync-docker-workspace.ts diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index b7f0a79..9229f49 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-35g","title":"Fix Docker deployment workspace lockfile drift","description":"Refresh deployment/docker workspace lockfile for Docker builds, add a drift guard for Docker-built workspaces, and document the separate deployment snapshot so frozen Bun installs cannot fail when repo dependencies change.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T06:02:06Z","created_by":"dirtydishes","updated_at":"2026-05-07T06:07:50Z","started_at":"2026-05-07T06:02:15Z","closed_at":"2026-05-07T06:07:50Z","close_reason":"Completed: synced deployment Docker workspace snapshot from repo root, refreshed deployment bun.lock, added sync/check scripts, and documented maintenance workflow. Local docker compose build validation is blocked here because Docker daemon is unavailable.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-2ij","title":"Harden tape virtualization, scoped focus, and live feed health","description":"Implement the coordinated tape stability plan across web and API.\n\nScope:\n- replace fixed-height tape virtualization with measured virtualization and virtual-end history loading\n- replace scrollHeight anchoring with key-based anchor restore\n- compose canonical tape lists across seed/live/history sources\n- preserve clicked contract/ticker context during scoped focus transitions\n- separate backend hot-channel health from scoped quiet empty states\n- shrink browser hot windows and modestly reduce server cache limits\n- add regression tests and development instrumentation\n\nAcceptance:\n- no giant blank spacer gaps during tape scrolling\n- scroll remains stable while live data and history mutate the list\n- clicked deep-history option/equity rows remain visible immediately after focus\n- narrow scopes do not surface Feed behind unless backend channel health is stale\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T05:35:18Z","created_by":"dirtydishes","updated_at":"2026-05-07T05:52:14Z","started_at":"2026-05-07T05:35:21Z","closed_at":"2026-05-07T05:52:14Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-uj7","title":"Fix home to tape navigation","description":"Home rail Tape navigation was not reliably switching to the tape route. Use browser-native top-level navigation for Home/Tape rail links so /tape remains reachable even if client router handling stalls.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T03:18:14Z","created_by":"dirtydishes","updated_at":"2026-05-07T03:18:21Z","started_at":"2026-05-07T03:18:20Z","closed_at":"2026-05-07T03:18:21Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-84s","title":"Implement seamless /tape live-to-history scroll gate","description":"Implement seamless live-to-ClickHouse scroll-gated history for /tape panes, including split live/history buffers in the web client, snapshot_limit support on live subscriptions, a bundled options support lookup endpoint, ClickHouse helpers for parity hydration, and test coverage for live head retention, background history loading, scoped options deep-hydration, and historical options decor restoration.\n","status":"in_progress","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-07T02:10:43Z","created_by":"dirtydishes","updated_at":"2026-05-07T02:10:47Z","started_at":"2026-05-07T02:10:47Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/deployment/docker/README.md b/deployment/docker/README.md index de7c805..dca5fbe 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -21,6 +21,7 @@ It is separate from the repo-root `docker-compose.yml`, which is still the light - `deployment/docker/Dockerfile.service`: shared Bun runtime image for most services - `deployment/docker/Dockerfile.ingest-options`: Bun runtime plus Python dependencies for Databento and IBKR adapters - `deployment/docker/Dockerfile.web`: multi-stage build for the Next.js web app +- `deployment/docker/workspace-root/`: deployment-specific workspace snapshot (`package.json`, `tsconfig.base.json`, `bun.lock`) used by Docker builds - `deployment/docker/clickhouse/listen.xml`: forces ClickHouse to listen on IPv4 for other containers on the Docker network - `deployment/docker/.env.example`: container-oriented environment template @@ -185,6 +186,22 @@ If NPM is on multiple networks and names collide (for example another stack also ## Updating the deployment +This deployment installs dependencies from `deployment/docker/workspace-root/bun.lock` (not the repo-root lockfile). + +When dependencies change in any workspace used by Docker builds, refresh and validate the deployment snapshot first: + +```bash +bun run sync:docker-workspace +bun run check:docker-workspace +``` + +Then validate the VPS build path: + +```bash +cd deployment/docker +docker compose build web +``` + When you pull new code: ```bash diff --git a/deployment/docker/workspace-root/bun.lock b/deployment/docker/workspace-root/bun.lock index d6e99c6..47fc572 100644 --- a/deployment/docker/workspace-root/bun.lock +++ b/deployment/docker/workspace-root/bun.lock @@ -12,6 +12,7 @@ "name": "@islandflow/web", "dependencies": { "@islandflow/types": "workspace:*", + "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", "next": "^14.2.4", "react": "^18.3.1", @@ -81,6 +82,7 @@ "@islandflow/bus": "workspace:*", "@islandflow/config": "workspace:*", "@islandflow/observability": "workspace:*", + "@islandflow/refdata": "workspace:*", "@islandflow/storage": "workspace:*", "@islandflow/types": "workspace:*", "redis": "^5.10.0", @@ -207,6 +209,10 @@ "@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="], + "@tanstack/react-virtual": ["@tanstack/react-virtual@3.13.24", "", { "dependencies": { "@tanstack/virtual-core": "3.14.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aIJvz5OSkhNIhZIpYivrxrPTKYsjW9Uzy+sP/mx0S3sev2HyvPb7xmjbYvokzEpfgYHy/HjzJ2zFAETuUfgCpg=="], + + "@tanstack/virtual-core": ["@tanstack/virtual-core@3.14.0", "", {}, "sha512-JLANqGy/D6k4Ujmh8Tr25lGimuOXNiaVyXaCAZS0W+1390sADdGnyUdSWNIfd49gebtIxGMij4IktRVzrdr12Q=="], + "@types/node": ["@types/node@20.19.27", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-N2clP5pJhB2YnZJ3PIHFk5RkygRX5WO/5f0WC08tp0wd+sv0rsJk3MqWn3CbNmT2J505a5336jaQj4ph1AdMug=="], "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], diff --git a/deployment/docker/workspace-root/package.json b/deployment/docker/workspace-root/package.json index 0d570a9..8240012 100644 --- a/deployment/docker/workspace-root/package.json +++ b/deployment/docker/workspace-root/package.json @@ -12,7 +12,9 @@ "dev:infra": "docker compose up", "dev:infra:down": "docker compose down", "dev:web": "bun --cwd=apps/web run dev", - "dev:services": "bun run scripts/dev-services.ts" + "dev:services": "bun run scripts/dev-services.ts", + "sync:docker-workspace": "bun run scripts/sync-docker-workspace.ts", + "check:docker-workspace": "bun run scripts/check-docker-workspace.ts" }, "devDependencies": { "typescript-language-server": "^5.1.3" diff --git a/deployment/docker/workspace-root/tsconfig.base.json b/deployment/docker/workspace-root/tsconfig.base.json index f98f46a..34b15d2 100644 --- a/deployment/docker/workspace-root/tsconfig.base.json +++ b/deployment/docker/workspace-root/tsconfig.base.json @@ -8,6 +8,6 @@ "isolatedModules": true, "resolveJsonModule": true, "skipLibCheck": true, - "noEmit": true - } + "noEmit": true, + }, } diff --git a/package.json b/package.json index 0d570a9..8240012 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,9 @@ "dev:infra": "docker compose up", "dev:infra:down": "docker compose down", "dev:web": "bun --cwd=apps/web run dev", - "dev:services": "bun run scripts/dev-services.ts" + "dev:services": "bun run scripts/dev-services.ts", + "sync:docker-workspace": "bun run scripts/sync-docker-workspace.ts", + "check:docker-workspace": "bun run scripts/check-docker-workspace.ts" }, "devDependencies": { "typescript-language-server": "^5.1.3" diff --git a/scripts/check-docker-workspace.ts b/scripts/check-docker-workspace.ts new file mode 100644 index 0000000..bc0d33e --- /dev/null +++ b/scripts/check-docker-workspace.ts @@ -0,0 +1,244 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +type DependencyMap = Record; + +type LockWorkspace = { + name?: string; + dependencies?: DependencyMap; + devDependencies?: DependencyMap; + optionalDependencies?: DependencyMap; + peerDependencies?: DependencyMap; +}; + +type BunLock = { + lockfileVersion?: number; + configVersion?: number; + workspaces?: Record; + packages?: Record; +}; + +type RootPackageManifest = { + workspaces?: string[]; +}; + +const repoRoot = path.resolve(import.meta.dir, ".."); +const deploymentRoot = path.join(repoRoot, "deployment/docker/workspace-root"); + +const rootPackagePath = path.join(repoRoot, "package.json"); +const deploymentPackagePath = path.join(deploymentRoot, "package.json"); +const rootTsconfigPath = path.join(repoRoot, "tsconfig.base.json"); +const deploymentTsconfigPath = path.join(deploymentRoot, "tsconfig.base.json"); +const rootLockPath = path.join(repoRoot, "bun.lock"); +const deploymentLockPath = path.join(deploymentRoot, "bun.lock"); + +const readUtf8 = async (filePath: string): Promise => { + return readFile(filePath, "utf8"); +}; + +const parseObjectLiteral = async (filePath: string): Promise => { + const raw = await readUtf8(filePath); + try { + const parsed = Function(`"use strict"; return (${raw});`)() as T; + return parsed; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to parse ${filePath}: ${message}`); + } +}; + +const stableSortObject = (value: unknown): unknown => { + if (Array.isArray(value)) { + return value.map(stableSortObject); + } + if (value && typeof value === "object") { + const entries = Object.entries(value as Record) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, nested]) => [key, stableSortObject(nested)] as const); + return Object.fromEntries(entries); + } + return value; +}; + +const stableStringify = (value: unknown): string => { + return JSON.stringify(stableSortObject(value)); +}; + +const listWorkspacePaths = async (workspacePatterns: string[]): Promise => { + const paths = new Set(); + + for (const pattern of workspacePatterns) { + const globPattern = pattern.endsWith("/") ? `${pattern}package.json` : `${pattern}/package.json`; + const glob = new Bun.Glob(globPattern); + for await (const match of glob.scan({ cwd: repoRoot })) { + const normalized = match.replaceAll("\\", "/"); + paths.add(path.posix.dirname(normalized)); + } + } + + return Array.from(paths).sort((a, b) => a.localeCompare(b)); +}; + +const normalizedDependencyMap = (input: DependencyMap | undefined): DependencyMap => { + if (!input) { + return {}; + } + return Object.fromEntries( + Object.entries(input) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, version]) => [name, version]) + ); +}; + +const formatDependencyDiff = ( + workspacePath: string, + section: string, + expected: DependencyMap, + actual: DependencyMap +): string[] => { + const issues: string[] = []; + const expectedKeys = new Set(Object.keys(expected)); + const actualKeys = new Set(Object.keys(actual)); + + for (const key of expectedKeys) { + if (!actualKeys.has(key)) { + issues.push(`${workspacePath} ${section}: missing ${key}@${expected[key]}`); + continue; + } + if (expected[key] !== actual[key]) { + issues.push( + `${workspacePath} ${section}: ${key} expected ${expected[key]} but found ${actual[key]}` + ); + } + } + + for (const key of actualKeys) { + if (!expectedKeys.has(key)) { + issues.push(`${workspacePath} ${section}: extra ${key}@${actual[key]}`); + } + } + + return issues; +}; + +const check = async (): Promise => { + const issues: string[] = []; + + const [rootPackage, deploymentPackage, rootTsconfig, deploymentTsconfig, rootLock, deploymentLock] = + await Promise.all([ + parseObjectLiteral(rootPackagePath), + parseObjectLiteral(deploymentPackagePath), + parseObjectLiteral(rootTsconfigPath), + parseObjectLiteral(deploymentTsconfigPath), + parseObjectLiteral(rootLockPath), + parseObjectLiteral(deploymentLockPath) + ]); + + const rootPackageSnapshot = stableStringify(rootPackage); + const deploymentPackageSnapshot = stableStringify(deploymentPackage); + if (rootPackageSnapshot !== deploymentPackageSnapshot) { + issues.push( + "deployment/docker/workspace-root/package.json does not match repo-root package.json" + ); + } + + const rootTsconfigSnapshot = stableStringify(rootTsconfig); + const deploymentTsconfigSnapshot = stableStringify(deploymentTsconfig); + if (rootTsconfigSnapshot !== deploymentTsconfigSnapshot) { + issues.push( + "deployment/docker/workspace-root/tsconfig.base.json does not match repo-root tsconfig.base.json" + ); + } + + const rootWorkspaces = rootLock.workspaces ?? {}; + const deploymentWorkspaces = deploymentLock.workspaces ?? {}; + + const workspacePatterns = rootPackage.workspaces ?? []; + const workspacePackagePaths = await listWorkspacePaths(workspacePatterns); + for (const workspacePath of workspacePackagePaths) { + const packageJsonPath = path.join(repoRoot, workspacePath, "package.json"); + const workspacePackage = (await parseObjectLiteral(packageJsonPath)) as LockWorkspace; + const deploymentWorkspace = deploymentWorkspaces[workspacePath]; + + if (!deploymentWorkspace) { + issues.push(`deployment lock is missing workspace entry: ${workspacePath}`); + continue; + } + + const sections: Array = [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies" + ]; + for (const section of sections) { + const expectedMap = normalizedDependencyMap(workspacePackage[section] as DependencyMap | undefined); + const actualMap = normalizedDependencyMap( + deploymentWorkspace[section] as DependencyMap | undefined + ); + issues.push(...formatDependencyDiff(workspacePath, section, expectedMap, actualMap)); + } + } + + const workspacePaths = Array.from( + new Set([...Object.keys(rootWorkspaces), ...Object.keys(deploymentWorkspaces)]) + ).sort((a, b) => a.localeCompare(b)); + + for (const workspacePath of workspacePaths) { + const rootWorkspace = rootWorkspaces[workspacePath]; + const deploymentWorkspace = deploymentWorkspaces[workspacePath]; + + if (!rootWorkspace) { + issues.push(`deployment lock has unexpected workspace entry: ${workspacePath}`); + continue; + } + if (!deploymentWorkspace) { + issues.push(`deployment lock is missing workspace entry: ${workspacePath}`); + continue; + } + + if ((rootWorkspace.name ?? "") !== (deploymentWorkspace.name ?? "")) { + issues.push( + `${workspacePath} name mismatch: expected ${rootWorkspace.name ?? "(none)"} but found ${ + deploymentWorkspace.name ?? "(none)" + }` + ); + } + + const sections: Array = [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies" + ]; + for (const section of sections) { + const expectedMap = normalizedDependencyMap(rootWorkspace[section] as DependencyMap | undefined); + const actualMap = normalizedDependencyMap( + deploymentWorkspace[section] as DependencyMap | undefined + ); + issues.push(...formatDependencyDiff(workspacePath, section, expectedMap, actualMap)); + } + } + + const rootPackagesSnapshot = stableStringify(rootLock.packages ?? {}); + const deploymentPackagesSnapshot = stableStringify(deploymentLock.packages ?? {}); + if (rootPackagesSnapshot !== deploymentPackagesSnapshot) { + issues.push( + "deployment/docker/workspace-root/bun.lock package resolutions differ from repo-root bun.lock" + ); + } + + if (issues.length > 0) { + console.error("Docker workspace snapshot is out of sync:"); + for (const issue of issues) { + console.error(`- ${issue}`); + } + console.error("Run: bun run sync:docker-workspace"); + return 1; + } + + console.log("Docker workspace snapshot is in sync."); + return 0; +}; + +process.exitCode = await check(); diff --git a/scripts/sync-docker-workspace.ts b/scripts/sync-docker-workspace.ts new file mode 100644 index 0000000..e20b293 --- /dev/null +++ b/scripts/sync-docker-workspace.ts @@ -0,0 +1,19 @@ +import { copyFile } from "node:fs/promises"; +import path from "node:path"; + +const repoRoot = path.resolve(import.meta.dir, ".."); +const deploymentRoot = path.join(repoRoot, "deployment/docker/workspace-root"); + +const filesToSync = [ + "package.json", + "bun.lock", + "tsconfig.base.json" +] as const; + +for (const fileName of filesToSync) { + const source = path.join(repoRoot, fileName); + const destination = path.join(deploymentRoot, fileName); + await copyFile(source, destination); + console.log(`synced ${fileName}`); +} +