dirtydishes/islandflow
1
0
Fork 0
Code Issues 2 Pull requests 4 Projects Releases Packages Wiki Activity Actions 1

speed up docker deploy builds

Browse source
This commit is contained in:
dirtydishes 2026-05-16 17:54:00 -04:00
parent 2abdd24e2c
commit 23ed3809cc
7 changed files with 349 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.beads/issues.jsonl
View file

@ -11,6 +11,7 @@
{"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-09a","title":"Speed up Docker deployment builds","description":"Implement the Docker deployment optimization plan from /Users/kell/Desktop/speed-up-docker.md: split dependency installation from source copy, add BuildKit caches, make scoped deploys build only their target services, update Docker deployment docs, validate, document the turn, commit, and push.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:50:24Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:53:48Z","started_at":"2026-05-16T21:50:37Z","closed_at":"2026-05-16T21:53:48Z","close_reason":"Implemented Docker dependency-layer caching, scoped deploy build/up flow, Docker docs updates, validation, and turn documentation. Follow-up islandflow-cnk tracks daemon-backed image build verification.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-0sa","title":"Fix live tape auto-hold, history seam, and remove manual pause control","description":"The live tape should automatically hold when the user scrolls away from the top, resume when they return to the top or use Jump to top, and keep older prints available seamlessly beyond the hot window. Manual Pause/Resume control is now redundant and should be removed from live tape panes. This work should also fix the current regression where paused/held tapes still mutate, and align the options tape with a strict 100-row hot head backed by ClickHouse history.","notes":"Implemented live scroll-hold with no live pause button, demand-loaded ClickHouse history, a 100-row options hot head, and cache-first scoped snapshots. Validated with bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts and bun --cwd=apps/web run build.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T18:12:51Z","created_by":"dirtydishes","updated_at":"2026-05-16T18:23:43Z","started_at":"2026-05-16T18:12:54Z","closed_at":"2026-05-16T18:23:43Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-2db","title":"Manually remove stale islandflow local-infra containers from VPS","description":"The live VPS still has an older compose project named islandflow created from the repo-root docker-compose.yml. Inspection shows it is separate from the supported islandflow-vps deployment stack and exposes NATS, ClickHouse, and Redis on host ports. Container removal commands currently hang when run as the delta user through Docker, so cleanup likely needs a focused maintenance window and possibly host-level intervention or a Docker daemon restart.","notes":"The duplicate islandflow compose project on the VPS was confirmed live during inspection. Nginx Proxy Manager routes public traffic only to islandflow-vps web/api by Docker name, so the stale islandflow project appears to be stray local-infra state rather than part of the supported production path. Attempts to remove the stale containers with docker compose down and docker rm -f as the delta user hung and timed out, so manual cleanup likely needs a maintenance window and possibly Docker daemon intervention.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:27:27Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:28:59Z","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-c87","title":"Clean up duplicate Islandflow Docker infra on VPS","description":"The live VPS is currently running both the production-style islandflow-vps Docker stack and an older root-level islandflow infra stack that publishes NATS, ClickHouse, and Redis on host ports. Investigate whether the older stack is unused, remove it safely if so, and update docs/deploy guidance so the server topology is clearer.","notes":"Inspected the live VPS and confirmed the duplicate compose project: islandflow-vps is the supported deployment stack, while a separate islandflow project from the repo-root docker-compose.yml still runs exposed NATS/ClickHouse/Redis containers. Verified Nginx Proxy Manager routes only to islandflow-vps web/api by Docker name. Attempted cleanup via docker compose down and docker rm -f on the stale islandflow containers, but those commands hung for the delta user and timed out. Added repo guardrails and docs so deploy warns when the duplicate project exists, and opened islandflow-2db for manual host-level cleanup during a maintenance window.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:16:05Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:28:07Z","started_at":"2026-05-16T01:16:09Z","closed_at":"2026-05-16T01:28:07Z","close_reason":"Completed the repo-side investigation and guardrails. Actual server-side container removal is blocked by hanging Docker operations and is tracked separately in islandflow-2db for a maintenance window.","dependency_count":0,"dependent_count":0,"comment_count":0}
@ -38,5 +39,6 @@
{"_type":"issue","id":"islandflow-zs0","title":"Migrate terminal UI to smart-money profiles","description":"Migrate apps/web terminal rendering to consume SmartMoneyEvent directly: primary profile, probability ladder, reason codes, and suppression/abstention state, while preserving legacy alert/classifier displays during the bridge.","status":"closed","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:23Z","created_by":"dirtydishes","updated_at":"2026-05-05T05:39:58Z","closed_at":"2026-05-05T05:39:58Z","close_reason":"Completed terminal smart-money profile migration","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-igk","title":"Add plan mode","description":"Implement a user-facing plan mode in the application so users can switch into planning before taking action. Scope to be clarified from existing app patterns.","status":"closed","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:22:37Z","created_by":"dirtydishes","updated_at":"2026-05-04T04:26:18Z","started_at":"2026-05-04T04:22:40Z","closed_at":"2026-05-04T04:26:18Z","close_reason":"Implemented as a global pi extension toggled with Shift+P","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-biq","title":"Finish raw live options delivery and filter/backpressure observability","description":"The smart-money signal path and Tape filters are in place, but the next firehose pass should finish server-side selective raw live delivery for options subscriptions and add explicit filtered-out/backpressure observability for API/web counters. This was discovered while landing islandflow-e4r.\n","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:28:58Z","created_by":"dirtydishes","updated_at":"2026-04-29T03:54:12Z","started_at":"2026-04-29T03:54:12Z","dependencies":[{"issue_id":"islandflow-biq","depends_on_id":"islandflow-e4r","type":"discovered-from","created_at":"2026-04-28T16:28:58Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-cnk","title":"Run Docker image build verification with active Docker daemon","description":"Targeted image builds could not run in the implementation session because the local Docker daemon was unavailable at unix:///Users/kell/.orbstack/run/docker.sock. When Docker or OrbStack is running, validate the refactored deployment Dockerfiles with: docker compose -f deployment/docker/docker-compose.yml build api; docker compose -f deployment/docker/docker-compose.yml build web; docker compose -f deployment/docker/docker-compose.yml build ingest-options.","status":"open","priority":3,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:53:41Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:53:41Z","dependencies":[{"issue_id":"islandflow-cnk","depends_on_id":"islandflow-09a","type":"discovered-from","created_at":"2026-05-16T17:53:40Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-38p","title":"Add native deployment unit templates and rollback helpers","description":"The deploy helper now supports --runtime native, but the repo still relies on operator-managed systemd units and manual rollback. Add checked-in native deployment templates or provisioning guidance for the expected units, and consider lightweight rollback/smoke-test helpers once the host-native path is exercised on the real VPS.","status":"open","priority":3,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:46:42Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:46:42Z","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-575","title":"Document smart-money event calendar env","description":"Document smart-money event-calendar environment configuration in env examples and README.\n","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T06:57:14Z","created_by":"dirtydishes","updated_at":"2026-05-05T06:57:57Z","started_at":"2026-05-05T06:57:17Z","closed_at":"2026-05-05T06:57:57Z","close_reason":"Documented event-calendar env variables","dependency_count":0,"dependent_count":0,"comment_count":0}

38
deployment/docker/Dockerfile.ingest-options
View file

@ -1,3 +1,5 @@
# syntax=docker/dockerfile:1.7
FROM oven/bun:1.3.11
WORKDIR /app
@ -9,15 +11,39 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
COPY --from=workspace package.json ./package.json
COPY --from=workspace bun.lock ./bun.lock
COPY --from=workspace tsconfig.base.json ./tsconfig.base.json
COPY --from=services . ./services
COPY --from=packages . ./packages
COPY --from=apps . ./apps
RUN apt-get update \
&& apt-get install -y --no-install-recommends python3 python3-pip python3-venv \
&& rm -rf /var/lib/apt/lists/* \
&& python3 -m venv "${VIRTUAL_ENV}" \
&& "${VIRTUAL_ENV}/bin/pip" install --no-cache-dir -r services/ingest-options/py/requirements.txt \
&& bun install --frozen-lockfile
&& python3 -m venv "${VIRTUAL_ENV}"
COPY --from=apps desktop/package.json ./apps/desktop/package.json
COPY --from=apps web/package.json ./apps/web/package.json
COPY --from=packages bus/package.json ./packages/bus/package.json
COPY --from=packages config/package.json ./packages/config/package.json
COPY --from=packages observability/package.json ./packages/observability/package.json
COPY --from=packages storage/package.json ./packages/storage/package.json
COPY --from=packages types/package.json ./packages/types/package.json
COPY --from=services api/package.json ./services/api/package.json
COPY --from=services candles/package.json ./services/candles/package.json
COPY --from=services compute/package.json ./services/compute/package.json
COPY --from=services eod-enricher/package.json ./services/eod-enricher/package.json
COPY --from=services ingest-equities/package.json ./services/ingest-equities/package.json
COPY --from=services ingest-options/package.json ./services/ingest-options/package.json
COPY --from=services ingest-options/py/requirements.txt ./services/ingest-options/py/requirements.txt
COPY --from=services refdata/package.json ./services/refdata/package.json
COPY --from=services replay/package.json ./services/replay/package.json
RUN --mount=type=cache,target=/root/.cache/pip \
"${VIRTUAL_ENV}/bin/pip" install -r services/ingest-options/py/requirements.txt
RUN --mount=type=cache,target=/root/.bun/install/cache \
bun install --frozen-lockfile
COPY --from=services . ./services
COPY --from=packages . ./packages
COPY --from=apps . ./apps
ENTRYPOINT ["bun"]

26
deployment/docker/Dockerfile.service
View file

@ -1,3 +1,5 @@
# syntax=docker/dockerfile:1.7
FROM oven/bun:1.3.11
WORKDIR /app
@ -7,10 +9,30 @@ ENV NODE_ENV=production
COPY --from=workspace package.json ./package.json
COPY --from=workspace bun.lock ./bun.lock
COPY --from=workspace tsconfig.base.json ./tsconfig.base.json
COPY --from=apps desktop/package.json ./apps/desktop/package.json
COPY --from=apps web/package.json ./apps/web/package.json
COPY --from=packages bus/package.json ./packages/bus/package.json
COPY --from=packages config/package.json ./packages/config/package.json
COPY --from=packages observability/package.json ./packages/observability/package.json
COPY --from=packages storage/package.json ./packages/storage/package.json
COPY --from=packages types/package.json ./packages/types/package.json
COPY --from=services api/package.json ./services/api/package.json
COPY --from=services candles/package.json ./services/candles/package.json
COPY --from=services compute/package.json ./services/compute/package.json
COPY --from=services eod-enricher/package.json ./services/eod-enricher/package.json
COPY --from=services ingest-equities/package.json ./services/ingest-equities/package.json
COPY --from=services ingest-options/package.json ./services/ingest-options/package.json
COPY --from=services refdata/package.json ./services/refdata/package.json
COPY --from=services replay/package.json ./services/replay/package.json
RUN --mount=type=cache,target=/root/.bun/install/cache \
bun install --frozen-lockfile
COPY --from=services . ./services
COPY --from=packages . ./packages
COPY --from=apps . ./apps
RUN bun install --frozen-lockfile
ENTRYPOINT ["bun"]

25
deployment/docker/Dockerfile.web
View file

@ -1,3 +1,5 @@
# syntax=docker/dockerfile:1.7
FROM oven/bun:1.3.11 AS build
WORKDIR /app
@ -13,11 +15,32 @@ ENV NEXT_PUBLIC_NBBO_MAX_AGE_MS=${NEXT_PUBLIC_NBBO_MAX_AGE_MS}
COPY --from=workspace package.json ./package.json
COPY --from=workspace bun.lock ./bun.lock
COPY --from=workspace tsconfig.base.json ./tsconfig.base.json
COPY --from=apps desktop/package.json ./apps/desktop/package.json
COPY --from=apps web/package.json ./apps/web/package.json
COPY --from=packages bus/package.json ./packages/bus/package.json
COPY --from=packages config/package.json ./packages/config/package.json
COPY --from=packages observability/package.json ./packages/observability/package.json
COPY --from=packages storage/package.json ./packages/storage/package.json
COPY --from=packages types/package.json ./packages/types/package.json
COPY --from=services api/package.json ./services/api/package.json
COPY --from=services candles/package.json ./services/candles/package.json
COPY --from=services compute/package.json ./services/compute/package.json
COPY --from=services eod-enricher/package.json ./services/eod-enricher/package.json
COPY --from=services ingest-equities/package.json ./services/ingest-equities/package.json
COPY --from=services ingest-options/package.json ./services/ingest-options/package.json
COPY --from=services refdata/package.json ./services/refdata/package.json
COPY --from=services replay/package.json ./services/replay/package.json
RUN --mount=type=cache,target=/root/.bun/install/cache \
bun install --frozen-lockfile
COPY --from=services . ./services
COPY --from=packages . ./packages
COPY --from=apps . ./apps
RUN bun install --frozen-lockfile
RUN bun run --cwd apps/web build
FROM oven/bun:1.3.11 AS runtime

35
deployment/docker/README.md
View file

@ -65,14 +65,16 @@ Important defaults:
3. Build and start the stack:
```bash
docker compose up -d --build
docker compose build api web compute candles ingest-options ingest-equities
docker compose up -d
```
If you are updating an existing deployment that already has failing `api` restart loops, do a full recreate so the ClickHouse config mount and dependency changes are applied cleanly:
```bash
docker compose down
docker compose up -d --build --force-recreate
docker compose build api web compute candles ingest-options ingest-equities
docker compose up -d --force-recreate
```
4. Confirm the containers are healthy:
@ -190,6 +192,19 @@ cd deployment/docker
docker compose build web
```
### Faster Docker builds
The app images are structured so dependency installation is isolated from source code changes:
- Docker first copies `package.json`, `bun.lock`, `tsconfig.base.json`, and workspace `package.json` files.
- `bun install --frozen-lockfile` runs in a cacheable layer with a BuildKit Bun cache mount.
- Source from `apps`, `services`, and `packages` is copied only after dependencies are installed.
- `ingest-options` also installs its Python sidecar dependencies from `services/ingest-options/py/requirements.txt` before source copy, using a BuildKit pip cache mount.
That means normal TypeScript edits should reuse dependency layers. The first build after a fresh server checkout, Docker cache cleanup, dependency change, or Python requirement change can still be slow; later deploys should spend their time on changed source and the specific service images being rolled out.
BuildKit cache mounts require a modern Docker Engine with Dockerfile frontend support. Docker Compose v2 on the VPS path enables this by default.
## Safe rollouts on `152.53.80.229`
The current live VPS uses Nginx Proxy Manager on the shared Docker network and routes public traffic to the Docker `web` and `api` containers by container name. Because of that, this Docker path remains the operationally correct default for the live server today.
@ -218,7 +233,7 @@ This flow:
- checks the server checkout before switching anything
- stops if the server has tracked local modifications
- allows the known untracked tarball at `deployment/docker/signal-cli-0.14.3-Linux-native.tar.gz`
- runs `git switch main`, `git pull --ff-only origin main`, and `docker compose up -d --build`
- runs `git switch main`, `git pull --ff-only origin main`, `docker compose build api web compute candles ingest-options ingest-equities`, and `docker compose up -d`
- verifies the stack with `docker compose ps`, recent service logs, container-local health checks, and public HTTPS checks
### Deploy the current local branch
@ -253,6 +268,14 @@ Examples:
./deploy main --runtime docker --web-only --no-build
```
Scoped Docker deploys now build only the selected image set and then restart only those services:
- `--web-only`: `docker compose build web`, then `docker compose up -d web`
- `--api-only`: `docker compose build api`, then `docker compose up -d api`
- `--services-only`: builds and restarts `api`, `compute`, `candles`, `ingest-options`, and `ingest-equities`
Use `--no-build` only when the image is already correct and you need Compose to recreate or restart containers, such as after changing server-side environment values that do not affect a Next.js build-time variable. Do not use `--no-build` for dependency changes, application source changes, or `NEXT_PUBLIC_*` changes.
### Escalation path
Use force recreate only when a normal refresh does not update the services cleanly:
@ -299,7 +322,8 @@ git switch main
git pull --ff-only origin main
cd /home/delta/islandflow/deployment/docker
docker compose up -d --build
docker compose build api web compute candles ingest-options ingest-equities
docker compose up -d
```
Deploy the current branch manually:
@ -314,7 +338,8 @@ git switch <current-branch> || git switch -c <current-branch> --track origin/<cu
git pull --ff-only origin <current-branch>
cd /home/delta/islandflow/deployment/docker
docker compose up -d --build
docker compose build api web compute candles ingest-options ingest-equities
docker compose up -d
```
If you changed only env values for the Bun services on the server:

219
docs/turns/2026-05-16-1752-speed-up-docker-deploys.html Normal file
View file

@ -0,0 +1,219 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Speed Up Docker Deploys</title>
<style>
:root {
color-scheme: dark;
--bg: #06080b;
--panel: #111820;
--panel-2: #0d141b;
--text: #e6edf4;
--muted: #90a0b2;
--faint: #6e7b8c;
--line: #ffffff14;
--accent: #f5a623;
--accent-soft: #f5a6231f;
--ok: #25c17a;
--warn: #ffb130;
}
body {
margin: 0;
background: var(--bg);
color: var(--text);
font: 15px/1.55 "IBM Plex Sans", system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
}
main {
max-width: 920px;
margin: 0 auto;
padding: 48px 24px 64px;
}
header {
border-bottom: 1px solid var(--line);
margin-bottom: 28px;
padding-bottom: 24px;
}
h1 {
margin: 0 0 10px;
font-size: 2rem;
line-height: 1.1;
}
h2 {
margin: 30px 0 10px;
font-size: 1rem;
letter-spacing: 0.08em;
text-transform: uppercase;
}
p {
max-width: 74ch;
margin: 0 0 12px;
}
ul {
margin: 0;
padding-left: 20px;
}
li {
margin: 7px 0;
}
code {
border: 1px solid var(--line);
border-radius: 6px;
background: var(--panel-2);
color: var(--text);
padding: 1px 5px;
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 0.92em;
}
pre {
overflow-x: auto;
border: 1px solid var(--line);
border-radius: 8px;
background: var(--panel-2);
padding: 14px 16px;
}
pre code {
border: 0;
padding: 0;
background: transparent;
}
.summary {
border: 1px solid var(--line);
border-radius: 8px;
background: var(--panel);
padding: 16px 18px;
}
.meta {
color: var(--muted);
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 0.78rem;
letter-spacing: 0.04em;
text-transform: uppercase;
}
.badge {
display: inline-block;
margin-right: 8px;
border: 1px solid var(--line);
border-radius: 999px;
background: var(--accent-soft);
color: var(--accent);
padding: 2px 8px;
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 0.72rem;
font-weight: 700;
letter-spacing: 0.08em;
text-transform: uppercase;
}
.ok {
color: var(--ok);
}
.warn {
color: var(--warn);
}
</style>
</head>
<body>
<main>
<header>
<div class="meta">2026-05-16 17:52 America/New_York</div>
<h1>Speed Up Docker Deploys</h1>
<p class="summary">
<span class="badge">Summary</span>
Docker app images now cache dependency installation separately from source changes, and Docker rollouts now build only the images required by the selected deploy scope before restarting containers.
</p>
</header>
<section>
<h2>Summary</h2>
<p>
Implemented the Docker deployment speed-up plan from <code>/Users/kell/Desktop/speed-up-docker.md</code>. The first build after this change may still be slow, but source-only changes should no longer invalidate the expensive Bun and Python dependency layers.
</p>
</section>
<section>
<h2>Changes Made</h2>
<ul>
<li>Refactored <code>deployment/docker/Dockerfile.service</code> to copy workspace manifests, run cached <code>bun install --frozen-lockfile</code>, then copy source.</li>
<li>Applied the same dependency-first build model to <code>deployment/docker/Dockerfile.web</code>, keeping the Next.js build after source copy.</li>
<li>Updated <code>deployment/docker/Dockerfile.ingest-options</code> with separate cached pip and Bun install layers before copying source.</li>
<li>Changed <code>scripts/deploy.ts</code> so Docker rollouts run explicit <code>docker compose build &lt;services&gt;</code> followed by <code>docker compose up -d &lt;services&gt;</code>.</li>
<li>Documented the faster-build model, scoped rollouts, and appropriate <code>--no-build</code> usage in <code>deployment/docker/README.md</code>.</li>
</ul>
</section>
<section>
<h2>Context</h2>
<p>
The previous Dockerfiles copied all app, service, and package source before dependency installation. That made nearly every code change invalidate <code>bun install</code>, increasing VPS deploy time. The deployment helper also used broad <code>up -d --build</code> behavior rather than a clean build phase scoped to the selected service set.
</p>
</section>
<section>
<h2>Important Implementation Details</h2>
<p>
Each app image now copies root deployment manifests plus every workspace <code>package.json</code> before installing dependencies. The source tree is copied only after the install layer is complete.
</p>
<pre><code>RUN --mount=type=cache,target=/root/.bun/install/cache \
bun install --frozen-lockfile</code></pre>
<p>
The <code>ingest-options</code> image also copies <code>services/ingest-options/py/requirements.txt</code> before source and uses a pip cache mount:
</p>
<pre><code>RUN --mount=type=cache,target=/root/.cache/pip \
"${VIRTUAL_ENV}/bin/pip" install -r services/ingest-options/py/requirements.txt</code></pre>
<p>
For full Docker deploys, the helper builds the six core app services explicitly. For scoped deploys, it builds and restarts only the requested services.
</p>
</section>
<section>
<h2>Expected Impact for End-Users</h2>
<p>
Users should see faster deployment turnaround after ordinary source edits because dependency installation is reused when manifests and locks have not changed. Scoped deploys should also disturb fewer containers, reducing restart surface for web-only, API-only, and backend-only updates.
</p>
</section>
<section>
<h2>Validation</h2>
<ul>
<li><span class="ok">Passed:</span> <code>bun run check:docker-workspace</code></li>
<li><span class="ok">Passed:</span> <code>./deploy --help</code></li>
<li><span class="ok">Passed:</span> <code>docker compose -f deployment/docker/docker-compose.yml config --quiet</code> with a temporary copy of <code>.env.example</code></li>
<li><span class="ok">Passed:</span> <code>bun --cwd=apps/web run build</code></li>
<li><span class="ok">Passed:</span> <code>bun test</code> with 222 passing tests</li>
<li><span class="warn">Not run:</span> targeted Docker image builds because this session could not connect to the Docker daemon at <code>unix:///Users/kell/.orbstack/run/docker.sock</code>.</li>
</ul>
</section>
<section>
<h2>Issues, Limitations, and Mitigations</h2>
<p>
Docker daemon access was unavailable locally, so image builds still need to be exercised on a machine with a running Docker daemon or during the next VPS rollout. Static Compose validation and repo test coverage passed, and the Dockerfiles use standard BuildKit cache mounts supported by modern Docker Compose v2.
</p>
</section>
<section>
<h2>Follow-up Work</h2>
<p>
No separate follow-up issue was created. The remaining verification is operational: run the targeted image builds once Docker or OrbStack is available.
</p>
</section>
</main>
</body>
</html>

26
scripts/deploy.ts
View file

@ -324,6 +324,15 @@ function dockerServicesForScope(scope: DeployScope): string[] {
}
}
function dockerBuildServicesForScope(scope: DeployScope): string[] {
switch (scope) {
case "full":
return [...DOCKER_CORE_SERVICES];
default:
return dockerServicesForScope(scope);
}
}
function dockerLogServicesForScope(scope: DeployScope): string[] {
switch (scope) {
case "web":
@ -565,15 +574,16 @@ function remoteDockerRollout(
forceRecreate: boolean,
noBuild: boolean
): void {
const services = dockerServicesForScope(scope);
const args = ["up", "-d"];
if (!noBuild) {
args.push("--build");
}
const rolloutServices = dockerServicesForScope(scope);
const upArgs = ["up", "-d"];
if (forceRecreate) {
args.push("--force-recreate");
upArgs.push("--force-recreate");
}
const command = `docker compose ${[...args, ...services].join(" ")}`;
const buildServices = dockerBuildServicesForScope(scope);
const buildCommand = noBuild
? null
: `docker compose build ${buildServices.join(" ")}`;
const upCommand = `docker compose ${[...upArgs, ...rolloutServices].join(" ")}`;
runRemoteScript(
"Remote Rollout",
@ -583,7 +593,7 @@ set -euo pipefail
${remoteGitUpdateScript(mode, branch)}
cd ${shellEscape(REMOTE_DOCKER_DEPLOYMENT)}
${command}
${buildCommand ? `${buildCommand}\n` : ""}${upCommand}
`
);
}