Merge pull request #33 from dirtydishes/deployment-fix

Add safe VPS deploy workflow
This commit is contained in:
dirtydishes 2026-05-08 04:13:33 -04:00 committed by GitHub
commit ed722e5936
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 401 additions and 16 deletions

View file

@ -1,3 +1,4 @@
{"_type":"issue","id":"islandflow-43i","title":"Implement safe VPS deploy modes","description":"Implement a safe local deploy entrypoint for the existing Islandflow VPS checkout. Add two rollout modes: deploy origin/main and deploy the current local branch. Use explicit SSH identity flags, preserve the shared npm-shared network topology, avoid destructive git cleanup on the server, allow the known untracked signal-cli tarball, and run standard remote plus public verification checks after compose rebuilds. Keep compatibility wrappers for the existing deployment helper scripts and document the workflow.\n","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T07:56:03Z","created_by":"dirtydishes","updated_at":"2026-05-08T08:01:32Z","started_at":"2026-05-08T07:56:08Z","closed_at":"2026-05-08T08:01:32Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-dil","title":"Run production baseline and post-rollout verification for load reduction","description":"Run the production verification checklist from the load-reduction plan on the VPS, capture baseline container/resource stats, validate replay remains disabled, and confirm JetStream/Redis behavior after rollout.\n\nThis follow-up is operational rather than code-local and could not be executed from the current workspace. It should compare pre/post CPU, RSS, Redis memory, and retention growth using the documented commands.\n","status":"open","priority":1,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:45:06Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:45:06Z","dependencies":[{"issue_id":"islandflow-dil","depends_on_id":"islandflow-1ln","type":"discovered-from","created_at":"2026-05-08T02:45:06Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-1ln","title":"Implement VPS load reduction plan","description":"Implement load-reduction plan across API, compute, logging, retention, and cache pruning.\n\nThis issue tracks the first-pass implementation of VPS load mitigations: lower live cache limits, async Redis write-behind in API live state, scoped cache eviction, reduced hot-path logging, bounded JetStream retention via shared config, in-memory rolling stats with async Redis snapshots, batched ClickHouse inserts for derived tables, and TTL/cardinality pruning for long-lived in-process maps.\n\nAcceptance:\n- Config surface for live limits, logging, rolling cache, and stream retention added\n- API live ingest avoids per-event full resort in monotonic case and avoids synchronous Redis writes per event\n- Compute rolling stats leave Redis hot path and derived ClickHouse writes batch\n- Long-lived caches/maps are pruned by TTL/cardinality\n- Tests cover monotonic/out-of-order live ingest, scoped eviction, rolling stats, and pruning behavior\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:27:41Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:46:23Z","started_at":"2026-05-08T06:27:54Z","closed_at":"2026-05-08T06:46:23Z","close_reason":"Implemented in code; rollout verification follow-up is islandflow-dil and Redis durability decision follow-up is islandflow-ybs","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-pre","title":"Fix contract-focused options tape hydration","description":"Implement contract-focused options tape hydration so focused contract views preserve the clicked seed row, stop reapplying broad flow filters in the Options pane, and use raw contract-scoped ClickHouse queries consistently across live snapshots, history, and replay. Includes frontend replay source-grouping changes and regression tests for focus seed durability, focused filtering, and contract-scoped API behavior.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T03:27:31Z","created_by":"dirtydishes","updated_at":"2026-05-08T03:37:18Z","started_at":"2026-05-08T03:27:35Z","closed_at":"2026-05-08T03:37:18Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
@ -12,6 +13,7 @@
{"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-dga","title":"Remove obsolete deploy wrappers","description":"Remove the legacy deployment helper wrappers now that the repo-standard local deploy entrypoint exists. Delete the obsolete deployment/docker/deploy.sh and deployment/docker/deploy-branch.sh scripts, update documentation to point only at ./deploy, and verify there are no remaining references to the old helpers.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T08:07:43Z","created_by":"dirtydishes","updated_at":"2026-05-08T08:08:12Z","started_at":"2026-05-08T08:07:52Z","closed_at":"2026-05-08T08:08:12Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-ybs","title":"Decide Redis AOF and cache/durable split after load rollout","description":"Decide whether the deployment Redis should keep AOF enabled or be split into cache vs durable roles after the first rollout data is available.\n\nThe current code changes reduce cache churn, but the operational durability/caching tradeoff still needs a production decision.\n","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:45:05Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:45:05Z","dependencies":[{"issue_id":"islandflow-ybs","depends_on_id":"islandflow-1ln","type":"discovered-from","created_at":"2026-05-08T02:45:04Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-vbk","title":"Remove deprecated Alpaca key-pair auth","description":"Remove legacy Alpaca key-pair authentication support and keep ALPACA_API_KEY as the only supported auth method across options/equities ingest and docs.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:19:51Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:21:10Z","started_at":"2026-05-05T07:19:54Z","closed_at":"2026-05-05T07:21:10Z","close_reason":"Removed key-pair auth and kept ALPACA_API_KEY only","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"islandflow-h47","title":"Support single-token Alpaca auth","description":"Support single-token Alpaca authentication across ingest adapters using ALPACA_API_KEY with fallback to ALPACA_KEY_ID/ALPACA_SECRET_KEY, and document env usage.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:12:22Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:13:54Z","started_at":"2026-05-05T07:12:25Z","closed_at":"2026-05-05T07:13:54Z","close_reason":"Added ALPACA_API_KEY support with key-pair fallback","dependency_count":0,"dependent_count":0,"comment_count":0}

5
deploy Executable file
View file

@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
exec bun run "$repo_root/scripts/deploy.ts" "$@"

View file

@ -202,22 +202,120 @@ cd deployment/docker
docker compose build web
```
When you pull new code:
## Safe rollouts on `152.53.80.229`
The checked-in deploy helper is meant to run from your local repo checkout, not from the VPS shell. It always targets:
- SSH host: `delta@152.53.80.229`
- SSH key: `~/.ssh/delta_ed25519`
- Live repo checkout: `/home/delta/islandflow`
- Live compose directory: `/home/delta/islandflow/deployment/docker`
- Shared proxy network: `npm-shared`
It preserves the current proxy topology, reuses the existing Docker Compose project, and avoids destructive cleanup on the server.
### Deploy `origin/main`
```bash
cd deployment/docker
./deploy main
```
This flow:
- fetches `origin` locally and shows the local branch state
- checks the server checkout before switching anything
- stops if the server has tracked local modifications
- allows the known untracked tarball at `deployment/docker/signal-cli-0.14.3-Linux-native.tar.gz`
- runs `git switch main`, `git pull --ff-only origin main`, and `docker compose up -d --build`
- verifies the stack with `docker compose ps`, recent service logs, container-local health checks, and public HTTPS checks
### Deploy the current local branch
```bash
./deploy current-branch
```
Alias:
```bash
./deploy current branch
```
This flow:
- requires a clean local working tree so you only deploy committed state
- pushes the current local branch to `origin`
- uses `git push -u origin <branch>` automatically when the branch has no upstream yet
- switches the server checkout to that same branch and keeps it there until you intentionally move it back
- runs the same rebuild and verification steps as `main`
### Escalation path
Use force recreate only when a normal refresh does not update the services cleanly:
```bash
./deploy main --force-recreate
./deploy current-branch --force-recreate
```
### Return the server to `main`
If the live checkout is on a branch deploy and you want normal production tracking again:
```bash
./deploy main
```
The helper always does the final public verification against:
- `https://flow.deltaisland.io`
- `https://api.flow.deltaisland.io/health`
It also uses container-local health checks inside `islandflow-vps-api-1` and `islandflow-vps-web-1`, because host loopback `127.0.0.1:4000` is not the right primary check for this topology.
## Manual server fallback
If you need to run the rollout steps manually over SSH, use the same live checkout and compose directory. Avoid `git clean -fd`, `git reset --hard`, proxy changes, or Docker network recreation during normal app rollouts.
Deploy `main` manually:
```bash
ssh -i ~/.ssh/delta_ed25519 -o IdentitiesOnly=yes delta@152.53.80.229
cd /home/delta/islandflow
git fetch origin
git switch main
git pull --ff-only origin main
cd /home/delta/islandflow/deployment/docker
docker compose up -d --build
```
If you changed only env values for the Bun services:
Deploy the current branch manually:
```bash
git push -u origin <current-branch> # omit -u if upstream already exists
ssh -i ~/.ssh/delta_ed25519 -o IdentitiesOnly=yes delta@152.53.80.229
cd /home/delta/islandflow
git fetch origin
git switch <current-branch> || git switch -c <current-branch> --track origin/<current-branch>
git pull --ff-only origin <current-branch>
cd /home/delta/islandflow/deployment/docker
docker compose up -d --build
```
If you changed only env values for the Bun services on the server:
```bash
cd /home/delta/islandflow/deployment/docker
docker compose up -d
```
If you changed `NEXT_PUBLIC_API_URL` or `NEXT_PUBLIC_NBBO_MAX_AGE_MS`, rebuild the web image because those are public Next.js build-time values:
```bash
cd /home/delta/islandflow/deployment/docker
docker compose build web
docker compose up -d web
```

View file

@ -1,6 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
git fetch
git pull
docker compose up -d --build --force-recreate

View file

@ -1,7 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
git fetch
git switch deployment
git pull
docker compose up -d --build --force-recreate

View file

@ -13,6 +13,9 @@
"dev:infra:down": "docker compose down",
"dev:web": "bun --cwd=apps/web run dev",
"dev:services": "bun run scripts/dev-services.ts",
"deploy": "bun run scripts/deploy.ts",
"deploy:main": "./deploy main",
"deploy:current-branch": "./deploy current-branch",
"sync:docker-workspace": "bun run scripts/sync-docker-workspace.ts",
"check:docker-workspace": "bun run scripts/check-docker-workspace.ts"
},

View file

@ -13,6 +13,9 @@
"dev:infra:down": "docker compose down",
"dev:web": "bun --cwd=apps/web run dev",
"dev:services": "bun run scripts/dev-services.ts",
"deploy": "bun run scripts/deploy.ts",
"deploy:main": "./deploy main",
"deploy:current-branch": "./deploy current-branch",
"sync:docker-workspace": "bun run scripts/sync-docker-workspace.ts",
"check:docker-workspace": "bun run scripts/check-docker-workspace.ts"
},

287
scripts/deploy.ts Normal file
View file

@ -0,0 +1,287 @@
#!/usr/bin/env bun
import { existsSync } from "node:fs";
import { spawnSync, type SpawnSyncOptions } from "node:child_process";
import path from "node:path";
import { fileURLToPath } from "node:url";
type DeployMode = "main" | "current-branch";
const REMOTE_HOST = "delta@152.53.80.229";
const REMOTE_REPO = "/home/delta/islandflow";
const REMOTE_DEPLOYMENT = "/home/delta/islandflow/deployment/docker";
const SSH_KEY = path.join(process.env.HOME ?? "", ".ssh", "delta_ed25519");
const SSH_OPTIONS = ["-i", SSH_KEY, "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes"];
const ALLOWED_REMOTE_UNTRACKED = "deployment/docker/signal-cli-0.14.3-Linux-native.tar.gz";
const API_CONTAINER = "islandflow-vps-api-1";
const WEB_CONTAINER = "islandflow-vps-web-1";
const PUBLIC_APP_URL = "https://flow.deltaisland.io";
const PUBLIC_API_HEALTH_URL = "https://api.flow.deltaisland.io/health";
const LOG_SERVICES = ["api", "web", "compute", "candles", "ingest-options", "ingest-equities"];
const scriptPath = fileURLToPath(import.meta.url);
const repoRoot = path.resolve(path.dirname(scriptPath), "..");
function usage(exitCode = 1): never {
console.error(`Usage:
./deploy main [--force-recreate]
./deploy current-branch [--force-recreate]
./deploy current branch [--force-recreate]
Modes:
main Deploy origin/main to the live server checkout.
current-branch Push the current local branch, switch the server to it, and deploy it.
Options:
--force-recreate Escalation path for docker compose when a normal refresh is not enough.
--help Show this help text.`);
process.exit(exitCode);
}
function section(title: string): void {
console.log(`\n== ${title} ==`);
}
function formatCommand(command: string, args: string[]): string {
return [command, ...args]
.map((part) => (/\s/.test(part) ? JSON.stringify(part) : part))
.join(" ");
}
function runChecked(command: string, args: string[], options: SpawnSyncOptions = {}): void {
console.log(`$ ${formatCommand(command, args)}`);
const result = spawnSync(command, args, {
cwd: repoRoot,
stdio: "inherit",
...options
});
if (result.status !== 0) {
process.exit(result.status ?? 1);
}
}
function captureChecked(command: string, args: string[], options: SpawnSyncOptions = {}): string {
const result = spawnSync(command, args, {
cwd: repoRoot,
encoding: "utf8",
stdio: ["inherit", "pipe", "pipe"],
...options
});
if (result.status !== 0) {
process.stderr.write(result.stderr ?? "");
process.exit(result.status ?? 1);
}
return result.stdout ?? "";
}
function runRemoteScript(title: string, script: string, args: string[] = []): void {
section(title);
const sshArgs = [...SSH_OPTIONS, REMOTE_HOST, "bash", "-s", "--", ...args];
console.log(`$ ${formatCommand("ssh", sshArgs)}`);
const result = spawnSync("ssh", sshArgs, {
cwd: repoRoot,
input: script,
encoding: "utf8",
stdio: ["pipe", "inherit", "inherit"]
});
if (result.status !== 0) {
process.exit(result.status ?? 1);
}
}
function parseArgs(rawArgs: string[]): { mode: DeployMode; forceRecreate: boolean } {
if (rawArgs.includes("--help") || rawArgs.includes("-h")) {
usage(0);
}
const forceRecreate = rawArgs.includes("--force-recreate");
const positional = rawArgs.filter((arg) => arg !== "--force-recreate");
if (positional.length === 1 && positional[0] === "main") {
return { mode: "main", forceRecreate };
}
if (
(positional.length === 1 && positional[0] === "current-branch") ||
(positional.length === 2 && positional[0] === "current" && positional[1] === "branch")
) {
return { mode: "current-branch", forceRecreate };
}
usage();
}
function assertSshKeyExists(): void {
if (!existsSync(SSH_KEY)) {
console.error(`Missing SSH key: ${SSH_KEY}`);
process.exit(1);
}
}
function localMainPrecheck(): void {
section("Local Precheck");
runChecked("git", ["fetch", "origin"]);
runChecked("git", ["status", "--short", "--branch"]);
runChecked("git", ["rev-parse", "--verify", "HEAD"]);
runChecked("git", ["rev-parse", "origin/main"]);
}
function currentBranchName(): string {
const branch = captureChecked("git", ["branch", "--show-current"]).trim();
if (!branch) {
console.error("Refusing branch deployment from a detached HEAD.");
process.exit(1);
}
return branch;
}
function localBranchPrecheck(branch: string): void {
section("Local Precheck");
runChecked("git", ["branch", "--show-current"]);
runChecked("git", ["status", "--short", "--branch"]);
runChecked("git", ["fetch", "origin"]);
const porcelain = captureChecked("git", ["status", "--porcelain=v1"]).trim();
if (porcelain) {
console.error(
`Refusing to deploy ${branch} with uncommitted local changes. Commit the intended state first.`
);
process.exit(1);
}
}
function publishCurrentBranch(branch: string): void {
section("Local Publish");
const upstreamResult = spawnSync(
"git",
["rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"],
{
cwd: repoRoot,
encoding: "utf8",
stdio: ["inherit", "pipe", "pipe"]
}
);
if (upstreamResult.status === 0) {
runChecked("git", ["push", "origin", branch]);
return;
}
runChecked("git", ["push", "-u", "origin", branch]);
}
function remotePrecheck(): void {
runRemoteScript(
"Remote Precheck",
`#!/usr/bin/env bash
set -euo pipefail
cd "${REMOTE_REPO}"
status="$(git status --porcelain=v1 --branch)"
git status --short --branch
git branch --show-current
while IFS= read -r line; do
[[ -z "$line" ]] && continue
case "$line" in
'## '*)
;;
'?? ${ALLOWED_REMOTE_UNTRACKED}')
;;
'?? '*)
echo "Refusing rollout: unexpected untracked path on server: \${line#?? }" >&2
exit 1
;;
*)
echo "Refusing rollout: tracked local modifications on server: $line" >&2
exit 1
;;
esac
done <<< "$status"
`
);
}
function remoteRollout(mode: DeployMode, branch: string | null, forceRecreate: boolean): void {
const composeArgs = forceRecreate ? "up -d --build --force-recreate" : "up -d --build";
const switchCommand =
mode === "main"
? `git switch main
git pull --ff-only origin main`
: `git switch ${shellEscape(branch!)} || git switch -c ${shellEscape(branch!)} --track origin/${shellEscape(branch!)}
git pull --ff-only origin ${shellEscape(branch!)}`;
runRemoteScript(
"Remote Rollout",
`#!/usr/bin/env bash
set -euo pipefail
cd "${REMOTE_REPO}"
git fetch origin
${switchCommand}
cd "${REMOTE_DEPLOYMENT}"
docker compose ${composeArgs}
`
);
}
function remoteVerification(): void {
runRemoteScript(
"Remote Verification",
`#!/usr/bin/env bash
set -euo pipefail
cd "${REMOTE_DEPLOYMENT}"
docker compose ps
docker compose logs --tail=100 ${LOG_SERVICES.join(" ")}
docker exec ${API_CONTAINER} bun -e 'const r = await fetch("http://127.0.0.1:4000/health"); console.log(await r.text())'
docker exec ${WEB_CONTAINER} bun -e 'const r = await fetch("http://127.0.0.1:3000/"); console.log(r.status)'
`
);
}
function publicVerification(): void {
section("Public Verification");
runChecked("curl", ["-I", "-fksS", PUBLIC_APP_URL]);
runChecked("curl", ["-fksS", PUBLIC_API_HEALTH_URL]);
}
function shellEscape(value: string): string {
if (value.length === 0) {
return "''";
}
return `'${value.replace(/'/g, `'\"'\"'`)}'`;
}
function main(): void {
const { mode, forceRecreate } = parseArgs(process.argv.slice(2));
assertSshKeyExists();
console.log(
mode === "main"
? "Deploying origin/main to the existing Islandflow VPS checkout."
: "Deploying the current local branch to the existing Islandflow VPS checkout."
);
if (mode === "main") {
localMainPrecheck();
remotePrecheck();
remoteRollout(mode, null, forceRecreate);
} else {
const branch = currentBranchName();
localBranchPrecheck(branch);
publishCurrentBranch(branch);
remotePrecheck();
remoteRollout(mode, branch, forceRecreate);
}
remoteVerification();
publicVerification();
}
main();