Fix production deploy network topology

.beads/issues.jsonl

@@ -1,3 +1,4 @@
+{"_type":"issue","id":"islandflow-kda","title":"Fix production compose shared-network topology","description":"Restore the production Docker topology so the merged deploy workflow actually matches the live proxy setup. Update deployment/docker/docker-compose.yml on the working branch so web and api attach to the shared npm-shared network instead of relying on loopback host port bindings, then validate the compose config and document any rollout implications.\n","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T11:08:48Z","created_by":"dirtydishes","updated_at":"2026-05-08T11:10:46Z","started_at":"2026-05-08T11:09:02Z","closed_at":"2026-05-08T11:10:46Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-43i","title":"Implement safe VPS deploy modes","description":"Implement a safe local deploy entrypoint for the existing Islandflow VPS checkout. Add two rollout modes: deploy origin/main and deploy the current local branch. Use explicit SSH identity flags, preserve the shared npm-shared network topology, avoid destructive git cleanup on the server, allow the known untracked signal-cli tarball, and run standard remote plus public verification checks after compose rebuilds. Keep compatibility wrappers for the existing deployment helper scripts and document the workflow.\n","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T07:56:03Z","created_by":"dirtydishes","updated_at":"2026-05-08T08:01:32Z","started_at":"2026-05-08T07:56:08Z","closed_at":"2026-05-08T08:01:32Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-dil","title":"Run production baseline and post-rollout verification for load reduction","description":"Run the production verification checklist from the load-reduction plan on the VPS, capture baseline container/resource stats, validate replay remains disabled, and confirm JetStream/Redis behavior after rollout.\n\nThis follow-up is operational rather than code-local and could not be executed from the current workspace. It should compare pre/post CPU, RSS, Redis memory, and retention growth using the documented commands.\n","status":"open","priority":1,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:45:06Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:45:06Z","dependencies":[{"issue_id":"islandflow-dil","depends_on_id":"islandflow-1ln","type":"discovered-from","created_at":"2026-05-08T02:45:06Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-1ln","title":"Implement VPS load reduction plan","description":"Implement load-reduction plan across API, compute, logging, retention, and cache pruning.\n\nThis issue tracks the first-pass implementation of VPS load mitigations: lower live cache limits, async Redis write-behind in API live state, scoped cache eviction, reduced hot-path logging, bounded JetStream retention via shared config, in-memory rolling stats with async Redis snapshots, batched ClickHouse inserts for derived tables, and TTL/cardinality pruning for long-lived in-process maps.\n\nAcceptance:\n- Config surface for live limits, logging, rolling cache, and stream retention added\n- API live ingest avoids per-event full resort in monotonic case and avoids synchronous Redis writes per event\n- Compute rolling stats leave Redis hot path and derived ClickHouse writes batch\n- Long-lived caches/maps are pruned by TTL/cardinality\n- Tests cover monotonic/out-of-order live ingest, scoped eviction, rolling stats, and pruning behavior\n","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:27:41Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:46:23Z","started_at":"2026-05-08T06:27:54Z","closed_at":"2026-05-08T06:46:23Z","close_reason":"Implemented in code; rollout verification follow-up is islandflow-dil and Redis durability decision follow-up is islandflow-ybs","dependency_count":0,"dependent_count":0,"comment_count":0}
@@ -13,6 +14,7 @@
 {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"islandflow-4sr","title":"Remove deprecated NPM deployment path","description":"The repo still carries a deprecated Nginx Proxy Manager deployment path under deployment/npm, and the Docker deployment docs/config still assume an external NPM shared network. Remove the obsolete NPM deployment path and update the Docker deployment to be the supported way to run Islandflow, including docs and compose/env defaults.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T08:12:30Z","created_by":"dirtydishes","updated_at":"2026-05-08T08:17:05Z","started_at":"2026-05-08T08:12:38Z","closed_at":"2026-05-08T08:17:05Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-dga","title":"Remove obsolete deploy wrappers","description":"Remove the legacy deployment helper wrappers now that the repo-standard local deploy entrypoint exists. Delete the obsolete deployment/docker/deploy.sh and deployment/docker/deploy-branch.sh scripts, update documentation to point only at ./deploy, and verify there are no remaining references to the old helpers.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-08T08:07:43Z","created_by":"dirtydishes","updated_at":"2026-05-08T08:08:12Z","started_at":"2026-05-08T08:07:52Z","closed_at":"2026-05-08T08:08:12Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-ybs","title":"Decide Redis AOF and cache/durable split after load rollout","description":"Decide whether the deployment Redis should keep AOF enabled or be split into cache vs durable roles after the first rollout data is available.\n\nThe current code changes reduce cache churn, but the operational durability/caching tradeoff still needs a production decision.\n","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-08T06:45:05Z","created_by":"dirtydishes","updated_at":"2026-05-08T06:45:05Z","dependencies":[{"issue_id":"islandflow-ybs","depends_on_id":"islandflow-1ln","type":"discovered-from","created_at":"2026-05-08T02:45:04Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
 {"_type":"issue","id":"islandflow-vbk","title":"Remove deprecated Alpaca key-pair auth","description":"Remove legacy Alpaca key-pair authentication support and keep ALPACA_API_KEY as the only supported auth method across options/equities ingest and docs.\n","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T07:19:51Z","created_by":"dirtydishes","updated_at":"2026-05-05T07:21:10Z","started_at":"2026-05-05T07:19:54Z","closed_at":"2026-05-05T07:21:10Z","close_reason":"Removed key-pair auth and kept ALPACA_API_KEY only","dependency_count":0,"dependent_count":0,"comment_count":0}

deployment/docker/docker-compose.yml

@@ -42,8 +42,9 @@ services:
     init: true
     expose:
       - "3000"
-    ports:
+    networks:
-      - "${WEB_BIND_IP:-127.0.0.1}:${WEB_HOST_PORT:-3000}:3000"
+      - default
+      - shared
     depends_on:
       api:
         condition: service_healthy
@@ -65,8 +66,9 @@ services:
     command: ["services/api/src/index.ts"]
     expose:
       - "4000"
-    ports:
+    networks:
-      - "${API_BIND_IP:-127.0.0.1}:${API_HOST_PORT:-4000}:4000"
+      - default
+      - shared
     healthcheck:
       test:
         [
@@ -164,6 +166,11 @@ services:
     volumes:
       - nats-data:/data
 
+networks:
+  shared:
+    external: true
+    name: ${NPM_SHARED_NETWORK:-npm-shared}
+
 volumes:
   clickhouse-data:
   redis-data:

scripts/deploy.ts

@@ -12,7 +12,10 @@ const REMOTE_REPO = "/home/delta/islandflow";
 const REMOTE_DEPLOYMENT = "/home/delta/islandflow/deployment/docker";
 const SSH_KEY = path.join(process.env.HOME ?? "", ".ssh", "delta_ed25519");
 const SSH_OPTIONS = ["-i", SSH_KEY, "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes"];
-const ALLOWED_REMOTE_UNTRACKED = "deployment/docker/signal-cli-0.14.3-Linux-native.tar.gz";
+const ALLOWED_REMOTE_UNTRACKED = new Set([
+  "deployment/docker/signal-cli-0.14.3-Linux-native.tar.gz",
+  "deployment/npm/"
+]);
 const API_CONTAINER = "islandflow-vps-api-1";
 const WEB_CONTAINER = "islandflow-vps-web-1";
 const PUBLIC_APP_URL = "https://flow.deltaisland.io";
@@ -190,11 +193,18 @@ while IFS= read -r line; do
   case "$line" in
     '## '*)
       ;;
-    '?? ${ALLOWED_REMOTE_UNTRACKED}')
-      ;;
     '?? '*)
-      echo "Refusing rollout: unexpected untracked path on server: \${line#?? }" >&2
+      path="\${line#?? }"
-      exit 1
+      case "$path" in
+${Array.from(ALLOWED_REMOTE_UNTRACKED)
+  .map((path) => `        ${shellPattern(path)})`)
+  .join("\n")}
+          ;;
+        *)
+          echo "Refusing rollout: unexpected untracked path on server: $path" >&2
+          exit 1
+          ;;
+      esac
       ;;
     *)
       echo "Refusing rollout: tracked local modifications on server: $line" >&2
@@ -258,6 +268,10 @@ function shellEscape(value: string): string {
   return `'${value.replace(/'/g, `'\"'\"'`)}'`;
 }
 
+function shellPattern(value: string): string {
+  return `'${value.replace(/'/g, `'\"'\"'`)}'`;
+}
+
 function main(): void {
   const { mode, forceRecreate } = parseArgs(process.argv.slice(2));
   assertSshKeyExists();