merge main into nextjs upgrade
This commit is contained in:
commit
171cf52518
40 changed files with 2355 additions and 131 deletions
|
|
@ -1,4 +1,4 @@
|
||||||
{"_type":"issue","id":"islandflow-g3a","title":"Reconcile PR merge conflicts","description":"Resolve the current pull request conflicts for the nextjs-upgrade branch, validate the result, document the turn, and push the reconciled branch.","status":"in_progress","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:44:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:44:56Z","started_at":"2026-05-19T18:44:56Z","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-g3a","title":"Reconcile PR merge conflicts","description":"Resolve the current pull request conflicts for the nextjs-upgrade branch, validate the result, document the turn, and push the reconciled branch.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:44:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:47:35Z","started_at":"2026-05-19T18:44:56Z","closed_at":"2026-05-19T18:47:35Z","close_reason":"Merged forgejo/main into nextjs-upgrade, resolved README and Beads conflicts, updated JetStream retention tests, validated deploy help, Docker workspace sync, API/bus tests, and web build, and added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
{"_type":"issue","id":"islandflow-jbi","title":"Hydrate alert evidence details from ClickHouse","description":"Alert detail drawers need to fetch persisted alert context from ClickHouse by trace id, including linked flow packets, option prints, preserved execution context, and explicit missing refs for UI diagnostics.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:55:43Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:01:58Z","started_at":"2026-05-17T14:55:53Z","closed_at":"2026-05-17T15:01:58Z","close_reason":"Implemented ClickHouse-backed alert context hydration across storage, API, terminal drawer, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-jbi","title":"Hydrate alert evidence details from ClickHouse","description":"Alert detail drawers need to fetch persisted alert context from ClickHouse by trace id, including linked flow packets, option prints, preserved execution context, and explicit missing refs for UI diagnostics.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:55:43Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:01:58Z","started_at":"2026-05-17T14:55:53Z","closed_at":"2026-05-17T15:01:58Z","close_reason":"Implemented ClickHouse-backed alert context hydration across storage, API, terminal drawer, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
{"_type":"issue","id":"islandflow-8kj","title":"Configure persistent beads Dolt remote on deltaisland server","description":"Install the beads and Dolt CLIs on the server, configure a persistent Dolt sync remote backed by the server-hosted Forgejo repository, verify refs/dolt/data publication, and document Nginx Proxy Manager / firewall considerations.","status":"closed","priority":1,"issue_type":"task","assignee":"delta","created_at":"2026-05-17T10:31:31Z","created_by":"delta","updated_at":"2026-05-17T10:37:47Z","started_at":"2026-05-17T10:32:16Z","closed_at":"2026-05-17T10:37:47Z","close_reason":"Installed bd and dolt on the server, configured the Forgejo-backed Dolt remote, published refs/dolt/data, and documented the setup.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-8kj","title":"Configure persistent beads Dolt remote on deltaisland server","description":"Install the beads and Dolt CLIs on the server, configure a persistent Dolt sync remote backed by the server-hosted Forgejo repository, verify refs/dolt/data publication, and document Nginx Proxy Manager / firewall considerations.","status":"closed","priority":1,"issue_type":"task","assignee":"delta","created_at":"2026-05-17T10:31:31Z","created_by":"delta","updated_at":"2026-05-17T10:37:47Z","started_at":"2026-05-17T10:32:16Z","closed_at":"2026-05-17T10:37:47Z","close_reason":"Installed bd and dolt on the server, configured the Forgejo-backed Dolt remote, published refs/dolt/data, and documented the setup.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
{"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
{"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0}
|
||||||
|
|
|
||||||
|
|
@ -168,10 +168,11 @@ Important deployment notes:
|
||||||
- Run the deploy helper from the local repo checkout, not from the VPS shell.
|
- Run the deploy helper from the local repo checkout, not from the VPS shell.
|
||||||
- Do not run the repo-root `docker-compose.yml` on the VPS. It is local infra only and can create duplicate exposed NATS, ClickHouse, and Redis containers on the server.
|
- Do not run the repo-root `docker-compose.yml` on the VPS. It is local infra only and can create duplicate exposed NATS, ClickHouse, and Redis containers on the server.
|
||||||
- The Docker stack lives in `deployment/docker` and is separate from local development infra.
|
- The Docker stack lives in `deployment/docker` and is separate from local development infra.
|
||||||
- Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, `--fast`, `--no-build`, and `--force-recreate`.
|
- Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, `--workers-only`, `--fast`, `--no-build`, and `--force-recreate`.
|
||||||
- `--fast` defaults to a services-only Docker rollout when no explicit scope is provided and trims public API route-suite verification while preserving remote service health checks.
|
- `--fast` defaults to a services-only Docker rollout when no explicit scope is provided and trims public API route-suite verification while preserving remote service health checks.
|
||||||
- `./deploy current-branch` requires a clean local working tree and pushes the branch before moving the server checkout.
|
- `./deploy current-branch` requires a clean local working tree and pushes the branch before moving the server checkout.
|
||||||
- The helper has Forgejo-aware remote resolution for deployments and branch pushes.
|
- The helper has Forgejo-aware remote resolution for deployments and branch pushes.
|
||||||
|
- When run from `/home/delta/islandflow` on the VPS itself, `./deploy` can execute locally instead of SSHing back into the same server.
|
||||||
- Native deployment is opt-in and experimental:
|
- Native deployment is opt-in and experimental:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|
@ -179,7 +180,7 @@ Important deployment notes:
|
||||||
./deploy current-branch --runtime native
|
./deploy current-branch --runtime native
|
||||||
```
|
```
|
||||||
|
|
||||||
Native deployment expects Bun, systemd units, host-reachable infra, and deliberate reverse-proxy changes. The open follow-up is to add native unit templates and rollback helpers.
|
Native deployment expects Bun, systemd units, host-reachable infra, and deliberate reverse-proxy changes. Native deploys are intended primarily for worker-only fast iteration until the public edge is cut over deliberately.
|
||||||
|
|
||||||
Read more:
|
Read more:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "bun run scripts/dev.ts",
|
"dev": "bun run scripts/dev.ts",
|
||||||
"build": "next build",
|
"build": "next build",
|
||||||
"start": "next start -p 3000"
|
"start": "next start"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@islandflow/types": "workspace:*",
|
"@islandflow/types": "workspace:*",
|
||||||
|
|
|
||||||
23
deployment/docker/.dockerignore
Normal file
23
deployment/docker/.dockerignore
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
.git
|
||||||
|
.github
|
||||||
|
.DS_Store
|
||||||
|
.bun
|
||||||
|
.tmp
|
||||||
|
node_modules
|
||||||
|
dist
|
||||||
|
coverage
|
||||||
|
logs
|
||||||
|
apps/web/.next
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
session-ses_*.md
|
||||||
|
token-usage-output.txt
|
||||||
|
signal-cli-*.tar.gz
|
||||||
|
*.tar
|
||||||
|
*.tar.gz
|
||||||
|
*.tgz
|
||||||
|
*.zip
|
||||||
|
__pycache__
|
||||||
|
.pytest_cache
|
||||||
|
!.env.example
|
||||||
|
!**/.env.example
|
||||||
|
|
@ -4,8 +4,10 @@ NATS_URL=nats://nats:4222
|
||||||
CLICKHOUSE_URL=http://clickhouse:8123
|
CLICKHOUSE_URL=http://clickhouse:8123
|
||||||
CLICKHOUSE_DATABASE=default
|
CLICKHOUSE_DATABASE=default
|
||||||
REDIS_URL=redis://redis:6379
|
REDIS_URL=redis://redis:6379
|
||||||
|
ISLANDFLOW_DATA_ROOT=/var/lib/islandflow
|
||||||
|
|
||||||
API_PORT=4000
|
API_PORT=4000
|
||||||
|
API_HOST=0.0.0.0
|
||||||
API_BIND_IP=127.0.0.1
|
API_BIND_IP=127.0.0.1
|
||||||
API_HOST_PORT=4000
|
API_HOST_PORT=4000
|
||||||
WEB_BIND_IP=127.0.0.1
|
WEB_BIND_IP=127.0.0.1
|
||||||
|
|
|
||||||
|
|
@ -60,4 +60,4 @@ COPY --from=build /app/packages ./packages
|
||||||
|
|
||||||
EXPOSE 3000
|
EXPOSE 3000
|
||||||
|
|
||||||
CMD ["bun", "run", "--cwd", "apps/web", "start"]
|
CMD ["bun", "run", "--cwd", "apps/web", "start", "--", "-H", "0.0.0.0", "-p", "3000"]
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,12 @@
|
||||||
|
|
||||||
This directory contains the Docker runtime for Islandflow VPS deployments.
|
This directory contains the Docker runtime for Islandflow VPS deployments.
|
||||||
|
|
||||||
Docker remains the default and recommended server rollout path, but the repo-root `deploy` helper can now target either:
|
Docker remains the default rollout path before native cutover and the rollback path after cutover. The repo-root `deploy` helper can target either:
|
||||||
|
|
||||||
- `--runtime docker` for this Docker Compose stack
|
- `--runtime docker` for this Docker Compose stack
|
||||||
- `--runtime native` for an experimental host-native Bun + systemd rollout described in `deployment/native/README.md`
|
- `--runtime native` for the host-native Bun + systemd rollout described in `deployment/native/README.md`
|
||||||
|
|
||||||
The repo no longer ships or supports a separate `deployment/npm` stack. If you want a reverse proxy, point it at the host ports published by this stack.
|
The public VPS edge remains Nginx Proxy Manager. Docker fallback can be reached either through the shared Docker network service names or the host ports published by this stack.
|
||||||
|
|
||||||
It is separate from the repo-root `docker-compose.yml`, which remains the lightweight local infra stack for development.
|
It is separate from the repo-root `docker-compose.yml`, which remains the lightweight local infra stack for development.
|
||||||
|
|
||||||
|
|
@ -17,7 +17,7 @@ Do not run the repo-root `docker-compose.yml` on the VPS. On the live server tha
|
||||||
|
|
||||||
- Builds and runs the full Islandflow stack with Docker Compose.
|
- Builds and runs the full Islandflow stack with Docker Compose.
|
||||||
- Publishes `web` and `api` to host ports, bound to loopback by default.
|
- Publishes `web` and `api` to host ports, bound to loopback by default.
|
||||||
- Runs ClickHouse, Redis, and NATS JetStream with persistent Docker volumes.
|
- Runs ClickHouse, Redis, and NATS JetStream with persistent host data under `ISLANDFLOW_DATA_ROOT`.
|
||||||
- Runs the core runtime services: `ingest-options`, `ingest-equities`, `compute`, `candles`, `api`, and `web`.
|
- Runs the core runtime services: `ingest-options`, `ingest-equities`, `compute`, `candles`, `api`, and `web`.
|
||||||
- Keeps `replay` opt-in through a Compose profile, because the current replay service starts immediately when the container is enabled.
|
- Keeps `replay` opt-in through a Compose profile, because the current replay service starts immediately when the container is enabled.
|
||||||
|
|
||||||
|
|
@ -56,6 +56,7 @@ cp .env.example .env
|
||||||
Important defaults:
|
Important defaults:
|
||||||
|
|
||||||
- `NATS_URL`, `CLICKHOUSE_URL`, and `REDIS_URL` should stay on the internal container hostnames unless you intentionally split infra out.
|
- `NATS_URL`, `CLICKHOUSE_URL`, and `REDIS_URL` should stay on the internal container hostnames unless you intentionally split infra out.
|
||||||
|
- `ISLANDFLOW_DATA_ROOT=/var/lib/islandflow` matches the native infra data root used by the VPS cutover helpers.
|
||||||
- `OPTIONS_INGEST_ADAPTER=synthetic` and `EQUITIES_INGEST_ADAPTER=synthetic` are the safest first-boot settings.
|
- `OPTIONS_INGEST_ADAPTER=synthetic` and `EQUITIES_INGEST_ADAPTER=synthetic` are the safest first-boot settings.
|
||||||
- `WEB_BIND_IP=127.0.0.1` and `API_BIND_IP=127.0.0.1` keep the published ports local to the host by default.
|
- `WEB_BIND_IP=127.0.0.1` and `API_BIND_IP=127.0.0.1` keep the published ports local to the host by default.
|
||||||
- `WEB_HOST_PORT=3000` and `API_HOST_PORT=4000` control the host-side published ports.
|
- `WEB_HOST_PORT=3000` and `API_HOST_PORT=4000` control the host-side published ports.
|
||||||
|
|
@ -213,17 +214,19 @@ BuildKit cache mounts require a modern Docker Engine with Dockerfile frontend su
|
||||||
|
|
||||||
## Safe rollouts on `152.53.80.229`
|
## Safe rollouts on `152.53.80.229`
|
||||||
|
|
||||||
The current live VPS uses Nginx Proxy Manager on the shared Docker network and routes public traffic to the Docker `web` and `api` containers by container name. Because of that, this Docker path remains the operationally correct default for the live server today.
|
The current live VPS uses Nginx Proxy Manager as the outer edge. Before native cutover, NPM routes Islandflow traffic to Docker service names. During cutover, `deployment/native/switch-npm-edge.sh native` retargets only the Islandflow proxy hosts to the NPM bridge gateway IP so NPM can reach native host ports. If needed, override the detected target with `ISLANDFLOW_NATIVE_HOST=<host-ip>`.
|
||||||
|
|
||||||
The deploy helper also warns if it detects a second compose project named `islandflow` on the server, because that usually means the repo-root local-infra stack was started on the VPS by mistake.
|
The deploy helper also warns if it detects a second compose project named `islandflow` on the server, because that usually means the repo-root local-infra stack was started on the VPS by mistake.
|
||||||
|
|
||||||
The checked-in deploy helper is meant to run from your local repo checkout, not from the VPS shell. It always targets:
|
The checked-in deploy helper normally runs from your local repo checkout and targets:
|
||||||
|
|
||||||
- SSH host: `delta@152.53.80.229`
|
- SSH host: `delta@152.53.80.229`
|
||||||
- SSH key: `~/.ssh/delta_ed25519`
|
- SSH key: `~/.ssh/delta_ed25519` by default
|
||||||
- Live repo checkout: `/home/delta/islandflow`
|
- Live repo checkout: `/home/delta/islandflow`
|
||||||
- Live compose directory: `/home/delta/islandflow/deployment/docker`
|
- Live compose directory: `/home/delta/islandflow/deployment/docker`
|
||||||
|
|
||||||
|
If you run `./deploy` from `/home/delta/islandflow` on the VPS itself, it now executes the remote steps locally instead of SSHing back into the same machine. You can still force SSH with `DEPLOY_FORCE_SSH=1`, or override the key path with `DEPLOY_SSH_KEY_PATH=/path/to/key`.
|
||||||
|
|
||||||
It preserves the current Docker Compose project and avoids destructive cleanup on the server.
|
It preserves the current Docker Compose project and avoids destructive cleanup on the server.
|
||||||
|
|
||||||
### Deploy `origin/main`
|
### Deploy `origin/main`
|
||||||
|
|
@ -271,6 +274,7 @@ Examples:
|
||||||
./deploy main --runtime docker --web-only
|
./deploy main --runtime docker --web-only
|
||||||
./deploy main --runtime docker --api-only
|
./deploy main --runtime docker --api-only
|
||||||
./deploy current-branch --runtime docker --services-only
|
./deploy current-branch --runtime docker --services-only
|
||||||
|
./deploy main --runtime docker --workers-only
|
||||||
./deploy main --runtime docker --fast
|
./deploy main --runtime docker --fast
|
||||||
./deploy main --runtime docker --web-only --no-build
|
./deploy main --runtime docker --web-only --no-build
|
||||||
```
|
```
|
||||||
|
|
@ -280,6 +284,7 @@ Scoped Docker deploys now build only the selected image set and then restart onl
|
||||||
- `--web-only`: `docker compose build web`, then `docker compose up -d web`
|
- `--web-only`: `docker compose build web`, then `docker compose up -d web`
|
||||||
- `--api-only`: `docker compose build api`, then `docker compose up -d api`
|
- `--api-only`: `docker compose build api`, then `docker compose up -d api`
|
||||||
- `--services-only`: builds and restarts `api`, `compute`, `candles`, `ingest-options`, and `ingest-equities`
|
- `--services-only`: builds and restarts `api`, `compute`, `candles`, `ingest-options`, and `ingest-equities`
|
||||||
|
- `--workers-only`: builds and restarts `compute`, `candles`, `ingest-options`, and `ingest-equities` without touching `web` or `api`
|
||||||
- `--fast`: when no explicit scope flag is given, treats the deploy as `--services-only` and skips the public API route suite for quicker completion. It still runs remote service health checks.
|
- `--fast`: when no explicit scope flag is given, treats the deploy as `--services-only` and skips the public API route suite for quicker completion. It still runs remote service health checks.
|
||||||
|
|
||||||
Use `--no-build` only when the image is already correct and you need Compose to recreate or restart containers, such as after changing server-side environment values that do not affect a Next.js build-time variable. Do not use `--no-build` for dependency changes, application source changes, or `NEXT_PUBLIC_*` changes.
|
Use `--no-build` only when the image is already correct and you need Compose to recreate or restart containers, such as after changing server-side environment values that do not affect a Next.js build-time variable. Do not use `--no-build` for dependency changes, application source changes, or `NEXT_PUBLIC_*` changes.
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,8 @@ services:
|
||||||
init: true
|
init: true
|
||||||
expose:
|
expose:
|
||||||
- "3000"
|
- "3000"
|
||||||
|
ports:
|
||||||
|
- "${WEB_BIND_IP:-127.0.0.1}:${WEB_HOST_PORT:-3000}:3000"
|
||||||
networks:
|
networks:
|
||||||
- default
|
- default
|
||||||
- shared
|
- shared
|
||||||
|
|
@ -64,8 +66,13 @@ services:
|
||||||
api:
|
api:
|
||||||
<<: *service-common
|
<<: *service-common
|
||||||
command: ["services/api/src/index.ts"]
|
command: ["services/api/src/index.ts"]
|
||||||
|
environment:
|
||||||
|
LOG_LEVEL: ${LOG_LEVEL:-warn}
|
||||||
|
API_HOST: 0.0.0.0
|
||||||
expose:
|
expose:
|
||||||
- "4000"
|
- "4000"
|
||||||
|
ports:
|
||||||
|
- "${API_BIND_IP:-127.0.0.1}:${API_HOST_PORT:-4000}:4000"
|
||||||
networks:
|
networks:
|
||||||
- default
|
- default
|
||||||
- shared
|
- shared
|
||||||
|
|
@ -132,7 +139,7 @@ services:
|
||||||
soft: 262144
|
soft: 262144
|
||||||
hard: 262144
|
hard: 262144
|
||||||
volumes:
|
volumes:
|
||||||
- clickhouse-data:/var/lib/clickhouse
|
- ${ISLANDFLOW_DATA_ROOT:-/var/lib/islandflow}/clickhouse:/var/lib/clickhouse
|
||||||
- ./clickhouse/listen.xml:/etc/clickhouse-server/config.d/listen.xml:ro
|
- ./clickhouse/listen.xml:/etc/clickhouse-server/config.d/listen.xml:ro
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test:
|
test:
|
||||||
|
|
@ -150,7 +157,7 @@ services:
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
command: ["redis-server", "--appendonly", "yes"]
|
command: ["redis-server", "--appendonly", "yes"]
|
||||||
volumes:
|
volumes:
|
||||||
- redis-data:/data
|
- ${ISLANDFLOW_DATA_ROOT:-/var/lib/islandflow}/redis:/data
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test:
|
test:
|
||||||
[
|
[
|
||||||
|
|
@ -168,14 +175,9 @@ services:
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
command: ["-js", "-sd", "/data"]
|
command: ["-js", "-sd", "/data"]
|
||||||
volumes:
|
volumes:
|
||||||
- nats-data:/data
|
- ${ISLANDFLOW_DATA_ROOT:-/var/lib/islandflow}/nats:/data
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
shared:
|
shared:
|
||||||
external: true
|
external: true
|
||||||
name: ${NPM_SHARED_NETWORK:-npm-shared}
|
name: ${NPM_SHARED_NETWORK:-npm-shared}
|
||||||
|
|
||||||
volumes:
|
|
||||||
clickhouse-data:
|
|
||||||
redis-data:
|
|
||||||
nats-data:
|
|
||||||
|
|
|
||||||
|
|
@ -1,29 +1,167 @@
|
||||||
# Native Deployment
|
# Native Deployment
|
||||||
|
|
||||||
This directory documents the experimental host-native Islandflow rollout path used by:
|
This directory documents the host-native Islandflow rollout path used by:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./deploy main --runtime native
|
./deploy main --runtime native
|
||||||
./deploy current-branch --runtime native
|
./deploy current-branch --runtime native
|
||||||
```
|
```
|
||||||
|
|
||||||
This runtime is intended for faster server iteration during the transition away from Docker-only app rollouts. It is not the recommended path for the current production VPS, which still uses Nginx Proxy Manager to reach the Docker `web` and `api` containers by container name on the shared Docker network. Local development should still prefer:
|
## Current operating model
|
||||||
|
|
||||||
- Docker for infra (`bun run dev:infra`)
|
Native runtime is now intended for a phased VPS cutover. Docker remains the supported rollback runtime, but Docker and native app services must not own the same Islandflow scope at the same time because the workers and API use durable JetStream consumers.
|
||||||
- native Bun services (`bun run dev:services`)
|
|
||||||
- native Next.js web (`bun run dev:web`)
|
Today, the recommended split is:
|
||||||
|
|
||||||
|
- **Nginx Proxy Manager** remains the public `:80/:443` edge
|
||||||
|
- **Native system services** own NATS, Redis, and ClickHouse after infra cutover
|
||||||
|
- **Native user services** own `web`, `api`, and workers after app cutover
|
||||||
|
- **Docker Compose** remains available as the rollback runtime
|
||||||
|
- local development stays:
|
||||||
|
- Docker infra: `bun run dev:infra`
|
||||||
|
- native backend services: `bun run dev:services`
|
||||||
|
- native web: `bun run dev:web`
|
||||||
|
|
||||||
## What native deploy means here
|
## What native deploy means here
|
||||||
|
|
||||||
The checked-in `deploy` helper assumes:
|
The checked-in `deploy` helper assumes:
|
||||||
|
|
||||||
- the live repo checkout is still `/home/delta/islandflow`
|
- the live repo checkout is `/home/delta/islandflow`
|
||||||
- Bun is installed on the VPS
|
- Bun is installed on the VPS
|
||||||
- app processes are managed by `systemd`
|
- app processes are managed by `systemd --user`
|
||||||
- infrastructure services such as NATS, ClickHouse, and Redis are already reachable from the host
|
- infrastructure services such as NATS, ClickHouse, and Redis are reachable from the host
|
||||||
- the web app runs from `apps/web` and is served with `next start -p 3000`
|
- the web app runs from `apps/web` and is served with `next start -p 3000`
|
||||||
|
|
||||||
The deploy script updates the repo checkout, optionally runs `bun install --frozen-lockfile`, optionally rebuilds the web app, restarts the target systemd units, and then verifies the services locally on the VPS plus through the public app URL.
|
The deploy script updates the repo checkout, optionally runs `bun install --frozen-lockfile`, optionally rebuilds the web app, restarts the target user units, verifies local health, and then runs public verification when the selected scope includes the public edge.
|
||||||
|
|
||||||
|
## Live audit status on 2026-05-18
|
||||||
|
|
||||||
|
The plan assumptions were audited on the VPS:
|
||||||
|
|
||||||
|
- `bun` is installed and available at `/home/delta/.bun/bin/bun`
|
||||||
|
- `systemctl --user` is available and the `delta` user has lingering enabled
|
||||||
|
- `/home/delta/islandflow/.env` exists
|
||||||
|
- public `https://flow.deltaisland.io/replay/options` routing is healthy again
|
||||||
|
- the previously reported duplicate `islandflow` compose project is not currently present in `docker compose ls`
|
||||||
|
- native Islandflow user units were not installed at the start of the audit; this change now provides and installs the checked-in user unit files, but they remain disabled until an operator enables a scope intentionally
|
||||||
|
|
||||||
|
That means native worker deploy support is now provisioned on the host, but native runtime should still be enabled scope-by-scope rather than started wholesale.
|
||||||
|
|
||||||
|
## Checked-in native ops assets
|
||||||
|
|
||||||
|
### Infra system units
|
||||||
|
|
||||||
|
Checked-in system service units and config live under:
|
||||||
|
|
||||||
|
- `deployment/native/systemd/system/islandflow-nats.service`
|
||||||
|
- `deployment/native/systemd/system/islandflow-redis.service`
|
||||||
|
- `deployment/native/systemd/system/islandflow-clickhouse.service`
|
||||||
|
- `deployment/native/config/redis.conf`
|
||||||
|
- `deployment/native/config/clickhouse-listen.xml`
|
||||||
|
|
||||||
|
Install and start them on the VPS with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./deployment/native/bootstrap-infra.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Or install and start manually:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ./deployment/native/install-infra-units.sh
|
||||||
|
sudo ./deployment/native/start-infra.sh
|
||||||
|
./deployment/native/check-native-infra.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The native infra services bind to loopback and use stable host data paths:
|
||||||
|
|
||||||
|
- NATS JetStream: `/var/lib/islandflow/nats`
|
||||||
|
- Redis: `/var/lib/islandflow/redis`
|
||||||
|
- ClickHouse: `/var/lib/islandflow/clickhouse`
|
||||||
|
|
||||||
|
The Docker fallback compose file uses the same `ISLANDFLOW_DATA_ROOT` default of `/var/lib/islandflow`, so rollback can preserve durable state when only one runtime is active.
|
||||||
|
|
||||||
|
### User unit templates
|
||||||
|
|
||||||
|
Checked-in unit files live under:
|
||||||
|
|
||||||
|
- `deployment/native/systemd/user/islandflow-web.service`
|
||||||
|
- `deployment/native/systemd/user/islandflow-api.service`
|
||||||
|
- `deployment/native/systemd/user/islandflow-compute.service`
|
||||||
|
- `deployment/native/systemd/user/islandflow-candles.service`
|
||||||
|
- `deployment/native/systemd/user/islandflow-ingest-options.service`
|
||||||
|
- `deployment/native/systemd/user/islandflow-ingest-equities.service`
|
||||||
|
|
||||||
|
These are written for the current VPS layout:
|
||||||
|
|
||||||
|
- repo root: `/home/delta/islandflow`
|
||||||
|
- Bun binary: `/home/delta/.bun/bin/bun`
|
||||||
|
- env file: `/home/delta/islandflow/.env`
|
||||||
|
|
||||||
|
### Install the units
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./deployment/native/install-user-units.sh
|
||||||
|
./deployment/native/install-user-units.sh workers
|
||||||
|
systemctl --user start islandflow-compute.service
|
||||||
|
```
|
||||||
|
|
||||||
|
Install script behavior:
|
||||||
|
|
||||||
|
- copies the checked-in unit files into `~/.config/systemd/user`
|
||||||
|
- reloads the user systemd daemon
|
||||||
|
- enables only the scope you explicitly request
|
||||||
|
- defaults to installing without enabling anything yet
|
||||||
|
|
||||||
|
### Smoke test helper
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./deployment/native/check-native-health.sh workers
|
||||||
|
./deployment/native/check-native-health.sh services
|
||||||
|
./deployment/native/check-native-health.sh full
|
||||||
|
```
|
||||||
|
|
||||||
|
This validates:
|
||||||
|
|
||||||
|
- native infra health for `full`, `api`, `services`, and `workers`
|
||||||
|
- `systemctl --user is-active` for the selected units
|
||||||
|
- local API health at `http://127.0.0.1:4000/health` when API scope is included
|
||||||
|
- local web health at `http://127.0.0.1:3000/` when web scope is included
|
||||||
|
|
||||||
|
### App cutover and edge switch helpers
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./deployment/native/cutover.sh full
|
||||||
|
./deployment/native/switch-npm-edge.sh native
|
||||||
|
./deployment/native/full-rollback.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The edge switch helper updates the Nginx Proxy Manager database entries for `flow.deltaisland.io` and `api.flow.deltaisland.io`, preserving the same-origin Islandflow API location matcher:
|
||||||
|
|
||||||
|
```nginx
|
||||||
|
^/(ws|replay|prints|joins|nbbo|dark|flow|candles|history)/
|
||||||
|
```
|
||||||
|
|
||||||
|
For native cutover, the helper targets the NPM bridge gateway IP by default, not `host.docker.internal`. NPM generates `proxy_pass` with a runtime-resolved `$server` variable, so Docker's `/etc/hosts` alias is not sufficient for these proxy hosts. On the current VPS that native target resolves to `172.18.0.1`, which reaches the host-native `3000` and `4000` listeners from the NPM container.
|
||||||
|
|
||||||
|
Switching back to Docker restores upstreams to the Compose service names `web:3000` and `api:4000`.
|
||||||
|
|
||||||
|
### Rollback helper
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./deployment/native/rollback.sh <git-ref> workers
|
||||||
|
./deployment/native/rollback.sh <git-ref> services
|
||||||
|
```
|
||||||
|
|
||||||
|
Rollback helper behavior:
|
||||||
|
|
||||||
|
- requires a clean repo state
|
||||||
|
- fetches refs
|
||||||
|
- switches the checkout to a detached target ref
|
||||||
|
- reruns `bun install --frozen-lockfile`
|
||||||
|
- rebuilds the web app only when web scope is included
|
||||||
|
- restarts the selected user units
|
||||||
|
- runs the native smoke checks
|
||||||
|
|
||||||
## Expected unit names
|
## Expected unit names
|
||||||
|
|
||||||
|
|
@ -54,87 +192,104 @@ Available overrides:
|
||||||
|
|
||||||
## systemctl invocation
|
## systemctl invocation
|
||||||
|
|
||||||
By default the deploy helper uses:
|
For the checked-in user units, use:
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo -n systemctl
|
|
||||||
```
|
|
||||||
|
|
||||||
If the server uses user units or another wrapper, override it locally before invoking `./deploy`:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
export DEPLOY_NATIVE_SYSTEMCTL_PREFIX="systemctl --user"
|
export DEPLOY_NATIVE_SYSTEMCTL_PREFIX="systemctl --user"
|
||||||
./deploy main --runtime native
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The deploy helper defaults to `sudo -n systemctl`, but that is only appropriate if you intentionally install matching system units.
|
||||||
|
|
||||||
## Partial native rollouts
|
## Partial native rollouts
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./deploy main --runtime native --web-only
|
./deploy main --runtime native --workers-only
|
||||||
./deploy main --runtime native --api-only
|
|
||||||
./deploy current-branch --runtime native --services-only
|
|
||||||
./deploy main --runtime native --fast
|
./deploy main --runtime native --fast
|
||||||
./deploy main --runtime native --web-only --no-build
|
./deploy main --runtime native --services-only
|
||||||
|
./deploy main --runtime native --web-only
|
||||||
|
./deploy current-branch --runtime native --workers-only --no-build
|
||||||
```
|
```
|
||||||
|
|
||||||
Scope behavior:
|
Scope behavior:
|
||||||
|
|
||||||
- default: restart web + API + backend services
|
- default: restart web + API + worker services
|
||||||
- `--web-only`: rebuild/restart only the web unit
|
- `--web-only`: rebuild/restart only the web unit
|
||||||
- `--api-only`: restart only the API unit
|
- `--api-only`: restart only the API unit
|
||||||
- `--services-only`: restart API + backend units without touching the web unit
|
- `--services-only`: restart API + worker units without touching the web unit
|
||||||
- `--fast`: when no explicit scope flag is provided, uses the same `--services-only` scope and trims verbose verification output for quicker completion
|
- `--workers-only`: restart only `compute`, `candles`, `ingest-options`, and `ingest-equities`
|
||||||
|
- `--fast`: when no explicit scope flag is provided, native deploys now default to `--workers-only`
|
||||||
- `--no-build`: skip `bun install --frozen-lockfile` and skip the web build step
|
- `--no-build`: skip `bun install --frozen-lockfile` and skip the web build step
|
||||||
|
|
||||||
## Current status
|
## Edge-cutover guardrail
|
||||||
|
|
||||||
On the current live VPS, native deploys should be treated as opt-in infrastructure work, not the default rollout path. Before a native deploy can succeed there, all of the following must be true at the same time:
|
Native deploys that touch the public web or API edge are intentionally blocked unless you acknowledge cutover readiness:
|
||||||
|
|
||||||
- Bun is installed on the host.
|
|
||||||
- The selected `systemctl` command works non-interactively.
|
|
||||||
- Islandflow systemd units exist for the requested scope.
|
|
||||||
- Host-native services can reach the intended NATS, ClickHouse, and Redis endpoints.
|
|
||||||
- If `web` or `api` move native, the reverse proxy topology is updated deliberately.
|
|
||||||
|
|
||||||
Until that is prepared intentionally, prefer:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./deploy main --runtime docker
|
export DEPLOY_NATIVE_EDGE_READY=1
|
||||||
./deploy current-branch --runtime docker
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Server preparation checklist
|
Without that variable, these commands are refused:
|
||||||
|
|
||||||
Before the first native rollout, ensure the VPS has:
|
- `./deploy main --runtime native`
|
||||||
|
- `./deploy main --runtime native --web-only`
|
||||||
|
- `./deploy main --runtime native --api-only`
|
||||||
|
- `./deploy main --runtime native --services-only`
|
||||||
|
|
||||||
1. Bun installed and on `PATH`
|
This keeps native app ownership explicit until infra, app health, and proxy routing are switched deliberately.
|
||||||
2. a working `/home/delta/islandflow/.env` (or unit-managed equivalent env source)
|
|
||||||
3. systemd units for each target service
|
|
||||||
4. the web unit configured to serve the built app on port `3000`
|
|
||||||
5. the API unit configured to serve health checks on port `4000`
|
|
||||||
6. infrastructure endpoints configured so the native services can reach NATS, ClickHouse, and Redis
|
|
||||||
|
|
||||||
## Verification
|
## Running deploy from the VPS itself
|
||||||
|
|
||||||
Native deploys verify:
|
If you run `./deploy` from `/home/delta/islandflow` on the live server, the deploy helper now executes the remote steps locally instead of SSHing back into the same machine.
|
||||||
|
|
||||||
- target units are active via `systemctl`
|
That means:
|
||||||
- recent unit status and journal output can be collected
|
|
||||||
- local `http://127.0.0.1:4000/health` when API scope is included
|
|
||||||
- local `http://127.0.0.1:3000/` when web scope is included
|
|
||||||
- the public app URL from the local machine after the rollout finishes
|
|
||||||
|
|
||||||
## Rollback
|
- no SSH key is required for on-server deploy execution
|
||||||
|
- timing and verification behavior stay the same
|
||||||
|
- you can still force SSH with `DEPLOY_FORCE_SSH=1`
|
||||||
|
- you can override the SSH key path with `DEPLOY_SSH_KEY_PATH=/path/to/key`
|
||||||
|
|
||||||
Rollback remains manual for now:
|
## Validation matrix
|
||||||
|
|
||||||
1. switch the server checkout back to the last known-good branch or commit
|
| Area | Native workers-only | Native edge cutover |
|
||||||
2. rerun the appropriate native deploy command
|
| --- | --- | --- |
|
||||||
3. if needed, restart only the affected units with `systemctl`
|
| Bun installed | required | required |
|
||||||
|
| `systemctl --user` works | required | required |
|
||||||
|
| Islandflow user units installed | worker units only | all units |
|
||||||
|
| Host access to NATS/ClickHouse/Redis | required | required |
|
||||||
|
| Proxy routes updated for `/prints`, `/history`, `/replay`, `/nbbo`, `/ws`, `/flow`, `/candles` | not required | required |
|
||||||
|
| Public app check | not required | required |
|
||||||
|
| Public API route suite | not required | required |
|
||||||
|
|
||||||
Docker remains the fallback and currently recommended runtime during the transition:
|
## Staged cutover plan
|
||||||
|
|
||||||
|
1. **Stage 1: native workers only**
|
||||||
|
- install user units
|
||||||
|
- validate `./deployment/native/check-native-health.sh workers`
|
||||||
|
- use `./deploy main --runtime native --fast`
|
||||||
|
2. **Stage 2: native API behind local-only verification**
|
||||||
|
- start `islandflow-api.service`
|
||||||
|
- confirm `curl http://127.0.0.1:4000/health`
|
||||||
|
- do not switch public routing yet
|
||||||
|
3. **Stage 3: deliberate public edge cutover**
|
||||||
|
- update proxy routing to native `web`/`api`
|
||||||
|
- export `DEPLOY_NATIVE_EDGE_READY=1`
|
||||||
|
- run full native deploy
|
||||||
|
- validate `bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io`
|
||||||
|
4. **Stage 4: decide final default runtime**
|
||||||
|
- keep Docker as fallback until native edge has proven stable
|
||||||
|
|
||||||
|
## Recommended current commands
|
||||||
|
|
||||||
|
Fast backend iteration before edge cutover:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export DEPLOY_NATIVE_SYSTEMCTL_PREFIX="systemctl --user"
|
||||||
|
./deploy main --runtime native --fast
|
||||||
|
```
|
||||||
|
|
||||||
|
Supported production path today:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./deploy main --runtime docker
|
./deploy main --runtime docker
|
||||||
|
|
|
||||||
24
deployment/native/bootstrap-infra.sh
Executable file
24
deployment/native/bootstrap-infra.sh
Executable file
|
|
@ -0,0 +1,24 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||||
|
|
||||||
|
if [[ "${EUID}" -eq 0 ]]; then
|
||||||
|
"$repo_root/deployment/native/install-infra-units.sh"
|
||||||
|
else
|
||||||
|
sudo "$repo_root/deployment/native/install-infra-units.sh"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Stopping Docker Islandflow services before native infra opens durable data."
|
||||||
|
(
|
||||||
|
cd "$repo_root/deployment/docker"
|
||||||
|
docker compose stop web api compute candles ingest-options ingest-equities nats redis clickhouse
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ "${EUID}" -eq 0 ]]; then
|
||||||
|
"$repo_root/deployment/native/start-infra.sh"
|
||||||
|
else
|
||||||
|
sudo "$repo_root/deployment/native/start-infra.sh"
|
||||||
|
fi
|
||||||
|
|
||||||
|
"$repo_root/deployment/native/check-native-infra.sh"
|
||||||
50
deployment/native/check-native-health.sh
Executable file
50
deployment/native/check-native-health.sh
Executable file
|
|
@ -0,0 +1,50 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
scope="${1:-full}"
|
||||||
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||||
|
units=()
|
||||||
|
|
||||||
|
case "$scope" in
|
||||||
|
full)
|
||||||
|
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||||
|
;;
|
||||||
|
web)
|
||||||
|
units=(islandflow-web.service)
|
||||||
|
;;
|
||||||
|
api)
|
||||||
|
units=(islandflow-api.service)
|
||||||
|
;;
|
||||||
|
services)
|
||||||
|
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||||
|
;;
|
||||||
|
workers)
|
||||||
|
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown scope: $scope" >&2
|
||||||
|
echo "Expected one of: full, web, api, services, workers" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
case "$scope" in
|
||||||
|
full|api|services|workers)
|
||||||
|
"$repo_root/deployment/native/check-native-infra.sh"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
for unit in "${units[@]}"; do
|
||||||
|
systemctl --user is-active --quiet "$unit"
|
||||||
|
echo "ok $unit"
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ " ${units[*]} " == *" islandflow-api.service "* ]]; then
|
||||||
|
curl -fksS http://127.0.0.1:4000/health >/dev/null
|
||||||
|
echo "ok api-health"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ " ${units[*]} " == *" islandflow-web.service "* ]]; then
|
||||||
|
curl -I -fksS http://127.0.0.1:3000/ >/dev/null
|
||||||
|
echo "ok web-health"
|
||||||
|
fi
|
||||||
24
deployment/native/check-native-infra.sh
Executable file
24
deployment/native/check-native-infra.sh
Executable file
|
|
@ -0,0 +1,24 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
systemctl is-active --quiet islandflow-nats.service
|
||||||
|
echo "ok islandflow-nats.service"
|
||||||
|
|
||||||
|
systemctl is-active --quiet islandflow-redis.service
|
||||||
|
echo "ok islandflow-redis.service"
|
||||||
|
|
||||||
|
systemctl is-active --quiet islandflow-clickhouse.service
|
||||||
|
echo "ok islandflow-clickhouse.service"
|
||||||
|
|
||||||
|
if command -v redis-cli >/dev/null 2>&1; then
|
||||||
|
redis-cli -h 127.0.0.1 -p 6379 ping | grep -q PONG
|
||||||
|
else
|
||||||
|
timeout 2 bash -c '</dev/tcp/127.0.0.1/6379'
|
||||||
|
fi
|
||||||
|
echo "ok redis-ping"
|
||||||
|
|
||||||
|
curl -fksS http://127.0.0.1:8123/ping | grep -q Ok
|
||||||
|
echo "ok clickhouse-ping"
|
||||||
|
|
||||||
|
timeout 2 bash -c '</dev/tcp/127.0.0.1/4222'
|
||||||
|
echo "ok nats-port"
|
||||||
6
deployment/native/config/clickhouse-listen.xml
Normal file
6
deployment/native/config/clickhouse-listen.xml
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
<clickhouse>
|
||||||
|
<listen_host>127.0.0.1</listen_host>
|
||||||
|
<path>/var/lib/islandflow/clickhouse/</path>
|
||||||
|
<tmp_path>/var/lib/islandflow/clickhouse/tmp/</tmp_path>
|
||||||
|
<user_files_path>/var/lib/islandflow/clickhouse/user_files/</user_files_path>
|
||||||
|
</clickhouse>
|
||||||
10
deployment/native/config/redis.conf
Normal file
10
deployment/native/config/redis.conf
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
bind 127.0.0.1
|
||||||
|
protected-mode yes
|
||||||
|
port 6379
|
||||||
|
dir /var/lib/islandflow/redis
|
||||||
|
appendonly yes
|
||||||
|
save 900 1
|
||||||
|
save 300 10
|
||||||
|
save 60 10000
|
||||||
|
loglevel notice
|
||||||
|
databases 16
|
||||||
34
deployment/native/cutover.sh
Executable file
34
deployment/native/cutover.sh
Executable file
|
|
@ -0,0 +1,34 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
scope="${1:-full}"
|
||||||
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||||
|
|
||||||
|
case "$scope" in
|
||||||
|
full|services|workers|api|web)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Usage: deployment/native/cutover.sh [full|services|workers|api|web]" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
echo "Stopping Docker-owned Islandflow app services before native ownership starts."
|
||||||
|
(
|
||||||
|
cd "$repo_root/deployment/docker"
|
||||||
|
docker compose stop web api compute candles ingest-options ingest-equities
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ "$scope" == "full" || "$scope" == "services" || "$scope" == "api" || "$scope" == "web" ]]; then
|
||||||
|
"$repo_root/deployment/native/check-native-infra.sh"
|
||||||
|
fi
|
||||||
|
|
||||||
|
systemctl --user restart $(case "$scope" in
|
||||||
|
full) echo islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;;
|
||||||
|
services) echo islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;;
|
||||||
|
workers) echo islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;;
|
||||||
|
api) echo islandflow-api.service ;;
|
||||||
|
web) echo islandflow-web.service ;;
|
||||||
|
esac)
|
||||||
|
|
||||||
|
"$repo_root/deployment/native/check-native-health.sh" "$scope"
|
||||||
27
deployment/native/full-rollback.sh
Executable file
27
deployment/native/full-rollback.sh
Executable file
|
|
@ -0,0 +1,27 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||||
|
|
||||||
|
echo "Stopping native app services."
|
||||||
|
systemctl --user stop islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service || true
|
||||||
|
|
||||||
|
echo "Stopping native infra before Docker reopens durable data."
|
||||||
|
if [[ "${EUID}" -eq 0 ]]; then
|
||||||
|
systemctl stop islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service || true
|
||||||
|
else
|
||||||
|
sudo systemctl stop islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Switching NPM Islandflow upstreams back to Docker service names."
|
||||||
|
"$repo_root/deployment/native/switch-npm-edge.sh" docker
|
||||||
|
|
||||||
|
echo "Restarting Docker Islandflow runtime."
|
||||||
|
(
|
||||||
|
cd "$repo_root/deployment/docker"
|
||||||
|
docker compose up -d web api compute candles ingest-options ingest-equities
|
||||||
|
)
|
||||||
|
|
||||||
|
curl -I -fksS "${DEPLOY_PUBLIC_APP_URL:-https://flow.deltaisland.io}" >/dev/null
|
||||||
|
curl -fksS "${DEPLOY_PUBLIC_API_HEALTH_URL:-https://api.flow.deltaisland.io/health}" >/dev/null
|
||||||
|
echo "Rollback validation passed."
|
||||||
72
deployment/native/install-infra-units.sh
Executable file
72
deployment/native/install-infra-units.sh
Executable file
|
|
@ -0,0 +1,72 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||||
|
system_unit_source_dir="$repo_root/deployment/native/systemd/system"
|
||||||
|
config_source_dir="$repo_root/deployment/native/config"
|
||||||
|
|
||||||
|
if [[ "${EUID}" -ne 0 ]]; then
|
||||||
|
echo "Run as root: sudo $0" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
resolve_binary() {
|
||||||
|
local name="$1"
|
||||||
|
local path=""
|
||||||
|
|
||||||
|
path="$(command -v "$name" 2>/dev/null || true)"
|
||||||
|
if [[ -n "$path" ]]; then
|
||||||
|
printf '%s\n' "$path"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
for candidate in "/usr/bin/$name" "/usr/sbin/$name" "/usr/local/bin/$name" "/usr/local/sbin/$name"; do
|
||||||
|
if [[ -x "$candidate" ]]; then
|
||||||
|
printf '%s\n' "$candidate"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
missing=()
|
||||||
|
for command in nats-server redis-server clickhouse-server; do
|
||||||
|
if ! resolve_binary "$command" >/dev/null; then
|
||||||
|
missing+=("$command")
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ ${#missing[@]} -gt 0 ]]; then
|
||||||
|
echo "Missing native infra binaries: ${missing[*]}" >&2
|
||||||
|
echo "Install NATS Server, Redis Server, and ClickHouse Server before bootstrapping native infra." >&2
|
||||||
|
echo "On Debian, Redis is usually available as redis-server; ClickHouse and NATS may require their vendor repositories or packaged binaries." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
ensure_system_user() {
|
||||||
|
local name="$1"
|
||||||
|
local home="$2"
|
||||||
|
|
||||||
|
getent group "$name" >/dev/null || groupadd --system "$name"
|
||||||
|
getent passwd "$name" >/dev/null || useradd --system --gid "$name" --home-dir "$home" --shell /usr/sbin/nologin "$name"
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_system_user nats /var/lib/islandflow/nats
|
||||||
|
ensure_system_user redis /var/lib/islandflow/redis
|
||||||
|
ensure_system_user clickhouse /var/lib/islandflow/clickhouse
|
||||||
|
|
||||||
|
install -d -m 0755 /etc/islandflow
|
||||||
|
install -m 0644 "$config_source_dir/redis.conf" /etc/islandflow/redis.conf
|
||||||
|
install -d -m 0755 /etc/clickhouse-server/config.d
|
||||||
|
install -m 0644 "$config_source_dir/clickhouse-listen.xml" /etc/clickhouse-server/config.d/islandflow-listen.xml
|
||||||
|
|
||||||
|
install -d -o nats -g nats -m 0750 /var/lib/islandflow/nats
|
||||||
|
install -d -o redis -g redis -m 0750 /var/lib/islandflow/redis
|
||||||
|
install -d -o clickhouse -g clickhouse -m 0750 /var/lib/islandflow/clickhouse
|
||||||
|
|
||||||
|
install -m 0644 "$system_unit_source_dir"/islandflow-*.service /etc/systemd/system/
|
||||||
|
systemctl daemon-reload
|
||||||
|
|
||||||
|
echo "Installed native infra system units and config."
|
||||||
|
echo "Start infra with: sudo deployment/native/start-infra.sh"
|
||||||
49
deployment/native/install-user-units.sh
Executable file
49
deployment/native/install-user-units.sh
Executable file
|
|
@ -0,0 +1,49 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
scope="${1:-none}"
|
||||||
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||||
|
unit_source_dir="$repo_root/deployment/native/systemd/user"
|
||||||
|
unit_target_dir="${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user"
|
||||||
|
units=()
|
||||||
|
|
||||||
|
case "$scope" in
|
||||||
|
none)
|
||||||
|
;;
|
||||||
|
full)
|
||||||
|
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||||
|
;;
|
||||||
|
web)
|
||||||
|
units=(islandflow-web.service)
|
||||||
|
;;
|
||||||
|
api)
|
||||||
|
units=(islandflow-api.service)
|
||||||
|
;;
|
||||||
|
services)
|
||||||
|
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||||
|
;;
|
||||||
|
workers)
|
||||||
|
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown scope: $scope" >&2
|
||||||
|
echo "Expected one of: none, full, web, api, services, workers" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
mkdir -p "$unit_target_dir"
|
||||||
|
cp "$unit_source_dir"/*.service "$unit_target_dir"/
|
||||||
|
|
||||||
|
systemctl --user daemon-reload
|
||||||
|
|
||||||
|
if [[ ${#units[@]} -gt 0 ]]; then
|
||||||
|
systemctl --user enable "${units[@]}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Installed Islandflow user units into $unit_target_dir"
|
||||||
|
if [[ ${#units[@]} -gt 0 ]]; then
|
||||||
|
echo "Enabled scope: $scope"
|
||||||
|
else
|
||||||
|
echo "No units enabled yet. Pass a scope such as workers when you are ready."
|
||||||
|
fi
|
||||||
57
deployment/native/rollback.sh
Executable file
57
deployment/native/rollback.sh
Executable file
|
|
@ -0,0 +1,57 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
if [[ $# -lt 1 || $# -gt 2 ]]; then
|
||||||
|
echo "Usage: deployment/native/rollback.sh <git-ref> [full|web|api|services|workers]" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
ref="$1"
|
||||||
|
scope="${2:-services}"
|
||||||
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||||
|
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
if [[ -n "$(git status --porcelain=v1)" ]]; then
|
||||||
|
echo "Refusing rollback with a dirty working tree." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
current_ref="$(git rev-parse --short HEAD)"
|
||||||
|
echo "Rolling back from $current_ref to $ref (scope: $scope)"
|
||||||
|
|
||||||
|
git fetch --all --prune
|
||||||
|
git switch --detach "$ref"
|
||||||
|
bun install --frozen-lockfile
|
||||||
|
|
||||||
|
if [[ "$scope" == "full" || "$scope" == "web" ]]; then
|
||||||
|
bun --cwd=apps/web run build
|
||||||
|
fi
|
||||||
|
|
||||||
|
case "$scope" in
|
||||||
|
full)
|
||||||
|
units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||||
|
;;
|
||||||
|
web)
|
||||||
|
units=(islandflow-web.service)
|
||||||
|
;;
|
||||||
|
api)
|
||||||
|
units=(islandflow-api.service)
|
||||||
|
;;
|
||||||
|
services)
|
||||||
|
units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||||
|
;;
|
||||||
|
workers)
|
||||||
|
units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown scope: $scope" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
systemctl --user restart "${units[@]}"
|
||||||
|
"$repo_root/deployment/native/check-native-health.sh" "$scope"
|
||||||
|
|
||||||
|
echo "Rollback complete. Repo is now detached at $(git rev-parse --short HEAD)."
|
||||||
|
echo "Return to tracked main later with: git switch main && git pull --ff-only <remote> main"
|
||||||
17
deployment/native/start-infra.sh
Executable file
17
deployment/native/start-infra.sh
Executable file
|
|
@ -0,0 +1,17 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
if [[ "${EUID}" -ne 0 ]]; then
|
||||||
|
echo "Run as root: sudo $0" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for unit in redis-server.service nats-server.service clickhouse-server.service; do
|
||||||
|
if systemctl list-unit-files "$unit" >/dev/null 2>&1; then
|
||||||
|
systemctl disable --now "$unit" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
systemctl reset-failed islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service || true
|
||||||
|
systemctl enable --now islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service
|
||||||
|
"$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/check-native-infra.sh"
|
||||||
9
deployment/native/stop-infra.sh
Executable file
9
deployment/native/stop-infra.sh
Executable file
|
|
@ -0,0 +1,9 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
if [[ "${EUID}" -ne 0 ]]; then
|
||||||
|
echo "Run as root: sudo $0" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
systemctl stop islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service
|
||||||
285
deployment/native/switch-npm-edge.sh
Executable file
285
deployment/native/switch-npm-edge.sh
Executable file
|
|
@ -0,0 +1,285 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
target="${1:-native}"
|
||||||
|
npm_root="${NPM_ROOT:-/home/delta/nginx-proxy-manager}"
|
||||||
|
db_path="${NPM_DB_PATH:-$npm_root/data/database.sqlite}"
|
||||||
|
app_domain="${ISLANDFLOW_APP_DOMAIN:-flow.deltaisland.io}"
|
||||||
|
api_domain="${ISLANDFLOW_API_DOMAIN:-api.flow.deltaisland.io}"
|
||||||
|
native_host="${ISLANDFLOW_NATIVE_HOST:-}"
|
||||||
|
docker_web_host="${ISLANDFLOW_DOCKER_WEB_HOST:-web}"
|
||||||
|
docker_api_host="${ISLANDFLOW_DOCKER_API_HOST:-api}"
|
||||||
|
web_port="${ISLANDFLOW_WEB_PORT:-3000}"
|
||||||
|
api_port="${ISLANDFLOW_API_PORT:-4000}"
|
||||||
|
restart_npm="${NPM_RESTART:-1}"
|
||||||
|
npm_container="${NPM_CONTAINER_NAME:-nginx-proxy-manager}"
|
||||||
|
sudo_cmd=()
|
||||||
|
|
||||||
|
case "$target" in
|
||||||
|
native|docker)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Usage: deployment/native/switch-npm-edge.sh [native|docker]" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
resolve_native_host() {
|
||||||
|
if [[ -n "$native_host" ]]; then
|
||||||
|
printf '%s\n' "$native_host"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
if command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' | grep -qx "$npm_container"; then
|
||||||
|
native_host="$(docker inspect "$npm_container" --format '{{range .NetworkSettings.Networks}}{{println .Gateway}}{{end}}' | sed '/^$/d' | head -n1)"
|
||||||
|
if [[ -n "$native_host" ]]; then
|
||||||
|
printf '%s\n' "$native_host"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Unable to determine the native upstream host for NPM." >&2
|
||||||
|
echo "Set ISLANDFLOW_NATIVE_HOST explicitly or start the $npm_container container first." >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ "$target" == "native" ]]; then
|
||||||
|
native_host="$(resolve_native_host)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -w "$db_path" || ! -w "$(dirname "$db_path")" ]]; then
|
||||||
|
if [[ "${EUID}" -eq 0 ]]; then
|
||||||
|
sudo_cmd=()
|
||||||
|
elif command -v sudo >/dev/null 2>&1; then
|
||||||
|
sudo_cmd=(sudo)
|
||||||
|
else
|
||||||
|
echo "NPM database path is not writable and sudo is unavailable: $db_path" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -f "$db_path" ]]; then
|
||||||
|
echo "NPM database not found: $db_path" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
backup="$db_path.before-islandflow-$target-$(date +%Y%m%d%H%M%S)"
|
||||||
|
"${sudo_cmd[@]}" cp "$db_path" "$backup"
|
||||||
|
echo "Backed up NPM database to $backup"
|
||||||
|
|
||||||
|
"${sudo_cmd[@]}" python3 - "$db_path" "$target" "$app_domain" "$api_domain" "$native_host" "$docker_web_host" "$docker_api_host" "$web_port" "$api_port" <<'PY'
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
|
||||||
|
db_path, target, app_domain, api_domain, native_host, docker_web_host, docker_api_host, web_port, api_port = sys.argv[1:]
|
||||||
|
web_host = native_host if target == "native" else docker_web_host
|
||||||
|
api_host = native_host if target == "native" else docker_api_host
|
||||||
|
|
||||||
|
advanced_config = f"""location ~ ^/(ws|replay|prints|joins|nbbo|dark|flow|candles|history)/ {{
|
||||||
|
set $forward_scheme http;
|
||||||
|
set $server "{api_host}";
|
||||||
|
set $port {api_port};
|
||||||
|
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection $http_connection;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
|
||||||
|
include conf.d/include/proxy.conf;
|
||||||
|
}}"""
|
||||||
|
|
||||||
|
def has_domain(raw, domain):
|
||||||
|
try:
|
||||||
|
return domain in json.loads(raw)
|
||||||
|
except Exception:
|
||||||
|
return domain in raw
|
||||||
|
|
||||||
|
con = sqlite3.connect(db_path)
|
||||||
|
cur = con.cursor()
|
||||||
|
rows = list(cur.execute("select id, domain_names from proxy_host where is_deleted = 0"))
|
||||||
|
app_ids = [row_id for row_id, domains in rows if has_domain(domains, app_domain)]
|
||||||
|
api_ids = [row_id for row_id, domains in rows if has_domain(domains, api_domain)]
|
||||||
|
|
||||||
|
if len(app_ids) != 1 or len(api_ids) != 1:
|
||||||
|
raise SystemExit(f"Expected one app and one API proxy host, found app={app_ids} api={api_ids}")
|
||||||
|
|
||||||
|
cur.execute(
|
||||||
|
"update proxy_host set forward_scheme = 'http', forward_host = ?, forward_port = ?, allow_websocket_upgrade = 1, advanced_config = ?, modified_on = datetime('now') where id = ?",
|
||||||
|
(web_host, int(web_port), advanced_config, app_ids[0]),
|
||||||
|
)
|
||||||
|
cur.execute(
|
||||||
|
"update proxy_host set forward_scheme = 'http', forward_host = ?, forward_port = ?, allow_websocket_upgrade = 1, modified_on = datetime('now') where id = ?",
|
||||||
|
(api_host, int(api_port), api_ids[0]),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
print(f"Updated {app_domain} -> {web_host}:{web_port}")
|
||||||
|
print(f"Updated {api_domain} -> {api_host}:{api_port}")
|
||||||
|
PY
|
||||||
|
|
||||||
|
if command -v python3 >/dev/null 2>&1; then
|
||||||
|
"${sudo_cmd[@]}" python3 - "$npm_root" "$db_path" "$target" "$app_domain" "$api_domain" "$native_host" "$docker_web_host" "$docker_api_host" "$web_port" "$api_port" <<'PY'
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
(
|
||||||
|
npm_root,
|
||||||
|
db_path,
|
||||||
|
target,
|
||||||
|
app_domain,
|
||||||
|
api_domain,
|
||||||
|
native_host,
|
||||||
|
docker_web_host,
|
||||||
|
docker_api_host,
|
||||||
|
web_port,
|
||||||
|
api_port,
|
||||||
|
) = sys.argv[1:]
|
||||||
|
|
||||||
|
web_host = native_host if target == "native" else docker_web_host
|
||||||
|
api_host = native_host if target == "native" else docker_api_host
|
||||||
|
|
||||||
|
def has_domain(raw, domain):
|
||||||
|
try:
|
||||||
|
return domain in json.loads(raw)
|
||||||
|
except Exception:
|
||||||
|
return domain in raw
|
||||||
|
|
||||||
|
def replace_nth(text, pattern, replacement, index):
|
||||||
|
matches = list(pattern.finditer(text))
|
||||||
|
if len(matches) < index:
|
||||||
|
raise SystemExit(f"Unable to rewrite generated proxy config; expected match {index} for {pattern.pattern!r}")
|
||||||
|
match = matches[index - 1]
|
||||||
|
return text[:match.start()] + replacement(match) + text[match.end():]
|
||||||
|
|
||||||
|
server_pattern = re.compile(r'^(?P<prefix>\s*set \$server\s+)".*?";\s*$', re.M)
|
||||||
|
port_pattern = re.compile(r'^(?P<prefix>\s*set \$port\s+)\d+;\s*$', re.M)
|
||||||
|
|
||||||
|
def replace_server(text, host, index):
|
||||||
|
return replace_nth(text, server_pattern, lambda m: f'{m.group("prefix")}"{host}";', index)
|
||||||
|
|
||||||
|
def replace_port(text, port, index):
|
||||||
|
return replace_nth(text, port_pattern, lambda m: f'{m.group("prefix")}{port};', index)
|
||||||
|
|
||||||
|
con = sqlite3.connect(db_path)
|
||||||
|
rows = list(con.execute("select id, domain_names from proxy_host where is_deleted = 0"))
|
||||||
|
app_ids = [row_id for row_id, domains in rows if has_domain(domains, app_domain)]
|
||||||
|
api_ids = [row_id for row_id, domains in rows if has_domain(domains, api_domain)]
|
||||||
|
if len(app_ids) != 1 or len(api_ids) != 1:
|
||||||
|
raise SystemExit(f"Expected one app and one API proxy host, found app={app_ids} api={api_ids}")
|
||||||
|
|
||||||
|
api_conf = Path(npm_root) / "data/nginx/proxy_host" / f"{api_ids[0]}.conf"
|
||||||
|
app_conf = Path(npm_root) / "data/nginx/proxy_host" / f"{app_ids[0]}.conf"
|
||||||
|
|
||||||
|
if api_conf.exists():
|
||||||
|
text = api_conf.read_text()
|
||||||
|
text = replace_server(text, api_host, 1)
|
||||||
|
text = replace_port(text, int(api_port), 1)
|
||||||
|
api_conf.write_text(text)
|
||||||
|
print(f"Synchronized {api_conf.name} -> {api_host}:{api_port}")
|
||||||
|
|
||||||
|
if app_conf.exists():
|
||||||
|
text = app_conf.read_text()
|
||||||
|
text = replace_server(text, web_host, 1)
|
||||||
|
text = replace_port(text, int(web_port), 1)
|
||||||
|
text = replace_server(text, api_host, 2)
|
||||||
|
text = replace_port(text, int(api_port), 2)
|
||||||
|
app_conf.write_text(text)
|
||||||
|
print(f"Synchronized {app_conf.name} -> {web_host}:{web_port} and API matcher -> {api_host}:{api_port}")
|
||||||
|
PY
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$restart_npm" == "0" ]]; then
|
||||||
|
echo "NPM container restart skipped because NPM_RESTART=0."
|
||||||
|
elif command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' | grep -qx nginx-proxy-manager; then
|
||||||
|
docker restart nginx-proxy-manager >/dev/null
|
||||||
|
echo "Restarted nginx-proxy-manager"
|
||||||
|
else
|
||||||
|
echo "NPM container restart skipped; restart it manually if it is not managed by Docker on this host."
|
||||||
|
fi
|
||||||
|
|
||||||
|
if command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' | grep -qx "$npm_container"; then
|
||||||
|
"${sudo_cmd[@]}" python3 - "$npm_root" "$db_path" "$target" "$app_domain" "$api_domain" "$native_host" "$docker_web_host" "$docker_api_host" "$web_port" "$api_port" <<'PY'
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
(
|
||||||
|
npm_root,
|
||||||
|
db_path,
|
||||||
|
target,
|
||||||
|
app_domain,
|
||||||
|
api_domain,
|
||||||
|
native_host,
|
||||||
|
docker_web_host,
|
||||||
|
docker_api_host,
|
||||||
|
web_port,
|
||||||
|
api_port,
|
||||||
|
) = sys.argv[1:]
|
||||||
|
|
||||||
|
web_host = native_host if target == "native" else docker_web_host
|
||||||
|
api_host = native_host if target == "native" else docker_api_host
|
||||||
|
|
||||||
|
def has_domain(raw, domain):
|
||||||
|
try:
|
||||||
|
return domain in json.loads(raw)
|
||||||
|
except Exception:
|
||||||
|
return domain in raw
|
||||||
|
|
||||||
|
def replace_nth(text, pattern, replacement, index):
|
||||||
|
matches = list(pattern.finditer(text))
|
||||||
|
if len(matches) < index:
|
||||||
|
raise SystemExit(f"Unable to rewrite generated proxy config; expected match {index} for {pattern.pattern!r}")
|
||||||
|
match = matches[index - 1]
|
||||||
|
return text[:match.start()] + replacement(match) + text[match.end():]
|
||||||
|
|
||||||
|
server_pattern = re.compile(r'^(?P<prefix>\s*set \$server\s+)".*?";\s*$', re.M)
|
||||||
|
port_pattern = re.compile(r'^(?P<prefix>\s*set \$port\s+)\d+;\s*$', re.M)
|
||||||
|
|
||||||
|
def replace_server(text, host, index):
|
||||||
|
return replace_nth(text, server_pattern, lambda m: f'{m.group("prefix")}"{host}";', index)
|
||||||
|
|
||||||
|
def replace_port(text, port, index):
|
||||||
|
return replace_nth(text, port_pattern, lambda m: f'{m.group("prefix")}{port};', index)
|
||||||
|
|
||||||
|
con = sqlite3.connect(db_path)
|
||||||
|
rows = list(con.execute("select id, domain_names from proxy_host where is_deleted = 0"))
|
||||||
|
app_ids = [row_id for row_id, domains in rows if has_domain(domains, app_domain)]
|
||||||
|
api_ids = [row_id for row_id, domains in rows if has_domain(domains, api_domain)]
|
||||||
|
if len(app_ids) != 1 or len(api_ids) != 1:
|
||||||
|
raise SystemExit(f"Expected one app and one API proxy host, found app={app_ids} api={api_ids}")
|
||||||
|
|
||||||
|
api_conf = Path(npm_root) / "data/nginx/proxy_host" / f"{api_ids[0]}.conf"
|
||||||
|
app_conf = Path(npm_root) / "data/nginx/proxy_host" / f"{app_ids[0]}.conf"
|
||||||
|
|
||||||
|
if api_conf.exists():
|
||||||
|
text = api_conf.read_text()
|
||||||
|
text = replace_server(text, api_host, 1)
|
||||||
|
text = replace_port(text, int(api_port), 1)
|
||||||
|
api_conf.write_text(text)
|
||||||
|
|
||||||
|
if app_conf.exists():
|
||||||
|
text = app_conf.read_text()
|
||||||
|
text = replace_server(text, web_host, 1)
|
||||||
|
text = replace_port(text, int(web_port), 1)
|
||||||
|
text = replace_server(text, api_host, 2)
|
||||||
|
text = replace_port(text, int(api_port), 2)
|
||||||
|
app_conf.write_text(text)
|
||||||
|
PY
|
||||||
|
reloaded=0
|
||||||
|
for _ in 1 2 3 4 5; do
|
||||||
|
if docker exec "$npm_container" nginx -s reload >/dev/null 2>&1; then
|
||||||
|
reloaded=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
if [[ "$reloaded" == "1" ]]; then
|
||||||
|
echo "Reloaded nginx-proxy-manager"
|
||||||
|
else
|
||||||
|
echo "Warning: nginx-proxy-manager reload did not succeed after restart; verify the container is healthy." >&2
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Islandflow ClickHouse
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
ExecStart=/usr/bin/env clickhouse-server --config-file=/etc/clickhouse-server/config.xml
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
User=clickhouse
|
||||||
|
Group=clickhouse
|
||||||
|
StateDirectory=clickhouse
|
||||||
|
LimitNOFILE=262144
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
18
deployment/native/systemd/system/islandflow-nats.service
Normal file
18
deployment/native/systemd/system/islandflow-nats.service
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Islandflow NATS JetStream
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
ExecStart=/usr/sbin/nats-server -js -sd /var/lib/islandflow/nats -a 127.0.0.1 -p 4222 -m 8222
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
User=nats
|
||||||
|
Group=nats
|
||||||
|
RuntimeDirectory=islandflow-nats
|
||||||
|
StateDirectory=islandflow/nats
|
||||||
|
LimitNOFILE=1048576
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
18
deployment/native/systemd/system/islandflow-redis.service
Normal file
18
deployment/native/systemd/system/islandflow-redis.service
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Islandflow Redis
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=notify
|
||||||
|
ExecStart=/usr/bin/env redis-server /etc/islandflow/redis.conf --supervised systemd --daemonize no
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
User=redis
|
||||||
|
Group=redis
|
||||||
|
RuntimeDirectory=islandflow-redis
|
||||||
|
StateDirectory=islandflow/redis
|
||||||
|
LimitNOFILE=65535
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
19
deployment/native/systemd/user/islandflow-api.service
Normal file
19
deployment/native/systemd/user/islandflow-api.service
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Islandflow API
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
WorkingDirectory=/home/delta/islandflow
|
||||||
|
Environment=API_HOST=0.0.0.0
|
||||||
|
Environment=API_PORT=4000
|
||||||
|
EnvironmentFile=/home/delta/islandflow/.env
|
||||||
|
ExecStart=/home/delta/.bun/bin/bun services/api/src/index.ts
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
KillSignal=SIGINT
|
||||||
|
TimeoutStopSec=20
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=default.target
|
||||||
17
deployment/native/systemd/user/islandflow-candles.service
Normal file
17
deployment/native/systemd/user/islandflow-candles.service
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Islandflow candles
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
WorkingDirectory=/home/delta/islandflow
|
||||||
|
EnvironmentFile=/home/delta/islandflow/.env
|
||||||
|
ExecStart=/home/delta/.bun/bin/bun services/candles/src/index.ts
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
KillSignal=SIGINT
|
||||||
|
TimeoutStopSec=20
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=default.target
|
||||||
17
deployment/native/systemd/user/islandflow-compute.service
Normal file
17
deployment/native/systemd/user/islandflow-compute.service
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Islandflow compute
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
WorkingDirectory=/home/delta/islandflow
|
||||||
|
EnvironmentFile=/home/delta/islandflow/.env
|
||||||
|
ExecStart=/home/delta/.bun/bin/bun services/compute/src/index.ts
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
KillSignal=SIGINT
|
||||||
|
TimeoutStopSec=20
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=default.target
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Islandflow ingest-equities
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
WorkingDirectory=/home/delta/islandflow
|
||||||
|
EnvironmentFile=/home/delta/islandflow/.env
|
||||||
|
ExecStart=/home/delta/.bun/bin/bun services/ingest-equities/src/index.ts
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
KillSignal=SIGINT
|
||||||
|
TimeoutStopSec=20
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=default.target
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Islandflow ingest-options
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
WorkingDirectory=/home/delta/islandflow
|
||||||
|
EnvironmentFile=/home/delta/islandflow/.env
|
||||||
|
Environment=OPTIONS_INGEST_ADAPTER=synthetic
|
||||||
|
ExecStart=/home/delta/.bun/bin/bun services/ingest-options/src/index.ts
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
KillSignal=SIGINT
|
||||||
|
TimeoutStopSec=20
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=default.target
|
||||||
19
deployment/native/systemd/user/islandflow-web.service
Normal file
19
deployment/native/systemd/user/islandflow-web.service
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Islandflow web
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
WorkingDirectory=/home/delta/islandflow
|
||||||
|
Environment=WEB_HOST=0.0.0.0
|
||||||
|
Environment=WEB_PORT=3000
|
||||||
|
EnvironmentFile=/home/delta/islandflow/.env
|
||||||
|
ExecStart=/bin/sh -lc 'cd /home/delta/islandflow/apps/web && exec /home/delta/.bun/bin/bun x next start -H "$WEB_HOST" -p "$WEB_PORT"'
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
KillSignal=SIGINT
|
||||||
|
TimeoutStopSec=20
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=default.target
|
||||||
93
docs/plans/2026-05-18-native-fast-iterative-deploy-plan.html
Normal file
93
docs/plans/2026-05-18-native-fast-iterative-deploy-plan.html
Normal file
|
|
@ -0,0 +1,93 @@
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<title>Plan: Native Fast Iterative Deployment</title>
|
||||||
|
<style>
|
||||||
|
body { font-family: Inter, system-ui, sans-serif; margin: 40px auto; max-width: 860px; line-height: 1.55; padding: 0 16px; }
|
||||||
|
h1, h2 { line-height: 1.2; }
|
||||||
|
.meta { color: #555; margin-bottom: 20px; }
|
||||||
|
section { margin: 22px 0; }
|
||||||
|
ul { padding-left: 20px; }
|
||||||
|
code { background: #f3f4f6; padding: 2px 6px; border-radius: 6px; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>Plan: Native, Fast, Iterative Deployment (Docker Optional)</h1>
|
||||||
|
<p class="meta">Date: 2026-05-18</p>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Plan Summary</h2>
|
||||||
|
<p>Define and execute a fast iteration deployment path centered on host-native services, while preserving Docker as a fallback/runtime option.</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Goals</h2>
|
||||||
|
<ul>
|
||||||
|
<li>Reduce deploy turnaround time immediately.</li>
|
||||||
|
<li>Identify concrete bottlenecks with timing evidence.</li>
|
||||||
|
<li>Stabilize proxy/runtime topology for reliable production rollouts.</li>
|
||||||
|
<li>Support both native and Docker strategies with explicit guardrails.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Proposed Changes</h2>
|
||||||
|
<ul>
|
||||||
|
<li>Use scoped fast deploys short-term.</li>
|
||||||
|
<li>Audit and remediate server-state blockers (duplicate compose/project drift).</li>
|
||||||
|
<li>Prepare native runtime prerequisites and checked-in operational assets.</li>
|
||||||
|
<li>Add deployment strategy prechecks, validation matrix, and staged cutover.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Relevant Context</h2>
|
||||||
|
<ul>
|
||||||
|
<li>Open issue <code>islandflow-2db</code>: stale duplicate compose stack cleanup.</li>
|
||||||
|
<li>Open issue <code>islandflow-sz8</code>: public <code>/replay/options</code> proxy regression.</li>
|
||||||
|
<li>Open issue <code>islandflow-38p</code>: native unit templates and rollback helpers.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Implementation Steps</h2>
|
||||||
|
<ol>
|
||||||
|
<li>Stop the bleeding immediately (current deploy loop).</li>
|
||||||
|
<li>Get hard timing data per deploy phase.</li>
|
||||||
|
<li>Live server state audit (when plan mode is off).</li>
|
||||||
|
<li>Resolve duplicate compose stack first (<code>islandflow-2db</code>).</li>
|
||||||
|
<li>Fix NPM proxy route regression (<code>islandflow-sz8</code>).</li>
|
||||||
|
<li>Define target iterative deployment model.</li>
|
||||||
|
<li>Prepare native runtime prerequisites on VPS.</li>
|
||||||
|
<li>Checked-in native ops assets (<code>islandflow-38p</code>).</li>
|
||||||
|
<li>Switch proxy topology for native mode carefully.</li>
|
||||||
|
<li>Deploy strategy guardrails.</li>
|
||||||
|
<li>Validation matrix.</li>
|
||||||
|
<li>Staged cutover plan.</li>
|
||||||
|
<li>Decision: final default runtime.</li>
|
||||||
|
<li>Decision: optimization priority.</li>
|
||||||
|
<li>Decision: immediate live audit kickoff.</li>
|
||||||
|
</ol>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Risks, Limitations, and Mitigations</h2>
|
||||||
|
<ul>
|
||||||
|
<li>Risk: native runtime not yet production-hardened. Mitigation: keep Docker fallback and explicit gating.</li>
|
||||||
|
<li>Risk: proxy misrouting breaks API routes. Mitigation: route checks and post-change smoke validation.</li>
|
||||||
|
<li>Risk: operational drift on VPS. Mitigation: preflight audits and documented rollback steps.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Open Questions</h2>
|
||||||
|
<ul>
|
||||||
|
<li>Should native become the default runtime now, or after hardening milestones?</li>
|
||||||
|
<li>Should backend iteration speed be prioritized ahead of web deploy speed?</li>
|
||||||
|
<li>Do we start immediate live server audit as soon as plan mode is disabled?</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
153
docs/turns/2026-05-18-native-fast-iterative-deploy.html
Normal file
153
docs/turns/2026-05-18-native-fast-iterative-deploy.html
Normal file
|
|
@ -0,0 +1,153 @@
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<title>2026-05-18: Native fast iterative deploy</title>
|
||||||
|
<style>
|
||||||
|
:root {
|
||||||
|
color-scheme: dark;
|
||||||
|
--bg: #0b1020;
|
||||||
|
--panel: #131a2b;
|
||||||
|
--panel-2: #182237;
|
||||||
|
--text: #eef3ff;
|
||||||
|
--muted: #a7b4d4;
|
||||||
|
--line: #2a3651;
|
||||||
|
--accent: #7dd3fc;
|
||||||
|
--good: #86efac;
|
||||||
|
--warn: #fbbf24;
|
||||||
|
}
|
||||||
|
* { box-sizing: border-box; }
|
||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
background: linear-gradient(180deg, #060914, var(--bg));
|
||||||
|
color: var(--text);
|
||||||
|
font: 16px/1.6 Inter, system-ui, sans-serif;
|
||||||
|
}
|
||||||
|
main {
|
||||||
|
max-width: 920px;
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 40px 20px 64px;
|
||||||
|
}
|
||||||
|
section {
|
||||||
|
margin-top: 22px;
|
||||||
|
padding: 22px 24px;
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
border-radius: 18px;
|
||||||
|
background: linear-gradient(180deg, var(--panel), var(--panel-2));
|
||||||
|
}
|
||||||
|
h1, h2 { line-height: 1.15; }
|
||||||
|
h1 { margin: 0 0 12px; font-size: 2rem; }
|
||||||
|
h2 { margin: 0 0 12px; font-size: 1.15rem; }
|
||||||
|
p, li { color: var(--text); }
|
||||||
|
.meta { color: var(--muted); margin-bottom: 18px; }
|
||||||
|
.lede { color: var(--muted); max-width: 72ch; }
|
||||||
|
code, pre { font: 13px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; }
|
||||||
|
code {
|
||||||
|
padding: 0.15rem 0.35rem;
|
||||||
|
border-radius: 8px;
|
||||||
|
background: rgba(125, 211, 252, 0.12);
|
||||||
|
color: var(--accent);
|
||||||
|
}
|
||||||
|
pre {
|
||||||
|
margin: 12px 0 0;
|
||||||
|
padding: 14px 16px;
|
||||||
|
overflow: auto;
|
||||||
|
border-radius: 14px;
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
background: #0a0f1d;
|
||||||
|
}
|
||||||
|
ul { margin: 0; padding-left: 1.2rem; }
|
||||||
|
.good { color: var(--good); }
|
||||||
|
.warn { color: var(--warn); }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<main>
|
||||||
|
<div class="meta">Turn document · 2026-05-18 03:29 EDT · Issues: islandflow-9rc, islandflow-38p, islandflow-bsg, islandflow-2db</div>
|
||||||
|
<h1>Native fast iterative deploy</h1>
|
||||||
|
<p class="lede">Implemented the native-first iterative deploy plan by adding deploy timing output, a safe worker-only native fast path, checked-in systemd user units and rollback helpers, server-local deploy execution, and updated live-operational documentation based on a fresh VPS audit.</p>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Summary</h2>
|
||||||
|
<p>The deploy flow now supports a safer native worker iteration model without requiring public edge cutover first. It can run directly from the VPS checkout without SSH, emits phase timings, includes checked-in native unit files plus install/rollback/smoke-test helpers, and documents the staged cutover path. During live audit, the previously reported <code>/replay/options</code> proxy issue and duplicate <code>islandflow</code> compose stack were both confirmed resolved on the host.</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Changes Made</h2>
|
||||||
|
<ul>
|
||||||
|
<li>Extended <code>scripts/deploy.ts</code> with deploy timing summaries for precheck, rollout, and verification phases.</li>
|
||||||
|
<li>Added <code>--workers-only</code> deploy scope for Docker and native runtimes.</li>
|
||||||
|
<li>Changed native <code>--fast</code> behavior so default full-scope fast deploys become worker-only instead of touching web/API.</li>
|
||||||
|
<li>Added native edge guardrails via <code>DEPLOY_NATIVE_EDGE_READY=1</code> before web/API native deploys are allowed.</li>
|
||||||
|
<li>Added local-server execution mode so <code>./deploy</code> can run from <code>/home/delta/islandflow</code> without SSHing back into the same host.</li>
|
||||||
|
<li>Added <code>DEPLOY_SSH_KEY_PATH</code> and <code>DEPLOY_FORCE_SSH</code> overrides for operators with non-default SSH setups.</li>
|
||||||
|
<li>Checked in native ops assets under <code>deployment/native/</code>:</li>
|
||||||
|
<li><code>install-user-units.sh</code>, <code>check-native-health.sh</code>, <code>rollback.sh</code></li>
|
||||||
|
<li>six user unit files in <code>deployment/native/systemd/user/</code></li>
|
||||||
|
<li>Updated <code>README.md</code>, <code>deployment/docker/README.md</code>, and <code>deployment/native/README.md</code> to document the worker-first model, local execution mode, validation matrix, and staged cutover guidance.</li>
|
||||||
|
<li>Synced <code>deployment/docker/workspace-root/package.json</code> so Docker workspace validation passes again.</li>
|
||||||
|
<li>Installed the checked-in user unit files onto the live VPS in disabled form under <code>~/.config/systemd/user</code>.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Context</h2>
|
||||||
|
<p>The plan targeted faster deployment iteration while avoiding a premature move of the public edge away from the current Docker + Nginx Proxy Manager topology. The practical target was to make native runtime useful immediately for backend-worker iteration, while leaving web/API cutover deliberate and reversible.</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Important Implementation Details</h2>
|
||||||
|
<ul>
|
||||||
|
<li>Native fast mode now defaults to <code>--workers-only</code>; Docker fast mode still defaults to <code>--services-only</code>.</li>
|
||||||
|
<li>Native deploys that include public web/API scope now fail fast unless <code>DEPLOY_NATIVE_EDGE_READY=1</code> is set.</li>
|
||||||
|
<li>Running from the live VPS checkout automatically switches deploy execution from SSH mode to local mode.</li>
|
||||||
|
<li>The checked-in native unit files are user units aimed at the current VPS layout: <code>/home/delta/islandflow</code> and <code>/home/delta/.bun/bin/bun</code>.</li>
|
||||||
|
<li><code>install-user-units.sh</code> now installs units safely without enabling anything by default; enabling is explicit and scope-based.</li>
|
||||||
|
<li><code>rollback.sh</code> intentionally uses a detached git ref to make one-off native rollback practical without rewriting branch history.</li>
|
||||||
|
</ul>
|
||||||
|
<pre>export DEPLOY_NATIVE_SYSTEMCTL_PREFIX="systemctl --user"
|
||||||
|
./deploy main --runtime native --fast
|
||||||
|
# resolves to worker-only native deploy before public edge cutover</pre>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Expected Impact for End-Users</h2>
|
||||||
|
<p>End-users should see indirect benefits first: faster backend iteration, safer operational changes, and clearer rollback paths. Public traffic behavior should remain unchanged until a deliberate native edge cutover is performed.</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Validation</h2>
|
||||||
|
<ul>
|
||||||
|
<li class="good">Passed: <code>bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io</code></li>
|
||||||
|
<li class="good">Passed: direct public <code>/replay/options</code> curl returned JSON</li>
|
||||||
|
<li class="good">Passed: live Nginx Proxy Manager config contains <code>/replay</code> in the API route matcher</li>
|
||||||
|
<li class="good">Passed: <code>docker compose ls</code> shows no duplicate <code>islandflow</code> project</li>
|
||||||
|
<li class="good">Passed: <code>bash -n deployment/native/install-user-units.sh deployment/native/check-native-health.sh deployment/native/rollback.sh</code></li>
|
||||||
|
<li class="good">Passed: <code>systemd-analyze verify deployment/native/systemd/user/*.service</code></li>
|
||||||
|
<li class="good">Passed: <code>bun run check:docker-workspace</code> after syncing workspace snapshot</li>
|
||||||
|
<li class="good">Passed: native edge guard refusal for <code>bun run scripts/deploy.ts main --runtime native --web-only --no-build</code></li>
|
||||||
|
<li class="good">Passed: <code>./deployment/native/install-user-units.sh</code> followed by <code>systemctl --user list-unit-files 'islandflow*'</code></li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Issues, Limitations, and Mitigations</h2>
|
||||||
|
<ul>
|
||||||
|
<li><span class="warn">Native units were installed but not enabled or started.</span> This is intentional to avoid conflicting with the current Docker production edge.</li>
|
||||||
|
<li><span class="warn">Public web/API native deploys are still gated.</span> Mitigation: explicit <code>DEPLOY_NATIVE_EDGE_READY=1</code> acknowledgment and staged cutover documentation.</li>
|
||||||
|
<li><span class="warn">Native worker runtime has not yet been exercised live against the existing Docker worker stack.</span> Mitigation: follow-up issue to soak worker-only native units before any default-runtime decision.</li>
|
||||||
|
<li><span class="warn">The known untracked Signal CLI tarball remains in the repo checkout.</span> This is already tolerated by the deploy helper allowlist and was not changed here.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Follow-up Work</h2>
|
||||||
|
<ul>
|
||||||
|
<li>Open follow-up: <code>islandflow-vvw</code> — stage native public-edge cutover after worker soak.</li>
|
||||||
|
<li>Decide whether native should ever replace Docker as the default runtime only after worker soak data and deliberate edge cutover validation.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
521
docs/turns/2026-05-18-native-public-edge-cutover.html
Normal file
521
docs/turns/2026-05-18-native-public-edge-cutover.html
Normal file
|
|
@ -0,0 +1,521 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<title>Turn Document - Native Public Edge Cutover</title>
|
||||||
|
<style>
|
||||||
|
:root {
|
||||||
|
color-scheme: dark;
|
||||||
|
--bg-core: #06080b;
|
||||||
|
--bg-elevated: #0b1016;
|
||||||
|
--bg-pane: #111820;
|
||||||
|
--bg-pane-2: #0d141b;
|
||||||
|
--bg-soft: rgba(255, 255, 255, 0.03);
|
||||||
|
--border-subtle: rgba(255, 255, 255, 0.12);
|
||||||
|
--border-strong: rgba(245, 166, 35, 0.32);
|
||||||
|
--text-primary: #e6edf4;
|
||||||
|
--text-dim: #90a0b2;
|
||||||
|
--text-faint: #6e7b8c;
|
||||||
|
--signal-amber: #f5a623;
|
||||||
|
--signal-amber-soft: rgba(245, 166, 35, 0.12);
|
||||||
|
--confirm-green: #25c17a;
|
||||||
|
--confirm-green-soft: rgba(37, 193, 122, 0.14);
|
||||||
|
--risk-red: #ff6b5f;
|
||||||
|
--risk-red-soft: rgba(255, 107, 95, 0.12);
|
||||||
|
--info-blue: #4da3ff;
|
||||||
|
--info-blue-soft: rgba(77, 163, 255, 0.12);
|
||||||
|
--shadow: 0 24px 60px rgba(0, 0, 0, 0.35);
|
||||||
|
}
|
||||||
|
|
||||||
|
* {
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
font-family: "IBM Plex Sans", "Segoe UI", sans-serif;
|
||||||
|
background:
|
||||||
|
radial-gradient(circle at top right, rgba(245, 166, 35, 0.12), transparent 28%),
|
||||||
|
linear-gradient(180deg, #06080b 0%, #0a1117 100%);
|
||||||
|
color: var(--text-primary);
|
||||||
|
}
|
||||||
|
|
||||||
|
main {
|
||||||
|
width: min(1080px, calc(100vw - 32px));
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 28px 0 48px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.hero {
|
||||||
|
background:
|
||||||
|
linear-gradient(140deg, rgba(245, 166, 35, 0.1), transparent 42%),
|
||||||
|
linear-gradient(180deg, rgba(255, 255, 255, 0.02), transparent 100%),
|
||||||
|
var(--bg-pane);
|
||||||
|
border: 1px solid var(--border-strong);
|
||||||
|
border-radius: 16px;
|
||||||
|
box-shadow: var(--shadow);
|
||||||
|
padding: 26px 28px;
|
||||||
|
margin-bottom: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.eyebrow,
|
||||||
|
h2,
|
||||||
|
.meta-label,
|
||||||
|
th {
|
||||||
|
font-family: "IBM Plex Mono", monospace;
|
||||||
|
text-transform: uppercase;
|
||||||
|
letter-spacing: 0.12em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.eyebrow {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
color: var(--signal-amber);
|
||||||
|
font-size: 0.72rem;
|
||||||
|
margin-bottom: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 {
|
||||||
|
margin: 0 0 10px;
|
||||||
|
font-family: "Quantico", "IBM Plex Sans", sans-serif;
|
||||||
|
font-size: clamp(2rem, 4vw, 3rem);
|
||||||
|
line-height: 1.05;
|
||||||
|
letter-spacing: 0.06em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lead {
|
||||||
|
margin: 0;
|
||||||
|
max-width: 72ch;
|
||||||
|
color: var(--text-dim);
|
||||||
|
line-height: 1.65;
|
||||||
|
}
|
||||||
|
|
||||||
|
.meta-grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
|
||||||
|
gap: 10px;
|
||||||
|
margin-top: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.meta-card {
|
||||||
|
padding: 12px 14px;
|
||||||
|
border-radius: 12px;
|
||||||
|
background: var(--bg-soft);
|
||||||
|
border: 1px solid var(--border-subtle);
|
||||||
|
}
|
||||||
|
|
||||||
|
.meta-label {
|
||||||
|
color: var(--text-faint);
|
||||||
|
font-size: 0.68rem;
|
||||||
|
margin-bottom: 6px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.meta-value {
|
||||||
|
color: var(--text-primary);
|
||||||
|
font-size: 0.95rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
section {
|
||||||
|
background: var(--bg-pane);
|
||||||
|
border: 1px solid var(--border-subtle);
|
||||||
|
border-radius: 16px;
|
||||||
|
padding: 22px 24px;
|
||||||
|
margin-bottom: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
h2 {
|
||||||
|
margin: 0 0 14px;
|
||||||
|
font-size: 0.76rem;
|
||||||
|
color: var(--signal-amber);
|
||||||
|
}
|
||||||
|
|
||||||
|
p,
|
||||||
|
li {
|
||||||
|
line-height: 1.65;
|
||||||
|
color: var(--text-dim);
|
||||||
|
}
|
||||||
|
|
||||||
|
ul {
|
||||||
|
margin: 0;
|
||||||
|
padding-left: 20px;
|
||||||
|
}
|
||||||
|
|
||||||
|
li + li {
|
||||||
|
margin-top: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
strong {
|
||||||
|
color: var(--text-primary);
|
||||||
|
}
|
||||||
|
|
||||||
|
code {
|
||||||
|
font-family: "IBM Plex Mono", monospace;
|
||||||
|
font-size: 0.92em;
|
||||||
|
color: var(--signal-amber);
|
||||||
|
}
|
||||||
|
|
||||||
|
pre {
|
||||||
|
margin: 12px 0 0;
|
||||||
|
padding: 14px 16px;
|
||||||
|
border-radius: 12px;
|
||||||
|
background: var(--bg-pane-2);
|
||||||
|
border: 1px solid var(--border-subtle);
|
||||||
|
overflow-x: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
pre code {
|
||||||
|
color: var(--text-primary);
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-card {
|
||||||
|
border-radius: 12px;
|
||||||
|
border: 1px solid var(--border-subtle);
|
||||||
|
padding: 14px;
|
||||||
|
background: var(--bg-pane-2);
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-card.good {
|
||||||
|
border-color: rgba(37, 193, 122, 0.32);
|
||||||
|
background: linear-gradient(180deg, var(--confirm-green-soft), transparent), var(--bg-pane-2);
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-card.warn {
|
||||||
|
border-color: rgba(77, 163, 255, 0.28);
|
||||||
|
background: linear-gradient(180deg, var(--info-blue-soft), transparent), var(--bg-pane-2);
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-title {
|
||||||
|
margin: 0 0 6px;
|
||||||
|
color: var(--text-primary);
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-copy {
|
||||||
|
margin: 0;
|
||||||
|
color: var(--text-dim);
|
||||||
|
}
|
||||||
|
|
||||||
|
table {
|
||||||
|
width: 100%;
|
||||||
|
border-collapse: collapse;
|
||||||
|
margin-top: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
th,
|
||||||
|
td {
|
||||||
|
text-align: left;
|
||||||
|
padding: 10px 0;
|
||||||
|
border-bottom: 1px solid var(--border-subtle);
|
||||||
|
vertical-align: top;
|
||||||
|
}
|
||||||
|
|
||||||
|
th {
|
||||||
|
color: var(--text-faint);
|
||||||
|
font-size: 0.68rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
td {
|
||||||
|
color: var(--text-dim);
|
||||||
|
}
|
||||||
|
|
||||||
|
.pill {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
border-radius: 999px;
|
||||||
|
padding: 4px 9px;
|
||||||
|
font-family: "IBM Plex Mono", monospace;
|
||||||
|
font-size: 0.7rem;
|
||||||
|
letter-spacing: 0.08em;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
.pill.good {
|
||||||
|
color: var(--confirm-green);
|
||||||
|
background: var(--confirm-green-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.pill.warn {
|
||||||
|
color: var(--info-blue);
|
||||||
|
background: var(--info-blue-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.pill.risk {
|
||||||
|
color: var(--risk-red);
|
||||||
|
background: var(--risk-red-soft);
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<main>
|
||||||
|
<section class="hero">
|
||||||
|
<div class="eyebrow">Islandflow Turn Document</div>
|
||||||
|
<h1>Native Public Edge Cutover</h1>
|
||||||
|
<p class="lead">
|
||||||
|
Completed the VPS native-first cutover for Islandflow infrastructure and app services while keeping Nginx
|
||||||
|
Proxy Manager as the outer edge and Docker as the rollback path. The final state now serves
|
||||||
|
<code>flow.deltaisland.io</code> and <code>api.flow.deltaisland.io</code> from the native web and API
|
||||||
|
processes, with verified public routing and a documented follow-up for the long-term API Cloudflare posture.
|
||||||
|
</p>
|
||||||
|
<div class="meta-grid">
|
||||||
|
<div class="meta-card">
|
||||||
|
<div class="meta-label">Generated</div>
|
||||||
|
<div class="meta-value">2026-05-18 19:52 EDT</div>
|
||||||
|
</div>
|
||||||
|
<div class="meta-card">
|
||||||
|
<div class="meta-label">Primary Issue</div>
|
||||||
|
<div class="meta-value"><code>islandflow-vvw</code></div>
|
||||||
|
</div>
|
||||||
|
<div class="meta-card">
|
||||||
|
<div class="meta-label">Follow-up</div>
|
||||||
|
<div class="meta-value"><code>islandflow-fl5</code></div>
|
||||||
|
</div>
|
||||||
|
<div class="meta-card">
|
||||||
|
<div class="meta-label">Runtime State</div>
|
||||||
|
<div class="meta-value">Native active, Docker retained for rollback</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Summary</h2>
|
||||||
|
<p>
|
||||||
|
The repository now contains the native infra units, native cutover scripts, Docker fallback adjustments, and
|
||||||
|
public-edge retargeting logic required to run Islandflow natively on the VPS. During validation, the live NPM
|
||||||
|
edge was switched from Docker container-name upstreams to native host ports, the host firewall was adjusted so
|
||||||
|
the NPM bridge could reach the native API, and the separate public API TLS problem was resolved by correcting
|
||||||
|
the Cloudflare DNS state for <code>api.flow.deltaisland.io</code>.
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Changes Made</h2>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
Added checked-in native infra operations under <code>deployment/native/</code>, including
|
||||||
|
<code>bootstrap-infra.sh</code>, <code>check-native-infra.sh</code>, <code>cutover.sh</code>,
|
||||||
|
<code>full-rollback.sh</code>, <code>start-infra.sh</code>, and the native system units for NATS, Redis,
|
||||||
|
and ClickHouse.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Extended native app runtime units so the web and API bind on host-reachable interfaces, and forced the
|
||||||
|
native options ingest service to use the synthetic adapter during the cutover.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Updated <code>services/api</code> to support explicit host binding through <code>API_HOST</code>, and fixed
|
||||||
|
JetStream retention conversion in <code>packages/bus</code> so native services can start cleanly with the
|
||||||
|
configured max-age values.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Updated the Docker fallback assets to publish loopback web/API ports, share durable host data under
|
||||||
|
<code>/var/lib/islandflow</code>, and document the native-to-Docker rollback path.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Reworked <code>deployment/native/switch-npm-edge.sh</code> so it targets the NPM bridge gateway IP instead
|
||||||
|
of <code>host.docker.internal</code>, handles the root-owned NPM SQLite database, synchronizes generated
|
||||||
|
<code>proxy_host</code> configs, and reloads NPM deterministically after the edge switch.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Created Beads follow-up issue <code>islandflow-fl5</code> for the remaining decision about whether
|
||||||
|
<code>api.flow.deltaisland.io</code> should remain DNS-only or be re-proxied through Cloudflare.
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Context</h2>
|
||||||
|
<p>
|
||||||
|
The migration started from a Docker-owned production baseline where NATS, Redis, ClickHouse, API, workers, and
|
||||||
|
web all ran in Compose, while NPM routed Islandflow traffic to Docker service names. That setup blocked a safe
|
||||||
|
native cutover for two reasons: the native services could not reach Docker-only infra reliably, and NPM could
|
||||||
|
not send public traffic to host-native processes without a deliberate upstream retarget.
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
The runtime model for this work is exclusive ownership. Native and Docker are not allowed to run the same API
|
||||||
|
or worker scopes in parallel because JetStream durable consumers would conflict. The objective was therefore a
|
||||||
|
phased handoff, not a mixed soak for the same queues.
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Important Implementation Details</h2>
|
||||||
|
<div class="status-grid">
|
||||||
|
<article class="status-card good">
|
||||||
|
<p class="status-title">NPM edge targeting</p>
|
||||||
|
<p class="status-copy">
|
||||||
|
NPM generates <code>proxy_pass</code> from a runtime-resolved <code>$server</code> variable, so the
|
||||||
|
Docker <code>/etc/hosts</code> alias for <code>host.docker.internal</code> was not sufficient. The switch
|
||||||
|
helper now detects the NPM bridge gateway and uses that IP for native upstreams.
|
||||||
|
</p>
|
||||||
|
</article>
|
||||||
|
<article class="status-card good">
|
||||||
|
<p class="status-title">Firewall path</p>
|
||||||
|
<p class="status-copy">
|
||||||
|
The host UFW policy already allowed port <code>3000</code> but not <code>4000</code>. The live fix was a
|
||||||
|
source-scoped allow for the NPM bridge subnet so the containerized edge could reach the native API.
|
||||||
|
</p>
|
||||||
|
</article>
|
||||||
|
<article class="status-card warn">
|
||||||
|
<p class="status-title">Cloudflare API hostname</p>
|
||||||
|
<p class="status-copy">
|
||||||
|
The API hostname failure was separate from the native cutover. The hostname is now a DNS-only
|
||||||
|
<code>A</code> record pointing at the VPS, which restored public TLS and health responses.
|
||||||
|
</p>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Area</th>
|
||||||
|
<th>Implementation detail</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td><strong>Native API</strong></td>
|
||||||
|
<td>
|
||||||
|
<code>services/api/src/index.ts</code> now accepts <code>API_HOST</code> and passes it to
|
||||||
|
<code>Bun.serve</code>. The native unit sets <code>API_HOST=0.0.0.0</code> and
|
||||||
|
<code>API_PORT=4000</code>.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><strong>Native web</strong></td>
|
||||||
|
<td>
|
||||||
|
The native web unit now starts from <code>apps/web</code> with
|
||||||
|
<code>bun x next start -H "$WEB_HOST" -p "$WEB_PORT"</code>, avoiding the earlier repo-root startup
|
||||||
|
failure and binding the service on <code>0.0.0.0:3000</code>.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><strong>JetStream retention</strong></td>
|
||||||
|
<td>
|
||||||
|
Native startup exposed a retention-unit bug. The shared bus layer now converts stream max-age values with
|
||||||
|
<code>nanos(...)</code> and formats them back with <code>millis(...)</code>.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><strong>Docker fallback</strong></td>
|
||||||
|
<td>
|
||||||
|
Docker Compose now uses <code>ISLANDFLOW_DATA_ROOT=/var/lib/islandflow</code>, publishes loopback
|
||||||
|
ports, and keeps the fallback runtime compatible with the same durable data directories as the native
|
||||||
|
services.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><strong>NPM switch helper</strong></td>
|
||||||
|
<td>
|
||||||
|
The helper now updates both the NPM database and the generated
|
||||||
|
<code>/data/nginx/proxy_host/*.conf</code> files, because a DB-only restart did not reliably rewrite the
|
||||||
|
live configs for Islandflow.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<pre><code>sudo ufw allow proto tcp from 172.18.0.0/16 to any port 4000 comment 'npm bridge to native api'</code></pre>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Expected Impact for End-Users</h2>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
Public web and API traffic now reaches the native Islandflow services, which removes Docker from the primary
|
||||||
|
live request path while keeping the outer edge unchanged.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Same-origin public API routes such as <code>/prints</code>, <code>/history</code>, <code>/replay</code>,
|
||||||
|
<code>/nbbo</code>, and <code>/ws/live</code> continue to resolve correctly through the main app hostname.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
Rollback remains fast and explicit: NPM can be pointed back at Docker service names and the Docker runtime
|
||||||
|
can reclaim the same durable data directories if native operation needs to be abandoned.
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Validation</h2>
|
||||||
|
<div class="status-grid">
|
||||||
|
<article class="status-card good">
|
||||||
|
<div class="pill good">Static checks</div>
|
||||||
|
<ul>
|
||||||
|
<li><code>bun run check:docker-workspace</code></li>
|
||||||
|
<li><code>docker compose -f deployment/docker/docker-compose.yml config --quiet</code></li>
|
||||||
|
<li><code>docker compose -f /home/delta/nginx-proxy-manager/docker-compose.yml config --quiet</code></li>
|
||||||
|
<li><code>bash -n deployment/native/*.sh</code></li>
|
||||||
|
<li><code>systemd-analyze verify deployment/native/systemd/user/*.service deployment/native/systemd/system/*.service</code></li>
|
||||||
|
<li><code>bun build services/api/src/index.ts --target=bun</code></li>
|
||||||
|
<li><code>bun build scripts/deploy.ts --target=bun</code></li>
|
||||||
|
</ul>
|
||||||
|
</article>
|
||||||
|
<article class="status-card good">
|
||||||
|
<div class="pill good">Native runtime</div>
|
||||||
|
<ul>
|
||||||
|
<li><code>./deployment/native/check-native-health.sh full</code></li>
|
||||||
|
<li><code>curl http://127.0.0.1:4000/health</code></li>
|
||||||
|
<li><code>curl -I http://127.0.0.1:3000/</code></li>
|
||||||
|
</ul>
|
||||||
|
</article>
|
||||||
|
<article class="status-card good">
|
||||||
|
<div class="pill good">Public edge</div>
|
||||||
|
<ul>
|
||||||
|
<li><code>curl -I -fksS https://flow.deltaisland.io</code></li>
|
||||||
|
<li><code>curl -fksS https://api.flow.deltaisland.io/health</code></li>
|
||||||
|
<li><code>bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io</code></li>
|
||||||
|
</ul>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Issues, Limitations, and Mitigations</h2>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
The native ingest-options service required an explicit synthetic-adapter override because the environment file
|
||||||
|
still pointed at an Alpaca adapter that was returning <code>401</code> responses. The service now starts
|
||||||
|
cleanly for native cutover, but production adapter selection remains an operational decision.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
The NPM helper still relies on direct config synchronization because NPM did not reliably regenerate the
|
||||||
|
Islandflow proxy files from SQLite changes alone. This is mitigated by keeping the synchronization logic
|
||||||
|
checked in and by reloading NPM as part of the helper itself.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
The final public API recovery currently leaves <code>api.flow.deltaisland.io</code> as a DNS-only hostname.
|
||||||
|
That restored service, but it changes the edge posture relative to the web hostname and should be reviewed
|
||||||
|
deliberately.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
A temporary Cloudflare API token was used to inspect and correct zone state during validation. That token
|
||||||
|
should be rotated outside this repository workflow.
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Follow-up Work</h2>
|
||||||
|
<ul>
|
||||||
|
<li>
|
||||||
|
<code>islandflow-fl5</code>: decide whether <code>api.flow.deltaisland.io</code> should remain DNS-only or
|
||||||
|
be re-proxied through Cloudflare, then re-validate TLS, websocket, and operational behavior for the chosen
|
||||||
|
posture.
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
After operational soak, decide whether native should become the default production runtime or remain a
|
||||||
|
supported alternative with Docker as the preferred steady-state runtime.
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
276
docs/turns/2026-05-19-reconcile-pr-conflicts.html
Normal file
276
docs/turns/2026-05-19-reconcile-pr-conflicts.html
Normal file
|
|
@ -0,0 +1,276 @@
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<title>Reconcile PR Conflicts</title>
|
||||||
|
<style>
|
||||||
|
:root {
|
||||||
|
color-scheme: dark;
|
||||||
|
--bg: oklch(13% 0.018 252);
|
||||||
|
--panel: oklch(18% 0.022 248);
|
||||||
|
--panel-2: oklch(22% 0.026 248);
|
||||||
|
--line: oklch(72% 0.03 248 / 0.18);
|
||||||
|
--text: oklch(93% 0.012 248);
|
||||||
|
--muted: oklch(72% 0.025 248);
|
||||||
|
--faint: oklch(58% 0.025 248);
|
||||||
|
--amber: oklch(78% 0.16 72);
|
||||||
|
--green: oklch(73% 0.16 154);
|
||||||
|
--blue: oklch(70% 0.14 245);
|
||||||
|
--red: oklch(68% 0.18 32);
|
||||||
|
}
|
||||||
|
|
||||||
|
* {
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
background: var(--bg);
|
||||||
|
color: var(--text);
|
||||||
|
font-family: "IBM Plex Sans", ui-sans-serif, system-ui, sans-serif;
|
||||||
|
line-height: 1.55;
|
||||||
|
}
|
||||||
|
|
||||||
|
main {
|
||||||
|
width: min(1120px, calc(100% - 32px));
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 48px 0 72px;
|
||||||
|
}
|
||||||
|
|
||||||
|
header {
|
||||||
|
border-bottom: 1px solid var(--line);
|
||||||
|
padding-bottom: 28px;
|
||||||
|
margin-bottom: 28px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.eyebrow,
|
||||||
|
h1,
|
||||||
|
h2,
|
||||||
|
code,
|
||||||
|
pre {
|
||||||
|
font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
}
|
||||||
|
|
||||||
|
.eyebrow {
|
||||||
|
color: var(--amber);
|
||||||
|
font-size: 0.78rem;
|
||||||
|
font-weight: 700;
|
||||||
|
letter-spacing: 0.12em;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 {
|
||||||
|
margin: 10px 0 12px;
|
||||||
|
font-size: clamp(2rem, 4vw, 3.2rem);
|
||||||
|
line-height: 1.06;
|
||||||
|
letter-spacing: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
h2 {
|
||||||
|
margin: 0 0 12px;
|
||||||
|
font-size: 1rem;
|
||||||
|
letter-spacing: 0.08em;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
p {
|
||||||
|
max-width: 76ch;
|
||||||
|
margin: 0 0 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
a {
|
||||||
|
color: var(--blue);
|
||||||
|
}
|
||||||
|
|
||||||
|
section {
|
||||||
|
padding: 24px 0;
|
||||||
|
border-bottom: 1px solid var(--line);
|
||||||
|
}
|
||||||
|
|
||||||
|
ul {
|
||||||
|
margin: 0;
|
||||||
|
padding-left: 20px;
|
||||||
|
}
|
||||||
|
|
||||||
|
li + li {
|
||||||
|
margin-top: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.summary {
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 1.05rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));
|
||||||
|
gap: 12px;
|
||||||
|
margin-top: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.tile {
|
||||||
|
background: var(--panel);
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
border-radius: 8px;
|
||||||
|
padding: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.tile strong {
|
||||||
|
display: block;
|
||||||
|
margin-bottom: 6px;
|
||||||
|
}
|
||||||
|
|
||||||
|
code {
|
||||||
|
color: var(--text);
|
||||||
|
background: oklch(100% 0 0 / 0.06);
|
||||||
|
border: 1px solid oklch(100% 0 0 / 0.08);
|
||||||
|
border-radius: 6px;
|
||||||
|
padding: 0.08rem 0.28rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
pre {
|
||||||
|
overflow: auto;
|
||||||
|
background: var(--panel-2);
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
border-radius: 8px;
|
||||||
|
padding: 14px;
|
||||||
|
font-size: 0.86rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
pre code {
|
||||||
|
background: transparent;
|
||||||
|
border: 0;
|
||||||
|
padding: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.diff-add {
|
||||||
|
color: var(--green);
|
||||||
|
}
|
||||||
|
|
||||||
|
.diff-del {
|
||||||
|
color: var(--red);
|
||||||
|
}
|
||||||
|
|
||||||
|
.diff-meta {
|
||||||
|
color: var(--faint);
|
||||||
|
}
|
||||||
|
|
||||||
|
.callout {
|
||||||
|
border: 1px solid oklch(78% 0.16 72 / 0.35);
|
||||||
|
background: oklch(78% 0.16 72 / 0.08);
|
||||||
|
border-radius: 8px;
|
||||||
|
padding: 14px;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<main>
|
||||||
|
<header>
|
||||||
|
<div class="eyebrow">Turn document • 2026-05-19 18:56 ET</div>
|
||||||
|
<h1>Reconcile PR Conflicts</h1>
|
||||||
|
<p class="summary">
|
||||||
|
Merged <code>forgejo/main</code> into <code>nextjs-upgrade</code>, resolved the checked-in Beads and README conflicts, kept the native deployment work from main, and updated the JetStream tests for the merged nanosecond retention behavior.
|
||||||
|
</p>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Summary</h2>
|
||||||
|
<p>
|
||||||
|
The PR branch now incorporates the current mainline deployment changes while preserving the Next.js upgrade branch. The only hand-edited conflict resolution was in <code>.beads/issues.jsonl</code> and <code>README.md</code>; the rest of the mainline merge applied cleanly.
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Changes Made</h2>
|
||||||
|
<ul>
|
||||||
|
<li>Resolved <code>.beads/issues.jsonl</code> by keeping issue records from both sides of the merge.</li>
|
||||||
|
<li>Resolved the README deployment workflow section by combining the branch’s command-oriented guidance with main’s newer worker-only, local-server, and native edge cutover notes.</li>
|
||||||
|
<li>Accepted mainline native deployment assets, Docker deployment refinements, API host binding support, deploy timing output, and worker-only deployment scope.</li>
|
||||||
|
<li>Adjusted <code>packages/bus/tests/jetstream.test.ts</code> so retention assertions expect NATS nanoseconds after the merged runtime change.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Context</h2>
|
||||||
|
<p>
|
||||||
|
The branch was clean before the merge, but Forgejo reported PR conflicts against <code>main</code>. Reproducing the merge locally showed conflicts in the Beads export file and the README deployment section. The automatic merge also brought in mainline native deployment work that touched deploy scripts, Docker deployment files, native systemd templates, public edge documentation, the API host setting, and JetStream retention units.
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Important Implementation Details</h2>
|
||||||
|
<div class="grid">
|
||||||
|
<div class="tile">
|
||||||
|
<strong>README resolution</strong>
|
||||||
|
<p>Kept Docker as the recommended VPS path, preserved explicit deploy commands, and added <code>--workers-only</code>, local server execution, and native worker iteration guidance.</p>
|
||||||
|
</div>
|
||||||
|
<div class="tile">
|
||||||
|
<strong>Beads resolution</strong>
|
||||||
|
<p>Removed conflict markers without dropping either branch’s issue records, so Beads history remains complete.</p>
|
||||||
|
</div>
|
||||||
|
<div class="tile">
|
||||||
|
<strong>Test repair</strong>
|
||||||
|
<p>Main now stores JetStream <code>max_age</code> in nanoseconds via NATS helpers. Tests now assert against <code>nanos(...)</code> instead of raw millisecond values.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Relevant Diff Snippets</h2>
|
||||||
|
<p>
|
||||||
|
Diff snippets are presented in the style of <a href="https://diffs.com/docs">diffs.com</a>, using structured additions and deletions for quick review.
|
||||||
|
</p>
|
||||||
|
<pre><code><span class="diff-meta">diff --git a/README.md b/README.md</span>
|
||||||
|
<span class="diff-del">- Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, `--fast`, `--no-build`, and `--force-recreate`.</span>
|
||||||
|
<span class="diff-add">+ Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, `--workers-only`, `--fast`, `--no-build`, and `--force-recreate`.</span>
|
||||||
|
<span class="diff-add">+ When run from `/home/delta/islandflow` on the VPS itself, `./deploy` can execute locally instead of SSHing back into the same server.</span>
|
||||||
|
<span class="diff-del">- Native deployment expects Bun, systemd units, host-reachable infra, and deliberate reverse-proxy changes. The open follow-up is to add native unit templates and rollback helpers.</span>
|
||||||
|
<span class="diff-add">+ Native deployment expects Bun, systemd units, host-reachable infra, and deliberate reverse-proxy changes. Native deploys are intended primarily for worker-only fast iteration until the public edge is cut over deliberately.</span></code></pre>
|
||||||
|
|
||||||
|
<pre><code><span class="diff-meta">diff --git a/packages/bus/tests/jetstream.test.ts b/packages/bus/tests/jetstream.test.ts</span>
|
||||||
|
<span class="diff-del">- import type { JetStreamManager, StreamConfig } from "nats";</span>
|
||||||
|
<span class="diff-add">+ import { nanos, type JetStreamManager, type StreamConfig } from "nats";</span>
|
||||||
|
<span class="diff-del">- max_age: 3_600_000,</span>
|
||||||
|
<span class="diff-add">+ max_age: nanos(3_600_000),</span>
|
||||||
|
<span class="diff-del">- max_age: 43_200_000,</span>
|
||||||
|
<span class="diff-add">+ max_age: nanos(43_200_000),</span></code></pre>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Expected Impact for End-Users</h2>
|
||||||
|
<p>
|
||||||
|
The PR should no longer show merge conflicts against main. Users and operators get the Next.js upgrade branch plus the newer deployment safety work from main, including worker-only native deploy guidance and current Docker deployment notes.
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Validation</h2>
|
||||||
|
<ul>
|
||||||
|
<li><code>git diff --check</code> passed.</li>
|
||||||
|
<li><code>bun run scripts/deploy.ts --help</code> passed.</li>
|
||||||
|
<li><code>bun run check:docker-workspace</code> passed.</li>
|
||||||
|
<li><code>bun test services/api/tests packages/bus/tests</code> passed with 45 tests.</li>
|
||||||
|
<li><code>bun --cwd=apps/web run build</code> passed on Next.js 16.2.6.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Issues, Limitations, and Mitigations</h2>
|
||||||
|
<div class="callout">
|
||||||
|
<p>
|
||||||
|
The first focused test run failed because the merged JetStream implementation correctly returned nanosecond retention values while the existing tests still expected milliseconds. The tests were updated to use the same NATS <code>nanos</code> helper as the runtime behavior, then the suite passed.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>Follow-up Work</h2>
|
||||||
|
<ul>
|
||||||
|
<li>No new follow-up was created from this reconciliation.</li>
|
||||||
|
<li>Existing deployment follow-ups remain in Beads, including native public edge posture and cutover decisions.</li>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -9,7 +9,9 @@ import {
|
||||||
type StreamUpdateConfig,
|
type StreamUpdateConfig,
|
||||||
JSONCodec,
|
JSONCodec,
|
||||||
type JsMsg,
|
type JsMsg,
|
||||||
createInbox
|
createInbox,
|
||||||
|
nanos,
|
||||||
|
millis
|
||||||
} from "nats";
|
} from "nats";
|
||||||
import { getKnownStreamDefinitions, getStreamDefinition, type StreamRetentionClass } from "./streams";
|
import { getKnownStreamDefinitions, getStreamDefinition, type StreamRetentionClass } from "./streams";
|
||||||
|
|
||||||
|
|
@ -164,13 +166,13 @@ export const resolveStreamRetention = (
|
||||||
): Pick<StreamConfig, "max_bytes" | "max_age"> => {
|
): Pick<StreamConfig, "max_bytes" | "max_age"> => {
|
||||||
if (streamClass === "raw") {
|
if (streamClass === "raw") {
|
||||||
return {
|
return {
|
||||||
max_age: parseBoundedNumber(env.STREAM_RAW_MAX_AGE_MS, 3_600_000),
|
max_age: nanos(parseBoundedNumber(env.STREAM_RAW_MAX_AGE_MS, 3_600_000)),
|
||||||
max_bytes: parseBoundedNumber(env.STREAM_RAW_MAX_BYTES, 536_870_912)
|
max_bytes: parseBoundedNumber(env.STREAM_RAW_MAX_BYTES, 536_870_912)
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
max_age: parseBoundedNumber(env.STREAM_DERIVED_MAX_AGE_MS, 43_200_000),
|
max_age: nanos(parseBoundedNumber(env.STREAM_DERIVED_MAX_AGE_MS, 43_200_000)),
|
||||||
max_bytes: parseBoundedNumber(env.STREAM_DERIVED_MAX_BYTES, 268_435_456)
|
max_bytes: parseBoundedNumber(env.STREAM_DERIVED_MAX_BYTES, 268_435_456)
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
@ -417,7 +419,7 @@ const formatBytes = (value: number): string => {
|
||||||
};
|
};
|
||||||
|
|
||||||
const formatRetentionSummary = (config: StreamConfig): string => {
|
const formatRetentionSummary = (config: StreamConfig): string => {
|
||||||
return `age=${formatDurationMs(Number(config.max_age))} bytes=${formatBytes(config.max_bytes)} replicas=${config.num_replicas} retention=${config.retention} discard=${config.discard}`;
|
return `age=${formatDurationMs(millis(Number(config.max_age)))} bytes=${formatBytes(config.max_bytes)} replicas=${config.num_replicas} retention=${config.retention} discard=${config.discard}`;
|
||||||
};
|
};
|
||||||
|
|
||||||
const formatReportLine = (
|
const formatReportLine = (
|
||||||
|
|
@ -442,12 +444,12 @@ const formatReportLine = (
|
||||||
const details = report.retentionDrift
|
const details = report.retentionDrift
|
||||||
.map((delta) => {
|
.map((delta) => {
|
||||||
const desiredValue = delta.field === "max_age"
|
const desiredValue = delta.field === "max_age"
|
||||||
? formatDurationMs(Number(delta.desired))
|
? formatDurationMs(millis(Number(delta.desired)))
|
||||||
: delta.field === "max_bytes"
|
: delta.field === "max_bytes"
|
||||||
? formatBytes(Number(delta.desired))
|
? formatBytes(Number(delta.desired))
|
||||||
: formatStructuredValue(delta.desired);
|
: formatStructuredValue(delta.desired);
|
||||||
const currentValue = delta.field === "max_age"
|
const currentValue = delta.field === "max_age"
|
||||||
? formatDurationMs(Number(delta.current))
|
? formatDurationMs(millis(Number(delta.current)))
|
||||||
: delta.field === "max_bytes"
|
: delta.field === "max_bytes"
|
||||||
? formatBytes(Number(delta.current))
|
? formatBytes(Number(delta.current))
|
||||||
: formatStructuredValue(delta.current);
|
: formatStructuredValue(delta.current);
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { describe, expect, it } from "bun:test";
|
import { describe, expect, it } from "bun:test";
|
||||||
import type { JetStreamManager, StreamConfig } from "nats";
|
import { nanos, type JetStreamManager, type StreamConfig } from "nats";
|
||||||
import {
|
import {
|
||||||
auditStreamConfig,
|
auditStreamConfig,
|
||||||
buildKnownStreamConfig,
|
buildKnownStreamConfig,
|
||||||
|
|
@ -52,14 +52,14 @@ const buildAllKnownConfigs = (env: Record<string, string | undefined> = {}) => {
|
||||||
describe("jetstream retention defaults", () => {
|
describe("jetstream retention defaults", () => {
|
||||||
it("resolves raw defaults to 60m and 512 MiB", () => {
|
it("resolves raw defaults to 60m and 512 MiB", () => {
|
||||||
expect(resolveStreamRetention("raw")).toEqual({
|
expect(resolveStreamRetention("raw")).toEqual({
|
||||||
max_age: 3_600_000,
|
max_age: nanos(3_600_000),
|
||||||
max_bytes: 536_870_912
|
max_bytes: 536_870_912
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("resolves derived defaults to 12h and 256 MiB", () => {
|
it("resolves derived defaults to 12h and 256 MiB", () => {
|
||||||
expect(resolveStreamRetention("derived")).toEqual({
|
expect(resolveStreamRetention("derived")).toEqual({
|
||||||
max_age: 43_200_000,
|
max_age: nanos(43_200_000),
|
||||||
max_bytes: 268_435_456
|
max_bytes: 268_435_456
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
@ -71,7 +71,7 @@ describe("jetstream retention defaults", () => {
|
||||||
STREAM_RAW_MAX_BYTES: "5678"
|
STREAM_RAW_MAX_BYTES: "5678"
|
||||||
})
|
})
|
||||||
).toEqual({
|
).toEqual({
|
||||||
max_age: 1234,
|
max_age: nanos(1234),
|
||||||
max_bytes: 5678
|
max_bytes: 5678
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
21
plans/2026-05-18-native-fast-iterative-deploy-plan.md
Normal file
21
plans/2026-05-18-native-fast-iterative-deploy-plan.md
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
# Native, Fast, Iterative Deployment Plan (Docker Optional)
|
||||||
|
|
||||||
|
Date: 2026-05-18
|
||||||
|
|
||||||
|
## Plan Steps (15)
|
||||||
|
|
||||||
|
1. ☐ Stop the bleeding immediately (current deploy loop).
|
||||||
|
2. ☐ Get hard timing data per deploy phase.
|
||||||
|
3. ☐ Live server state audit (when plan mode is off).
|
||||||
|
4. ☐ Resolve duplicate compose stack first (islandflow-2db).
|
||||||
|
5. ☐ Fix NPM proxy route regression (islandflow-sz8).
|
||||||
|
6. ☐ Define target iterative deployment model.
|
||||||
|
7. ☐ Prepare native runtime prerequisites on VPS.
|
||||||
|
8. ☐ Checked-in native ops assets (islandflow-38p).
|
||||||
|
9. ☐ Switch proxy topology for native mode carefully.
|
||||||
|
10. ☐ Deploy strategy guardrails.
|
||||||
|
11. ☐ Validation matrix.
|
||||||
|
12. ☐ Staged cutover plan.
|
||||||
|
13. ☐ Decision: final default runtime.
|
||||||
|
14. ☐ Decision: optimization priority.
|
||||||
|
15. ☐ Decision: immediate live audit kickoff.
|
||||||
|
|
@ -7,7 +7,7 @@ import { fileURLToPath } from "node:url";
|
||||||
|
|
||||||
type DeployMode = "main" | "current-branch";
|
type DeployMode = "main" | "current-branch";
|
||||||
type DeployRuntime = "docker" | "native";
|
type DeployRuntime = "docker" | "native";
|
||||||
type DeployScope = "full" | "web" | "api" | "services";
|
type DeployScope = "full" | "web" | "api" | "services" | "workers";
|
||||||
|
|
||||||
type DeployOptions = {
|
type DeployOptions = {
|
||||||
mode: DeployMode;
|
mode: DeployMode;
|
||||||
|
|
@ -18,10 +18,18 @@ type DeployOptions = {
|
||||||
noBuild: boolean;
|
noBuild: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type PhaseTiming = {
|
||||||
|
name: string;
|
||||||
|
durationMs: number;
|
||||||
|
};
|
||||||
|
|
||||||
const REMOTE_HOST = "delta@152.53.80.229";
|
const REMOTE_HOST = "delta@152.53.80.229";
|
||||||
const REMOTE_REPO = "/home/delta/islandflow";
|
const REMOTE_REPO = "/home/delta/islandflow";
|
||||||
const REMOTE_DOCKER_DEPLOYMENT = "/home/delta/islandflow/deployment/docker";
|
const REMOTE_DOCKER_DEPLOYMENT = "/home/delta/islandflow/deployment/docker";
|
||||||
const SSH_KEY = path.join(process.env.HOME ?? "", ".ssh", "delta_ed25519");
|
const SSH_KEY =
|
||||||
|
process.env.DEPLOY_SSH_KEY_PATH?.trim() ||
|
||||||
|
path.join(process.env.HOME ?? "", ".ssh", "delta_ed25519");
|
||||||
|
const DEPLOY_FORCE_SSH = process.env.DEPLOY_FORCE_SSH?.trim() === "1";
|
||||||
const SSH_OPTIONS = [
|
const SSH_OPTIONS = [
|
||||||
"-i",
|
"-i",
|
||||||
SSH_KEY,
|
SSH_KEY,
|
||||||
|
|
@ -38,6 +46,7 @@ const PUBLIC_APP_URL =
|
||||||
const PUBLIC_API_HEALTH_URL =
|
const PUBLIC_API_HEALTH_URL =
|
||||||
process.env.DEPLOY_PUBLIC_API_HEALTH_URL?.trim() || null;
|
process.env.DEPLOY_PUBLIC_API_HEALTH_URL?.trim() || null;
|
||||||
const DEPLOY_GIT_REMOTE_OVERRIDE = process.env.DEPLOY_GIT_REMOTE?.trim() || null;
|
const DEPLOY_GIT_REMOTE_OVERRIDE = process.env.DEPLOY_GIT_REMOTE?.trim() || null;
|
||||||
|
const DEPLOY_NATIVE_EDGE_READY = process.env.DEPLOY_NATIVE_EDGE_READY?.trim() === "1";
|
||||||
const NATIVE_SYSTEMCTL_PREFIX =
|
const NATIVE_SYSTEMCTL_PREFIX =
|
||||||
process.env.DEPLOY_NATIVE_SYSTEMCTL_PREFIX?.trim() || "sudo -n systemctl";
|
process.env.DEPLOY_NATIVE_SYSTEMCTL_PREFIX?.trim() || "sudo -n systemctl";
|
||||||
const NATIVE_UNITS = {
|
const NATIVE_UNITS = {
|
||||||
|
|
@ -68,15 +77,22 @@ const DOCKER_BACKEND_SERVICES = [
|
||||||
"ingest-equities",
|
"ingest-equities",
|
||||||
"ingest-news"
|
"ingest-news"
|
||||||
] as const;
|
] as const;
|
||||||
|
const DOCKER_WORKER_SERVICES = [
|
||||||
|
"compute",
|
||||||
|
"candles",
|
||||||
|
"ingest-options",
|
||||||
|
"ingest-equities"
|
||||||
|
] as const;
|
||||||
|
|
||||||
const scriptPath = fileURLToPath(import.meta.url);
|
const scriptPath = fileURLToPath(import.meta.url);
|
||||||
const repoRoot = path.resolve(path.dirname(scriptPath), "..");
|
const repoRoot = path.resolve(path.dirname(scriptPath), "..");
|
||||||
|
const isLocalServerExecution = !DEPLOY_FORCE_SSH && repoRoot === REMOTE_REPO;
|
||||||
|
|
||||||
function usage(exitCode = 1): never {
|
function usage(exitCode = 1): never {
|
||||||
console.error(`Usage:
|
console.error(`Usage:
|
||||||
./deploy main [--runtime docker|native] [--web-only|--api-only|--services-only] [--fast] [--no-build] [--force-recreate]
|
./deploy main [--runtime docker|native] [--web-only|--api-only|--services-only|--workers-only] [--fast] [--no-build] [--force-recreate]
|
||||||
./deploy current-branch [--runtime docker|native] [--web-only|--api-only|--services-only] [--fast] [--no-build] [--force-recreate]
|
./deploy current-branch [--runtime docker|native] [--web-only|--api-only|--services-only|--workers-only] [--fast] [--no-build] [--force-recreate]
|
||||||
./deploy current branch [--runtime docker|native] [--web-only|--api-only|--services-only] [--fast] [--no-build] [--force-recreate]
|
./deploy current branch [--runtime docker|native] [--web-only|--api-only|--services-only|--workers-only] [--fast] [--no-build] [--force-recreate]
|
||||||
|
|
||||||
Modes:
|
Modes:
|
||||||
main Deploy <remote>/main to the live server checkout.
|
main Deploy <remote>/main to the live server checkout.
|
||||||
|
|
@ -91,18 +107,22 @@ Scopes:
|
||||||
--web-only Deploy only the Next.js web surface.
|
--web-only Deploy only the Next.js web surface.
|
||||||
--api-only Deploy only the API service.
|
--api-only Deploy only the API service.
|
||||||
--services-only Deploy API + backend services without the web service.
|
--services-only Deploy API + backend services without the web service.
|
||||||
|
--workers-only Deploy compute/candles/ingest workers without touching web or API.
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
--runtime <name> Explicit runtime selector (docker or native).
|
--runtime <name> Explicit runtime selector (docker or native).
|
||||||
--fast Prefer a quicker rollout profile (defaults full scope to --services-only and skips public API route suite).
|
--fast Prefer a quicker rollout profile (defaults full scope to --services-only for docker and --workers-only for native, and skips the public API route suite when API scope is included).
|
||||||
--no-build Skip docker image builds or native bun install/web build steps.
|
--no-build Skip docker image builds or native bun install/web build steps.
|
||||||
--force-recreate Docker-only escalation path for docker compose when a normal refresh is not enough.
|
--force-recreate Docker-only escalation path for docker compose when a normal refresh is not enough.
|
||||||
--help Show this help text.
|
--help Show this help text.
|
||||||
|
|
||||||
Environment:
|
Environment:
|
||||||
DEPLOY_GIT_REMOTE Override git remote used for deploy fetch/pull/push (auto-detected by default).
|
DEPLOY_GIT_REMOTE Override git remote used for deploy fetch/pull/push (auto-detected by default).
|
||||||
|
DEPLOY_SSH_KEY_PATH Override the SSH key used for remote execution.
|
||||||
|
DEPLOY_FORCE_SSH Set to 1 to force SSH even when running from the live server checkout.
|
||||||
DEPLOY_PUBLIC_APP_URL Override the public app URL (default: https://flow.deltaisland.io).
|
DEPLOY_PUBLIC_APP_URL Override the public app URL (default: https://flow.deltaisland.io).
|
||||||
DEPLOY_PUBLIC_API_HEALTH_URL Optional separate public API health URL for two-origin deployments.
|
DEPLOY_PUBLIC_API_HEALTH_URL Optional separate public API health URL for two-origin deployments.
|
||||||
|
DEPLOY_NATIVE_EDGE_READY Set to 1 to allow native rollouts that include the public web or API edge.
|
||||||
DEPLOY_NATIVE_SYSTEMCTL_PREFIX Override systemctl invocation for native rollouts (default: sudo -n systemctl).
|
DEPLOY_NATIVE_SYSTEMCTL_PREFIX Override systemctl invocation for native rollouts (default: sudo -n systemctl).
|
||||||
DEPLOY_NATIVE_WEB_UNIT Override native web systemd unit name.
|
DEPLOY_NATIVE_WEB_UNIT Override native web systemd unit name.
|
||||||
DEPLOY_NATIVE_API_UNIT Override native api systemd unit name.
|
DEPLOY_NATIVE_API_UNIT Override native api systemd unit name.
|
||||||
|
|
@ -118,6 +138,32 @@ function section(title: string): void {
|
||||||
console.log(`\n== ${title} ==`);
|
console.log(`\n== ${title} ==`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatDuration(durationMs: number): string {
|
||||||
|
if (durationMs < 1000) {
|
||||||
|
return `${durationMs}ms`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return `${(durationMs / 1000).toFixed(2)}s`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function timedPhase<T>(timings: PhaseTiming[], name: string, fn: () => T): T {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
try {
|
||||||
|
return fn();
|
||||||
|
} finally {
|
||||||
|
timings.push({ name, durationMs: Date.now() - startedAt });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function printTimingSummary(timings: PhaseTiming[]): void {
|
||||||
|
section("Deploy Timings");
|
||||||
|
const totalMs = timings.reduce((sum, timing) => sum + timing.durationMs, 0);
|
||||||
|
for (const timing of timings) {
|
||||||
|
console.log(`[deploy] ${timing.name}: ${formatDuration(timing.durationMs)}`);
|
||||||
|
}
|
||||||
|
console.log(`[deploy] total: ${formatDuration(totalMs)}`);
|
||||||
|
}
|
||||||
|
|
||||||
function formatCommand(command: string, args: string[]): string {
|
function formatCommand(command: string, args: string[]): string {
|
||||||
return [command, ...args]
|
return [command, ...args]
|
||||||
.map((part) => (/\s/.test(part) ? JSON.stringify(part) : part))
|
.map((part) => (/\s/.test(part) ? JSON.stringify(part) : part))
|
||||||
|
|
@ -184,6 +230,23 @@ function runRemoteScript(
|
||||||
args: string[] = []
|
args: string[] = []
|
||||||
): void {
|
): void {
|
||||||
section(title);
|
section(title);
|
||||||
|
|
||||||
|
if (isLocalServerExecution) {
|
||||||
|
const localArgs = ["-s", "--", ...args];
|
||||||
|
console.log(`$ ${formatCommand("bash", localArgs)} # local server execution`);
|
||||||
|
const result = spawnSync("bash", localArgs, {
|
||||||
|
cwd: repoRoot,
|
||||||
|
input: script,
|
||||||
|
encoding: "utf8",
|
||||||
|
stdio: ["pipe", "inherit", "inherit"]
|
||||||
|
});
|
||||||
|
|
||||||
|
if (result.status !== 0) {
|
||||||
|
process.exit(result.status ?? 1);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const sshArgs = [...SSH_OPTIONS, REMOTE_HOST, "bash", "-s", "--", ...args];
|
const sshArgs = [...SSH_OPTIONS, REMOTE_HOST, "bash", "-s", "--", ...args];
|
||||||
console.log(`$ ${formatCommand("ssh", sshArgs)}`);
|
console.log(`$ ${formatCommand("ssh", sshArgs)}`);
|
||||||
const result = spawnSync("ssh", sshArgs, {
|
const result = spawnSync("ssh", sshArgs, {
|
||||||
|
|
@ -225,11 +288,14 @@ function parseScope(rawArgs: string[]): DeployScope {
|
||||||
const scopes = [
|
const scopes = [
|
||||||
rawArgs.includes("--web-only") ? "web" : null,
|
rawArgs.includes("--web-only") ? "web" : null,
|
||||||
rawArgs.includes("--api-only") ? "api" : null,
|
rawArgs.includes("--api-only") ? "api" : null,
|
||||||
rawArgs.includes("--services-only") ? "services" : null
|
rawArgs.includes("--services-only") ? "services" : null,
|
||||||
|
rawArgs.includes("--workers-only") ? "workers" : null
|
||||||
].filter((value): value is Exclude<DeployScope, "full"> => value !== null);
|
].filter((value): value is Exclude<DeployScope, "full"> => value !== null);
|
||||||
|
|
||||||
if (scopes.length > 1) {
|
if (scopes.length > 1) {
|
||||||
console.error("Choose only one deploy scope flag: --web-only, --api-only, or --services-only.");
|
console.error(
|
||||||
|
"Choose only one deploy scope flag: --web-only, --api-only, --services-only, or --workers-only."
|
||||||
|
);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -254,6 +320,7 @@ function parseArgs(rawArgs: string[]): DeployOptions {
|
||||||
arg !== "--web-only" &&
|
arg !== "--web-only" &&
|
||||||
arg !== "--api-only" &&
|
arg !== "--api-only" &&
|
||||||
arg !== "--services-only" &&
|
arg !== "--services-only" &&
|
||||||
|
arg !== "--workers-only" &&
|
||||||
arg !== "--runtime" &&
|
arg !== "--runtime" &&
|
||||||
rawArgs[index - 1] !== "--runtime" &&
|
rawArgs[index - 1] !== "--runtime" &&
|
||||||
!arg.startsWith("--runtime=")
|
!arg.startsWith("--runtime=")
|
||||||
|
|
@ -286,8 +353,13 @@ function parseArgs(rawArgs: string[]): DeployOptions {
|
||||||
}
|
}
|
||||||
|
|
||||||
function assertSshKeyExists(): void {
|
function assertSshKeyExists(): void {
|
||||||
|
if (isLocalServerExecution) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (!existsSync(SSH_KEY)) {
|
if (!existsSync(SSH_KEY)) {
|
||||||
console.error(`Missing SSH key: ${SSH_KEY}`);
|
console.error(`Missing SSH key: ${SSH_KEY}`);
|
||||||
|
console.error("Set DEPLOY_SSH_KEY_PATH or run from the live server checkout without DEPLOY_FORCE_SSH.");
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -402,14 +474,16 @@ function describeScope(scope: DeployScope): string {
|
||||||
return "api only";
|
return "api only";
|
||||||
case "services":
|
case "services":
|
||||||
return "api + backend services";
|
return "api + backend services";
|
||||||
|
case "workers":
|
||||||
|
return "worker services only";
|
||||||
default:
|
default:
|
||||||
return "full stack";
|
return "full stack";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function effectiveScope(scope: DeployScope, fast: boolean): DeployScope {
|
function effectiveScope(scope: DeployScope, runtime: DeployRuntime, fast: boolean): DeployScope {
|
||||||
if (fast && scope === "full") {
|
if (fast && scope === "full") {
|
||||||
return "services";
|
return runtime === "native" ? "workers" : "services";
|
||||||
}
|
}
|
||||||
return scope;
|
return scope;
|
||||||
}
|
}
|
||||||
|
|
@ -422,6 +496,10 @@ function scopeIncludesApi(scope: DeployScope): boolean {
|
||||||
return scope === "full" || scope === "api" || scope === "services";
|
return scope === "full" || scope === "api" || scope === "services";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function scopeTouchesPublicEdge(scope: DeployScope): boolean {
|
||||||
|
return scopeIncludesWeb(scope) || scopeIncludesApi(scope);
|
||||||
|
}
|
||||||
|
|
||||||
function dockerServicesForScope(scope: DeployScope): string[] {
|
function dockerServicesForScope(scope: DeployScope): string[] {
|
||||||
switch (scope) {
|
switch (scope) {
|
||||||
case "web":
|
case "web":
|
||||||
|
|
@ -430,6 +508,8 @@ function dockerServicesForScope(scope: DeployScope): string[] {
|
||||||
return ["api"];
|
return ["api"];
|
||||||
case "services":
|
case "services":
|
||||||
return [...DOCKER_BACKEND_SERVICES];
|
return [...DOCKER_BACKEND_SERVICES];
|
||||||
|
case "workers":
|
||||||
|
return [...DOCKER_WORKER_SERVICES];
|
||||||
default:
|
default:
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
@ -452,6 +532,8 @@ function dockerLogServicesForScope(scope: DeployScope): string[] {
|
||||||
return ["api"];
|
return ["api"];
|
||||||
case "services":
|
case "services":
|
||||||
return [...DOCKER_BACKEND_SERVICES];
|
return [...DOCKER_BACKEND_SERVICES];
|
||||||
|
case "workers":
|
||||||
|
return [...DOCKER_WORKER_SERVICES];
|
||||||
default:
|
default:
|
||||||
return [...DOCKER_CORE_SERVICES];
|
return [...DOCKER_CORE_SERVICES];
|
||||||
}
|
}
|
||||||
|
|
@ -472,6 +554,13 @@ function nativeUnitsForScope(scope: DeployScope): string[] {
|
||||||
NATIVE_UNITS.ingestEquities,
|
NATIVE_UNITS.ingestEquities,
|
||||||
NATIVE_UNITS.ingestNews
|
NATIVE_UNITS.ingestNews
|
||||||
];
|
];
|
||||||
|
case "workers":
|
||||||
|
return [
|
||||||
|
NATIVE_UNITS.compute,
|
||||||
|
NATIVE_UNITS.candles,
|
||||||
|
NATIVE_UNITS.ingestOptions,
|
||||||
|
NATIVE_UNITS.ingestEquities
|
||||||
|
];
|
||||||
default:
|
default:
|
||||||
return [
|
return [
|
||||||
NATIVE_UNITS.web,
|
NATIVE_UNITS.web,
|
||||||
|
|
@ -500,19 +589,46 @@ function localDockerWorkspaceSnapshotPrecheck(): void {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function localRuntimePrecheck(runtime: DeployRuntime, noBuild: boolean): void {
|
function assertNativeEdgeReady(scope: DeployScope): void {
|
||||||
|
if (!scopeTouchesPublicEdge(scope) || DEPLOY_NATIVE_EDGE_READY) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error(
|
||||||
|
"Refusing native deploy that touches public web/API scope before edge cutover is acknowledged."
|
||||||
|
);
|
||||||
|
console.error(
|
||||||
|
"Set DEPLOY_NATIVE_EDGE_READY=1 only after proxy routing and native units for the public edge are intentionally prepared."
|
||||||
|
);
|
||||||
|
console.error(
|
||||||
|
"For fast iterative backend deploys before cutover, use --runtime native --workers-only or --runtime native --fast."
|
||||||
|
);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
function localRuntimePrecheck(runtime: DeployRuntime, scope: DeployScope, noBuild: boolean): void {
|
||||||
if (runtime === "docker" && !noBuild) {
|
if (runtime === "docker" && !noBuild) {
|
||||||
localDockerWorkspaceSnapshotPrecheck();
|
localDockerWorkspaceSnapshotPrecheck();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (runtime === "native") {
|
||||||
|
assertNativeEdgeReady(scope);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function localMainPrecheck(remote: string, runtime: DeployRuntime, noBuild: boolean): void {
|
function localMainPrecheck(
|
||||||
|
remote: string,
|
||||||
|
runtime: DeployRuntime,
|
||||||
|
scope: DeployScope,
|
||||||
|
noBuild: boolean
|
||||||
|
): void {
|
||||||
section("Local Precheck");
|
section("Local Precheck");
|
||||||
runChecked("git", ["fetch", remote]);
|
runChecked("git", ["fetch", remote]);
|
||||||
runChecked("git", ["status", "--short", "--branch"]);
|
runChecked("git", ["status", "--short", "--branch"]);
|
||||||
runChecked("git", ["rev-parse", "--verify", "HEAD"]);
|
runChecked("git", ["rev-parse", "--verify", "HEAD"]);
|
||||||
runChecked("git", ["rev-parse", `${remote}/main`]);
|
runChecked("git", ["rev-parse", `${remote}/main`]);
|
||||||
localRuntimePrecheck(runtime, noBuild);
|
localRuntimePrecheck(runtime, scope, noBuild);
|
||||||
}
|
}
|
||||||
|
|
||||||
function currentBranchName(): string {
|
function currentBranchName(): string {
|
||||||
|
|
@ -528,6 +644,7 @@ function localBranchPrecheck(
|
||||||
remote: string,
|
remote: string,
|
||||||
branch: string,
|
branch: string,
|
||||||
runtime: DeployRuntime,
|
runtime: DeployRuntime,
|
||||||
|
scope: DeployScope,
|
||||||
noBuild: boolean
|
noBuild: boolean
|
||||||
): void {
|
): void {
|
||||||
section("Local Precheck");
|
section("Local Precheck");
|
||||||
|
|
@ -543,7 +660,7 @@ function localBranchPrecheck(
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
localRuntimePrecheck(runtime, noBuild);
|
localRuntimePrecheck(runtime, scope, noBuild);
|
||||||
}
|
}
|
||||||
|
|
||||||
function publishCurrentBranch(remote: string, branch: string): void {
|
function publishCurrentBranch(remote: string, branch: string): void {
|
||||||
|
|
@ -809,6 +926,10 @@ function remoteNativeVerification(scope: DeployScope, fast: boolean): void {
|
||||||
const units = nativeUnitsForScope(scope).map((value) => shellEscape(value)).join(" ");
|
const units = nativeUnitsForScope(scope).map((value) => shellEscape(value)).join(" ");
|
||||||
const checks: string[] = [];
|
const checks: string[] = [];
|
||||||
|
|
||||||
|
if (scope === "full" || scope === "api" || scope === "services" || scope === "workers") {
|
||||||
|
checks.push("./deployment/native/check-native-infra.sh");
|
||||||
|
}
|
||||||
|
|
||||||
if (scopeIncludesApi(scope)) {
|
if (scopeIncludesApi(scope)) {
|
||||||
checks.push('curl -fksS http://127.0.0.1:4000/health');
|
checks.push('curl -fksS http://127.0.0.1:4000/health');
|
||||||
}
|
}
|
||||||
|
|
@ -843,10 +964,10 @@ function remoteVerification(runtime: DeployRuntime, scope: DeployScope, fast: bo
|
||||||
|
|
||||||
function publicVerification(scope: DeployScope, fast: boolean): void {
|
function publicVerification(scope: DeployScope, fast: boolean): void {
|
||||||
section("Public Verification");
|
section("Public Verification");
|
||||||
if (!fast || scopeIncludesWeb(scope)) {
|
if (scopeIncludesWeb(scope)) {
|
||||||
runChecked("curl", ["-I", "-fksS", PUBLIC_APP_URL]);
|
runChecked("curl", ["-I", "-fksS", PUBLIC_APP_URL]);
|
||||||
} else {
|
} else {
|
||||||
console.log("[deploy] Fast mode: skipping public app HEAD check because web scope is not included.");
|
console.log("[deploy] Skipping public app HEAD check because web scope is not included.");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (scopeIncludesApi(scope) && PUBLIC_API_HEALTH_URL) {
|
if (scopeIncludesApi(scope) && PUBLIC_API_HEALTH_URL) {
|
||||||
|
|
@ -867,7 +988,8 @@ function publicVerification(scope: DeployScope, fast: boolean): void {
|
||||||
|
|
||||||
function main(): void {
|
function main(): void {
|
||||||
const options = parseArgs(process.argv.slice(2));
|
const options = parseArgs(process.argv.slice(2));
|
||||||
const scope = effectiveScope(options.scope, options.fast);
|
const scope = effectiveScope(options.scope, options.runtime, options.fast);
|
||||||
|
const timings: PhaseTiming[] = [];
|
||||||
const currentBranch = options.mode === "current-branch" ? currentBranchName() : null;
|
const currentBranch = options.mode === "current-branch" ? currentBranchName() : null;
|
||||||
const deployRemote = resolveDeployRemote(options.mode, currentBranch);
|
const deployRemote = resolveDeployRemote(options.mode, currentBranch);
|
||||||
assertSshKeyExists();
|
assertSshKeyExists();
|
||||||
|
|
@ -878,22 +1000,33 @@ function main(): void {
|
||||||
`via ${describeRuntime(options.runtime)} (${describeScope(scope)}${options.fast ? ", fast mode" : ""}).`
|
`via ${describeRuntime(options.runtime)} (${describeScope(scope)}${options.fast ? ", fast mode" : ""}).`
|
||||||
);
|
);
|
||||||
console.log(`[deploy] Using git remote: ${deployRemote}`);
|
console.log(`[deploy] Using git remote: ${deployRemote}`);
|
||||||
|
console.log(
|
||||||
|
`[deploy] Execution mode: ${isLocalServerExecution ? "local server checkout" : `ssh to ${REMOTE_HOST}`}`
|
||||||
|
);
|
||||||
if (options.fast && options.scope === "full") {
|
if (options.fast && options.scope === "full") {
|
||||||
console.log("[deploy] Fast mode changed default full scope to --services-only.");
|
console.log(
|
||||||
|
`[deploy] Fast mode changed default full scope to ${options.runtime === "native" ? "--workers-only" : "--services-only"}.`
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (options.mode === "main") {
|
if (options.mode === "main") {
|
||||||
localMainPrecheck(deployRemote, options.runtime, options.noBuild);
|
timedPhase(timings, "local precheck", () =>
|
||||||
remoteGitPrecheck();
|
localMainPrecheck(deployRemote, options.runtime, scope, options.noBuild)
|
||||||
remoteRuntimePrecheck(options.runtime, scope);
|
);
|
||||||
remoteRollout(
|
timedPhase(timings, "remote git precheck", () => remoteGitPrecheck());
|
||||||
options.mode,
|
timedPhase(timings, "remote runtime precheck", () =>
|
||||||
deployRemote,
|
remoteRuntimePrecheck(options.runtime, scope)
|
||||||
options.runtime,
|
);
|
||||||
null,
|
timedPhase(timings, "remote rollout", () =>
|
||||||
scope,
|
remoteRollout(
|
||||||
options.forceRecreate,
|
options.mode,
|
||||||
options.noBuild
|
deployRemote,
|
||||||
|
options.runtime,
|
||||||
|
null,
|
||||||
|
scope,
|
||||||
|
options.forceRecreate,
|
||||||
|
options.noBuild
|
||||||
|
)
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
const branch = currentBranch;
|
const branch = currentBranch;
|
||||||
|
|
@ -901,23 +1034,34 @@ function main(): void {
|
||||||
console.error("Unable to resolve current branch for current-branch deploy mode.");
|
console.error("Unable to resolve current branch for current-branch deploy mode.");
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
localBranchPrecheck(deployRemote, branch, options.runtime, options.noBuild);
|
timedPhase(timings, "local precheck", () =>
|
||||||
publishCurrentBranch(deployRemote, branch);
|
localBranchPrecheck(deployRemote, branch, options.runtime, scope, options.noBuild)
|
||||||
remoteGitPrecheck();
|
);
|
||||||
remoteRuntimePrecheck(options.runtime, scope);
|
timedPhase(timings, "local publish", () => publishCurrentBranch(deployRemote, branch));
|
||||||
remoteRollout(
|
timedPhase(timings, "remote git precheck", () => remoteGitPrecheck());
|
||||||
options.mode,
|
timedPhase(timings, "remote runtime precheck", () =>
|
||||||
deployRemote,
|
remoteRuntimePrecheck(options.runtime, scope)
|
||||||
options.runtime,
|
);
|
||||||
branch,
|
timedPhase(timings, "remote rollout", () =>
|
||||||
scope,
|
remoteRollout(
|
||||||
options.forceRecreate,
|
options.mode,
|
||||||
options.noBuild
|
deployRemote,
|
||||||
|
options.runtime,
|
||||||
|
branch,
|
||||||
|
scope,
|
||||||
|
options.forceRecreate,
|
||||||
|
options.noBuild
|
||||||
|
)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
remoteVerification(options.runtime, scope, options.fast);
|
timedPhase(timings, "remote verification", () =>
|
||||||
publicVerification(scope, options.fast);
|
remoteVerification(options.runtime, scope, options.fast)
|
||||||
|
);
|
||||||
|
timedPhase(timings, "public verification", () =>
|
||||||
|
publicVerification(scope, options.fast)
|
||||||
|
);
|
||||||
|
printTimingSummary(timings);
|
||||||
}
|
}
|
||||||
|
|
||||||
main();
|
main();
|
||||||
|
|
|
||||||
|
|
@ -145,6 +145,7 @@ const DeliverPolicySchema = z.enum(["new", "all", "last", "last_per_subject"]);
|
||||||
|
|
||||||
const envSchema = z.object({
|
const envSchema = z.object({
|
||||||
API_PORT: z.coerce.number().int().positive().default(4000),
|
API_PORT: z.coerce.number().int().positive().default(4000),
|
||||||
|
API_HOST: z.string().min(1).default("127.0.0.1"),
|
||||||
NATS_URL: z.string().default("nats://127.0.0.1:4222"),
|
NATS_URL: z.string().default("nats://127.0.0.1:4222"),
|
||||||
CLICKHOUSE_URL: z.string().default("http://127.0.0.1:8123"),
|
CLICKHOUSE_URL: z.string().default("http://127.0.0.1:8123"),
|
||||||
CLICKHOUSE_DATABASE: z.string().default("default"),
|
CLICKHOUSE_DATABASE: z.string().default("default"),
|
||||||
|
|
@ -1349,6 +1350,7 @@ const run = async () => {
|
||||||
};
|
};
|
||||||
|
|
||||||
const server = Bun.serve<WsData | LiveWsData>({
|
const server = Bun.serve<WsData | LiveWsData>({
|
||||||
|
hostname: env.API_HOST,
|
||||||
port: env.API_PORT,
|
port: env.API_PORT,
|
||||||
fetch: async (req: Request, serverRef: any) => {
|
fetch: async (req: Request, serverRef: any) => {
|
||||||
const url = new URL(req.url);
|
const url = new URL(req.url);
|
||||||
|
|
@ -2045,7 +2047,7 @@ const run = async () => {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
logger.info("api listening", { port: server.port });
|
logger.info("api listening", { host: env.API_HOST, port: server.port });
|
||||||
|
|
||||||
const shutdown = async (signal: string) => {
|
const shutdown = async (signal: string) => {
|
||||||
if (state.shutdownPromise) {
|
if (state.shutdownPromise) {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue