From d589858c03c6de8aa105fc9e7432a0720ba27b46 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 18 May 2026 03:34:24 -0400 Subject: [PATCH 01/26] Implement native fast iterative deploy workflow --- .beads/issues.jsonl | 15 +- README.md | 4 +- deployment/docker/README.md | 8 +- deployment/native/README.md | 216 ++++++++++++----- deployment/native/check-native-health.sh | 43 ++++ deployment/native/install-user-units.sh | 49 ++++ deployment/native/rollback.sh | 57 +++++ .../systemd/user/islandflow-api.service | 17 ++ .../systemd/user/islandflow-candles.service | 17 ++ .../systemd/user/islandflow-compute.service | 17 ++ .../user/islandflow-ingest-equities.service | 17 ++ .../user/islandflow-ingest-options.service | 17 ++ .../systemd/user/islandflow-web.service | 17 ++ ...-18-native-fast-iterative-deploy-plan.html | 93 ++++++++ ...26-05-18-native-fast-iterative-deploy.html | 153 ++++++++++++ ...05-18-native-fast-iterative-deploy-plan.md | 21 ++ scripts/deploy.ts | 222 ++++++++++++++---- 17 files changed, 873 insertions(+), 110 deletions(-) create mode 100755 deployment/native/check-native-health.sh create mode 100755 deployment/native/install-user-units.sh create mode 100755 deployment/native/rollback.sh create mode 100644 deployment/native/systemd/user/islandflow-api.service create mode 100644 deployment/native/systemd/user/islandflow-candles.service create mode 100644 deployment/native/systemd/user/islandflow-compute.service create mode 100644 deployment/native/systemd/user/islandflow-ingest-equities.service create mode 100644 deployment/native/systemd/user/islandflow-ingest-options.service create mode 100644 deployment/native/systemd/user/islandflow-web.service create mode 100644 docs/plans/2026-05-18-native-fast-iterative-deploy-plan.html create mode 100644 docs/turns/2026-05-18-native-fast-iterative-deploy.html create mode 100644 plans/2026-05-18-native-fast-iterative-deploy-plan.md diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index e025c4d..16eabf1 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,4 +1,4 @@ -{"_type":"issue","id":"islandflow-jbi","title":"Hydrate alert evidence details from ClickHouse","description":"Alert detail drawers need to fetch persisted alert context from ClickHouse by trace id, including linked flow packets, option prints, preserved execution context, and explicit missing refs for UI diagnostics.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:55:43Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:01:58Z","started_at":"2026-05-17T14:55:53Z","closed_at":"2026-05-17T15:01:58Z","close_reason":"Implemented ClickHouse-backed alert context hydration across storage, API, terminal drawer, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-9rc","title":"Implement native fast iterative deploy plan","description":"Implement the checked-in plan at plans/2026-05-18-native-fast-iterative-deploy-plan.md. Cover deploy-phase timing instrumentation, native deployment operational assets, deploy guardrails, validation/cutover documentation, and any required live VPS remediation that is safely actionable from this session. Track follow-up items separately if anything cannot be completed in-repo or on the live host.","status":"in_progress","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T07:15:19Z","created_by":"dirtydishes","updated_at":"2026-05-18T07:15:25Z","started_at":"2026-05-18T07:15:25Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8kj","title":"Configure persistent beads Dolt remote on deltaisland server","description":"Install the beads and Dolt CLIs on the server, configure a persistent Dolt sync remote backed by the server-hosted Forgejo repository, verify refs/dolt/data publication, and document Nginx Proxy Manager / firewall considerations.","status":"closed","priority":1,"issue_type":"task","assignee":"delta","created_at":"2026-05-17T10:31:31Z","created_by":"delta","updated_at":"2026-05-17T10:37:47Z","started_at":"2026-05-17T10:32:16Z","closed_at":"2026-05-17T10:37:47Z","close_reason":"Installed bd and dolt on the server, configured the Forgejo-backed Dolt remote, published refs/dolt/data, and documented the setup.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-k4f","title":"Gate deploy script on docker workspace snapshot sync","description":"Prevent frozen-lockfile build failures during deploy by adding a local preflight in scripts/deploy.ts that runs bun run check:docker-workspace and aborts with a clear sync+commit remediation message when stale.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:01:44Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:04:11Z","started_at":"2026-05-15T23:01:48Z","closed_at":"2026-05-15T23:04:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -13,14 +13,11 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-1ei","title":"Make deploy helper remote-aware for Forgejo","description":"Why: scripts/deploy.ts hardcodes git remote name origin for fetch/pull/push and branch verification, but this repository now uses forgejo/github remotes and may not have an origin remote. What: update deploy.ts to resolve the deploy git remote robustly (Forgejo-aware), use it across local prechecks, branch publish, and remote rollout git operations, and keep behavior explicit in output.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T03:20:12Z","created_by":"dirtydishes","updated_at":"2026-05-18T03:22:39Z","started_at":"2026-05-18T03:20:16Z","closed_at":"2026-05-18T03:22:39Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-xod","title":"Add --fast mode to deploy helper","description":"Why: full main deploys rebuild all images and run full verification, which is slow for routine rollouts. What: add a --fast flag to scripts/deploy.ts with explicit behavior that short-circuits slow steps while preserving basic safety checks; update help text/docs for discoverability.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T02:50:47Z","created_by":"dirtydishes","updated_at":"2026-05-18T02:53:41Z","started_at":"2026-05-18T02:50:50Z","closed_at":"2026-05-18T02:53:41Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-cif","title":"hydrate alert evidence context from clickhouse","description":"Implement alert detail hydration from ClickHouse with a new context endpoint and frontend drawer evidence resolution. Includes storage lookup by alert trace_id/evidence refs, unresolved refs diagnostics, API route GET /flow/alerts/:trace_id/context, terminal evidence hydration + loading states/copy updates, and tests across storage/api/web.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T00:15:55Z","created_by":"dirtydishes","updated_at":"2026-05-18T00:17:38Z","started_at":"2026-05-18T00:16:00Z","closed_at":"2026-05-18T00:17:38Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-4e9","title":"Polish terminal view","description":"Improve the Islandflow web terminal view with a focused UI polish pass aligned to the product design system.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T15:18:18Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:25:02Z","started_at":"2026-05-17T15:18:21Z","closed_at":"2026-05-17T15:25:02Z","close_reason":"Polished terminal shell styling, responsive Tape actions, and documented the turn.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-lyt","title":"Summarize 2026-05-16 git activity for standup","description":"Create a grounded standup summary for yesterday's git activity, anchored to commits, changed files, and any linked PR context if present. Produce the required HTML document in docs/general and complete the beads + git handoff workflow.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:02:57Z","created_by":"dirtydishes","updated_at":"2026-05-17T14:05:37Z","started_at":"2026-05-17T14:03:09Z","closed_at":"2026-05-17T14:05:37Z","close_reason":"Created docs/general standup summary for 2026-05-16 git activity, grounded to commits and changed files, and prepared the repo handoff workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-sz8","title":"Fix public /replay/options proxy regression","description":"## Summary\nThe new deploy-time public route checker added in commit 1424a27 (\"fix durable options history routing\") currently fails against https://flow.deltaisland.io because GET /replay/options returns HTML instead of JSON.\n\n## Evidence\n- `bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io` fails on `/replay/options?view=signal\u0026after_ts=0\u0026after_seq=0\u0026limit=1` with `returned non-JSON content (text/html; charset=UTF-8)`\n- `services/api/src/index.ts` implements `GET /replay/options`, so the HTML response indicates the request is landing on the web app instead of the API service\n- `deployment/docker/README.md` documents that same-origin proxy mode must include `/replay/*` in the API route matcher\n\n## Minimal Fix\nUpdate the live reverse proxy / edge route matcher for flow.deltaisland.io so `/replay/*` is forwarded to the API host, then rerun `bun run check:public-api-routes`.\n\n## Notes\nThis looks like a production proxy configuration regression rather than an in-repo application bug.","status":"open","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-17T13:06:11Z","created_by":"dirtydishes","updated_at":"2026-05-17T13:06:11Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-vvw","title":"Stage native public-edge cutover after worker soak","description":"Why this issue exists and what needs to be done:\\n- The native deploy path is now provisioned for worker-first iteration, with checked-in user units, rollback helpers, and edge guardrails\\n- Remaining work is to enable and soak native worker units, validate duplicate-processing behavior, then deliberately cut over the public web/api edge if warranted\\n- Final acceptance should include deciding whether Docker or native becomes the default runtime after operational evidence","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-18T07:32:35Z","created_by":"dirtydishes","updated_at":"2026-05-18T07:32:35Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-bsg","title":"Fix public /replay/options proxy regression","description":"Restore correct public routing for GET /replay/options on flow.deltaisland.io. The app currently serves HTML for that API path, which indicates edge/proxy routing drift. Update the live proxy topology or deployment assets as needed, then validate with bun run scripts/check-public-api-routes.ts.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T07:15:19Z","created_by":"dirtydishes","updated_at":"2026-05-18T07:32:51Z","started_at":"2026-05-18T07:15:24Z","closed_at":"2026-05-18T07:32:51Z","close_reason":"Audited the live VPS and reverse proxy on 2026-05-18: public /replay/options now returns JSON, bun run scripts/check-public-api-routes.ts passes, and the active Nginx Proxy Manager config includes /replay in the API route matcher. No in-repo app code change was required.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-9j5","title":"Prepare PR for deploy allowlist cleanup","description":"Why this issue exists and what needs to be done:\\n- Package current deploy allowlist cleanup into a reviewable PR with multiple commits\\n- Add required turn documentation in docs/turns\\n- Run validation and push all artifacts","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T15:44:12Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:53:55Z","started_at":"2026-05-17T15:44:22Z","closed_at":"2026-05-17T15:53:55Z","close_reason":"Packaged deploy allowlist cleanup into multi-commit PR branch with required turn documentation and push workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0sa","title":"Fix live tape auto-hold, history seam, and remove manual pause control","description":"The live tape should automatically hold when the user scrolls away from the top, resume when they return to the top or use Jump to top, and keep older prints available seamlessly beyond the hot window. Manual Pause/Resume control is now redundant and should be removed from live tape panes. This work should also fix the current regression where paused/held tapes still mutate, and align the options tape with a strict 100-row hot head backed by ClickHouse history.","notes":"Implemented live scroll-hold with no live pause button, demand-loaded ClickHouse history, a 100-row options hot head, and cache-first scoped snapshots. Validated with bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts and bun --cwd=apps/web run build.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T18:12:51Z","created_by":"dirtydishes","updated_at":"2026-05-16T18:23:43Z","started_at":"2026-05-16T18:12:54Z","closed_at":"2026-05-16T18:23:43Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-2db","title":"Manually remove stale islandflow local-infra containers from VPS","description":"The live VPS still has an older compose project named islandflow created from the repo-root docker-compose.yml. Inspection shows it is separate from the supported islandflow-vps deployment stack and exposes NATS, ClickHouse, and Redis on host ports. Container removal commands currently hang when run as the delta user through Docker, so cleanup likely needs a focused maintenance window and possibly host-level intervention or a Docker daemon restart.","notes":"The duplicate islandflow compose project on the VPS was confirmed live during inspection. Nginx Proxy Manager routes public traffic only to islandflow-vps web/api by Docker name, so the stale islandflow project appears to be stray local-infra state rather than part of the supported production path. Attempts to remove the stale containers with docker compose down and docker rm -f as the delta user hung and timed out, so manual cleanup likely needs a maintenance window and possibly Docker daemon intervention.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:27:27Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:28:59Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-2db","title":"Manually remove stale islandflow local-infra containers from VPS","description":"The live VPS still has an older compose project named islandflow created from the repo-root docker-compose.yml. Inspection shows it is separate from the supported islandflow-vps deployment stack and exposes NATS, ClickHouse, and Redis on host ports. Container removal commands currently hang when run as the delta user through Docker, so cleanup likely needs a focused maintenance window and possibly host-level intervention or a Docker daemon restart.","notes":"The duplicate islandflow compose project on the VPS was confirmed live during inspection. Nginx Proxy Manager routes public traffic only to islandflow-vps web/api by Docker name, so the stale islandflow project appears to be stray local-infra state rather than part of the supported production path. Attempts to remove the stale containers with docker compose down and docker rm -f as the delta user hung and timed out, so manual cleanup likely needs a maintenance window and possibly Docker daemon intervention.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:27:27Z","created_by":"dirtydishes","updated_at":"2026-05-18T07:32:48Z","started_at":"2026-05-18T07:15:25Z","closed_at":"2026-05-18T07:32:48Z","close_reason":"Audited the live VPS on 2026-05-18: docker compose ls and container labels no longer show a duplicate islandflow compose project, so the stale local-infra stack cleanup appears to already be resolved on the host.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-c87","title":"Clean up duplicate Islandflow Docker infra on VPS","description":"The live VPS is currently running both the production-style islandflow-vps Docker stack and an older root-level islandflow infra stack that publishes NATS, ClickHouse, and Redis on host ports. Investigate whether the older stack is unused, remove it safely if so, and update docs/deploy guidance so the server topology is clearer.","notes":"Inspected the live VPS and confirmed the duplicate compose project: islandflow-vps is the supported deployment stack, while a separate islandflow project from the repo-root docker-compose.yml still runs exposed NATS/ClickHouse/Redis containers. Verified Nginx Proxy Manager routes only to islandflow-vps web/api by Docker name. Attempted cleanup via docker compose down and docker rm -f on the stale islandflow containers, but those commands hung for the delta user and timed out. Added repo guardrails and docs so deploy warns when the duplicate project exists, and opened islandflow-2db for manual host-level cleanup during a maintenance window.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:16:05Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:28:07Z","started_at":"2026-05-16T01:16:09Z","closed_at":"2026-05-16T01:28:07Z","close_reason":"Completed the repo-side investigation and guardrails. Actual server-side container removal is blocked by hanging Docker operations and is tracked separately in islandflow-2db for a maintenance window.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-4gj","title":"Clarify Docker-first deploy workflow and mark native runtime experimental","description":"After inspecting the live VPS, native deployment is not ready for routine use: Nginx Proxy Manager routes to Docker container names, Bun is not installed on the host, sudo systemctl is not passwordless, and no Islandflow units exist. Update deploy messaging and docs so Docker remains the clearly recommended deployment path and native runtime is labeled experimental/future-facing with server prerequisites called out.","notes":"Updated deploy messaging and docs after live VPS inspection. scripts/deploy.ts now marks Docker as the default and recommended runtime, labels native as experimental, switches native systemctl default to sudo -n systemctl, and prints explicit native precheck failures for missing Bun/systemctl access/units. Updated README.md, deployment/docker/README.md, and deployment/native/README.md to reflect the current Docker + Nginx Proxy Manager topology. Validation: ./deploy --help, ./deploy main --runtime native --no-build (fails fast with Bun-missing message), bun run check:docker-workspace.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T01:10:11Z","created_by":"dirtydishes","updated_at":"2026-05-16T01:12:39Z","started_at":"2026-05-16T01:10:14Z","closed_at":"2026-05-16T01:12:39Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-7p2","title":"Fix deploy wrapper argument forwarding for runtime flags","description":"The repo-root deploy wrapper currently invokes bun run without a -- separator, so flags like --runtime native are treated as Bun CLI flags instead of script arguments. Update the wrapper so ./deploy main --runtime native forwards arguments correctly to scripts/deploy.ts.","notes":"Cherry-picked the dual-runtime deploy workflow onto main and fixed the repo-root deploy wrapper to call Bun with a -- separator so flags like --runtime native are forwarded to scripts/deploy.ts correctly. Validation: ./deploy --help, ./deploy main --runtime native --force-recreate guard, bun run check:docker-workspace.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T00:51:05Z","created_by":"dirtydishes","updated_at":"2026-05-16T00:52:34Z","started_at":"2026-05-16T00:51:10Z","closed_at":"2026-05-16T00:52:34Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -47,5 +44,5 @@ {"_type":"issue","id":"islandflow-igk","title":"Add plan mode","description":"Implement a user-facing plan mode in the application so users can switch into planning before taking action. Scope to be clarified from existing app patterns.","status":"closed","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:22:37Z","created_by":"dirtydishes","updated_at":"2026-05-04T04:26:18Z","started_at":"2026-05-04T04:22:40Z","closed_at":"2026-05-04T04:26:18Z","close_reason":"Implemented as a global pi extension toggled with Shift+P","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-biq","title":"Finish raw live options delivery and filter/backpressure observability","description":"The smart-money signal path and Tape filters are in place, but the next firehose pass should finish server-side selective raw live delivery for options subscriptions and add explicit filtered-out/backpressure observability for API/web counters. This was discovered while landing islandflow-e4r.\n","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:28:58Z","created_by":"dirtydishes","updated_at":"2026-04-29T03:54:12Z","started_at":"2026-04-29T03:54:12Z","dependencies":[{"issue_id":"islandflow-biq","depends_on_id":"islandflow-e4r","type":"discovered-from","created_at":"2026-04-28T16:28:58Z","created_by":"auto-import","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-zsy","title":"Expose Forgejo SSH on a direct DNS hostname","description":"git.deltaisland.io currently resolves through Cloudflare's proxy, so SSH on port 2222 does not complete even though the Forgejo container is listening on the host. If SSH-based git/beads workflows are desired, add a DNS-only hostname (or adjust the existing record) that points directly at the server for Forgejo SSH.","status":"open","priority":3,"issue_type":"task","created_at":"2026-05-17T10:34:06Z","created_by":"delta","updated_at":"2026-05-17T10:34:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-38p","title":"Add native deployment unit templates and rollback helpers","description":"The deploy helper now supports --runtime native, but the repo still relies on operator-managed systemd units and manual rollback. Add checked-in native deployment templates or provisioning guidance for the expected units, and consider lightweight rollback/smoke-test helpers once the host-native path is exercised on the real VPS.","status":"open","priority":3,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:46:42Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:46:42Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-38p","title":"Add native deployment unit templates and rollback helpers","description":"The deploy helper now supports --runtime native, but the repo still relies on operator-managed systemd units and manual rollback. Add checked-in native deployment templates or provisioning guidance for the expected units, and consider lightweight rollback/smoke-test helpers once the host-native path is exercised on the real VPS.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:46:42Z","created_by":"dirtydishes","updated_at":"2026-05-18T07:34:02Z","started_at":"2026-05-18T07:15:25Z","closed_at":"2026-05-18T07:34:02Z","close_reason":"Added checked-in native user unit templates, install/smoke-test/rollback helpers, updated native deploy docs with worker-first guidance, and installed the unit files onto the VPS in disabled form.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-575","title":"Document smart-money event calendar env","description":"Document smart-money event-calendar environment configuration in env examples and README.\n","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T06:57:14Z","created_by":"dirtydishes","updated_at":"2026-05-05T06:57:57Z","started_at":"2026-05-05T06:57:17Z","closed_at":"2026-05-05T06:57:57Z","close_reason":"Documented event-calendar env variables","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/README.md b/README.md index 50063d9..98d0936 100644 --- a/README.md +++ b/README.md @@ -129,8 +129,10 @@ This keeps Docker in the local workflow where it helps most (NATS, ClickHouse, R - `./deploy main` keeps the current VPS Docker rollout path as the default and recommended path. - Do not run the repo-root `docker-compose.yml` on the VPS. That file is for local infra only and can create duplicate exposed NATS, ClickHouse, and Redis containers on the server. - `./deploy main --runtime native` targets an experimental host-native Bun + systemd deployment. +- Native deploys are now intended primarily for worker-only fast iteration until the public edge is cut over deliberately. - `./deploy current-branch` and `./deploy current-branch --runtime native` keep branch deploys available during the transition, but Docker remains the supported path for the current VPS. -- Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, and `--no-build`. +- Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, `--workers-only`, and `--no-build`. +- When run from `/home/delta/islandflow` on the VPS itself, `./deploy` can execute locally instead of SSHing back into the same server. - Docker runtime details live in `deployment/docker/README.md`. - Native runtime expectations and prerequisites live in `deployment/native/README.md`. diff --git a/deployment/docker/README.md b/deployment/docker/README.md index 2b167da..ed80c53 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -217,13 +217,15 @@ The current live VPS uses Nginx Proxy Manager on the shared Docker network and r The deploy helper also warns if it detects a second compose project named `islandflow` on the server, because that usually means the repo-root local-infra stack was started on the VPS by mistake. -The checked-in deploy helper is meant to run from your local repo checkout, not from the VPS shell. It always targets: +The checked-in deploy helper normally runs from your local repo checkout and targets: - SSH host: `delta@152.53.80.229` -- SSH key: `~/.ssh/delta_ed25519` +- SSH key: `~/.ssh/delta_ed25519` by default - Live repo checkout: `/home/delta/islandflow` - Live compose directory: `/home/delta/islandflow/deployment/docker` +If you run `./deploy` from `/home/delta/islandflow` on the VPS itself, it now executes the remote steps locally instead of SSHing back into the same machine. You can still force SSH with `DEPLOY_FORCE_SSH=1`, or override the key path with `DEPLOY_SSH_KEY_PATH=/path/to/key`. + It preserves the current Docker Compose project and avoids destructive cleanup on the server. ### Deploy `origin/main` @@ -271,6 +273,7 @@ Examples: ./deploy main --runtime docker --web-only ./deploy main --runtime docker --api-only ./deploy current-branch --runtime docker --services-only +./deploy main --runtime docker --workers-only ./deploy main --runtime docker --fast ./deploy main --runtime docker --web-only --no-build ``` @@ -280,6 +283,7 @@ Scoped Docker deploys now build only the selected image set and then restart onl - `--web-only`: `docker compose build web`, then `docker compose up -d web` - `--api-only`: `docker compose build api`, then `docker compose up -d api` - `--services-only`: builds and restarts `api`, `compute`, `candles`, `ingest-options`, and `ingest-equities` +- `--workers-only`: builds and restarts `compute`, `candles`, `ingest-options`, and `ingest-equities` without touching `web` or `api` - `--fast`: when no explicit scope flag is given, treats the deploy as `--services-only` and skips the public API route suite for quicker completion. It still runs remote service health checks. Use `--no-build` only when the image is already correct and you need Compose to recreate or restart containers, such as after changing server-side environment values that do not affect a Next.js build-time variable. Do not use `--no-build` for dependency changes, application source changes, or `NEXT_PUBLIC_*` changes. diff --git a/deployment/native/README.md b/deployment/native/README.md index a9903cc..4e2dd52 100644 --- a/deployment/native/README.md +++ b/deployment/native/README.md @@ -1,29 +1,114 @@ # Native Deployment -This directory documents the experimental host-native Islandflow rollout path used by: +This directory documents the host-native Islandflow rollout path used by: ```bash ./deploy main --runtime native ./deploy current-branch --runtime native ``` -This runtime is intended for faster server iteration during the transition away from Docker-only app rollouts. It is not the recommended path for the current production VPS, which still uses Nginx Proxy Manager to reach the Docker `web` and `api` containers by container name on the shared Docker network. Local development should still prefer: +## Current operating model -- Docker for infra (`bun run dev:infra`) -- native Bun services (`bun run dev:services`) -- native Next.js web (`bun run dev:web`) +Native runtime is now intended for **fast iterative backend deploys first**, while Docker remains the supported public production edge until a deliberate cutover is completed. + +Today, the recommended split is: + +- **Docker runtime** for the live public `web` + `api` path +- **Native runtime** for worker-only iteration (`compute`, `candles`, `ingest-options`, `ingest-equities`) +- local development stays: + - Docker infra: `bun run dev:infra` + - native backend services: `bun run dev:services` + - native web: `bun run dev:web` ## What native deploy means here The checked-in `deploy` helper assumes: -- the live repo checkout is still `/home/delta/islandflow` +- the live repo checkout is `/home/delta/islandflow` - Bun is installed on the VPS -- app processes are managed by `systemd` -- infrastructure services such as NATS, ClickHouse, and Redis are already reachable from the host +- app processes are managed by `systemd --user` +- infrastructure services such as NATS, ClickHouse, and Redis are reachable from the host - the web app runs from `apps/web` and is served with `next start -p 3000` -The deploy script updates the repo checkout, optionally runs `bun install --frozen-lockfile`, optionally rebuilds the web app, restarts the target systemd units, and then verifies the services locally on the VPS plus through the public app URL. +The deploy script updates the repo checkout, optionally runs `bun install --frozen-lockfile`, optionally rebuilds the web app, restarts the target user units, verifies local health, and then runs public verification when the selected scope includes the public edge. + +## Live audit status on 2026-05-18 + +The plan assumptions were audited on the VPS: + +- `bun` is installed and available at `/home/delta/.bun/bin/bun` +- `systemctl --user` is available and the `delta` user has lingering enabled +- `/home/delta/islandflow/.env` exists +- public `https://flow.deltaisland.io/replay/options` routing is healthy again +- the previously reported duplicate `islandflow` compose project is not currently present in `docker compose ls` +- native Islandflow user units were not installed at the start of the audit; this change now provides and installs the checked-in user unit files, but they remain disabled until an operator enables a scope intentionally + +That means native worker deploy support is now provisioned on the host, but native runtime should still be enabled scope-by-scope rather than started wholesale. + +## Checked-in native ops assets + +### User unit templates + +Checked-in unit files live under: + +- `deployment/native/systemd/user/islandflow-web.service` +- `deployment/native/systemd/user/islandflow-api.service` +- `deployment/native/systemd/user/islandflow-compute.service` +- `deployment/native/systemd/user/islandflow-candles.service` +- `deployment/native/systemd/user/islandflow-ingest-options.service` +- `deployment/native/systemd/user/islandflow-ingest-equities.service` + +These are written for the current VPS layout: + +- repo root: `/home/delta/islandflow` +- Bun binary: `/home/delta/.bun/bin/bun` +- env file: `/home/delta/islandflow/.env` + +### Install the units + +```bash +./deployment/native/install-user-units.sh +./deployment/native/install-user-units.sh workers +systemctl --user start islandflow-compute.service +``` + +Install script behavior: + +- copies the checked-in unit files into `~/.config/systemd/user` +- reloads the user systemd daemon +- enables only the scope you explicitly request +- defaults to installing without enabling anything yet + +### Smoke test helper + +```bash +./deployment/native/check-native-health.sh workers +./deployment/native/check-native-health.sh services +./deployment/native/check-native-health.sh full +``` + +This validates: + +- `systemctl --user is-active` for the selected units +- local API health at `http://127.0.0.1:4000/health` when API scope is included +- local web health at `http://127.0.0.1:3000/` when web scope is included + +### Rollback helper + +```bash +./deployment/native/rollback.sh workers +./deployment/native/rollback.sh services +``` + +Rollback helper behavior: + +- requires a clean repo state +- fetches refs +- switches the checkout to a detached target ref +- reruns `bun install --frozen-lockfile` +- rebuilds the web app only when web scope is included +- restarts the selected user units +- runs the native smoke checks ## Expected unit names @@ -54,87 +139,104 @@ Available overrides: ## systemctl invocation -By default the deploy helper uses: - -```bash -sudo -n systemctl -``` - -If the server uses user units or another wrapper, override it locally before invoking `./deploy`: +For the checked-in user units, use: ```bash export DEPLOY_NATIVE_SYSTEMCTL_PREFIX="systemctl --user" -./deploy main --runtime native ``` +The deploy helper defaults to `sudo -n systemctl`, but that is only appropriate if you intentionally install matching system units. + ## Partial native rollouts Examples: ```bash -./deploy main --runtime native --web-only -./deploy main --runtime native --api-only -./deploy current-branch --runtime native --services-only +./deploy main --runtime native --workers-only ./deploy main --runtime native --fast -./deploy main --runtime native --web-only --no-build +./deploy main --runtime native --services-only +./deploy main --runtime native --web-only +./deploy current-branch --runtime native --workers-only --no-build ``` Scope behavior: -- default: restart web + API + backend services +- default: restart web + API + worker services - `--web-only`: rebuild/restart only the web unit - `--api-only`: restart only the API unit -- `--services-only`: restart API + backend units without touching the web unit -- `--fast`: when no explicit scope flag is provided, uses the same `--services-only` scope and trims verbose verification output for quicker completion +- `--services-only`: restart API + worker units without touching the web unit +- `--workers-only`: restart only `compute`, `candles`, `ingest-options`, and `ingest-equities` +- `--fast`: when no explicit scope flag is provided, native deploys now default to `--workers-only` - `--no-build`: skip `bun install --frozen-lockfile` and skip the web build step -## Current status +## Edge-cutover guardrail -On the current live VPS, native deploys should be treated as opt-in infrastructure work, not the default rollout path. Before a native deploy can succeed there, all of the following must be true at the same time: - -- Bun is installed on the host. -- The selected `systemctl` command works non-interactively. -- Islandflow systemd units exist for the requested scope. -- Host-native services can reach the intended NATS, ClickHouse, and Redis endpoints. -- If `web` or `api` move native, the reverse proxy topology is updated deliberately. - -Until that is prepared intentionally, prefer: +Native deploys that touch the public web or API edge are intentionally blocked unless you acknowledge cutover readiness: ```bash -./deploy main --runtime docker -./deploy current-branch --runtime docker +export DEPLOY_NATIVE_EDGE_READY=1 ``` -## Server preparation checklist +Without that variable, these commands are refused: -Before the first native rollout, ensure the VPS has: +- `./deploy main --runtime native` +- `./deploy main --runtime native --web-only` +- `./deploy main --runtime native --api-only` +- `./deploy main --runtime native --services-only` -1. Bun installed and on `PATH` -2. a working `/home/delta/islandflow/.env` (or unit-managed equivalent env source) -3. systemd units for each target service -4. the web unit configured to serve the built app on port `3000` -5. the API unit configured to serve health checks on port `4000` -6. infrastructure endpoints configured so the native services can reach NATS, ClickHouse, and Redis +This keeps the native path focused on safe worker iteration until proxy routing and public unit ownership are switched deliberately. -## Verification +## Running deploy from the VPS itself -Native deploys verify: +If you run `./deploy` from `/home/delta/islandflow` on the live server, the deploy helper now executes the remote steps locally instead of SSHing back into the same machine. -- target units are active via `systemctl` -- recent unit status and journal output can be collected -- local `http://127.0.0.1:4000/health` when API scope is included -- local `http://127.0.0.1:3000/` when web scope is included -- the public app URL from the local machine after the rollout finishes +That means: -## Rollback +- no SSH key is required for on-server deploy execution +- timing and verification behavior stay the same +- you can still force SSH with `DEPLOY_FORCE_SSH=1` +- you can override the SSH key path with `DEPLOY_SSH_KEY_PATH=/path/to/key` -Rollback remains manual for now: +## Validation matrix -1. switch the server checkout back to the last known-good branch or commit -2. rerun the appropriate native deploy command -3. if needed, restart only the affected units with `systemctl` +| Area | Native workers-only | Native edge cutover | +| --- | --- | --- | +| Bun installed | required | required | +| `systemctl --user` works | required | required | +| Islandflow user units installed | worker units only | all units | +| Host access to NATS/ClickHouse/Redis | required | required | +| Proxy routes updated for `/prints`, `/history`, `/replay`, `/nbbo`, `/ws`, `/flow`, `/candles` | not required | required | +| Public app check | not required | required | +| Public API route suite | not required | required | -Docker remains the fallback and currently recommended runtime during the transition: +## Staged cutover plan + +1. **Stage 1: native workers only** + - install user units + - validate `./deployment/native/check-native-health.sh workers` + - use `./deploy main --runtime native --fast` +2. **Stage 2: native API behind local-only verification** + - start `islandflow-api.service` + - confirm `curl http://127.0.0.1:4000/health` + - do not switch public routing yet +3. **Stage 3: deliberate public edge cutover** + - update proxy routing to native `web`/`api` + - export `DEPLOY_NATIVE_EDGE_READY=1` + - run full native deploy + - validate `bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io` +4. **Stage 4: decide final default runtime** + - keep Docker as fallback until native edge has proven stable + +## Recommended current commands + +Fast backend iteration before edge cutover: + +```bash +export DEPLOY_NATIVE_SYSTEMCTL_PREFIX="systemctl --user" +./deploy main --runtime native --fast +``` + +Supported production path today: ```bash ./deploy main --runtime docker diff --git a/deployment/native/check-native-health.sh b/deployment/native/check-native-health.sh new file mode 100755 index 0000000..1d070e5 --- /dev/null +++ b/deployment/native/check-native-health.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +scope="${1:-full}" +units=() + +case "$scope" in + full) + units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + ;; + web) + units=(islandflow-web.service) + ;; + api) + units=(islandflow-api.service) + ;; + services) + units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + ;; + workers) + units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + ;; + *) + echo "Unknown scope: $scope" >&2 + echo "Expected one of: full, web, api, services, workers" >&2 + exit 1 + ;; +esac + +for unit in "${units[@]}"; do + systemctl --user is-active --quiet "$unit" + echo "ok $unit" +done + +if [[ " ${units[*]} " == *" islandflow-api.service "* ]]; then + curl -fksS http://127.0.0.1:4000/health >/dev/null + echo "ok api-health" +fi + +if [[ " ${units[*]} " == *" islandflow-web.service "* ]]; then + curl -I -fksS http://127.0.0.1:3000/ >/dev/null + echo "ok web-health" +fi diff --git a/deployment/native/install-user-units.sh b/deployment/native/install-user-units.sh new file mode 100755 index 0000000..350cab1 --- /dev/null +++ b/deployment/native/install-user-units.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +scope="${1:-none}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +unit_source_dir="$repo_root/deployment/native/systemd/user" +unit_target_dir="${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user" +units=() + +case "$scope" in + none) + ;; + full) + units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + ;; + web) + units=(islandflow-web.service) + ;; + api) + units=(islandflow-api.service) + ;; + services) + units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + ;; + workers) + units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + ;; + *) + echo "Unknown scope: $scope" >&2 + echo "Expected one of: none, full, web, api, services, workers" >&2 + exit 1 + ;; +esac + +mkdir -p "$unit_target_dir" +cp "$unit_source_dir"/*.service "$unit_target_dir"/ + +systemctl --user daemon-reload + +if [[ ${#units[@]} -gt 0 ]]; then + systemctl --user enable "${units[@]}" +fi + +echo "Installed Islandflow user units into $unit_target_dir" +if [[ ${#units[@]} -gt 0 ]]; then + echo "Enabled scope: $scope" +else + echo "No units enabled yet. Pass a scope such as workers when you are ready." +fi \ No newline at end of file diff --git a/deployment/native/rollback.sh b/deployment/native/rollback.sh new file mode 100755 index 0000000..fb472d9 --- /dev/null +++ b/deployment/native/rollback.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 || $# -gt 2 ]]; then + echo "Usage: deployment/native/rollback.sh [full|web|api|services|workers]" >&2 + exit 1 +fi + +ref="$1" +scope="${2:-services}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +cd "$repo_root" + +if [[ -n "$(git status --porcelain=v1)" ]]; then + echo "Refusing rollback with a dirty working tree." >&2 + exit 1 +fi + +current_ref="$(git rev-parse --short HEAD)" +echo "Rolling back from $current_ref to $ref (scope: $scope)" + +git fetch --all --prune +git switch --detach "$ref" +bun install --frozen-lockfile + +if [[ "$scope" == "full" || "$scope" == "web" ]]; then + bun --cwd=apps/web run build +fi + +case "$scope" in + full) + units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + ;; + web) + units=(islandflow-web.service) + ;; + api) + units=(islandflow-api.service) + ;; + services) + units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + ;; + workers) + units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + ;; + *) + echo "Unknown scope: $scope" >&2 + exit 1 + ;; +esac + +systemctl --user restart "${units[@]}" +"$repo_root/deployment/native/check-native-health.sh" "$scope" + +echo "Rollback complete. Repo is now detached at $(git rev-parse --short HEAD)." +echo "Return to tracked main later with: git switch main && git pull --ff-only main" diff --git a/deployment/native/systemd/user/islandflow-api.service b/deployment/native/systemd/user/islandflow-api.service new file mode 100644 index 0000000..5a74500 --- /dev/null +++ b/deployment/native/systemd/user/islandflow-api.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow API +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun services/api/src/index.ts +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/deployment/native/systemd/user/islandflow-candles.service b/deployment/native/systemd/user/islandflow-candles.service new file mode 100644 index 0000000..585b37c --- /dev/null +++ b/deployment/native/systemd/user/islandflow-candles.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow candles +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun services/candles/src/index.ts +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/deployment/native/systemd/user/islandflow-compute.service b/deployment/native/systemd/user/islandflow-compute.service new file mode 100644 index 0000000..603f252 --- /dev/null +++ b/deployment/native/systemd/user/islandflow-compute.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow compute +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun services/compute/src/index.ts +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/deployment/native/systemd/user/islandflow-ingest-equities.service b/deployment/native/systemd/user/islandflow-ingest-equities.service new file mode 100644 index 0000000..837a04f --- /dev/null +++ b/deployment/native/systemd/user/islandflow-ingest-equities.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow ingest-equities +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun services/ingest-equities/src/index.ts +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/deployment/native/systemd/user/islandflow-ingest-options.service b/deployment/native/systemd/user/islandflow-ingest-options.service new file mode 100644 index 0000000..eac0a6c --- /dev/null +++ b/deployment/native/systemd/user/islandflow-ingest-options.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow ingest-options +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun services/ingest-options/src/index.ts +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/deployment/native/systemd/user/islandflow-web.service b/deployment/native/systemd/user/islandflow-web.service new file mode 100644 index 0000000..6e79177 --- /dev/null +++ b/deployment/native/systemd/user/islandflow-web.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow web +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun --cwd apps/web run start +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/docs/plans/2026-05-18-native-fast-iterative-deploy-plan.html b/docs/plans/2026-05-18-native-fast-iterative-deploy-plan.html new file mode 100644 index 0000000..98fff10 --- /dev/null +++ b/docs/plans/2026-05-18-native-fast-iterative-deploy-plan.html @@ -0,0 +1,93 @@ + + + + + + Plan: Native Fast Iterative Deployment + + + +

Plan: Native, Fast, Iterative Deployment (Docker Optional)

+

Date: 2026-05-18

+ +
+

Plan Summary

+

Define and execute a fast iteration deployment path centered on host-native services, while preserving Docker as a fallback/runtime option.

+
+ +
+

Goals

+
    +
  • Reduce deploy turnaround time immediately.
    • +
  • Identify concrete bottlenecks with timing evidence.
    • +
  • Stabilize proxy/runtime topology for reliable production rollouts.
    • +
  • Support both native and Docker strategies with explicit guardrails.
    • +
+
+ +
+

Proposed Changes

+
    +
  • Use scoped fast deploys short-term.
    • +
  • Audit and remediate server-state blockers (duplicate compose/project drift).
    • +
  • Prepare native runtime prerequisites and checked-in operational assets.
    • +
  • Add deployment strategy prechecks, validation matrix, and staged cutover.
    • +
+
+ +
+

Relevant Context

+
    +
  • Open issue islandflow-2db: stale duplicate compose stack cleanup.
    • +
  • Open issue islandflow-sz8: public /replay/options proxy regression.
    • +
  • Open issue islandflow-38p: native unit templates and rollback helpers.
    • +
+
+ +
+

Implementation Steps

+
    +
  1. Stop the bleeding immediately (current deploy loop).
    2. +
  2. Get hard timing data per deploy phase.
    3. +
  3. Live server state audit (when plan mode is off).
    4. +
  4. Resolve duplicate compose stack first (islandflow-2db).
    5. +
  5. Fix NPM proxy route regression (islandflow-sz8).
    6. +
  6. Define target iterative deployment model.
    7. +
  7. Prepare native runtime prerequisites on VPS.
    8. +
  8. Checked-in native ops assets (islandflow-38p).
    9. +
  9. Switch proxy topology for native mode carefully.
    10. +
  10. Deploy strategy guardrails.
    11. +
  11. Validation matrix.
    12. +
  12. Staged cutover plan.
    13. +
  13. Decision: final default runtime.
    14. +
  14. Decision: optimization priority.
    15. +
  15. Decision: immediate live audit kickoff.
    16. +
+
+ +
+

Risks, Limitations, and Mitigations

+
    +
  • Risk: native runtime not yet production-hardened. Mitigation: keep Docker fallback and explicit gating.
    • +
  • Risk: proxy misrouting breaks API routes. Mitigation: route checks and post-change smoke validation.
    • +
  • Risk: operational drift on VPS. Mitigation: preflight audits and documented rollback steps.
    • +
+
+ +
+

Open Questions

+
    +
  • Should native become the default runtime now, or after hardening milestones?
    • +
  • Should backend iteration speed be prioritized ahead of web deploy speed?
    • +
  • Do we start immediate live server audit as soon as plan mode is disabled?
    • +
+
+ + diff --git a/docs/turns/2026-05-18-native-fast-iterative-deploy.html b/docs/turns/2026-05-18-native-fast-iterative-deploy.html new file mode 100644 index 0000000..45cba6c --- /dev/null +++ b/docs/turns/2026-05-18-native-fast-iterative-deploy.html @@ -0,0 +1,153 @@ + + + + + + 2026-05-18: Native fast iterative deploy + + + +
+
Turn document · 2026-05-18 03:29 EDT · Issues: islandflow-9rc, islandflow-38p, islandflow-bsg, islandflow-2db
+

Native fast iterative deploy

+

Implemented the native-first iterative deploy plan by adding deploy timing output, a safe worker-only native fast path, checked-in systemd user units and rollback helpers, server-local deploy execution, and updated live-operational documentation based on a fresh VPS audit.

+ +
+

Summary

+

The deploy flow now supports a safer native worker iteration model without requiring public edge cutover first. It can run directly from the VPS checkout without SSH, emits phase timings, includes checked-in native unit files plus install/rollback/smoke-test helpers, and documents the staged cutover path. During live audit, the previously reported /replay/options proxy issue and duplicate islandflow compose stack were both confirmed resolved on the host.

+
+ +
+

Changes Made

+
    +
  • Extended scripts/deploy.ts with deploy timing summaries for precheck, rollout, and verification phases.
    • +
  • Added --workers-only deploy scope for Docker and native runtimes.
    • +
  • Changed native --fast behavior so default full-scope fast deploys become worker-only instead of touching web/API.
    • +
  • Added native edge guardrails via DEPLOY_NATIVE_EDGE_READY=1 before web/API native deploys are allowed.
    • +
  • Added local-server execution mode so ./deploy can run from /home/delta/islandflow without SSHing back into the same host.
    • +
  • Added DEPLOY_SSH_KEY_PATH and DEPLOY_FORCE_SSH overrides for operators with non-default SSH setups.
    • +
  • Checked in native ops assets under deployment/native/:
    • +
  • install-user-units.sh, check-native-health.sh, rollback.sh
    • +
  • six user unit files in deployment/native/systemd/user/
    • +
  • Updated README.md, deployment/docker/README.md, and deployment/native/README.md to document the worker-first model, local execution mode, validation matrix, and staged cutover guidance.
    • +
  • Synced deployment/docker/workspace-root/package.json so Docker workspace validation passes again.
    • +
  • Installed the checked-in user unit files onto the live VPS in disabled form under ~/.config/systemd/user.
    • +
+
+ +
+

Context

+

The plan targeted faster deployment iteration while avoiding a premature move of the public edge away from the current Docker + Nginx Proxy Manager topology. The practical target was to make native runtime useful immediately for backend-worker iteration, while leaving web/API cutover deliberate and reversible.

+
+ +
+

Important Implementation Details

+
    +
  • Native fast mode now defaults to --workers-only; Docker fast mode still defaults to --services-only.
    • +
  • Native deploys that include public web/API scope now fail fast unless DEPLOY_NATIVE_EDGE_READY=1 is set.
    • +
  • Running from the live VPS checkout automatically switches deploy execution from SSH mode to local mode.
    • +
  • The checked-in native unit files are user units aimed at the current VPS layout: /home/delta/islandflow and /home/delta/.bun/bin/bun.
    • +
  • install-user-units.sh now installs units safely without enabling anything by default; enabling is explicit and scope-based.
    • +
  • rollback.sh intentionally uses a detached git ref to make one-off native rollback practical without rewriting branch history.
    • +
+ 
export DEPLOY_NATIVE_SYSTEMCTL_PREFIX="systemctl --user"
+./deploy main --runtime native --fast
+# resolves to worker-only native deploy before public edge cutover
+
+ +
+

Expected Impact for End-Users

+

End-users should see indirect benefits first: faster backend iteration, safer operational changes, and clearer rollback paths. Public traffic behavior should remain unchanged until a deliberate native edge cutover is performed.

+
+ +
+

Validation

+
    +
  • Passed: bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io
    • +
  • Passed: direct public /replay/options curl returned JSON
    • +
  • Passed: live Nginx Proxy Manager config contains /replay in the API route matcher
    • +
  • Passed: docker compose ls shows no duplicate islandflow project
    • +
  • Passed: bash -n deployment/native/install-user-units.sh deployment/native/check-native-health.sh deployment/native/rollback.sh
    • +
  • Passed: systemd-analyze verify deployment/native/systemd/user/*.service
    • +
  • Passed: bun run check:docker-workspace after syncing workspace snapshot
    • +
  • Passed: native edge guard refusal for bun run scripts/deploy.ts main --runtime native --web-only --no-build
    • +
  • Passed: ./deployment/native/install-user-units.sh followed by systemctl --user list-unit-files 'islandflow*'
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • Native units were installed but not enabled or started. This is intentional to avoid conflicting with the current Docker production edge.
    • +
  • Public web/API native deploys are still gated. Mitigation: explicit DEPLOY_NATIVE_EDGE_READY=1 acknowledgment and staged cutover documentation.
    • +
  • Native worker runtime has not yet been exercised live against the existing Docker worker stack. Mitigation: follow-up issue to soak worker-only native units before any default-runtime decision.
    • +
  • The known untracked Signal CLI tarball remains in the repo checkout. This is already tolerated by the deploy helper allowlist and was not changed here.
    • +
+
+ +
+

Follow-up Work

+
    +
  • Open follow-up: islandflow-vvw — stage native public-edge cutover after worker soak.
    • +
  • Decide whether native should ever replace Docker as the default runtime only after worker soak data and deliberate edge cutover validation.
    • +
+
+
+ + diff --git a/plans/2026-05-18-native-fast-iterative-deploy-plan.md b/plans/2026-05-18-native-fast-iterative-deploy-plan.md new file mode 100644 index 0000000..0e09102 --- /dev/null +++ b/plans/2026-05-18-native-fast-iterative-deploy-plan.md @@ -0,0 +1,21 @@ +# Native, Fast, Iterative Deployment Plan (Docker Optional) + +Date: 2026-05-18 + +## Plan Steps (15) + +1. ☐ Stop the bleeding immediately (current deploy loop). +2. ☐ Get hard timing data per deploy phase. +3. ☐ Live server state audit (when plan mode is off). +4. ☐ Resolve duplicate compose stack first (islandflow-2db). +5. ☐ Fix NPM proxy route regression (islandflow-sz8). +6. ☐ Define target iterative deployment model. +7. ☐ Prepare native runtime prerequisites on VPS. +8. ☐ Checked-in native ops assets (islandflow-38p). +9. ☐ Switch proxy topology for native mode carefully. +10. ☐ Deploy strategy guardrails. +11. ☐ Validation matrix. +12. ☐ Staged cutover plan. +13. ☐ Decision: final default runtime. +14. ☐ Decision: optimization priority. +15. ☐ Decision: immediate live audit kickoff. diff --git a/scripts/deploy.ts b/scripts/deploy.ts index 68d260a..043122e 100644 --- a/scripts/deploy.ts +++ b/scripts/deploy.ts @@ -7,7 +7,7 @@ import { fileURLToPath } from "node:url"; type DeployMode = "main" | "current-branch"; type DeployRuntime = "docker" | "native"; -type DeployScope = "full" | "web" | "api" | "services"; +type DeployScope = "full" | "web" | "api" | "services" | "workers"; type DeployOptions = { mode: DeployMode; @@ -18,10 +18,18 @@ type DeployOptions = { noBuild: boolean; }; +type PhaseTiming = { + name: string; + durationMs: number; +}; + const REMOTE_HOST = "delta@152.53.80.229"; const REMOTE_REPO = "/home/delta/islandflow"; const REMOTE_DOCKER_DEPLOYMENT = "/home/delta/islandflow/deployment/docker"; -const SSH_KEY = path.join(process.env.HOME ?? "", ".ssh", "delta_ed25519"); +const SSH_KEY = + process.env.DEPLOY_SSH_KEY_PATH?.trim() || + path.join(process.env.HOME ?? "", ".ssh", "delta_ed25519"); +const DEPLOY_FORCE_SSH = process.env.DEPLOY_FORCE_SSH?.trim() === "1"; const SSH_OPTIONS = [ "-i", SSH_KEY, @@ -38,6 +46,7 @@ const PUBLIC_APP_URL = const PUBLIC_API_HEALTH_URL = process.env.DEPLOY_PUBLIC_API_HEALTH_URL?.trim() || null; const DEPLOY_GIT_REMOTE_OVERRIDE = process.env.DEPLOY_GIT_REMOTE?.trim() || null; +const DEPLOY_NATIVE_EDGE_READY = process.env.DEPLOY_NATIVE_EDGE_READY?.trim() === "1"; const NATIVE_SYSTEMCTL_PREFIX = process.env.DEPLOY_NATIVE_SYSTEMCTL_PREFIX?.trim() || "sudo -n systemctl"; const NATIVE_UNITS = { @@ -65,15 +74,22 @@ const DOCKER_BACKEND_SERVICES = [ "ingest-options", "ingest-equities" ] as const; +const DOCKER_WORKER_SERVICES = [ + "compute", + "candles", + "ingest-options", + "ingest-equities" +] as const; const scriptPath = fileURLToPath(import.meta.url); const repoRoot = path.resolve(path.dirname(scriptPath), ".."); +const isLocalServerExecution = !DEPLOY_FORCE_SSH && repoRoot === REMOTE_REPO; function usage(exitCode = 1): never { console.error(`Usage: - ./deploy main [--runtime docker|native] [--web-only|--api-only|--services-only] [--fast] [--no-build] [--force-recreate] - ./deploy current-branch [--runtime docker|native] [--web-only|--api-only|--services-only] [--fast] [--no-build] [--force-recreate] - ./deploy current branch [--runtime docker|native] [--web-only|--api-only|--services-only] [--fast] [--no-build] [--force-recreate] + ./deploy main [--runtime docker|native] [--web-only|--api-only|--services-only|--workers-only] [--fast] [--no-build] [--force-recreate] + ./deploy current-branch [--runtime docker|native] [--web-only|--api-only|--services-only|--workers-only] [--fast] [--no-build] [--force-recreate] + ./deploy current branch [--runtime docker|native] [--web-only|--api-only|--services-only|--workers-only] [--fast] [--no-build] [--force-recreate] Modes: main Deploy /main to the live server checkout. @@ -88,18 +104,22 @@ Scopes: --web-only Deploy only the Next.js web surface. --api-only Deploy only the API service. --services-only Deploy API + backend services without the web service. + --workers-only Deploy compute/candles/ingest workers without touching web or API. Options: --runtime Explicit runtime selector (docker or native). - --fast Prefer a quicker rollout profile (defaults full scope to --services-only and skips public API route suite). + --fast Prefer a quicker rollout profile (defaults full scope to --services-only for docker and --workers-only for native, and skips the public API route suite when API scope is included). --no-build Skip docker image builds or native bun install/web build steps. --force-recreate Docker-only escalation path for docker compose when a normal refresh is not enough. --help Show this help text. Environment: DEPLOY_GIT_REMOTE Override git remote used for deploy fetch/pull/push (auto-detected by default). + DEPLOY_SSH_KEY_PATH Override the SSH key used for remote execution. + DEPLOY_FORCE_SSH Set to 1 to force SSH even when running from the live server checkout. DEPLOY_PUBLIC_APP_URL Override the public app URL (default: https://flow.deltaisland.io). DEPLOY_PUBLIC_API_HEALTH_URL Optional separate public API health URL for two-origin deployments. + DEPLOY_NATIVE_EDGE_READY Set to 1 to allow native rollouts that include the public web or API edge. DEPLOY_NATIVE_SYSTEMCTL_PREFIX Override systemctl invocation for native rollouts (default: sudo -n systemctl). DEPLOY_NATIVE_WEB_UNIT Override native web systemd unit name. DEPLOY_NATIVE_API_UNIT Override native api systemd unit name. @@ -114,6 +134,32 @@ function section(title: string): void { console.log(`\n== ${title} ==`); } +function formatDuration(durationMs: number): string { + if (durationMs < 1000) { + return `${durationMs}ms`; + } + + return `${(durationMs / 1000).toFixed(2)}s`; +} + +function timedPhase(timings: PhaseTiming[], name: string, fn: () => T): T { + const startedAt = Date.now(); + try { + return fn(); + } finally { + timings.push({ name, durationMs: Date.now() - startedAt }); + } +} + +function printTimingSummary(timings: PhaseTiming[]): void { + section("Deploy Timings"); + const totalMs = timings.reduce((sum, timing) => sum + timing.durationMs, 0); + for (const timing of timings) { + console.log(`[deploy] ${timing.name}: ${formatDuration(timing.durationMs)}`); + } + console.log(`[deploy] total: ${formatDuration(totalMs)}`); +} + function formatCommand(command: string, args: string[]): string { return [command, ...args] .map((part) => (/\s/.test(part) ? JSON.stringify(part) : part)) @@ -180,6 +226,23 @@ function runRemoteScript( args: string[] = [] ): void { section(title); + + if (isLocalServerExecution) { + const localArgs = ["-s", "--", ...args]; + console.log(`$ ${formatCommand("bash", localArgs)} # local server execution`); + const result = spawnSync("bash", localArgs, { + cwd: repoRoot, + input: script, + encoding: "utf8", + stdio: ["pipe", "inherit", "inherit"] + }); + + if (result.status !== 0) { + process.exit(result.status ?? 1); + } + return; + } + const sshArgs = [...SSH_OPTIONS, REMOTE_HOST, "bash", "-s", "--", ...args]; console.log(`$ ${formatCommand("ssh", sshArgs)}`); const result = spawnSync("ssh", sshArgs, { @@ -221,11 +284,14 @@ function parseScope(rawArgs: string[]): DeployScope { const scopes = [ rawArgs.includes("--web-only") ? "web" : null, rawArgs.includes("--api-only") ? "api" : null, - rawArgs.includes("--services-only") ? "services" : null + rawArgs.includes("--services-only") ? "services" : null, + rawArgs.includes("--workers-only") ? "workers" : null ].filter((value): value is Exclude => value !== null); if (scopes.length > 1) { - console.error("Choose only one deploy scope flag: --web-only, --api-only, or --services-only."); + console.error( + "Choose only one deploy scope flag: --web-only, --api-only, --services-only, or --workers-only." + ); process.exit(1); } @@ -250,6 +316,7 @@ function parseArgs(rawArgs: string[]): DeployOptions { arg !== "--web-only" && arg !== "--api-only" && arg !== "--services-only" && + arg !== "--workers-only" && arg !== "--runtime" && rawArgs[index - 1] !== "--runtime" && !arg.startsWith("--runtime=") @@ -282,8 +349,13 @@ function parseArgs(rawArgs: string[]): DeployOptions { } function assertSshKeyExists(): void { + if (isLocalServerExecution) { + return; + } + if (!existsSync(SSH_KEY)) { console.error(`Missing SSH key: ${SSH_KEY}`); + console.error("Set DEPLOY_SSH_KEY_PATH or run from the live server checkout without DEPLOY_FORCE_SSH."); process.exit(1); } } @@ -398,14 +470,16 @@ function describeScope(scope: DeployScope): string { return "api only"; case "services": return "api + backend services"; + case "workers": + return "worker services only"; default: return "full stack"; } } -function effectiveScope(scope: DeployScope, fast: boolean): DeployScope { +function effectiveScope(scope: DeployScope, runtime: DeployRuntime, fast: boolean): DeployScope { if (fast && scope === "full") { - return "services"; + return runtime === "native" ? "workers" : "services"; } return scope; } @@ -418,6 +492,10 @@ function scopeIncludesApi(scope: DeployScope): boolean { return scope === "full" || scope === "api" || scope === "services"; } +function scopeTouchesPublicEdge(scope: DeployScope): boolean { + return scopeIncludesWeb(scope) || scopeIncludesApi(scope); +} + function dockerServicesForScope(scope: DeployScope): string[] { switch (scope) { case "web": @@ -426,6 +504,8 @@ function dockerServicesForScope(scope: DeployScope): string[] { return ["api"]; case "services": return [...DOCKER_BACKEND_SERVICES]; + case "workers": + return [...DOCKER_WORKER_SERVICES]; default: return []; } @@ -448,6 +528,8 @@ function dockerLogServicesForScope(scope: DeployScope): string[] { return ["api"]; case "services": return [...DOCKER_BACKEND_SERVICES]; + case "workers": + return [...DOCKER_WORKER_SERVICES]; default: return [...DOCKER_CORE_SERVICES]; } @@ -467,6 +549,13 @@ function nativeUnitsForScope(scope: DeployScope): string[] { NATIVE_UNITS.ingestOptions, NATIVE_UNITS.ingestEquities ]; + case "workers": + return [ + NATIVE_UNITS.compute, + NATIVE_UNITS.candles, + NATIVE_UNITS.ingestOptions, + NATIVE_UNITS.ingestEquities + ]; default: return [ NATIVE_UNITS.web, @@ -494,19 +583,46 @@ function localDockerWorkspaceSnapshotPrecheck(): void { } } -function localRuntimePrecheck(runtime: DeployRuntime, noBuild: boolean): void { +function assertNativeEdgeReady(scope: DeployScope): void { + if (!scopeTouchesPublicEdge(scope) || DEPLOY_NATIVE_EDGE_READY) { + return; + } + + console.error( + "Refusing native deploy that touches public web/API scope before edge cutover is acknowledged." + ); + console.error( + "Set DEPLOY_NATIVE_EDGE_READY=1 only after proxy routing and native units for the public edge are intentionally prepared." + ); + console.error( + "For fast iterative backend deploys before cutover, use --runtime native --workers-only or --runtime native --fast." + ); + process.exit(1); +} + +function localRuntimePrecheck(runtime: DeployRuntime, scope: DeployScope, noBuild: boolean): void { if (runtime === "docker" && !noBuild) { localDockerWorkspaceSnapshotPrecheck(); + return; + } + + if (runtime === "native") { + assertNativeEdgeReady(scope); } } -function localMainPrecheck(remote: string, runtime: DeployRuntime, noBuild: boolean): void { +function localMainPrecheck( + remote: string, + runtime: DeployRuntime, + scope: DeployScope, + noBuild: boolean +): void { section("Local Precheck"); runChecked("git", ["fetch", remote]); runChecked("git", ["status", "--short", "--branch"]); runChecked("git", ["rev-parse", "--verify", "HEAD"]); runChecked("git", ["rev-parse", `${remote}/main`]); - localRuntimePrecheck(runtime, noBuild); + localRuntimePrecheck(runtime, scope, noBuild); } function currentBranchName(): string { @@ -522,6 +638,7 @@ function localBranchPrecheck( remote: string, branch: string, runtime: DeployRuntime, + scope: DeployScope, noBuild: boolean ): void { section("Local Precheck"); @@ -537,7 +654,7 @@ function localBranchPrecheck( process.exit(1); } - localRuntimePrecheck(runtime, noBuild); + localRuntimePrecheck(runtime, scope, noBuild); } function publishCurrentBranch(remote: string, branch: string): void { @@ -861,7 +978,8 @@ function publicVerification(scope: DeployScope, fast: boolean): void { function main(): void { const options = parseArgs(process.argv.slice(2)); - const scope = effectiveScope(options.scope, options.fast); + const scope = effectiveScope(options.scope, options.runtime, options.fast); + const timings: PhaseTiming[] = []; const currentBranch = options.mode === "current-branch" ? currentBranchName() : null; const deployRemote = resolveDeployRemote(options.mode, currentBranch); assertSshKeyExists(); @@ -872,22 +990,33 @@ function main(): void { `via ${describeRuntime(options.runtime)} (${describeScope(scope)}${options.fast ? ", fast mode" : ""}).` ); console.log(`[deploy] Using git remote: ${deployRemote}`); + console.log( + `[deploy] Execution mode: ${isLocalServerExecution ? "local server checkout" : `ssh to ${REMOTE_HOST}`}` + ); if (options.fast && options.scope === "full") { - console.log("[deploy] Fast mode changed default full scope to --services-only."); + console.log( + `[deploy] Fast mode changed default full scope to ${options.runtime === "native" ? "--workers-only" : "--services-only"}.` + ); } if (options.mode === "main") { - localMainPrecheck(deployRemote, options.runtime, options.noBuild); - remoteGitPrecheck(); - remoteRuntimePrecheck(options.runtime, scope); - remoteRollout( - options.mode, - deployRemote, - options.runtime, - null, - scope, - options.forceRecreate, - options.noBuild + timedPhase(timings, "local precheck", () => + localMainPrecheck(deployRemote, options.runtime, scope, options.noBuild) + ); + timedPhase(timings, "remote git precheck", () => remoteGitPrecheck()); + timedPhase(timings, "remote runtime precheck", () => + remoteRuntimePrecheck(options.runtime, scope) + ); + timedPhase(timings, "remote rollout", () => + remoteRollout( + options.mode, + deployRemote, + options.runtime, + null, + scope, + options.forceRecreate, + options.noBuild + ) ); } else { const branch = currentBranch; @@ -895,23 +1024,34 @@ function main(): void { console.error("Unable to resolve current branch for current-branch deploy mode."); process.exit(1); } - localBranchPrecheck(deployRemote, branch, options.runtime, options.noBuild); - publishCurrentBranch(deployRemote, branch); - remoteGitPrecheck(); - remoteRuntimePrecheck(options.runtime, scope); - remoteRollout( - options.mode, - deployRemote, - options.runtime, - branch, - scope, - options.forceRecreate, - options.noBuild + timedPhase(timings, "local precheck", () => + localBranchPrecheck(deployRemote, branch, options.runtime, scope, options.noBuild) + ); + timedPhase(timings, "local publish", () => publishCurrentBranch(deployRemote, branch)); + timedPhase(timings, "remote git precheck", () => remoteGitPrecheck()); + timedPhase(timings, "remote runtime precheck", () => + remoteRuntimePrecheck(options.runtime, scope) + ); + timedPhase(timings, "remote rollout", () => + remoteRollout( + options.mode, + deployRemote, + options.runtime, + branch, + scope, + options.forceRecreate, + options.noBuild + ) ); } - remoteVerification(options.runtime, scope, options.fast); - publicVerification(scope, options.fast); + timedPhase(timings, "remote verification", () => + remoteVerification(options.runtime, scope, options.fast) + ); + timedPhase(timings, "public verification", () => + publicVerification(scope, options.fast) + ); + printTimingSummary(timings); } main(); From 62aae7087899359c299b726c4df6e3ce41ce1764 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 18 May 2026 09:05:40 -0400 Subject: [PATCH 02/26] docs(general): add 2026-05-17 standup summary --- .beads/issues.jsonl | 3 + ...2026-05-18-standup-summary-2026-05-17.html | 549 ++++++++++++++++++ 2 files changed, 552 insertions(+) create mode 100644 docs/general/2026-05-18-standup-summary-2026-05-17.html diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index e025c4d..629eb06 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -13,6 +13,8 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-k8i","title":"Fix duplicate alert context import in API entrypoint","description":"Recent alert-context work introduced a duplicate fetchAlertContextByTraceId import in services/api/src/index.ts, which risks breaking TypeScript compilation and API startup. Remove the duplicate import and validate the affected API/web tests.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:58Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:03:40Z","started_at":"2026-05-18T13:02:02Z","closed_at":"2026-05-18T13:03:40Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-lk9","title":"Fix PR creation workflow after Forgejo migration","description":"## Why\\nCreating pull requests with fails after the repository moved primary collaboration from GitHub to Forgejo. The current workflow still assumes GitHub GraphQL PR creation semantics, which do not work against the Forgejo remote.\\n\\n## What\\nInvestigate the current PR creation path, identify remaining GitHub-specific assumptions, and update the repo workflow/scripts/docs so contributors can reliably publish branches and open PRs in the Forgejo-based setup.\\n\\n## Acceptance Criteria\\n- The repo no longer instructs contributors to use a broken GitHub-specific PR creation path for Forgejo branches\\n- There is a documented and preferably scripted way to create the equivalent review request against Forgejo\\n- Validation demonstrates the new workflow behaves correctly or clearly documents any remaining platform limitation","status":"in_progress","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T10:26:47Z","created_by":"dirtydishes","updated_at":"2026-05-18T10:26:53Z","started_at":"2026-05-18T10:26:53Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-1ei","title":"Make deploy helper remote-aware for Forgejo","description":"Why: scripts/deploy.ts hardcodes git remote name origin for fetch/pull/push and branch verification, but this repository now uses forgejo/github remotes and may not have an origin remote. What: update deploy.ts to resolve the deploy git remote robustly (Forgejo-aware), use it across local prechecks, branch publish, and remote rollout git operations, and keep behavior explicit in output.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T03:20:12Z","created_by":"dirtydishes","updated_at":"2026-05-18T03:22:39Z","started_at":"2026-05-18T03:20:16Z","closed_at":"2026-05-18T03:22:39Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-xod","title":"Add --fast mode to deploy helper","description":"Why: full main deploys rebuild all images and run full verification, which is slow for routine rollouts. What: add a --fast flag to scripts/deploy.ts with explicit behavior that short-circuits slow steps while preserving basic safety checks; update help text/docs for discoverability.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T02:50:47Z","created_by":"dirtydishes","updated_at":"2026-05-18T02:53:41Z","started_at":"2026-05-18T02:50:50Z","closed_at":"2026-05-18T02:53:41Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-cif","title":"hydrate alert evidence context from clickhouse","description":"Implement alert detail hydration from ClickHouse with a new context endpoint and frontend drawer evidence resolution. Includes storage lookup by alert trace_id/evidence refs, unresolved refs diagnostics, API route GET /flow/alerts/:trace_id/context, terminal evidence hydration + loading states/copy updates, and tests across storage/api/web.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T00:15:55Z","created_by":"dirtydishes","updated_at":"2026-05-18T00:17:38Z","started_at":"2026-05-18T00:16:00Z","closed_at":"2026-05-18T00:17:38Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -46,6 +48,7 @@ {"_type":"issue","id":"islandflow-zs0","title":"Migrate terminal UI to smart-money profiles","description":"Migrate apps/web terminal rendering to consume SmartMoneyEvent directly: primary profile, probability ladder, reason codes, and suppression/abstention state, while preserving legacy alert/classifier displays during the bridge.","status":"closed","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:23Z","created_by":"dirtydishes","updated_at":"2026-05-05T05:39:58Z","closed_at":"2026-05-05T05:39:58Z","close_reason":"Completed terminal smart-money profile migration","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-igk","title":"Add plan mode","description":"Implement a user-facing plan mode in the application so users can switch into planning before taking action. Scope to be clarified from existing app patterns.","status":"closed","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:22:37Z","created_by":"dirtydishes","updated_at":"2026-05-04T04:26:18Z","started_at":"2026-05-04T04:22:40Z","closed_at":"2026-05-04T04:26:18Z","close_reason":"Implemented as a global pi extension toggled with Shift+P","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-biq","title":"Finish raw live options delivery and filter/backpressure observability","description":"The smart-money signal path and Tape filters are in place, but the next firehose pass should finish server-side selective raw live delivery for options subscriptions and add explicit filtered-out/backpressure observability for API/web counters. This was discovered while landing islandflow-e4r.\n","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:28:58Z","created_by":"dirtydishes","updated_at":"2026-04-29T03:54:12Z","started_at":"2026-04-29T03:54:12Z","dependencies":[{"issue_id":"islandflow-biq","depends_on_id":"islandflow-e4r","type":"discovered-from","created_at":"2026-04-28T16:28:58Z","created_by":"auto-import","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-x70","title":"Create 2026-05-17 git standup summary","description":"Why this issue exists and what needs to be done:\\n- Produce the daily automation summary for 2026-05-17 git activity.\\n- Ground statements in commits, PRs, and touched files only.\\n- Create a user-readable HTML document in docs/general and update automation memory.\\n- Complete the Beads sync and git push workflow after documenting the run.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:43Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:05:37Z","started_at":"2026-05-18T13:01:53Z","closed_at":"2026-05-18T13:05:37Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-zsy","title":"Expose Forgejo SSH on a direct DNS hostname","description":"git.deltaisland.io currently resolves through Cloudflare's proxy, so SSH on port 2222 does not complete even though the Forgejo container is listening on the host. If SSH-based git/beads workflows are desired, add a DNS-only hostname (or adjust the existing record) that points directly at the server for Forgejo SSH.","status":"open","priority":3,"issue_type":"task","created_at":"2026-05-17T10:34:06Z","created_by":"delta","updated_at":"2026-05-17T10:34:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-38p","title":"Add native deployment unit templates and rollback helpers","description":"The deploy helper now supports --runtime native, but the repo still relies on operator-managed systemd units and manual rollback. Add checked-in native deployment templates or provisioning guidance for the expected units, and consider lightweight rollback/smoke-test helpers once the host-native path is exercised on the real VPS.","status":"open","priority":3,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:46:42Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:46:42Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-575","title":"Document smart-money event calendar env","description":"Document smart-money event-calendar environment configuration in env examples and README.\n","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T06:57:14Z","created_by":"dirtydishes","updated_at":"2026-05-05T06:57:57Z","started_at":"2026-05-05T06:57:17Z","closed_at":"2026-05-05T06:57:57Z","close_reason":"Documented event-calendar env variables","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/docs/general/2026-05-18-standup-summary-2026-05-17.html b/docs/general/2026-05-18-standup-summary-2026-05-17.html new file mode 100644 index 0000000..ba21b1b --- /dev/null +++ b/docs/general/2026-05-18-standup-summary-2026-05-17.html @@ -0,0 +1,549 @@ + + + + + + Standup Summary for 2026-05-17 + + + +
+
+ Git Standup Summary +

Repository activity recorded for 2026-05-17

+

+ Yesterday's git history shows three main themes: frontend and API work to hydrate alert evidence from + ClickHouse, deploy workflow changes in scripts/deploy.ts, and Beads/Dolt remote setup plus + documentation updates. This summary is grounded in the commits, merged PRs, and touched files visible in the + repository history for 2026-05-17. +

+
+
+ Commit Count + 20 commits on 2026-05-17 +
+
+ Merges + 5 pull request merges +
+
+ File Footprint + 22 distinct paths touched +
+
+ Most Revisited + .beads/issues.jsonl, scripts/deploy.ts, apps/web/app/terminal.tsx +
+
+
+ +
+

Summary

+
+
+ Alert context from ClickHouse landed and was merged twice through follow-up PRs. + The core implementation appeared in commit c0b5b6d and merge PR #41 + (3e08955), then was extended in 58e57fa and merged through #43 + (a27d499) and a documentation polish PR #44 (49efc24). +
+
+ Deploy tooling changed in three steps. + The day included an allowlist tightening in 5ddfbfa, a new fast deploy mode in + 75ed6f3, and Forgejo-aware remote resolution in 6e6788b, all centered on + scripts/deploy.ts. +
+
+ Process and reporting work was visible alongside feature work. + Beads Dolt remote configuration was added in 37bd393, revised in d0d8bd4 and + cd0a1dd, and yesterday's prior standup report was added in 0416194. +
+
+
+ +
+

Changes Made

+
+
+
+ Frontend + API + c0b5b6d + 11:02 EDT +
+

Hydrate alert evidence from ClickHouse

+

+ Commit c0b5b6d added ClickHouse-backed alert context across storage, API, tests, and the + terminal UI. The same change set was merged as PR #41 in 3e08955. +

+
+ packages/storage/src/clickhouse.ts + services/api/src/alert-context.ts + services/api/src/index.ts + apps/web/app/terminal.tsx + apps/web/app/terminal.test.ts + packages/storage/tests/alerts.test.ts +
+
+ +
+
+ Deploy workflow + 5ddfbfa + 11:45 EDT +
+

Tighten deploy remote untracked allowlist

+

+ Commit 5ddfbfa, later merged as PR #42 in 8b166a5, narrowed the + remote untracked allowlist in scripts/deploy.ts. Two follow-up documentation commits, + 8631a53 and 219d3fd, recorded and corrected the validation notes for that + change. +

+
+ scripts/deploy.ts + docs/turns/2026-05-17-deploy-allowlist-pr-packaging.html +
+
+ +
+
+ Integration + 58e57fa + 20:18 EDT +
+

Add ClickHouse alert context hydration for alert drawers

+

+ Commit 58e57fa extended the earlier alert-context work, adding drawer-specific hydration in + the web app and API. A merge-conflict resolution commit dc932cf combined this with the + deploy allowlist branch before PR #43 merged in a27d499. +

+
+ apps/web/app/terminal.tsx + packages/storage/src/clickhouse.ts + services/api/src/index.ts + docs/turns/2026-05-17-clickhouse-alert-context.html +
+
+ +
+
+ Deploy workflow + 75ed6f3 + 22:53 EDT +
+

Add fast deploy mode for routine rollouts

+

+ Commit 75ed6f3 added a faster deploy path and updated both deployment readmes. Minutes + later, commit 6e6788b made deploy remote resolution Forgejo-aware, again in + scripts/deploy.ts. +

+
+ scripts/deploy.ts + deployment/docker/README.md + deployment/native/README.md + docs/turns/2026-05-17-add-fast-deploy-mode.html + docs/turns/2026-05-17-forgejo-deploy-remote-resolution.html +
+
+ +
+
+ Repo operations + 37bd393 + 06:41 EDT +
+

Beads remote setup and daily reporting

+

+ Commit 37bd393 configured the Beads Dolt remote in .beads/config.yaml, then + commits d0d8bd4 and cd0a1dd revised the same sync settings. Commit + 0416194 added the standup summary document for 2026-05-16 activity in + docs/general. +

+
+ .beads/config.yaml + .beads/issues.jsonl + docs/general/2026-05-17-standup-summary-2026-05-16.html +
+
+
+
+ +
+

Context

+
+
+ Merged PRs +
    +
  • #40 merged in 88b2c33: live tape scroll stability and related deploy/image work.
    • +
  • #41 merged in 3e08955: initial ClickHouse alert evidence hydration.
    • +
  • #42 merged in 8b166a5: deploy allowlist packaging follow-through.
    • +
  • #43 merged in a27d499: alert drawer hydration follow-up.
    • +
  • #44 merged in 49efc24: turn-document polish for alert context.
    • +
+
+
+ Most Touched Areas +
    +
  • .beads/issues.jsonl changed in 9 commits, reflecting issue tracking churn throughout the day.
    • +
  • scripts/deploy.ts changed in 3 direct commits tied to deploy safety and speed.
    • +
  • apps/web/app/terminal.tsx changed in 3 direct commits tied to live tape behavior and alert context.
    • +
  • Documentation output expanded across docs/turns and docs/general alongside implementation work.
    • +
+
+
+
+ +
+

Important Implementation Details

+
    +
  • + The ClickHouse alert-context work was not isolated to one layer. Commits c0b5b6d and + 58e57fa touched storage access, API wiring, UI presentation, and dedicated tests, which + makes this the clearest full-stack change in yesterday's history. +
    • +
  • + The deploy changes were incremental rather than a single rewrite. The history shows a narrowing change in + 5ddfbfa, an operator-speed path in 75ed6f3, and remote detection logic in + 6e6788b. +
    • +
  • + Merge commit dc932cf explicitly resolved conflicts between the alert-context and deploy + allowlist branches before later PR merges landed, so yesterday's main branch activity included integration + work as well as feature work. +
    • +
  • + Commit 073c1de created an empty forgejo.test path. The git history shows the file + creation, but no test content in that commit. +
    • +
+
+ +
+

Expected Impact for End-Users

+
    +
  • + User-facing terminal behavior changed in two visible ways: live tape scroll stability from + d334e16/#40 and richer alert evidence context from c0b5b6d, + 58e57fa, and the follow-up merges. +
    • +
  • + Deploy workflow commits affected operator tooling rather than customer-facing product screens. Those changes + should matter most to maintainers using scripts/deploy.ts and the deployment readmes. +
    • +
  • + Beads remote configuration and standup-report commits affected internal workflow and documentation, not + runtime product behavior. +
    • +
+
+ +
+

Validation

+
    +
  • + The turn document added in d334e16 records + bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts passing and + bun --cwd=apps/web run build passing. +
    • +
  • + The turn document added in c0b5b6d records + bun test packages/storage/tests, + bun test services/api/tests, + bun test apps/web/app/terminal.test.ts, and + bun --cwd=apps/web run build. +
    • +
  • + The polished turn document merged in 49efc24 records those alert-context validations as + passing. +
    • +
  • + The deploy allowlist turn document created in 8631a53 and corrected in 219d3fd + explicitly notes that a repository-wide bun test run reported failures at that point. +
    • +
  • + Later deploy-related turn documents added in 75ed6f3 and 6e6788b record full + bun test passing, with the Forgejo remote document stating 232 passing, + 0 failing. +
    • +
  • + This automation run only created documentation. No additional code validation command was run for this + summary itself. +
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • + This document summarizes repository history only. It does not infer goals beyond what commit subjects, PR + titles, merge structure, and touched files show. +
    • +
  • + Some PR context is visible only through merge commits. For example, PR #40 bundles scroll + stability with deploy and Docker-path changes, so the summary reports the merged file footprint rather than + inferring which portion dominated the review. +
    • +
  • + Validation evidence comes from committed turn documents, not from re-running every historical command during + this automation. +
    • +
+
+ +
+

Follow-up Work

+

+ No new follow-up Beads issue was created from the git summary itself. The Beads task for this automation run + is islandflow-x70, which tracks creation of this standup document and will be closed as part of + the session sync. +

+
+
+ + From 906fe411c9daffde038285c3a29f02d07c351e6c Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 18 May 2026 16:55:31 -0400 Subject: [PATCH 03/26] add alpaca news wire across ingest api and web --- .beads/issues.jsonl | 1 + apps/web/app/globals.css | 74 +++++ apps/web/app/news/page.tsx | 7 + apps/web/app/terminal.test.ts | 21 +- apps/web/app/terminal.tsx | 305 +++++++++++++++++++- bun.lock | 13 + deployment/docker/Dockerfile.ingest-options | 1 + deployment/docker/Dockerfile.service | 1 + deployment/docker/Dockerfile.web | 1 + deployment/docker/docker-compose.yml | 4 + deployment/docker/workspace-root/bun.lock | 13 + docs/turns/2026-05-18-news-wire-view.html | 152 ++++++++++ packages/bus/src/streams.ts | 5 +- packages/bus/src/subjects.ts | 2 + packages/storage/src/clickhouse.ts | 141 +++++++++ packages/storage/src/index.ts | 1 + packages/storage/src/news.ts | 102 +++++++ packages/storage/tests/news.test.ts | 78 +++++ packages/types/src/events.ts | 23 ++ packages/types/src/live.ts | 8 +- packages/types/tests/live.test.ts | 26 +- scripts/deploy.ts | 18 +- scripts/dev-services.ts | 1 + scripts/dev.ts | 1 + services/api/src/index.ts | 54 +++- services/api/src/live.ts | 65 +++-- services/ingest-news/package.json | 16 + services/ingest-news/src/index.ts | 216 ++++++++++++++ services/ingest-news/src/symbols.ts | 70 +++++ services/ingest-news/tests/symbols.test.ts | 30 ++ services/ingest-news/tsconfig.json | 7 + 31 files changed, 1407 insertions(+), 50 deletions(-) create mode 100644 apps/web/app/news/page.tsx create mode 100644 docs/turns/2026-05-18-news-wire-view.html create mode 100644 packages/storage/src/news.ts create mode 100644 packages/storage/tests/news.test.ts create mode 100644 services/ingest-news/package.json create mode 100644 services/ingest-news/src/index.ts create mode 100644 services/ingest-news/src/symbols.ts create mode 100644 services/ingest-news/tests/symbols.test.ts create mode 100644 services/ingest-news/tsconfig.json diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 629eb06..9909cdd 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -13,6 +13,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-8fn","title":"implement alpaca-backed news wire view","description":"Why this issue exists and what needs to be done:\\nAdd an Alpaca-powered live news pipeline, API, storage, and web experience, including a dedicated /news route, Home preview, live fanout, history pagination, ticker resolution, and replay-mode live-only empty states.\\n\\nAcceptance criteria:\\n- normalized NewsStory contract and live channel exist\\n- ingest-news service backfills and streams Alpaca news\\n- API persists, serves, and fans out news\\n- web app exposes /news plus Home preview and drawer\\n- tests cover types, storage, API, and key UI behaviors\\n- turn documentation is added\\n\\nDesign:\\nReuse Islandflow drawer, chips, panes, and terminal styling; keep news live-only in v1 replay mode.\\n\\nNotes:\\nImplement client-side ticker filtering in v1 and expose latest revision only per provider+story_id.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T20:37:13Z","created_by":"dirtydishes","updated_at":"2026-05-18T20:55:11Z","started_at":"2026-05-18T20:37:20Z","closed_at":"2026-05-18T20:55:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-k8i","title":"Fix duplicate alert context import in API entrypoint","description":"Recent alert-context work introduced a duplicate fetchAlertContextByTraceId import in services/api/src/index.ts, which risks breaking TypeScript compilation and API startup. Remove the duplicate import and validate the affected API/web tests.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:58Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:03:40Z","started_at":"2026-05-18T13:02:02Z","closed_at":"2026-05-18T13:03:40Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lk9","title":"Fix PR creation workflow after Forgejo migration","description":"## Why\\nCreating pull requests with fails after the repository moved primary collaboration from GitHub to Forgejo. The current workflow still assumes GitHub GraphQL PR creation semantics, which do not work against the Forgejo remote.\\n\\n## What\\nInvestigate the current PR creation path, identify remaining GitHub-specific assumptions, and update the repo workflow/scripts/docs so contributors can reliably publish branches and open PRs in the Forgejo-based setup.\\n\\n## Acceptance Criteria\\n- The repo no longer instructs contributors to use a broken GitHub-specific PR creation path for Forgejo branches\\n- There is a documented and preferably scripted way to create the equivalent review request against Forgejo\\n- Validation demonstrates the new workflow behaves correctly or clearly documents any remaining platform limitation","status":"in_progress","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T10:26:47Z","created_by":"dirtydishes","updated_at":"2026-05-18T10:26:53Z","started_at":"2026-05-18T10:26:53Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-1ei","title":"Make deploy helper remote-aware for Forgejo","description":"Why: scripts/deploy.ts hardcodes git remote name origin for fetch/pull/push and branch verification, but this repository now uses forgejo/github remotes and may not have an origin remote. What: update deploy.ts to resolve the deploy git remote robustly (Forgejo-aware), use it across local prechecks, branch publish, and remote rollout git operations, and keep behavior explicit in output.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T03:20:12Z","created_by":"dirtydishes","updated_at":"2026-05-18T03:22:39Z","started_at":"2026-05-18T03:20:16Z","closed_at":"2026-05-18T03:22:39Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index 64b6f16..cf6746b 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -708,7 +708,12 @@ h3 { grid-template-columns: repeat(3, minmax(0, 1fr)); } +.page-grid-news { + grid-template-columns: minmax(0, 1fr); +} + .page-grid-home > :nth-child(3), +.page-grid-home > :nth-child(4), .page-grid-tape > :nth-child(1), .page-grid-replay > :nth-child(1) { grid-column: 1 / -1; @@ -933,6 +938,7 @@ h3 { } .page-grid-home > :nth-child(3), +.page-grid-home > :nth-child(4), .page-grid-replay > :not(:first-child) { height: clamp(430px, 58vh, 760px); } @@ -1747,6 +1753,72 @@ h3 { gap: 10px; } +.terminal-link-button { + text-decoration: none; +} + +.news-list { + display: flex; + flex-direction: column; + gap: 10px; +} + +.news-row { + width: 100%; + display: flex; + flex-direction: column; + gap: 8px; + padding: 14px 16px; + border: 1px solid var(--border); + border-radius: 12px; + background: oklch(0.18 0.012 250 / 0.6); + color: var(--text); + text-align: left; + transition: border-color 150ms ease, background 150ms ease; +} + +.news-row:hover { + border-color: var(--accent-soft); + background: oklch(0.2 0.015 250 / 0.75); +} + +.news-row-head, +.news-row-meta { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + flex-wrap: wrap; +} + +.news-row h3 { + margin: 0; + font-size: 0.96rem; + font-weight: 600; +} + +.news-row-time { + color: var(--text-dim); + font-family: var(--font-mono), monospace; + font-size: 0.78rem; +} + +.news-row-meta { + color: var(--text-dim); + font-size: 0.78rem; +} + +.news-drawer-body a { + color: var(--accent); +} + +.news-drawer-body p, +.news-drawer-body ul, +.news-drawer-body ol, +.news-drawer-body blockquote { + margin: 0 0 12px; +} + .synthetic-status-grid strong, .synthetic-hit-row strong { font-family: var(--font-mono), monospace; @@ -1964,6 +2036,7 @@ h3 { } .page-grid-home > :nth-child(3), + .page-grid-home > :nth-child(4), .page-grid-tape > :nth-child(1), .page-grid-replay > :nth-child(1) { grid-column: auto; @@ -1973,6 +2046,7 @@ h3 { .page-grid-home > :nth-child(1), .page-grid-home > :nth-child(2), .page-grid-home > :nth-child(3), + .page-grid-home > :nth-child(4), .page-grid-signals > .terminal-pane, .page-grid-replay > :not(:first-child), .page-grid-tape > :first-child, diff --git a/apps/web/app/news/page.tsx b/apps/web/app/news/page.tsx new file mode 100644 index 0000000..7e06aa8 --- /dev/null +++ b/apps/web/app/news/page.tsx @@ -0,0 +1,7 @@ +import { NewsRoute } from "../terminal"; + +export const dynamic = "force-dynamic"; + +export default function Page() { + return ; +} diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index 2be3da8..63918f2 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -247,6 +247,15 @@ describe("live manifest", () => { ]); }); + it("includes news subscriptions on home and /news", () => { + expect(getLiveManifest("/", "SPY", 60000, buildDefaultFlowFilters()).map((subscription) => subscription.channel)).toContain( + "news" + ); + expect(getLiveManifest("/news", "SPY", 60000, buildDefaultFlowFilters()).map((subscription) => subscription.channel)).toEqual([ + "news" + ]); + }); + it("scopes /charts subscriptions to chart channels only", () => { const channels = getLiveManifest("/charts", "SPY", 60000, buildDefaultFlowFilters()).map( (subscription) => subscription.channel @@ -431,6 +440,13 @@ describe("route feature map", () => { expect(features.equityOverlay).toBe(true); expect(features.alerts).toBe(false); }); + + it("maps /news to the dedicated news pane", () => { + const features = getRouteFeatures("/news"); + expect(features.news).toBe(true); + expect(features.showNewsPane).toBe(true); + expect(features.showAlertsPane).toBe(false); + }); }); describe("fixed tape virtualization config", () => { @@ -461,10 +477,11 @@ describe("dark underlying route dependency helper", () => { }); describe("terminal navigation", () => { - it("exposes only Home and Tape as top-level destinations", () => { + it("exposes Home, Tape, and News as top-level destinations", () => { expect(NAV_ITEMS).toEqual([ { href: "/", label: "Home" }, - { href: "/tape", label: "Tape" } + { href: "/tape", label: "Tape" }, + { href: "/news", label: "News" } ]); }); }); diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index e1ee74c..218e149 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -33,6 +33,7 @@ import type { LiveServerMessage, LiveHotChannelHealthMap, LiveSubscription, + NewsStory, OptionFlowFilters, OptionFlowView, OptionNbboSide, @@ -158,6 +159,7 @@ type RouteFeatures = { nbbo: boolean; equities: boolean; flow: boolean; + news: boolean; alerts: boolean; smartMoney: boolean; classifierHits: boolean; @@ -168,6 +170,7 @@ type RouteFeatures = { showOptionsPane: boolean; showEquitiesPane: boolean; showFlowPane: boolean; + showNewsPane: boolean; showAlertsPane: boolean; showClassifierPane: boolean; showDarkPane: boolean; @@ -187,6 +190,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { const includeEquitiesFallback = shouldIncludeEquitiesForDarkUnderlyingFallback(); const normalizedPath = pathname === "/tape" || + pathname === "/news" || pathname === "/signals" || pathname === "/charts" || pathname === "/replay" @@ -200,6 +204,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { nbbo: true, equities: true, flow: true, + news: false, alerts: false, smartMoney: false, classifierHits: false, @@ -210,6 +215,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { showOptionsPane: true, showEquitiesPane: true, showFlowPane: true, + showNewsPane: false, showAlertsPane: false, showClassifierPane: false, showDarkPane: false, @@ -220,12 +226,41 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { needsAlertEvidencePrefetch: false, needsDarkUnderlying: false }; + case "/news": + return { + options: false, + nbbo: false, + equities: false, + flow: false, + news: true, + alerts: false, + smartMoney: false, + classifierHits: false, + inferredDark: false, + equityJoins: false, + equityCandles: false, + equityOverlay: false, + showOptionsPane: false, + showEquitiesPane: false, + showFlowPane: false, + showNewsPane: true, + showAlertsPane: false, + showClassifierPane: false, + showDarkPane: false, + showChartPane: false, + showFocusPane: false, + showReplayConsole: false, + needsClassifierDecor: false, + needsAlertEvidencePrefetch: false, + needsDarkUnderlying: false + }; case "/signals": return { options: false, nbbo: false, equities: includeEquitiesFallback, flow: false, + news: false, alerts: true, smartMoney: true, classifierHits: true, @@ -236,6 +271,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { showOptionsPane: false, showEquitiesPane: false, showFlowPane: false, + showNewsPane: false, showAlertsPane: true, showClassifierPane: true, showDarkPane: true, @@ -252,6 +288,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { nbbo: false, equities: includeEquitiesFallback, flow: false, + news: false, alerts: false, smartMoney: true, classifierHits: false, @@ -262,6 +299,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { showOptionsPane: false, showEquitiesPane: false, showFlowPane: false, + showNewsPane: false, showAlertsPane: false, showClassifierPane: false, showDarkPane: false, @@ -278,6 +316,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { nbbo: false, equities: false, flow: false, + news: false, alerts: false, smartMoney: false, classifierHits: false, @@ -288,6 +327,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { showOptionsPane: true, showEquitiesPane: false, showFlowPane: true, + showNewsPane: false, showAlertsPane: true, showClassifierPane: false, showDarkPane: false, @@ -305,6 +345,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { nbbo: false, equities: true, flow: false, + news: true, alerts: true, smartMoney: true, classifierHits: false, @@ -315,6 +356,7 @@ export const getRouteFeatures = (pathname: string): RouteFeatures => { showOptionsPane: false, showEquitiesPane: true, showFlowPane: false, + showNewsPane: true, showAlertsPane: true, showClassifierPane: false, showDarkPane: false, @@ -332,6 +374,7 @@ const EMPTY_ALERT_EVENTS: AlertEvent[] = []; const EMPTY_CLASSIFIER_HIT_EVENTS: ClassifierHitEvent[] = []; const EMPTY_SMART_MONEY_EVENTS: SmartMoneyEvent[] = []; const EMPTY_INFERRED_DARK_EVENTS: InferredDarkEvent[] = []; +const EMPTY_NEWS_STORIES: NewsStory[] = []; type CandlestickSeries = ReturnType; @@ -1194,6 +1237,44 @@ const formatDateTime = (ts: number): string => { return `${date.toLocaleDateString()} ${date.toLocaleTimeString()}`; }; +const isSameLocalDay = (left: number, right: number): boolean => { + const a = new Date(left); + const b = new Date(right); + return ( + a.getFullYear() === b.getFullYear() && + a.getMonth() === b.getMonth() && + a.getDate() === b.getDate() + ); +}; + +export const formatNewsTimestamp = (ts: number, now = Date.now()): string => { + const date = new Date(ts); + return isSameLocalDay(ts, now) + ? date.toLocaleTimeString([], { hour: "numeric", minute: "2-digit" }) + : date.toLocaleString([], { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" }); +}; + +const sanitizeNewsHtml = (value: string): { html: string; fallbackText: string; sanitized: boolean } => { + const fallbackText = value + .replace(//gi, " ") + .replace(//gi, " ") + .replace(/<[^>]+>/g, " ") + .replace(/\s+/g, " ") + .trim(); + + try { + const sanitized = value + .replace(//gi, "") + .replace(//gi, "") + .replace(/\son\w+=(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, "") + .replace(/\shref=(["'])javascript:[\s\S]*?\1/gi, ' href="#"') + .replace(/<(?!\/?(p|div|section|article|span|strong|em|b|i|ul|ol|li|br|a|h1|h2|h3|h4|blockquote)\b)[^>]*>/gi, ""); + return { html: sanitized, fallbackText, sanitized: true }; + } catch { + return { html: "", fallbackText, sanitized: false }; + } +}; + const humanizeClassifierId = (value: string): string => { if (!value) { return "Classifier"; @@ -2870,6 +2951,7 @@ type LiveSessionState = { smartMoneyHistory: SmartMoneyEvent[]; classifierHitsHistory: ClassifierHitEvent[]; alertsHistory: AlertEvent[]; + newsHistory: NewsStory[]; inferredDarkHistory: InferredDarkEvent[]; options: OptionPrint[]; nbbo: OptionNBBO[]; @@ -2880,6 +2962,7 @@ type LiveSessionState = { smartMoney: SmartMoneyEvent[]; classifierHits: ClassifierHitEvent[]; alerts: AlertEvent[]; + news: NewsStory[]; inferredDark: InferredDarkEvent[]; chartCandles: EquityCandle[]; chartOverlay: EquityPrint[]; @@ -2900,6 +2983,7 @@ const LIVE_HISTORY_ENDPOINTS: Partial([]); const [classifierHits, setClassifierHits] = useState([]); const [alerts, setAlerts] = useState([]); + const [news, setNews] = useState([]); const [inferredDark, setInferredDark] = useState([]); const [optionsHistory, setOptionsHistory] = useState([]); const [nbboHistory, setNbboHistory] = useState([]); @@ -3142,6 +3230,7 @@ const useLiveSession = ( const [smartMoneyHistory, setSmartMoneyHistory] = useState([]); const [classifierHitsHistory, setClassifierHitsHistory] = useState([]); const [alertsHistory, setAlertsHistory] = useState([]); + const [newsHistory, setNewsHistory] = useState([]); const [inferredDarkHistory, setInferredDarkHistory] = useState([]); const [chartCandles, setChartCandles] = useState([]); const [chartOverlay, setChartOverlay] = useState([]); @@ -3154,6 +3243,7 @@ const useLiveSession = ( const smartMoneyRef = useRef([]); const classifierHitsRef = useRef([]); const alertsRef = useRef([]); + const newsRef = useRef([]); const inferredDarkRef = useRef([]); const chartCandlesRef = useRef([]); const chartOverlayRef = useRef([]); @@ -3165,6 +3255,7 @@ const useLiveSession = ( const smartMoneyHistoryRef = useRef([]); const classifierHitsHistoryRef = useRef([]); const alertsHistoryRef = useRef([]); + const newsHistoryRef = useRef([]); const inferredDarkHistoryRef = useRef([]); const socketRef = useRef(null); const reconnectRef = useRef(null); @@ -3218,6 +3309,7 @@ const useLiveSession = ( setSmartMoney([]); setClassifierHits([]); setAlerts([]); + setNews([]); setInferredDark([]); setOptionsHistory([]); setNbboHistory([]); @@ -3227,6 +3319,7 @@ const useLiveSession = ( setSmartMoneyHistory([]); setClassifierHitsHistory([]); setAlertsHistory([]); + setNewsHistory([]); setInferredDarkHistory([]); setChartCandles([]); setChartOverlay([]); @@ -3239,6 +3332,7 @@ const useLiveSession = ( smartMoneyRef.current = []; classifierHitsRef.current = []; alertsRef.current = []; + newsRef.current = []; inferredDarkRef.current = []; chartCandlesRef.current = []; chartOverlayRef.current = []; @@ -3250,6 +3344,7 @@ const useLiveSession = ( smartMoneyHistoryRef.current = []; classifierHitsHistoryRef.current = []; alertsHistoryRef.current = []; + newsHistoryRef.current = []; inferredDarkHistoryRef.current = []; subscribedKeysRef.current = new Set(); subscribedMapRef.current = new Map(); @@ -3403,6 +3498,12 @@ const useLiveSession = ( ref: alertsHistoryRef }); break; + case "news": + mergeItems(setNews, newsRef, items as NewsStory[], LIVE_OPTIONS_HEAD_LIMIT, { + setter: setNewsHistory, + ref: newsHistoryRef + }); + break; case "inferred-dark": mergeItems(setInferredDark, inferredDarkRef, items as InferredDarkEvent[], LIVE_HOT_WINDOW, { setter: setInferredDarkHistory, @@ -3694,6 +3795,9 @@ const useLiveSession = ( case "alerts": mergeOlder(setAlertsHistory, alertsHistoryRef, alertsRef.current); break; + case "news": + mergeOlder(setNewsHistory, newsHistoryRef, newsRef.current); + break; case "inferred-dark": mergeOlder(setInferredDarkHistory, inferredDarkHistoryRef, inferredDarkRef.current); break; @@ -3735,6 +3839,7 @@ const useLiveSession = ( smartMoneyHistory, classifierHitsHistory, alertsHistory, + newsHistory, inferredDarkHistory, options, nbbo, @@ -3745,6 +3850,7 @@ const useLiveSession = ( smartMoney, classifierHits, alerts, + news, inferredDark, chartCandles, chartOverlay @@ -4822,6 +4928,69 @@ const AlertDrawer = ({ alert, flowPacket, evidence, contextStatus, onClose }: Al ); }; +type NewsDrawerProps = { + story: NewsStory; + onClose: () => void; +}; + +const NewsDrawer = ({ story, onClose }: NewsDrawerProps) => { + const body = sanitizeNewsHtml(story.content_html); + + return ( + + ); +}; + type ClassifierHitDrawerProps = { hit: ClassifierHitEvent; flowPacket: FlowPacket | null; @@ -5178,6 +5347,7 @@ const useTerminalState = () => { const [mode, setMode] = useState("live"); const [replaySource, setReplaySource] = useState(null); const [selectedAlert, setSelectedAlert] = useState(null); + const [selectedNewsStory, setSelectedNewsStory] = useState(null); const [selectedDarkEvent, setSelectedDarkEvent] = useState(null); const [selectedClassifierHit, setSelectedClassifierHit] = useState(null); const [selectedSmartMoneyEvent, setSelectedSmartMoneyEvent] = useState(null); @@ -5274,12 +5444,13 @@ const useTerminalState = () => { }, [mode]); useEffect(() => { - if (!selectedAlert && !selectedClassifierHit && !selectedDarkEvent && !selectedSmartMoneyEvent) { + if (!selectedAlert && !selectedNewsStory && !selectedClassifierHit && !selectedDarkEvent && !selectedSmartMoneyEvent) { return; } const dismissDrawers = () => { setSelectedAlert(null); + setSelectedNewsStory(null); setSelectedClassifierHit(null); setSelectedSmartMoneyEvent(null); setSelectedDarkEvent(null); @@ -5305,7 +5476,7 @@ const useTerminalState = () => { document.removeEventListener("mousedown", handlePointerDown); document.removeEventListener("keydown", handleKeyDown); }; - }, [selectedAlert, selectedClassifierHit, selectedDarkEvent, selectedSmartMoneyEvent]); + }, [selectedAlert, selectedNewsStory, selectedClassifierHit, selectedDarkEvent, selectedSmartMoneyEvent]); const optionsScroll = useListScroll(); const equitiesScroll = useListScroll(); @@ -5540,6 +5711,14 @@ const useTerminalState = () => { ) : equityJoins; const flowFeed = mode === "live" ? liveFlow : flow; + const newsFeed = + mode === "live" + ? toStaticTapeState( + liveSession.status, + composeTapeItems([], liveSession.news, liveSession.newsHistory), + liveSession.lastUpdate + ) + : toStaticTapeState("disconnected", [], null); const alertsFeed = mode === "live" ? toStaticTapeState( @@ -6490,6 +6669,16 @@ const useTerminalState = () => { routeFeatures.needsAlertEvidencePrefetch ]); + const filteredNews = useMemo(() => { + if (!routeFeatures.news && !routeFeatures.showNewsPane) { + return EMPTY_NEWS_STORIES; + } + if (tickerSet.size === 0) { + return newsFeed.items; + } + return newsFeed.items.filter((story) => story.resolved_symbols.some((symbol) => matchesTicker(symbol))); + }, [matchesTicker, newsFeed.items, routeFeatures.news, routeFeatures.showNewsPane, tickerSet]); + const visibleAlerts = useMemo(() => { if (routeFeatures.needsAlertEvidencePrefetch) { return filteredAlerts.slice(0, 12); @@ -6767,6 +6956,7 @@ const useTerminalState = () => { (hit: ClassifierHitEvent) => { const alert = findAlertForClassifierHit(hit); if (alert) { + setSelectedNewsStory(null); setSelectedClassifierHit(null); setSelectedDarkEvent(null); setSelectedSmartMoneyEvent(null); @@ -6774,6 +6964,7 @@ const useTerminalState = () => { return; } + setSelectedNewsStory(null); setSelectedAlert(null); setSelectedDarkEvent(null); setSelectedSmartMoneyEvent(null); @@ -6783,6 +6974,7 @@ const useTerminalState = () => { ); const openFromSmartMoneyEvent = useCallback((event: SmartMoneyEvent) => { + setSelectedNewsStory(null); setSelectedAlert(null); setSelectedClassifierHit(null); setSelectedDarkEvent(null); @@ -6797,6 +6989,7 @@ const useTerminalState = () => { ); const handleDarkMarkerClick = useCallback((event: InferredDarkEvent) => { + setSelectedNewsStory(null); setSelectedAlert(null); setSelectedClassifierHit(null); setSelectedSmartMoneyEvent(null); @@ -6817,6 +7010,9 @@ const useTerminalState = () => { if (routeFeatures.flow || routeFeatures.showFlowPane) { updates.push(flowFeed.lastUpdate); } + if (routeFeatures.news || routeFeatures.showNewsPane) { + updates.push(newsFeed.lastUpdate); + } if (routeFeatures.alerts || routeFeatures.showAlertsPane) { updates.push(alertsFeed.lastUpdate); } @@ -6839,6 +7035,8 @@ const useTerminalState = () => { routeFeatures.showFocusPane, routeFeatures.flow, routeFeatures.showFlowPane, + routeFeatures.news, + routeFeatures.showNewsPane, routeFeatures.alerts, routeFeatures.showAlertsPane, routeFeatures.smartMoney, @@ -6849,6 +7047,7 @@ const useTerminalState = () => { equitiesFeed.lastUpdate, inferredDarkFeed.lastUpdate, flowFeed.lastUpdate, + newsFeed.lastUpdate, alertsFeed.lastUpdate, smartMoneyFeed.lastUpdate, classifierHitsFeed.lastUpdate @@ -6861,6 +7060,8 @@ const useTerminalState = () => { setReplaySource, selectedAlert, setSelectedAlert, + selectedNewsStory, + setSelectedNewsStory, selectedDarkEvent, setSelectedDarkEvent, selectedClassifierHit, @@ -6887,6 +7088,7 @@ const useTerminalState = () => { equityJoins: equityJoinsFeed, nbbo: nbboFeed, inferredDark: inferredDarkFeed, + news: newsFeed, flow: flowFeed, alerts: alertsFeed, smartMoney: smartMoneyFeed, @@ -6920,6 +7122,7 @@ const useTerminalState = () => { equitiesScopedQuiet, equitiesSilentWarning, filteredInferredDark, + filteredNews, filteredFlow, filteredAlerts, filteredSmartMoneyEvents, @@ -6953,7 +7156,8 @@ const useTerminal = (): TerminalState => { export const NAV_ITEMS = [ { href: "/", label: "Home" }, - { href: "/tape", label: "Tape" } + { href: "/tape", label: "Tape" }, + { href: "/news", label: "News" } ] as const; type PageFrameProps = { @@ -7780,6 +7984,7 @@ const AlertsPane = memo(({ state, limit, withStrip = false, className }: AlertsP data-tape-key={key} style={{ transform: `translateY(${start}px)` }} onClick={() => { + state.setSelectedNewsStory(null); state.setSelectedDarkEvent(null); state.setSelectedClassifierHit(null); state.setSelectedSmartMoneyEvent(null); @@ -7806,6 +8011,83 @@ const AlertsPane = memo(({ state, limit, withStrip = false, className }: AlertsP ); }); +type NewsPaneProps = { + state: TerminalState; + limit?: number; + className?: string; +}; + +const NewsPane = memo(({ state, limit, className }: NewsPaneProps) => { + const items = limit ? state.filteredNews.slice(0, limit) : state.filteredNews; + const canLoadOlder = state.mode === "live" && !limit && items.length > 0; + + return ( + + View all + + ) : ( +
+ + {state.mode === "live" ? "Live wire" : "Live-only in v1"} +
+ ) + } + actions={ + canLoadOlder ? ( + + ) : null + } + > + {state.mode === "replay" ? ( +
News is live-only in v1.
+ ) : items.length === 0 ? ( +
+ {state.tickerSet.size > 0 ? "No news stories match the current filter." : "Waiting for live news stories."} +
+ ) : ( +
+ {items.map((story) => ( + + ))} +
+ )} + + ); +}); + type ClassifierPaneProps = { state: TerminalState; limit?: number; @@ -8016,6 +8298,7 @@ const DarkPane = memo(({ state, limit, className }: DarkPaneProps) => { data-tape-key={key} style={{ transform: `translateY(${start}px)` }} onClick={() => { + state.setSelectedNewsStory(null); state.setSelectedAlert(null); state.setSelectedClassifierHit(null); state.setSelectedSmartMoneyEvent(null); @@ -8624,6 +8907,10 @@ export function TerminalAppShell({ children }: { children: ReactNode }) { /> ) : null} + {state.selectedNewsStory ? ( + state.setSelectedNewsStory(null)} /> + ) : null} + {state.selectedClassifierHit ? ( + ); } +export function NewsRoute() { + const state = useTerminal(); + return ( + +
+ +
+ + ); +} + export function TapeRoute() { const state = useTerminal(); return ( diff --git a/bun.lock b/bun.lock index 46160a7..35e00d7 100644 --- a/bun.lock +++ b/bun.lock @@ -121,6 +121,17 @@ "zod": "^3.23.8", }, }, + "services/ingest-news": { + "name": "@islandflow/ingest-news", + "dependencies": { + "@islandflow/bus": "workspace:*", + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + "@islandflow/types": "workspace:*", + "ws": "^8.18.3", + "zod": "^3.23.8", + }, + }, "services/ingest-options": { "name": "@islandflow/ingest-options", "dependencies": { @@ -250,6 +261,8 @@ "@islandflow/ingest-equities": ["@islandflow/ingest-equities@workspace:services/ingest-equities"], + "@islandflow/ingest-news": ["@islandflow/ingest-news@workspace:services/ingest-news"], + "@islandflow/ingest-options": ["@islandflow/ingest-options@workspace:services/ingest-options"], "@islandflow/observability": ["@islandflow/observability@workspace:packages/observability"], diff --git a/deployment/docker/Dockerfile.ingest-options b/deployment/docker/Dockerfile.ingest-options index 52cba59..212b96b 100644 --- a/deployment/docker/Dockerfile.ingest-options +++ b/deployment/docker/Dockerfile.ingest-options @@ -31,6 +31,7 @@ COPY --from=services candles/package.json ./services/candles/package.json COPY --from=services compute/package.json ./services/compute/package.json COPY --from=services eod-enricher/package.json ./services/eod-enricher/package.json COPY --from=services ingest-equities/package.json ./services/ingest-equities/package.json +COPY --from=services ingest-news/package.json ./services/ingest-news/package.json COPY --from=services ingest-options/package.json ./services/ingest-options/package.json COPY --from=services ingest-options/py/requirements.txt ./services/ingest-options/py/requirements.txt COPY --from=services refdata/package.json ./services/refdata/package.json diff --git a/deployment/docker/Dockerfile.service b/deployment/docker/Dockerfile.service index e0fcf72..4a7d9f1 100644 --- a/deployment/docker/Dockerfile.service +++ b/deployment/docker/Dockerfile.service @@ -24,6 +24,7 @@ COPY --from=services candles/package.json ./services/candles/package.json COPY --from=services compute/package.json ./services/compute/package.json COPY --from=services eod-enricher/package.json ./services/eod-enricher/package.json COPY --from=services ingest-equities/package.json ./services/ingest-equities/package.json +COPY --from=services ingest-news/package.json ./services/ingest-news/package.json COPY --from=services ingest-options/package.json ./services/ingest-options/package.json COPY --from=services refdata/package.json ./services/refdata/package.json COPY --from=services replay/package.json ./services/replay/package.json diff --git a/deployment/docker/Dockerfile.web b/deployment/docker/Dockerfile.web index 33723ae..37443d9 100644 --- a/deployment/docker/Dockerfile.web +++ b/deployment/docker/Dockerfile.web @@ -30,6 +30,7 @@ COPY --from=services candles/package.json ./services/candles/package.json COPY --from=services compute/package.json ./services/compute/package.json COPY --from=services eod-enricher/package.json ./services/eod-enricher/package.json COPY --from=services ingest-equities/package.json ./services/ingest-equities/package.json +COPY --from=services ingest-news/package.json ./services/ingest-news/package.json COPY --from=services ingest-options/package.json ./services/ingest-options/package.json COPY --from=services refdata/package.json ./services/refdata/package.json COPY --from=services replay/package.json ./services/replay/package.json diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index 96598ba..37682f6 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -115,6 +115,10 @@ services: <<: *service-common command: ["services/ingest-equities/src/index.ts"] + ingest-news: + <<: *service-common + command: ["services/ingest-news/src/index.ts"] + replay: <<: *service-common profiles: ["replay"] diff --git a/deployment/docker/workspace-root/bun.lock b/deployment/docker/workspace-root/bun.lock index 46160a7..35e00d7 100644 --- a/deployment/docker/workspace-root/bun.lock +++ b/deployment/docker/workspace-root/bun.lock @@ -121,6 +121,17 @@ "zod": "^3.23.8", }, }, + "services/ingest-news": { + "name": "@islandflow/ingest-news", + "dependencies": { + "@islandflow/bus": "workspace:*", + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + "@islandflow/types": "workspace:*", + "ws": "^8.18.3", + "zod": "^3.23.8", + }, + }, "services/ingest-options": { "name": "@islandflow/ingest-options", "dependencies": { @@ -250,6 +261,8 @@ "@islandflow/ingest-equities": ["@islandflow/ingest-equities@workspace:services/ingest-equities"], + "@islandflow/ingest-news": ["@islandflow/ingest-news@workspace:services/ingest-news"], + "@islandflow/ingest-options": ["@islandflow/ingest-options@workspace:services/ingest-options"], "@islandflow/observability": ["@islandflow/observability@workspace:packages/observability"], diff --git a/docs/turns/2026-05-18-news-wire-view.html b/docs/turns/2026-05-18-news-wire-view.html new file mode 100644 index 0000000..be02f26 --- /dev/null +++ b/docs/turns/2026-05-18-news-wire-view.html @@ -0,0 +1,152 @@ + + + + + + Turn Report: News Wire View via Alpaca Feed + + + +
+

Created 2026-05-18 · Task: News Wire View via Alpaca Feed

+

News Wire View via Alpaca Feed

+
+ Summary +

+ Added an Alpaca-backed live news pipeline end to end: normalized NewsStory types, + a dedicated JetStream subject/stream, ClickHouse storage helpers with latest-revision semantics, + a new services/ingest-news service, API endpoints and live fanout, and a web + /news route plus Home preview with a right-side story drawer. +

+
+ +
+

Changes Made

+
    +
  • Added NewsStorySchema, the news live channel, and subscription parsing support in packages/types.
    • +
  • Added bus constants for the flow.news subject and NEWS stream.
    • +
  • Added ClickHouse news storage helpers, including recent, before-cursor, and after-cursor queries that collapse provider revisions to the latest row per provider + story_id.
    • +
  • Created services/ingest-news with Alpaca REST backfill, Alpaca websocket streaming, normalization, and deterministic ticker resolution.
    • +
  • Extended the API service to persist live news in the shared cache, expose GET /news and GET /history/news, and fan out news events on /ws/live.
    • +
  • Added a top-level /news route, primary nav entry, Home preview pane, replay-mode live-only empty states, and a sanitized full-story drawer.
    • +
  • Updated dev and deployment wiring so the new service is included in local runners and the Docker workspace snapshot.
    • +
+
+ +
+

Context

+

+ The plan called for a free-provider v1 news surface that behaves like the rest of Islandflow: + compact, evidence-first, and live-native. The implementation keeps replay intentionally out of scope + for news while still integrating news into the same live manifest, history pagination, rail navigation, + and drawer language used elsewhere in the terminal. +

+
+ +
+

Important Implementation Details

+
    +
  • Ticker resolution prefers provider symbols first, then falls back only to structured patterns in provider HTML: ticker anchors, EXCHANGE:SYM, and $SYM.
    • +
  • News history uses published_ts as the visible cursor while revisions are collapsed with a window function over provider, story_id ordered by updated_ts, ingest_ts, and seq.
    • +
  • The web drawer sanitizes provider HTML by removing scripts, inline event handlers, and unsupported tags; if sanitization yields nothing useful, the drawer falls back to stripped plain text.
    • +
  • Replay mode intentionally renders a clear empty state for news on both Home and /news instead of pretending news is replay-synced.
    • +
+ 
resolved_symbols = provider_symbols
+  or ticker anchors in content_html
+  or EXCHANGE:SYM matches
+  or $SYM matches
+
+ +
+

Expected Impact for End-Users

+

+ Traders can now monitor a dedicated live news wire inside Islandflow, spot symbol-linked headlines from + the Home view, and open full stories in-context without leaving the app. The displayed ticker chips are + grounded in stored provider and derived symbol metadata, which makes the feed safer to filter and trust. +

+
+ +
+

Validation

+
    +
  • Ran targeted Bun tests covering types, storage, API live-state behavior, ingest-news symbol resolution, route wiring, and terminal helpers.
    • +
  • Built the Next.js web app with bun --cwd=apps/web run build.
    • +
  • Ran bun run check:docker-workspace after syncing the deployment workspace snapshot.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • Replay support remains intentionally absent in v1; the UI now states that explicitly instead of showing misleading empty historical behavior.
    • +
  • The sanitizer is intentionally conservative and custom, which keeps dependencies light but may strip some harmless provider formatting.
    • +
  • The ingest service assumes Alpaca’s current REST and websocket news contracts; if Alpaca changes those payload shapes, the normalization layer will need adjustment.
    • +
+
+ +
+

Follow-up Work

+
    +
  • No additional follow-up issue was required during this turn.
    • +
  • Future extensions are still available behind the same contract: multi-provider aggregation, server-side symbol filtering, and replay-aware news history.
    • +
+
+
+ + diff --git a/packages/bus/src/streams.ts b/packages/bus/src/streams.ts index eeb8116..b23c125 100644 --- a/packages/bus/src/streams.ts +++ b/packages/bus/src/streams.ts @@ -7,6 +7,7 @@ import { STREAM_EQUITY_QUOTES, STREAM_FLOW_PACKETS, STREAM_INFERRED_DARK, + STREAM_NEWS, STREAM_OPTION_NBBO, STREAM_OPTION_PRINTS, STREAM_OPTION_SIGNAL_PRINTS, @@ -19,6 +20,7 @@ import { SUBJECT_EQUITY_QUOTES, SUBJECT_FLOW_PACKETS, SUBJECT_INFERRED_DARK, + SUBJECT_NEWS, SUBJECT_OPTION_NBBO, SUBJECT_OPTION_PRINTS, SUBJECT_OPTION_SIGNAL_PRINTS, @@ -53,7 +55,8 @@ export const STREAM_CATALOG: readonly KnownStreamDefinition[] = [ retentionClass: "derived" }, { name: STREAM_CLASSIFIER_HITS, subject: SUBJECT_CLASSIFIER_HITS, retentionClass: "derived" }, - { name: STREAM_ALERTS, subject: SUBJECT_ALERTS, retentionClass: "derived" } + { name: STREAM_ALERTS, subject: SUBJECT_ALERTS, retentionClass: "derived" }, + { name: STREAM_NEWS, subject: SUBJECT_NEWS, retentionClass: "derived" } ]; const STREAM_CATALOG_BY_NAME = new Map(STREAM_CATALOG.map((definition) => [definition.name, definition])); diff --git a/packages/bus/src/subjects.ts b/packages/bus/src/subjects.ts index 6b21afd..956d357 100644 --- a/packages/bus/src/subjects.ts +++ b/packages/bus/src/subjects.ts @@ -22,3 +22,5 @@ export const STREAM_CLASSIFIER_HITS = "CLASSIFIER_HITS"; export const SUBJECT_CLASSIFIER_HITS = "flow.classifier_hits"; export const STREAM_ALERTS = "ALERTS"; export const SUBJECT_ALERTS = "flow.alerts"; +export const STREAM_NEWS = "NEWS"; +export const SUBJECT_NEWS = "flow.news"; diff --git a/packages/storage/src/clickhouse.ts b/packages/storage/src/clickhouse.ts index bc0061e..af469d7 100644 --- a/packages/storage/src/clickhouse.ts +++ b/packages/storage/src/clickhouse.ts @@ -7,6 +7,7 @@ import { EquityPrintJoinSchema, InferredDarkEventSchema, FlowPacketSchema, + NewsStorySchema, OptionNBBOSchema, OptionPrintSchema, SmartMoneyEventSchema @@ -20,6 +21,7 @@ import type { EquityPrintJoin, InferredDarkEvent, FlowPacket, + NewsStory, SmartMoneyEvent, OptionNBBO, OptionPrint, @@ -91,6 +93,13 @@ import { toSmartMoneyEventRecord, type SmartMoneyEventRecord } from "./smart-money-events"; +import { + NEWS_TABLE, + newsTableDDL, + fromNewsRecord, + toNewsRecord, + type NewsRecord +} from "./news"; export type ClickHouseOptions = { url: string; @@ -320,6 +329,12 @@ export const ensureAlertsTable = async (client: ClickHouseClient): Promise } }; +export const ensureNewsTable = async (client: ClickHouseClient): Promise => { + await client.exec({ + query: newsTableDDL() + }); +}; + export const insertOptionPrint = async ( client: ClickHouseClient, print: OptionPrint @@ -449,6 +464,15 @@ export const insertAlert = async (client: ClickHouseClient, alert: AlertEvent): }); }; +export const insertNewsStory = async (client: ClickHouseClient, story: NewsStory): Promise => { + const record = toNewsRecord(story); + await client.insert({ + table: NEWS_TABLE, + values: [record], + format: "JSONEachRow" + }); +}; + export type ClickHouseBatchWriterOptions = { flushIntervalMs?: number; maxRows?: number; @@ -600,6 +624,13 @@ export const enqueueAlertInsert = ( writer.enqueue(ALERTS_TABLE, toAlertRecord(alert)); }; +export const enqueueNewsStoryInsert = ( + writer: ClickHouseBatchWriter, + story: NewsStory +): void => { + writer.enqueue(NEWS_TABLE, toNewsRecord(story)); +}; + const clampLimit = (limit: number): number => { if (!Number.isFinite(limit)) { return 100; @@ -1016,6 +1047,32 @@ const normalizeAlertRow = (row: unknown): AlertRecord | null => { }; }; +const normalizeNewsRow = (row: unknown): NewsRecord | null => { + if (!row || typeof row !== "object") { + return null; + } + + const record = row as Record; + return { + source_ts: coerceNumber(record.source_ts) as number, + ingest_ts: coerceNumber(record.ingest_ts) as number, + seq: coerceNumber(record.seq) as number, + trace_id: String(record.trace_id ?? ""), + story_id: coerceNumber(record.story_id) as number, + provider: String(record.provider ?? ""), + source: String(record.source ?? ""), + headline: String(record.headline ?? ""), + summary: String(record.summary ?? ""), + content_html: String(record.content_html ?? ""), + url: String(record.url ?? ""), + published_ts: coerceNumber(record.published_ts) as number, + updated_ts: coerceNumber(record.updated_ts) as number, + provider_symbols_json: String(record.provider_symbols_json ?? "[]"), + resolved_symbols_json: String(record.resolved_symbols_json ?? "[]"), + symbol_resolution: String(record.symbol_resolution ?? "none") as NewsRecord["symbol_resolution"] + }; +}; + export const fetchRecentOptionPrints = async ( client: ClickHouseClient, limit: number, @@ -1207,6 +1264,50 @@ export const fetchRecentAlerts = async ( return AlertEventSchema.array().parse(alerts); }; +const latestNewsSelect = ` +SELECT + source_ts, + ingest_ts, + seq, + trace_id, + story_id, + provider, + source, + headline, + summary, + content_html, + url, + published_ts, + updated_ts, + provider_symbols_json, + resolved_symbols_json, + symbol_resolution +FROM ( + SELECT + *, + row_number() OVER (PARTITION BY provider, story_id ORDER BY updated_ts DESC, ingest_ts DESC, seq DESC) AS revision_rank + FROM ${NEWS_TABLE} +) +WHERE revision_rank = 1 +`; + +export const fetchRecentNews = async ( + client: ClickHouseClient, + limit: number +): Promise => { + const safeLimit = clampLimit(limit); + const result = await client.query({ + query: `${latestNewsSelect} ORDER BY published_ts DESC, story_id DESC LIMIT ${safeLimit}`, + format: "JSONEachRow" + }); + + const rows = await result.json(); + const records = rows + .map(normalizeNewsRow) + .filter((record): record is NewsRecord => record !== null); + return NewsStorySchema.array().parse(records.map(fromNewsRecord)); +}; + const normalizeAlertEvidenceRefs = (refs: string[]): string[] => { return Array.from(new Set(refs.map((ref) => ref.trim()).filter(Boolean))); }; @@ -1600,6 +1701,27 @@ export const fetchAlertsAfter = async ( return AlertEventSchema.array().parse(alerts); }; +export const fetchNewsAfter = async ( + client: ClickHouseClient, + afterTs: number, + afterSeq: number, + limit: number +): Promise => { + const safeLimit = clampLimit(limit); + const safeAfterTs = clampCursor(afterTs); + const safeAfterSeq = clampCursor(afterSeq); + const result = await client.query({ + query: `${latestNewsSelect} AND (published_ts, seq) > (${safeAfterTs}, ${safeAfterSeq}) ORDER BY published_ts ASC, seq ASC LIMIT ${safeLimit}`, + format: "JSONEachRow" + }); + + const rows = await result.json(); + const records = rows + .map(normalizeNewsRow) + .filter((record): record is NewsRecord => record !== null); + return NewsStorySchema.array().parse(records.map(fromNewsRecord)); +}; + export const fetchOptionPrintsBefore = async ( client: ClickHouseClient, beforeTs: number, @@ -1778,6 +1900,25 @@ export const fetchAlertsBefore = async ( return AlertEventSchema.array().parse(records.map(fromAlertRecord)); }; +export const fetchNewsBefore = async ( + client: ClickHouseClient, + beforeTs: number, + beforeSeq: number, + limit: number +): Promise => { + const safeLimit = clampLimit(limit); + const result = await client.query({ + query: `${latestNewsSelect} AND ${buildBeforeTupleCondition("published_ts", "seq", beforeTs, beforeSeq)} ORDER BY published_ts DESC, seq DESC LIMIT ${safeLimit}`, + format: "JSONEachRow" + }); + + const rows = await result.json(); + const records = rows + .map(normalizeNewsRow) + .filter((record): record is NewsRecord => record !== null); + return NewsStorySchema.array().parse(records.map(fromNewsRecord)); +}; + export const fetchInferredDarkBefore = async ( client: ClickHouseClient, beforeTs: number, diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index 4fefabc..810d67c 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -10,3 +10,4 @@ export * from "./equity-print-joins"; export * from "./inferred-dark"; export * from "./option-prints"; export * from "./option-nbbo"; +export * from "./news"; diff --git a/packages/storage/src/news.ts b/packages/storage/src/news.ts new file mode 100644 index 0000000..cf92f40 --- /dev/null +++ b/packages/storage/src/news.ts @@ -0,0 +1,102 @@ +import type { NewsStory, NewsSymbolResolution } from "@islandflow/types"; + +export const NEWS_TABLE = "news"; + +export type NewsRecord = { + source_ts: number; + ingest_ts: number; + seq: number; + trace_id: string; + story_id: number; + provider: string; + source: string; + headline: string; + summary: string; + content_html: string; + url: string; + published_ts: number; + updated_ts: number; + provider_symbols_json: string; + resolved_symbols_json: string; + symbol_resolution: NewsSymbolResolution; +}; + +export const newsTableDDL = (): string => { + return ` +CREATE TABLE IF NOT EXISTS ${NEWS_TABLE} ( + source_ts UInt64, + ingest_ts UInt64, + seq UInt64, + trace_id String, + story_id UInt64, + provider String, + source String, + headline String, + summary String, + content_html String, + url String, + published_ts UInt64, + updated_ts UInt64, + provider_symbols_json String, + resolved_symbols_json String, + symbol_resolution String +) +ENGINE = ReplacingMergeTree(updated_ts) +ORDER BY (provider, story_id, updated_ts, seq) +`; +}; + +const safeStringArray = (value: string): string[] => { + try { + const parsed = JSON.parse(value); + if (Array.isArray(parsed)) { + return parsed.map((entry) => String(entry)); + } + } catch { + // ignore + } + + return []; +}; + +export const toNewsRecord = (story: NewsStory): NewsRecord => { + return { + source_ts: story.source_ts, + ingest_ts: story.ingest_ts, + seq: story.seq, + trace_id: story.trace_id, + story_id: story.story_id, + provider: story.provider, + source: story.source, + headline: story.headline, + summary: story.summary, + content_html: story.content_html, + url: story.url, + published_ts: story.published_ts, + updated_ts: story.updated_ts, + provider_symbols_json: JSON.stringify(story.provider_symbols), + resolved_symbols_json: JSON.stringify(story.resolved_symbols), + symbol_resolution: story.symbol_resolution + }; +}; + +export const fromNewsRecord = (record: NewsRecord): NewsStory => { + return { + source_ts: record.source_ts, + ingest_ts: record.ingest_ts, + seq: record.seq, + trace_id: record.trace_id, + story_id: record.story_id, + provider: record.provider, + source: record.source, + headline: record.headline, + summary: record.summary, + content_html: record.content_html, + url: record.url, + published_ts: record.published_ts, + updated_ts: record.updated_ts, + provider_symbols: safeStringArray(record.provider_symbols_json), + resolved_symbols: safeStringArray(record.resolved_symbols_json), + symbol_resolution: record.symbol_resolution + }; +}; diff --git a/packages/storage/tests/news.test.ts b/packages/storage/tests/news.test.ts new file mode 100644 index 0000000..c5b71c8 --- /dev/null +++ b/packages/storage/tests/news.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "bun:test"; +import type { ClickHouseClient } from "../src/clickhouse"; +import { + NEWS_TABLE, + fromNewsRecord, + newsTableDDL, + toNewsRecord +} from "../src/news"; +import { + fetchNewsAfter, + fetchNewsBefore, + fetchRecentNews +} from "../src/clickhouse"; + +const makeClient = (resolver: (query: string) => unknown[]): ClickHouseClient => + ({ + exec: async () => {}, + insert: async () => {}, + ping: async () => ({ success: true }), + close: async () => {}, + query: async ({ query }: { query: string }) => ({ + async json() { + return resolver(query) as T; + } + }) + }) as ClickHouseClient; + +const story = { + source_ts: 100, + ingest_ts: 101, + seq: 3, + trace_id: "alpaca:77", + story_id: 77, + provider: "alpaca", + source: "Benzinga", + headline: "TSLA rises", + summary: "Summary", + content_html: "

TSLA rises

", + url: "https://example.com/story", + published_ts: 100, + updated_ts: 120, + provider_symbols: ["TSLA"], + resolved_symbols: ["TSLA", "AAPL"], + symbol_resolution: "mixed" as const +}; + +describe("news storage helpers", () => { + it("includes the correct table name in the DDL", () => { + const ddl = newsTableDDL(); + expect(ddl).toContain(NEWS_TABLE); + expect(ddl).toContain("ReplacingMergeTree"); + }); + + it("round-trips news records", () => { + const record = toNewsRecord(story); + const restored = fromNewsRecord(record); + expect(restored).toEqual(story); + }); + + it("uses latest-revision selection for recent and cursor queries", async () => { + const queries: string[] = []; + const client = makeClient((query) => { + queries.push(query); + return [toNewsRecord(story)]; + }); + + const recent = await fetchRecentNews(client, 10); + const before = await fetchNewsBefore(client, 200, 10, 10); + const after = await fetchNewsAfter(client, 50, 1, 10); + + expect(recent[0]?.trace_id).toBe("alpaca:77"); + expect(before[0]?.story_id).toBe(77); + expect(after[0]?.updated_ts).toBe(120); + expect(queries[0]).toContain("row_number() OVER"); + expect(queries[1]).toContain("published_ts"); + expect(queries[2]).toContain("(published_ts, seq) > (50, 1)"); + }); +}); diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index c15dc7b..0556bd8 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -262,3 +262,26 @@ export const InferredDarkEventSchema = EventMetaSchema.merge( ); export type InferredDarkEvent = z.infer; + +export const NewsSymbolResolutionSchema = z.enum(["provider", "derived", "mixed", "none"]); + +export type NewsSymbolResolution = z.infer; + +export const NewsStorySchema = EventMetaSchema.merge( + z.object({ + story_id: z.number().int().nonnegative(), + provider: z.string().min(1), + source: z.string().min(1), + headline: z.string().min(1), + summary: z.string(), + content_html: z.string(), + url: z.string().url().or(z.literal("")), + published_ts: z.number().int().nonnegative(), + updated_ts: z.number().int().nonnegative(), + provider_symbols: z.array(z.string().min(1)), + resolved_symbols: z.array(z.string().min(1)), + symbol_resolution: NewsSymbolResolutionSchema + }) +); + +export type NewsStory = z.infer; diff --git a/packages/types/src/live.ts b/packages/types/src/live.ts index 0787c84..10ac486 100644 --- a/packages/types/src/live.ts +++ b/packages/types/src/live.ts @@ -8,6 +8,7 @@ import { EquityQuoteSchema, FlowPacketSchema, InferredDarkEventSchema, + NewsStorySchema, OptionNBBOSchema, OptionPrintSchema, SmartMoneyEventSchema @@ -34,7 +35,8 @@ export const LiveGenericChannelSchema = z.enum([ "smart-money", "classifier-hits", "alerts", - "inferred-dark" + "inferred-dark", + "news" ]); export const LiveChannelSchema = z.enum([ @@ -48,6 +50,7 @@ export const LiveChannelSchema = z.enum([ "classifier-hits", "alerts", "inferred-dark", + "news", "equity-candles", "equity-overlay" ]); @@ -91,7 +94,7 @@ export const LiveSubscriptionSchema = z.discriminatedUnion("channel", [ snapshot_limit: z.number().int().positive().optional() }), z.object({ - channel: z.enum(["nbbo", "equity-quotes", "equity-joins", "classifier-hits", "alerts", "inferred-dark"]), + channel: z.enum(["nbbo", "equity-quotes", "equity-joins", "classifier-hits", "alerts", "inferred-dark", "news"]), snapshot_limit: z.number().int().positive().optional() }), z.object({ @@ -123,6 +126,7 @@ const livePayloadSchemas = { "classifier-hits": ClassifierHitEventSchema, alerts: AlertEventSchema, "inferred-dark": InferredDarkEventSchema, + news: NewsStorySchema, "equity-candles": EquityCandleSchema, "equity-overlay": EquityPrintSchema } as const; diff --git a/packages/types/tests/live.test.ts b/packages/types/tests/live.test.ts index 075eab1..ef254b4 100644 --- a/packages/types/tests/live.test.ts +++ b/packages/types/tests/live.test.ts @@ -9,6 +9,7 @@ import { describe("live protocol types", () => { it("builds stable keys for generic and parameterized subscriptions", () => { expect(getSubscriptionKey({ channel: "flow" })).toBe("flow|{}"); + expect(getSubscriptionKey({ channel: "news" })).toBe("news"); expect( getSubscriptionKey({ channel: "options", @@ -53,12 +54,13 @@ describe("live protocol types", () => { op: "subscribe", subscriptions: [ { channel: "flow", filters: { nbboSides: ["AA", "A"], minNotional: 50000 } }, + { channel: "news", snapshot_limit: 100 }, { channel: "equity-candles", underlying_id: "SPY", interval_ms: 60000 } ] }); expect(parsed.op).toBe("subscribe"); - expect(parsed.subscriptions).toHaveLength(2); + expect(parsed.subscriptions).toHaveLength(3); }); it("validates snapshot and event server messages", () => { @@ -74,18 +76,24 @@ describe("live protocol types", () => { }); const event = LiveServerMessageSchema.parse({ op: "event", - subscription: { channel: "equity-overlay", underlying_id: "SPY" }, + subscription: { channel: "news" }, item: { source_ts: 100, ingest_ts: 101, seq: 1, - trace_id: "eq-1", - ts: 100, - underlying_id: "SPY", - price: 500, - size: 10, - exchange: "X", - offExchangeFlag: true + trace_id: "alpaca:1", + story_id: 1, + provider: "alpaca", + source: "Benzinga", + headline: "TSLA rises", + summary: "", + content_html: "

TSLA rises

", + url: "https://example.com/story", + published_ts: 100, + updated_ts: 100, + provider_symbols: ["TSLA"], + resolved_symbols: ["TSLA"], + symbol_resolution: "provider" }, watermark: cursor }); diff --git a/scripts/deploy.ts b/scripts/deploy.ts index 68d260a..5b94d95 100644 --- a/scripts/deploy.ts +++ b/scripts/deploy.ts @@ -48,7 +48,8 @@ const NATIVE_UNITS = { ingestOptions: process.env.DEPLOY_NATIVE_INGEST_OPTIONS_UNIT?.trim() || "islandflow-ingest-options", ingestEquities: - process.env.DEPLOY_NATIVE_INGEST_EQUITIES_UNIT?.trim() || "islandflow-ingest-equities" + process.env.DEPLOY_NATIVE_INGEST_EQUITIES_UNIT?.trim() || "islandflow-ingest-equities", + ingestNews: process.env.DEPLOY_NATIVE_INGEST_NEWS_UNIT?.trim() || "islandflow-ingest-news" } as const; const DOCKER_CORE_SERVICES = [ "api", @@ -56,14 +57,16 @@ const DOCKER_CORE_SERVICES = [ "compute", "candles", "ingest-options", - "ingest-equities" + "ingest-equities", + "ingest-news" ] as const; const DOCKER_BACKEND_SERVICES = [ "api", "compute", "candles", "ingest-options", - "ingest-equities" + "ingest-equities", + "ingest-news" ] as const; const scriptPath = fileURLToPath(import.meta.url); @@ -106,7 +109,8 @@ Environment: DEPLOY_NATIVE_COMPUTE_UNIT Override native compute systemd unit name. DEPLOY_NATIVE_CANDLES_UNIT Override native candles systemd unit name. DEPLOY_NATIVE_INGEST_OPTIONS_UNIT Override native ingest-options systemd unit name. - DEPLOY_NATIVE_INGEST_EQUITIES_UNIT Override native ingest-equities systemd unit name.`); + DEPLOY_NATIVE_INGEST_EQUITIES_UNIT Override native ingest-equities systemd unit name. + DEPLOY_NATIVE_INGEST_NEWS_UNIT Override native ingest-news systemd unit name.`); process.exit(exitCode); } @@ -465,7 +469,8 @@ function nativeUnitsForScope(scope: DeployScope): string[] { NATIVE_UNITS.compute, NATIVE_UNITS.candles, NATIVE_UNITS.ingestOptions, - NATIVE_UNITS.ingestEquities + NATIVE_UNITS.ingestEquities, + NATIVE_UNITS.ingestNews ]; default: return [ @@ -474,7 +479,8 @@ function nativeUnitsForScope(scope: DeployScope): string[] { NATIVE_UNITS.compute, NATIVE_UNITS.candles, NATIVE_UNITS.ingestOptions, - NATIVE_UNITS.ingestEquities + NATIVE_UNITS.ingestEquities, + NATIVE_UNITS.ingestNews ]; } } diff --git a/scripts/dev-services.ts b/scripts/dev-services.ts index 09cd381..2bcb641 100644 --- a/scripts/dev-services.ts +++ b/scripts/dev-services.ts @@ -222,6 +222,7 @@ process.on("SIGHUP", () => handleSignal("SIGHUP")); const tasks: ChildSpec[] = [ { name: "ingest-options", cmd: ["bun", "run", "dev"], cwd: "services/ingest-options" }, { name: "ingest-equities", cmd: ["bun", "run", "dev"], cwd: "services/ingest-equities" }, + { name: "ingest-news", cmd: ["bun", "run", "dev"], cwd: "services/ingest-news" }, { name: "compute", cmd: ["bun", "run", "dev"], cwd: "services/compute" }, { name: "candles", cmd: ["bun", "run", "dev"], cwd: "services/candles" }, { name: "refdata", cmd: ["bun", "run", "dev"], cwd: "services/refdata" }, diff --git a/scripts/dev.ts b/scripts/dev.ts index 64406d6..1d031a7 100644 --- a/scripts/dev.ts +++ b/scripts/dev.ts @@ -325,6 +325,7 @@ const serviceTasks: ChildSpec[] = [ { name: "web", cmd: ["bun", "run", "dev"], cwd: "apps/web" }, { name: "ingest-options", cmd: ["bun", "run", "dev"], cwd: "services/ingest-options" }, { name: "ingest-equities", cmd: ["bun", "run", "dev"], cwd: "services/ingest-equities" }, + { name: "ingest-news", cmd: ["bun", "run", "dev"], cwd: "services/ingest-news" }, { name: "compute", cmd: ["bun", "run", "dev"], cwd: "services/compute" }, { name: "candles", cmd: ["bun", "run", "dev"], cwd: "services/candles" }, { name: "refdata", cmd: ["bun", "run", "dev"], cwd: "services/refdata" }, diff --git a/services/api/src/index.ts b/services/api/src/index.ts index 433222a..c0a9c79 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -9,6 +9,7 @@ import { SUBJECT_EQUITY_QUOTES, SUBJECT_INFERRED_DARK, SUBJECT_FLOW_PACKETS, + SUBJECT_NEWS, SUBJECT_SMART_MONEY_EVENTS, SUBJECT_OPTION_NBBO, SUBJECT_OPTION_SIGNAL_PRINTS, @@ -20,6 +21,7 @@ import { STREAM_EQUITY_QUOTES, STREAM_INFERRED_DARK, STREAM_FLOW_PACKETS, + STREAM_NEWS, STREAM_SMART_MONEY_EVENTS, STREAM_OPTION_NBBO, STREAM_OPTION_SIGNAL_PRINTS, @@ -35,6 +37,7 @@ import { import { createClickHouseClient, ensureAlertsTable, + ensureNewsTable, ensureClassifierHitsTable, ensureEquityCandlesTable, ensureEquityPrintJoinsTable, @@ -48,6 +51,8 @@ import { fetchAlertsAfter, fetchAlertsBefore, fetchAlertContextByTraceId, + fetchNewsAfter, + fetchNewsBefore, fetchClassifierHitsAfter, fetchClassifierHitsBefore, fetchSmartMoneyEventsAfter, @@ -58,6 +63,7 @@ import { fetchFlowPacketsByMemberTraceIds, fetchFlowPacketsBefore, fetchRecentAlerts, + fetchRecentNews, fetchRecentClassifierHits, fetchRecentSmartMoneyEvents, fetchRecentEquityPrintJoins, @@ -99,6 +105,7 @@ import { EquityQuoteSchema, FeedSnapshot, InferredDarkEventSchema, + NewsStorySchema, LiveClientMessageSchema, LiveServerMessage, LiveSubscription, @@ -676,7 +683,8 @@ const run = async () => { STREAM_FLOW_PACKETS, STREAM_SMART_MONEY_EVENTS, STREAM_CLASSIFIER_HITS, - STREAM_ALERTS + STREAM_ALERTS, + STREAM_NEWS ], { logger } ); @@ -719,6 +727,7 @@ const run = async () => { await ensureSmartMoneyEventsTable(clickhouse); await ensureClassifierHitsTable(clickhouse); await ensureAlertsTable(clickhouse); + await ensureNewsTable(clickhouse); }); let redis: ReturnType | null = null; @@ -843,6 +852,11 @@ const run = async () => { subject: SUBJECT_ALERTS, stream: STREAM_ALERTS, durableName: "api-alerts" + }, + { + subject: SUBJECT_NEWS, + stream: STREAM_NEWS, + durableName: "api-news" } ] as const; @@ -991,10 +1005,16 @@ const run = async () => { consumerBindings[10].durableName ); + const newsSubscription = await subscribeWithReset( + consumerBindings[11].subject, + consumerBindings[11].stream, + consumerBindings[11].durableName + ); + const fanoutLive = async ( subscription: LiveSubscription, item: unknown, - ingestChannel: "options" | "nbbo" | "equities" | "equity-quotes" | "equity-candles" | "equity-overlay" | "equity-joins" | "flow" | "classifier-hits" | "alerts" | "inferred-dark" + ingestChannel: "options" | "nbbo" | "equities" | "equity-quotes" | "equity-candles" | "equity-overlay" | "equity-joins" | "flow" | "classifier-hits" | "alerts" | "inferred-dark" | "news" ) => { const watermark = await liveState.ingest(ingestChannel, item); @@ -1252,6 +1272,21 @@ const run = async () => { } }; + const pumpNews = async () => { + for await (const msg of newsSubscription.messages) { + try { + const payload = NewsStorySchema.parse(newsSubscription.decode(msg)); + await fanoutLive({ channel: "news" }, payload, "news"); + msg.ack(); + } catch (error) { + logger.error("failed to process news story", { + error: error instanceof Error ? error.message : String(error) + }); + msg.term(); + } + } + }; + void pumpOptions(); void pumpOptionNbbo(); void pumpEquities(); @@ -1263,6 +1298,7 @@ const run = async () => { void pumpSmartMoney(); void pumpClassifierHits(); void pumpAlerts(); + void pumpNews(); const buildSyntheticStatusBody = () => { const derived = @@ -1490,6 +1526,12 @@ const run = async () => { return jsonResponse({ data }); } + if (req.method === "GET" && url.pathname === "/news") { + const limit = parseLimit(url.searchParams.get("limit") ?? "100"); + const data = await fetchRecentNews(clickhouse, limit); + return jsonResponse({ data }); + } + if (req.method === "GET" && isAlertContextPath(url.pathname)) { try { const traceId = parseAlertContextTraceIdPath(url.pathname); @@ -1607,6 +1649,14 @@ const run = async () => { ); } + if (req.method === "GET" && url.pathname === "/history/news") { + const { beforeTs, beforeSeq, limit } = parseBeforeParams(url); + const data = await fetchNewsBefore(clickhouse, beforeTs, beforeSeq, limit); + return jsonResponse( + buildHistoryResponse(data, (item) => ({ ts: item.published_ts, seq: item.seq })) + ); + } + if (req.method === "GET" && /^\/flow\/packets\/[^/]+$/.test(url.pathname)) { const id = decodeURIComponent(url.pathname.slice("/flow/packets/".length)); const data = await fetchFlowPacketById(clickhouse, id); diff --git a/services/api/src/live.ts b/services/api/src/live.ts index 024935e..c8d2886 100644 --- a/services/api/src/live.ts +++ b/services/api/src/live.ts @@ -8,6 +8,7 @@ import { fetchRecentEquityQuotes, fetchRecentFlowPackets, fetchRecentInferredDark, + fetchRecentNews, fetchRecentOptionNBBO, fetchRecentSmartMoneyEvents, type ClickHouseClient @@ -25,6 +26,7 @@ import { FeedSnapshot, FlowPacketSchema, InferredDarkEventSchema, + NewsStorySchema, LiveChannelHealth, LiveGenericChannel, LiveHotChannel, @@ -40,6 +42,7 @@ import { type EquityCandle, type EquityPrint, type LiveChannel, + type NewsStory, type OptionPrint } from "@islandflow/types"; import { createMetrics } from "@islandflow/observability"; @@ -63,7 +66,8 @@ const GENERIC_LIMIT_ENV_KEYS: Record = { "smart-money": "LIVE_LIMIT_SMART_MONEY", "classifier-hits": "LIVE_LIMIT_CLASSIFIER_HITS", alerts: "LIVE_LIMIT_ALERTS", - "inferred-dark": "LIVE_LIMIT_INFERRED_DARK" + "inferred-dark": "LIVE_LIMIT_INFERRED_DARK", + news: "LIVE_LIMIT_NEWS" }; const CHART_LIMITS = { @@ -81,7 +85,8 @@ const DEFAULT_LIVE_LIMITS: GenericLiveLimits = { "smart-money": 300, "classifier-hits": 300, alerts: 300, - "inferred-dark": 300 + "inferred-dark": 300, + news: 100 }; const DEFAULT_SCOPED_CACHE_MAX_KEYS = 32; @@ -196,16 +201,28 @@ export const resolveGenericLiveLimits = (env: NodeJS.ProcessEnv = process.env): env, "inferred-dark", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS["inferred-dark"] - ) + ), + news: parseGenericLimit(env, "news", env.LIVE_LIMIT_DEFAULT ? liveLimitDefault : DEFAULT_LIVE_LIMITS.news) }; }; -const parsePositiveInt = (value: string | undefined, fallback: number): number => { - const parsed = Number(value); - if (!Number.isFinite(parsed)) { - return fallback; +const extractFreshnessTs = (channel: LiveGenericChannel, item: any): number | null => { + switch (channel) { + case "options": + case "nbbo": + case "equities": + case "equity-quotes": + return typeof item.ts === "number" ? item.ts : null; + case "flow": + case "classifier-hits": + case "alerts": + case "inferred-dark": + return typeof item.source_ts === "number" ? item.source_ts : null; + case "news": + return typeof item.published_ts === "number" ? item.published_ts : null; + default: + return null; } - return Math.max(1, Math.floor(parsed)); }; export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): LiveStateConfig => ({ @@ -217,6 +234,13 @@ export const resolveLiveStateConfig = (env: NodeJS.ProcessEnv = process.env): Li ), redisFlushMaxItems: parsePositiveInt(env.LIVE_REDIS_FLUSH_MAX_ITEMS, DEFAULT_REDIS_FLUSH_MAX_ITEMS) }); +const parsePositiveInt = (value: string | undefined, fallback: number): number => { + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + return fallback; + } + return Math.max(1, Math.floor(parsed)); +}; type RedisLike = Pick< RedisClientType, @@ -318,6 +342,14 @@ const getGenericConfig = (limits: GenericLiveLimits): { parse: (value) => InferredDarkEventSchema.parse(value), cursor: (item) => ({ ts: item.source_ts, seq: item.seq }), fetchRecent: fetchRecentInferredDark + }, + news: { + redisKey: "live:news", + cursorField: "news", + limit: limits.news, + parse: (value) => NewsStorySchema.parse(value), + cursor: (item) => ({ ts: item.published_ts, seq: item.seq }), + fetchRecent: fetchRecentNews } }); @@ -371,23 +403,6 @@ const normalizeGenericItems = ( return sortGenericItems(items, config.cursor).slice(0, config.limit); }; -const extractFreshnessTs = (channel: LiveGenericChannel, item: any): number | null => { - switch (channel) { - case "options": - case "nbbo": - case "equities": - case "equity-quotes": - return typeof item.ts === "number" ? item.ts : null; - case "flow": - case "classifier-hits": - case "alerts": - case "inferred-dark": - return typeof item.source_ts === "number" ? item.source_ts : null; - default: - return null; - } -}; - const isWithinLiveFeedLookback = ( channel: LiveGenericChannel, item: unknown, diff --git a/services/ingest-news/package.json b/services/ingest-news/package.json new file mode 100644 index 0000000..050f40b --- /dev/null +++ b/services/ingest-news/package.json @@ -0,0 +1,16 @@ +{ + "name": "@islandflow/ingest-news", + "private": true, + "type": "module", + "scripts": { + "dev": "bun run src/index.ts" + }, + "dependencies": { + "@islandflow/bus": "workspace:*", + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + "@islandflow/types": "workspace:*", + "ws": "^8.18.3", + "zod": "^3.23.8" + } +} diff --git a/services/ingest-news/src/index.ts b/services/ingest-news/src/index.ts new file mode 100644 index 0000000..3f91ee2 --- /dev/null +++ b/services/ingest-news/src/index.ts @@ -0,0 +1,216 @@ +import { readEnv } from "@islandflow/config"; +import { createLogger } from "@islandflow/observability"; +import { + SUBJECT_NEWS, + STREAM_NEWS, + connectJetStreamWithRetry, + ensureKnownStreams, + publishJson +} from "@islandflow/bus"; +import { NewsStorySchema, type NewsStory } from "@islandflow/types"; +import WebSocket from "ws"; +import { z } from "zod"; +import { resolveNewsSymbols } from "./symbols"; + +const service = "ingest-news"; +const logger = createLogger({ service }); + +const envSchema = z.object({ + NATS_URL: z.string().default("nats://127.0.0.1:4222"), + ALPACA_API_KEY: z.string().default(""), + ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"), + ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"), + ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(200).default(100), + ALPACA_NEWS_WEBSOCKET_PATH: z.string().default("/v1beta1/news") +}); + +const env = readEnv(envSchema); + +type AlpacaNewsItem = { + id?: number; + headline?: string; + summary?: string; + content?: string; + author?: string; + created_at?: string; + updated_at?: string; + url?: string; + symbols?: string[]; + source?: string; +}; + +type AlpacaNewsResponse = { + news?: AlpacaNewsItem[]; +}; + +const buildHeaders = (): Record => ({ + Authorization: `Bearer ${env.ALPACA_API_KEY}` +}); + +const parseTimestamp = (value: string | undefined): number => { + const parsed = value ? Date.parse(value) : Number.NaN; + return Number.isFinite(parsed) ? parsed : Date.now(); +}; + +const toStory = (item: AlpacaNewsItem, seq: number): NewsStory | null => { + const storyId = Number(item.id); + if (!Number.isFinite(storyId) || storyId < 0) { + return null; + } + + const provider = "alpaca"; + const contentHtml = item.content ?? ""; + const symbols = resolveNewsSymbols(item.symbols ?? [], contentHtml); + const publishedTs = parseTimestamp(item.created_at); + const updatedTs = parseTimestamp(item.updated_at ?? item.created_at); + + return NewsStorySchema.parse({ + source_ts: publishedTs, + ingest_ts: Date.now(), + seq, + trace_id: `${provider}:${storyId}`, + story_id: storyId, + provider, + source: item.source?.trim() || item.author?.trim() || "Alpaca News", + headline: item.headline?.trim() || `Story ${storyId}`, + summary: item.summary?.trim() || "", + content_html: contentHtml, + url: item.url?.trim() || "", + published_ts: publishedTs, + updated_ts: updatedTs, + provider_symbols: symbols.provider_symbols, + resolved_symbols: symbols.resolved_symbols, + symbol_resolution: symbols.symbol_resolution + }); +}; + +const fetchBackfill = async (): Promise => { + const url = new URL("/v1beta1/news", env.ALPACA_REST_URL); + url.searchParams.set("sort", "desc"); + url.searchParams.set("limit", env.ALPACA_NEWS_BACKFILL_LIMIT.toString()); + + const response = await fetch(url.toString(), { + headers: buildHeaders() + }); + + if (!response.ok) { + throw new Error(`alpaca news backfill failed (${response.status})`); + } + + const payload = (await response.json()) as AlpacaNewsResponse; + return Array.isArray(payload.news) ? payload.news : []; +}; + +const decodePayload = (data: WebSocket.RawData): unknown => { + if (typeof data === "string") { + return JSON.parse(data) as unknown; + } + if (data instanceof ArrayBuffer) { + return JSON.parse(new TextDecoder().decode(new Uint8Array(data))) as unknown; + } + if (ArrayBuffer.isView(data)) { + return JSON.parse(new TextDecoder().decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))) as unknown; + } + return JSON.parse(new TextDecoder().decode(new Uint8Array(data as ArrayBuffer))) as unknown; +}; + +const run = async () => { + if (!env.ALPACA_API_KEY) { + throw new Error("ALPACA_API_KEY is required for ingest-news."); + } + + const { nc, js, jsm } = await connectJetStreamWithRetry( + { + servers: env.NATS_URL, + name: service + }, + { attempts: 120, delayMs: 500 } + ); + + await ensureKnownStreams(jsm, [STREAM_NEWS], { logger }); + + let seq = 0; + const publishStory = async (item: AlpacaNewsItem) => { + seq += 1; + const story = toStory(item, seq); + if (!story) { + return; + } + await publishJson(js, SUBJECT_NEWS, story); + }; + + const backfill = await fetchBackfill(); + for (const item of backfill.reverse()) { + await publishStory(item); + } + + const wsUrl = new URL(env.ALPACA_NEWS_WEBSOCKET_PATH, env.ALPACA_WS_BASE_URL).toString(); + const ws = new WebSocket(wsUrl, { + headers: buildHeaders() + }); + + ws.on("open", () => { + ws.send( + JSON.stringify({ + action: "auth", + key: env.ALPACA_API_KEY, + secret: "" + }) + ); + }); + + ws.on("message", (raw) => { + let payload: unknown; + try { + payload = decodePayload(raw); + } catch (error) { + logger.warn("failed to decode alpaca news message", { + error: error instanceof Error ? error.message : String(error) + }); + return; + } + + if (!Array.isArray(payload)) { + return; + } + + for (const entry of payload) { + if (!entry || typeof entry !== "object") { + continue; + } + const message = entry as Record; + if (message.T === "success") { + const msg = typeof message.msg === "string" ? message.msg : ""; + if (msg === "authenticated") { + ws.send(JSON.stringify({ action: "subscribe", news: ["*"] })); + } + continue; + } + if (message.T === "subscription" || message.T === "error") { + continue; + } + void publishStory(message as AlpacaNewsItem).catch((error) => { + logger.error("failed to publish alpaca news story", { + error: error instanceof Error ? error.message : String(error) + }); + }); + } + }); + + const shutdown = async (signal: string) => { + logger.info("shutting down", { signal }); + ws.close(); + await nc.drain(); + process.exit(0); + }; + + process.on("SIGINT", () => void shutdown("SIGINT")); + process.on("SIGTERM", () => void shutdown("SIGTERM")); +}; + +void run().catch((error) => { + logger.error("service crashed", { + error: error instanceof Error ? error.message : String(error) + }); + process.exit(1); +}); diff --git a/services/ingest-news/src/symbols.ts b/services/ingest-news/src/symbols.ts new file mode 100644 index 0000000..e1537fd --- /dev/null +++ b/services/ingest-news/src/symbols.ts @@ -0,0 +1,70 @@ +import type { NewsSymbolResolution } from "@islandflow/types"; + +const TICKER_ANCHOR_RE = />\s*([A-Z]{1,5})\s*<\/a>/g; +const EXCHANGE_TICKER_RE = /\b(?:NASDAQ|NYSE|NYSEAMERICAN|AMEX|OTC|CBOE):([A-Z]{1,5})\b/g; +const DOLLAR_TICKER_RE = /\$([A-Z]{1,5})\b/g; + +const normalizeSymbols = (symbols: string[]): string[] => { + const seen = new Set(); + const normalized: string[] = []; + + for (const entry of symbols) { + const symbol = entry.trim().toUpperCase(); + if (!symbol || !/^[A-Z]{1,5}$/.test(symbol) || seen.has(symbol)) { + continue; + } + seen.add(symbol); + normalized.push(symbol); + } + + return normalized; +}; + +const collectMatches = (value: string, regex: RegExp): string[] => { + regex.lastIndex = 0; + const matches: string[] = []; + let match: RegExpExecArray | null = null; + while ((match = regex.exec(value)) !== null) { + matches.push(match[1] ?? ""); + } + return matches; +}; + +export const resolveNewsSymbols = ( + providerSymbols: string[], + contentHtml: string +): { + provider_symbols: string[]; + resolved_symbols: string[]; + symbol_resolution: NewsSymbolResolution; +} => { + const normalizedProvider = normalizeSymbols(providerSymbols); + const derived = normalizeSymbols([ + ...collectMatches(contentHtml, TICKER_ANCHOR_RE), + ...collectMatches(contentHtml, EXCHANGE_TICKER_RE), + ...collectMatches(contentHtml, DOLLAR_TICKER_RE) + ]); + + if (normalizedProvider.length > 0) { + const merged = normalizeSymbols([...normalizedProvider, ...derived]); + return { + provider_symbols: normalizedProvider, + resolved_symbols: merged, + symbol_resolution: derived.length > 0 ? "mixed" : "provider" + }; + } + + if (derived.length > 0) { + return { + provider_symbols: [], + resolved_symbols: derived, + symbol_resolution: "derived" + }; + } + + return { + provider_symbols: [], + resolved_symbols: [], + symbol_resolution: "none" + }; +}; diff --git a/services/ingest-news/tests/symbols.test.ts b/services/ingest-news/tests/symbols.test.ts new file mode 100644 index 0000000..4f3994e --- /dev/null +++ b/services/ingest-news/tests/symbols.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "bun:test"; +import { resolveNewsSymbols } from "../src/symbols"; + +describe("resolveNewsSymbols", () => { + it("prefers provider symbols when present", () => { + const result = resolveNewsSymbols(["tsla", "aapl"], "

No extra tickers here.

"); + expect(result.provider_symbols).toEqual(["TSLA", "AAPL"]); + expect(result.resolved_symbols).toEqual(["TSLA", "AAPL"]); + expect(result.symbol_resolution).toBe("provider"); + }); + + it("falls back to ticker anchors", () => { + const result = resolveNewsSymbols([], 'TSLA'); + expect(result.resolved_symbols).toEqual(["TSLA"]); + expect(result.symbol_resolution).toBe("derived"); + }); + + it("falls back to exchange and dollar patterns", () => { + const result = resolveNewsSymbols([], "

NASDAQ:TSLA met with $IBM executives.

"); + expect(result.resolved_symbols).toEqual(["TSLA", "IBM"]); + expect(result.symbol_resolution).toBe("derived"); + }); + + it("dedupes and uppercases merged symbols", () => { + const result = resolveNewsSymbols(["tsla"], "

$TSLA and NASDAQ:TSLA

"); + expect(result.provider_symbols).toEqual(["TSLA"]); + expect(result.resolved_symbols).toEqual(["TSLA"]); + expect(result.symbol_resolution).toBe("mixed"); + }); +}); diff --git a/services/ingest-news/tsconfig.json b/services/ingest-news/tsconfig.json new file mode 100644 index 0000000..43ef119 --- /dev/null +++ b/services/ingest-news/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": [] + }, + "include": ["src/**/*.ts", "tests/**/*.ts"] +} From bdb9d9a95a10e2df736d412a0083004e69752eee Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 18 May 2026 19:55:27 -0400 Subject: [PATCH 04/26] Implement native public edge cutover --- .beads/issues.jsonl | 5 +- apps/web/package.json | 2 +- deployment/docker/.dockerignore | 23 + deployment/docker/.env.example | 2 + deployment/docker/Dockerfile.web | 2 +- deployment/docker/README.md | 11 +- deployment/docker/docker-compose.yml | 18 +- deployment/native/README.md | 61 +- deployment/native/bootstrap-infra.sh | 24 + deployment/native/check-native-health.sh | 7 + deployment/native/check-native-infra.sh | 24 + .../native/config/clickhouse-listen.xml | 6 + deployment/native/config/redis.conf | 10 + deployment/native/cutover.sh | 34 ++ deployment/native/full-rollback.sh | 27 + deployment/native/install-infra-units.sh | 72 +++ deployment/native/start-infra.sh | 17 + deployment/native/stop-infra.sh | 9 + deployment/native/switch-npm-edge.sh | 285 ++++++++++ .../system/islandflow-clickhouse.service | 17 + .../systemd/system/islandflow-nats.service | 18 + .../systemd/system/islandflow-redis.service | 18 + .../systemd/user/islandflow-api.service | 2 + .../user/islandflow-ingest-options.service | 1 + .../systemd/user/islandflow-web.service | 4 +- ...2026-05-18-native-public-edge-cutover.html | 521 ++++++++++++++++++ packages/bus/src/jetstream.ts | 14 +- scripts/deploy.ts | 8 +- services/api/src/index.ts | 4 +- 29 files changed, 1215 insertions(+), 31 deletions(-) create mode 100644 deployment/docker/.dockerignore create mode 100755 deployment/native/bootstrap-infra.sh create mode 100755 deployment/native/check-native-infra.sh create mode 100644 deployment/native/config/clickhouse-listen.xml create mode 100644 deployment/native/config/redis.conf create mode 100755 deployment/native/cutover.sh create mode 100755 deployment/native/full-rollback.sh create mode 100755 deployment/native/install-infra-units.sh create mode 100755 deployment/native/start-infra.sh create mode 100755 deployment/native/stop-infra.sh create mode 100755 deployment/native/switch-npm-edge.sh create mode 100644 deployment/native/systemd/system/islandflow-clickhouse.service create mode 100644 deployment/native/systemd/system/islandflow-nats.service create mode 100644 deployment/native/systemd/system/islandflow-redis.service create mode 100644 docs/turns/2026-05-18-native-public-edge-cutover.html diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 16eabf1..00b065c 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,4 +1,4 @@ -{"_type":"issue","id":"islandflow-9rc","title":"Implement native fast iterative deploy plan","description":"Implement the checked-in plan at plans/2026-05-18-native-fast-iterative-deploy-plan.md. Cover deploy-phase timing instrumentation, native deployment operational assets, deploy guardrails, validation/cutover documentation, and any required live VPS remediation that is safely actionable from this session. Track follow-up items separately if anything cannot be completed in-repo or on the live host.","status":"in_progress","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T07:15:19Z","created_by":"dirtydishes","updated_at":"2026-05-18T07:15:25Z","started_at":"2026-05-18T07:15:25Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-9rc","title":"Implement native fast iterative deploy plan","description":"Implement the checked-in plan at plans/2026-05-18-native-fast-iterative-deploy-plan.md. Cover deploy-phase timing instrumentation, native deployment operational assets, deploy guardrails, validation/cutover documentation, and any required live VPS remediation that is safely actionable from this session. Track follow-up items separately if anything cannot be completed in-repo or on the live host.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T07:15:19Z","created_by":"dirtydishes","updated_at":"2026-05-18T07:34:03Z","started_at":"2026-05-18T07:15:25Z","closed_at":"2026-05-18T07:34:03Z","close_reason":"Implemented the native fast iterative deploy plan with deploy timing summaries, worker-only native fast mode, edge-cutover guardrails, local-on-server execution support, checked-in native ops assets, live audit findings, and turn documentation. Remaining cutover work is tracked in islandflow-vvw.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8kj","title":"Configure persistent beads Dolt remote on deltaisland server","description":"Install the beads and Dolt CLIs on the server, configure a persistent Dolt sync remote backed by the server-hosted Forgejo repository, verify refs/dolt/data publication, and document Nginx Proxy Manager / firewall considerations.","status":"closed","priority":1,"issue_type":"task","assignee":"delta","created_at":"2026-05-17T10:31:31Z","created_by":"delta","updated_at":"2026-05-17T10:37:47Z","started_at":"2026-05-17T10:32:16Z","closed_at":"2026-05-17T10:37:47Z","close_reason":"Installed bd and dolt on the server, configured the Forgejo-backed Dolt remote, published refs/dolt/data, and documented the setup.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-k4f","title":"Gate deploy script on docker workspace snapshot sync","description":"Prevent frozen-lockfile build failures during deploy by adding a local preflight in scripts/deploy.ts that runs bun run check:docker-workspace and aborts with a clear sync+commit remediation message when stale.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:01:44Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:04:11Z","started_at":"2026-05-15T23:01:48Z","closed_at":"2026-05-15T23:04:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -13,7 +13,8 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-vvw","title":"Stage native public-edge cutover after worker soak","description":"Why this issue exists and what needs to be done:\\n- The native deploy path is now provisioned for worker-first iteration, with checked-in user units, rollback helpers, and edge guardrails\\n- Remaining work is to enable and soak native worker units, validate duplicate-processing behavior, then deliberately cut over the public web/api edge if warranted\\n- Final acceptance should include deciding whether Docker or native becomes the default runtime after operational evidence","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-18T07:32:35Z","created_by":"dirtydishes","updated_at":"2026-05-18T07:32:35Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-fl5","title":"Decide final public posture for api.flow.deltaisland.io after native cutover","description":"Why this issue exists and what needs to be done:\\n- Native cutover now works end-to-end through Nginx Proxy Manager and the public API hostname now resolves directly to the VPS\\n- The API hostname was left DNS-only in Cloudflare during incident resolution, while the web hostname still uses the Cloudflare proxy\\n- We need to decide whether api.flow.deltaisland.io should remain direct-to-origin or be re-proxied through Cloudflare, then validate TLS, websocket, and operational behavior for the chosen posture","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-18T23:51:21Z","created_by":"dirtydishes","updated_at":"2026-05-18T23:51:21Z","dependencies":[{"issue_id":"islandflow-fl5","depends_on_id":"islandflow-vvw","type":"discovered-from","created_at":"2026-05-18T19:52:32Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-vvw","title":"Stage native public-edge cutover after worker soak","description":"Why this issue exists and what needs to be done:\\n- The native deploy path is now provisioned for worker-first iteration, with checked-in user units, rollback helpers, and edge guardrails\\n- Remaining work is to enable and soak native worker units, validate duplicate-processing behavior, then deliberately cut over the public web/api edge if warranted\\n- Final acceptance should include deciding whether Docker or native becomes the default runtime after operational evidence","notes":"2026-05-18: native infra, native app services, NPM public-edge retargeting, Docker rollback helpers, and Cloudflare/DNS API hostname recovery were implemented and verified. Public checks now pass for flow.deltaisland.io and api.flow.deltaisland.io. Remaining follow-up: decide whether api.flow.deltaisland.io should remain DNS-only or be re-proxied through Cloudflare under islandflow-fl5.","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T07:32:35Z","created_by":"dirtydishes","updated_at":"2026-05-18T23:52:32Z","started_at":"2026-05-18T23:51:20Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-bsg","title":"Fix public /replay/options proxy regression","description":"Restore correct public routing for GET /replay/options on flow.deltaisland.io. The app currently serves HTML for that API path, which indicates edge/proxy routing drift. Update the live proxy topology or deployment assets as needed, then validate with bun run scripts/check-public-api-routes.ts.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T07:15:19Z","created_by":"dirtydishes","updated_at":"2026-05-18T07:32:51Z","started_at":"2026-05-18T07:15:24Z","closed_at":"2026-05-18T07:32:51Z","close_reason":"Audited the live VPS and reverse proxy on 2026-05-18: public /replay/options now returns JSON, bun run scripts/check-public-api-routes.ts passes, and the active Nginx Proxy Manager config includes /replay in the API route matcher. No in-repo app code change was required.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-9j5","title":"Prepare PR for deploy allowlist cleanup","description":"Why this issue exists and what needs to be done:\\n- Package current deploy allowlist cleanup into a reviewable PR with multiple commits\\n- Add required turn documentation in docs/turns\\n- Run validation and push all artifacts","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T15:44:12Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:53:55Z","started_at":"2026-05-17T15:44:22Z","closed_at":"2026-05-17T15:53:55Z","close_reason":"Packaged deploy allowlist cleanup into multi-commit PR branch with required turn documentation and push workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0sa","title":"Fix live tape auto-hold, history seam, and remove manual pause control","description":"The live tape should automatically hold when the user scrolls away from the top, resume when they return to the top or use Jump to top, and keep older prints available seamlessly beyond the hot window. Manual Pause/Resume control is now redundant and should be removed from live tape panes. This work should also fix the current regression where paused/held tapes still mutate, and align the options tape with a strict 100-row hot head backed by ClickHouse history.","notes":"Implemented live scroll-hold with no live pause button, demand-loaded ClickHouse history, a 100-row options hot head, and cache-first scoped snapshots. Validated with bun test apps/web/app/terminal.test.ts services/api/tests/live.test.ts and bun --cwd=apps/web run build.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T18:12:51Z","created_by":"dirtydishes","updated_at":"2026-05-16T18:23:43Z","started_at":"2026-05-16T18:12:54Z","closed_at":"2026-05-16T18:23:43Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/apps/web/package.json b/apps/web/package.json index 8ab6906..91611ea 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": "bun run scripts/dev.ts", "build": "next build", - "start": "next start -p 3000" + "start": "next start" }, "dependencies": { "@islandflow/types": "workspace:*", diff --git a/deployment/docker/.dockerignore b/deployment/docker/.dockerignore new file mode 100644 index 0000000..8fd5de7 --- /dev/null +++ b/deployment/docker/.dockerignore @@ -0,0 +1,23 @@ +.git +.github +.DS_Store +.bun +.tmp +node_modules +dist +coverage +logs +apps/web/.next +.env +.env.* +session-ses_*.md +token-usage-output.txt +signal-cli-*.tar.gz +*.tar +*.tar.gz +*.tgz +*.zip +__pycache__ +.pytest_cache +!.env.example +!**/.env.example diff --git a/deployment/docker/.env.example b/deployment/docker/.env.example index eee9cef..1a3eb84 100644 --- a/deployment/docker/.env.example +++ b/deployment/docker/.env.example @@ -4,8 +4,10 @@ NATS_URL=nats://nats:4222 CLICKHOUSE_URL=http://clickhouse:8123 CLICKHOUSE_DATABASE=default REDIS_URL=redis://redis:6379 +ISLANDFLOW_DATA_ROOT=/var/lib/islandflow API_PORT=4000 +API_HOST=0.0.0.0 API_BIND_IP=127.0.0.1 API_HOST_PORT=4000 WEB_BIND_IP=127.0.0.1 diff --git a/deployment/docker/Dockerfile.web b/deployment/docker/Dockerfile.web index 33723ae..efd186b 100644 --- a/deployment/docker/Dockerfile.web +++ b/deployment/docker/Dockerfile.web @@ -59,4 +59,4 @@ COPY --from=build /app/packages ./packages EXPOSE 3000 -CMD ["bun", "run", "--cwd", "apps/web", "start"] +CMD ["bun", "run", "--cwd", "apps/web", "start", "--", "-H", "0.0.0.0", "-p", "3000"] diff --git a/deployment/docker/README.md b/deployment/docker/README.md index ed80c53..9b36220 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -2,12 +2,12 @@ This directory contains the Docker runtime for Islandflow VPS deployments. -Docker remains the default and recommended server rollout path, but the repo-root `deploy` helper can now target either: +Docker remains the default rollout path before native cutover and the rollback path after cutover. The repo-root `deploy` helper can target either: - `--runtime docker` for this Docker Compose stack -- `--runtime native` for an experimental host-native Bun + systemd rollout described in `deployment/native/README.md` +- `--runtime native` for the host-native Bun + systemd rollout described in `deployment/native/README.md` -The repo no longer ships or supports a separate `deployment/npm` stack. If you want a reverse proxy, point it at the host ports published by this stack. +The public VPS edge remains Nginx Proxy Manager. Docker fallback can be reached either through the shared Docker network service names or the host ports published by this stack. It is separate from the repo-root `docker-compose.yml`, which remains the lightweight local infra stack for development. @@ -17,7 +17,7 @@ Do not run the repo-root `docker-compose.yml` on the VPS. On the live server tha - Builds and runs the full Islandflow stack with Docker Compose. - Publishes `web` and `api` to host ports, bound to loopback by default. -- Runs ClickHouse, Redis, and NATS JetStream with persistent Docker volumes. +- Runs ClickHouse, Redis, and NATS JetStream with persistent host data under `ISLANDFLOW_DATA_ROOT`. - Runs the core runtime services: `ingest-options`, `ingest-equities`, `compute`, `candles`, `api`, and `web`. - Keeps `replay` opt-in through a Compose profile, because the current replay service starts immediately when the container is enabled. @@ -56,6 +56,7 @@ cp .env.example .env Important defaults: - `NATS_URL`, `CLICKHOUSE_URL`, and `REDIS_URL` should stay on the internal container hostnames unless you intentionally split infra out. +- `ISLANDFLOW_DATA_ROOT=/var/lib/islandflow` matches the native infra data root used by the VPS cutover helpers. - `OPTIONS_INGEST_ADAPTER=synthetic` and `EQUITIES_INGEST_ADAPTER=synthetic` are the safest first-boot settings. - `WEB_BIND_IP=127.0.0.1` and `API_BIND_IP=127.0.0.1` keep the published ports local to the host by default. - `WEB_HOST_PORT=3000` and `API_HOST_PORT=4000` control the host-side published ports. @@ -213,7 +214,7 @@ BuildKit cache mounts require a modern Docker Engine with Dockerfile frontend su ## Safe rollouts on `152.53.80.229` -The current live VPS uses Nginx Proxy Manager on the shared Docker network and routes public traffic to the Docker `web` and `api` containers by container name. Because of that, this Docker path remains the operationally correct default for the live server today. +The current live VPS uses Nginx Proxy Manager as the outer edge. Before native cutover, NPM routes Islandflow traffic to Docker service names. During cutover, `deployment/native/switch-npm-edge.sh native` retargets only the Islandflow proxy hosts to the NPM bridge gateway IP so NPM can reach native host ports. If needed, override the detected target with `ISLANDFLOW_NATIVE_HOST=`. The deploy helper also warns if it detects a second compose project named `islandflow` on the server, because that usually means the repo-root local-infra stack was started on the VPS by mistake. diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index 96598ba..1fbf251 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -42,6 +42,8 @@ services: init: true expose: - "3000" + ports: + - "${WEB_BIND_IP:-127.0.0.1}:${WEB_HOST_PORT:-3000}:3000" networks: - default - shared @@ -64,8 +66,13 @@ services: api: <<: *service-common command: ["services/api/src/index.ts"] + environment: + LOG_LEVEL: ${LOG_LEVEL:-warn} + API_HOST: 0.0.0.0 expose: - "4000" + ports: + - "${API_BIND_IP:-127.0.0.1}:${API_HOST_PORT:-4000}:4000" networks: - default - shared @@ -128,7 +135,7 @@ services: soft: 262144 hard: 262144 volumes: - - clickhouse-data:/var/lib/clickhouse + - ${ISLANDFLOW_DATA_ROOT:-/var/lib/islandflow}/clickhouse:/var/lib/clickhouse - ./clickhouse/listen.xml:/etc/clickhouse-server/config.d/listen.xml:ro healthcheck: test: @@ -146,7 +153,7 @@ services: restart: unless-stopped command: ["redis-server", "--appendonly", "yes"] volumes: - - redis-data:/data + - ${ISLANDFLOW_DATA_ROOT:-/var/lib/islandflow}/redis:/data healthcheck: test: [ @@ -164,14 +171,9 @@ services: restart: unless-stopped command: ["-js", "-sd", "/data"] volumes: - - nats-data:/data + - ${ISLANDFLOW_DATA_ROOT:-/var/lib/islandflow}/nats:/data networks: shared: external: true name: ${NPM_SHARED_NETWORK:-npm-shared} - -volumes: - clickhouse-data: - redis-data: - nats-data: diff --git a/deployment/native/README.md b/deployment/native/README.md index 4e2dd52..c421c51 100644 --- a/deployment/native/README.md +++ b/deployment/native/README.md @@ -9,12 +9,14 @@ This directory documents the host-native Islandflow rollout path used by: ## Current operating model -Native runtime is now intended for **fast iterative backend deploys first**, while Docker remains the supported public production edge until a deliberate cutover is completed. +Native runtime is now intended for a phased VPS cutover. Docker remains the supported rollback runtime, but Docker and native app services must not own the same Islandflow scope at the same time because the workers and API use durable JetStream consumers. Today, the recommended split is: -- **Docker runtime** for the live public `web` + `api` path -- **Native runtime** for worker-only iteration (`compute`, `candles`, `ingest-options`, `ingest-equities`) +- **Nginx Proxy Manager** remains the public `:80/:443` edge +- **Native system services** own NATS, Redis, and ClickHouse after infra cutover +- **Native user services** own `web`, `api`, and workers after app cutover +- **Docker Compose** remains available as the rollback runtime - local development stays: - Docker infra: `bun run dev:infra` - native backend services: `bun run dev:services` @@ -47,6 +49,38 @@ That means native worker deploy support is now provisioned on the host, but nati ## Checked-in native ops assets +### Infra system units + +Checked-in system service units and config live under: + +- `deployment/native/systemd/system/islandflow-nats.service` +- `deployment/native/systemd/system/islandflow-redis.service` +- `deployment/native/systemd/system/islandflow-clickhouse.service` +- `deployment/native/config/redis.conf` +- `deployment/native/config/clickhouse-listen.xml` + +Install and start them on the VPS with: + +```bash +./deployment/native/bootstrap-infra.sh +``` + +Or install and start manually: + +```bash +sudo ./deployment/native/install-infra-units.sh +sudo ./deployment/native/start-infra.sh +./deployment/native/check-native-infra.sh +``` + +The native infra services bind to loopback and use stable host data paths: + +- NATS JetStream: `/var/lib/islandflow/nats` +- Redis: `/var/lib/islandflow/redis` +- ClickHouse: `/var/lib/islandflow/clickhouse` + +The Docker fallback compose file uses the same `ISLANDFLOW_DATA_ROOT` default of `/var/lib/islandflow`, so rollback can preserve durable state when only one runtime is active. + ### User unit templates Checked-in unit files live under: @@ -89,10 +123,29 @@ Install script behavior: This validates: +- native infra health for `full`, `api`, `services`, and `workers` - `systemctl --user is-active` for the selected units - local API health at `http://127.0.0.1:4000/health` when API scope is included - local web health at `http://127.0.0.1:3000/` when web scope is included +### App cutover and edge switch helpers + +```bash +./deployment/native/cutover.sh full +./deployment/native/switch-npm-edge.sh native +./deployment/native/full-rollback.sh +``` + +The edge switch helper updates the Nginx Proxy Manager database entries for `flow.deltaisland.io` and `api.flow.deltaisland.io`, preserving the same-origin Islandflow API location matcher: + +```nginx +^/(ws|replay|prints|joins|nbbo|dark|flow|candles|history)/ +``` + +For native cutover, the helper targets the NPM bridge gateway IP by default, not `host.docker.internal`. NPM generates `proxy_pass` with a runtime-resolved `$server` variable, so Docker's `/etc/hosts` alias is not sufficient for these proxy hosts. On the current VPS that native target resolves to `172.18.0.1`, which reaches the host-native `3000` and `4000` listeners from the NPM container. + +Switching back to Docker restores upstreams to the Compose service names `web:3000` and `api:4000`. + ### Rollback helper ```bash @@ -184,7 +237,7 @@ Without that variable, these commands are refused: - `./deploy main --runtime native --api-only` - `./deploy main --runtime native --services-only` -This keeps the native path focused on safe worker iteration until proxy routing and public unit ownership are switched deliberately. +This keeps native app ownership explicit until infra, app health, and proxy routing are switched deliberately. ## Running deploy from the VPS itself diff --git a/deployment/native/bootstrap-infra.sh b/deployment/native/bootstrap-infra.sh new file mode 100755 index 0000000..dfc3422 --- /dev/null +++ b/deployment/native/bootstrap-infra.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +if [[ "${EUID}" -eq 0 ]]; then + "$repo_root/deployment/native/install-infra-units.sh" +else + sudo "$repo_root/deployment/native/install-infra-units.sh" +fi + +echo "Stopping Docker Islandflow services before native infra opens durable data." +( + cd "$repo_root/deployment/docker" + docker compose stop web api compute candles ingest-options ingest-equities nats redis clickhouse +) + +if [[ "${EUID}" -eq 0 ]]; then + "$repo_root/deployment/native/start-infra.sh" +else + sudo "$repo_root/deployment/native/start-infra.sh" +fi + +"$repo_root/deployment/native/check-native-infra.sh" diff --git a/deployment/native/check-native-health.sh b/deployment/native/check-native-health.sh index 1d070e5..13582bc 100755 --- a/deployment/native/check-native-health.sh +++ b/deployment/native/check-native-health.sh @@ -2,6 +2,7 @@ set -euo pipefail scope="${1:-full}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" units=() case "$scope" in @@ -27,6 +28,12 @@ case "$scope" in ;; esac +case "$scope" in + full|api|services|workers) + "$repo_root/deployment/native/check-native-infra.sh" + ;; +esac + for unit in "${units[@]}"; do systemctl --user is-active --quiet "$unit" echo "ok $unit" diff --git a/deployment/native/check-native-infra.sh b/deployment/native/check-native-infra.sh new file mode 100755 index 0000000..bfdc998 --- /dev/null +++ b/deployment/native/check-native-infra.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +systemctl is-active --quiet islandflow-nats.service +echo "ok islandflow-nats.service" + +systemctl is-active --quiet islandflow-redis.service +echo "ok islandflow-redis.service" + +systemctl is-active --quiet islandflow-clickhouse.service +echo "ok islandflow-clickhouse.service" + +if command -v redis-cli >/dev/null 2>&1; then + redis-cli -h 127.0.0.1 -p 6379 ping | grep -q PONG +else + timeout 2 bash -c ' + 127.0.0.1 + /var/lib/islandflow/clickhouse/ + /var/lib/islandflow/clickhouse/tmp/ + /var/lib/islandflow/clickhouse/user_files/ + diff --git a/deployment/native/config/redis.conf b/deployment/native/config/redis.conf new file mode 100644 index 0000000..8a39ba6 --- /dev/null +++ b/deployment/native/config/redis.conf @@ -0,0 +1,10 @@ +bind 127.0.0.1 +protected-mode yes +port 6379 +dir /var/lib/islandflow/redis +appendonly yes +save 900 1 +save 300 10 +save 60 10000 +loglevel notice +databases 16 diff --git a/deployment/native/cutover.sh b/deployment/native/cutover.sh new file mode 100755 index 0000000..fcff377 --- /dev/null +++ b/deployment/native/cutover.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +scope="${1:-full}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +case "$scope" in + full|services|workers|api|web) + ;; + *) + echo "Usage: deployment/native/cutover.sh [full|services|workers|api|web]" >&2 + exit 1 + ;; +esac + +echo "Stopping Docker-owned Islandflow app services before native ownership starts." +( + cd "$repo_root/deployment/docker" + docker compose stop web api compute candles ingest-options ingest-equities +) + +if [[ "$scope" == "full" || "$scope" == "services" || "$scope" == "api" || "$scope" == "web" ]]; then + "$repo_root/deployment/native/check-native-infra.sh" +fi + +systemctl --user restart $(case "$scope" in + full) echo islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;; + services) echo islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;; + workers) echo islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;; + api) echo islandflow-api.service ;; + web) echo islandflow-web.service ;; +esac) + +"$repo_root/deployment/native/check-native-health.sh" "$scope" diff --git a/deployment/native/full-rollback.sh b/deployment/native/full-rollback.sh new file mode 100755 index 0000000..77a78af --- /dev/null +++ b/deployment/native/full-rollback.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +echo "Stopping native app services." +systemctl --user stop islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service || true + +echo "Stopping native infra before Docker reopens durable data." +if [[ "${EUID}" -eq 0 ]]; then + systemctl stop islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service || true +else + sudo systemctl stop islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service || true +fi + +echo "Switching NPM Islandflow upstreams back to Docker service names." +"$repo_root/deployment/native/switch-npm-edge.sh" docker + +echo "Restarting Docker Islandflow runtime." +( + cd "$repo_root/deployment/docker" + docker compose up -d web api compute candles ingest-options ingest-equities +) + +curl -I -fksS "${DEPLOY_PUBLIC_APP_URL:-https://flow.deltaisland.io}" >/dev/null +curl -fksS "${DEPLOY_PUBLIC_API_HEALTH_URL:-https://api.flow.deltaisland.io/health}" >/dev/null +echo "Rollback validation passed." diff --git a/deployment/native/install-infra-units.sh b/deployment/native/install-infra-units.sh new file mode 100755 index 0000000..2a9ab85 --- /dev/null +++ b/deployment/native/install-infra-units.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +system_unit_source_dir="$repo_root/deployment/native/systemd/system" +config_source_dir="$repo_root/deployment/native/config" + +if [[ "${EUID}" -ne 0 ]]; then + echo "Run as root: sudo $0" >&2 + exit 1 +fi + +resolve_binary() { + local name="$1" + local path="" + + path="$(command -v "$name" 2>/dev/null || true)" + if [[ -n "$path" ]]; then + printf '%s\n' "$path" + return 0 + fi + + for candidate in "/usr/bin/$name" "/usr/sbin/$name" "/usr/local/bin/$name" "/usr/local/sbin/$name"; do + if [[ -x "$candidate" ]]; then + printf '%s\n' "$candidate" + return 0 + fi + done + + return 1 +} + +missing=() +for command in nats-server redis-server clickhouse-server; do + if ! resolve_binary "$command" >/dev/null; then + missing+=("$command") + fi +done + +if [[ ${#missing[@]} -gt 0 ]]; then + echo "Missing native infra binaries: ${missing[*]}" >&2 + echo "Install NATS Server, Redis Server, and ClickHouse Server before bootstrapping native infra." >&2 + echo "On Debian, Redis is usually available as redis-server; ClickHouse and NATS may require their vendor repositories or packaged binaries." >&2 + exit 1 +fi + +ensure_system_user() { + local name="$1" + local home="$2" + + getent group "$name" >/dev/null || groupadd --system "$name" + getent passwd "$name" >/dev/null || useradd --system --gid "$name" --home-dir "$home" --shell /usr/sbin/nologin "$name" +} + +ensure_system_user nats /var/lib/islandflow/nats +ensure_system_user redis /var/lib/islandflow/redis +ensure_system_user clickhouse /var/lib/islandflow/clickhouse + +install -d -m 0755 /etc/islandflow +install -m 0644 "$config_source_dir/redis.conf" /etc/islandflow/redis.conf +install -d -m 0755 /etc/clickhouse-server/config.d +install -m 0644 "$config_source_dir/clickhouse-listen.xml" /etc/clickhouse-server/config.d/islandflow-listen.xml + +install -d -o nats -g nats -m 0750 /var/lib/islandflow/nats +install -d -o redis -g redis -m 0750 /var/lib/islandflow/redis +install -d -o clickhouse -g clickhouse -m 0750 /var/lib/islandflow/clickhouse + +install -m 0644 "$system_unit_source_dir"/islandflow-*.service /etc/systemd/system/ +systemctl daemon-reload + +echo "Installed native infra system units and config." +echo "Start infra with: sudo deployment/native/start-infra.sh" diff --git a/deployment/native/start-infra.sh b/deployment/native/start-infra.sh new file mode 100755 index 0000000..8f78791 --- /dev/null +++ b/deployment/native/start-infra.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${EUID}" -ne 0 ]]; then + echo "Run as root: sudo $0" >&2 + exit 1 +fi + +for unit in redis-server.service nats-server.service clickhouse-server.service; do + if systemctl list-unit-files "$unit" >/dev/null 2>&1; then + systemctl disable --now "$unit" >/dev/null 2>&1 || true + fi +done + +systemctl reset-failed islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service || true +systemctl enable --now islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service +"$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/check-native-infra.sh" diff --git a/deployment/native/stop-infra.sh b/deployment/native/stop-infra.sh new file mode 100755 index 0000000..91a488d --- /dev/null +++ b/deployment/native/stop-infra.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${EUID}" -ne 0 ]]; then + echo "Run as root: sudo $0" >&2 + exit 1 +fi + +systemctl stop islandflow-nats.service islandflow-redis.service islandflow-clickhouse.service diff --git a/deployment/native/switch-npm-edge.sh b/deployment/native/switch-npm-edge.sh new file mode 100755 index 0000000..c9fcd93 --- /dev/null +++ b/deployment/native/switch-npm-edge.sh @@ -0,0 +1,285 @@ +#!/usr/bin/env bash +set -euo pipefail + +target="${1:-native}" +npm_root="${NPM_ROOT:-/home/delta/nginx-proxy-manager}" +db_path="${NPM_DB_PATH:-$npm_root/data/database.sqlite}" +app_domain="${ISLANDFLOW_APP_DOMAIN:-flow.deltaisland.io}" +api_domain="${ISLANDFLOW_API_DOMAIN:-api.flow.deltaisland.io}" +native_host="${ISLANDFLOW_NATIVE_HOST:-}" +docker_web_host="${ISLANDFLOW_DOCKER_WEB_HOST:-web}" +docker_api_host="${ISLANDFLOW_DOCKER_API_HOST:-api}" +web_port="${ISLANDFLOW_WEB_PORT:-3000}" +api_port="${ISLANDFLOW_API_PORT:-4000}" +restart_npm="${NPM_RESTART:-1}" +npm_container="${NPM_CONTAINER_NAME:-nginx-proxy-manager}" +sudo_cmd=() + +case "$target" in + native|docker) + ;; + *) + echo "Usage: deployment/native/switch-npm-edge.sh [native|docker]" >&2 + exit 1 + ;; +esac + +resolve_native_host() { + if [[ -n "$native_host" ]]; then + printf '%s\n' "$native_host" + return + fi + + if command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' | grep -qx "$npm_container"; then + native_host="$(docker inspect "$npm_container" --format '{{range .NetworkSettings.Networks}}{{println .Gateway}}{{end}}' | sed '/^$/d' | head -n1)" + if [[ -n "$native_host" ]]; then + printf '%s\n' "$native_host" + return + fi + fi + + echo "Unable to determine the native upstream host for NPM." >&2 + echo "Set ISLANDFLOW_NATIVE_HOST explicitly or start the $npm_container container first." >&2 + exit 1 +} + +if [[ "$target" == "native" ]]; then + native_host="$(resolve_native_host)" +fi + +if [[ ! -w "$db_path" || ! -w "$(dirname "$db_path")" ]]; then + if [[ "${EUID}" -eq 0 ]]; then + sudo_cmd=() + elif command -v sudo >/dev/null 2>&1; then + sudo_cmd=(sudo) + else + echo "NPM database path is not writable and sudo is unavailable: $db_path" >&2 + exit 1 + fi +fi + +if [[ ! -f "$db_path" ]]; then + echo "NPM database not found: $db_path" >&2 + exit 1 +fi + +backup="$db_path.before-islandflow-$target-$(date +%Y%m%d%H%M%S)" +"${sudo_cmd[@]}" cp "$db_path" "$backup" +echo "Backed up NPM database to $backup" + +"${sudo_cmd[@]}" python3 - "$db_path" "$target" "$app_domain" "$api_domain" "$native_host" "$docker_web_host" "$docker_api_host" "$web_port" "$api_port" <<'PY' +import json +import sqlite3 +import sys + +db_path, target, app_domain, api_domain, native_host, docker_web_host, docker_api_host, web_port, api_port = sys.argv[1:] +web_host = native_host if target == "native" else docker_web_host +api_host = native_host if target == "native" else docker_api_host + +advanced_config = f"""location ~ ^/(ws|replay|prints|joins|nbbo|dark|flow|candles|history)/ {{ + set $forward_scheme http; + set $server "{api_host}"; + set $port {api_port}; + + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $http_connection; + proxy_http_version 1.1; + + include conf.d/include/proxy.conf; +}}""" + +def has_domain(raw, domain): + try: + return domain in json.loads(raw) + except Exception: + return domain in raw + +con = sqlite3.connect(db_path) +cur = con.cursor() +rows = list(cur.execute("select id, domain_names from proxy_host where is_deleted = 0")) +app_ids = [row_id for row_id, domains in rows if has_domain(domains, app_domain)] +api_ids = [row_id for row_id, domains in rows if has_domain(domains, api_domain)] + +if len(app_ids) != 1 or len(api_ids) != 1: + raise SystemExit(f"Expected one app and one API proxy host, found app={app_ids} api={api_ids}") + +cur.execute( + "update proxy_host set forward_scheme = 'http', forward_host = ?, forward_port = ?, allow_websocket_upgrade = 1, advanced_config = ?, modified_on = datetime('now') where id = ?", + (web_host, int(web_port), advanced_config, app_ids[0]), +) +cur.execute( + "update proxy_host set forward_scheme = 'http', forward_host = ?, forward_port = ?, allow_websocket_upgrade = 1, modified_on = datetime('now') where id = ?", + (api_host, int(api_port), api_ids[0]), +) +con.commit() +print(f"Updated {app_domain} -> {web_host}:{web_port}") +print(f"Updated {api_domain} -> {api_host}:{api_port}") +PY + +if command -v python3 >/dev/null 2>&1; then + "${sudo_cmd[@]}" python3 - "$npm_root" "$db_path" "$target" "$app_domain" "$api_domain" "$native_host" "$docker_web_host" "$docker_api_host" "$web_port" "$api_port" <<'PY' +import json +import re +import sqlite3 +import sys +from pathlib import Path + +( + npm_root, + db_path, + target, + app_domain, + api_domain, + native_host, + docker_web_host, + docker_api_host, + web_port, + api_port, +) = sys.argv[1:] + +web_host = native_host if target == "native" else docker_web_host +api_host = native_host if target == "native" else docker_api_host + +def has_domain(raw, domain): + try: + return domain in json.loads(raw) + except Exception: + return domain in raw + +def replace_nth(text, pattern, replacement, index): + matches = list(pattern.finditer(text)) + if len(matches) < index: + raise SystemExit(f"Unable to rewrite generated proxy config; expected match {index} for {pattern.pattern!r}") + match = matches[index - 1] + return text[:match.start()] + replacement(match) + text[match.end():] + +server_pattern = re.compile(r'^(?P\s*set \$server\s+)".*?";\s*$', re.M) +port_pattern = re.compile(r'^(?P\s*set \$port\s+)\d+;\s*$', re.M) + +def replace_server(text, host, index): + return replace_nth(text, server_pattern, lambda m: f'{m.group("prefix")}"{host}";', index) + +def replace_port(text, port, index): + return replace_nth(text, port_pattern, lambda m: f'{m.group("prefix")}{port};', index) + +con = sqlite3.connect(db_path) +rows = list(con.execute("select id, domain_names from proxy_host where is_deleted = 0")) +app_ids = [row_id for row_id, domains in rows if has_domain(domains, app_domain)] +api_ids = [row_id for row_id, domains in rows if has_domain(domains, api_domain)] +if len(app_ids) != 1 or len(api_ids) != 1: + raise SystemExit(f"Expected one app and one API proxy host, found app={app_ids} api={api_ids}") + +api_conf = Path(npm_root) / "data/nginx/proxy_host" / f"{api_ids[0]}.conf" +app_conf = Path(npm_root) / "data/nginx/proxy_host" / f"{app_ids[0]}.conf" + +if api_conf.exists(): + text = api_conf.read_text() + text = replace_server(text, api_host, 1) + text = replace_port(text, int(api_port), 1) + api_conf.write_text(text) + print(f"Synchronized {api_conf.name} -> {api_host}:{api_port}") + +if app_conf.exists(): + text = app_conf.read_text() + text = replace_server(text, web_host, 1) + text = replace_port(text, int(web_port), 1) + text = replace_server(text, api_host, 2) + text = replace_port(text, int(api_port), 2) + app_conf.write_text(text) + print(f"Synchronized {app_conf.name} -> {web_host}:{web_port} and API matcher -> {api_host}:{api_port}") +PY +fi + +if [[ "$restart_npm" == "0" ]]; then + echo "NPM container restart skipped because NPM_RESTART=0." +elif command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' | grep -qx nginx-proxy-manager; then + docker restart nginx-proxy-manager >/dev/null + echo "Restarted nginx-proxy-manager" +else + echo "NPM container restart skipped; restart it manually if it is not managed by Docker on this host." +fi + +if command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' | grep -qx "$npm_container"; then + "${sudo_cmd[@]}" python3 - "$npm_root" "$db_path" "$target" "$app_domain" "$api_domain" "$native_host" "$docker_web_host" "$docker_api_host" "$web_port" "$api_port" <<'PY' +import json +import re +import sqlite3 +import sys +from pathlib import Path + +( + npm_root, + db_path, + target, + app_domain, + api_domain, + native_host, + docker_web_host, + docker_api_host, + web_port, + api_port, +) = sys.argv[1:] + +web_host = native_host if target == "native" else docker_web_host +api_host = native_host if target == "native" else docker_api_host + +def has_domain(raw, domain): + try: + return domain in json.loads(raw) + except Exception: + return domain in raw + +def replace_nth(text, pattern, replacement, index): + matches = list(pattern.finditer(text)) + if len(matches) < index: + raise SystemExit(f"Unable to rewrite generated proxy config; expected match {index} for {pattern.pattern!r}") + match = matches[index - 1] + return text[:match.start()] + replacement(match) + text[match.end():] + +server_pattern = re.compile(r'^(?P\s*set \$server\s+)".*?";\s*$', re.M) +port_pattern = re.compile(r'^(?P\s*set \$port\s+)\d+;\s*$', re.M) + +def replace_server(text, host, index): + return replace_nth(text, server_pattern, lambda m: f'{m.group("prefix")}"{host}";', index) + +def replace_port(text, port, index): + return replace_nth(text, port_pattern, lambda m: f'{m.group("prefix")}{port};', index) + +con = sqlite3.connect(db_path) +rows = list(con.execute("select id, domain_names from proxy_host where is_deleted = 0")) +app_ids = [row_id for row_id, domains in rows if has_domain(domains, app_domain)] +api_ids = [row_id for row_id, domains in rows if has_domain(domains, api_domain)] +if len(app_ids) != 1 or len(api_ids) != 1: + raise SystemExit(f"Expected one app and one API proxy host, found app={app_ids} api={api_ids}") + +api_conf = Path(npm_root) / "data/nginx/proxy_host" / f"{api_ids[0]}.conf" +app_conf = Path(npm_root) / "data/nginx/proxy_host" / f"{app_ids[0]}.conf" + +if api_conf.exists(): + text = api_conf.read_text() + text = replace_server(text, api_host, 1) + text = replace_port(text, int(api_port), 1) + api_conf.write_text(text) + +if app_conf.exists(): + text = app_conf.read_text() + text = replace_server(text, web_host, 1) + text = replace_port(text, int(web_port), 1) + text = replace_server(text, api_host, 2) + text = replace_port(text, int(api_port), 2) + app_conf.write_text(text) +PY + reloaded=0 + for _ in 1 2 3 4 5; do + if docker exec "$npm_container" nginx -s reload >/dev/null 2>&1; then + reloaded=1 + break + fi + sleep 1 + done + if [[ "$reloaded" == "1" ]]; then + echo "Reloaded nginx-proxy-manager" + else + echo "Warning: nginx-proxy-manager reload did not succeed after restart; verify the container is healthy." >&2 + fi +fi diff --git a/deployment/native/systemd/system/islandflow-clickhouse.service b/deployment/native/systemd/system/islandflow-clickhouse.service new file mode 100644 index 0000000..79f8ed2 --- /dev/null +++ b/deployment/native/systemd/system/islandflow-clickhouse.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow ClickHouse +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/bin/env clickhouse-server --config-file=/etc/clickhouse-server/config.xml +Restart=always +RestartSec=5 +User=clickhouse +Group=clickhouse +StateDirectory=clickhouse +LimitNOFILE=262144 + +[Install] +WantedBy=multi-user.target diff --git a/deployment/native/systemd/system/islandflow-nats.service b/deployment/native/systemd/system/islandflow-nats.service new file mode 100644 index 0000000..a23eefc --- /dev/null +++ b/deployment/native/systemd/system/islandflow-nats.service @@ -0,0 +1,18 @@ +[Unit] +Description=Islandflow NATS JetStream +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/sbin/nats-server -js -sd /var/lib/islandflow/nats -a 127.0.0.1 -p 4222 -m 8222 +Restart=always +RestartSec=2 +User=nats +Group=nats +RuntimeDirectory=islandflow-nats +StateDirectory=islandflow/nats +LimitNOFILE=1048576 + +[Install] +WantedBy=multi-user.target diff --git a/deployment/native/systemd/system/islandflow-redis.service b/deployment/native/systemd/system/islandflow-redis.service new file mode 100644 index 0000000..3e63d74 --- /dev/null +++ b/deployment/native/systemd/system/islandflow-redis.service @@ -0,0 +1,18 @@ +[Unit] +Description=Islandflow Redis +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +ExecStart=/usr/bin/env redis-server /etc/islandflow/redis.conf --supervised systemd --daemonize no +Restart=always +RestartSec=2 +User=redis +Group=redis +RuntimeDirectory=islandflow-redis +StateDirectory=islandflow/redis +LimitNOFILE=65535 + +[Install] +WantedBy=multi-user.target diff --git a/deployment/native/systemd/user/islandflow-api.service b/deployment/native/systemd/user/islandflow-api.service index 5a74500..1e6cc99 100644 --- a/deployment/native/systemd/user/islandflow-api.service +++ b/deployment/native/systemd/user/islandflow-api.service @@ -6,6 +6,8 @@ Wants=network-online.target [Service] Type=simple WorkingDirectory=/home/delta/islandflow +Environment=API_HOST=0.0.0.0 +Environment=API_PORT=4000 EnvironmentFile=/home/delta/islandflow/.env ExecStart=/home/delta/.bun/bin/bun services/api/src/index.ts Restart=always diff --git a/deployment/native/systemd/user/islandflow-ingest-options.service b/deployment/native/systemd/user/islandflow-ingest-options.service index eac0a6c..10107b1 100644 --- a/deployment/native/systemd/user/islandflow-ingest-options.service +++ b/deployment/native/systemd/user/islandflow-ingest-options.service @@ -7,6 +7,7 @@ Wants=network-online.target Type=simple WorkingDirectory=/home/delta/islandflow EnvironmentFile=/home/delta/islandflow/.env +Environment=OPTIONS_INGEST_ADAPTER=synthetic ExecStart=/home/delta/.bun/bin/bun services/ingest-options/src/index.ts Restart=always RestartSec=2 diff --git a/deployment/native/systemd/user/islandflow-web.service b/deployment/native/systemd/user/islandflow-web.service index 6e79177..ce75e0b 100644 --- a/deployment/native/systemd/user/islandflow-web.service +++ b/deployment/native/systemd/user/islandflow-web.service @@ -6,8 +6,10 @@ Wants=network-online.target [Service] Type=simple WorkingDirectory=/home/delta/islandflow +Environment=WEB_HOST=0.0.0.0 +Environment=WEB_PORT=3000 EnvironmentFile=/home/delta/islandflow/.env -ExecStart=/home/delta/.bun/bin/bun --cwd apps/web run start +ExecStart=/bin/sh -lc 'cd /home/delta/islandflow/apps/web && exec /home/delta/.bun/bin/bun x next start -H "$WEB_HOST" -p "$WEB_PORT"' Restart=always RestartSec=2 KillSignal=SIGINT diff --git a/docs/turns/2026-05-18-native-public-edge-cutover.html b/docs/turns/2026-05-18-native-public-edge-cutover.html new file mode 100644 index 0000000..8d2d2b1 --- /dev/null +++ b/docs/turns/2026-05-18-native-public-edge-cutover.html @@ -0,0 +1,521 @@ + + + + + + Turn Document - Native Public Edge Cutover + + + +
+
+
Islandflow Turn Document
+

Native Public Edge Cutover

+

+ Completed the VPS native-first cutover for Islandflow infrastructure and app services while keeping Nginx + Proxy Manager as the outer edge and Docker as the rollback path. The final state now serves + flow.deltaisland.io and api.flow.deltaisland.io from the native web and API + processes, with verified public routing and a documented follow-up for the long-term API Cloudflare posture. +

+
+
+
Generated
+
2026-05-18 19:52 EDT
+
+
+
Primary Issue
+
islandflow-vvw
+
+
+
Follow-up
+
islandflow-fl5
+
+
+
Runtime State
+
Native active, Docker retained for rollback
+
+
+
+ +
+

Summary

+

+ The repository now contains the native infra units, native cutover scripts, Docker fallback adjustments, and + public-edge retargeting logic required to run Islandflow natively on the VPS. During validation, the live NPM + edge was switched from Docker container-name upstreams to native host ports, the host firewall was adjusted so + the NPM bridge could reach the native API, and the separate public API TLS problem was resolved by correcting + the Cloudflare DNS state for api.flow.deltaisland.io. +

+
+ +
+

Changes Made

+
    +
  • + Added checked-in native infra operations under deployment/native/, including + bootstrap-infra.sh, check-native-infra.sh, cutover.sh, + full-rollback.sh, start-infra.sh, and the native system units for NATS, Redis, + and ClickHouse. +
    • +
  • + Extended native app runtime units so the web and API bind on host-reachable interfaces, and forced the + native options ingest service to use the synthetic adapter during the cutover. +
    • +
  • + Updated services/api to support explicit host binding through API_HOST, and fixed + JetStream retention conversion in packages/bus so native services can start cleanly with the + configured max-age values. +
    • +
  • + Updated the Docker fallback assets to publish loopback web/API ports, share durable host data under + /var/lib/islandflow, and document the native-to-Docker rollback path. +
    • +
  • + Reworked deployment/native/switch-npm-edge.sh so it targets the NPM bridge gateway IP instead + of host.docker.internal, handles the root-owned NPM SQLite database, synchronizes generated + proxy_host configs, and reloads NPM deterministically after the edge switch. +
    • +
  • + Created Beads follow-up issue islandflow-fl5 for the remaining decision about whether + api.flow.deltaisland.io should remain DNS-only or be re-proxied through Cloudflare. +
    • +
+
+ +
+

Context

+

+ The migration started from a Docker-owned production baseline where NATS, Redis, ClickHouse, API, workers, and + web all ran in Compose, while NPM routed Islandflow traffic to Docker service names. That setup blocked a safe + native cutover for two reasons: the native services could not reach Docker-only infra reliably, and NPM could + not send public traffic to host-native processes without a deliberate upstream retarget. +

+

+ The runtime model for this work is exclusive ownership. Native and Docker are not allowed to run the same API + or worker scopes in parallel because JetStream durable consumers would conflict. The objective was therefore a + phased handoff, not a mixed soak for the same queues. +

+
+ +
+

Important Implementation Details

+
+
+

NPM edge targeting

+

+ NPM generates proxy_pass from a runtime-resolved $server variable, so the + Docker /etc/hosts alias for host.docker.internal was not sufficient. The switch + helper now detects the NPM bridge gateway and uses that IP for native upstreams. +

+
+
+

Firewall path

+

+ The host UFW policy already allowed port 3000 but not 4000. The live fix was a + source-scoped allow for the NPM bridge subnet so the containerized edge could reach the native API. +

+
+
+

Cloudflare API hostname

+

+ The API hostname failure was separate from the native cutover. The hostname is now a DNS-only + A record pointing at the VPS, which restored public TLS and health responses. +

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AreaImplementation detail
Native API + services/api/src/index.ts now accepts API_HOST and passes it to + Bun.serve. The native unit sets API_HOST=0.0.0.0 and + API_PORT=4000. +
Native web + The native web unit now starts from apps/web with + bun x next start -H "$WEB_HOST" -p "$WEB_PORT", avoiding the earlier repo-root startup + failure and binding the service on 0.0.0.0:3000. +
JetStream retention + Native startup exposed a retention-unit bug. The shared bus layer now converts stream max-age values with + nanos(...) and formats them back with millis(...). +
Docker fallback + Docker Compose now uses ISLANDFLOW_DATA_ROOT=/var/lib/islandflow, publishes loopback + ports, and keeps the fallback runtime compatible with the same durable data directories as the native + services. +
NPM switch helper + The helper now updates both the NPM database and the generated + /data/nginx/proxy_host/*.conf files, because a DB-only restart did not reliably rewrite the + live configs for Islandflow. +
+ + 
sudo ufw allow proto tcp from 172.18.0.0/16 to any port 4000 comment 'npm bridge to native api'
+
+ +
+

Expected Impact for End-Users

+
    +
  • + Public web and API traffic now reaches the native Islandflow services, which removes Docker from the primary + live request path while keeping the outer edge unchanged. +
    • +
  • + Same-origin public API routes such as /prints, /history, /replay, + /nbbo, and /ws/live continue to resolve correctly through the main app hostname. +
    • +
  • + Rollback remains fast and explicit: NPM can be pointed back at Docker service names and the Docker runtime + can reclaim the same durable data directories if native operation needs to be abandoned. +
    • +
+
+ +
+

Validation

+
+
+
Static checks
+
    +
  • bun run check:docker-workspace
    • +
  • docker compose -f deployment/docker/docker-compose.yml config --quiet
    • +
  • docker compose -f /home/delta/nginx-proxy-manager/docker-compose.yml config --quiet
    • +
  • bash -n deployment/native/*.sh
    • +
  • systemd-analyze verify deployment/native/systemd/user/*.service deployment/native/systemd/system/*.service
    • +
  • bun build services/api/src/index.ts --target=bun
    • +
  • bun build scripts/deploy.ts --target=bun
    • +
+
+
+
Native runtime
+
    +
  • ./deployment/native/check-native-health.sh full
    • +
  • curl http://127.0.0.1:4000/health
    • +
  • curl -I http://127.0.0.1:3000/
    • +
+
+
+
Public edge
+
    +
  • curl -I -fksS https://flow.deltaisland.io
    • +
  • curl -fksS https://api.flow.deltaisland.io/health
    • +
  • bun run scripts/check-public-api-routes.ts https://flow.deltaisland.io
    • +
+
+
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • + The native ingest-options service required an explicit synthetic-adapter override because the environment file + still pointed at an Alpaca adapter that was returning 401 responses. The service now starts + cleanly for native cutover, but production adapter selection remains an operational decision. +
    • +
  • + The NPM helper still relies on direct config synchronization because NPM did not reliably regenerate the + Islandflow proxy files from SQLite changes alone. This is mitigated by keeping the synchronization logic + checked in and by reloading NPM as part of the helper itself. +
    • +
  • + The final public API recovery currently leaves api.flow.deltaisland.io as a DNS-only hostname. + That restored service, but it changes the edge posture relative to the web hostname and should be reviewed + deliberately. +
    • +
  • + A temporary Cloudflare API token was used to inspect and correct zone state during validation. That token + should be rotated outside this repository workflow. +
    • +
+
+ +
+

Follow-up Work

+
    +
  • + islandflow-fl5: decide whether api.flow.deltaisland.io should remain DNS-only or + be re-proxied through Cloudflare, then re-validate TLS, websocket, and operational behavior for the chosen + posture. +
    • +
  • + After operational soak, decide whether native should become the default production runtime or remain a + supported alternative with Docker as the preferred steady-state runtime. +
    • +
+
+
+ + diff --git a/packages/bus/src/jetstream.ts b/packages/bus/src/jetstream.ts index 2eaf6a0..04bfa85 100644 --- a/packages/bus/src/jetstream.ts +++ b/packages/bus/src/jetstream.ts @@ -9,7 +9,9 @@ import { type StreamUpdateConfig, JSONCodec, type JsMsg, - createInbox + createInbox, + nanos, + millis } from "nats"; import { getKnownStreamDefinitions, getStreamDefinition, type StreamRetentionClass } from "./streams"; @@ -164,13 +166,13 @@ export const resolveStreamRetention = ( ): Pick => { if (streamClass === "raw") { return { - max_age: parseBoundedNumber(env.STREAM_RAW_MAX_AGE_MS, 3_600_000), + max_age: nanos(parseBoundedNumber(env.STREAM_RAW_MAX_AGE_MS, 3_600_000)), max_bytes: parseBoundedNumber(env.STREAM_RAW_MAX_BYTES, 536_870_912) }; } return { - max_age: parseBoundedNumber(env.STREAM_DERIVED_MAX_AGE_MS, 43_200_000), + max_age: nanos(parseBoundedNumber(env.STREAM_DERIVED_MAX_AGE_MS, 43_200_000)), max_bytes: parseBoundedNumber(env.STREAM_DERIVED_MAX_BYTES, 268_435_456) }; }; @@ -417,7 +419,7 @@ const formatBytes = (value: number): string => { }; const formatRetentionSummary = (config: StreamConfig): string => { - return `age=${formatDurationMs(Number(config.max_age))} bytes=${formatBytes(config.max_bytes)} replicas=${config.num_replicas} retention=${config.retention} discard=${config.discard}`; + return `age=${formatDurationMs(millis(Number(config.max_age)))} bytes=${formatBytes(config.max_bytes)} replicas=${config.num_replicas} retention=${config.retention} discard=${config.discard}`; }; const formatReportLine = ( @@ -442,12 +444,12 @@ const formatReportLine = ( const details = report.retentionDrift .map((delta) => { const desiredValue = delta.field === "max_age" - ? formatDurationMs(Number(delta.desired)) + ? formatDurationMs(millis(Number(delta.desired))) : delta.field === "max_bytes" ? formatBytes(Number(delta.desired)) : formatStructuredValue(delta.desired); const currentValue = delta.field === "max_age" - ? formatDurationMs(Number(delta.current)) + ? formatDurationMs(millis(Number(delta.current))) : delta.field === "max_bytes" ? formatBytes(Number(delta.current)) : formatStructuredValue(delta.current); diff --git a/scripts/deploy.ts b/scripts/deploy.ts index 043122e..e6f3a5c 100644 --- a/scripts/deploy.ts +++ b/scripts/deploy.ts @@ -920,6 +920,10 @@ function remoteNativeVerification(scope: DeployScope, fast: boolean): void { const units = nativeUnitsForScope(scope).map((value) => shellEscape(value)).join(" "); const checks: string[] = []; + if (scope === "full" || scope === "api" || scope === "services" || scope === "workers") { + checks.push("./deployment/native/check-native-infra.sh"); + } + if (scopeIncludesApi(scope)) { checks.push('curl -fksS http://127.0.0.1:4000/health'); } @@ -954,10 +958,10 @@ function remoteVerification(runtime: DeployRuntime, scope: DeployScope, fast: bo function publicVerification(scope: DeployScope, fast: boolean): void { section("Public Verification"); - if (!fast || scopeIncludesWeb(scope)) { + if (scopeIncludesWeb(scope)) { runChecked("curl", ["-I", "-fksS", PUBLIC_APP_URL]); } else { - console.log("[deploy] Fast mode: skipping public app HEAD check because web scope is not included."); + console.log("[deploy] Skipping public app HEAD check because web scope is not included."); } if (scopeIncludesApi(scope) && PUBLIC_API_HEALTH_URL) { diff --git a/services/api/src/index.ts b/services/api/src/index.ts index 433222a..41761a7 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -138,6 +138,7 @@ const DeliverPolicySchema = z.enum(["new", "all", "last", "last_per_subject"]); const envSchema = z.object({ API_PORT: z.coerce.number().int().positive().default(4000), + API_HOST: z.string().min(1).default("127.0.0.1"), NATS_URL: z.string().default("nats://127.0.0.1:4222"), CLICKHOUSE_URL: z.string().default("http://127.0.0.1:8123"), CLICKHOUSE_DATABASE: z.string().default("default"), @@ -1313,6 +1314,7 @@ const run = async () => { }; const server = Bun.serve({ + hostname: env.API_HOST, port: env.API_PORT, fetch: async (req: Request, serverRef: any) => { const url = new URL(req.url); @@ -1995,7 +1997,7 @@ const run = async () => { } }); - logger.info("api listening", { port: server.port }); + logger.info("api listening", { host: env.API_HOST, port: server.port }); const shutdown = async (signal: string) => { if (state.shutdownPromise) { From 04baecebe0574ff6ae1a1ed8552271d08c330bba Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Mon, 18 May 2026 21:32:44 -0400 Subject: [PATCH 05/26] update turn docs and beads workflow --- AGENTS.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3ab1cf0..8f1971b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,8 +117,9 @@ Each turn document must include these sections: 2. **Changes Made** 3. **Context** 4. **Important Implementation Details** -5. **Expected Impact for End-Users** -5. **Validation** +5. **Relevant Diff Snippets** +6. **Expected Impact for End-Users** +7. **Validation** 6. **Issues, Limitations, and Mitigations** 7. **Follow-up Work** From 8173b05c1c71318d5ab10696bfe1fd2d790e7427 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 07:05:55 -0400 Subject: [PATCH 06/26] upgrade next.js to 16.2.6 and react 19 --- .beads/issues.jsonl | 1 + AGENTS server.md | 174 ++++++++++++++++++++++++++++++++++++++++++++ AGENTS.md | 5 +- 3 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 AGENTS server.md diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 9909cdd..e7a99aa 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -13,6 +13,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-lib","title":"Upgrade apps/web to Next.js 16.2.6","description":"Upgrade the web app dependency stack to Next.js 16.2.6 with React 19, refresh Bun and mirrored Docker workspace lockfiles, keep runtime behavior unchanged, fix any focused web test fallout, validate the web build and targeted route tests, and document the completed work.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:04:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:04:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8fn","title":"implement alpaca-backed news wire view","description":"Why this issue exists and what needs to be done:\\nAdd an Alpaca-powered live news pipeline, API, storage, and web experience, including a dedicated /news route, Home preview, live fanout, history pagination, ticker resolution, and replay-mode live-only empty states.\\n\\nAcceptance criteria:\\n- normalized NewsStory contract and live channel exist\\n- ingest-news service backfills and streams Alpaca news\\n- API persists, serves, and fans out news\\n- web app exposes /news plus Home preview and drawer\\n- tests cover types, storage, API, and key UI behaviors\\n- turn documentation is added\\n\\nDesign:\\nReuse Islandflow drawer, chips, panes, and terminal styling; keep news live-only in v1 replay mode.\\n\\nNotes:\\nImplement client-side ticker filtering in v1 and expose latest revision only per provider+story_id.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T20:37:13Z","created_by":"dirtydishes","updated_at":"2026-05-18T20:55:11Z","started_at":"2026-05-18T20:37:20Z","closed_at":"2026-05-18T20:55:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-k8i","title":"Fix duplicate alert context import in API entrypoint","description":"Recent alert-context work introduced a duplicate fetchAlertContextByTraceId import in services/api/src/index.ts, which risks breaking TypeScript compilation and API startup. Remove the duplicate import and validate the affected API/web tests.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:58Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:03:40Z","started_at":"2026-05-18T13:02:02Z","closed_at":"2026-05-18T13:03:40Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lk9","title":"Fix PR creation workflow after Forgejo migration","description":"## Why\\nCreating pull requests with fails after the repository moved primary collaboration from GitHub to Forgejo. The current workflow still assumes GitHub GraphQL PR creation semantics, which do not work against the Forgejo remote.\\n\\n## What\\nInvestigate the current PR creation path, identify remaining GitHub-specific assumptions, and update the repo workflow/scripts/docs so contributors can reliably publish branches and open PRs in the Forgejo-based setup.\\n\\n## Acceptance Criteria\\n- The repo no longer instructs contributors to use a broken GitHub-specific PR creation path for Forgejo branches\\n- There is a documented and preferably scripted way to create the equivalent review request against Forgejo\\n- Validation demonstrates the new workflow behaves correctly or clearly documents any remaining platform limitation","status":"in_progress","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T10:26:47Z","created_by":"dirtydishes","updated_at":"2026-05-18T10:26:53Z","started_at":"2026-05-18T10:26:53Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/AGENTS server.md b/AGENTS server.md new file mode 100644 index 0000000..08a484a --- /dev/null +++ b/AGENTS server.md @@ -0,0 +1,174 @@ + +## Beads Issue Tracker + +This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands. + +### Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work +bd close # Complete work +``` + +### Rules + +- Use `bd` for ALL task tracking — do NOT use TodoWrite, TaskCreate, or markdown TODO lists +- Run `bd prime` for detailed command reference and session close protocol +- Use `bd remember` for persistent knowledge — do NOT use MEMORY.md files + +## Session Completion + +**When ending a work session**, you MUST complete ALL steps below. Work is NOT complete until `git push` succeeds. + +**MANDATORY WORKFLOW:** + +1. **File issues for remaining work** - Create issues for anything that needs follow-up +2. **Run quality gates** (if code changed) - Tests, linters, builds +3. **Update issue status** - Close finished work, update in-progress items +4. **PUSH TO REMOTE** - This is MANDATORY: + ```bash + git pull --rebase + bd dolt push + git push + git status # MUST show "up to date with origin" + ``` +5. **Clean up** - Clear stashes, prune remote branches +6. **Verify** - All changes committed AND pushed +7. **Hand off** - Provide context for next session + +**CRITICAL RULES:** +- Work is NOT complete until `git push` succeeds +- NEVER stop before pushing - that leaves work stranded locally +- NEVER say "ready to push when you are" - YOU must push +- If push fails, resolve and retry until it succeeds + + +## Minimal Repo Operating Instructions + +This is a Bun + TypeScript monorepo for an event-sourced market-data pipeline: +- Flow: ingest services publish to NATS/JetStream, compute/candles derive events, API serves REST/WS, web consumes live/replay streams. +- Main folders: `services/*` (runtime services), `packages/*` (shared libs/types/storage), `apps/web` (Next.js UI). +- Infra dependency: local dev assumes Docker services (NATS, ClickHouse, Redis) are available. + +Use these repo-specific commands: +- Install deps: `bun install` +- Start full stack: `bun run dev` +- Start infra only: `bun run dev:infra` +- Start backend services only: `bun run dev:services` +- Start web only: `bun run dev:web` + +Testing and validation in this repo are Bun-first: +- Run tests: `bun test` +- Run scoped tests: `bun test services/compute/tests` (or another package/service path) +- Validate web production build when UI code changes: `bun --cwd=apps/web run build` + +Working style that avoids common problems here: +- Prefer editing in the touched workspace (`services/`, `packages/`, `apps/web`) and keep shared contract changes in `packages/types`. +- Keep `.env` aligned with `.env.example`; adapters default to synthetic modes for local development. +- Dev runners persist child PID state in `.tmp/`; if a previous run crashed, restart via the standard `bun run dev*` commands so stale processes are cleaned up. + +## Required Turn Documentation + +At the end of every completed implementation task, before final handoff, create a user-readable HTML document describing the work. + +This documentation is mandatory whenever code, configuration, tests, or project files were changed. + +### Location + +Save the document in: + +```text +docs/turns/ +``` +## Important: If you are not working inside a git repository, save the document to `~/dev/docs/turns/` + +Use a clear timestamped filename: + +```text +docs/turns/YYYY-MM-DD-short-task-name.html +``` + +Example: + +```text +docs/turns/2026-05-14-add-market-replay-controls.html +``` + +### Format + +Use the impeccable skill to structure the document as clean, readable HTML. + +If the impeccable skill is unavailable, still create a well-structured standalone HTML file with: + +- A concise summary at the top +- A detailed explanation of what changed +- Relevant context or background +- Specific code snippets or examples when helpful +- Issues, limitations, tradeoffs, or mitigations +- Validation performed, including tests, builds, linters, or manual checks +- Any remaining follow-up work, with corresponding Beads issue IDs when applicable + +### Required Sections + +Each turn document must include these sections: + +1. **Summary** +2. **Changes Made** +3. **Context** +4. **Important Implementation Details** +5. **Relevant Diff Snippets** +6. **Expected Impact for End-Users** +7. **Validation** +8. **Issues, Limitations, and Mitigations** +9. **Follow-up Work** + +### Completion Rule + +A task is not complete until: + +1. The Beads workflow is updated +2. The turn document is created in `docs/turns` +3. Relevant quality gates have passed or failures are documented +4. Changes are committed +5. `bd dolt push` succeeds +6. `git push` succeeds +7. `git status` shows the branch is up to date with origin + +For trivial changes, the document may be brief, but it must still exist and clearly explain what changed and how it was validated. + +## Plan Mode Documentation + +When working in plan mode, do not modify implementation files. + +At the end of plan mode, provide a concise summary of the plan and ask the user whether they want to proceed with implementation. + +If the user asks to save the plan, create a user-readable HTML plan document in: + +```text +docs/plans/ +``` + +Use a clear timestamped filename: + +```text +docs/plans/YYYY-MM-DD-short-plan-name.html +``` + +The plan document should be labeled clearly as a plan and should include: + +1. **Plan Summary** +2. **Goals** +3. **Proposed Changes** +4. **Relevant Context** +5. **Implementation Steps** +6. **Risks, Limitations, and Mitigations** +7. **Open Questions** + +Always do the following when you finish a task, finish the beads workflow and and make a commit: +- Document the changes in a user-readable format +- Use the impeccable skill to structure the document as HTML +- Create a clear, concise summary of the changes at the top, followed by a detailed description of the changes, including any relevant context or background as well as specific code snippets or examples. +- Note any relevant issues or limitations that were addressed or mitigated by the changes. +- The HTML file should be stored in the `docs/turns` directory. It should include the current date and time, as well as a brief explanation of changes. e.g. docs/turns/YYYY-MM-DD-{description}.html diff --git a/AGENTS.md b/AGENTS.md index 8f1971b..08a484a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,6 +82,7 @@ Save the document in: ```text docs/turns/ ``` +## Important: If you are not working inside a git repository, save the document to `~/dev/docs/turns/` Use a clear timestamped filename: @@ -120,8 +121,8 @@ Each turn document must include these sections: 5. **Relevant Diff Snippets** 6. **Expected Impact for End-Users** 7. **Validation** -6. **Issues, Limitations, and Mitigations** -7. **Follow-up Work** +8. **Issues, Limitations, and Mitigations** +9. **Follow-up Work** ### Completion Rule From 728ca5569dc27b5f476ca91bddc47a1773c60431 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 07:12:06 -0400 Subject: [PATCH 07/26] update beads --- .beads/issues.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index e7a99aa..493492f 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -13,7 +13,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-lib","title":"Upgrade apps/web to Next.js 16.2.6","description":"Upgrade the web app dependency stack to Next.js 16.2.6 with React 19, refresh Bun and mirrored Docker workspace lockfiles, keep runtime behavior unchanged, fix any focused web test fallout, validate the web build and targeted route tests, and document the completed work.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:04:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:04:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-lib","title":"Upgrade apps/web to Next.js 16.2.6","description":"Upgrade the web app dependency stack to Next.js 16.2.6 with React 19, refresh Bun and mirrored Docker workspace lockfiles, keep runtime behavior unchanged, fix any focused web test fallout, validate the web build and targeted route tests, and document the completed work.","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:04:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:04:57Z","started_at":"2026-05-19T11:04:57Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8fn","title":"implement alpaca-backed news wire view","description":"Why this issue exists and what needs to be done:\\nAdd an Alpaca-powered live news pipeline, API, storage, and web experience, including a dedicated /news route, Home preview, live fanout, history pagination, ticker resolution, and replay-mode live-only empty states.\\n\\nAcceptance criteria:\\n- normalized NewsStory contract and live channel exist\\n- ingest-news service backfills and streams Alpaca news\\n- API persists, serves, and fans out news\\n- web app exposes /news plus Home preview and drawer\\n- tests cover types, storage, API, and key UI behaviors\\n- turn documentation is added\\n\\nDesign:\\nReuse Islandflow drawer, chips, panes, and terminal styling; keep news live-only in v1 replay mode.\\n\\nNotes:\\nImplement client-side ticker filtering in v1 and expose latest revision only per provider+story_id.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T20:37:13Z","created_by":"dirtydishes","updated_at":"2026-05-18T20:55:11Z","started_at":"2026-05-18T20:37:20Z","closed_at":"2026-05-18T20:55:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-k8i","title":"Fix duplicate alert context import in API entrypoint","description":"Recent alert-context work introduced a duplicate fetchAlertContextByTraceId import in services/api/src/index.ts, which risks breaking TypeScript compilation and API startup. Remove the duplicate import and validate the affected API/web tests.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:58Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:03:40Z","started_at":"2026-05-18T13:02:02Z","closed_at":"2026-05-18T13:03:40Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lk9","title":"Fix PR creation workflow after Forgejo migration","description":"## Why\\nCreating pull requests with fails after the repository moved primary collaboration from GitHub to Forgejo. The current workflow still assumes GitHub GraphQL PR creation semantics, which do not work against the Forgejo remote.\\n\\n## What\\nInvestigate the current PR creation path, identify remaining GitHub-specific assumptions, and update the repo workflow/scripts/docs so contributors can reliably publish branches and open PRs in the Forgejo-based setup.\\n\\n## Acceptance Criteria\\n- The repo no longer instructs contributors to use a broken GitHub-specific PR creation path for Forgejo branches\\n- There is a documented and preferably scripted way to create the equivalent review request against Forgejo\\n- Validation demonstrates the new workflow behaves correctly or clearly documents any remaining platform limitation","status":"in_progress","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T10:26:47Z","created_by":"dirtydishes","updated_at":"2026-05-18T10:26:53Z","started_at":"2026-05-18T10:26:53Z","dependency_count":0,"dependent_count":0,"comment_count":0} From b6fa2f0d179ad18c402b40c93a9ba0c1a946ead1 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 07:31:41 -0400 Subject: [PATCH 08/26] upgrade web to nextjs 16 --- .beads/issues.jsonl | 2 +- apps/web/app/terminal.tsx | 6 +- apps/web/next-env.d.ts | 3 +- apps/web/package.json | 9 +- bun.lock | 113 ++++++--- deployment/docker/workspace-root/bun.lock | 113 ++++++--- docs/turns/2026-05-19-upgrade-nextjs-16.html | 229 +++++++++++++++++++ 7 files changed, 394 insertions(+), 81 deletions(-) create mode 100644 docs/turns/2026-05-19-upgrade-nextjs-16.html diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 493492f..550d304 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -13,7 +13,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-lib","title":"Upgrade apps/web to Next.js 16.2.6","description":"Upgrade the web app dependency stack to Next.js 16.2.6 with React 19, refresh Bun and mirrored Docker workspace lockfiles, keep runtime behavior unchanged, fix any focused web test fallout, validate the web build and targeted route tests, and document the completed work.","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:04:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:04:57Z","started_at":"2026-05-19T11:04:57Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-lib","title":"Upgrade apps/web to Next.js 16.2.6","description":"Upgrade the web app dependency stack to Next.js 16.2.6 with React 19, refresh Bun and mirrored Docker workspace lockfiles, keep runtime behavior unchanged, fix any focused web test fallout, validate the web build and targeted route tests, and document the completed work.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:04:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:31:23Z","started_at":"2026-05-19T11:04:57Z","closed_at":"2026-05-19T11:31:23Z","close_reason":"Upgraded apps/web to Next.js 16.2.6 with React 19, refreshed Bun lockfiles including the Docker workspace mirror, fixed the React 19 nullable ref type issue, and validated the web build, focused tests, Docker workspace sync, and route smoke checks.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8fn","title":"implement alpaca-backed news wire view","description":"Why this issue exists and what needs to be done:\\nAdd an Alpaca-powered live news pipeline, API, storage, and web experience, including a dedicated /news route, Home preview, live fanout, history pagination, ticker resolution, and replay-mode live-only empty states.\\n\\nAcceptance criteria:\\n- normalized NewsStory contract and live channel exist\\n- ingest-news service backfills and streams Alpaca news\\n- API persists, serves, and fans out news\\n- web app exposes /news plus Home preview and drawer\\n- tests cover types, storage, API, and key UI behaviors\\n- turn documentation is added\\n\\nDesign:\\nReuse Islandflow drawer, chips, panes, and terminal styling; keep news live-only in v1 replay mode.\\n\\nNotes:\\nImplement client-side ticker filtering in v1 and expose latest revision only per provider+story_id.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T20:37:13Z","created_by":"dirtydishes","updated_at":"2026-05-18T20:55:11Z","started_at":"2026-05-18T20:37:20Z","closed_at":"2026-05-18T20:55:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-k8i","title":"Fix duplicate alert context import in API entrypoint","description":"Recent alert-context work introduced a duplicate fetchAlertContextByTraceId import in services/api/src/index.ts, which risks breaking TypeScript compilation and API startup. Remove the duplicate import and validate the affected API/web tests.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:58Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:03:40Z","started_at":"2026-05-18T13:02:02Z","closed_at":"2026-05-18T13:03:40Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lk9","title":"Fix PR creation workflow after Forgejo migration","description":"## Why\\nCreating pull requests with fails after the repository moved primary collaboration from GitHub to Forgejo. The current workflow still assumes GitHub GraphQL PR creation semantics, which do not work against the Forgejo remote.\\n\\n## What\\nInvestigate the current PR creation path, identify remaining GitHub-specific assumptions, and update the repo workflow/scripts/docs so contributors can reliably publish branches and open PRs in the Forgejo-based setup.\\n\\n## Acceptance Criteria\\n- The repo no longer instructs contributors to use a broken GitHub-specific PR creation path for Forgejo branches\\n- There is a documented and preferably scripted way to create the equivalent review request against Forgejo\\n- Validation demonstrates the new workflow behaves correctly or clearly documents any remaining platform limitation","status":"in_progress","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T10:26:47Z","created_by":"dirtydishes","updated_at":"2026-05-18T10:26:53Z","started_at":"2026-05-18T10:26:53Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 218e149..3bec184 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -1749,7 +1749,7 @@ export const getOptionTableSnapshot = ( }; type ListScrollState = { - listRef: React.RefObject; + listRef: React.RefObject; listNode: HTMLDivElement | null; setListRef: (node: HTMLDivElement | null) => void; isAtTop: boolean; @@ -1854,7 +1854,7 @@ const useListScroll = (): ListScrollState => { }; const useScrollAnchor = ( - listRef: React.RefObject, + listRef: React.RefObject, isAtTopRef: React.MutableRefObject ) => { const pendingRef = useRef<{ @@ -1996,7 +1996,7 @@ type TapeVirtualRow = { const useTapeVirtualList = ( items: T[], - listRef: React.RefObject, + listRef: React.RefObject, config: TapeVirtualListConfig ): TapeVirtualListResult => { const virtualizer = useVirtualizer({ diff --git a/apps/web/next-env.d.ts b/apps/web/next-env.d.ts index 40c3d68..9edff1c 100644 --- a/apps/web/next-env.d.ts +++ b/apps/web/next-env.d.ts @@ -1,5 +1,6 @@ /// /// +import "./.next/types/routes.d.ts"; // NOTE: This file should not be edited -// see https://nextjs.org/docs/app/building-your-application/configuring/typescript for more information. +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/apps/web/package.json b/apps/web/package.json index 8ab6906..c6a605e 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -11,13 +11,14 @@ "@islandflow/types": "workspace:*", "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", - "next": "^14.2.4", - "react": "^18.3.1", - "react-dom": "^18.3.1" + "next": "^16.2.6", + "react": "^19.2.0", + "react-dom": "^19.2.0" }, "devDependencies": { "@types/node": "^20.14.10", - "@types/react": "^18.3.3", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", "typescript": "^5.5.4" } } diff --git a/bun.lock b/bun.lock index 35e00d7..80788c9 100644 --- a/bun.lock +++ b/bun.lock @@ -26,13 +26,14 @@ "@islandflow/types": "workspace:*", "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", - "next": "^14.2.4", - "react": "^18.3.1", - "react-dom": "^18.3.1", + "next": "^16.2.6", + "react": "^19.2.0", + "react-dom": "^19.2.0", }, "devDependencies": { "@types/node": "^20.14.10", - "@types/react": "^18.3.3", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", "typescript": "^5.5.4", }, }, @@ -215,8 +216,60 @@ "@electron/windows-sign": ["@electron/windows-sign@1.2.2", "", { "dependencies": { "cross-dirname": "^0.1.0", "debug": "^4.3.4", "fs-extra": "^11.1.1", "minimist": "^1.2.8", "postject": "^1.0.0-alpha.6" }, "bin": { "electron-windows-sign": "bin/electron-windows-sign.js" } }, "sha512-dfZeox66AvdPtb2lD8OsIIQh12Tp0GNCRUDfBHIKGpbmopZto2/A8nSpYYLoedPIHpqkeblZ/k8OV0Gy7PYuyQ=="], + "@emnapi/runtime": ["@emnapi/runtime@1.10.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="], + "@gar/promisify": ["@gar/promisify@1.1.3", "", {}, "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw=="], + "@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.2.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="], + + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.2.4", "", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.2.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="], + + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="], + + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="], + + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], + "@inquirer/checkbox": ["@inquirer/checkbox@3.0.1", "", { "dependencies": { "@inquirer/core": "^9.2.1", "@inquirer/figures": "^1.0.6", "@inquirer/type": "^2.0.0", "ansi-escapes": "^4.3.2", "yoctocolors-cjs": "^2.1.2" } }, "sha512-0hm2nrToWUdD6/UHnel/UKGdk1//ke5zGUpHIvk5ZWmaKezlGxZkOJXNSWsdxO/rEqTkbB3lNC2J6nBElV2aAQ=="], "@inquirer/confirm": ["@inquirer/confirm@4.0.1", "", { "dependencies": { "@inquirer/core": "^9.2.1", "@inquirer/type": "^2.0.0" } }, "sha512-46yL28o2NJ9doViqOy0VDcoTzng7rAb6yPQKU7VDLqkmbCaH4JqK4yk4XqlzNWy9PVC5pG1ZUXPBQv+VqnYs2w=="], @@ -293,25 +346,23 @@ "@msgpack/msgpack": ["@msgpack/msgpack@3.1.3", "", {}, "sha512-47XIizs9XZXvuJgoaJUIE2lFoID8ugvc0jzSHP+Ptfk8nTbnR8g788wv48N03Kx0UkAv559HWRQ3yzOgzlRNUA=="], - "@next/env": ["@next/env@14.2.35", "", {}, "sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ=="], + "@next/env": ["@next/env@16.2.6", "", {}, "sha512-gd8HoHN4ufj73WmR3JmVolrpJR47ILK6LouP5xElPglaVxir6e1a7VzvTvDWkOoPXT9rkkTzyCxBu4yeZfZwcw=="], - "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@14.2.33", "", { "os": "darwin", "cpu": "arm64" }, "sha512-HqYnb6pxlsshoSTubdXKu15g3iivcbsMXg4bYpjL2iS/V6aQot+iyF4BUc2qA/J/n55YtvE4PHMKWBKGCF/+wA=="], + "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@16.2.6", "", { "os": "darwin", "cpu": "arm64" }, "sha512-ZJGkkcNfYgrrMkqOdZ7zoLa1TOy0qpcMfk/z4Mh/FKUz40gVO+HNQWqmLxf67Z5WB64DRp0dhEbyHfel+6sJUg=="], - "@next/swc-darwin-x64": ["@next/swc-darwin-x64@14.2.33", "", { "os": "darwin", "cpu": "x64" }, "sha512-8HGBeAE5rX3jzKvF593XTTFg3gxeU4f+UWnswa6JPhzaR6+zblO5+fjltJWIZc4aUalqTclvN2QtTC37LxvZAA=="], + "@next/swc-darwin-x64": ["@next/swc-darwin-x64@16.2.6", "", { "os": "darwin", "cpu": "x64" }, "sha512-v/YLBHIY132Ced3puBJ7YJKw1lqsCrgcNo2aRJlCEyQrrCeRJlvGlnmxhPxNQI3KE3N1DN5r9TPNPvka3nq5RQ=="], - "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@14.2.33", "", { "os": "linux", "cpu": "arm64" }, "sha512-JXMBka6lNNmqbkvcTtaX8Gu5by9547bukHQvPoLe9VRBx1gHwzf5tdt4AaezW85HAB3pikcvyqBToRTDA4DeLw=="], + "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@16.2.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-RPOvqlYBbcQjkz9VQQDZ2T2bARIjXZV1KFlt+V2Mr6SW/e4I9fcKsaA0hdyf2FHoTlsV2xnBd5Y912rP/1Ce6w=="], - "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@14.2.33", "", { "os": "linux", "cpu": "arm64" }, "sha512-Bm+QulsAItD/x6Ih8wGIMfRJy4G73tu1HJsrccPW6AfqdZd0Sfm5Imhgkgq2+kly065rYMnCOxTBvmvFY1BKfg=="], + "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@16.2.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-URUTu1+dMkxJsPFgm+OeEvq9wf5sujw0EvgYy80TDGHTSLTnIHeqb0Eu8A3sC95IRgjejQL+kC4mw+4yPxiAXA=="], - "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@14.2.33", "", { "os": "linux", "cpu": "x64" }, "sha512-FnFn+ZBgsVMbGDsTqo8zsnRzydvsGV8vfiWwUo1LD8FTmPTdV+otGSWKc4LJec0oSexFnCYVO4hX8P8qQKaSlg=="], + "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@16.2.6", "", { "os": "linux", "cpu": "x64" }, "sha512-DOj182mPV8G3UkrayLoREM5YEYI+Dk5wv7Ox9xl1fFibAELEsFD0lDPfHIeILlutMMfdyhlzYPELG3peuKaurw=="], - "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@14.2.33", "", { "os": "linux", "cpu": "x64" }, "sha512-345tsIWMzoXaQndUTDv1qypDRiebFxGYx9pYkhwY4hBRaOLt8UGfiWKr9FSSHs25dFIf8ZqIFaPdy5MljdoawA=="], + "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@16.2.6", "", { "os": "linux", "cpu": "x64" }, "sha512-HKQ5SP/V/ub73UvF7n/zeJlxk2kLmtL7Wzrg4WfmkjmNos5onJ2tKu7yZOPdL18A6Svfn3max29ym+ry7NkK4g=="], - "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@14.2.33", "", { "os": "win32", "cpu": "arm64" }, "sha512-nscpt0G6UCTkrT2ppnJnFsYbPDQwmum4GNXYTeoTIdsmMydSKFz9Iny2jpaRupTb+Wl298+Rh82WKzt9LCcqSQ=="], + "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@16.2.6", "", { "os": "win32", "cpu": "arm64" }, "sha512-LZXpTlPyS5v7HhSmnvsLGP3iIYgYOBnc8r8ArlT55sGHV89bR2HlDdBjWQ+PY6SJMmk8TuVGFuxalnP3k/0Dwg=="], - "@next/swc-win32-ia32-msvc": ["@next/swc-win32-ia32-msvc@14.2.33", "", { "os": "win32", "cpu": "ia32" }, "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q=="], - - "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@14.2.33", "", { "os": "win32", "cpu": "x64" }, "sha512-nOjfZMy8B94MdisuzZo9/57xuFVLHJaDj5e/xrduJp9CV2/HrfxTRH2fbyLe+K9QT41WBLUd4iXX3R7jBp0EUg=="], + "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@16.2.6", "", { "os": "win32", "cpu": "x64" }, "sha512-F0+4i0h9J6C4eE3EAPWsoCk7UW/dbzOjyzxY0qnDUOYFu6FFmdZ6l97/XdV3/Nz3VYyO7UWjyEJUXkGqcoXfMA=="], "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], @@ -335,9 +386,7 @@ "@sindresorhus/is": ["@sindresorhus/is@4.6.0", "", {}, "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw=="], - "@swc/counter": ["@swc/counter@0.1.3", "", {}, "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="], - - "@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="], + "@swc/helpers": ["@swc/helpers@0.5.15", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g=="], "@szmarczak/http-timer": ["@szmarczak/http-timer@4.0.6", "", { "dependencies": { "defer-to-connect": "^2.0.0" } }, "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w=="], @@ -365,9 +414,9 @@ "@types/node": ["@types/node@24.12.4", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GUUEShf+PBCGW2KaXwcIt3Yk+e3pkKwWKb9GSyM9WQVE+ep2jzmHdGsHzu4wgcZy5fN9FBdVzjpBQsYlpfpgLA=="], - "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], + "@types/react": ["@types/react@19.2.14", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w=="], - "@types/react": ["@types/react@18.3.27", "", { "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" } }, "sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w=="], + "@types/react-dom": ["@types/react-dom@19.2.3", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="], "@types/responselike": ["@types/responselike@1.0.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw=="], @@ -465,15 +514,13 @@ "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], - "busboy": ["busboy@1.6.0", "", { "dependencies": { "streamsearch": "^1.1.0" } }, "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA=="], - "cacache": ["cacache@16.1.3", "", { "dependencies": { "@npmcli/fs": "^2.1.0", "@npmcli/move-file": "^2.0.0", "chownr": "^2.0.0", "fs-minipass": "^2.1.0", "glob": "^8.0.1", "infer-owner": "^1.0.4", "lru-cache": "^7.7.1", "minipass": "^3.1.6", "minipass-collect": "^1.0.2", "minipass-flush": "^1.0.5", "minipass-pipeline": "^1.2.4", "mkdirp": "^1.0.4", "p-map": "^4.0.0", "promise-inflight": "^1.0.1", "rimraf": "^3.0.2", "ssri": "^9.0.0", "tar": "^6.1.11", "unique-filename": "^2.0.0" } }, "sha512-/+Emcj9DAXxX4cwlLmRI9c166RuL3w30zp4R7Joiv2cQTtTtA+jeuCAjH3ZlGnYS3tKENSrKhAzVVP9GVyzeYQ=="], "cacheable-lookup": ["cacheable-lookup@5.0.4", "", {}, "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA=="], "cacheable-request": ["cacheable-request@7.0.4", "", { "dependencies": { "clone-response": "^1.0.2", "get-stream": "^5.1.0", "http-cache-semantics": "^4.0.0", "keyv": "^4.0.0", "lowercase-keys": "^2.0.0", "normalize-url": "^6.0.1", "responselike": "^2.0.0" } }, "sha512-v+p6ongsrp0yTGbJXjgxPow2+DL93DASP4kXCDKb8/bwRtt9OEF3whggkkDkGNzgcWy2XaF4a8nZglC7uElscg=="], - "caniuse-lite": ["caniuse-lite@1.0.30001761", "", {}, "sha512-JF9ptu1vP2coz98+5051jZ4PwQgd2ni8A+gYSN7EA7dPKIMf0pDlSUxhdmVOaV3/fYK5uWBkgSXJaRLr4+3A6g=="], + "caniuse-lite": ["caniuse-lite@1.0.30001792", "", {}, "sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw=="], "chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], @@ -713,8 +760,6 @@ "jiti": ["jiti@2.7.0", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="], - "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], - "json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="], "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], @@ -745,8 +790,6 @@ "log-update": ["log-update@5.0.1", "", { "dependencies": { "ansi-escapes": "^5.0.0", "cli-cursor": "^4.0.0", "slice-ansi": "^5.0.0", "strip-ansi": "^7.0.1", "wrap-ansi": "^8.0.1" } }, "sha512-5UtUDQ/6edw4ofyljDNcOVJQ4c7OjDro4h3y8e1GQL5iYElYclVHJ3zeWchylvMaKnDbDilC8irOVyexnA/Slw=="], - "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], - "lowercase-keys": ["lowercase-keys@2.0.0", "", {}, "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="], "lru-cache": ["lru-cache@7.18.3", "", {}, "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="], @@ -803,7 +846,7 @@ "neo-async": ["neo-async@2.6.2", "", {}, "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="], - "next": ["next@14.2.35", "", { "dependencies": { "@next/env": "14.2.35", "@swc/helpers": "0.5.5", "busboy": "1.6.0", "caniuse-lite": "^1.0.30001579", "graceful-fs": "^4.2.11", "postcss": "8.4.31", "styled-jsx": "5.1.1" }, "optionalDependencies": { "@next/swc-darwin-arm64": "14.2.33", "@next/swc-darwin-x64": "14.2.33", "@next/swc-linux-arm64-gnu": "14.2.33", "@next/swc-linux-arm64-musl": "14.2.33", "@next/swc-linux-x64-gnu": "14.2.33", "@next/swc-linux-x64-musl": "14.2.33", "@next/swc-win32-arm64-msvc": "14.2.33", "@next/swc-win32-ia32-msvc": "14.2.33", "@next/swc-win32-x64-msvc": "14.2.33" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.41.2", "react": "^18.2.0", "react-dom": "^18.2.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-KhYd2Hjt/O1/1aZVX3dCwGXM1QmOV4eNM2UTacK5gipDdPN/oHHK/4oVGy7X8GMfPMsUTUEmGlsy0EY1YGAkig=="], + "next": ["next@16.2.6", "", { "dependencies": { "@next/env": "16.2.6", "@swc/helpers": "0.5.15", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", "styled-jsx": "5.1.6" }, "optionalDependencies": { "@next/swc-darwin-arm64": "16.2.6", "@next/swc-darwin-x64": "16.2.6", "@next/swc-linux-arm64-gnu": "16.2.6", "@next/swc-linux-arm64-musl": "16.2.6", "@next/swc-linux-x64-gnu": "16.2.6", "@next/swc-linux-x64-musl": "16.2.6", "@next/swc-win32-arm64-msvc": "16.2.6", "@next/swc-win32-x64-msvc": "16.2.6", "sharp": "^0.34.5" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "babel-plugin-react-compiler", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-qOVgKJg1+At15NpeUP+eJgCHvTCgXsogweq87Ri/Ix7PkqQHg4sdaXmSFqKlgaIXE4kW0g25LE68W87UANlHtw=="], "nice-try": ["nice-try@1.0.5", "", {}, "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ=="], @@ -897,9 +940,9 @@ "quick-lru": ["quick-lru@5.1.1", "", {}, "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA=="], - "react": ["react@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="], + "react": ["react@19.2.6", "", {}, "sha512-sfWGGfavi0xr8Pg0sVsyHMAOziVYKgPLNrS7ig+ivMNb3wbCBw3KxtflsGBAwD3gYQlE/AEZsTLgToRrSCjb0Q=="], - "react-dom": ["react-dom@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" }, "peerDependencies": { "react": "^18.3.1" } }, "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw=="], + "react-dom": ["react-dom@19.2.6", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.6" } }, "sha512-0prMI+hvBbPjsWnxDLxlCGyM8PN6UuWjEUCYmZhO67xIV9Xasa/r/vDnq+Xyq4Lo27g8QSbO5YzARu0D1Sps3g=="], "read-binary-file-arch": ["read-binary-file-arch@1.0.6", "", { "dependencies": { "debug": "^4.3.4" }, "bin": { "read-binary-file-arch": "cli.js" } }, "sha512-BNg9EN3DD3GsDXX7Aa8O4p92sryjkmzYYgmgTAc6CA4uGLEDzFfxOxugu21akOxpcXHiEgsYkC6nPsQvLLLmEg=="], @@ -943,7 +986,7 @@ "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], - "scheduler": ["scheduler@0.23.2", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ=="], + "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], "schema-utils": ["schema-utils@4.3.3", "", { "dependencies": { "@types/json-schema": "^7.0.9", "ajv": "^8.9.0", "ajv-formats": "^2.1.1", "ajv-keywords": "^5.1.0" } }, "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA=="], @@ -953,6 +996,8 @@ "serialize-error": ["serialize-error@7.0.1", "", { "dependencies": { "type-fest": "^0.13.1" } }, "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw=="], + "sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], @@ -985,8 +1030,6 @@ "ssri": ["ssri@9.0.1", "", { "dependencies": { "minipass": "^3.1.1" } }, "sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q=="], - "streamsearch": ["streamsearch@1.1.0", "", {}, "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg=="], - "string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], @@ -999,7 +1042,7 @@ "strip-outer": ["strip-outer@1.0.1", "", { "dependencies": { "escape-string-regexp": "^1.0.2" } }, "sha512-k55yxKHwaXnpYGsOzg4Vl8+tDrWylxDEpknGjhTiZB8dFRU5rTo9CAzeycivxV3s+zlTKwrs6WxMxR95n26kwg=="], - "styled-jsx": ["styled-jsx@5.1.1", "", { "dependencies": { "client-only": "0.0.1" }, "peerDependencies": { "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0" } }, "sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw=="], + "styled-jsx": ["styled-jsx@5.1.6", "", { "dependencies": { "client-only": "0.0.1" }, "peerDependencies": { "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0" } }, "sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA=="], "sumchecker": ["sumchecker@3.0.1", "", { "dependencies": { "debug": "^4.1.0" } }, "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg=="], @@ -1141,8 +1184,6 @@ "ansi-escapes/type-fest": ["type-fest@0.21.3", "", {}, "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w=="], - "browserslist/caniuse-lite": ["caniuse-lite@1.0.30001792", "", {}, "sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw=="], - "cacache/glob": ["glob@8.1.0", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^5.0.1", "once": "^1.3.0" } }, "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ=="], "cacache/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="], diff --git a/deployment/docker/workspace-root/bun.lock b/deployment/docker/workspace-root/bun.lock index 35e00d7..80788c9 100644 --- a/deployment/docker/workspace-root/bun.lock +++ b/deployment/docker/workspace-root/bun.lock @@ -26,13 +26,14 @@ "@islandflow/types": "workspace:*", "@tanstack/react-virtual": "^3.13.24", "lightweight-charts": "^4.2.0", - "next": "^14.2.4", - "react": "^18.3.1", - "react-dom": "^18.3.1", + "next": "^16.2.6", + "react": "^19.2.0", + "react-dom": "^19.2.0", }, "devDependencies": { "@types/node": "^20.14.10", - "@types/react": "^18.3.3", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", "typescript": "^5.5.4", }, }, @@ -215,8 +216,60 @@ "@electron/windows-sign": ["@electron/windows-sign@1.2.2", "", { "dependencies": { "cross-dirname": "^0.1.0", "debug": "^4.3.4", "fs-extra": "^11.1.1", "minimist": "^1.2.8", "postject": "^1.0.0-alpha.6" }, "bin": { "electron-windows-sign": "bin/electron-windows-sign.js" } }, "sha512-dfZeox66AvdPtb2lD8OsIIQh12Tp0GNCRUDfBHIKGpbmopZto2/A8nSpYYLoedPIHpqkeblZ/k8OV0Gy7PYuyQ=="], + "@emnapi/runtime": ["@emnapi/runtime@1.10.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="], + "@gar/promisify": ["@gar/promisify@1.1.3", "", {}, "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw=="], + "@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.2.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="], + + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.2.4", "", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.2.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="], + + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="], + + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="], + + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], + "@inquirer/checkbox": ["@inquirer/checkbox@3.0.1", "", { "dependencies": { "@inquirer/core": "^9.2.1", "@inquirer/figures": "^1.0.6", "@inquirer/type": "^2.0.0", "ansi-escapes": "^4.3.2", "yoctocolors-cjs": "^2.1.2" } }, "sha512-0hm2nrToWUdD6/UHnel/UKGdk1//ke5zGUpHIvk5ZWmaKezlGxZkOJXNSWsdxO/rEqTkbB3lNC2J6nBElV2aAQ=="], "@inquirer/confirm": ["@inquirer/confirm@4.0.1", "", { "dependencies": { "@inquirer/core": "^9.2.1", "@inquirer/type": "^2.0.0" } }, "sha512-46yL28o2NJ9doViqOy0VDcoTzng7rAb6yPQKU7VDLqkmbCaH4JqK4yk4XqlzNWy9PVC5pG1ZUXPBQv+VqnYs2w=="], @@ -293,25 +346,23 @@ "@msgpack/msgpack": ["@msgpack/msgpack@3.1.3", "", {}, "sha512-47XIizs9XZXvuJgoaJUIE2lFoID8ugvc0jzSHP+Ptfk8nTbnR8g788wv48N03Kx0UkAv559HWRQ3yzOgzlRNUA=="], - "@next/env": ["@next/env@14.2.35", "", {}, "sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ=="], + "@next/env": ["@next/env@16.2.6", "", {}, "sha512-gd8HoHN4ufj73WmR3JmVolrpJR47ILK6LouP5xElPglaVxir6e1a7VzvTvDWkOoPXT9rkkTzyCxBu4yeZfZwcw=="], - "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@14.2.33", "", { "os": "darwin", "cpu": "arm64" }, "sha512-HqYnb6pxlsshoSTubdXKu15g3iivcbsMXg4bYpjL2iS/V6aQot+iyF4BUc2qA/J/n55YtvE4PHMKWBKGCF/+wA=="], + "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@16.2.6", "", { "os": "darwin", "cpu": "arm64" }, "sha512-ZJGkkcNfYgrrMkqOdZ7zoLa1TOy0qpcMfk/z4Mh/FKUz40gVO+HNQWqmLxf67Z5WB64DRp0dhEbyHfel+6sJUg=="], - "@next/swc-darwin-x64": ["@next/swc-darwin-x64@14.2.33", "", { "os": "darwin", "cpu": "x64" }, "sha512-8HGBeAE5rX3jzKvF593XTTFg3gxeU4f+UWnswa6JPhzaR6+zblO5+fjltJWIZc4aUalqTclvN2QtTC37LxvZAA=="], + "@next/swc-darwin-x64": ["@next/swc-darwin-x64@16.2.6", "", { "os": "darwin", "cpu": "x64" }, "sha512-v/YLBHIY132Ced3puBJ7YJKw1lqsCrgcNo2aRJlCEyQrrCeRJlvGlnmxhPxNQI3KE3N1DN5r9TPNPvka3nq5RQ=="], - "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@14.2.33", "", { "os": "linux", "cpu": "arm64" }, "sha512-JXMBka6lNNmqbkvcTtaX8Gu5by9547bukHQvPoLe9VRBx1gHwzf5tdt4AaezW85HAB3pikcvyqBToRTDA4DeLw=="], + "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@16.2.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-RPOvqlYBbcQjkz9VQQDZ2T2bARIjXZV1KFlt+V2Mr6SW/e4I9fcKsaA0hdyf2FHoTlsV2xnBd5Y912rP/1Ce6w=="], - "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@14.2.33", "", { "os": "linux", "cpu": "arm64" }, "sha512-Bm+QulsAItD/x6Ih8wGIMfRJy4G73tu1HJsrccPW6AfqdZd0Sfm5Imhgkgq2+kly065rYMnCOxTBvmvFY1BKfg=="], + "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@16.2.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-URUTu1+dMkxJsPFgm+OeEvq9wf5sujw0EvgYy80TDGHTSLTnIHeqb0Eu8A3sC95IRgjejQL+kC4mw+4yPxiAXA=="], - "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@14.2.33", "", { "os": "linux", "cpu": "x64" }, "sha512-FnFn+ZBgsVMbGDsTqo8zsnRzydvsGV8vfiWwUo1LD8FTmPTdV+otGSWKc4LJec0oSexFnCYVO4hX8P8qQKaSlg=="], + "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@16.2.6", "", { "os": "linux", "cpu": "x64" }, "sha512-DOj182mPV8G3UkrayLoREM5YEYI+Dk5wv7Ox9xl1fFibAELEsFD0lDPfHIeILlutMMfdyhlzYPELG3peuKaurw=="], - "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@14.2.33", "", { "os": "linux", "cpu": "x64" }, "sha512-345tsIWMzoXaQndUTDv1qypDRiebFxGYx9pYkhwY4hBRaOLt8UGfiWKr9FSSHs25dFIf8ZqIFaPdy5MljdoawA=="], + "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@16.2.6", "", { "os": "linux", "cpu": "x64" }, "sha512-HKQ5SP/V/ub73UvF7n/zeJlxk2kLmtL7Wzrg4WfmkjmNos5onJ2tKu7yZOPdL18A6Svfn3max29ym+ry7NkK4g=="], - "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@14.2.33", "", { "os": "win32", "cpu": "arm64" }, "sha512-nscpt0G6UCTkrT2ppnJnFsYbPDQwmum4GNXYTeoTIdsmMydSKFz9Iny2jpaRupTb+Wl298+Rh82WKzt9LCcqSQ=="], + "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@16.2.6", "", { "os": "win32", "cpu": "arm64" }, "sha512-LZXpTlPyS5v7HhSmnvsLGP3iIYgYOBnc8r8ArlT55sGHV89bR2HlDdBjWQ+PY6SJMmk8TuVGFuxalnP3k/0Dwg=="], - "@next/swc-win32-ia32-msvc": ["@next/swc-win32-ia32-msvc@14.2.33", "", { "os": "win32", "cpu": "ia32" }, "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q=="], - - "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@14.2.33", "", { "os": "win32", "cpu": "x64" }, "sha512-nOjfZMy8B94MdisuzZo9/57xuFVLHJaDj5e/xrduJp9CV2/HrfxTRH2fbyLe+K9QT41WBLUd4iXX3R7jBp0EUg=="], + "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@16.2.6", "", { "os": "win32", "cpu": "x64" }, "sha512-F0+4i0h9J6C4eE3EAPWsoCk7UW/dbzOjyzxY0qnDUOYFu6FFmdZ6l97/XdV3/Nz3VYyO7UWjyEJUXkGqcoXfMA=="], "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], @@ -335,9 +386,7 @@ "@sindresorhus/is": ["@sindresorhus/is@4.6.0", "", {}, "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw=="], - "@swc/counter": ["@swc/counter@0.1.3", "", {}, "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="], - - "@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="], + "@swc/helpers": ["@swc/helpers@0.5.15", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g=="], "@szmarczak/http-timer": ["@szmarczak/http-timer@4.0.6", "", { "dependencies": { "defer-to-connect": "^2.0.0" } }, "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w=="], @@ -365,9 +414,9 @@ "@types/node": ["@types/node@24.12.4", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GUUEShf+PBCGW2KaXwcIt3Yk+e3pkKwWKb9GSyM9WQVE+ep2jzmHdGsHzu4wgcZy5fN9FBdVzjpBQsYlpfpgLA=="], - "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], + "@types/react": ["@types/react@19.2.14", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w=="], - "@types/react": ["@types/react@18.3.27", "", { "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" } }, "sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w=="], + "@types/react-dom": ["@types/react-dom@19.2.3", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="], "@types/responselike": ["@types/responselike@1.0.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw=="], @@ -465,15 +514,13 @@ "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], - "busboy": ["busboy@1.6.0", "", { "dependencies": { "streamsearch": "^1.1.0" } }, "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA=="], - "cacache": ["cacache@16.1.3", "", { "dependencies": { "@npmcli/fs": "^2.1.0", "@npmcli/move-file": "^2.0.0", "chownr": "^2.0.0", "fs-minipass": "^2.1.0", "glob": "^8.0.1", "infer-owner": "^1.0.4", "lru-cache": "^7.7.1", "minipass": "^3.1.6", "minipass-collect": "^1.0.2", "minipass-flush": "^1.0.5", "minipass-pipeline": "^1.2.4", "mkdirp": "^1.0.4", "p-map": "^4.0.0", "promise-inflight": "^1.0.1", "rimraf": "^3.0.2", "ssri": "^9.0.0", "tar": "^6.1.11", "unique-filename": "^2.0.0" } }, "sha512-/+Emcj9DAXxX4cwlLmRI9c166RuL3w30zp4R7Joiv2cQTtTtA+jeuCAjH3ZlGnYS3tKENSrKhAzVVP9GVyzeYQ=="], "cacheable-lookup": ["cacheable-lookup@5.0.4", "", {}, "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA=="], "cacheable-request": ["cacheable-request@7.0.4", "", { "dependencies": { "clone-response": "^1.0.2", "get-stream": "^5.1.0", "http-cache-semantics": "^4.0.0", "keyv": "^4.0.0", "lowercase-keys": "^2.0.0", "normalize-url": "^6.0.1", "responselike": "^2.0.0" } }, "sha512-v+p6ongsrp0yTGbJXjgxPow2+DL93DASP4kXCDKb8/bwRtt9OEF3whggkkDkGNzgcWy2XaF4a8nZglC7uElscg=="], - "caniuse-lite": ["caniuse-lite@1.0.30001761", "", {}, "sha512-JF9ptu1vP2coz98+5051jZ4PwQgd2ni8A+gYSN7EA7dPKIMf0pDlSUxhdmVOaV3/fYK5uWBkgSXJaRLr4+3A6g=="], + "caniuse-lite": ["caniuse-lite@1.0.30001792", "", {}, "sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw=="], "chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], @@ -713,8 +760,6 @@ "jiti": ["jiti@2.7.0", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="], - "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], - "json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="], "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], @@ -745,8 +790,6 @@ "log-update": ["log-update@5.0.1", "", { "dependencies": { "ansi-escapes": "^5.0.0", "cli-cursor": "^4.0.0", "slice-ansi": "^5.0.0", "strip-ansi": "^7.0.1", "wrap-ansi": "^8.0.1" } }, "sha512-5UtUDQ/6edw4ofyljDNcOVJQ4c7OjDro4h3y8e1GQL5iYElYclVHJ3zeWchylvMaKnDbDilC8irOVyexnA/Slw=="], - "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], - "lowercase-keys": ["lowercase-keys@2.0.0", "", {}, "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="], "lru-cache": ["lru-cache@7.18.3", "", {}, "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="], @@ -803,7 +846,7 @@ "neo-async": ["neo-async@2.6.2", "", {}, "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="], - "next": ["next@14.2.35", "", { "dependencies": { "@next/env": "14.2.35", "@swc/helpers": "0.5.5", "busboy": "1.6.0", "caniuse-lite": "^1.0.30001579", "graceful-fs": "^4.2.11", "postcss": "8.4.31", "styled-jsx": "5.1.1" }, "optionalDependencies": { "@next/swc-darwin-arm64": "14.2.33", "@next/swc-darwin-x64": "14.2.33", "@next/swc-linux-arm64-gnu": "14.2.33", "@next/swc-linux-arm64-musl": "14.2.33", "@next/swc-linux-x64-gnu": "14.2.33", "@next/swc-linux-x64-musl": "14.2.33", "@next/swc-win32-arm64-msvc": "14.2.33", "@next/swc-win32-ia32-msvc": "14.2.33", "@next/swc-win32-x64-msvc": "14.2.33" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.41.2", "react": "^18.2.0", "react-dom": "^18.2.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-KhYd2Hjt/O1/1aZVX3dCwGXM1QmOV4eNM2UTacK5gipDdPN/oHHK/4oVGy7X8GMfPMsUTUEmGlsy0EY1YGAkig=="], + "next": ["next@16.2.6", "", { "dependencies": { "@next/env": "16.2.6", "@swc/helpers": "0.5.15", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", "styled-jsx": "5.1.6" }, "optionalDependencies": { "@next/swc-darwin-arm64": "16.2.6", "@next/swc-darwin-x64": "16.2.6", "@next/swc-linux-arm64-gnu": "16.2.6", "@next/swc-linux-arm64-musl": "16.2.6", "@next/swc-linux-x64-gnu": "16.2.6", "@next/swc-linux-x64-musl": "16.2.6", "@next/swc-win32-arm64-msvc": "16.2.6", "@next/swc-win32-x64-msvc": "16.2.6", "sharp": "^0.34.5" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "babel-plugin-react-compiler", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-qOVgKJg1+At15NpeUP+eJgCHvTCgXsogweq87Ri/Ix7PkqQHg4sdaXmSFqKlgaIXE4kW0g25LE68W87UANlHtw=="], "nice-try": ["nice-try@1.0.5", "", {}, "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ=="], @@ -897,9 +940,9 @@ "quick-lru": ["quick-lru@5.1.1", "", {}, "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA=="], - "react": ["react@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="], + "react": ["react@19.2.6", "", {}, "sha512-sfWGGfavi0xr8Pg0sVsyHMAOziVYKgPLNrS7ig+ivMNb3wbCBw3KxtflsGBAwD3gYQlE/AEZsTLgToRrSCjb0Q=="], - "react-dom": ["react-dom@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" }, "peerDependencies": { "react": "^18.3.1" } }, "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw=="], + "react-dom": ["react-dom@19.2.6", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.6" } }, "sha512-0prMI+hvBbPjsWnxDLxlCGyM8PN6UuWjEUCYmZhO67xIV9Xasa/r/vDnq+Xyq4Lo27g8QSbO5YzARu0D1Sps3g=="], "read-binary-file-arch": ["read-binary-file-arch@1.0.6", "", { "dependencies": { "debug": "^4.3.4" }, "bin": { "read-binary-file-arch": "cli.js" } }, "sha512-BNg9EN3DD3GsDXX7Aa8O4p92sryjkmzYYgmgTAc6CA4uGLEDzFfxOxugu21akOxpcXHiEgsYkC6nPsQvLLLmEg=="], @@ -943,7 +986,7 @@ "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], - "scheduler": ["scheduler@0.23.2", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ=="], + "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], "schema-utils": ["schema-utils@4.3.3", "", { "dependencies": { "@types/json-schema": "^7.0.9", "ajv": "^8.9.0", "ajv-formats": "^2.1.1", "ajv-keywords": "^5.1.0" } }, "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA=="], @@ -953,6 +996,8 @@ "serialize-error": ["serialize-error@7.0.1", "", { "dependencies": { "type-fest": "^0.13.1" } }, "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw=="], + "sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], @@ -985,8 +1030,6 @@ "ssri": ["ssri@9.0.1", "", { "dependencies": { "minipass": "^3.1.1" } }, "sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q=="], - "streamsearch": ["streamsearch@1.1.0", "", {}, "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg=="], - "string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], @@ -999,7 +1042,7 @@ "strip-outer": ["strip-outer@1.0.1", "", { "dependencies": { "escape-string-regexp": "^1.0.2" } }, "sha512-k55yxKHwaXnpYGsOzg4Vl8+tDrWylxDEpknGjhTiZB8dFRU5rTo9CAzeycivxV3s+zlTKwrs6WxMxR95n26kwg=="], - "styled-jsx": ["styled-jsx@5.1.1", "", { "dependencies": { "client-only": "0.0.1" }, "peerDependencies": { "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0" } }, "sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw=="], + "styled-jsx": ["styled-jsx@5.1.6", "", { "dependencies": { "client-only": "0.0.1" }, "peerDependencies": { "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0" } }, "sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA=="], "sumchecker": ["sumchecker@3.0.1", "", { "dependencies": { "debug": "^4.1.0" } }, "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg=="], @@ -1141,8 +1184,6 @@ "ansi-escapes/type-fest": ["type-fest@0.21.3", "", {}, "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w=="], - "browserslist/caniuse-lite": ["caniuse-lite@1.0.30001792", "", {}, "sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw=="], - "cacache/glob": ["glob@8.1.0", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^5.0.1", "once": "^1.3.0" } }, "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ=="], "cacache/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="], diff --git a/docs/turns/2026-05-19-upgrade-nextjs-16.html b/docs/turns/2026-05-19-upgrade-nextjs-16.html new file mode 100644 index 0000000..cdbb2f1 --- /dev/null +++ b/docs/turns/2026-05-19-upgrade-nextjs-16.html @@ -0,0 +1,229 @@ + + + + + + Upgrade apps/web to Next.js 16.2.6 + + + + + + +
+
+

Turn document · 2026-05-19

+

Upgrade apps/web to Next.js 16.2.6

+

The web app now builds and passes focused validation on Next.js 16.2.6 with React 19. The change keeps route behavior and synthetic admin proxy behavior intact while refreshing the root and Docker workspace Bun lockfiles.

+
+ +
+

Summary

+

Upgraded apps/web from the Next 14 / React 18 stack to Next 16.2.6 and React 19.2.x. The Bun lockfile was refreshed, the Docker workspace lock snapshot was synced, and a React 19 nullable ref type issue exposed by the Next 16 build was fixed.

+
+ +
+

Changes Made

+
    +
  • Updated apps/web/package.json to request next ^16.2.6, react ^19.2.0, and react-dom ^19.2.0.
    • +
  • Updated React type dependencies to @types/react ^19.2.7 and added @types/react-dom ^19.2.3.
    • +
  • Ran bun install, which resolved Next to 16.2.6 and React/React DOM to 19.2.6 in bun.lock.
    • +
  • Ran bun run sync:docker-workspace so deployment/docker/workspace-root/bun.lock matches the root lock snapshot.
    • +
  • Adjusted the terminal list ref types to accept HTMLDivElement | null, matching React 19's stricter ref object typing.
    • +
  • Allowed Next 16 to regenerate apps/web/next-env.d.ts with its updated TypeScript reference comment and generated route type import.
    • +
+
+ +
+

Context

+

The requested upgrade was intentionally dependency-focused. No routes, backend contracts, environment variable names, or shared package exports were changed. Before editing, the web build and the targeted route tests passed on the previous locked Next 14.2.35 stack.

+
+ +
+

Important Implementation Details

+

No broad codemod was run. The only source-code change was a targeted type correction in apps/web/app/terminal.tsx. Next 16's build now runs with Turbopack by default in this project and completed successfully after the ref typing was narrowed to the actual nullable runtime value.

+

The Docker workspace sync changed the mirrored lockfile, but did not need to rewrite the mirrored package manifest or TypeScript base config.

+
+ +
+

Relevant Diff Snippets

+ 
"next": "^16.2.6",
+"react": "^19.2.0",
+"react-dom": "^19.2.0",
+"@types/react": "^19.2.7",
+"@types/react-dom": "^19.2.3"
+ 
type ListScrollState = {
+  listRef: React.RefObject<HTMLDivElement | null>;
+  listNode: HTMLDivElement | null;
+  setListRef: (node: HTMLDivElement | null) => void;
+};
+
+ +
+

Expected Impact for End-Users

+

There should be no intentional user-facing behavior change. The expected visible behavior remains: /, /tape, and /news render the terminal app; /signals, /charts, and /replay redirect to /; synthetic admin API routes keep their gated proxy behavior.

+
+ +
+

Validation

+
    +
  • Baseline before edits: bun --cwd=apps/web run build passed on Next 14.2.35.
    • +
  • Baseline before edits: bun test apps/web/app/routes.test.ts passed, 3 tests.
    • +
  • Baseline before edits: bun test apps/web/app/terminal.test.ts passed, 70 tests.
    • +
  • Baseline before edits: bun test apps/web/app/api/admin/synthetic/routes.test.ts passed, 4 tests.
    • +
  • After upgrade: bun --cwd=apps/web run build passed on Next 16.2.6.
    • +
  • After upgrade: bun test apps/web/app/routes.test.ts passed, 3 tests.
    • +
  • After upgrade: bun test apps/web/app/terminal.test.ts passed, 70 tests.
    • +
  • After upgrade: bun test apps/web/app/api/admin/synthetic/routes.test.ts passed, 4 tests.
    • +
  • After upgrade: bun run check:docker-workspace passed.
    • +
  • Manual smoke: bun run dev:web served Next 16.2.6 on localhost:3000.
    • +
  • Manual smoke: browser checks confirmed /, /tape, and /news render with title Islandflow Terminal.
    • +
  • Manual smoke: /signals, /charts, and /replay returned 307 redirects to /.
    • +
  • Manual smoke: synthetic admin status and control routes returned gated 404 responses when the internal UI flag was off.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+

During dev:web smoke testing, the browser logged a live socket channel validation warning because only the web app was running, not the full backend service stack. Route rendering, redirect behavior, and gated synthetic admin proxy behavior were still verified. A full-stack live feed verification can be done separately with bun run dev if needed.

+

The upgrade did not include a full monorepo test run because the acceptance bar was intentionally web-focused.

+
+ +
+

Follow-up Work

+
    +
  • No required follow-up Beads issue was opened for this upgrade.
    • +
  • Optional: run a full-stack live feed smoke with infra and services running if you want runtime stream confidence beyond the web-focused acceptance checks.
    • +
  • Optional: run the full monorepo bun test suite before a larger release branch merge.
    • +
+
+ +
+

Helpful Links

+ +
+
+ + From 82fd29f1a451a46b7f90a89201e78966581aebe7 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 07:40:18 -0400 Subject: [PATCH 09/26] update readme for current project state --- .beads/issues.jsonl | 1 + README.md | 389 +++++++++--------- ...5-19-0739-update-readme-current-state.html | 259 ++++++++++++ 3 files changed, 449 insertions(+), 200 deletions(-) create mode 100644 docs/turns/2026-05-19-0739-update-readme-current-state.html diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 550d304..61aef8b 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -13,6 +13,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-6iq","title":"Update README for current project state","description":"Resolve README merge conflicts and document the current project state, including the smart money classification taxonomy, Next.js update, and deployment workflow changes.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:37:24Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:40:01Z","started_at":"2026-05-19T11:37:31Z","closed_at":"2026-05-19T11:40:01Z","close_reason":"README conflict resolved and current project state documented, including smart-money taxonomy, Next.js update, and deployment workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lib","title":"Upgrade apps/web to Next.js 16.2.6","description":"Upgrade the web app dependency stack to Next.js 16.2.6 with React 19, refresh Bun and mirrored Docker workspace lockfiles, keep runtime behavior unchanged, fix any focused web test fallout, validate the web build and targeted route tests, and document the completed work.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:04:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:31:23Z","started_at":"2026-05-19T11:04:57Z","closed_at":"2026-05-19T11:31:23Z","close_reason":"Upgraded apps/web to Next.js 16.2.6 with React 19, refreshed Bun lockfiles including the Docker workspace mirror, fixed the React 19 nullable ref type issue, and validated the web build, focused tests, Docker workspace sync, and route smoke checks.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8fn","title":"implement alpaca-backed news wire view","description":"Why this issue exists and what needs to be done:\\nAdd an Alpaca-powered live news pipeline, API, storage, and web experience, including a dedicated /news route, Home preview, live fanout, history pagination, ticker resolution, and replay-mode live-only empty states.\\n\\nAcceptance criteria:\\n- normalized NewsStory contract and live channel exist\\n- ingest-news service backfills and streams Alpaca news\\n- API persists, serves, and fans out news\\n- web app exposes /news plus Home preview and drawer\\n- tests cover types, storage, API, and key UI behaviors\\n- turn documentation is added\\n\\nDesign:\\nReuse Islandflow drawer, chips, panes, and terminal styling; keep news live-only in v1 replay mode.\\n\\nNotes:\\nImplement client-side ticker filtering in v1 and expose latest revision only per provider+story_id.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T20:37:13Z","created_by":"dirtydishes","updated_at":"2026-05-18T20:55:11Z","started_at":"2026-05-18T20:37:20Z","closed_at":"2026-05-18T20:55:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-k8i","title":"Fix duplicate alert context import in API entrypoint","description":"Recent alert-context work introduced a duplicate fetchAlertContextByTraceId import in services/api/src/index.ts, which risks breaking TypeScript compilation and API startup. Remove the duplicate import and validate the affected API/web tests.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:58Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:03:40Z","started_at":"2026-05-18T13:02:02Z","closed_at":"2026-05-18T13:03:40Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/README.md b/README.md index 50063d9..d7b8ace 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,12 @@ > **Pre-alpha warning** This project is in an early pre-alpha state. It will not perform consistently or as expected, and APIs, behavior, and data contracts may change without notice. -This repository contains a Bun + TypeScript monorepo for a personal-use, event-sourced market microstructure research platform focused on: +Islandflow is a Bun + TypeScript monorepo for a personal-use, event-sourced market microstructure research platform focused on: - options prints + NBBO, - off-exchange equity prints, -- explainable rule-based flow classification, +- market news context, +- explainable smart-money flow classification, - deterministic replay, - evidence-linked UI inspection. @@ -19,124 +20,175 @@ This repository contains a Bun + TypeScript monorepo for a personal-use, event-s Implemented now: - Bun workspaces with shared packages for schemas, bus, config, observability, and ClickHouse access. -- Infra orchestration via Docker Compose (NATS JetStream, ClickHouse, Redis). -- Options ingest service with adapters: - - synthetic stream, - - Alpaca options (dev-focused, bounded contracts), - - IBKR bridge (Python sidecar), - - Databento historical replay adapter (Python sidecar). -- Equities ingest service with adapters: - - synthetic stream, - - Alpaca equities trades/quotes. -- Compute service: - - deterministic option print clustering into `FlowPacket`s, - - NBBO join quality features and aggressor-mix metrics, - - rolling baselines in Redis, - - structure summarization and structure packet emission, - - rule-based classifiers + confidence-scored alert events, - - dark-style inferred events from equity prints/quotes, - - equity print-to-quote join events. -- Candles service: - - server-side equity candle aggregation, - - ClickHouse persistence, - - optional Redis hot cache, - - NATS publication. -- Replay service: - - deterministic republishing from ClickHouse to NATS, - - multi-stream merge with stable tie-break ordering, - - speed/start/end controls. -- API service: - - REST endpoints for recent + cursor pagination, - - REST range endpoints for chart windows, - - REST replay-oriented endpoints, - - WebSocket channels for options, NBBO, equities, quotes, joins, flow, classifier hits, alerts, inferred dark, and candles. -- Next.js web app: - - live tape/workspace views, - - replay controls and status, - - signals and chart-focused routes, - - evidence-centric terminal UI. -- Refdata + EOD enricher service entrypoints are present but currently scaffolds (lifecycle/logging only). +- Infra orchestration via Docker Compose for local NATS JetStream, ClickHouse, and Redis. +- Options ingest service with synthetic, Alpaca options, IBKR bridge, and Databento historical replay adapters. +- Equities ingest service with synthetic and Alpaca equities trades/quotes adapters. +- News ingest service for Alpaca news backfill and websocket publication. +- Compute service for deterministic parent-event reconstruction, flow packets, NBBO quality features, rolling baselines, smart-money profile scoring, compatibility classifier hits, alerts, inferred dark-style events, and equity print-to-quote joins. +- Candles service for server-side equity candle aggregation, ClickHouse persistence, optional Redis hot cache, and NATS publication. +- Replay service for deterministic ClickHouse-to-NATS republishing with multi-stream merge, stable tie-break ordering, speed, start, and end controls. +- API service with REST endpoints, cursor pagination, replay/history endpoints, live hot-cache hydration, and WebSocket channels for options, NBBO, equities, quotes, joins, flow, classifier hits, alerts, smart-money events, inferred dark, candles, and news. +- Next.js web app upgraded to Next.js `16.2.6`, React `19.2.0`, and React DOM `19.2.0`. +- Evidence-centric terminal UI, live/replay controls, chart-focused routes, news view, profile-aware smart-money display, and alert-context hydration. +- Thin Electron desktop shell in `apps/desktop` that can wrap the hosted app or local web UI. +- Refdata + EOD enricher service entrypoints are present, with refdata able to validate or refresh the event-calendar cache. Planned / not yet complete: - production-grade licensed feed integrations and entitlement workflow, - richer refdata/corp-action enrichment, - secure deployment/auth hardening, -- deeper structure + calibration workflows from `PLAN.md`. +- native deployment unit templates and rollback helpers, +- signed/notarized desktop distribution and richer desktop-native features, +- deeper calibration workflows from `PLAN.md` and `SMART_MONEY_REBUILD_PLAN.md`. ## Core Principles -- **Explainability first** — inferred outputs are evidence-backed and human-readable. -- **Event sourcing** — raw and derived events persist to support replay. -- **Determinism** — replay behavior tracks live pipeline logic. -- **Microstructure awareness** — bounded joins, confidence scoring, and explicit uncertainty. -- **Bun-first tooling** — runtime/package/scripts all use Bun. +- **Explainability first**: inferred outputs are evidence-backed and human-readable. +- **Event sourcing**: raw and derived events persist to support replay. +- **Determinism**: replay behavior tracks live pipeline logic. +- **Microstructure awareness**: bounded joins, confidence scoring, and explicit uncertainty. +- **Taxonomy over folklore**: "smart money" is modeled as participant-style hypotheses, not a single binary label. +- **Bun-first tooling**: runtime, package management, scripts, and tests use Bun. + +## Smart-Money Classification Taxonomy + +Islandflow now emits first-class `SmartMoneyEvent` records instead of treating old classifier hits as the final semantic object. `FlowPacket` remains the clustering bridge, while smart-money events carry typed features, profile scores, confidence bands, directions, reason codes, abstention state, and suppression reasons. + +Public profile IDs: + +| Profile ID | Meaning | Common evidence | +| --- | --- | --- | +| `institutional_directional` | Large directional parent flow with stronger institutional-style conviction. | premium, size, sweep/burst behavior, aggressor imbalance, quote quality, not short-dated retail-chase context | +| `retail_whale` | Large retail-style speculative bursts, often short-dated or attention-driven. | short-dated OTM concentration, burst prints, IV shock, lower premium than institutional blocks | +| `event_driven` | Flow aligned to known upcoming events. | event-calendar proximity, expiry after event, pre-event concentration, spread/IV pressure | +| `vol_seller` | Premium-selling or short-volatility structure evidence. | sell-side premium, straddles/strangles, neutral direction | +| `arbitrage` | Multi-leg or symmetric structures with low directional exposure. | matched leg symmetry, same-size legs, near-flat directional bias | +| `hedge_reactive` | Hedge or dealer-reaction style flow around short-dated ATM/gamma context. | 0-2 DTE, near-ATM contracts, underlying move linkage, size | + +Compatibility surfaces remain in place: + +- `ClassifierHitEvent` is derived from `SmartMoneyEvent.primary_profile_id`. +- `AlertEvent` may include `primary_profile_id` and `profile_scores`. +- Legacy classifier and alert endpoints still work. + +Primary smart-money access paths: + +```text +/flow/smart-money +/history/smart-money +/replay/smart-money +/ws/smart-money +``` + +The classifier intentionally abstains when evidence is weak or quote context is stale/missing. Suppression guards cover stale quotes, complex/special prints, retail-frenzy directional confusion, hedge-reactive short-dated ATM contexts, and arbitrage symmetry. ## Monorepo Layout - `apps/web` — Next.js UI shell/routes. -- `apps/desktop` — Electron desktop shell that loads the hosted Islandflow app. +- `apps/desktop` — Electron desktop shell that loads the hosted or local Islandflow app. - `services/ingest-options` — options print/NBBO ingest adapters. - `services/ingest-equities` — equity print/quote ingest adapters. -- `services/compute` — clustering, structures, classifiers, alerts, inferred dark. +- `services/ingest-news` — Alpaca news backfill and websocket ingest. +- `services/compute` — parent-event reconstruction, flow packets, smart-money scoring, alerts, inferred dark. - `services/candles` — server-side candle aggregation + cache. -- `services/replay` — ClickHouse → NATS replay streamer. +- `services/replay` — ClickHouse to NATS replay streamer. - `services/api` — REST + WebSocket gateway. -- `services/refdata` — scaffold service. +- `services/refdata` — event-calendar validation/provider refresh scaffolding. - `services/eod-enricher` — scaffold service. - `packages/types` — shared event schemas/types. - `packages/storage` — ClickHouse tables/queries. - `packages/bus` — NATS/JetStream helpers. - `packages/config` — env parsing. - `packages/observability` — logger + metrics facade. +- `deployment/docker` — supported VPS Docker Compose runtime. +- `deployment/native` — experimental host-native Bun + systemd deployment notes. ## Build and Run Install dependencies: -- `bun install` +```bash +bun install +``` Start infrastructure only: -- `docker compose up -d` +```bash +bun run dev:infra +``` Create env file: -- copy `.env.example` to `.env` and set provider credentials as needed. +```bash +cp .env.example .env +``` Start infra + all services + web: -- `bun run dev` +```bash +bun run dev +``` -Start services only (assumes infra is already running): +Start services only, assuming infra is already running: -- `bun run dev:services` +```bash +bun run dev:services +``` Start web only: -- `bun run dev:web` +```bash +bun run dev:web +``` Recommended fast iteration loop: -- `bun run dev:infra` for Docker-backed infra only -- `bun run dev:services` for native Bun backend services -- `bun run dev:web` for the local Next.js UI +```bash +bun run dev:infra +bun run dev:services +bun run dev:web +``` -This keeps Docker in the local workflow where it helps most (NATS, ClickHouse, Redis) without forcing the app services themselves into slower container rebuild/restart loops. +This keeps Docker in the local workflow where it helps most, for NATS, ClickHouse, and Redis, while keeping the app services in native Bun/Next.js loops. ## Deployment Workflow -- `./deploy main` keeps the current VPS Docker rollout path as the default and recommended path. -- Do not run the repo-root `docker-compose.yml` on the VPS. That file is for local infra only and can create duplicate exposed NATS, ClickHouse, and Redis containers on the server. -- `./deploy main --runtime native` targets an experimental host-native Bun + systemd deployment. -- `./deploy current-branch` and `./deploy current-branch --runtime native` keep branch deploys available during the transition, but Docker remains the supported path for the current VPS. -- Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, and `--no-build`. -- Docker runtime details live in `deployment/docker/README.md`. -- Native runtime expectations and prerequisites live in `deployment/native/README.md`. +Docker remains the supported and recommended path for the current VPS. + +```bash +./deploy main +./deploy main --runtime docker +./deploy current-branch +./deploy current-branch --runtime docker +``` + +Important deployment notes: + +- Run the deploy helper from the local repo checkout, not from the VPS shell. +- Do not run the repo-root `docker-compose.yml` on the VPS. It is local infra only and can create duplicate exposed NATS, ClickHouse, and Redis containers on the server. +- The Docker stack lives in `deployment/docker` and is separate from local development infra. +- Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, `--fast`, `--no-build`, and `--force-recreate`. +- `--fast` defaults to a services-only Docker rollout when no explicit scope is provided and trims public API route-suite verification while preserving remote service health checks. +- `./deploy current-branch` requires a clean local working tree and pushes the branch before moving the server checkout. +- The helper has Forgejo-aware remote resolution for deployments and branch pushes. +- Native deployment is opt-in and experimental: + +```bash +./deploy main --runtime native +./deploy current-branch --runtime native +``` + +Native deployment expects Bun, systemd units, host-reachable infra, and deliberate reverse-proxy changes. The open follow-up is to add native unit templates and rollback helpers. + +Read more: + +- `deployment/docker/README.md` +- `deployment/native/README.md` ## Desktop Shell -Islandflow also includes a thin Electron desktop shell in `apps/desktop`. +Islandflow includes a thin Electron desktop shell in `apps/desktop`. What it is: @@ -144,37 +196,35 @@ What it is: - a native app window plus packaging/distribution shell, - a way to run the existing web UI inside Electron without local backend services. -What it is not: +What it is not yet: - a bundled backend runtime, -- a packaged local Next.js frontend in v1, -- a desktop feature layer with notifications, preferences, or auto-updates yet. +- a packaged local Next.js frontend, +- a desktop feature layer with notifications, preferences, auto-updates, signing, or notarization. Run the desktop shell against a local web UI: -- `bun run dev:desktop` - -This starts the local Next.js app, defaults `NEXT_PUBLIC_API_URL` to `https://flow.deltaisland.io` unless you already set it, waits for port `3000`, and then launches Electron against `http://127.0.0.1:3000`. +```bash +bun run dev:desktop +``` Run the desktop shell directly against the hosted app: -- `bun run dev:desktop:remote` +```bash +bun run dev:desktop:remote +``` Package the desktop shell: -- `bun run package:desktop` -- `bun run make:desktop` +```bash +bun run package:desktop +bun run make:desktop +``` Desktop-specific environment: - `ISLANDFLOW_DESKTOP_START_URL` is only used by the Electron shell and is restricted to trusted Islandflow app origins. -- `NEXT_PUBLIC_API_URL` remains the web app's API/WebSocket origin control and should usually point at `https://flow.deltaisland.io` when developing the local UI inside Electron. - -Current desktop limitations: - -- v1 builds are unsigned internal macOS artifacts only, -- Forge currently makes a simple zip distributable for the current host architecture, -- signing, notarization, auto-updates, remembered window state, and richer native integrations are intentionally deferred. +- `NEXT_PUBLIC_API_URL` remains the web app API/WebSocket origin control and usually points at `https://flow.deltaisland.io` when developing local UI inside Electron. ## Environment Configuration @@ -196,32 +246,27 @@ All runtime configuration comes from `.env`. | `OPTIONS_INGEST_ADAPTER` | `synthetic` | Options ingest source: `synthetic`, `alpaca`, `ibkr`, or `databento`. | | `EQUITIES_INGEST_ADAPTER` | `synthetic` | Equities ingest source: `synthetic` or `alpaca`. | | `EMIT_INTERVAL_MS` | `1000` | Emit cadence for synthetic ingest adapters. | -| `SYNTHETIC_MARKET_MODE` | `realistic` | Shared synthetic profile (`realistic`, `active`, `firehose`) used when per-service override is unset. | -| `SYNTHETIC_OPTIONS_MODE` | empty | Options-only synthetic profile override; falls back to `SYNTHETIC_MARKET_MODE`. | -| `SYNTHETIC_EQUITIES_MODE` | empty | Equities-only synthetic profile override; falls back to `SYNTHETIC_MARKET_MODE`. | +| `SYNTHETIC_MARKET_MODE` | `realistic` | Shared synthetic profile: `realistic`, `active`, or `firehose`. | +| `SYNTHETIC_OPTIONS_MODE` | empty | Options-only synthetic profile override. | +| `SYNTHETIC_EQUITIES_MODE` | empty | Equities-only synthetic profile override. | -Synthetic profile intent: -- `realistic`: default local mode with lower synthetic burstiness/noise. -- `active`: busier demo flow while still readable. -- `firehose`: stress mode for throughput/backpressure/hot-window behavior. - -### Options ingest adapter configuration +### Alpaca and news configuration | Variable | Default | What it controls | | --- | --- | --- | -| `ALPACA_API_KEY` | empty | Single-token Alpaca API auth for options/equities adapters. Use this when your account provides one API key value. | -| `ALPACA_REST_URL` | `https://data.alpaca.markets` | Alpaca REST base URL for contract discovery/reference calls. | -| `ALPACA_WS_BASE_URL` | `wss://stream.data.alpaca.markets/v1beta1` (options), `wss://stream.data.alpaca.markets` (equities) | Alpaca websocket base URL. | -| `ALPACA_FEED` | `indicative` | Options feed tier for Alpaca options (`indicative` or `opra`). | +| `ALPACA_API_KEY` | empty | Single-token Alpaca API auth for options, equities, and news adapters. | +| `ALPACA_REST_URL` | `https://data.alpaca.markets` | Alpaca REST base URL. | +| `ALPACA_WS_BASE_URL` | `wss://stream.data.alpaca.markets/v1beta1` for options, `wss://stream.data.alpaca.markets` for equities/news | Alpaca websocket base URL. | +| `ALPACA_FEED` | `indicative` | Options feed tier: `indicative` or `opra`. | | `ALPACA_UNDERLYINGS` | `SPY,NVDA,AAPL` | Comma-separated symbols targeted by Alpaca ingest. | | `ALPACA_STRIKES_PER_SIDE` | `8` | Contracts selected per side of spot for Alpaca options chain sampling. | | `ALPACA_MAX_DTE_DAYS` | `30` | Max days-to-expiry included for Alpaca options contract selection. | | `ALPACA_MONEYNESS_PCT` | `0.06` | Primary moneyness filter for Alpaca options contract selection. | | `ALPACA_MONEYNESS_FALLBACK_PCT` | `0.1` | Wider fallback moneyness filter if candidate set is too sparse. | | `ALPACA_MAX_QUOTES` | `200` | Upper bound on selected Alpaca options contracts/quotes per cycle. | -| `ALPACA_EQUITIES_FEED` | `iex` | Alpaca equities feed (`iex` free tier, `sip` paid consolidated feed). | - -For Alpaca adapters, configure `ALPACA_API_KEY`. +| `ALPACA_EQUITIES_FEED` | `iex` | Alpaca equities feed: `iex` or `sip`. | +| `ALPACA_NEWS_BACKFILL_LIMIT` | `100` | Alpaca news stories fetched on startup, capped at 200. | +| `ALPACA_NEWS_WEBSOCKET_PATH` | `/v1beta1/news` | Alpaca news websocket path. | ### Databento replay adapter configuration @@ -236,7 +281,7 @@ For Alpaca adapters, configure `ALPACA_API_KEY`. | `DATABENTO_SYMBOLS` | `ALL` | Symbol selection forwarded to Databento sidecar query. | | `DATABENTO_STYPE_IN` | `raw_symbol` | Databento input symbology type. | | `DATABENTO_STYPE_OUT` | `raw_symbol` | Databento output symbology type. | -| `DATABENTO_LIMIT` | `0` | Max Databento records (`0` means no explicit limit). | +| `DATABENTO_LIMIT` | `0` | Max Databento records, where `0` means no explicit limit. | | `DATABENTO_PRICE_SCALE` | `1` | Multiplier applied to decoded prices from sidecar output. | | `DATABENTO_PYTHON_BIN` | `python3` | Python executable used to run Databento sidecar script. | @@ -248,9 +293,9 @@ For Alpaca adapters, configure `ALPACA_API_KEY`. | `IBKR_PORT` | `7497` | TWS/Gateway port for IBKR bridge. | | `IBKR_CLIENT_ID` | `0` | IBKR client id used by the bridge connection. | | `IBKR_SYMBOL` | `SPY` | Underlying symbol requested from IBKR. | -| `IBKR_EXPIRY` | `20250117` | Option expiry (YYYYMMDD) requested from IBKR. | +| `IBKR_EXPIRY` | `20250117` | Option expiry requested from IBKR. | | `IBKR_STRIKE` | `450` | Strike requested from IBKR. | -| `IBKR_RIGHT` | `C` | Option side (`C` or `P`). | +| `IBKR_RIGHT` | `C` | Option side: `C` or `P`. | | `IBKR_EXCHANGE` | `SMART` | IBKR exchange routing code. | | `IBKR_CURRENCY` | `USD` | Contract currency. | | `IBKR_PYTHON_BIN` | `python3` | Python executable used for IBKR sidecar. | @@ -259,133 +304,77 @@ For Alpaca adapters, configure `ALPACA_API_KEY`. | Variable | Default | What it controls | | --- | --- | --- | -| `OPTIONS_SIGNAL_MODE` | `smart-money` | Signal pass policy (`smart-money`, `balanced`, `all`) for options prints. | +| `OPTIONS_SIGNAL_MODE` | `smart-money` | Signal pass policy: `smart-money`, `balanced`, or `all`. | | `OPTIONS_SIGNAL_MIN_NOTIONAL` | `10000` | Base minimum notional for most signal candidates. | | `OPTIONS_SIGNAL_ETF_MIN_NOTIONAL` | `50000` | ETF-specific minimum notional for signal inclusion. | -| `OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL` | `25000` | Minimum notional for bid-side (`B`/`BB`) or sweep/ISO thresholds. | +| `OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL` | `25000` | Minimum notional for bid-side or sweep/ISO thresholds. | | `OPTIONS_SIGNAL_MID_MIN_NOTIONAL` | `20000` | Minimum notional for non-sweep/non-ISO `MID` prints. | | `OPTIONS_SIGNAL_NBBO_MAX_AGE_MS` | `1500` | NBBO freshness threshold used during signal classification. | -| `OPTIONS_SIGNAL_ETF_UNDERLYINGS` | `SPY,QQQ,IWM,DIA,TLT,GLD,SLV,XLF,XLE,XLV,XLI,XLP,XLU,XLY,SMH,ARKK` | Comma-separated underlyings treated as ETFs by signal filters. | +| `OPTIONS_SIGNAL_ETF_UNDERLYINGS` | `SPY,QQQ,IWM,DIA,TLT,GLD,SLV,XLF,XLE,XLV,XLI,XLP,XLU,XLY,SMH,ARKK` | ETF underlyings treated specially by signal filters. | -Default `smart-money` policy rejects lower-information prints and keeps high-confidence/high-notional/sweep-style flow; `balanced` lowers thresholds; `all` bypasses filtering. +Default `smart-money` policy rejects lower-information prints and keeps higher-confidence, higher-notional, sweep-style flow. `balanced` lowers thresholds. `all` bypasses filtering. -### Compute/classifier/dark-inference configuration +### Compute, classifier, and dark-inference configuration | Variable | Default | What it controls | | --- | --- | --- | -| `CLUSTER_WINDOW_MS` | `500` | Time window used to cluster nearby option prints into a packet candidate. | -| `COMPUTE_DELIVER_POLICY` | `new` | Consumer start policy for compute stream subscriptions (`new`, `all`, `last`, `last_per_subject`). | -| `COMPUTE_CONSUMER_RESET` | `false` | If true, resets durable consumer position for compute on startup. | +| `CLUSTER_WINDOW_MS` | `500` | Time window used to cluster nearby option prints into packet candidates. | +| `COMPUTE_DELIVER_POLICY` | `new` | Consumer start policy for compute subscriptions. | +| `COMPUTE_CONSUMER_RESET` | `false` | Resets durable consumer position for compute on startup when true. | | `NBBO_MAX_AGE_MS` | `1000` | Max NBBO age accepted when enriching option prints in compute. | | `ROLLING_WINDOW_SIZE` | `50` | Number of observations retained per rolling metric key. | | `ROLLING_TTL_SEC` | `86400` | Redis TTL for rolling metric keys. | | `EQUITY_QUOTE_MAX_AGE_MS` | `1000` | Max quote staleness when joining equity prints for inference. | | `DARK_INFER_WINDOW_MS` | `60000` | Sliding window length for dark-style inference accumulation. | -| `DARK_INFER_COOLDOWN_MS` | `30000` | Cooldown before emitting repeated dark inferences for same symbol/pattern. | -| `DARK_INFER_MIN_BLOCK_SIZE` | `2000` | Minimum single-print size for block-style dark inference evidence. | -| `DARK_INFER_MIN_ACCUM_SIZE` | `3000` | Minimum aggregate size for accumulation-style dark inference evidence. | -| `DARK_INFER_MIN_ACCUM_COUNT` | `4` | Minimum print count for accumulation-style dark inference. | -| `DARK_INFER_MIN_PRINT_SIZE` | `200` | Minimum print size considered as dark inference evidence. | -| `DARK_INFER_MAX_EVIDENCE` | `20` | Max evidence items attached to one inferred dark event. | -| `DARK_INFER_MAX_SPREAD_PCT` | `0.005` | Maximum spread percentage allowed for dark inference confidence. | -| `CLASSIFIER_SWEEP_MIN_PREMIUM` | `40000` | Minimum premium to trigger sweep classifier logic. | -| `CLASSIFIER_SWEEP_MIN_COUNT` | `3` | Minimum child prints in cluster for sweep classifier hit. | -| `CLASSIFIER_SWEEP_MIN_PREMIUM_Z` | `2` | Min premium z-score for sweep classifier confirmation. | -| `CLASSIFIER_SPIKE_MIN_PREMIUM` | `20000` | Minimum premium for spike classifier logic. | -| `CLASSIFIER_SPIKE_MIN_SIZE` | `400` | Minimum total size for spike classifier logic. | -| `CLASSIFIER_SPIKE_MIN_PREMIUM_Z` | `2.5` | Min premium z-score for spike classifier confirmation. | -| `CLASSIFIER_SPIKE_MIN_SIZE_Z` | `2` | Min size z-score for spike classifier confirmation. | -| `CLASSIFIER_Z_MIN_SAMPLES` | `12` | Minimum rolling sample count before z-score gating applies. | -| `CLASSIFIER_MIN_NBBO_COVERAGE` | `0.5` | Required fraction of prints in cluster with valid NBBO context. | -| `CLASSIFIER_MIN_AGGRESSOR_RATIO` | `0.55` | Minimum aggressor-side ratio for classifier confidence. | -| `CLASSIFIER_0DTE_MAX_ATM_PCT` | `0.01` | Max distance-from-ATM to qualify as near-ATM 0DTE event. | -| `CLASSIFIER_0DTE_MIN_PREMIUM` | `20000` | Minimum premium for 0DTE classifier events. | -| `CLASSIFIER_0DTE_MIN_SIZE` | `400` | Minimum size for 0DTE classifier events. | -| `SMART_MONEY_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file used by compute to enrich event-driven smart-money profile features. | -| `REFDATA_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file for refdata service startup validation; falls back to `SMART_MONEY_EVENT_CALENDAR_PATH` when unset. | -| `REFDATA_EVENT_CALENDAR_PROVIDER` | empty | Set to `alpha_vantage` to have refdata refresh the calendar cache from Alpha Vantage. | -| `ALPHA_VANTAGE_API_KEY` | empty | Alpha Vantage key used when `REFDATA_EVENT_CALENDAR_PROVIDER=alpha_vantage`. | -| `ALPHA_VANTAGE_EARNINGS_HORIZON` | `3month` | Alpha Vantage earnings horizon: `3month`, `6month`, or `12month`. | -| `ALPHA_VANTAGE_EARNINGS_SYMBOL` | empty | Optional single-symbol Alpha Vantage earnings query; empty fetches the full scheduled earnings list. | -| `REFDATA_EVENT_CALENDAR_REFRESH_MS` | `86400000` | Refdata refresh cadence for provider-backed event-calendar cache writes. | +| `DARK_INFER_COOLDOWN_MS` | `30000` | Cooldown before repeated dark inferences for same symbol/pattern. | +| `SMART_MONEY_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file used by compute. | +| `REFDATA_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar path for refdata; falls back to `SMART_MONEY_EVENT_CALENDAR_PATH`. | +| `REFDATA_EVENT_CALENDAR_PROVIDER` | empty | Set to `alpha_vantage` to refresh event-calendar cache from Alpha Vantage. | +| `ALPHA_VANTAGE_API_KEY` | empty | Alpha Vantage key for provider-backed event-calendar refresh. | -Event-calendar rows may use `symbol`, `underlying`, or `underlying_id`; `event_date`, `event_time`, or `event_ts`; and `announced_ts`, `available_ts`, `as_of_ts`, or `created_ts`. Compute only uses events already available at the packet timestamp, so missing or unavailable rows leave event-alignment features as neutral `null` values. - -### Candle service configuration - -| Variable | Default | What it controls | -| --- | --- | --- | -| `CANDLE_INTERVALS_MS` | `60000,300000` | Comma-separated candle intervals generated from equity prints. | -| `CANDLE_MAX_LATE_MS` | `0` | Allowed lateness for out-of-order prints before candle rejection/roll policy applies. | -| `CANDLE_CACHE_LIMIT` | `2000` | Max cached candles per `(underlying, interval)` in Redis (`0` disables cache). | -| `CANDLE_DELIVER_POLICY` | `new` | Consumer start policy for candle service (`new`, `all`, `last`, `last_per_subject`). | -| `CANDLE_CONSUMER_RESET` | `false` | If true, resets candle durable consumer position on startup. | - -### API + live cache configuration +### API, live cache, and web client | Variable | Default | What it controls | | --- | --- | --- | | `API_PORT` | `4000` | API service listen port. | -| `REST_DEFAULT_LIMIT` | `200` | Default record count when a REST endpoint omits `limit`. | -| `API_DELIVER_POLICY` | `new` | JetStream consumer start policy used by API live subscribers (`new`, `all`, `last`, `last_per_subject`). | -| `API_CONSUMER_RESET` | `false` | If true, API resets/recreates its live durable consumers on startup. | -| `LIVE_LIMIT_OPTIONS` | `10000` | In-memory/Redis live cache depth for options channel (clamped `1..100000`). | -| `LIVE_LIMIT_NBBO` | `10000` | Live cache depth for options NBBO channel (clamped `1..100000`). | -| `LIVE_LIMIT_EQUITIES` | `10000` | Live cache depth for equities channel (clamped `1..100000`). | -| `LIVE_LIMIT_EQUITY_QUOTES` | `10000` | Live cache depth for equity quotes channel (clamped `1..100000`). | -| `LIVE_LIMIT_EQUITY_JOINS` | `10000` | Live cache depth for equity join channel (clamped `1..100000`). | -| `LIVE_LIMIT_FLOW` | `10000` | Live cache depth for flow packet channel (clamped `1..100000`). | -| `LIVE_LIMIT_CLASSIFIER_HITS` | `10000` | Live cache depth for classifier hits channel (clamped `1..100000`). | -| `LIVE_LIMIT_ALERTS` | `10000` | Live cache depth for alerts channel (clamped `1..100000`). | -| `LIVE_LIMIT_INFERRED_DARK` | `10000` | Live cache depth for inferred dark channel (clamped `1..100000`). | - -### Web client configuration (`NEXT_PUBLIC_*`) - -| Variable | Default | What it controls | -| --- | --- | --- | -| `NEXT_PUBLIC_API_URL` | auto-detected (`window.location.origin` in browser; `http://127.0.0.1:4000` fallback) | Explicit base URL for API/WS calls from the web app. | -| `NEXT_PUBLIC_LIVE_HOT_WINDOW` | `2000` | Max hot-window items retained for non-options live streams in UI state (`100..100000`). | -| `NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS` | `25000` | Dedicated max hot-window items retained for options prints (`100..100000`). | -| `NEXT_PUBLIC_NBBO_MAX_AGE_MS` | `1000` | Frontend NBBO staleness threshold used for UI status/placement logic. | -| `NEXT_PUBLIC_LIVE_EQUITIES_SILENT_WARNING_MS` | `25000` | Delay before warning when equities stream is quiet (`5000..300000`). | -| `NEXT_PUBLIC_PINNED_EVIDENCE_TTL_MS` | `1200000` | TTL for pinned evidence objects in UI (`60000..7200000`). | -| `NEXT_PUBLIC_PINNED_EVIDENCE_MAX_ITEMS` | `4000` | Maximum pinned evidence cache size in UI (`100..50000`). | -| `NEXT_PUBLIC_FLOW_FILTER_PRESET` | `smart-money` | Default flow filter preset applied on page load (`smart-money`, `balanced`, `all`). | +| `REST_DEFAULT_LIMIT` | `200` | Default REST record count. | +| `API_DELIVER_POLICY` | `new` | JetStream consumer start policy used by API live subscribers. | +| `API_CONSUMER_RESET` | `false` | Resets/recreates API live durable consumers on startup when true. | +| `LIVE_LIMIT_DEFAULT` | `1000` | Optional generic live cache depth default. | +| `LIVE_LIMIT_FLOW` | `500` | Live cache depth for flow packet events unless overridden. | +| `LIVE_LIMIT_SMART_MONEY` | `300` | Live cache depth for smart-money events unless overridden. | +| `LIVE_LIMIT_OPTIONS` | `1000` | Live cache depth for options channel unless overridden. | +| `LIVE_LIMIT_ALERTS` | `300` | Live cache depth for alerts channel unless overridden. | +| `LIVE_LIMIT_NEWS` | `100` | Live cache depth for news channel unless overridden. | +| `NEXT_PUBLIC_API_URL` | auto-detected in browser, `http://127.0.0.1:4000` fallback | Explicit base URL for API/WS calls from the web app. | +| `NEXT_PUBLIC_LIVE_HOT_WINDOW` | `600` | Max hot-window items retained for non-options live streams in UI state. | +| `NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS` | `1200` | Dedicated max hot-window items retained for options prints. | +| `NEXT_PUBLIC_NBBO_MAX_AGE_MS` | `1000` | Frontend NBBO staleness threshold. | +| `NEXT_PUBLIC_FLOW_FILTER_PRESET` | `smart-money` | Default flow filter preset: `smart-money`, `balanced`, or `all`. | ### Replay and testing controls | Variable | Default | What it controls | | --- | --- | --- | -| `REPLAY_ENABLED` | `false` | Dev-script toggle: starts replay service in `bun run dev` when truthy. | -| `REPLAY_STREAMS` | `options,nbbo,equities,equity-quotes` | Replay stream selection (`all` or comma list of supported aliases). | -| `REPLAY_START_TS` | `0` | Replay lower-bound timestamp; `0` means from earliest stored data. | -| `REPLAY_END_TS` | `0` | Replay upper-bound timestamp; `0` means no explicit end bound. | -| `REPLAY_SPEED` | `1` | Replay speed multiplier relative to original event timing. | -| `REPLAY_BATCH_SIZE` | `200` | Batch fetch size per replay stream pull. | -| `REPLAY_LOG_EVERY` | `1000` | Progress log interval (emitted event count). | +| `REPLAY_ENABLED` | `false` | Starts replay service in `bun run dev` when truthy. | +| `REPLAY_STREAMS` | `options,nbbo,equities,equity-quotes` | Replay stream selection. | +| `REPLAY_START_TS` | `0` | Replay lower-bound timestamp. | +| `REPLAY_END_TS` | `0` | Replay upper-bound timestamp. | +| `REPLAY_SPEED` | `1` | Replay speed multiplier. | +| `REPLAY_BATCH_SIZE` | `200` | Batch fetch size per stream. | +| `REPLAY_LOG_EVERY` | `1000` | Progress log interval. | | `TESTING_MODE` | `false` | Enables ingest publish throttling for deterministic/lower-volume test runs. | | `TESTING_THROTTLE_MS` | `200` | Minimum delay between emitted events while `TESTING_MODE=true`. | ## Quick Notes -- Python dependencies are required only for IBKR/Databento sidecars (`services/ingest-options/py/requirements.txt`). +- Python dependencies are required only for IBKR/Databento sidecars: `services/ingest-options/py/requirements.txt`. - Candle construction is server-side; the client consumes prebuilt OHLC events. -- Option prints now persist as enriched raw rows and can be queried as either: - - `view=signal` — default live/UI path and compute input. - - `view=raw` — audit/debug path that preserves every stored print. -- The default Tape page options/packets posture is now stock-only, hides `B` / `BB`, keeps calls and puts visible, and applies in-memory min-notional controls immediately. -- Live retention uses a two-tier model: - - ClickHouse is durable server history; Redis is a bounded hot cache per live generic channel. - - `LIVE_LIMIT_*` controls initial snapshot/hot-cache depth, not total persisted history. - - Browser state is only a rendering window and UI preferences, not a market-data database. - - Devices connected to the same API hydrate from the same server-seen history. - - UI keeps a bounded hot window for rendering performance around the signal view rather than raw noise. - - Options prints can use a deeper dedicated cap via `NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS` without raising every other feed. - - Alert/drawer evidence is pinned and hydrated by id/trace so details remain inspectable after hot-window eviction. -- Firehose-readiness strategy: - - preserve raw ingest for storage/replay, - - feed compute and default live UI from the filtered signal path, - - add filterable live subscription contracts now so selective delivery can move server-side without reshaping the protocol later. +- Option prints persist as enriched raw rows and can be queried as `view=signal` or `view=raw`. +- The default Tape page options/packets posture is stock-only, hides `B` / `BB`, keeps calls and puts visible, and applies in-memory min-notional controls immediately. +- Live retention uses ClickHouse for durable server history, Redis for bounded hot cache, and browser state for rendering windows/preferences. +- Alert and drawer evidence is pinned and hydrated by id/trace so details remain inspectable after hot-window eviction. +- Firehose readiness keeps raw ingest for storage/replay, routes default compute/UI through filtered signals, and keeps subscription contracts ready for server-side selective delivery. - This repository is for personal, non-redistributed usage. ## Useful Examples diff --git a/docs/turns/2026-05-19-0739-update-readme-current-state.html b/docs/turns/2026-05-19-0739-update-readme-current-state.html new file mode 100644 index 0000000..77e0a2a --- /dev/null +++ b/docs/turns/2026-05-19-0739-update-readme-current-state.html @@ -0,0 +1,259 @@ + + + + + + README Current-State Update + + + +
+
+
Turn document · 2026-05-19 07:39 America/New_York
+

README Current-State Update

+

+ Resolved the README merge conflict and rewrote the project overview so it matches the current Islandflow codebase, including the smart-money taxonomy, Next.js 16 update, news ingest, desktop shell, and current deployment posture. +

+
+ README.md + smart-money taxonomy + Next.js 16.2.6 + deployment docs +
+
+ +
+

Summary

+

+ The README no longer contains conflict markers. It now gives a concise but current description of the platform, its runtime services, public smart-money categories, environment knobs, and supported deployment workflow. +

+
+ +
+

Changes Made

+
    +
  • Resolved the conflicted README by preserving the useful project-state content and removing stale simplified sections.
    • +
  • Added a first-class smart-money taxonomy section for the six public profiles: institutional_directional, retail_whale, event_driven, vol_seller, arbitrage, and hedge_reactive.
    • +
  • Documented that smart-money events are now the semantic object, while legacy classifier hits and alerts remain compatibility surfaces.
    • +
  • Updated the current implementation state to include Alpaca news ingest, profile-aware UI behavior, alert-context hydration, and the Electron shell.
    • +
  • Recorded the Next.js update to 16.2.6 with React and React DOM 19.2.0.
    • +
  • Clarified deployment: Docker is still the supported VPS path, native Bun/systemd rollout is experimental, and scoped deploy flags are available.
    • +
  • Aligned live-cache and web hot-window defaults with the current env examples and API defaults.
    • +
+
+ +
+

Context

+

+ Recent commits showed the README branch was carrying a Next.js upgrade, Alpaca news support, smart-money event work, and deployment helper changes. The prior README mixed both sides of a merge conflict and did not explain the newer taxonomy-driven classifier model. +

+
+ +
+

Important Implementation Details

+

+ The README intentionally treats FlowPacket as an intermediate clustering bridge and SmartMoneyEvent as the current semantic surface. It also documents abstention and suppression behavior so readers do not mistake every large print for a forced smart-money label. +

+

+ Deployment language now matches the current operations docs: ./deploy main defaults to the Docker path, --runtime native is available but experimental, and native rollout still depends on systemd units and reverse-proxy preparation. +

+
+ +
+

Relevant Diff Snippets

+

+ Diff snippets are formatted for readability in the same spirit as diffs.com, with only the most relevant README changes shown here. +

+ 
+## Smart-Money Classification Taxonomy
++
++Islandflow now emits first-class `SmartMoneyEvent` records instead of treating old classifier hits as the final semantic object.
++
++| Profile ID | Meaning | Common evidence |
++| --- | --- | --- |
++| `institutional_directional` | Large directional parent flow with stronger institutional-style conviction. | premium, size, sweep/burst behavior, aggressor imbalance, quote quality |
++| `retail_whale` | Large retail-style speculative bursts, often short-dated or attention-driven. | short-dated OTM concentration, burst prints, IV shock |
++| `event_driven` | Flow aligned to known upcoming events. | event-calendar proximity, expiry after event, pre-event concentration |
++| `vol_seller` | Premium-selling or short-volatility structure evidence. | sell-side premium, straddles/strangles |
++| `arbitrage` | Multi-leg or symmetric structures with low directional exposure. | matched leg symmetry, near-flat directional bias |
++| `hedge_reactive` | Hedge or dealer-reaction style flow around short-dated ATM/gamma context. | 0-2 DTE, near-ATM contracts, underlying move linkage |
+ 
+## Deployment Workflow
++
++Docker remains the supported and recommended path for the current VPS.
++
++./deploy main
++./deploy main --runtime docker
++./deploy current-branch
++./deploy current-branch --runtime docker
++
++Native deployment is opt-in and experimental:
++
++./deploy main --runtime native
++./deploy current-branch --runtime native
+
+ +
+

Expected Impact for End-Users

+

+ New contributors or future sessions should be able to read the README and understand what Islandflow currently does, which service owns each capability, how the smart-money labels should be interpreted, and which deployment command is appropriate for the VPS. +

+
+ +
+

Validation

+
    +
  • Confirmed no merge conflict markers remain with rg -n "<<<<<<<|=======|>>>>>>>" README.md.
    • +
  • Ran git diff --check; no whitespace or patch-format issues were reported.
    • +
  • Ran focused tests: bun test packages/types/tests/options-flow.test.ts packages/types/tests/live.test.ts packages/storage/tests/smart-money-events.test.ts services/compute/tests/parent-events.test.ts.
    • +
  • Focused test result: 12 pass, 0 fail.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • This was documentation-only, so no full production web build was run. The focused tests cover the smart-money/type/storage claims most relevant to the README update.
    • +
  • The README summarizes environment variables instead of listing every low-level classifier and dark-inference threshold. Detailed knobs remain available in .env.example and service code.
    • +
  • Native deployment remains experimental; the README calls that out directly and points to the dedicated native deployment document.
    • +
+
+ +
+

Follow-up Work

+
    +
  • islandflow-38p: add native deployment unit templates and rollback helpers.
    • +
  • islandflow-932: continue desktop follow-up native features.
    • +
  • islandflow-2db: manually remove stale local-infra containers from the VPS when doing server hygiene.
    • +
+
+
+ + From a790a2815cd23ab7f7b08b719d56a47b6ef69ed8 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 08:05:30 -0400 Subject: [PATCH 10/26] clarify repo turn documentation scope --- .beads/issues.jsonl | 1 + AGENTS.md | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 61aef8b..40c5966 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -13,6 +13,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-lm6","title":"Clarify repo turn documentation scope","description":"Update AGENTS.md so repository turn documentation clearly uses repo-local docs/turns and impeccable styling, without inheriting global non-repo computer-task styling.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T12:05:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T12:05:07Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-6iq","title":"Update README for current project state","description":"Resolve README merge conflicts and document the current project state, including the smart money classification taxonomy, Next.js update, and deployment workflow changes.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:37:24Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:40:01Z","started_at":"2026-05-19T11:37:31Z","closed_at":"2026-05-19T11:40:01Z","close_reason":"README conflict resolved and current project state documented, including smart-money taxonomy, Next.js update, and deployment workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lib","title":"Upgrade apps/web to Next.js 16.2.6","description":"Upgrade the web app dependency stack to Next.js 16.2.6 with React 19, refresh Bun and mirrored Docker workspace lockfiles, keep runtime behavior unchanged, fix any focused web test fallout, validate the web build and targeted route tests, and document the completed work.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:04:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:31:23Z","started_at":"2026-05-19T11:04:57Z","closed_at":"2026-05-19T11:31:23Z","close_reason":"Upgraded apps/web to Next.js 16.2.6 with React 19, refreshed Bun lockfiles including the Docker workspace mirror, fixed the React 19 nullable ref type issue, and validated the web build, focused tests, Docker workspace sync, and route smoke checks.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8fn","title":"implement alpaca-backed news wire view","description":"Why this issue exists and what needs to be done:\\nAdd an Alpaca-powered live news pipeline, API, storage, and web experience, including a dedicated /news route, Home preview, live fanout, history pagination, ticker resolution, and replay-mode live-only empty states.\\n\\nAcceptance criteria:\\n- normalized NewsStory contract and live channel exist\\n- ingest-news service backfills and streams Alpaca news\\n- API persists, serves, and fans out news\\n- web app exposes /news plus Home preview and drawer\\n- tests cover types, storage, API, and key UI behaviors\\n- turn documentation is added\\n\\nDesign:\\nReuse Islandflow drawer, chips, panes, and terminal styling; keep news live-only in v1 replay mode.\\n\\nNotes:\\nImplement client-side ticker filtering in v1 and expose latest revision only per provider+story_id.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T20:37:13Z","created_by":"dirtydishes","updated_at":"2026-05-18T20:55:11Z","started_at":"2026-05-18T20:37:20Z","closed_at":"2026-05-18T20:55:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/AGENTS.md b/AGENTS.md index 08a484a..fe8ffca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,7 +82,6 @@ Save the document in: ```text docs/turns/ ``` -## Important: If you are not working inside a git repository, save the document to `~/dev/docs/turns/` Use a clear timestamped filename: @@ -98,9 +97,11 @@ docs/turns/2026-05-14-add-market-replay-controls.html ### Format -Use the impeccable skill to structure the document as clean, readable HTML. +Use the `impeccable` skill to structure and style the document as clean, readable HTML. -If the impeccable skill is unavailable, still create a well-structured standalone HTML file with: +For this repository, `impeccable` is the styling and layout authority for turn documents when available. Do not apply global non-repo computer-task house styling to repository turn documents. + +If the `impeccable` skill is unavailable or blocked by an actual tool/file error, still create a well-structured standalone HTML file with: - A concise summary at the top - A detailed explanation of what changed From cb2de93ddec07ad9932120eca28fa214b9ded48b Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 08:06:10 -0400 Subject: [PATCH 11/26] clarify repo turn doc rules --- ...5-19-0805-clarify-repo-turn-doc-rules.html | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 docs/turns/2026-05-19-0805-clarify-repo-turn-doc-rules.html diff --git a/docs/turns/2026-05-19-0805-clarify-repo-turn-doc-rules.html b/docs/turns/2026-05-19-0805-clarify-repo-turn-doc-rules.html new file mode 100644 index 0000000..9342851 --- /dev/null +++ b/docs/turns/2026-05-19-0805-clarify-repo-turn-doc-rules.html @@ -0,0 +1,200 @@ + + + + + + Clarify Repo Turn Documentation Rules + + + +
+
+
Turn document · 2026-05-19 08:05 America/New_York
+

Clarify Repo Turn Documentation Rules

+

+ Updated the repository instructions so Islandflow turn documents are clearly repo-local and styled through impeccable, without inheriting global non-repo computer-task styling. +

+
+ +
+

Summary

+

+ The repo AGENTS.md now removes a stray non-repo location rule and explicitly states that impeccable is the styling and layout authority for Islandflow turn documents when available. +

+
+ +
+

Changes Made

+
    +
  • Removed the confusing instruction to save non-repo documentation under ~/dev/docs/turns/.
    • +
  • Clarified that repository turn documents stay in docs/turns/.
    • +
  • Updated the format rule to say impeccable handles both structure and styling.
    • +
  • Added an explicit guard against applying global non-repo computer-task house styling to this repository's turn documents.
    • +
  • Clarified that the fallback standalone HTML path only applies when impeccable is unavailable or blocked by an actual error.
    • +
+
+ +
+

Context

+

+ The global agent instructions now distinguish repository implementation documentation from non-repo computer-task documentation. This repo file needed a small cleanup so it would not reintroduce ambiguity about location or styling. +

+
+ +
+

Important Implementation Details

+

+ This was a documentation-only change in AGENTS.md. It changes future agent behavior but does not alter runtime code, tests, deployment scripts, or application behavior. +

+
+ +
+

Relevant Diff Snippets

+ 
-## Important: If you are not working inside a git repository, save the document to `~/dev/docs/turns/`
+
+-Use the impeccable skill to structure the document as clean, readable HTML.
++Use the `impeccable` skill to structure and style the document as clean, readable HTML.
++
++For this repository, `impeccable` is the styling and layout authority for turn documents when available. Do not apply global non-repo computer-task house styling to repository turn documents.
+
+-If the impeccable skill is unavailable, still create a well-structured standalone HTML file with:
++If the `impeccable` skill is unavailable or blocked by an actual tool/file error, still create a well-structured standalone HTML file with:
+
+ +
+

Expected Impact for End-Users

+

+ Future Islandflow turns should produce documentation in the repo's docs/turns/ folder and let impeccable drive the visual treatment, making repo documentation less likely to inherit global computer-task styling. +

+
+ +
+

Validation

+
    +
  • Reviewed the AGENTS.md diff after patching.
    • +
  • Ran git diff --check with no whitespace errors.
    • +
  • No application test suite was run because this change only updates repository instructions.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+

+ This clarification depends on future agents reading both global and repo instructions. The new wording is intentionally direct about repo scope, location, and styling to reduce that risk. +

+
+ +
+

Follow-up Work

+

+ No follow-up issue is required for this patch. The related Beads task for this documentation cleanup is islandflow-lm6. +

+
+
+ + From 328974b374a130e2e31503189322763c396f28a8 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 08:06:33 -0400 Subject: [PATCH 12/26] update beads for repo doc rules --- .beads/issues.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 40c5966..aa74dd2 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -13,7 +13,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-lm6","title":"Clarify repo turn documentation scope","description":"Update AGENTS.md so repository turn documentation clearly uses repo-local docs/turns and impeccable styling, without inheriting global non-repo computer-task styling.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T12:05:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T12:05:07Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-lm6","title":"Clarify repo turn documentation scope","description":"Update AGENTS.md so repository turn documentation clearly uses repo-local docs/turns and impeccable styling, without inheriting global non-repo computer-task styling.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T12:05:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T12:06:12Z","started_at":"2026-05-19T12:05:14Z","closed_at":"2026-05-19T12:06:12Z","close_reason":"Verified AGENTS.md now scopes repo turn docs to docs/turns and makes impeccable the styling authority; added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-6iq","title":"Update README for current project state","description":"Resolve README merge conflicts and document the current project state, including the smart money classification taxonomy, Next.js update, and deployment workflow changes.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:37:24Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:40:01Z","started_at":"2026-05-19T11:37:31Z","closed_at":"2026-05-19T11:40:01Z","close_reason":"README conflict resolved and current project state documented, including smart-money taxonomy, Next.js update, and deployment workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lib","title":"Upgrade apps/web to Next.js 16.2.6","description":"Upgrade the web app dependency stack to Next.js 16.2.6 with React 19, refresh Bun and mirrored Docker workspace lockfiles, keep runtime behavior unchanged, fix any focused web test fallout, validate the web build and targeted route tests, and document the completed work.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:04:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:31:23Z","started_at":"2026-05-19T11:04:57Z","closed_at":"2026-05-19T11:31:23Z","close_reason":"Upgraded apps/web to Next.js 16.2.6 with React 19, refreshed Bun lockfiles including the Docker workspace mirror, fixed the React 19 nullable ref type issue, and validated the web build, focused tests, Docker workspace sync, and route smoke checks.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8fn","title":"implement alpaca-backed news wire view","description":"Why this issue exists and what needs to be done:\\nAdd an Alpaca-powered live news pipeline, API, storage, and web experience, including a dedicated /news route, Home preview, live fanout, history pagination, ticker resolution, and replay-mode live-only empty states.\\n\\nAcceptance criteria:\\n- normalized NewsStory contract and live channel exist\\n- ingest-news service backfills and streams Alpaca news\\n- API persists, serves, and fans out news\\n- web app exposes /news plus Home preview and drawer\\n- tests cover types, storage, API, and key UI behaviors\\n- turn documentation is added\\n\\nDesign:\\nReuse Islandflow drawer, chips, panes, and terminal styling; keep news live-only in v1 replay mode.\\n\\nNotes:\\nImplement client-side ticker filtering in v1 and expose latest revision only per provider+story_id.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T20:37:13Z","created_by":"dirtydishes","updated_at":"2026-05-18T20:55:11Z","started_at":"2026-05-18T20:37:20Z","closed_at":"2026-05-18T20:55:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} From 75ff4f489f67b54fb4dfa15125d77554a650e4a4 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 14:42:56 -0400 Subject: [PATCH 13/26] docs(daily-git): add 2026-05-18 standup summary --- .beads/issues.jsonl | 1 + ...2026-05-19-standup-summary-2026-05-18.html | 566 ++++++++++++++++++ 2 files changed, 567 insertions(+) create mode 100644 docs/daily-git/2026-05-19-standup-summary-2026-05-18.html diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index aa74dd2..c61c799 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -52,6 +52,7 @@ {"_type":"issue","id":"islandflow-zs0","title":"Migrate terminal UI to smart-money profiles","description":"Migrate apps/web terminal rendering to consume SmartMoneyEvent directly: primary profile, probability ladder, reason codes, and suppression/abstention state, while preserving legacy alert/classifier displays during the bridge.","status":"closed","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:23Z","created_by":"dirtydishes","updated_at":"2026-05-05T05:39:58Z","closed_at":"2026-05-05T05:39:58Z","close_reason":"Completed terminal smart-money profile migration","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-igk","title":"Add plan mode","description":"Implement a user-facing plan mode in the application so users can switch into planning before taking action. Scope to be clarified from existing app patterns.","status":"closed","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:22:37Z","created_by":"dirtydishes","updated_at":"2026-05-04T04:26:18Z","started_at":"2026-05-04T04:22:40Z","closed_at":"2026-05-04T04:26:18Z","close_reason":"Implemented as a global pi extension toggled with Shift+P","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-biq","title":"Finish raw live options delivery and filter/backpressure observability","description":"The smart-money signal path and Tape filters are in place, but the next firehose pass should finish server-side selective raw live delivery for options subscriptions and add explicit filtered-out/backpressure observability for API/web counters. This was discovered while landing islandflow-e4r.\n","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:28:58Z","created_by":"dirtydishes","updated_at":"2026-04-29T03:54:12Z","started_at":"2026-04-29T03:54:12Z","dependencies":[{"issue_id":"islandflow-biq","depends_on_id":"islandflow-e4r","type":"discovered-from","created_at":"2026-04-28T16:28:58Z","created_by":"auto-import","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-2df","title":"Publish 2026-05-18 git standup summary","description":"Why: the daily automation needs a grounded standup summary for May 18, 2026. What: review commits from 2026-05-18, create a scannable HTML summary in docs/daily-git, and capture only commit/file-backed statements.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:41:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:42:42Z","started_at":"2026-05-19T18:41:10Z","closed_at":"2026-05-19T18:42:42Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-x70","title":"Create 2026-05-17 git standup summary","description":"Why this issue exists and what needs to be done:\\n- Produce the daily automation summary for 2026-05-17 git activity.\\n- Ground statements in commits, PRs, and touched files only.\\n- Create a user-readable HTML document in docs/general and update automation memory.\\n- Complete the Beads sync and git push workflow after documenting the run.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:43Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:05:37Z","started_at":"2026-05-18T13:01:53Z","closed_at":"2026-05-18T13:05:37Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-zsy","title":"Expose Forgejo SSH on a direct DNS hostname","description":"git.deltaisland.io currently resolves through Cloudflare's proxy, so SSH on port 2222 does not complete even though the Forgejo container is listening on the host. If SSH-based git/beads workflows are desired, add a DNS-only hostname (or adjust the existing record) that points directly at the server for Forgejo SSH.","status":"open","priority":3,"issue_type":"task","created_at":"2026-05-17T10:34:06Z","created_by":"delta","updated_at":"2026-05-17T10:34:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-38p","title":"Add native deployment unit templates and rollback helpers","description":"The deploy helper now supports --runtime native, but the repo still relies on operator-managed systemd units and manual rollback. Add checked-in native deployment templates or provisioning guidance for the expected units, and consider lightweight rollback/smoke-test helpers once the host-native path is exercised on the real VPS.","status":"open","priority":3,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:46:42Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:46:42Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html b/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html new file mode 100644 index 0000000..5a33e66 --- /dev/null +++ b/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html @@ -0,0 +1,566 @@ + + + + + + Daily Git Summary for 2026-05-18 + + + +
+
+ Daily Git Summary +

Standup summary for Monday, May 18, 2026

+

+ Git history for May 18 shows four commits. One feature commit introduced an Alpaca-backed news wire across ingest, + storage, API, and web surfaces; the other three commits updated workflow docs, beads state, and the previous + standup summary. +

+
+
+ Commits +
4
+
+
+ Files Touched +
35
+
+
+ Insertions +
1963
+
+
+ Deletions +
52
+
+
+
+ +
+

Summary

+
+
+ Primary Delivery +

+ Commit 906fe411 added a new services/ingest-news service, news persistence in + packages/storage, API endpoints in services/api, and a live news view in + apps/web/app/terminal.tsx plus apps/web/app/news/page.tsx. +

+
+
+ Docs And Workflow +

+ Commits 62aae708, 687a2170, and 04baeceb updated the previous standup + report, beads state, deployment/docker/workspace-root/package.json, and the repo-level + AGENTS.md instructions. +

+
+
+ Standup Framing +

+ Yesterday’s visible product work centered on making live Alpaca news available end to end. The remaining + activity was project hygiene and documentation. +

+
+
+
+ +
+

Changes Made

+
+
+
+ update beads + 687a2170 + 2026-05-18 03:15 -0400 + 1 file +
+

+ Added one line to deployment/docker/workspace-root/package.json. The local git history does not + show more context beyond the file touch and commit subject. +

+
+ +
+
+ docs(general): add 2026-05-17 standup summary + 62aae708 + 2026-05-18 09:05 -0400 + 2 files +
+

+ Added the prior day’s report at docs/general/2026-05-18-standup-summary-2026-05-17.html and + updated .beads/issues.jsonl. +

+
+ docs/general/2026-05-18-standup-summary-2026-05-17.html + .beads/issues.jsonl +
+
+ +
+
+ add alpaca news wire across ingest api and web + 906fe411 + 2026-05-18 16:55 -0400 + 31 files + +1407 / -50 +
+
    +
  • + Added a new ingest service in services/ingest-news/src/index.ts that backfills Alpaca news, + subscribes to the Alpaca news websocket, resolves symbols, and publishes NewsStory payloads to + NATS. +
    • +
  • + Extended shared contracts in packages/types/src/events.ts and + packages/types/src/live.ts, plus new storage support in + packages/storage/src/news.ts and packages/storage/src/clickhouse.ts. +
    • +
  • + Wired the API to store, fan out, and expose news via /news and /history/news in + services/api/src/index.ts and live-session updates in services/api/src/live.ts. +
    • +
  • + Added a web route in apps/web/app/news/page.tsx, a news pane and drawer in + apps/web/app/terminal.tsx, and related styling in apps/web/app/globals.css. +
    • +
  • + Updated runtime packaging and local/dev deployment surfaces, including + deployment/docker/docker-compose.yml, Dockerfiles, scripts/dev.ts, and + scripts/deploy.ts. +
    • +
  • + Added tests in packages/storage/tests/news.test.ts, + services/ingest-news/tests/symbols.test.ts, and adjusted + apps/web/app/terminal.test.ts plus packages/types/tests/live.test.ts. +
    • +
+
+ services/ingest-news/src/index.ts + packages/storage/src/news.ts + services/api/src/index.ts + apps/web/app/terminal.tsx + apps/web/app/news/page.tsx + apps/web/app/globals.css +
+
+ +
+
+ update turn docs and beads workflow + 04baeceb + 2026-05-18 21:32 -0400 + 1 file +
+

+ Edited AGENTS.md to update turn-document and beads workflow guidance. +

+
+
+
+ +
+

Context

+

+ This summary is based on local git history between 2026-05-18 00:00 -0400 and + 2026-05-19 00:00 -0400. The repository uses Bun, TypeScript, NATS/JetStream, ClickHouse, and a Next.js + web app, so the main feature commit spans service ingestion, shared types, persistence, API delivery, and the UI. +

+
+ +
+

Important Implementation Details

+
+
+

News ingestion was introduced as a first-class service

+

+ services/ingest-news/src/index.ts authenticates against Alpaca, backfills recent news, subscribes + to live updates, resolves symbols, validates payloads with NewsStorySchema, and publishes them onto + the repo’s bus layer. +

+ 
const backfill = await fetchBackfill();
+for (const item of backfill.reverse()) {
+  await publishStory(item);
+}
+
+if (msg === "authenticated") {
+  ws.send(JSON.stringify({ action: "subscribe", news: ["*"] }));
+}
+
+ +
+

API and live session support were expanded for news

+

+ services/api/src/index.ts now ensures the news table exists, subscribes to a news consumer, fans + out live updates, and exposes both recent and paginated history endpoints. +

+ 
if (req.method === "GET" && url.pathname === "/news") {
+  const limit = parseLimit(url.searchParams.get("limit") ?? "100");
+  const data = await fetchRecentNews(clickhouse, limit);
+  return jsonResponse({ data });
+}
+
+ +
+

The web terminal gained a dedicated news surface

+

+ apps/web/app/terminal.tsx added a live-only news pane, a per-story drawer, history loading, and a + new /news route entry point via apps/web/app/news/page.tsx. +

+ 
if (features.news) {
+  subscriptions.push({ channel: "news", snapshot_limit: LIVE_OPTIONS_HEAD_LIMIT });
+}
+
+export function NewsRoute() {
+  const state = useTerminal();
+  return (
+    <PageFrame title="News">
+      <div className="page-grid page-grid-news">
+        <NewsPane state={state} className="news-pane-full" />
+      </div>
+    </PageFrame>
+  );
+}
+
+
+
+ +
+

Expected Impact for End-Users

+
+
+ Live Terminal +

+ Users now have a dedicated news wire surface in the web terminal, including summary rows, story details, and + a direct link to the source article. +

+
+
+ Coverage +

+ News is now available alongside the repo’s existing live feeds, with shared symbol resolution and storage that + make the data retrievable through API history endpoints. +

+
+
+ Current Boundary +

+ The UI copy in the news pane explicitly marks news as live-only in v1, so replay users should not expect the + same behavior there yet. +

+
+
+
+ +
+

Validation

+
    +
  • Reviewed local git history with git log --since='2026-05-18 00:00' --until='2026-05-19 00:00'.
    • +
  • Used git log --stat, git show, and file-level history to anchor each summary item to specific commits and files.
    • +
  • No builds or tests were run for this reporting task because the work product is a git summary document, not a behavior change.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • + This report is grounded in local commit metadata only. No pull request identifiers were present in the inspected + git history, so the summary references commits and files instead of PR numbers. +
    • +
  • + The update beads commit touched only deployment/docker/workspace-root/package.json in + visible git output, so this report does not infer intent beyond that recorded file change. +
    • +
  • + Counts here describe May 18 commits only and exclude any uncommitted work present after that date. +
    • +
+
+ +
+

Follow-up Work

+
    +
  • + No new product follow-up items were derived from this reporting pass. The only beads item created for this task + is islandflow-2df, which tracks publication of this summary document. +
    • +
+
+
+ + From 8d39fb72a456bd43a5b9187f5f98a19e2ca33057 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 14:45:06 -0400 Subject: [PATCH 14/26] track pr conflict reconciliation --- .beads/issues.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index c61c799..67dab61 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-g3a","title":"Reconcile PR merge conflicts","description":"Resolve the current pull request conflicts for the nextjs-upgrade branch, validate the result, document the turn, and push the reconciled branch.","status":"in_progress","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:44:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:44:56Z","started_at":"2026-05-19T18:44:56Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-jbi","title":"Hydrate alert evidence details from ClickHouse","description":"Alert detail drawers need to fetch persisted alert context from ClickHouse by trace id, including linked flow packets, option prints, preserved execution context, and explicit missing refs for UI diagnostics.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:55:43Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:01:58Z","started_at":"2026-05-17T14:55:53Z","closed_at":"2026-05-17T15:01:58Z","close_reason":"Implemented ClickHouse-backed alert context hydration across storage, API, terminal drawer, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8kj","title":"Configure persistent beads Dolt remote on deltaisland server","description":"Install the beads and Dolt CLIs on the server, configure a persistent Dolt sync remote backed by the server-hosted Forgejo repository, verify refs/dolt/data publication, and document Nginx Proxy Manager / firewall considerations.","status":"closed","priority":1,"issue_type":"task","assignee":"delta","created_at":"2026-05-17T10:31:31Z","created_by":"delta","updated_at":"2026-05-17T10:37:47Z","started_at":"2026-05-17T10:32:16Z","closed_at":"2026-05-17T10:37:47Z","close_reason":"Installed bd and dolt on the server, configured the Forgejo-backed Dolt remote, published refs/dolt/data, and documented the setup.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} From 276d48950d62d8e13deaa45931ee5504b5d162a9 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 14:55:38 -0400 Subject: [PATCH 15/26] docs(daily-git): regenerate 2026-05-18 summary after merge --- .beads/issues.jsonl | 1 + ...2026-05-19-standup-summary-2026-05-18.html | 586 ++++++++---------- 2 files changed, 252 insertions(+), 335 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 2b26525..a1ec579 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -53,6 +53,7 @@ {"_type":"issue","id":"islandflow-zs0","title":"Migrate terminal UI to smart-money profiles","description":"Migrate apps/web terminal rendering to consume SmartMoneyEvent directly: primary profile, probability ladder, reason codes, and suppression/abstention state, while preserving legacy alert/classifier displays during the bridge.","status":"closed","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:23Z","created_by":"dirtydishes","updated_at":"2026-05-05T05:39:58Z","closed_at":"2026-05-05T05:39:58Z","close_reason":"Completed terminal smart-money profile migration","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-igk","title":"Add plan mode","description":"Implement a user-facing plan mode in the application so users can switch into planning before taking action. Scope to be clarified from existing app patterns.","status":"closed","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:22:37Z","created_by":"dirtydishes","updated_at":"2026-05-04T04:26:18Z","started_at":"2026-05-04T04:22:40Z","closed_at":"2026-05-04T04:26:18Z","close_reason":"Implemented as a global pi extension toggled with Shift+P","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-biq","title":"Finish raw live options delivery and filter/backpressure observability","description":"The smart-money signal path and Tape filters are in place, but the next firehose pass should finish server-side selective raw live delivery for options subscriptions and add explicit filtered-out/backpressure observability for API/web counters. This was discovered while landing islandflow-e4r.\n","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:28:58Z","created_by":"dirtydishes","updated_at":"2026-04-29T03:54:12Z","started_at":"2026-04-29T03:54:12Z","dependencies":[{"issue_id":"islandflow-biq","depends_on_id":"islandflow-e4r","type":"discovered-from","created_at":"2026-04-28T16:28:58Z","created_by":"auto-import","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-0ty","title":"Recreate May 18 standup summary after merge","description":"Regenerate docs/daily-git/2026-05-19-standup-summary-2026-05-18.html using merged history so it reflects all commits in the May 18 window, including native deployment and merge commits.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:53:48Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:55:33Z","started_at":"2026-05-19T18:53:52Z","closed_at":"2026-05-19T18:55:33Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-2df","title":"Publish 2026-05-18 git standup summary","description":"Why: the daily automation needs a grounded standup summary for May 18, 2026. What: review commits from 2026-05-18, create a scannable HTML summary in docs/daily-git, and capture only commit/file-backed statements.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:41:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:42:42Z","started_at":"2026-05-19T18:41:10Z","closed_at":"2026-05-19T18:42:42Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-x70","title":"Create 2026-05-17 git standup summary","description":"Why this issue exists and what needs to be done:\\n- Produce the daily automation summary for 2026-05-17 git activity.\\n- Ground statements in commits, PRs, and touched files only.\\n- Create a user-readable HTML document in docs/general and update automation memory.\\n- Complete the Beads sync and git push workflow after documenting the run.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:43Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:05:37Z","started_at":"2026-05-18T13:01:53Z","closed_at":"2026-05-18T13:05:37Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-zsy","title":"Expose Forgejo SSH on a direct DNS hostname","description":"git.deltaisland.io currently resolves through Cloudflare's proxy, so SSH on port 2222 does not complete even though the Forgejo container is listening on the host. If SSH-based git/beads workflows are desired, add a DNS-only hostname (or adjust the existing record) that points directly at the server for Forgejo SSH.","status":"open","priority":3,"issue_type":"task","created_at":"2026-05-17T10:34:06Z","created_by":"delta","updated_at":"2026-05-17T10:34:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html b/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html index 5a33e66..1d6e914 100644 --- a/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html +++ b/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html @@ -3,27 +3,23 @@ - Daily Git Summary for 2026-05-18 + Daily Git Summary for 2026-05-18 (Merged View) @@ -283,56 +238,54 @@
Daily Git Summary -

Standup summary for Monday, May 18, 2026

+

Standup summary for Monday, May 18, 2026 (after merge)

- Git history for May 18 shows four commits. One feature commit introduced an Alpaca-backed news wire across ingest, - storage, API, and web surfaces; the other three commits updated workflow docs, beads state, and the previous - standup summary. + This regenerated report uses merged history for the full May 18 local-day window + (2026-05-18 00:00 -0400 through 2026-05-19 00:00 -0400). It now includes eight commits, + including native deployment work and the merge commit that landed that line of work on main.

-
+
Commits -
4
+
8
- Files Touched -
35
+ Unique Files +
68
Insertions -
1963
+
4244
Deletions -
52
+
194

Summary

-
-
- Primary Delivery +
+
+ User-facing delivery

- Commit 906fe411 added a new services/ingest-news service, news persistence in - packages/storage, API endpoints in services/api, and a live news view in - apps/web/app/terminal.tsx plus apps/web/app/news/page.tsx. + Commit 906fe411 added Alpaca news wire support across ingest, storage, API, and web terminal/news + route surfaces.

-
- Docs And Workflow +
+ Platform and deployment delivery

- Commits 62aae708, 687a2170, and 04baeceb updated the previous standup - report, beads state, deployment/docker/workspace-root/package.json, and the repo-level - AGENTS.md instructions. + Commits d589858c and bdb9d9a9 added native deployment workflow, infra/user units, + cutover, rollback, and health-check scripts, then merged via 8f0794dd (PR #2).

-
- Standup Framing +
+ Workflow and docs updates

- Yesterday’s visible product work centered on making live Alpaca news available end to end. The remaining - activity was project hygiene and documentation. + Commits 687a2170, 62aae708, 48095fce, and 04baeceb updated + beads/docs instructions and added turn/standup documentation.

@@ -341,94 +294,99 @@

Changes Made

-
-
+
+
update beads - 687a2170 - 2026-05-18 03:15 -0400 + 687a2170 + 2026-05-18 03:15 -0400 1 file
-

- Added one line to deployment/docker/workspace-root/package.json. The local git history does not - show more context beyond the file touch and commit subject. -

+

Touched deployment/docker/workspace-root/package.json with one-line change.

-
-
- docs(general): add 2026-05-17 standup summary - 62aae708 - 2026-05-18 09:05 -0400 +
+
+ Implement native fast iterative deploy workflow + d589858c + 2026-05-18 03:34 -0400 + 17 files + +873 / -110 +
+
    +
  • Expanded scripts/deploy.ts for native deploy runtime behavior.
    • +
  • Added native user-unit templates and rollback/health tooling in deployment/native/.
    • +
  • Added associated plan and turn documents in docs/plans and docs/turns.
    • +
+
+ +
+
+ fix(api): remove duplicate alert context import + 48095fce + 2026-05-18 09:04 -0400 2 files
-

- Added the prior day’s report at docs/general/2026-05-18-standup-summary-2026-05-17.html and - updated .beads/issues.jsonl. -

-
- docs/general/2026-05-18-standup-summary-2026-05-17.html - .beads/issues.jsonl -
+

Removed duplicate import in services/api/src/index.ts and added a turn doc.

-
-
+
+
+ docs(general): add 2026-05-17 standup summary + 62aae708 + 2026-05-18 09:05 -0400 + 2 files +
+

Added docs/general/2026-05-18-standup-summary-2026-05-17.html and updated beads state.

+
+ +
+
add alpaca news wire across ingest api and web - 906fe411 - 2026-05-18 16:55 -0400 + 906fe411 + 2026-05-18 16:55 -0400 31 files +1407 / -50
    -
  • - Added a new ingest service in services/ingest-news/src/index.ts that backfills Alpaca news, - subscribes to the Alpaca news websocket, resolves symbols, and publishes NewsStory payloads to - NATS. -
    • -
  • - Extended shared contracts in packages/types/src/events.ts and - packages/types/src/live.ts, plus new storage support in - packages/storage/src/news.ts and packages/storage/src/clickhouse.ts. -
    • -
  • - Wired the API to store, fan out, and expose news via /news and /history/news in - services/api/src/index.ts and live-session updates in services/api/src/live.ts. -
    • -
  • - Added a web route in apps/web/app/news/page.tsx, a news pane and drawer in - apps/web/app/terminal.tsx, and related styling in apps/web/app/globals.css. -
    • -
  • - Updated runtime packaging and local/dev deployment surfaces, including - deployment/docker/docker-compose.yml, Dockerfiles, scripts/dev.ts, and - scripts/deploy.ts. -
    • -
  • - Added tests in packages/storage/tests/news.test.ts, - services/ingest-news/tests/symbols.test.ts, and adjusted - apps/web/app/terminal.test.ts plus packages/types/tests/live.test.ts. -
    • +
  • Created services/ingest-news and wired Alpaca backfill/websocket ingestion.
    • +
  • Added news types/storage contracts in packages/types and packages/storage.
    • +
  • Extended API live/history endpoints and web terminal/news route rendering.
-
- services/ingest-news/src/index.ts - packages/storage/src/news.ts - services/api/src/index.ts - apps/web/app/terminal.tsx - apps/web/app/news/page.tsx - apps/web/app/globals.css -
-
-
+
+
+ Implement native public edge cutover + bdb9d9a9 + 2026-05-18 19:55 -0400 + 29 files + +1215 / -31 +
+
    +
  • Added native infra system units and scripts for bootstrap/start/stop/cutover/full rollback.
    • +
  • Updated deploy docs and runtime config files under deployment/native/config.
    • +
  • Added turn doc docs/turns/2026-05-18-native-public-edge-cutover.html.
    • +
+
+ +
+
+ Merge pull request 'Native public edge cutover with Docker rollback path' (#2) + 8f0794dd + 2026-05-19 00:09 +0000 + merge commit +
+

Merged native-deploy into main within the May 18 US/Eastern day window.

+
+ +
+
update turn docs and beads workflow - 04baeceb - 2026-05-18 21:32 -0400 + 04baeceb + 2026-05-18 21:32 -0400 1 file
-

- Edited AGENTS.md to update turn-document and beads workflow guidance. -

+

Updated repository-level instructions in AGENTS.md.

@@ -436,92 +394,59 @@

Context

- This summary is based on local git history between 2026-05-18 00:00 -0400 and - 2026-05-19 00:00 -0400. The repository uses Bun, TypeScript, NATS/JetStream, ClickHouse, and a Next.js - web app, so the main feature commit spans service ingestion, shared types, persistence, API delivery, and the UI. + The earlier report was generated before merged history included the native deployment branch on main. + This recreation uses git log --all over the same date window, so it captures both feature work and + merged operational/deployment work visible after PR merge.

Important Implementation Details

-
-
-

News ingestion was introduced as a first-class service

+
+
+

News wire ingestion and delivery path

- services/ingest-news/src/index.ts authenticates against Alpaca, backfills recent news, subscribes - to live updates, resolves symbols, validates payloads with NewsStorySchema, and publishes them onto - the repo’s bus layer. -

- 
const backfill = await fetchBackfill();
-for (const item of backfill.reverse()) {
-  await publishStory(item);
-}
-
-if (msg === "authenticated") {
-  ws.send(JSON.stringify({ action: "subscribe", news: ["*"] }));
-}
-
- -
-

API and live session support were expanded for news

-

- services/api/src/index.ts now ensures the news table exists, subscribes to a news consumer, fans - out live updates, and exposes both recent and paginated history endpoints. -

- 
if (req.method === "GET" && url.pathname === "/news") {
-  const limit = parseLimit(url.searchParams.get("limit") ?? "100");
-  const data = await fetchRecentNews(clickhouse, limit);
-  return jsonResponse({ data });
-}
-
- -
-

The web terminal gained a dedicated news surface

-

- apps/web/app/terminal.tsx added a live-only news pane, a per-story drawer, history loading, and a - new /news route entry point via apps/web/app/news/page.tsx. + The news pipeline added a new ingest service and API fanout channel, then exposed UI surfaces in + /news and terminal panes. 

if (features.news) {
   subscriptions.push({ channel: "news", snapshot_limit: LIVE_OPTIONS_HEAD_LIMIT });
-}
-
-export function NewsRoute() {
-  const state = useTerminal();
-  return (
-    <PageFrame title="News">
-      <div className="page-grid page-grid-news">
-        <NewsPane state={state} className="news-pane-full" />
-      </div>
-    </PageFrame>
-  );
 }
-
+
+
+

Native deployment hardening

+

+ Deployment scripts and unit templates now include direct scripts for cutover and rollback, with infra and + service checks under deployment/native/. +

+ 
deployment/native/cutover.sh
+deployment/native/full-rollback.sh
+deployment/native/install-infra-units.sh
+
+
+

Merged history effect on standup scope

+

+ The merged view increased the standup scope from 4 to 8 commits and from 35 to 68 unique files touched for the + same local-day window. +

+

Expected Impact for End-Users

-
-
- Live Terminal -

- Users now have a dedicated news wire surface in the web terminal, including summary rows, story details, and - a direct link to the source article. -

+
+
+ Trading UI users +

Live news wire data is now available in terminal surfaces alongside existing market/event feeds.

-
- Coverage -

- News is now available alongside the repo’s existing live feeds, with shared symbol resolution and storage that - make the data retrievable through API history endpoints. -

+
+ Operators +

Native deployment and rollback procedures now have first-class scripted and documented paths.

-
- Current Boundary -

- The UI copy in the news pane explicitly marks news as live-only in v1, so replay users should not expect the - same behavior there yet. -

+
+ Team reporting +

This standup report now matches merged repository history instead of pre-merge branch-local history.

@@ -529,36 +454,27 @@ export function NewsRoute() {

Validation

    -
  • Reviewed local git history with git log --since='2026-05-18 00:00' --until='2026-05-19 00:00'.
    • -
  • Used git log --stat, git show, and file-level history to anchor each summary item to specific commits and files.
    • -
  • No builds or tests were run for this reporting task because the work product is a git summary document, not a behavior change.
    • +
  • Used git fetch --all --prune before recomputing history.
    • +
  • Used git log --all over the May 18 ET window to include merged commits.
    • +
  • Used git log --stat --summary and --numstat to ground file and line-count statements.
    • +
  • No build/test commands were run because this task only regenerates reporting documentation.

Issues, Limitations, and Mitigations

    -
  • - This report is grounded in local commit metadata only. No pull request identifiers were present in the inspected - git history, so the summary references commits and files instead of PR numbers. -
    • -
  • - The update beads commit touched only deployment/docker/workspace-root/package.json in - visible git output, so this report does not infer intent beyond that recorded file change. -
    • -
  • - Counts here describe May 18 commits only and exclude any uncommitted work present after that date. -
    • +
  • This report describes commit history only and does not infer intent beyond commit messages and touched files.
    • +
  • Commit 8f0794dd is timestamped in UTC; it still falls on May 18 in US/Eastern, so it is included.
    • +
  • Metrics are based on local git history at regeneration time and can change if additional backdated commits appear.

Follow-up Work

    -
  • - No new product follow-up items were derived from this reporting pass. The only beads item created for this task - is islandflow-2df, which tracks publication of this summary document. -
    • +
  • This regeneration is tracked by beads issue islandflow-0ty.
    • +
  • No additional follow-up work was identified during this documentation-only task.
From 4bacf2c2f8ee48cbeecda7985bfccd1772e777f3 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 14:59:58 -0400 Subject: [PATCH 16/26] publish docs index and github pages workflow --- .beads/issues.jsonl | 1 + .github/workflows/docs-pages.yml | 56 ++ docs/index.html | 638 ++++++++++++++++++ .../2026-05-19-publish-docs-pages-index.html | 195 ++++++ scripts/generate-docs-index.mjs | 421 ++++++++++++ 5 files changed, 1311 insertions(+) create mode 100644 .github/workflows/docs-pages.yml create mode 100644 docs/index.html create mode 100644 docs/turns/2026-05-19-publish-docs-pages-index.html create mode 100644 scripts/generate-docs-index.mjs diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index a1ec579..e6a01f4 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -14,6 +14,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-tqk","title":"publish docs/ to github pages with navigable index","description":"Set up docs deployment so repository docs are published to dirtydishes.github.io/islandflow/docs with a nicer, browsable experience than a raw file listing.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:56:02Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:59:55Z","started_at":"2026-05-19T18:56:04Z","closed_at":"2026-05-19T18:59:55Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lm6","title":"Clarify repo turn documentation scope","description":"Update AGENTS.md so repository turn documentation clearly uses repo-local docs/turns and impeccable styling, without inheriting global non-repo computer-task styling.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T12:05:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T12:06:12Z","started_at":"2026-05-19T12:05:14Z","closed_at":"2026-05-19T12:06:12Z","close_reason":"Verified AGENTS.md now scopes repo turn docs to docs/turns and makes impeccable the styling authority; added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-6iq","title":"Update README for current project state","description":"Resolve README merge conflicts and document the current project state, including the smart money classification taxonomy, Next.js update, and deployment workflow changes.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:37:24Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:40:01Z","started_at":"2026-05-19T11:37:31Z","closed_at":"2026-05-19T11:40:01Z","close_reason":"README conflict resolved and current project state documented, including smart-money taxonomy, Next.js update, and deployment workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lib","title":"Upgrade apps/web to Next.js 16.2.6","description":"Upgrade the web app dependency stack to Next.js 16.2.6 with React 19, refresh Bun and mirrored Docker workspace lockfiles, keep runtime behavior unchanged, fix any focused web test fallout, validate the web build and targeted route tests, and document the completed work.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:04:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:31:23Z","started_at":"2026-05-19T11:04:57Z","closed_at":"2026-05-19T11:31:23Z","close_reason":"Upgraded apps/web to Next.js 16.2.6 with React 19, refreshed Bun lockfiles including the Docker workspace mirror, fixed the React 19 nullable ref type issue, and validated the web build, focused tests, Docker workspace sync, and route smoke checks.","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/.github/workflows/docs-pages.yml b/.github/workflows/docs-pages.yml new file mode 100644 index 0000000..9c4db98 --- /dev/null +++ b/.github/workflows/docs-pages.yml @@ -0,0 +1,56 @@ +name: Publish Docs + +on: + push: + branches: + - main + paths: + - "docs/**" + - "scripts/generate-docs-index.mjs" + - ".github/workflows/docs-pages.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure Pages + uses: actions/configure-pages@v5 + + - name: Build docs index + run: node scripts/generate-docs-index.mjs + + - name: Prepare static site payload + run: | + mkdir -p site/docs + cp -R docs/. site/docs/ + printf '%s\n' 'Islandflow DocsContinue to docs' > site/index.html + touch site/.nojekyll + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + needs: build + runs-on: ubuntu-latest + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..211c5ac --- /dev/null +++ b/docs/index.html @@ -0,0 +1,638 @@ + + + + + + Islandflow Docs + + + +
+
+

Islandflow docs index

+

A browsable index of files under docs/ with filtering and grouped navigation.

+
+ +
+
35 of 35 files shown
+ + +
+ +
+
+

turns 28

+ +
+ + +
+

daily-git 1

+ +
+ + +
+

general 2

+ +
+ + +
+

plans 2

+ +
+ + +
+

root 2

+ +
+
+

No files match that filter.

+
+ + + + diff --git a/docs/turns/2026-05-19-publish-docs-pages-index.html b/docs/turns/2026-05-19-publish-docs-pages-index.html new file mode 100644 index 0000000..9946b33 --- /dev/null +++ b/docs/turns/2026-05-19-publish-docs-pages-index.html @@ -0,0 +1,195 @@ + + + + + + Turn Report - Publish Docs to GitHub Pages + + + +
+

Publish docs/ to GitHub Pages with navigable index

+

Completed on May 19, 2026 at 9:38 AM ET.

+ +
+

Summary

+

+ Added an automated docs publishing flow to GitHub Pages and generated a new + docs/index.html browsing experience so docs are easy to navigate at + /islandflow/docs/. +

+
+ +
+

Changes Made

+
    +
  • Added scripts/generate-docs-index.mjs to build a browsable index of files under docs/.
    • +
  • Added .github/workflows/docs-pages.yml to publish docs to GitHub Pages on pushes to main.
    • +
  • Generated docs/index.html from current docs content.
    • +
  • Configured deployment artifact layout so docs are available at /docs/ under the project Pages site.
    • +
+
+ +
+

Context

+

+ The repository already stores operational and implementation documentation under + docs/, but there was no dedicated GitHub Pages pipeline and no curated + index page for discovery. This task focused on syncing that folder to Pages and + making it easy to browse by category and filename. +

+
+ +
+

Important Implementation Details

+
    +
  • The index generator excludes hidden files and avoids self-including docs/index.html.
    • +
  • Files are grouped by first path segment (turns, general, plans, and others) with quick category chips.
    • +
  • The index includes client-side filtering so users can search docs by path text in-browser.
    • +
  • Pages deployment packages a site/ payload where docs are copied into site/docs and root redirects to ./docs/.
    • +
+
+ +
+

Relevant Diff Snippets

+

+ Snippets are shown in a compact style aligned with diffs.com presentation patterns. +

+ 
+++ .github/workflows/docs-pages.yml
+name: Publish Docs
+on:
+  push:
+    branches: [main]
+    paths:
+      - "docs/**"
+      - "scripts/generate-docs-index.mjs"
+      - ".github/workflows/docs-pages.yml"
+  workflow_dispatch:
+
+jobs:
+  build:
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/configure-pages@v5
+      - run: node scripts/generate-docs-index.mjs
+      - run: cp -R docs/. site/docs/
+      - uses: actions/upload-pages-artifact@v3
+  deploy:
+    needs: build
+    steps:
+      - uses: actions/deploy-pages@v4
+ 
+++ scripts/generate-docs-index.mjs
+const files = await collectDocsFiles(docsDir);
+const html = renderDocument(files);
+await fs.writeFile(outputFile, html, "utf8");
+
+// Generated index features:
+// - grouped sections
+// - search filter
+// - file size and modified time metadata
+// - links preserving docs folder structure
+
+ +
+

Expected Impact for End-Users

+
    +
  • Docs are reachable via a stable Pages URL path: dirtydishes.github.io/islandflow/docs/.
    • +
  • Readers can quickly scan categories and search by filename instead of relying on raw directory browsing.
    • +
  • New docs added to the repository are published automatically on main pushes.
    • +
+
+ +
+

Validation

+
    +
  • Ran node scripts/generate-docs-index.mjs successfully.
    • +
  • Ran node --check scripts/generate-docs-index.mjs for syntax validation.
    • +
  • Confirmed generated index contains expected navigation/search markers and category anchors.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • GitHub Pages must be enabled for this repository and set to GitHub Actions deployment.
    • +
  • The index reflects files present at build time and does not include full-text search inside documents.
    • +
  • Markdown files are linked as-is; rendering behavior depends on GitHub Pages static hosting behavior.
    • +
+
+ +
+

Follow-up Work

+
    +
  • Add a docs landing page summary for key collections (turn docs, runbooks, daily notes).
    • +
  • Optionally add link-checking in CI for docs URLs and local references.
    • +
  • Consider tagging docs with metadata for richer filtering by date, topic, and type.
    • +
+
+
+ + diff --git a/scripts/generate-docs-index.mjs b/scripts/generate-docs-index.mjs new file mode 100644 index 0000000..cf64a9d --- /dev/null +++ b/scripts/generate-docs-index.mjs @@ -0,0 +1,421 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; + +const docsDir = path.resolve(process.cwd(), "docs"); +const outputFile = path.join(docsDir, "index.html"); + +const dateFormatter = new Intl.DateTimeFormat("en-US", { + dateStyle: "medium", + timeStyle: "short", +}); + +function escapeHtml(value) { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function formatBytes(bytes) { + if (bytes < 1024) { + return `${bytes} B`; + } + + const units = ["KB", "MB", "GB"]; + let size = bytes / 1024; + let unitIndex = 0; + + while (size >= 1024 && unitIndex < units.length - 1) { + size /= 1024; + unitIndex += 1; + } + + return `${size.toFixed(size >= 10 ? 0 : 1)} ${units[unitIndex]}`; +} + +function docsHref(relativePath) { + const encoded = relativePath + .split("/") + .map((part) => encodeURIComponent(part)) + .join("/"); + return `./${encoded}`; +} + +async function collectDocsFiles(rootDir, currentDir = rootDir, acc = []) { + const entries = await fs.readdir(currentDir, { withFileTypes: true }); + const sortedEntries = entries.sort((a, b) => a.name.localeCompare(b.name)); + + for (const entry of sortedEntries) { + if (entry.name.startsWith(".")) { + continue; + } + + const absolutePath = path.join(currentDir, entry.name); + const relativePath = path.relative(rootDir, absolutePath).replaceAll(path.sep, "/"); + + if (relativePath === "index.html") { + continue; + } + + if (entry.isDirectory()) { + await collectDocsFiles(rootDir, absolutePath, acc); + continue; + } + + if (entry.isFile()) { + const stats = await fs.stat(absolutePath); + + acc.push({ + relativePath, + category: relativePath.includes("/") ? relativePath.split("/")[0] : "root", + sizeBytes: stats.size, + modifiedAt: stats.mtime, + }); + } + } + + return acc; +} + +function groupByCategory(items) { + const groups = new Map(); + for (const item of items) { + if (!groups.has(item.category)) { + groups.set(item.category, []); + } + groups.get(item.category).push(item); + } + return groups; +} + +function sortedCategories(groups) { + const preferredOrder = ["turns", "daily-git", "general", "plans", "root"]; + const groupNames = [...groups.keys()]; + return groupNames.sort((a, b) => { + const aIndex = preferredOrder.indexOf(a); + const bIndex = preferredOrder.indexOf(b); + + if (aIndex !== -1 || bIndex !== -1) { + if (aIndex === -1) return 1; + if (bIndex === -1) return -1; + return aIndex - bIndex; + } + + return a.localeCompare(b); + }); +} + +function renderDocument(items) { + const sortedItems = [...items].sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime()); + const groups = groupByCategory(sortedItems); + const categories = sortedCategories(groups); + const totalCount = sortedItems.length; + + const categoryChips = categories + .map((category) => { + const count = groups.get(category).length; + return `${escapeHtml( + category + )} ${count}`; + }) + .join("\n"); + + const groupsMarkup = categories + .map((category) => { + const entries = groups.get(category); + const entryMarkup = entries + .map((entry) => { + const extension = path.extname(entry.relativePath).replace(".", "") || "file"; + const searchable = `${entry.relativePath} ${category}`.toLowerCase(); + return ` +
  • + ${escapeHtml( + entry.relativePath + )} +
    + ${escapeHtml(extension)} + ${escapeHtml(formatBytes(entry.sizeBytes))} + ${escapeHtml(dateFormatter.format(entry.modifiedAt))} +
    +
    • + `; + }) + .join("\n"); + + return ` +
    +

    ${escapeHtml(category)} ${entries.length}

    +
      + ${entryMarkup} +
    +
    + `; + }) + .join("\n"); + + return ` + + + + + Islandflow Docs + + + +
    +
    +

    Islandflow docs index

    +

    A browsable index of files under docs/ with filtering and grouped navigation.

    +
    + +
    +
    ${totalCount} of ${totalCount} files shown
    + + +
    + +
    ${groupsMarkup}
    +

    No files match that filter.

    +
    + + + + +`; +} + +async function main() { + const files = await collectDocsFiles(docsDir); + const html = renderDocument(files); + await fs.writeFile(outputFile, html, "utf8"); + console.log(`Generated ${outputFile} with ${files.length} entries.`); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); From bca74d1811ae84da2bdb57cab91fb3ac74025951 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 17:00:15 -0400 Subject: [PATCH 17/26] update beads for codex forgejo status patch --- .beads/issues.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index e6a01f4..6d09bd4 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-jor","title":"Support Forgejo pull request status in desktop git panel","description":"The desktop app currently reports pull request status unavailable when a repository only has a Forgejo remote. Add native Forgejo/Gitea-style remote detection and pull request status lookup so Forgejo-only repositories can show PR state in the Codex app git panel.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T20:55:15Z","created_by":"dirtydishes","updated_at":"2026-05-19T20:59:46Z","started_at":"2026-05-19T20:55:25Z","closed_at":"2026-05-19T20:59:46Z","close_reason":"Patched the installed Codex desktop app bundle with a Forgejo PR status fallback and documented the local change.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-g3a","title":"Reconcile PR merge conflicts","description":"Resolve the current pull request conflicts for the nextjs-upgrade branch, validate the result, document the turn, and push the reconciled branch.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:44:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:47:35Z","started_at":"2026-05-19T18:44:56Z","closed_at":"2026-05-19T18:47:35Z","close_reason":"Merged forgejo/main into nextjs-upgrade, resolved README and Beads conflicts, updated JetStream retention tests, validated deploy help, Docker workspace sync, API/bus tests, and web build, and added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-jbi","title":"Hydrate alert evidence details from ClickHouse","description":"Alert detail drawers need to fetch persisted alert context from ClickHouse by trace id, including linked flow packets, option prints, preserved execution context, and explicit missing refs for UI diagnostics.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:55:43Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:01:58Z","started_at":"2026-05-17T14:55:53Z","closed_at":"2026-05-17T15:01:58Z","close_reason":"Implemented ClickHouse-backed alert context hydration across storage, API, terminal drawer, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8kj","title":"Configure persistent beads Dolt remote on deltaisland server","description":"Install the beads and Dolt CLIs on the server, configure a persistent Dolt sync remote backed by the server-hosted Forgejo repository, verify refs/dolt/data publication, and document Nginx Proxy Manager / firewall considerations.","status":"closed","priority":1,"issue_type":"task","assignee":"delta","created_at":"2026-05-17T10:31:31Z","created_by":"delta","updated_at":"2026-05-17T10:37:47Z","started_at":"2026-05-17T10:32:16Z","closed_at":"2026-05-17T10:37:47Z","close_reason":"Installed bd and dolt on the server, configured the Forgejo-backed Dolt remote, published refs/dolt/data, and documented the setup.","dependency_count":0,"dependent_count":0,"comment_count":0} From 4b8eaae0ee8b882948d312fb8a9d88f61db09cb7 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 19:28:33 -0400 Subject: [PATCH 18/26] document native options recovery and clean up the unit override --- .beads/issues.jsonl | 3 + deployment/native/README.md | 2 + .../user/islandflow-ingest-options.service | 1 - ...19-native-options-recovery-guardrails.html | 183 ++++++++++++++++++ 4 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 docs/turns/2026-05-19-native-options-recovery-guardrails.html diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 6d09bd4..3df43cf 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -15,6 +15,9 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-wf5","title":"Harden native options provider configuration after synthetic recovery","description":"Native production recovery restored OPTIONS_INGEST_ADAPTER=synthetic because the current Alpaca setup fails authentication and crash-loops ingest-options. Follow up by deciding whether production options should remain synthetic or move to a supported live provider auth path, then add a deploy-time smoke test or config validation that catches provider auth failures before native cutover.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:27:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-m83","title":"Restore options ingestion and print generation on native deployment","description":"After moving the production/VPS deployment from Docker-managed services to the native runtime, the options feed appears behind and fresh option prints are not reaching the UI. Investigate the native deployment path on the server, identify the ingestion or compute breakage, apply the required code and/or host configuration changes, validate that fresh option prints resume, and document any follow-up operational work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:20:01Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:52Z","started_at":"2026-05-19T23:20:10Z","closed_at":"2026-05-19T23:27:52Z","close_reason":"Restored native options ingest by switching the VPS back to the last known-good synthetic adapter, verified fresh option prints and compute output, and documented the native env precedence gotcha.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-o1v","title":"Add SCM provider layer with Forgejo detection","description":"Implement provider-aware source-control detection and mirror-aware guardrails for repo automation so Forgejo remotes are treated as authoritative when present.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:04:33Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:06:55Z","started_at":"2026-05-19T23:04:35Z","closed_at":"2026-05-19T23:06:55Z","close_reason":"created by mistake during interrupted turn; no implementation was started","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-tqk","title":"publish docs/ to github pages with navigable index","description":"Set up docs deployment so repository docs are published to dirtydishes.github.io/islandflow/docs with a nicer, browsable experience than a raw file listing.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:56:02Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:59:55Z","started_at":"2026-05-19T18:56:04Z","closed_at":"2026-05-19T18:59:55Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lm6","title":"Clarify repo turn documentation scope","description":"Update AGENTS.md so repository turn documentation clearly uses repo-local docs/turns and impeccable styling, without inheriting global non-repo computer-task styling.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T12:05:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T12:06:12Z","started_at":"2026-05-19T12:05:14Z","closed_at":"2026-05-19T12:06:12Z","close_reason":"Verified AGENTS.md now scopes repo turn docs to docs/turns and makes impeccable the styling authority; added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-6iq","title":"Update README for current project state","description":"Resolve README merge conflicts and document the current project state, including the smart money classification taxonomy, Next.js update, and deployment workflow changes.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:37:24Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:40:01Z","started_at":"2026-05-19T11:37:31Z","closed_at":"2026-05-19T11:40:01Z","close_reason":"README conflict resolved and current project state documented, including smart-money taxonomy, Next.js update, and deployment workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/deployment/native/README.md b/deployment/native/README.md index c421c51..569cdb8 100644 --- a/deployment/native/README.md +++ b/deployment/native/README.md @@ -98,6 +98,8 @@ These are written for the current VPS layout: - Bun binary: `/home/delta/.bun/bin/bun` - env file: `/home/delta/islandflow/.env` +Important: treat `/home/delta/islandflow/.env` as the effective source of truth for adapter selection. The Bun-launched services read that file directly at runtime, so a conflicting `OPTIONS_INGEST_ADAPTER` value in `.env` can still win over a systemd-only override and push `ingest-options` onto the wrong provider path. + ### Install the units ```bash diff --git a/deployment/native/systemd/user/islandflow-ingest-options.service b/deployment/native/systemd/user/islandflow-ingest-options.service index 10107b1..eac0a6c 100644 --- a/deployment/native/systemd/user/islandflow-ingest-options.service +++ b/deployment/native/systemd/user/islandflow-ingest-options.service @@ -7,7 +7,6 @@ Wants=network-online.target Type=simple WorkingDirectory=/home/delta/islandflow EnvironmentFile=/home/delta/islandflow/.env -Environment=OPTIONS_INGEST_ADAPTER=synthetic ExecStart=/home/delta/.bun/bin/bun services/ingest-options/src/index.ts Restart=always RestartSec=2 diff --git a/docs/turns/2026-05-19-native-options-recovery-guardrails.html b/docs/turns/2026-05-19-native-options-recovery-guardrails.html new file mode 100644 index 0000000..441ade2 --- /dev/null +++ b/docs/turns/2026-05-19-native-options-recovery-guardrails.html @@ -0,0 +1,183 @@ + + + + + + 2026-05-19 Native Options Recovery Guardrails + + + +
    +
    +

    Native Options Recovery Guardrails

    +

    + The production outage turned out to be a native deployment config mismatch, not a data-pipeline code failure. I restored the VPS to the last known-good synthetic options adapter, then tightened the checked-in native deployment assets so they no longer imply a systemd override will beat the repo .env. +

    +
    Generated 2026-05-19 19:24 EDT
    +
    + +
    +

    Summary

    +

    + The repo-side change is small and targeted: remove the misleading Environment=OPTIONS_INGEST_ADAPTER=synthetic line from the checked-in native ingest-options unit, and document that Bun-launched services effectively take adapter selection from /home/delta/islandflow/.env. +

    +
    + +
    +

    Changes Made

    +
      +
    • Removed the checked-in systemd override from deployment/native/systemd/user/islandflow-ingest-options.service.
      • +
    • Added an explicit env-precedence warning to deployment/native/README.md.
      • +
    • Captured the live diagnosis that the native server had drifted to OPTIONS_INGEST_ADAPTER=alpaca while the prior Docker deployment was running synthetic options.
      • +
    +
    + +
    +

    Context

    +

    + On the VPS, islandflow-ingest-options.service was crash-looping with repeated 401 Unauthorized responses from Alpaca while the rest of the native stack stayed healthy. The previous Docker-owned islandflow-vps-ingest-options-1 container showed OPTIONS_INGEST_ADAPTER=synthetic, which explains why the UI had been healthy before the runtime transition. +

    +
    + +
    +

    Important Implementation Details

    +
      +
    • The checked-in unit already referenced /home/delta/islandflow/.env, and Bun's runtime env loading meant a conflicting adapter value there still won in practice.
      • +
    • The static key currently stored as ALPACA_API_KEY does not authenticate the failing market-data snapshot request as a Bearer token.
      • +
    • Because the real outage fix required a server-side .env correction, this repo patch focuses on preventing operator confusion during the next native cutover.
      • +
    +
    + +
    +

    Relevant Diff Snippets

    +

    Unified diff blocks below are formatted for diffs-compatible rendering.

    + 
    diff --git a/deployment/native/README.md b/deployment/native/README.md
+@@ -98,6 +98,8 @@ These are written for the current VPS layout:
+ - Bun binary: `/home/delta/.bun/bin/bun`
+ - env file: `/home/delta/islandflow/.env`
+ 
++Important: treat `/home/delta/islandflow/.env` as the effective source of truth for adapter selection. The Bun-launched services read that file directly at runtime, so a conflicting `OPTIONS_INGEST_ADAPTER` value in `.env` can still win over a systemd-only override and push `ingest-options` onto the wrong provider path.
++
+ ### Install the units
+
+diff --git a/deployment/native/systemd/user/islandflow-ingest-options.service b/deployment/native/systemd/user/islandflow-ingest-options.service
+@@ -7,7 +7,6 @@ Wants=network-online.target
+ Type=simple
+ WorkingDirectory=/home/delta/islandflow
+ EnvironmentFile=/home/delta/islandflow/.env
+-Environment=OPTIONS_INGEST_ADAPTER=synthetic
+ ExecStart=/home/delta/.bun/bin/bun services/ingest-options/src/index.ts
    +
    + +
    +

    Expected Impact for End-Users

    +

    + End users should not see the options tape stall the next time native units are installed or audited by following the checked-in assets. Operators now have a clearer paper trail that the actual runtime adapter comes from the deployment env file. +

    +
    + +
    +

    Validation

    +
      +
    • Verified the native outage mode on the VPS: islandflow-ingest-options.service crash-looped on Alpaca 401 responses.
      • +
    • Confirmed the previous Docker container had been running OPTIONS_INGEST_ADAPTER=synthetic.
      • +
    • After the server-side env fix, confirmed fresh rows in default.option_prints and new compute emissions in the native logs.
      • +
    • Ran git diff to verify the repo change stayed scoped to the deployment README and the checked-in user unit.
      • +
    +
    + +
    +

    Issues, Limitations, and Mitigations

    +
      +
    • The repo patch does not add new credential support for Alpaca. It only documents the current env-precedence behavior and removes a misleading override.
      • +
    • The live server is restored with synthetic options, which matches the last known-good Docker behavior, but it is not a true live Alpaca ingest path.
      • +
    +
    + +
    +

    Follow-up Work

    +
      +
    • islandflow-wf5: Decide whether production options should remain synthetic or move to a fully supported live provider configuration.
      • +
    • islandflow-wf5: If Alpaca live data is still desired, add a validated auth flow and a deploy-time smoke test that catches provider auth failures before cutover.
      • +
    +
    +
    + + From e70835e9c4ea335dac42493249c996cbd560bcf8 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 19:40:20 -0400 Subject: [PATCH 19/26] fix native deploy ssh assumptions --- .beads/issues.jsonl | 1 + ...05-19-harden-native-ssh-deploy-checks.html | 191 ++++++++++++++++++ scripts/deploy.ts | 14 ++ 3 files changed, 206 insertions(+) create mode 100644 docs/turns/2026-05-19-harden-native-ssh-deploy-checks.html diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 3df43cf..59c55f5 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -15,6 +15,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-fmg","title":"Fix native deploy SSH path and verification cwd assumptions","description":"Native deploys over SSH assumed bun was already on PATH and that remote verification would run from the repository root. On the live VPS, non-login SSH shells omitted /home/delta/.bun/bin and remote native verification could not find deployment/native/check-native-infra.sh because it ran from the home directory. Update the deploy helper to prepend /Users/kell/.bun/bin when present and cd into the repo before native verification checks run.","status":"open","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:38:32Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:38:32Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-wf5","title":"Harden native options provider configuration after synthetic recovery","description":"Native production recovery restored OPTIONS_INGEST_ADAPTER=synthetic because the current Alpaca setup fails authentication and crash-loops ingest-options. Follow up by deciding whether production options should remain synthetic or move to a supported live provider auth path, then add a deploy-time smoke test or config validation that catches provider auth failures before native cutover.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:27:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-m83","title":"Restore options ingestion and print generation on native deployment","description":"After moving the production/VPS deployment from Docker-managed services to the native runtime, the options feed appears behind and fresh option prints are not reaching the UI. Investigate the native deployment path on the server, identify the ingestion or compute breakage, apply the required code and/or host configuration changes, validate that fresh option prints resume, and document any follow-up operational work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:20:01Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:52Z","started_at":"2026-05-19T23:20:10Z","closed_at":"2026-05-19T23:27:52Z","close_reason":"Restored native options ingest by switching the VPS back to the last known-good synthetic adapter, verified fresh option prints and compute output, and documented the native env precedence gotcha.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-o1v","title":"Add SCM provider layer with Forgejo detection","description":"Implement provider-aware source-control detection and mirror-aware guardrails for repo automation so Forgejo remotes are treated as authoritative when present.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:04:33Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:06:55Z","started_at":"2026-05-19T23:04:35Z","closed_at":"2026-05-19T23:06:55Z","close_reason":"created by mistake during interrupted turn; no implementation was started","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/docs/turns/2026-05-19-harden-native-ssh-deploy-checks.html b/docs/turns/2026-05-19-harden-native-ssh-deploy-checks.html new file mode 100644 index 0000000..7cee829 --- /dev/null +++ b/docs/turns/2026-05-19-harden-native-ssh-deploy-checks.html @@ -0,0 +1,191 @@ + + + + + + 2026-05-19 Harden Native SSH Deploy Checks + + + +
    +
    +

    Harden Native SSH Deploy Checks

    +

    + Native deploys over SSH were failing for avoidable operator reasons: the remote shell did not inherit Bun's install path, and native verification assumed it was already running from the repository root before it called checked-in health scripts. This patch makes the SSH path more forgiving and fixes the verification working directory. +

    +
    Generated 2026-05-19 19:38 EDT
    +
    + +
    +

    Summary

    +

    + Updated scripts/deploy.ts so native SSH deploys prepend $HOME/.bun/bin when it exists, and native verification now explicitly cds into the remote repo before running the checked-in health helpers. +

    +
    + +
    +

    Changes Made

    +
      +
    • Prepended $HOME/.bun/bin during native remote precheck when available.
      • +
    • Prepended $HOME/.bun/bin during native remote rollout when available.
      • +
    • Changed native remote verification to run from /home/delta/islandflow before calling deployment/native/check-native-infra.sh.
      • +
    +
    + +
    +

    Context

    +

    + During a live native rollout, the deploy helper failed first because the non-login SSH shell could not find bun even though it was installed under the deploy user's home directory. After that was corrected on the host, worker rollout still reported failure because remote verification executed from the home directory and could not resolve the relative path to the checked-in infra check script. +

    +
    + +
    +

    Important Implementation Details

    +
      +
    • The fallback only adjusts PATH when $HOME/.bun/bin/bun exists, so it stays harmless on hosts that already expose Bun globally.
      • +
    • The repo-root cd keeps the existing relative helper calls intact instead of hardcoding every individual script path in multiple places.
      • +
    • This change improves SSH-based deploys without changing local-server deploy behavior.
      • +
    +
    + +
    +

    Relevant Diff Snippets

    +

    Unified diff blocks below are formatted for diffs-compatible rendering.

    + 
    diff --git a/scripts/deploy.ts b/scripts/deploy.ts
+@@ -754,6 +754,10 @@ set -euo pipefail
+ 
+ cd ${shellEscape(REMOTE_REPO)}
+ 
++if [[ -x "$HOME/.bun/bin/bun" ]]; then
++  export PATH="$HOME/.bun/bin:$PATH"
++fi
++
+ if ! command -v bun >/dev/null 2>&1; then
+
+@@ -855,6 +859,10 @@ set -euo pipefail
+ 
++if [[ -x "$HOME/.bun/bin/bun" ]]; then
++  export PATH="$HOME/.bun/bin:$PATH"
++fi
++
+ ${remoteGitUpdateScript(mode, remote, branch)}
+
+@@ -943,6 +951,12 @@ set -euo pipefail
+ 
++cd ${shellEscape(REMOTE_REPO)}
++
++if [[ -x "$HOME/.bun/bin/bun" ]]; then
++  export PATH="$HOME/.bun/bin:$PATH"
++fi
++
+ declare -a units=(${units})
    +
    + +
    +

    Expected Impact for End-Users

    +

    + End users should see fewer failed native deploy attempts and fewer partial restarts caused by tooling assumptions rather than application health. This lowers the odds of avoidable downtime during native rollouts. +

    +
    + +
    +

    Validation

    +
      +
    • Observed the original failures during live rollout: missing bun in SSH PATH and missing deployment/native/check-native-infra.sh during remote verification.
      • +
    • Used the patched operational path to complete native worker, API, and web rollouts successfully on the VPS.
      • +
    • Verified API health at http://127.0.0.1:4000/health and web health at both http://127.0.0.1:3000/ and https://flow.deltaisland.io.
      • +
    +
    + +
    +

    Issues, Limitations, and Mitigations

    +
      +
    • This patch does not solve the separate ingest-news credential problem. Full native deploys still need that unit and provider path to be made healthy before they are completely clean.
      • +
    • The VPS also needed a host-level Bun symlink during this recovery. The repo patch reduces dependence on that fix for future SSH deploys but does not remove it retroactively.
      • +
    +
    + +
    +

    Follow-up Work

    +
      +
    • islandflow-fmg: Keep the deploy helper aligned with the actual VPS runtime assumptions and add regression checks around native verification paths.
      • +
    • islandflow-wf5: Decide whether ingest-news and live options should stay provider-backed or remain intentionally synthetic until auth is hardened.
      • +
    +
    +
    + + diff --git a/scripts/deploy.ts b/scripts/deploy.ts index e703c49..169f7a9 100644 --- a/scripts/deploy.ts +++ b/scripts/deploy.ts @@ -754,6 +754,10 @@ set -euo pipefail cd ${shellEscape(REMOTE_REPO)} +if [[ -x "$HOME/.bun/bin/bun" ]]; then + export PATH="$HOME/.bun/bin:$PATH" +fi + if ! command -v bun >/dev/null 2>&1; then echo "Refusing native rollout: bun is not installed on the server." >&2 echo "The current supported VPS path remains --runtime docker." >&2 @@ -855,6 +859,10 @@ function remoteNativeRollout( `#!/usr/bin/env bash set -euo pipefail +if [[ -x "$HOME/.bun/bin/bun" ]]; then + export PATH="$HOME/.bun/bin:$PATH" +fi + ${remoteGitUpdateScript(mode, remote, branch)} cd ${shellEscape(REMOTE_REPO)} @@ -943,6 +951,12 @@ function remoteNativeVerification(scope: DeployScope, fast: boolean): void { `#!/usr/bin/env bash set -euo pipefail +cd ${shellEscape(REMOTE_REPO)} + +if [[ -x "$HOME/.bun/bin/bun" ]]; then + export PATH="$HOME/.bun/bin:$PATH" +fi + declare -a units=(${units}) for unit in "\${units[@]}"; do ${NATIVE_SYSTEMCTL_PREFIX} is-active --quiet "$unit" From e9739f5dc9a251407e933a42495b3fc33a9a39aa Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 19:40:52 -0400 Subject: [PATCH 20/26] update beads for native deploy ssh fix --- .beads/issues.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 59c55f5..3ce8c65 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -15,7 +15,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-fmg","title":"Fix native deploy SSH path and verification cwd assumptions","description":"Native deploys over SSH assumed bun was already on PATH and that remote verification would run from the repository root. On the live VPS, non-login SSH shells omitted /home/delta/.bun/bin and remote native verification could not find deployment/native/check-native-infra.sh because it ran from the home directory. Update the deploy helper to prepend /Users/kell/.bun/bin when present and cd into the repo before native verification checks run.","status":"open","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:38:32Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:38:32Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-fmg","title":"Fix native deploy SSH path and verification cwd assumptions","description":"Native deploys over SSH assumed bun was already on PATH and that remote verification would run from the repository root. On the live VPS, non-login SSH shells omitted /home/delta/.bun/bin and remote native verification could not find deployment/native/check-native-infra.sh because it ran from the home directory. Update the deploy helper to prepend /Users/kell/.bun/bin when present and cd into the repo before native verification checks run.","status":"closed","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:38:32Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:40:33Z","closed_at":"2026-05-19T23:40:33Z","close_reason":"Updated native SSH deploy flow to prepend Bun's home install path when present and run native verification from the repo root before health scripts.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-wf5","title":"Harden native options provider configuration after synthetic recovery","description":"Native production recovery restored OPTIONS_INGEST_ADAPTER=synthetic because the current Alpaca setup fails authentication and crash-loops ingest-options. Follow up by deciding whether production options should remain synthetic or move to a supported live provider auth path, then add a deploy-time smoke test or config validation that catches provider auth failures before native cutover.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:27:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-m83","title":"Restore options ingestion and print generation on native deployment","description":"After moving the production/VPS deployment from Docker-managed services to the native runtime, the options feed appears behind and fresh option prints are not reaching the UI. Investigate the native deployment path on the server, identify the ingestion or compute breakage, apply the required code and/or host configuration changes, validate that fresh option prints resume, and document any follow-up operational work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:20:01Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:52Z","started_at":"2026-05-19T23:20:10Z","closed_at":"2026-05-19T23:27:52Z","close_reason":"Restored native options ingest by switching the VPS back to the last known-good synthetic adapter, verified fresh option prints and compute output, and documented the native env precedence gotcha.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-o1v","title":"Add SCM provider layer with Forgejo detection","description":"Implement provider-aware source-control detection and mirror-aware guardrails for repo automation so Forgejo remotes are treated as authoritative when present.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:04:33Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:06:55Z","started_at":"2026-05-19T23:04:35Z","closed_at":"2026-05-19T23:06:55Z","close_reason":"created by mistake during interrupted turn; no implementation was started","dependency_count":0,"dependent_count":0,"comment_count":0} From 7d25608b35784e135b4a03bc018a14107acc1bc1 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 19:57:56 -0400 Subject: [PATCH 21/26] fix alpaca news auth and native worker wiring --- .beads/issues.jsonl | 1 + .env.example | 4 + README.md | 6 +- deployment/docker/.env.example | 4 + deployment/docker/README.md | 8 +- deployment/native/README.md | 5 +- deployment/native/check-native-health.sh | 6 +- deployment/native/cutover.sh | 8 +- deployment/native/full-rollback.sh | 4 +- deployment/native/install-user-units.sh | 8 +- deployment/native/rollback.sh | 6 +- .../user/islandflow-ingest-news.service | 17 +++++ packages/config/src/alpaca.ts | 76 +++++++++++++++++++ packages/config/src/index.ts | 1 + packages/config/tests/alpaca.test.ts | 65 ++++++++++++++++ scripts/deploy.ts | 6 +- .../ingest-equities/src/adapters/alpaca.ts | 29 +++---- services/ingest-equities/src/index.ts | 17 +++-- services/ingest-news/src/index.ts | 35 +++++---- .../ingest-options/src/adapters/alpaca.ts | 42 ++++++---- services/ingest-options/src/index.ts | 17 +++-- 21 files changed, 285 insertions(+), 80 deletions(-) create mode 100644 deployment/native/systemd/user/islandflow-ingest-news.service create mode 100644 packages/config/src/alpaca.ts create mode 100644 packages/config/tests/alpaca.test.ts diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 3ce8c65..b82115f 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -15,6 +15,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-laq","title":"fix native alpaca news deploy and auth","description":"Why this issue exists and what needs to be done:\\n\\nNative Islandflow rollout is incomplete because services/ingest-news is not healthy on the VPS. The checked-in native user units and helper scripts do not fully include ingest-news, and the current service uses bearer-style auth that returns 401 against Alpaca news endpoints.\\n\\nThis task should verify the current Alpaca news auth requirements against official docs, update the repo code and native deployment assets as needed, install and enable the missing VPS unit, verify news events flow end-to-end, and document the work.","status":"in_progress","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:47:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:47:12Z","started_at":"2026-05-19T23:47:12Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-fmg","title":"Fix native deploy SSH path and verification cwd assumptions","description":"Native deploys over SSH assumed bun was already on PATH and that remote verification would run from the repository root. On the live VPS, non-login SSH shells omitted /home/delta/.bun/bin and remote native verification could not find deployment/native/check-native-infra.sh because it ran from the home directory. Update the deploy helper to prepend /Users/kell/.bun/bin when present and cd into the repo before native verification checks run.","status":"closed","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:38:32Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:40:33Z","closed_at":"2026-05-19T23:40:33Z","close_reason":"Updated native SSH deploy flow to prepend Bun's home install path when present and run native verification from the repo root before health scripts.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-wf5","title":"Harden native options provider configuration after synthetic recovery","description":"Native production recovery restored OPTIONS_INGEST_ADAPTER=synthetic because the current Alpaca setup fails authentication and crash-loops ingest-options. Follow up by deciding whether production options should remain synthetic or move to a supported live provider auth path, then add a deploy-time smoke test or config validation that catches provider auth failures before native cutover.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:27:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-m83","title":"Restore options ingestion and print generation on native deployment","description":"After moving the production/VPS deployment from Docker-managed services to the native runtime, the options feed appears behind and fresh option prints are not reaching the UI. Investigate the native deployment path on the server, identify the ingestion or compute breakage, apply the required code and/or host configuration changes, validate that fresh option prints resume, and document any follow-up operational work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:20:01Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:52Z","started_at":"2026-05-19T23:20:10Z","closed_at":"2026-05-19T23:27:52Z","close_reason":"Restored native options ingest by switching the VPS back to the last known-good synthetic adapter, verified fresh option prints and compute output, and documented the native env precedence gotcha.","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/.env.example b/.env.example index d42f715..be20b62 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,10 @@ REDIS_URL=redis://127.0.0.1:6379 # Options ingest OPTIONS_INGEST_ADAPTER=synthetic ALPACA_API_KEY= +ALPACA_API_KEY_ID= +ALPACA_KEY_ID= +ALPACA_API_SECRET_KEY= +ALPACA_SECRET_KEY= ALPACA_REST_URL=https://data.alpaca.markets ALPACA_WS_BASE_URL=wss://stream.data.alpaca.markets/v1beta1 ALPACA_FEED=indicative diff --git a/README.md b/README.md index 02417aa..9456d1b 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,11 @@ All runtime configuration comes from `.env`. | Variable | Default | What it controls | | --- | --- | --- | -| `ALPACA_API_KEY` | empty | Single-token Alpaca API auth for options, equities, and news adapters. | +| `ALPACA_API_KEY` | empty | Legacy single-token fallback kept for older Alpaca setups. Prefer explicit key ID + secret vars for current Alpaca auth. | +| `ALPACA_API_KEY_ID` | empty | Preferred Alpaca key ID used for market-data REST and websocket auth. | +| `ALPACA_KEY_ID` | empty | Alternate name accepted for the Alpaca key ID. | +| `ALPACA_API_SECRET_KEY` | empty | Preferred Alpaca secret key paired with `ALPACA_API_KEY_ID`. | +| `ALPACA_SECRET_KEY` | empty | Alternate name accepted for the Alpaca secret key. | | `ALPACA_REST_URL` | `https://data.alpaca.markets` | Alpaca REST base URL. | | `ALPACA_WS_BASE_URL` | `wss://stream.data.alpaca.markets/v1beta1` for options, `wss://stream.data.alpaca.markets` for equities/news | Alpaca websocket base URL. | | `ALPACA_FEED` | `indicative` | Options feed tier: `indicative` or `opra`. | diff --git a/deployment/docker/.env.example b/deployment/docker/.env.example index 1a3eb84..4972ada 100644 --- a/deployment/docker/.env.example +++ b/deployment/docker/.env.example @@ -27,6 +27,10 @@ NEXT_PUBLIC_NBBO_MAX_AGE_MS=1000 # Options ingest OPTIONS_INGEST_ADAPTER=synthetic ALPACA_API_KEY= +ALPACA_API_KEY_ID= +ALPACA_KEY_ID= +ALPACA_API_SECRET_KEY= +ALPACA_SECRET_KEY= ALPACA_REST_URL=https://data.alpaca.markets ALPACA_WS_BASE_URL=wss://stream.data.alpaca.markets/v1beta1 ALPACA_FEED=indicative diff --git a/deployment/docker/README.md b/deployment/docker/README.md index 9b36220..644798b 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -161,8 +161,10 @@ Set the adapter values and credentials in `.env`: - `OPTIONS_INGEST_ADAPTER=alpaca` - `EQUITIES_INGEST_ADAPTER=alpaca` -- `ALPACA_KEY_ID=...` -- `ALPACA_SECRET_KEY=...` +- `ALPACA_API_KEY_ID=...` +- `ALPACA_API_SECRET_KEY=...` + +The older single-variable `ALPACA_API_KEY` fallback is still accepted for legacy setups, but Alpaca's current market-data auth expects a key ID plus secret key pair. ### Databento mode @@ -284,7 +286,7 @@ Scoped Docker deploys now build only the selected image set and then restart onl - `--web-only`: `docker compose build web`, then `docker compose up -d web` - `--api-only`: `docker compose build api`, then `docker compose up -d api` - `--services-only`: builds and restarts `api`, `compute`, `candles`, `ingest-options`, and `ingest-equities` -- `--workers-only`: builds and restarts `compute`, `candles`, `ingest-options`, and `ingest-equities` without touching `web` or `api` +- `--workers-only`: builds and restarts `compute`, `candles`, `ingest-options`, `ingest-equities`, and `ingest-news` without touching `web` or `api` - `--fast`: when no explicit scope flag is given, treats the deploy as `--services-only` and skips the public API route suite for quicker completion. It still runs remote service health checks. Use `--no-build` only when the image is already correct and you need Compose to recreate or restart containers, such as after changing server-side environment values that do not affect a Next.js build-time variable. Do not use `--no-build` for dependency changes, application source changes, or `NEXT_PUBLIC_*` changes. diff --git a/deployment/native/README.md b/deployment/native/README.md index 569cdb8..219f952 100644 --- a/deployment/native/README.md +++ b/deployment/native/README.md @@ -91,6 +91,7 @@ Checked-in unit files live under: - `deployment/native/systemd/user/islandflow-candles.service` - `deployment/native/systemd/user/islandflow-ingest-options.service` - `deployment/native/systemd/user/islandflow-ingest-equities.service` +- `deployment/native/systemd/user/islandflow-ingest-news.service` These are written for the current VPS layout: @@ -175,6 +176,7 @@ Default unit names used by `scripts/deploy.ts`: - `islandflow-candles` - `islandflow-ingest-options` - `islandflow-ingest-equities` +- `islandflow-ingest-news` Override them from your local shell before running `./deploy` if the server uses different names: @@ -191,6 +193,7 @@ Available overrides: - `DEPLOY_NATIVE_CANDLES_UNIT` - `DEPLOY_NATIVE_INGEST_OPTIONS_UNIT` - `DEPLOY_NATIVE_INGEST_EQUITIES_UNIT` +- `DEPLOY_NATIVE_INGEST_NEWS_UNIT` ## systemctl invocation @@ -220,7 +223,7 @@ Scope behavior: - `--web-only`: rebuild/restart only the web unit - `--api-only`: restart only the API unit - `--services-only`: restart API + worker units without touching the web unit -- `--workers-only`: restart only `compute`, `candles`, `ingest-options`, and `ingest-equities` +- `--workers-only`: restart only `compute`, `candles`, `ingest-options`, `ingest-equities`, and `ingest-news` - `--fast`: when no explicit scope flag is provided, native deploys now default to `--workers-only` - `--no-build`: skip `bun install --frozen-lockfile` and skip the web build step diff --git a/deployment/native/check-native-health.sh b/deployment/native/check-native-health.sh index 13582bc..e78270a 100755 --- a/deployment/native/check-native-health.sh +++ b/deployment/native/check-native-health.sh @@ -7,7 +7,7 @@ units=() case "$scope" in full) - units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) ;; web) units=(islandflow-web.service) @@ -16,10 +16,10 @@ case "$scope" in units=(islandflow-api.service) ;; services) - units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) ;; workers) - units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) ;; *) echo "Unknown scope: $scope" >&2 diff --git a/deployment/native/cutover.sh b/deployment/native/cutover.sh index fcff377..5971f12 100755 --- a/deployment/native/cutover.sh +++ b/deployment/native/cutover.sh @@ -16,7 +16,7 @@ esac echo "Stopping Docker-owned Islandflow app services before native ownership starts." ( cd "$repo_root/deployment/docker" - docker compose stop web api compute candles ingest-options ingest-equities + docker compose stop web api compute candles ingest-options ingest-equities ingest-news ) if [[ "$scope" == "full" || "$scope" == "services" || "$scope" == "api" || "$scope" == "web" ]]; then @@ -24,9 +24,9 @@ if [[ "$scope" == "full" || "$scope" == "services" || "$scope" == "api" || "$sco fi systemctl --user restart $(case "$scope" in - full) echo islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;; - services) echo islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;; - workers) echo islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service ;; + full) echo islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service ;; + services) echo islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service ;; + workers) echo islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service ;; api) echo islandflow-api.service ;; web) echo islandflow-web.service ;; esac) diff --git a/deployment/native/full-rollback.sh b/deployment/native/full-rollback.sh index 77a78af..9cac62b 100755 --- a/deployment/native/full-rollback.sh +++ b/deployment/native/full-rollback.sh @@ -4,7 +4,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" echo "Stopping native app services." -systemctl --user stop islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service || true +systemctl --user stop islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service || true echo "Stopping native infra before Docker reopens durable data." if [[ "${EUID}" -eq 0 ]]; then @@ -19,7 +19,7 @@ echo "Switching NPM Islandflow upstreams back to Docker service names." echo "Restarting Docker Islandflow runtime." ( cd "$repo_root/deployment/docker" - docker compose up -d web api compute candles ingest-options ingest-equities + docker compose up -d web api compute candles ingest-options ingest-equities ingest-news ) curl -I -fksS "${DEPLOY_PUBLIC_APP_URL:-https://flow.deltaisland.io}" >/dev/null diff --git a/deployment/native/install-user-units.sh b/deployment/native/install-user-units.sh index 350cab1..558ff93 100755 --- a/deployment/native/install-user-units.sh +++ b/deployment/native/install-user-units.sh @@ -11,7 +11,7 @@ case "$scope" in none) ;; full) - units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) ;; web) units=(islandflow-web.service) @@ -20,10 +20,10 @@ case "$scope" in units=(islandflow-api.service) ;; services) - units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) ;; workers) - units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) ;; *) echo "Unknown scope: $scope" >&2 @@ -46,4 +46,4 @@ if [[ ${#units[@]} -gt 0 ]]; then echo "Enabled scope: $scope" else echo "No units enabled yet. Pass a scope such as workers when you are ready." -fi \ No newline at end of file +fi diff --git a/deployment/native/rollback.sh b/deployment/native/rollback.sh index fb472d9..0721b50 100755 --- a/deployment/native/rollback.sh +++ b/deployment/native/rollback.sh @@ -30,7 +30,7 @@ fi case "$scope" in full) - units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + units=(islandflow-web.service islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) ;; web) units=(islandflow-web.service) @@ -39,10 +39,10 @@ case "$scope" in units=(islandflow-api.service) ;; services) - units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + units=(islandflow-api.service islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) ;; workers) - units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service) + units=(islandflow-compute.service islandflow-candles.service islandflow-ingest-options.service islandflow-ingest-equities.service islandflow-ingest-news.service) ;; *) echo "Unknown scope: $scope" >&2 diff --git a/deployment/native/systemd/user/islandflow-ingest-news.service b/deployment/native/systemd/user/islandflow-ingest-news.service new file mode 100644 index 0000000..bca11a3 --- /dev/null +++ b/deployment/native/systemd/user/islandflow-ingest-news.service @@ -0,0 +1,17 @@ +[Unit] +Description=Islandflow ingest-news +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/delta/islandflow +EnvironmentFile=/home/delta/islandflow/.env +ExecStart=/home/delta/.bun/bin/bun services/ingest-news/src/index.ts +Restart=always +RestartSec=2 +KillSignal=SIGINT +TimeoutStopSec=20 + +[Install] +WantedBy=default.target diff --git a/packages/config/src/alpaca.ts b/packages/config/src/alpaca.ts new file mode 100644 index 0000000..697d65b --- /dev/null +++ b/packages/config/src/alpaca.ts @@ -0,0 +1,76 @@ +export type AlpacaCredentials = { + keyId: string; + secret: string; + legacyToken: string; + usesLegacyBearer: boolean; +}; + +type AlpacaCredentialEnv = { + ALPACA_API_KEY?: string; + ALPACA_API_KEY_ID?: string; + ALPACA_KEY_ID?: string; + ALPACA_API_SECRET_KEY?: string; + ALPACA_SECRET_KEY?: string; +}; + +const normalize = (value: string | undefined): string => value?.trim() ?? ""; + +export const resolveAlpacaCredentials = ( + env: AlpacaCredentialEnv +): AlpacaCredentials => { + const legacyToken = normalize(env.ALPACA_API_KEY); + const explicitKeyId = + normalize(env.ALPACA_API_KEY_ID) || normalize(env.ALPACA_KEY_ID); + const secret = + normalize(env.ALPACA_API_SECRET_KEY) || normalize(env.ALPACA_SECRET_KEY); + const keyId = explicitKeyId || legacyToken; + const usesLegacyBearer = !explicitKeyId && !secret && legacyToken.length > 0; + + return { + keyId, + secret, + legacyToken, + usesLegacyBearer + }; +}; + +export const hasAlpacaCredentials = (credentials: AlpacaCredentials): boolean => { + if (credentials.usesLegacyBearer) { + return credentials.legacyToken.length > 0; + } + + return credentials.keyId.length > 0 && credentials.secret.length > 0; +}; + +export const buildAlpacaAuthHeaders = ( + credentials: AlpacaCredentials +): Record => { + if (credentials.usesLegacyBearer) { + return { + Authorization: `Bearer ${credentials.legacyToken}` + }; + } + + return { + "APCA-API-KEY-ID": credentials.keyId, + "APCA-API-SECRET-KEY": credentials.secret + }; +}; + +export const buildAlpacaWebSocketAuthMessage = ( + credentials: AlpacaCredentials +): { action: "auth"; key: string; secret: string } => { + if (credentials.usesLegacyBearer) { + return { + action: "auth", + key: credentials.legacyToken, + secret: "" + }; + } + + return { + action: "auth", + key: credentials.keyId, + secret: credentials.secret + }; +}; diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index 77b0d3c..577271f 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -1 +1,2 @@ export * from "./env"; +export * from "./alpaca"; diff --git a/packages/config/tests/alpaca.test.ts b/packages/config/tests/alpaca.test.ts new file mode 100644 index 0000000..9c48f12 --- /dev/null +++ b/packages/config/tests/alpaca.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "bun:test"; +import { + buildAlpacaAuthHeaders, + buildAlpacaWebSocketAuthMessage, + hasAlpacaCredentials, + resolveAlpacaCredentials +} from "../src/alpaca"; + +describe("resolveAlpacaCredentials", () => { + it("prefers explicit key-id and secret vars", () => { + const credentials = resolveAlpacaCredentials({ + ALPACA_API_KEY: "legacy-token", + ALPACA_API_KEY_ID: "key-id", + ALPACA_API_SECRET_KEY: "secret" + }); + + expect(credentials).toEqual({ + keyId: "key-id", + secret: "secret", + legacyToken: "legacy-token", + usesLegacyBearer: false + }); + expect(hasAlpacaCredentials(credentials)).toBe(true); + expect(buildAlpacaAuthHeaders(credentials)).toEqual({ + "APCA-API-KEY-ID": "key-id", + "APCA-API-SECRET-KEY": "secret" + }); + expect(buildAlpacaWebSocketAuthMessage(credentials)).toEqual({ + action: "auth", + key: "key-id", + secret: "secret" + }); + }); + + it("supports the older bearer-token fallback when no secret exists", () => { + const credentials = resolveAlpacaCredentials({ + ALPACA_API_KEY: "legacy-token" + }); + + expect(credentials.usesLegacyBearer).toBe(true); + expect(hasAlpacaCredentials(credentials)).toBe(true); + expect(buildAlpacaAuthHeaders(credentials)).toEqual({ + Authorization: "Bearer legacy-token" + }); + expect(buildAlpacaWebSocketAuthMessage(credentials)).toEqual({ + action: "auth", + key: "legacy-token", + secret: "" + }); + }); + + it("supports alternate secret env names", () => { + const credentials = resolveAlpacaCredentials({ + ALPACA_KEY_ID: "short-key", + ALPACA_SECRET_KEY: "short-secret" + }); + + expect(credentials).toEqual({ + keyId: "short-key", + secret: "short-secret", + legacyToken: "", + usesLegacyBearer: false + }); + }); +}); diff --git a/scripts/deploy.ts b/scripts/deploy.ts index 169f7a9..8a5b9c7 100644 --- a/scripts/deploy.ts +++ b/scripts/deploy.ts @@ -81,7 +81,8 @@ const DOCKER_WORKER_SERVICES = [ "compute", "candles", "ingest-options", - "ingest-equities" + "ingest-equities", + "ingest-news" ] as const; const scriptPath = fileURLToPath(import.meta.url); @@ -559,7 +560,8 @@ function nativeUnitsForScope(scope: DeployScope): string[] { NATIVE_UNITS.compute, NATIVE_UNITS.candles, NATIVE_UNITS.ingestOptions, - NATIVE_UNITS.ingestEquities + NATIVE_UNITS.ingestEquities, + NATIVE_UNITS.ingestNews ]; default: return [ diff --git a/services/ingest-equities/src/adapters/alpaca.ts b/services/ingest-equities/src/adapters/alpaca.ts index 672347f..7a1447f 100644 --- a/services/ingest-equities/src/adapters/alpaca.ts +++ b/services/ingest-equities/src/adapters/alpaca.ts @@ -1,3 +1,8 @@ +import { + buildAlpacaAuthHeaders, + buildAlpacaWebSocketAuthMessage, + type AlpacaCredentials +} from "@islandflow/config"; import { createLogger } from "@islandflow/observability"; import type { EquityPrint, EquityQuote } from "@islandflow/types"; import type { EquityIngestAdapter, EquityIngestHandlers } from "./types"; @@ -6,7 +11,7 @@ import WebSocket from "ws"; export type AlpacaEquitiesFeed = "iex" | "sip"; export type AlpacaEquitiesAdapterConfig = { - apiKey: string; + credentials: AlpacaCredentials; restUrl: string; wsBaseUrl: string; feed: AlpacaEquitiesFeed; @@ -62,12 +67,6 @@ const normalizeSymbols = (symbols: string[]): string[] => { return result; }; -const buildHeaders = (config: AlpacaEquitiesAdapterConfig): Record => { - return { - Authorization: `Bearer ${config.apiKey}` - }; -}; - const parseTimestamp = (value: string): number => { const parsed = Date.parse(value); if (Number.isFinite(parsed)) { @@ -157,7 +156,7 @@ const fetchExchangeMeta = async (config: AlpacaEquitiesAdapterConfig): Promise { - if (!config.apiKey) { - throw new Error("Alpaca equities adapter requires ALPACA_API_KEY."); + if (!config.credentials.keyId) { + throw new Error("Alpaca equities adapter requires Alpaca credentials."); } const symbols = normalizeSymbols(config.symbols); @@ -196,7 +195,7 @@ export const createAlpacaEquitiesAdapter = ( const exchangeNameMap = await fetchExchangeMeta(config); const wsUrl = buildWsUrl(config.wsBaseUrl, config.feed); const ws = new WebSocket(wsUrl, { - headers: buildHeaders(config) + headers: buildAlpacaAuthHeaders(config.credentials) }); let seq = 0; @@ -204,13 +203,7 @@ export const createAlpacaEquitiesAdapter = ( let authenticated = false; ws.on("open", () => { - ws.send( - JSON.stringify({ - action: "auth", - key: config.apiKey, - secret: "" - }) - ); + ws.send(JSON.stringify(buildAlpacaWebSocketAuthMessage(config.credentials))); }); const subscribe = () => { diff --git a/services/ingest-equities/src/index.ts b/services/ingest-equities/src/index.ts index f098b15..1b708ae 100644 --- a/services/ingest-equities/src/index.ts +++ b/services/ingest-equities/src/index.ts @@ -1,4 +1,4 @@ -import { readEnv } from "@islandflow/config"; +import { hasAlpacaCredentials, readEnv, resolveAlpacaCredentials } from "@islandflow/config"; import { createLogger } from "@islandflow/observability"; import { SUBJECT_EQUITY_PRINTS, @@ -47,6 +47,10 @@ const envSchema = z.object({ // Alpaca (equities) ALPACA_API_KEY: z.string().default(""), + ALPACA_API_KEY_ID: z.string().default(""), + ALPACA_KEY_ID: z.string().default(""), + ALPACA_API_SECRET_KEY: z.string().default(""), + ALPACA_SECRET_KEY: z.string().default(""), ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"), ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"), ALPACA_UNDERLYINGS: z.string().default("SPY,NVDA,AAPL"), @@ -70,6 +74,7 @@ const envSchema = z.object({ }); const env = readEnv(envSchema); +const alpacaCredentials = resolveAlpacaCredentials(env); const syntheticModes = resolveSyntheticMarketModes({ syntheticMarketMode: env.SYNTHETIC_MARKET_MODE, syntheticEquitiesMode: env.SYNTHETIC_EQUITIES_MODE @@ -175,13 +180,15 @@ const selectAdapter = ( } if (name === "alpaca") { - if (!env.ALPACA_API_KEY) { - logger.warn("alpaca credentials missing; set ALPACA_API_KEY"); - throw new Error("ALPACA_API_KEY is required for the alpaca adapter."); + if (!hasAlpacaCredentials(alpacaCredentials)) { + logger.warn("alpaca credentials missing; set ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY"); + throw new Error( + "Alpaca equities adapter requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or legacy ALPACA_API_KEY)." + ); } return createAlpacaEquitiesAdapter({ - apiKey: env.ALPACA_API_KEY, + credentials: alpacaCredentials, restUrl: env.ALPACA_REST_URL, wsBaseUrl: env.ALPACA_WS_BASE_URL, feed: env.ALPACA_EQUITIES_FEED, diff --git a/services/ingest-news/src/index.ts b/services/ingest-news/src/index.ts index 3f91ee2..c73cfe0 100644 --- a/services/ingest-news/src/index.ts +++ b/services/ingest-news/src/index.ts @@ -1,4 +1,10 @@ -import { readEnv } from "@islandflow/config"; +import { + buildAlpacaAuthHeaders, + buildAlpacaWebSocketAuthMessage, + hasAlpacaCredentials, + readEnv, + resolveAlpacaCredentials +} from "@islandflow/config"; import { createLogger } from "@islandflow/observability"; import { SUBJECT_NEWS, @@ -18,6 +24,10 @@ const logger = createLogger({ service }); const envSchema = z.object({ NATS_URL: z.string().default("nats://127.0.0.1:4222"), ALPACA_API_KEY: z.string().default(""), + ALPACA_API_KEY_ID: z.string().default(""), + ALPACA_KEY_ID: z.string().default(""), + ALPACA_API_SECRET_KEY: z.string().default(""), + ALPACA_SECRET_KEY: z.string().default(""), ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"), ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"), ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(200).default(100), @@ -25,6 +35,7 @@ const envSchema = z.object({ }); const env = readEnv(envSchema); +const alpacaCredentials = resolveAlpacaCredentials(env); type AlpacaNewsItem = { id?: number; @@ -43,10 +54,6 @@ type AlpacaNewsResponse = { news?: AlpacaNewsItem[]; }; -const buildHeaders = (): Record => ({ - Authorization: `Bearer ${env.ALPACA_API_KEY}` -}); - const parseTimestamp = (value: string | undefined): number => { const parsed = value ? Date.parse(value) : Number.NaN; return Number.isFinite(parsed) ? parsed : Date.now(); @@ -90,7 +97,7 @@ const fetchBackfill = async (): Promise => { url.searchParams.set("limit", env.ALPACA_NEWS_BACKFILL_LIMIT.toString()); const response = await fetch(url.toString(), { - headers: buildHeaders() + headers: buildAlpacaAuthHeaders(alpacaCredentials) }); if (!response.ok) { @@ -115,8 +122,10 @@ const decodePayload = (data: WebSocket.RawData): unknown => { }; const run = async () => { - if (!env.ALPACA_API_KEY) { - throw new Error("ALPACA_API_KEY is required for ingest-news."); + if (!hasAlpacaCredentials(alpacaCredentials)) { + throw new Error( + "Alpaca news requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or ALPACA_KEY_ID / ALPACA_SECRET_KEY)." + ); } const { nc, js, jsm } = await connectJetStreamWithRetry( @@ -146,17 +155,11 @@ const run = async () => { const wsUrl = new URL(env.ALPACA_NEWS_WEBSOCKET_PATH, env.ALPACA_WS_BASE_URL).toString(); const ws = new WebSocket(wsUrl, { - headers: buildHeaders() + headers: buildAlpacaAuthHeaders(alpacaCredentials) }); ws.on("open", () => { - ws.send( - JSON.stringify({ - action: "auth", - key: env.ALPACA_API_KEY, - secret: "" - }) - ); + ws.send(JSON.stringify(buildAlpacaWebSocketAuthMessage(alpacaCredentials))); }); ws.on("message", (raw) => { diff --git a/services/ingest-options/src/adapters/alpaca.ts b/services/ingest-options/src/adapters/alpaca.ts index dce7702..00645b8 100644 --- a/services/ingest-options/src/adapters/alpaca.ts +++ b/services/ingest-options/src/adapters/alpaca.ts @@ -1,4 +1,9 @@ import { decode, encode } from "@msgpack/msgpack"; +import { + buildAlpacaAuthHeaders, + buildAlpacaWebSocketAuthMessage, + type AlpacaCredentials +} from "@islandflow/config"; import { createLogger } from "@islandflow/observability"; import type { OptionIngestAdapter, OptionIngestHandlers } from "./types"; import WebSocket from "ws"; @@ -6,7 +11,7 @@ import WebSocket from "ws"; type AlpacaFeed = "indicative" | "opra"; type AlpacaOptionsAdapterConfig = { - apiKey: string; + credentials: AlpacaCredentials; restUrl: string; wsBaseUrl: string; feed: AlpacaFeed; @@ -147,18 +152,12 @@ const normalizeUnderlyings = (value: string[]): string[] => { return result; }; -const buildHeaders = (config: AlpacaOptionsAdapterConfig): Record => { - return { - Authorization: `Bearer ${config.apiKey}` - }; -}; - const fetchJson = async ( url: URL, config: AlpacaOptionsAdapterConfig ): Promise => { const response = await fetch(url.toString(), { - headers: buildHeaders(config) + headers: buildAlpacaAuthHeaders(config.credentials) }); if (!response.ok) { @@ -398,8 +397,8 @@ export const createAlpacaOptionsAdapter = ( return { name: "alpaca", start: async (handlers: OptionIngestHandlers) => { - if (!config.apiKey) { - throw new Error("Alpaca adapter requires ALPACA_API_KEY."); + if (!config.credentials.keyId) { + throw new Error("Alpaca adapter requires Alpaca credentials."); } const underlyings = normalizeUnderlyings(config.underlyings); @@ -485,15 +484,22 @@ export const createAlpacaOptionsAdapter = ( const wsUrl = `${wsBase}/${config.feed}`; const ws = new WebSocket(wsUrl, { headers: { - ...buildHeaders(config), + ...buildAlpacaAuthHeaders(config.credentials), "Content-Type": "application/msgpack" } }); let seq = 0; let stopped = false; + let subscribed = false; + + const subscribe = () => { + if (subscribed) { + return; + } + + subscribed = true; - ws.on("open", () => { const subscribe: Record = { action: "subscribe", trades: selectedSymbols @@ -504,6 +510,10 @@ export const createAlpacaOptionsAdapter = ( } ws.send(encode(subscribe)); + }; + + ws.on("open", () => { + ws.send(encode(buildAlpacaWebSocketAuthMessage(config.credentials))); }); ws.on("message", (data) => { @@ -583,7 +593,13 @@ export const createAlpacaOptionsAdapter = ( if (type === "error") { logger.error("alpaca stream error", { message }); - } else if (type === "success" || type === "subscription") { + } else if (type === "success") { + const status = (message as { msg?: string }).msg ?? ""; + if (status === "authenticated") { + subscribe(); + } + logger.info("alpaca stream status", { message }); + } else if (type === "subscription") { logger.info("alpaca stream status", { message }); } } diff --git a/services/ingest-options/src/index.ts b/services/ingest-options/src/index.ts index a52661f..301632e 100644 --- a/services/ingest-options/src/index.ts +++ b/services/ingest-options/src/index.ts @@ -1,4 +1,4 @@ -import { readEnv } from "@islandflow/config"; +import { hasAlpacaCredentials, readEnv, resolveAlpacaCredentials } from "@islandflow/config"; import { createLogger } from "@islandflow/observability"; import { SUBJECT_OPTION_NBBO, @@ -55,6 +55,10 @@ const envSchema = z.object({ CLICKHOUSE_DATABASE: z.string().default("default"), OPTIONS_INGEST_ADAPTER: z.string().min(1).default("synthetic"), ALPACA_API_KEY: z.string().default(""), + ALPACA_API_KEY_ID: z.string().default(""), + ALPACA_KEY_ID: z.string().default(""), + ALPACA_API_SECRET_KEY: z.string().default(""), + ALPACA_SECRET_KEY: z.string().default(""), ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"), ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets/v1beta1"), ALPACA_FEED: z.enum(["indicative", "opra"]).default("indicative"), @@ -120,6 +124,7 @@ const envSchema = z.object({ }); const env = readEnv(envSchema); +const alpacaCredentials = resolveAlpacaCredentials(env); const syntheticModes = resolveSyntheticMarketModes({ syntheticMarketMode: env.SYNTHETIC_MARKET_MODE, syntheticOptionsMode: env.SYNTHETIC_OPTIONS_MODE @@ -277,15 +282,17 @@ const selectAdapter = ( } if (name === "alpaca") { - if (!env.ALPACA_API_KEY) { - logger.warn("alpaca credentials missing; set ALPACA_API_KEY"); - throw new Error("ALPACA_API_KEY is required for the alpaca adapter."); + if (!hasAlpacaCredentials(alpacaCredentials)) { + logger.warn("alpaca credentials missing; set ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY"); + throw new Error( + "Alpaca adapter requires ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY (or legacy ALPACA_API_KEY)." + ); } const underlyings = env.ALPACA_UNDERLYINGS.split(",").map((symbol) => symbol.trim()); return createAlpacaOptionsAdapter({ - apiKey: env.ALPACA_API_KEY, + credentials: alpacaCredentials, restUrl: env.ALPACA_REST_URL, wsBaseUrl: env.ALPACA_WS_BASE_URL, feed: env.ALPACA_FEED, From 93b9152345bda8fcd9d055d1927cb4d834f25c20 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 20:02:35 -0400 Subject: [PATCH 22/26] persist news stories and request article content --- README.md | 2 +- services/api/src/index.ts | 4 +++- services/ingest-news/src/index.ts | 16 +++++++++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9456d1b..6b3b7fc 100644 --- a/README.md +++ b/README.md @@ -270,7 +270,7 @@ All runtime configuration comes from `.env`. | `ALPACA_MONEYNESS_FALLBACK_PCT` | `0.1` | Wider fallback moneyness filter if candidate set is too sparse. | | `ALPACA_MAX_QUOTES` | `200` | Upper bound on selected Alpaca options contracts/quotes per cycle. | | `ALPACA_EQUITIES_FEED` | `iex` | Alpaca equities feed: `iex` or `sip`. | -| `ALPACA_NEWS_BACKFILL_LIMIT` | `100` | Alpaca news stories fetched on startup, capped at 200. | +| `ALPACA_NEWS_BACKFILL_LIMIT` | `50` | Alpaca news stories fetched on startup, capped at 50 by the Alpaca News API. | | `ALPACA_NEWS_WEBSOCKET_PATH` | `/v1beta1/news` | Alpaca news websocket path. | ### Databento replay adapter configuration diff --git a/services/api/src/index.ts b/services/api/src/index.ts index f481626..562fb6b 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -92,7 +92,8 @@ import { fetchNearestOptionNBBOForPrints, fetchSmartMoneyEventsByPacketIds, fetchClassifierHitsByPacketIds, - fetchRecentOptionPrints + fetchRecentOptionPrints, + insertNewsStory } from "@islandflow/storage"; import type { EquityPrintQueryFilters } from "@islandflow/storage"; import { @@ -1277,6 +1278,7 @@ const run = async () => { for await (const msg of newsSubscription.messages) { try { const payload = NewsStorySchema.parse(newsSubscription.decode(msg)); + await insertNewsStory(clickhouse, payload); await fanoutLive({ channel: "news" }, payload, "news"); msg.ack(); } catch (error) { diff --git a/services/ingest-news/src/index.ts b/services/ingest-news/src/index.ts index c73cfe0..95cca42 100644 --- a/services/ingest-news/src/index.ts +++ b/services/ingest-news/src/index.ts @@ -30,13 +30,21 @@ const envSchema = z.object({ ALPACA_SECRET_KEY: z.string().default(""), ALPACA_REST_URL: z.string().default("https://data.alpaca.markets"), ALPACA_WS_BASE_URL: z.string().default("wss://stream.data.alpaca.markets"), - ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(200).default(100), + ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(50).default(50), ALPACA_NEWS_WEBSOCKET_PATH: z.string().default("/v1beta1/news") }); const env = readEnv(envSchema); const alpacaCredentials = resolveAlpacaCredentials(env); +const escapeHtml = (value: string): string => + value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); + type AlpacaNewsItem = { id?: number; headline?: string; @@ -66,7 +74,8 @@ const toStory = (item: AlpacaNewsItem, seq: number): NewsStory | null => { } const provider = "alpaca"; - const contentHtml = item.content ?? ""; + const summary = item.summary?.trim() ?? ""; + const contentHtml = item.content?.trim() || (summary ? `

    ${escapeHtml(summary)}

    ` : ""); const symbols = resolveNewsSymbols(item.symbols ?? [], contentHtml); const publishedTs = parseTimestamp(item.created_at); const updatedTs = parseTimestamp(item.updated_at ?? item.created_at); @@ -80,7 +89,7 @@ const toStory = (item: AlpacaNewsItem, seq: number): NewsStory | null => { provider, source: item.source?.trim() || item.author?.trim() || "Alpaca News", headline: item.headline?.trim() || `Story ${storyId}`, - summary: item.summary?.trim() || "", + summary, content_html: contentHtml, url: item.url?.trim() || "", published_ts: publishedTs, @@ -95,6 +104,7 @@ const fetchBackfill = async (): Promise => { const url = new URL("/v1beta1/news", env.ALPACA_REST_URL); url.searchParams.set("sort", "desc"); url.searchParams.set("limit", env.ALPACA_NEWS_BACKFILL_LIMIT.toString()); + url.searchParams.set("include_content", "true"); const response = await fetch(url.toString(), { headers: buildAlpacaAuthHeaders(alpacaCredentials) From 3632f362720a27eed604dee4a19528913d3c28d9 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 20:05:37 -0400 Subject: [PATCH 23/26] document native alpaca news repair --- .beads/issues.jsonl | 2 +- .../2026-05-19-fix-native-alpaca-news.html | 233 ++++++++++++++++++ 2 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 docs/turns/2026-05-19-fix-native-alpaca-news.html diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index b82115f..57fbdd7 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -15,7 +15,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-laq","title":"fix native alpaca news deploy and auth","description":"Why this issue exists and what needs to be done:\\n\\nNative Islandflow rollout is incomplete because services/ingest-news is not healthy on the VPS. The checked-in native user units and helper scripts do not fully include ingest-news, and the current service uses bearer-style auth that returns 401 against Alpaca news endpoints.\\n\\nThis task should verify the current Alpaca news auth requirements against official docs, update the repo code and native deployment assets as needed, install and enable the missing VPS unit, verify news events flow end-to-end, and document the work.","status":"in_progress","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:47:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:47:12Z","started_at":"2026-05-19T23:47:12Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-laq","title":"fix native alpaca news deploy and auth","description":"Why this issue exists and what needs to be done:\\n\\nNative Islandflow rollout is incomplete because services/ingest-news is not healthy on the VPS. The checked-in native user units and helper scripts do not fully include ingest-news, and the current service uses bearer-style auth that returns 401 against Alpaca news endpoints.\\n\\nThis task should verify the current Alpaca news auth requirements against official docs, update the repo code and native deployment assets as needed, install and enable the missing VPS unit, verify news events flow end-to-end, and document the work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:47:07Z","created_by":"dirtydishes","updated_at":"2026-05-20T00:05:20Z","started_at":"2026-05-19T23:47:12Z","closed_at":"2026-05-20T00:05:20Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-fmg","title":"Fix native deploy SSH path and verification cwd assumptions","description":"Native deploys over SSH assumed bun was already on PATH and that remote verification would run from the repository root. On the live VPS, non-login SSH shells omitted /home/delta/.bun/bin and remote native verification could not find deployment/native/check-native-infra.sh because it ran from the home directory. Update the deploy helper to prepend /Users/kell/.bun/bin when present and cd into the repo before native verification checks run.","status":"closed","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:38:32Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:40:33Z","closed_at":"2026-05-19T23:40:33Z","close_reason":"Updated native SSH deploy flow to prepend Bun's home install path when present and run native verification from the repo root before health scripts.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-wf5","title":"Harden native options provider configuration after synthetic recovery","description":"Native production recovery restored OPTIONS_INGEST_ADAPTER=synthetic because the current Alpaca setup fails authentication and crash-loops ingest-options. Follow up by deciding whether production options should remain synthetic or move to a supported live provider auth path, then add a deploy-time smoke test or config validation that catches provider auth failures before native cutover.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:27:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-m83","title":"Restore options ingestion and print generation on native deployment","description":"After moving the production/VPS deployment from Docker-managed services to the native runtime, the options feed appears behind and fresh option prints are not reaching the UI. Investigate the native deployment path on the server, identify the ingestion or compute breakage, apply the required code and/or host configuration changes, validate that fresh option prints resume, and document any follow-up operational work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:20:01Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:52Z","started_at":"2026-05-19T23:20:10Z","closed_at":"2026-05-19T23:27:52Z","close_reason":"Restored native options ingest by switching the VPS back to the last known-good synthetic adapter, verified fresh option prints and compute output, and documented the native env precedence gotcha.","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/docs/turns/2026-05-19-fix-native-alpaca-news.html b/docs/turns/2026-05-19-fix-native-alpaca-news.html new file mode 100644 index 0000000..ddecc1a --- /dev/null +++ b/docs/turns/2026-05-19-fix-native-alpaca-news.html @@ -0,0 +1,233 @@ + + + + + + Turn Report: Fix Native Alpaca News + + + +
    +

    Created 2026-05-19 20:05 EDT · Branch: alpaca-news · Issue: islandflow-laq

    +

    Fix Native Alpaca News

    +
    +

    + Restored the native Alpaca news pipeline on the VPS by correcting Alpaca auth to use key ID + secret, + adding the missing native islandflow-ingest-news unit and worker-scope wiring, fixing the + Alpaca news backfill defaults to match the current API contract, requesting article content explicitly, + and repairing API-side news persistence so the feed is both live and queryable. +

    +
    + VPS unit installed and enabled + Alpaca auth aligned to current docs + Live news confirmed + ClickHouse news history confirmed +
    +
    + +
    +

    Summary

    +

    + The original native news rollout failed for two separate reasons: the repo never fully wired + ingest-news into the native worker templates, and the service was still using bearer-style + Alpaca auth plus an oversized backfill limit that Alpaca's current News API rejects. After the service + started flowing again, one more pipeline gap appeared: the API fanned news out live but never persisted it + to ClickHouse, so /news stayed empty even when headlines showed up in the UI. +

    +
    + +
    +

    Changes Made

    +
      +
    • Added shared Alpaca credential helpers in packages/config with support for official key ID + secret auth and a legacy bearer fallback.
      • +
    • Rewired the Alpaca news, options, and equities adapters to use the shared auth model instead of hardcoded bearer headers and empty websocket secrets.
      • +
    • Added the checked-in native user unit deployment/native/systemd/user/islandflow-ingest-news.service.
      • +
    • Updated native install, health, cutover, rollback, and deploy-scope scripts so worker/native rollouts include ingest-news.
      • +
    • Corrected the native and Docker env/docs story to advertise current Alpaca credential names.
      • +
    • Lowered the default Alpaca news backfill limit from 100 to 50 to match the current endpoint contract.
      • +
    • Requested include_content=true for Alpaca news backfill and added a safe summary fallback when article content is missing.
      • +
    • Fixed API-side persistence by inserting each consumed news story into ClickHouse before live fanout.
      • +
    • On the VPS, created a fresh .env backup, added ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY, set ALPACA_NEWS_BACKFILL_LIMIT=50, switched the server checkout to alpaca-news, installed the new user unit, and restarted api plus ingest-news.
      • +
    +
    + +
    +

    Context

    +

    + Alpaca's current official auth docs require the APCA-API-KEY-ID and + APCA-API-SECRET-KEY header pair for market-data requests, and the current News endpoint + documents a limit range of 1..50 plus optional + include_content. This turn aligned Islandflow's native news path with those present-day + contracts instead of relying on the older single-token assumption that had drifted into the repo. +

    +
    + +
    +

    Important Implementation Details

    +
      +
    • The shared helper prefers ALPACA_API_KEY_ID + ALPACA_API_SECRET_KEY, also accepts ALPACA_KEY_ID + ALPACA_SECRET_KEY, and only falls back to legacy bearer auth when no secret is present.
      • +
    • The news backfill now requests article bodies explicitly. When Alpaca still omits full content, the service emits an escaped summary paragraph instead of a blank story body.
      • +
    • The native worker scope now treats ingest-news as a first-class worker everywhere the repo previously only handled options and equities.
      • +
    • The API now persists each consumed news story into ClickHouse before live fanout, which restores /news and history behavior without removing the live websocket path.
      • +
    +
    + +
    +

    Relevant Diff Snippets

    + 
    diff --git a/packages/config/src/alpaca.ts b/packages/config/src/alpaca.ts
++export const buildAlpacaAuthHeaders = (credentials) => ({
++  "APCA-API-KEY-ID": credentials.keyId,
++  "APCA-API-SECRET-KEY": credentials.secret
++})
++export const buildAlpacaWebSocketAuthMessage = (credentials) => ({
++  action: "auth",
++  key: credentials.keyId,
++  secret: credentials.secret
++})
    + 
    diff --git a/services/ingest-news/src/index.ts b/services/ingest-news/src/index.ts
+-  ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(200).default(100),
++  ALPACA_NEWS_BACKFILL_LIMIT: z.coerce.number().int().positive().max(50).default(50),
++  url.searchParams.set("include_content", "true");
++  const contentHtml = item.content?.trim() || (summary ? `<p>${escapeHtml(summary)}</p>` : "");
    + 
    diff --git a/services/api/src/index.ts b/services/api/src/index.ts
+   const payload = NewsStorySchema.parse(newsSubscription.decode(msg));
++  await insertNewsStory(clickhouse, payload);
+   await fanoutLive({ channel: "news" }, payload, "news");
+   msg.ack();
    +

    These snippets are included in a diff-style rendering format for fast review.

    +
    + +
    +

    Expected Impact for End-Users

    +

    + Native Islandflow deployments on the VPS now have a real Alpaca-backed news worker instead of a missing unit + and a crash loop. News stories populate with actual article body content in the feed more reliably, and the + API's /news path can serve persisted recent stories instead of only depending on live websocket + state. +

    +
    + +
    +

    Validation

    +
      +
    • Ran local targeted tests: bun test packages/config/tests packages/storage/tests/news.test.ts services/ingest-news/tests services/ingest-equities/tests and all passed.
      • +
    • Ran bun run check:docker-workspace and confirmed the Docker workspace snapshot stayed in sync.
      • +
    • Verified against current Alpaca docs that market-data auth uses key ID + secret and that the news endpoint limit is capped at 50.
      • +
    • On the VPS, confirmed the new islandflow-ingest-news.service unit is installed, enabled, and active under systemd --user.
      • +
    • Queried Alpaca directly from the VPS with the configured credentials and confirmed GET https://data.alpaca.markets/v1beta1/news?limit=1&sort=desc returned HTTP 200.
      • +
    • Restarted the VPS api and ingest-news services after the persistence fix so the API would store newly republished backfill stories.
      • +
    • Verified VPS API output: GET http://127.0.0.1:4000/news?limit=3 returned 3 recent real Alpaca stories with non-empty content_html payloads.
      • +
    • Verified ClickHouse persistence: SELECT count(), max(story_id), max(published_ts) FROM news returned 50 rows after the republished backfill.
      • +
    +
    + +
    +

    Issues, Limitations, and Mitigations

    +
      +
    • The server checkout still carries an unrelated untracked file, deployment/docker/signal-cli-0.14.3-Linux-native.tar.gz. It does not block the news fix, but it is repo hygiene debt on the VPS checkout.
      • +
    • The shared Alpaca helper keeps a legacy bearer fallback so older setups do not fail immediately, but the repo documentation now treats key ID + secret as the supported path.
      • +
    • Some Alpaca/Benzinga stories may still omit full content. The summary fallback prevents a blank drawer in those cases, but it cannot synthesize text Alpaca does not send.
      • +
    +
    + +
    +

    Follow-up Work

    +
      +
    • No new follow-up Beads issue was required to ship this repair.
      • +
    • If native Alpaca options or equities are re-enabled later, the shared credential changes in this turn already cover the same key ID + secret auth model.
      • +
    • If the team wants historical news beyond the startup backfill, the next logical extension is a scheduled catch-up cursor instead of only restart-time republishing.
      • +
    +
    +
    + + From adba1f6b5ac458e79e1c13cde9698217821394ff Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Wed, 20 May 2026 02:59:53 -0400 Subject: [PATCH 24/26] fix historical alert flow packet resolution --- .beads/issues.jsonl | 1 + apps/web/app/terminal.test.ts | 29 ++ apps/web/app/terminal.tsx | 48 +- ...6-05-20-fix-alert-flow-packet-history.html | 412 ++++++++++++++++++ 4 files changed, 473 insertions(+), 17 deletions(-) create mode 100644 docs/turns/2026-05-20-fix-alert-flow-packet-history.html diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 57fbdd7..245689b 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-yza","title":"Persist historical flow packets for alert detail replay","description":"## Why\nAlert details can show a missing persisted flow packet when the packet is no longer present in the Redis hot cache, even though the associated historical alert and evidence were loaded from ClickHouse.\n\n## What needs to be done\nTrace the API path that resolves alert detail flow packets, compare Redis hot-cache lookups with ClickHouse historical fetches, and ensure historical flow packet payloads are treated as first-class persisted data with context preserved when replaying or loading older alerts.\n\n## Acceptance Criteria\n- Alert detail flow packets load for historical alerts even when the packet is absent from Redis hot cache\n- Historical ClickHouse-backed flow packet responses preserve the context required by the UI\n- Relevant automated tests cover the regression or the gap is explicitly documented","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-20T06:52:04Z","created_by":"dirtydishes","updated_at":"2026-05-20T06:59:26Z","started_at":"2026-05-20T06:52:09Z","closed_at":"2026-05-20T06:59:26Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-jor","title":"Support Forgejo pull request status in desktop git panel","description":"The desktop app currently reports pull request status unavailable when a repository only has a Forgejo remote. Add native Forgejo/Gitea-style remote detection and pull request status lookup so Forgejo-only repositories can show PR state in the Codex app git panel.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T20:55:15Z","created_by":"dirtydishes","updated_at":"2026-05-19T20:59:46Z","started_at":"2026-05-19T20:55:25Z","closed_at":"2026-05-19T20:59:46Z","close_reason":"Patched the installed Codex desktop app bundle with a Forgejo PR status fallback and documented the local change.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-g3a","title":"Reconcile PR merge conflicts","description":"Resolve the current pull request conflicts for the nextjs-upgrade branch, validate the result, document the turn, and push the reconciled branch.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:44:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:47:35Z","started_at":"2026-05-19T18:44:56Z","closed_at":"2026-05-19T18:47:35Z","close_reason":"Merged forgejo/main into nextjs-upgrade, resolved README and Beads conflicts, updated JetStream retention tests, validated deploy help, Docker workspace sync, API/bus tests, and web build, and added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-jbi","title":"Hydrate alert evidence details from ClickHouse","description":"Alert detail drawers need to fetch persisted alert context from ClickHouse by trace id, including linked flow packets, option prints, preserved execution context, and explicit missing refs for UI diagnostics.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:55:43Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:01:58Z","started_at":"2026-05-17T14:55:53Z","closed_at":"2026-05-17T15:01:58Z","close_reason":"Implemented ClickHouse-backed alert context hydration across storage, API, terminal drawer, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/apps/web/app/terminal.test.ts b/apps/web/app/terminal.test.ts index 63918f2..92a9904 100644 --- a/apps/web/app/terminal.test.ts +++ b/apps/web/app/terminal.test.ts @@ -43,6 +43,8 @@ import { shouldClearOptionFocusSeed, smartMoneyProfileLabel, smartMoneyToneForProfile, + getAlertFlowPacketRefs, + resolveAlertFlowPacket, statusLabel, toggleFilterValue } from "./terminal"; @@ -133,6 +135,33 @@ describe("alert context hydration helpers", () => { expect(evidence.prints.get("print:1")?.execution_nbbo_bid).toBe(1.2); expect(evidence.prints.get("print:1")?.execution_underlying_spot).toBe(450.05); }); + + it("finds flow-packet refs even when they are not first in alert evidence", () => { + const alert = makeAlert({ + evidence_refs: ["smartmoney:single_leg_event:flowpacket:1", "flowpacket:1", "print:1"] + }); + + expect(getAlertFlowPacketRefs(alert)).toEqual(["flowpacket:1"]); + }); + + it("resolves the primary alert flow packet from hydrated historical context", () => { + const packet = { + trace_id: "flowpacket:1", + id: "flowpacket:1", + members: ["print:1"], + source_ts: 1, + ingest_ts: 2, + seq: 1, + features: {}, + join_quality: {} + } as any; + const alert = makeAlert({ + evidence_refs: ["smartmoney:single_leg_event:flowpacket:1", "flowpacket:1", "print:1"] + }); + const packets = new Map([[packet.id, packet]]); + + expect(resolveAlertFlowPacket(alert, packets)).toBe(packet); + }); }); describe("live manifest", () => { diff --git a/apps/web/app/terminal.tsx b/apps/web/app/terminal.tsx index 3bec184..3057f58 100644 --- a/apps/web/app/terminal.tsx +++ b/apps/web/app/terminal.tsx @@ -4753,6 +4753,26 @@ export const collectAlertContextEvidence = ( return { packets, prints }; }; +export const getAlertFlowPacketRefs = ( + alert: Pick +): string[] => { + return alert.evidence_refs.filter((ref) => ref.startsWith("flowpacket:")); +}; + +export const resolveAlertFlowPacket = ( + alert: Pick, + packets: Map +): FlowPacket | null => { + for (const ref of getAlertFlowPacketRefs(alert)) { + const packet = packets.get(ref); + if (packet) { + return packet; + } + } + + return null; +}; + type DarkEvidenceItem = | { kind: "join"; id: string; join: EquityPrintJoin } | { kind: "unknown"; id: string }; @@ -6014,8 +6034,7 @@ const useTerminalState = () => { if (!selectedAlert) { return null; } - const packetId = selectedAlert.evidence_refs[0]; - return packetId ? resolvedFlowPacketMap.get(packetId) ?? null : null; + return resolveAlertFlowPacket(selectedAlert, resolvedFlowPacketMap); }, [selectedAlert, resolvedFlowPacketMap]); const selectedDarkEvidence = useMemo((): DarkEvidenceItem[] => { @@ -6427,12 +6446,9 @@ const useTerminalState = () => { return fromTrace; } - const packetId = alert.evidence_refs[0]; - if (packetId) { - const packet = resolvedFlowPacketMap.get(packetId); - if (packet) { - return extractUnderlying(extractPacketContract(packet)); - } + const packet = resolveAlertFlowPacket(alert, resolvedFlowPacketMap); + if (packet) { + return extractUnderlying(extractPacketContract(packet)); } for (const ref of alert.evidence_refs) { @@ -6704,9 +6720,7 @@ const useTerminalState = () => { return; } - const visiblePacketIds = visibleAlerts - .map((alert) => alert.evidence_refs[0] ?? null) - .filter((id): id is string => Boolean(id) && id.startsWith("flowpacket:")); + const visiblePacketIds = visibleAlerts.flatMap((alert) => getAlertFlowPacketRefs(alert)); const missingPacketIds = Array.from(new Set(visiblePacketIds)).filter( (id) => !resolvedFlowPacketMap.has(id) ); @@ -6788,9 +6802,10 @@ const useTerminalState = () => { const activePinnedFlowKeys = useMemo(() => { const keys = new Set(); - const selectedAlertPacketId = selectedAlert?.evidence_refs[0]; - if (selectedAlertPacketId) { - keys.add(selectedAlertPacketId); + if (selectedAlert) { + for (const packetId of getAlertFlowPacketRefs(selectedAlert)) { + keys.add(packetId); + } } if (selectedClassifierPacketId) { keys.add(selectedClassifierPacketId); @@ -6799,8 +6814,7 @@ const useTerminalState = () => { keys.add(packetId); } for (const alert of visibleAlerts) { - const packetId = alert.evidence_refs[0]; - if (packetId) { + for (const packetId of getAlertFlowPacketRefs(alert)) { keys.add(packetId); } } @@ -6945,7 +6959,7 @@ const useTerminalState = () => { const desiredTrace = `alert:${packetId}`; return ( alertsFeed.items.find( - (item) => item.trace_id === desiredTrace || item.evidence_refs[0] === packetId + (item) => item.trace_id === desiredTrace || getAlertFlowPacketRefs(item).includes(packetId) ) ?? null ); }, diff --git a/docs/turns/2026-05-20-fix-alert-flow-packet-history.html b/docs/turns/2026-05-20-fix-alert-flow-packet-history.html new file mode 100644 index 0000000..d7e2b30 --- /dev/null +++ b/docs/turns/2026-05-20-fix-alert-flow-packet-history.html @@ -0,0 +1,412 @@ + + + + + + Fix historical alert flow packet persistence in the web terminal + + + + + + +
    +
    +

    Turn Document · 2026-05-20 02:56 EDT

    +

    Historical Alert Flow Packets Persist Again

    +

    Alert detail drawers now resolve persisted flow packets from ClickHouse-backed historical context instead of assuming the first evidence reference is the packet. This restores packet visibility for replayed and older alerts after their Redis hot-cache entries have aged out.

    +
    + Beads: islandflow-yza + Surface: apps/web terminal + Validation: tests + prod build +
    +
    + +
    +
    +

    Summary

    +

    The web terminal was assuming alert.evidence_refs[0] always pointed at a flow packet. For compute-generated alerts, the first evidence ref is often the smart-money event id, with the actual packet id later in the list. That made persisted historical packets look missing even when ClickHouse context had already hydrated them successfully.

    +
    + +
    +

    Changes Made

    +
      +
    • Added shared alert helpers in apps/web/app/terminal.tsx to extract all flow-packet refs from an alert and resolve the first hydrated packet semantically.
      • +
    • Switched the alert drawer's selected packet lookup to use the shared resolver instead of the first evidence ref.
      • +
    • Updated alert-underlying inference, visible-alert prefetch, pinned-flow retention keys, and classifier-hit-to-alert matching to use the same alert packet semantics.
      • +
    • Added focused regression coverage in apps/web/app/terminal.test.ts for alerts whose packet ref is not the first evidence entry.
      • +
    +
    + +
    +

    Context

    +

    Islandflow alert detail views combine live Redis retention with ClickHouse historical hydration. Once a packet leaves the hot cache, the UI must treat ClickHouse-loaded evidence as first-class persisted context, not as a degraded fallback. The bug was in the web client’s interpretation of alert evidence ordering, not in the persistence of the packet itself.

    +
    + Historical packet context was already present. The terminal simply was not selecting it unless the packet id happened to be the first evidence ref. +
    +
    + +
    +

    Important Implementation Details

    +
      +
    • The fix is backward-compatible with already-persisted alerts because it tolerates existing evidence ordering instead of rewriting stored records.
      • +
    • The shared resolver centralizes the packet-selection rule so replay, pinning, and alert navigation do not drift apart again.
      • +
    • The classifier-hit alert matching path now finds alerts by any embedded packet ref, which improves consistency when opening related alert context from signal panes.
      • +
    +
    + +
    +

    Relevant Diff Snippets

    +
    +
    +

    apps/web/app/terminal.tsx · alert packet resolution

    +
    + 
    -const packetId = selectedAlert.evidence_refs[0];
+-return packetId ? resolvedFlowPacketMap.get(packetId) ?? null : null;
++return resolveAlertFlowPacket(selectedAlert, resolvedFlowPacketMap);
    +
    + +
    +

    apps/web/app/terminal.tsx · prefetch and alert matching

    +
    + 
    -const visiblePacketIds = visibleAlerts
+-  .map((alert) => alert.evidence_refs[0] ?? null)
+-  .filter((id): id is string => Boolean(id) && id.startsWith("flowpacket:"));
++const visiblePacketIds = visibleAlerts.flatMap((alert) => getAlertFlowPacketRefs(alert));
+
+-alertsFeed.items.find((item) => item.trace_id === desiredTrace || item.evidence_refs[0] === packetId)
++alertsFeed.items.find(
++  (item) => item.trace_id === desiredTrace || getAlertFlowPacketRefs(item).includes(packetId)
++)
    +
    +
    +

    These snippets are rendered client-side with Diffs using the same old/new code blocks shown in the fallback text if the library cannot load.

    +
    + +
    +

    Expected Impact for End-Users

    +

    Older or replayed alerts should now show their persisted flow packet summary in the detail drawer even after the Redis hot cache no longer has that packet. Users investigating signal history should keep the same evidence continuity they get from live data: packet summary, print context, and related alert linkage stay intact.

    +
    + +
    +

    Validation

    +
      +
    • bun test apps/web/app/terminal.test.ts passed with 72 tests.
      • +
    • bun --cwd=apps/web run build passed on Next.js 16.2.6.
      • +
    • The new tests specifically cover alerts where a smart-money event id precedes the packet id in evidence_refs.
      • +
    +
    + +
    +

    Issues, Limitations, and Mitigations

    +
      +
    • This change does not alter how compute persists alert evidence ordering. Instead, it makes the terminal resilient to existing and future mixed evidence lists.
      • +
    • The Diffs rendering in this document loads from the published package at view time. A plain-text fallback is included directly in the HTML so the document remains readable offline.
      • +
    • No full monorepo test sweep was run because the change was isolated to the web terminal alert-context path.
      • +
    +
    + +
    +

    Follow-up Work

    +
      +
    • No additional Beads issue was required for this fix.
      • +
    • Optional: audit whether compute should emit packet ids before higher-level event ids in evidence_refs for simpler downstream consumers.
      • +
    • Optional: add a small integration test around alert drawer selection if the web app gains component-level interaction tests later.
      • +
    +
    +
    +
    + + + + From d9c8b53b695046dbc1c6d6e7db77c25cbe79c47f Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Wed, 20 May 2026 10:41:13 -0400 Subject: [PATCH 25/26] add desktop codex login and analyst copilot --- .beads/issues.jsonl | 20 + apps/desktop/package.json | 3 + apps/desktop/src/desktop-ai-ipc.ts | 8 + apps/desktop/src/desktop-ai.test.ts | 174 +++ apps/desktop/src/desktop-ai.ts | 1222 +++++++++++++++++ apps/desktop/src/main.ts | 96 +- apps/desktop/src/preload.ts | 43 + apps/desktop/tsconfig.json | 4 +- apps/web/app/charts/page.tsx | 4 +- apps/web/app/desktop-ai-panels.tsx | 924 +++++++++++++ apps/web/app/desktop-ai.tsx | 179 +++ apps/web/app/globals.css | 377 ++++- apps/web/app/layout.tsx | 5 +- apps/web/app/replay/page.tsx | 4 +- apps/web/app/routes.test.ts | 38 +- apps/web/app/settings/page.tsx | 7 + apps/web/app/signals/page.tsx | 4 +- apps/web/app/terminal.test.ts | 26 +- apps/web/app/terminal.tsx | 254 +++- bun.lock | 3 + ...05-20-codex-desktop-login-and-copilot.html | 538 ++++++++ packages/types/src/desktop-ai.ts | 303 ++++ packages/types/src/index.ts | 1 + 23 files changed, 4142 insertions(+), 95 deletions(-) create mode 100644 apps/desktop/src/desktop-ai-ipc.ts create mode 100644 apps/desktop/src/desktop-ai.test.ts create mode 100644 apps/desktop/src/desktop-ai.ts create mode 100644 apps/desktop/src/preload.ts create mode 100644 apps/web/app/desktop-ai-panels.tsx create mode 100644 apps/web/app/desktop-ai.tsx create mode 100644 apps/web/app/settings/page.tsx create mode 100644 docs/turns/2026-05-20-codex-desktop-login-and-copilot.html create mode 100644 packages/types/src/desktop-ai.ts diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index e025c4d..8653c73 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,6 @@ +{"_type":"issue","id":"islandflow-yza","title":"Persist historical flow packets for alert detail replay","description":"## Why\nAlert details can show a missing persisted flow packet when the packet is no longer present in the Redis hot cache, even though the associated historical alert and evidence were loaded from ClickHouse.\n\n## What needs to be done\nTrace the API path that resolves alert detail flow packets, compare Redis hot-cache lookups with ClickHouse historical fetches, and ensure historical flow packet payloads are treated as first-class persisted data with context preserved when replaying or loading older alerts.\n\n## Acceptance Criteria\n- Alert detail flow packets load for historical alerts even when the packet is absent from Redis hot cache\n- Historical ClickHouse-backed flow packet responses preserve the context required by the UI\n- Relevant automated tests cover the regression or the gap is explicitly documented","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-20T06:52:04Z","created_by":"dirtydishes","updated_at":"2026-05-20T06:59:26Z","started_at":"2026-05-20T06:52:09Z","closed_at":"2026-05-20T06:59:26Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-jor","title":"Support Forgejo pull request status in desktop git panel","description":"The desktop app currently reports pull request status unavailable when a repository only has a Forgejo remote. Add native Forgejo/Gitea-style remote detection and pull request status lookup so Forgejo-only repositories can show PR state in the Codex app git panel.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T20:55:15Z","created_by":"dirtydishes","updated_at":"2026-05-19T20:59:46Z","started_at":"2026-05-19T20:55:25Z","closed_at":"2026-05-19T20:59:46Z","close_reason":"Patched the installed Codex desktop app bundle with a Forgejo PR status fallback and documented the local change.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-g3a","title":"Reconcile PR merge conflicts","description":"Resolve the current pull request conflicts for the nextjs-upgrade branch, validate the result, document the turn, and push the reconciled branch.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:44:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:47:35Z","started_at":"2026-05-19T18:44:56Z","closed_at":"2026-05-19T18:47:35Z","close_reason":"Merged forgejo/main into nextjs-upgrade, resolved README and Beads conflicts, updated JetStream retention tests, validated deploy help, Docker workspace sync, API/bus tests, and web build, and added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-jbi","title":"Hydrate alert evidence details from ClickHouse","description":"Alert detail drawers need to fetch persisted alert context from ClickHouse by trace id, including linked flow packets, option prints, preserved execution context, and explicit missing refs for UI diagnostics.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:55:43Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:01:58Z","started_at":"2026-05-17T14:55:53Z","closed_at":"2026-05-17T15:01:58Z","close_reason":"Implemented ClickHouse-backed alert context hydration across storage, API, terminal drawer, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8kj","title":"Configure persistent beads Dolt remote on deltaisland server","description":"Install the beads and Dolt CLIs on the server, configure a persistent Dolt sync remote backed by the server-hosted Forgejo repository, verify refs/dolt/data publication, and document Nginx Proxy Manager / firewall considerations.","status":"closed","priority":1,"issue_type":"task","assignee":"delta","created_at":"2026-05-17T10:31:31Z","created_by":"delta","updated_at":"2026-05-17T10:37:47Z","started_at":"2026-05-17T10:32:16Z","closed_at":"2026-05-17T10:37:47Z","close_reason":"Installed bd and dolt on the server, configured the Forgejo-backed Dolt remote, published refs/dolt/data, and documented the setup.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -13,6 +16,19 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-6tn","title":"Add Codex desktop login and usage bridge","description":"Implement a desktop-only Codex integration for the Islandflow Electron app using the official codex app-server with managed ChatGPT login, native IPC, settings UI, usage tracking, and clean web degradation.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-20T14:01:36Z","created_by":"dirtydishes","updated_at":"2026-05-20T14:40:49Z","started_at":"2026-05-20T14:01:48Z","closed_at":"2026-05-20T14:40:49Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-laq","title":"fix native alpaca news deploy and auth","description":"Why this issue exists and what needs to be done:\\n\\nNative Islandflow rollout is incomplete because services/ingest-news is not healthy on the VPS. The checked-in native user units and helper scripts do not fully include ingest-news, and the current service uses bearer-style auth that returns 401 against Alpaca news endpoints.\\n\\nThis task should verify the current Alpaca news auth requirements against official docs, update the repo code and native deployment assets as needed, install and enable the missing VPS unit, verify news events flow end-to-end, and document the work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:47:07Z","created_by":"dirtydishes","updated_at":"2026-05-20T00:05:20Z","started_at":"2026-05-19T23:47:12Z","closed_at":"2026-05-20T00:05:20Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-fmg","title":"Fix native deploy SSH path and verification cwd assumptions","description":"Native deploys over SSH assumed bun was already on PATH and that remote verification would run from the repository root. On the live VPS, non-login SSH shells omitted /home/delta/.bun/bin and remote native verification could not find deployment/native/check-native-infra.sh because it ran from the home directory. Update the deploy helper to prepend /Users/kell/.bun/bin when present and cd into the repo before native verification checks run.","status":"closed","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:38:32Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:40:33Z","closed_at":"2026-05-19T23:40:33Z","close_reason":"Updated native SSH deploy flow to prepend Bun's home install path when present and run native verification from the repo root before health scripts.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-wf5","title":"Harden native options provider configuration after synthetic recovery","description":"Native production recovery restored OPTIONS_INGEST_ADAPTER=synthetic because the current Alpaca setup fails authentication and crash-loops ingest-options. Follow up by deciding whether production options should remain synthetic or move to a supported live provider auth path, then add a deploy-time smoke test or config validation that catches provider auth failures before native cutover.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:27:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-m83","title":"Restore options ingestion and print generation on native deployment","description":"After moving the production/VPS deployment from Docker-managed services to the native runtime, the options feed appears behind and fresh option prints are not reaching the UI. Investigate the native deployment path on the server, identify the ingestion or compute breakage, apply the required code and/or host configuration changes, validate that fresh option prints resume, and document any follow-up operational work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:20:01Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:52Z","started_at":"2026-05-19T23:20:10Z","closed_at":"2026-05-19T23:27:52Z","close_reason":"Restored native options ingest by switching the VPS back to the last known-good synthetic adapter, verified fresh option prints and compute output, and documented the native env precedence gotcha.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-o1v","title":"Add SCM provider layer with Forgejo detection","description":"Implement provider-aware source-control detection and mirror-aware guardrails for repo automation so Forgejo remotes are treated as authoritative when present.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:04:33Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:06:55Z","started_at":"2026-05-19T23:04:35Z","closed_at":"2026-05-19T23:06:55Z","close_reason":"created by mistake during interrupted turn; no implementation was started","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-tqk","title":"publish docs/ to github pages with navigable index","description":"Set up docs deployment so repository docs are published to dirtydishes.github.io/islandflow/docs with a nicer, browsable experience than a raw file listing.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:56:02Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:59:55Z","started_at":"2026-05-19T18:56:04Z","closed_at":"2026-05-19T18:59:55Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-lm6","title":"Clarify repo turn documentation scope","description":"Update AGENTS.md so repository turn documentation clearly uses repo-local docs/turns and impeccable styling, without inheriting global non-repo computer-task styling.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T12:05:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T12:06:12Z","started_at":"2026-05-19T12:05:14Z","closed_at":"2026-05-19T12:06:12Z","close_reason":"Verified AGENTS.md now scopes repo turn docs to docs/turns and makes impeccable the styling authority; added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-6iq","title":"Update README for current project state","description":"Resolve README merge conflicts and document the current project state, including the smart money classification taxonomy, Next.js update, and deployment workflow changes.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:37:24Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:40:01Z","started_at":"2026-05-19T11:37:31Z","closed_at":"2026-05-19T11:40:01Z","close_reason":"README conflict resolved and current project state documented, including smart-money taxonomy, Next.js update, and deployment workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-lib","title":"Upgrade apps/web to Next.js 16.2.6","description":"Upgrade the web app dependency stack to Next.js 16.2.6 with React 19, refresh Bun and mirrored Docker workspace lockfiles, keep runtime behavior unchanged, fix any focused web test fallout, validate the web build and targeted route tests, and document the completed work.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:04:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:31:23Z","started_at":"2026-05-19T11:04:57Z","closed_at":"2026-05-19T11:31:23Z","close_reason":"Upgraded apps/web to Next.js 16.2.6 with React 19, refreshed Bun lockfiles including the Docker workspace mirror, fixed the React 19 nullable ref type issue, and validated the web build, focused tests, Docker workspace sync, and route smoke checks.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-8fn","title":"implement alpaca-backed news wire view","description":"Why this issue exists and what needs to be done:\\nAdd an Alpaca-powered live news pipeline, API, storage, and web experience, including a dedicated /news route, Home preview, live fanout, history pagination, ticker resolution, and replay-mode live-only empty states.\\n\\nAcceptance criteria:\\n- normalized NewsStory contract and live channel exist\\n- ingest-news service backfills and streams Alpaca news\\n- API persists, serves, and fans out news\\n- web app exposes /news plus Home preview and drawer\\n- tests cover types, storage, API, and key UI behaviors\\n- turn documentation is added\\n\\nDesign:\\nReuse Islandflow drawer, chips, panes, and terminal styling; keep news live-only in v1 replay mode.\\n\\nNotes:\\nImplement client-side ticker filtering in v1 and expose latest revision only per provider+story_id.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T20:37:13Z","created_by":"dirtydishes","updated_at":"2026-05-18T20:55:11Z","started_at":"2026-05-18T20:37:20Z","closed_at":"2026-05-18T20:55:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-k8i","title":"Fix duplicate alert context import in API entrypoint","description":"Recent alert-context work introduced a duplicate fetchAlertContextByTraceId import in services/api/src/index.ts, which risks breaking TypeScript compilation and API startup. Remove the duplicate import and validate the affected API/web tests.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:58Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:03:40Z","started_at":"2026-05-18T13:02:02Z","closed_at":"2026-05-18T13:03:40Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-lk9","title":"Fix PR creation workflow after Forgejo migration","description":"## Why\\nCreating pull requests with fails after the repository moved primary collaboration from GitHub to Forgejo. The current workflow still assumes GitHub GraphQL PR creation semantics, which do not work against the Forgejo remote.\\n\\n## What\\nInvestigate the current PR creation path, identify remaining GitHub-specific assumptions, and update the repo workflow/scripts/docs so contributors can reliably publish branches and open PRs in the Forgejo-based setup.\\n\\n## Acceptance Criteria\\n- The repo no longer instructs contributors to use a broken GitHub-specific PR creation path for Forgejo branches\\n- There is a documented and preferably scripted way to create the equivalent review request against Forgejo\\n- Validation demonstrates the new workflow behaves correctly or clearly documents any remaining platform limitation","status":"in_progress","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T10:26:47Z","created_by":"dirtydishes","updated_at":"2026-05-18T10:26:53Z","started_at":"2026-05-18T10:26:53Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-1ei","title":"Make deploy helper remote-aware for Forgejo","description":"Why: scripts/deploy.ts hardcodes git remote name origin for fetch/pull/push and branch verification, but this repository now uses forgejo/github remotes and may not have an origin remote. What: update deploy.ts to resolve the deploy git remote robustly (Forgejo-aware), use it across local prechecks, branch publish, and remote rollout git operations, and keep behavior explicit in output.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T03:20:12Z","created_by":"dirtydishes","updated_at":"2026-05-18T03:22:39Z","started_at":"2026-05-18T03:20:16Z","closed_at":"2026-05-18T03:22:39Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-xod","title":"Add --fast mode to deploy helper","description":"Why: full main deploys rebuild all images and run full verification, which is slow for routine rollouts. What: add a --fast flag to scripts/deploy.ts with explicit behavior that short-circuits slow steps while preserving basic safety checks; update help text/docs for discoverability.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T02:50:47Z","created_by":"dirtydishes","updated_at":"2026-05-18T02:53:41Z","started_at":"2026-05-18T02:50:50Z","closed_at":"2026-05-18T02:53:41Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-cif","title":"hydrate alert evidence context from clickhouse","description":"Implement alert detail hydration from ClickHouse with a new context endpoint and frontend drawer evidence resolution. Includes storage lookup by alert trace_id/evidence refs, unresolved refs diagnostics, API route GET /flow/alerts/:trace_id/context, terminal evidence hydration + loading states/copy updates, and tests across storage/api/web.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T00:15:55Z","created_by":"dirtydishes","updated_at":"2026-05-18T00:17:38Z","started_at":"2026-05-18T00:16:00Z","closed_at":"2026-05-18T00:17:38Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -46,6 +62,10 @@ {"_type":"issue","id":"islandflow-zs0","title":"Migrate terminal UI to smart-money profiles","description":"Migrate apps/web terminal rendering to consume SmartMoneyEvent directly: primary profile, probability ladder, reason codes, and suppression/abstention state, while preserving legacy alert/classifier displays during the bridge.","status":"closed","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:23Z","created_by":"dirtydishes","updated_at":"2026-05-05T05:39:58Z","closed_at":"2026-05-05T05:39:58Z","close_reason":"Completed terminal smart-money profile migration","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-igk","title":"Add plan mode","description":"Implement a user-facing plan mode in the application so users can switch into planning before taking action. Scope to be clarified from existing app patterns.","status":"closed","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:22:37Z","created_by":"dirtydishes","updated_at":"2026-05-04T04:26:18Z","started_at":"2026-05-04T04:22:40Z","closed_at":"2026-05-04T04:26:18Z","close_reason":"Implemented as a global pi extension toggled with Shift+P","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-biq","title":"Finish raw live options delivery and filter/backpressure observability","description":"The smart-money signal path and Tape filters are in place, but the next firehose pass should finish server-side selective raw live delivery for options subscriptions and add explicit filtered-out/backpressure observability for API/web counters. This was discovered while landing islandflow-e4r.\n","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:28:58Z","created_by":"dirtydishes","updated_at":"2026-04-29T03:54:12Z","started_at":"2026-04-29T03:54:12Z","dependencies":[{"issue_id":"islandflow-biq","depends_on_id":"islandflow-e4r","type":"discovered-from","created_at":"2026-04-28T16:28:58Z","created_by":"auto-import","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-8vr","title":"Summarize 2026-05-19 git activity for standup","description":"Create the daily git summary for 2026-05-19 in docs/general using yesterday's commits, touched files, and validation evidence only.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-20T13:02:41Z","created_by":"dirtydishes","updated_at":"2026-05-20T13:04:50Z","started_at":"2026-05-20T13:02:47Z","closed_at":"2026-05-20T13:04:50Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-0ty","title":"Recreate May 18 standup summary after merge","description":"Regenerate docs/daily-git/2026-05-19-standup-summary-2026-05-18.html using merged history so it reflects all commits in the May 18 window, including native deployment and merge commits.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:53:48Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:55:33Z","started_at":"2026-05-19T18:53:52Z","closed_at":"2026-05-19T18:55:33Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-2df","title":"Publish 2026-05-18 git standup summary","description":"Why: the daily automation needs a grounded standup summary for May 18, 2026. What: review commits from 2026-05-18, create a scannable HTML summary in docs/daily-git, and capture only commit/file-backed statements.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:41:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:42:42Z","started_at":"2026-05-19T18:41:10Z","closed_at":"2026-05-19T18:42:42Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-x70","title":"Create 2026-05-17 git standup summary","description":"Why this issue exists and what needs to be done:\\n- Produce the daily automation summary for 2026-05-17 git activity.\\n- Ground statements in commits, PRs, and touched files only.\\n- Create a user-readable HTML document in docs/general and update automation memory.\\n- Complete the Beads sync and git push workflow after documenting the run.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:43Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:05:37Z","started_at":"2026-05-18T13:01:53Z","closed_at":"2026-05-18T13:05:37Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-zsy","title":"Expose Forgejo SSH on a direct DNS hostname","description":"git.deltaisland.io currently resolves through Cloudflare's proxy, so SSH on port 2222 does not complete even though the Forgejo container is listening on the host. If SSH-based git/beads workflows are desired, add a DNS-only hostname (or adjust the existing record) that points directly at the server for Forgejo SSH.","status":"open","priority":3,"issue_type":"task","created_at":"2026-05-17T10:34:06Z","created_by":"delta","updated_at":"2026-05-17T10:34:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-38p","title":"Add native deployment unit templates and rollback helpers","description":"The deploy helper now supports --runtime native, but the repo still relies on operator-managed systemd units and manual rollback. Add checked-in native deployment templates or provisioning guidance for the expected units, and consider lightweight rollback/smoke-test helpers once the host-native path is exercised on the real VPS.","status":"open","priority":3,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:46:42Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:46:42Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-575","title":"Document smart-money event calendar env","description":"Document smart-money event-calendar environment configuration in env examples and README.\n","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T06:57:14Z","created_by":"dirtydishes","updated_at":"2026-05-05T06:57:57Z","started_at":"2026-05-05T06:57:17Z","closed_at":"2026-05-05T06:57:57Z","close_reason":"Documented event-calendar env variables","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/apps/desktop/package.json b/apps/desktop/package.json index c46915b..8a3c3e5 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -12,6 +12,9 @@ "package": "bun run build && electron-forge package", "make": "bun run build && electron-forge make" }, + "dependencies": { + "@islandflow/types": "workspace:*" + }, "devDependencies": { "@electron-forge/cli": "^7.8.1", "@electron-forge/core": "^7.11.1", diff --git a/apps/desktop/src/desktop-ai-ipc.ts b/apps/desktop/src/desktop-ai-ipc.ts new file mode 100644 index 0000000..25e53f6 --- /dev/null +++ b/apps/desktop/src/desktop-ai-ipc.ts @@ -0,0 +1,8 @@ +export const DESKTOP_AI_STATE_CHANNEL = "islandflow:desktop-ai:state"; +export const DESKTOP_AI_GET_STATE = "islandflow:desktop-ai:get-state"; +export const DESKTOP_AI_LOGIN_BROWSER = "islandflow:desktop-ai:login-browser"; +export const DESKTOP_AI_LOGIN_DEVICE = "islandflow:desktop-ai:login-device"; +export const DESKTOP_AI_CANCEL_LOGIN = "islandflow:desktop-ai:cancel-login"; +export const DESKTOP_AI_LOGOUT = "islandflow:desktop-ai:logout"; +export const DESKTOP_AI_UPDATE_PREFERENCES = "islandflow:desktop-ai:update-preferences"; +export const DESKTOP_AI_RUN_TASK = "islandflow:desktop-ai:run-task"; diff --git a/apps/desktop/src/desktop-ai.test.ts b/apps/desktop/src/desktop-ai.test.ts new file mode 100644 index 0000000..53f4822 --- /dev/null +++ b/apps/desktop/src/desktop-ai.test.ts @@ -0,0 +1,174 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { createAppServerChildEnv, IslandflowDesktopAiService, summarizeRateLimit } from "./desktop-ai.js"; + +const tempDirs: string[] = []; + +const makeTempDir = async (): Promise => { + const dir = await mkdtemp(path.join(tmpdir(), "islandflow-desktop-ai-")); + tempDirs.push(dir); + return dir; +}; + +afterEach(async () => { + await Promise.all( + tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })) + ); +}); + +describe("desktop ai auth environment", () => { + it("scrubs global OpenAI keys for managed ChatGPT sessions", () => { + const env = createAppServerChildEnv("managed-chatgpt", { + OPENAI_API_KEY: "openai-test", + CODEX_API_KEY: "codex-test", + HOME: "/tmp/home" + }); + + expect(env.OPENAI_API_KEY).toBeUndefined(); + expect(env.CODEX_API_KEY).toBeUndefined(); + expect(env.HOME).toBe("/tmp/home"); + }); + + it("preserves keys for api-key mode", () => { + const env = createAppServerChildEnv("api-key", { + OPENAI_API_KEY: "openai-test", + CODEX_API_KEY: "codex-test" + }); + + expect(env.OPENAI_API_KEY).toBe("openai-test"); + expect(env.CODEX_API_KEY).toBe("codex-test"); + }); +}); + +describe("desktop ai usage and state tracking", () => { + it("records exact token usage notifications into usage rollups", async () => { + const dir = await makeTempDir(); + const service = new IslandflowDesktopAiService(dir, async () => {}, () => {}); + const internal = service as any; + + internal.state.account.email = "analyst@example.com"; + internal.state.account.planType = "plus"; + internal.state.preferences.model = "gpt-5.4"; + internal.state.tasks = [ + { + taskId: "task-1", + kind: "smart-money-explain", + title: "Explain smart money event", + subtitle: "AAPL", + status: "running", + createdAt: Date.now(), + updatedAt: Date.now(), + threadId: "thread-1", + turnId: "turn-1", + model: "gpt-5.4", + reasoningEffort: "high", + text: "", + error: null, + compiledScreen: null + } + ]; + internal.activeTasksByThreadId.set("thread-1", { + taskId: "task-1", + taskKind: "smart-money-explain", + taskTitle: "Explain smart money event", + profileId: "managed-chatgpt" + }); + + await internal.handleNotification("thread/tokenUsage/updated", { + threadId: "thread-1", + turnId: "turn-1", + tokenUsage: { + total: { + totalTokens: 1800, + inputTokens: 1000, + cachedInputTokens: 500, + outputTokens: 250, + reasoningOutputTokens: 50 + }, + last: { + totalTokens: 1800, + inputTokens: 1000, + cachedInputTokens: 500, + outputTokens: 250, + reasoningOutputTokens: 50 + } + } + }); + + expect(service.getState().usage.today.breakdown).toEqual({ + totalTokens: 1800, + inputTokens: 1000, + cachedInputTokens: 500, + outputTokens: 250, + reasoningOutputTokens: 50 + }); + expect(service.getState().usage.today.turnCount).toBe(1); + expect(service.getState().usage.recentTurns[0]?.normalizedCostUsd).toBeCloseTo(0.007125, 6); + }); + + it("stores rate-limit snapshots with reset times", async () => { + const dir = await makeTempDir(); + const service = new IslandflowDesktopAiService(dir, async () => {}, () => {}); + const internal = service as any; + + await internal.handleNotification("account/rateLimits/updated", { + rateLimits: { + limitId: "chatgpt_plus", + limitName: "ChatGPT Plus", + primary: { + usedPercent: 38.4, + windowDurationMins: 180, + resetsAt: 1_710_000_000_000 + }, + secondary: { + usedPercent: 12.1, + windowDurationMins: 1440, + resetsAt: 1_710_003_600_000 + }, + planType: "plus" + } + }); + + expect(service.getState().rateLimitsByLimitId.chatgpt_plus).toEqual( + summarizeRateLimit({ + limitId: "chatgpt_plus", + limitName: "ChatGPT Plus", + primary: { + usedPercent: 38.4, + windowDurationMins: 180, + resetsAt: 1_710_000_000_000 + }, + secondary: { + usedPercent: 12.1, + windowDurationMins: 1440, + resetsAt: 1_710_003_600_000 + }, + planType: "plus" + }) + ); + }); + + it("clears local account state on logout", async () => { + const dir = await makeTempDir(); + const service = new IslandflowDesktopAiService(dir, async () => {}, () => {}); + const internal = service as any; + + internal.client = { + request: async () => ({}) + }; + internal.state.account.loggedIn = true; + internal.state.account.email = "analyst@example.com"; + internal.state.account.planType = "plus"; + internal.state.account.login = { status: "browser_pending", message: "Waiting", loginId: "login-1", authUrl: "https://example.com" }; + + await service.logout(); + + expect(service.getState().account.loggedIn).toBe(false); + expect(service.getState().account.email).toBeNull(); + expect(service.getState().account.planType).toBeNull(); + expect(service.getState().account.login).toEqual({ status: "idle", message: "Logged out." }); + }); +}); diff --git a/apps/desktop/src/desktop-ai.ts b/apps/desktop/src/desktop-ai.ts new file mode 100644 index 0000000..8d695a2 --- /dev/null +++ b/apps/desktop/src/desktop-ai.ts @@ -0,0 +1,1222 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { + IslandflowAiCompiledScreenSchema, + IslandflowAiProfileModeSchema, + IslandflowAiReasoningEffortSchema, + IslandflowAiTaskRequestSchema, + type IslandflowAiCompiledScreen, + type IslandflowAiModelSummary, + type IslandflowAiPlanType, + type IslandflowAiPricing, + type IslandflowAiProfileMode, + type IslandflowAiRateLimitSnapshot, + type IslandflowAiReasoningEffort, + type IslandflowAiState, + type IslandflowAiTaskKind, + type IslandflowAiTaskRequest, + type IslandflowAiTaskSnapshot, + type IslandflowAiTokenBreakdown, + type IslandflowAiUsageRollup, + type IslandflowAiUsageTurnRecord +} from "@islandflow/types"; + +const MANAGED_CHATGPT_PROFILE_ID = "managed-chatgpt"; +const WORKSPACE_PROVIDER_PROFILE_ID = "workspace-provider"; +const APP_SERVER_SERVICE_NAME = "Islandflow Analyst Copilot"; +const APP_SERVER_SANDBOX_CWD = "copilot-sandbox"; +const PREFERENCES_FILE = "copilot-preferences.json"; +const USAGE_FILE = "copilot-usage.json"; +const DEFAULT_REASONING = IslandflowAiReasoningEffortSchema.parse("high"); + +const EMPTY_BREAKDOWN: IslandflowAiTokenBreakdown = { + totalTokens: 0, + inputTokens: 0, + cachedInputTokens: 0, + outputTokens: 0, + reasoningOutputTokens: 0 +}; + +type JsonRpcSuccess = { + id: number; + result: unknown; +}; + +type JsonRpcFailure = { + id: number; + error: { + message?: string; + code?: number; + data?: unknown; + }; +}; + +type JsonRpcNotification = { + method: string; + params?: unknown; + id?: never; +}; + +type JsonRpcServerRequest = { + id: number; + method: string; + params?: unknown; +}; + +type JsonRpcMessage = JsonRpcSuccess | JsonRpcFailure | JsonRpcNotification | JsonRpcServerRequest; + +type CodexModelRecord = { + id: string; + model: string; + displayName: string; + description: string; + hidden: boolean; + isDefault: boolean; + supportedReasoningEfforts: Array<{ reasoningEffort: IslandflowAiReasoningEffort }>; + defaultReasoningEffort: IslandflowAiReasoningEffort | null; +}; + +type CodexThreadStartResult = { + thread: { + id: string; + }; + model: string; + reasoningEffort: IslandflowAiReasoningEffort | null; +}; + +type CodexTurnStartResult = { + turn: { + id: string; + }; +}; + +type PersistedUsageStore = { + version: 1; + turns: Record; +}; + +type PersistedPreferences = { + model: string | null; + reasoningEffort: IslandflowAiReasoningEffort | null; +}; + +type OpenExternalFn = (url: string) => Promise; + +type ActiveTaskContext = { + taskId: string; + taskKind: IslandflowAiTaskKind; + taskTitle: string; + profileId: string; +}; + +const MODEL_PRICING: Record = { + "gpt-5.5": { + inputUsdPer1MTokens: 5, + cachedInputUsdPer1MTokens: 0.5, + outputUsdPer1MTokens: 30, + sourceLabel: "OpenAI GPT-5.5 model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5.5" + }, + "gpt-5.4": { + inputUsdPer1MTokens: 2.5, + cachedInputUsdPer1MTokens: 0.25, + outputUsdPer1MTokens: 15, + sourceLabel: "OpenAI GPT-5.4 model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5.4" + }, + "gpt-5.4-mini": { + inputUsdPer1MTokens: 0.75, + cachedInputUsdPer1MTokens: 0.075, + outputUsdPer1MTokens: 4.5, + sourceLabel: "OpenAI GPT-5.4 mini model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5.4-mini" + }, + "gpt-5.3-codex": { + inputUsdPer1MTokens: 1.75, + cachedInputUsdPer1MTokens: 0.175, + outputUsdPer1MTokens: 14, + sourceLabel: "OpenAI GPT-5.3-Codex model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5.3-codex" + }, + "gpt-5.2": { + inputUsdPer1MTokens: 1.75, + cachedInputUsdPer1MTokens: 0.175, + outputUsdPer1MTokens: 14, + sourceLabel: "OpenAI GPT-5.2 model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5.2" + }, + "gpt-5.2-codex": { + inputUsdPer1MTokens: 1.75, + cachedInputUsdPer1MTokens: 0.175, + outputUsdPer1MTokens: 14, + sourceLabel: "OpenAI GPT-5.2-Codex model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5.2-codex" + }, + "gpt-5-codex": { + inputUsdPer1MTokens: 1.25, + cachedInputUsdPer1MTokens: 0.125, + outputUsdPer1MTokens: 10, + sourceLabel: "OpenAI GPT-5-Codex model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5-codex" + }, + "codex-mini-latest": { + inputUsdPer1MTokens: 1.5, + cachedInputUsdPer1MTokens: 0.375, + outputUsdPer1MTokens: 6, + sourceLabel: "OpenAI codex-mini-latest model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/codex-mini-latest" + } +}; + +const createEmptyUsageRollup = (): IslandflowAiUsageRollup => ({ + breakdown: { ...EMPTY_BREAKDOWN }, + normalizedCostUsd: 0, + turnCount: 0, + activeDays: 0 +}); + +const createInitialState = (): IslandflowAiState => ({ + desktopAvailable: true, + transportStatus: "starting", + transportError: null, + profiles: [ + { + id: MANAGED_CHATGPT_PROFILE_ID, + label: "Managed ChatGPT login", + description: "User-scoped ChatGPT or Codex sign-in managed by the official app-server.", + mode: "managed-chatgpt", + enabled: true, + selected: true, + statusLabel: "Active" + }, + { + id: WORKSPACE_PROVIDER_PROFILE_ID, + label: "Workspace provider slot", + description: "Reserved for future shared API-key or enterprise access-token flows.", + mode: "workspace-provider", + enabled: false, + selected: false, + statusLabel: "Reserved" + } + ], + selectedProfileId: MANAGED_CHATGPT_PROFILE_ID, + account: { + loggedIn: false, + email: null, + planType: null, + authMode: null, + requiresOpenaiAuth: true, + login: { status: "idle", message: null } + }, + preferences: { + model: null, + reasoningEffort: DEFAULT_REASONING + }, + models: [], + rateLimitsByLimitId: {}, + usage: { + today: createEmptyUsageRollup(), + lifetime: createEmptyUsageRollup(), + recentTurns: [] + }, + tasks: [], + updatedAt: Date.now() +}); + +const buildUsageKey = (threadId: string, turnId: string): string => `${threadId}:${turnId}`; + +const normalizeBreakdown = (value: Partial | null | undefined): IslandflowAiTokenBreakdown => ({ + totalTokens: value?.totalTokens ?? 0, + inputTokens: value?.inputTokens ?? 0, + cachedInputTokens: value?.cachedInputTokens ?? 0, + outputTokens: value?.outputTokens ?? 0, + reasoningOutputTokens: value?.reasoningOutputTokens ?? 0 +}); + +const addBreakdowns = ( + left: IslandflowAiTokenBreakdown, + right: IslandflowAiTokenBreakdown +): IslandflowAiTokenBreakdown => ({ + totalTokens: left.totalTokens + right.totalTokens, + inputTokens: left.inputTokens + right.inputTokens, + cachedInputTokens: left.cachedInputTokens + right.cachedInputTokens, + outputTokens: left.outputTokens + right.outputTokens, + reasoningOutputTokens: left.reasoningOutputTokens + right.reasoningOutputTokens +}); + +const isoDayKey = (timestampMs: number): string => new Date(timestampMs).toISOString().slice(0, 10); + +const sanitizeJsonText = (value: string): string => { + const trimmed = value.trim(); + if (trimmed.startsWith("```")) { + return trimmed.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, ""); + } + return trimmed; +}; + +const estimateNormalizedCost = ( + model: string | null, + breakdown: IslandflowAiTokenBreakdown +): number | null => { + if (!model) { + return null; + } + const pricing = MODEL_PRICING[model]; + if (!pricing) { + return null; + } + const outputBillableTokens = breakdown.outputTokens + breakdown.reasoningOutputTokens; + const usd = + (breakdown.inputTokens / 1_000_000) * pricing.inputUsdPer1MTokens + + (breakdown.cachedInputTokens / 1_000_000) * pricing.cachedInputUsdPer1MTokens + + (outputBillableTokens / 1_000_000) * pricing.outputUsdPer1MTokens; + return Number(usd.toFixed(6)); +}; + +const compactTaskList = (tasks: IslandflowAiTaskSnapshot[]): IslandflowAiTaskSnapshot[] => + [...tasks].sort((left, right) => right.updatedAt - left.updatedAt).slice(0, 24); + +export const summarizeRateLimit = (snapshot: any): IslandflowAiRateLimitSnapshot => ({ + limitId: typeof snapshot?.limitId === "string" ? snapshot.limitId : null, + limitName: typeof snapshot?.limitName === "string" ? snapshot.limitName : null, + primary: snapshot?.primary + ? { + usedPercent: Number(snapshot.primary.usedPercent ?? 0), + windowDurationMins: + snapshot.primary.windowDurationMins === null || snapshot.primary.windowDurationMins === undefined + ? null + : Number(snapshot.primary.windowDurationMins), + resetsAt: + snapshot.primary.resetsAt === null || snapshot.primary.resetsAt === undefined + ? null + : Number(snapshot.primary.resetsAt) + } + : null, + secondary: snapshot?.secondary + ? { + usedPercent: Number(snapshot.secondary.usedPercent ?? 0), + windowDurationMins: + snapshot.secondary.windowDurationMins === null || snapshot.secondary.windowDurationMins === undefined + ? null + : Number(snapshot.secondary.windowDurationMins), + resetsAt: + snapshot.secondary.resetsAt === null || snapshot.secondary.resetsAt === undefined + ? null + : Number(snapshot.secondary.resetsAt) + } + : null, + planType: snapshot?.planType ?? null, + reachedType: snapshot?.rateLimitReachedType ?? null, + hasCredits: + snapshot?.credits?.hasCredits === undefined || snapshot?.credits?.hasCredits === null + ? null + : Boolean(snapshot.credits.hasCredits), + unlimitedCredits: + snapshot?.credits?.unlimited === undefined || snapshot?.credits?.unlimited === null + ? null + : Boolean(snapshot.credits.unlimited), + creditsBalance: typeof snapshot?.credits?.balance === "string" ? snapshot.credits.balance : null +}); + +export const createAppServerChildEnv = ( + profileMode: IslandflowAiProfileMode, + baseEnv: NodeJS.ProcessEnv = process.env +): NodeJS.ProcessEnv => { + const childEnv = { ...baseEnv }; + if (profileMode !== "api-key") { + delete childEnv.OPENAI_API_KEY; + delete childEnv.CODEX_API_KEY; + } + return childEnv; +}; + +const createTaskSnapshot = (request: IslandflowAiTaskRequest): Pick => { + switch (request.kind) { + case "smart-money-explain": + return { + kind: request.kind, + title: "Explain smart money event", + subtitle: `${request.context.event.underlying_id} · ${request.context.event.primary_direction}` + }; + case "smart-money-skeptic": + return { + kind: request.kind, + title: "Counter-thesis pass", + subtitle: `${request.context.event.underlying_id} · skepticism` + }; + case "smart-money-burst-summary": + return { + kind: request.kind, + title: "Burst summary", + subtitle: `${request.context.event.underlying_id} · related packets` + }; + case "watchlist-synthesis": + return { + kind: request.kind, + title: "Watchlist synthesis", + subtitle: `${request.context.event.underlying_id} · setups` + }; + case "replay-postmortem": + return { + kind: request.kind, + title: "Replay postmortem", + subtitle: `${request.context.ticker ?? "All symbols"} · replay session` + }; + case "screen-compile": + return { + kind: request.kind, + title: "Natural-language screen", + subtitle: request.context.prompt + }; + } + + throw new Error("Unsupported Copilot task kind."); +}; + +const SMART_MONEY_RUBRIC = [ + "Treat the deterministic classifier and event payload as the source of truth.", + "Act as an evidence interpreter, not the live classifier.", + "Use only the provided structured payloads, do not call tools or inspect the filesystem.", + "Lead with the clearest thesis, but include uncertainty, missing evidence, and alternate explanations.", + "Prefer practical market structure language: aggressor side, concentration, event timing, IV shock, NBBO quality, and packet construction.", + "Do not pretend to know price action or fundamentals beyond the supplied data.", + "When the data suggests retail frenzy, dealer hedging, volatility selling, or arbitrage, say so plainly.", + "Keep the answer terse, structured, and useful under pressure." +].join("\n"); + +const BASE_INSTRUCTIONS = [ + "You are Islandflow Analyst Copilot.", + "Work only from the structured Islandflow context provided in the user message.", + "Never call tools, never browse, and never inspect files.", + "If evidence is missing or ambiguous, say that directly." +].join("\n"); + +const buildUserPrompt = (request: IslandflowAiTaskRequest): string => { + switch (request.kind) { + case "smart-money-explain": + return [ + "Explain this selected smart-money event for a trader who wants the key evidence fast.", + "Output sections named Thesis, Evidence, Caveats, and What To Watch.", + JSON.stringify(request.context, null, 2) + ].join("\n\n"); + case "smart-money-skeptic": + return [ + "Run a skepticism pass on this selected smart-money event.", + "Output sections named Why It Might Be Wrong, Alternate Microstructure Explanations, Missing Evidence, and Confidence Check.", + JSON.stringify(request.context, null, 2) + ].join("\n\n"); + case "smart-money-burst-summary": + return [ + "Summarize the burst across the related packets for this selected smart-money event.", + "Output sections named Burst Read, Packet Relationships, Quality Flags, and Trading Relevance.", + JSON.stringify(request.context, null, 2) + ].join("\n\n"); + case "watchlist-synthesis": + return [ + "Turn this event into a practical watchlist and setup brief.", + "Output sections named Watchlist, Trigger Levels Or Conditions, Invalidations, and Session Notes.", + JSON.stringify(request.context, null, 2) + ].join("\n\n"); + case "replay-postmortem": + return [ + "Write a replay postmortem from this structured replay slice.", + "Output sections named Session Read, Best Evidence, What Was Noise, and Follow-up Questions.", + JSON.stringify(request.context, null, 2) + ].join("\n\n"); + case "screen-compile": + return [ + "Compile this natural-language screen into the existing Islandflow filter model where possible.", + "Return only valid JSON that matches the requested schema.", + JSON.stringify(request.context, null, 2) + ].join("\n\n"); + } + + throw new Error("Unsupported Copilot task kind."); +}; + +const buildScreenOutputSchema = () => ({ + type: "object", + additionalProperties: false, + required: ["compiledFilters", "rationale", "unhandledClauses", "sanitizedPrompt"], + properties: { + compiledFilters: { + anyOf: [ + { + type: "object", + additionalProperties: false, + properties: { + view: { type: "string", enum: ["signal", "raw"] }, + securityTypes: { + type: "array", + items: { type: "string", enum: ["stock", "etf"] } + }, + nbboSides: { + type: "array", + items: { type: "string", enum: ["AA", "A", "MID", "B", "BB", "MISSING", "STALE"] } + }, + optionTypes: { + type: "array", + items: { type: "string", enum: ["call", "put"] } + }, + minNotional: { type: "number", minimum: 0 } + } + }, + { type: "null" } + ] + }, + rationale: { type: "string" }, + unhandledClauses: { + type: "array", + items: { type: "string" } + }, + sanitizedPrompt: { type: "string" } + } +}); + +const createUsageStore = (): PersistedUsageStore => ({ + version: 1, + turns: {} +}); + +class CodexAppServerClient { + private child: ChildProcessWithoutNullStreams | null = null; + private readonly pending = new Map< + number, + { resolve: (value: any) => void; reject: (error: Error) => void; timeout: ReturnType } + >(); + private buffer = ""; + private nextId = 1; + + constructor( + private readonly sandboxCwd: string, + private readonly onNotification: (method: string, params: unknown) => Promise | void, + private readonly onExit: (reason: string) => Promise | void + ) {} + + async start(profileMode: IslandflowAiProfileMode): Promise { + if (this.child) { + return; + } + + await mkdir(this.sandboxCwd, { recursive: true }); + + this.child = spawn("codex", ["app-server"], { + stdio: ["pipe", "pipe", "pipe"], + env: createAppServerChildEnv(profileMode) + }); + + this.child.stdout.setEncoding("utf8"); + this.child.stderr.setEncoding("utf8"); + + this.child.stdout.on("data", (chunk: string) => { + this.buffer += chunk; + void this.flushBuffer(); + }); + + this.child.stderr.on("data", (chunk: string) => { + console.warn(`[desktop-ai] ${chunk.trim()}`); + }); + + this.child.once("exit", (code, signal) => { + this.child = null; + this.buffer = ""; + for (const [id, pending] of this.pending.entries()) { + clearTimeout(pending.timeout); + pending.reject(new Error(`Codex app-server exited before replying to request ${id}.`)); + } + this.pending.clear(); + void this.onExit(`app-server exited${code !== null ? ` (${code})` : ""}${signal ? ` via ${signal}` : ""}`); + }); + + await this.request("initialize", { + clientInfo: { + name: "islandflow-desktop", + title: "Islandflow Desktop", + version: "0.1.0" + }, + capabilities: { + experimentalApi: true, + requestAttestation: false, + optOutNotificationMethods: [ + "app/list/updated", + "remoteControl/status/changed", + "skills/changed", + "plugin/installed" + ] + } + }); + + this.notify("initialized"); + } + + async stop(): Promise { + if (!this.child) { + return; + } + this.child.kill("SIGTERM"); + this.child = null; + } + + async request(method: string, params: unknown): Promise { + if (!this.child) { + throw new Error("Codex app-server is not running."); + } + + const id = this.nextId++; + const payload = JSON.stringify({ id, method, params }) + "\n"; + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.pending.delete(id); + reject(new Error(`Timed out waiting for ${method}.`)); + }, 30_000); + + this.pending.set(id, { resolve, reject, timeout }); + this.child?.stdin.write(payload); + }); + } + + private notify(method: string, params?: unknown): void { + if (!this.child) { + return; + } + this.child.stdin.write(JSON.stringify(params === undefined ? { method } : { method, params }) + "\n"); + } + + private async flushBuffer(): Promise { + while (true) { + const newlineIndex = this.buffer.indexOf("\n"); + if (newlineIndex === -1) { + return; + } + + const line = this.buffer.slice(0, newlineIndex).trim(); + this.buffer = this.buffer.slice(newlineIndex + 1); + if (!line) { + continue; + } + + const message = JSON.parse(line) as JsonRpcMessage; + if ("id" in message && "result" in message) { + const pending = this.pending.get(message.id); + if (pending) { + clearTimeout(pending.timeout); + this.pending.delete(message.id); + pending.resolve(message.result); + } + continue; + } + + if ("id" in message && "error" in message) { + const pending = this.pending.get(message.id); + if (pending) { + clearTimeout(pending.timeout); + this.pending.delete(message.id); + pending.reject(new Error(message.error.message ?? `Request ${message.id} failed.`)); + } + continue; + } + + if (typeof (message as Partial).id === "number" && "method" in message) { + this.respondUnsupported(message as JsonRpcServerRequest); + continue; + } + + if ("method" in message) { + await this.onNotification(message.method, message.params); + } + } + } + + private respondUnsupported(message: JsonRpcServerRequest): void { + if (!this.child) { + return; + } + this.child.stdin.write( + JSON.stringify({ + id: message.id, + error: { + message: `Islandflow desktop does not support server request ${message.method}.` + } + }) + "\n" + ); + } +} + +export class IslandflowDesktopAiService { + private readonly preferencesPath: string; + private readonly usagePath: string; + private readonly sandboxCwd: string; + private readonly client: CodexAppServerClient; + private readonly activeTasksByThreadId = new Map(); + private usageStore: PersistedUsageStore = createUsageStore(); + private state: IslandflowAiState = createInitialState(); + private serviceTier: string | null = null; + private started = false; + + constructor( + userDataPath: string, + private readonly openExternalUrl: OpenExternalFn, + private readonly publishState: (state: IslandflowAiState) => void + ) { + this.preferencesPath = path.join(userDataPath, PREFERENCES_FILE); + this.usagePath = path.join(userDataPath, USAGE_FILE); + this.sandboxCwd = path.join(userDataPath, APP_SERVER_SANDBOX_CWD); + this.client = new CodexAppServerClient( + this.sandboxCwd, + async (method, params) => { + await this.handleNotification(method, params); + }, + async (reason) => { + this.state.transportStatus = "restarting"; + this.state.transportError = reason; + this.failActiveTasks(reason); + this.emitState(); + } + ); + } + + async start(): Promise { + if (this.started) { + return; + } + this.started = true; + await mkdir(path.dirname(this.preferencesPath), { recursive: true }); + await mkdir(this.sandboxCwd, { recursive: true }); + await this.loadPreferences(); + await this.loadUsageStore(); + await this.ensureClientReady(); + } + + getState(): IslandflowAiState { + return this.state; + } + + async loginWithBrowser(): Promise { + await this.start(); + await this.ensureClientReady(); + + const result = await this.client.request("account/login/start", { + type: "chatgpt", + codexStreamlinedLogin: true + }); + + this.state.account.login = { + status: "browser_pending", + message: "Waiting for browser sign-in to complete.", + loginId: String(result.loginId), + authUrl: String(result.authUrl) + }; + this.emitState(); + await this.openExternalUrl(String(result.authUrl)); + } + + async loginWithDeviceCode(): Promise { + await this.start(); + await this.ensureClientReady(); + + const result = await this.client.request("account/login/start", { + type: "chatgptDeviceCode" + }); + + this.state.account.login = { + status: "device_code_pending", + message: "Enter the device code in your browser to finish sign-in.", + loginId: String(result.loginId), + verificationUrl: String(result.verificationUrl), + userCode: String(result.userCode) + }; + this.emitState(); + await this.openExternalUrl(String(result.verificationUrl)); + } + + async cancelLogin(): Promise { + const login = this.state.account.login; + if (login.status !== "browser_pending" && login.status !== "device_code_pending") { + return; + } + await this.client.request("account/login/cancel", { loginId: login.loginId }); + this.state.account.login = { status: "idle", message: "Login cancelled." }; + this.emitState(); + } + + async logout(): Promise { + await this.client.request("account/logout", {}); + this.state.account.loggedIn = false; + this.state.account.email = null; + this.state.account.planType = null; + this.state.account.login = { status: "idle", message: "Logged out." }; + this.emitState(); + } + + async updatePreferences( + next: Partial<{ model: string | null; reasoningEffort: IslandflowAiReasoningEffort | null }> + ): Promise { + this.state.preferences = { + model: next.model === undefined ? this.state.preferences.model : next.model, + reasoningEffort: + next.reasoningEffort === undefined + ? this.state.preferences.reasoningEffort + : next.reasoningEffort + }; + await this.savePreferences(); + this.emitState(); + } + + async runTask(rawRequest: unknown): Promise<{ taskId: string }> { + await this.start(); + + if (!this.state.account.loggedIn) { + throw new Error("Log into a ChatGPT or Codex account first."); + } + + const request = IslandflowAiTaskRequestSchema.parse(rawRequest); + const meta = createTaskSnapshot(request); + const taskId = `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; + const task: IslandflowAiTaskSnapshot = { + taskId, + kind: meta.kind, + title: meta.title, + subtitle: meta.subtitle, + status: "queued", + createdAt: Date.now(), + updatedAt: Date.now(), + threadId: null, + turnId: null, + model: this.state.preferences.model, + reasoningEffort: this.state.preferences.reasoningEffort, + text: "", + error: null, + compiledScreen: null + }; + + this.state.tasks = compactTaskList([task, ...this.state.tasks]); + this.emitState(); + + try { + await this.ensureClientReady(); + + const thread = await this.client.request("thread/start", { + model: this.state.preferences.model ?? undefined, + cwd: this.sandboxCwd, + approvalPolicy: "never", + sandbox: "read-only", + serviceName: APP_SERVER_SERVICE_NAME, + baseInstructions: BASE_INSTRUCTIONS, + developerInstructions: SMART_MONEY_RUBRIC, + ephemeral: true, + serviceTier: this.serviceTier ?? undefined + }); + + this.activeTasksByThreadId.set(thread.thread.id, { + taskId, + taskKind: task.kind, + taskTitle: task.title, + profileId: this.state.selectedProfileId + }); + this.patchTask(taskId, { + status: "running", + threadId: thread.thread.id, + model: thread.model, + reasoningEffort: thread.reasoningEffort ?? this.state.preferences.reasoningEffort + }); + + const turn = await this.client.request("turn/start", { + threadId: thread.thread.id, + input: [ + { + type: "text", + text: buildUserPrompt(request), + text_elements: [] + } + ], + model: this.state.preferences.model ?? undefined, + effort: this.state.preferences.reasoningEffort ?? undefined, + serviceTier: this.serviceTier ?? undefined, + outputSchema: request.kind === "screen-compile" ? buildScreenOutputSchema() : undefined + }); + + this.patchTask(taskId, { + turnId: turn.turn.id + }); + + return { taskId }; + } catch (error) { + this.patchTask(taskId, { + status: "failed", + error: error instanceof Error ? error.message : String(error) + }); + throw error; + } + } + + private async ensureClientReady(): Promise { + const selectedProfile = this.resolveSelectedProfileMode(); + this.state.transportStatus = this.state.transportStatus === "restarting" ? "restarting" : "starting"; + this.emitState(); + + try { + await this.client.start(selectedProfile); + await this.refreshServerState(); + this.state.transportStatus = "ready"; + this.state.transportError = null; + this.emitState(); + } catch (error) { + this.state.transportStatus = "error"; + this.state.transportError = error instanceof Error ? error.message : String(error); + this.emitState(); + throw error; + } + } + + private async refreshServerState(): Promise { + const [config, models, account, auth, rateLimits] = await Promise.all([ + this.client.request("config/read", {}), + this.client.request("model/list", {}), + this.client.request("account/read", { refreshToken: false }), + this.client.request("getAuthStatus", {}), + this.client.request("account/rateLimits/read", {}) + ]); + + this.serviceTier = typeof config?.config?.service_tier === "string" ? config.config.service_tier : null; + const configModel = typeof config?.config?.model === "string" ? config.config.model : null; + const configReasoning = + config?.config?.model_reasoning_effort === null || config?.config?.model_reasoning_effort === undefined + ? null + : IslandflowAiReasoningEffortSchema.parse(config.config.model_reasoning_effort); + + if (!this.state.preferences.model) { + this.state.preferences.model = configModel; + } + if (!this.state.preferences.reasoningEffort) { + this.state.preferences.reasoningEffort = configReasoning ?? DEFAULT_REASONING; + } + + this.state.models = (Array.isArray(models?.data) ? models.data : []) + .filter((model: CodexModelRecord) => !model.hidden) + .map((model: CodexModelRecord): IslandflowAiModelSummary => ({ + id: model.id, + model: model.model, + displayName: model.displayName, + description: model.description, + isDefault: Boolean(model.isDefault), + supportedReasoningEfforts: model.supportedReasoningEfforts.map((entry) => entry.reasoningEffort), + defaultReasoningEffort: model.defaultReasoningEffort, + pricing: MODEL_PRICING[model.model] ?? null + })); + + this.state.account.loggedIn = Boolean(account?.account); + this.state.account.email = + account?.account?.type === "chatgpt" && typeof account.account.email === "string" + ? account.account.email + : null; + this.state.account.planType = + account?.account?.type === "chatgpt" ? (account.account.planType as IslandflowAiPlanType) : null; + this.state.account.authMode = auth?.authMethod ?? null; + this.state.account.requiresOpenaiAuth = Boolean(account?.requiresOpenaiAuth ?? auth?.requiresOpenaiAuth ?? true); + if (this.state.account.login.status === "idle") { + this.state.account.login = { + status: "idle", + message: this.state.account.loggedIn ? "Connected." : null + }; + } + + this.state.rateLimitsByLimitId = this.normalizeRateLimitBuckets(rateLimits); + this.rebuildUsageDashboard(); + } + + private normalizeRateLimitBuckets(payload: any): Record { + const bucketEntries = Object.entries(payload?.rateLimitsByLimitId ?? {}); + if (bucketEntries.length === 0 && payload?.rateLimits) { + const single = summarizeRateLimit(payload.rateLimits); + return { + [single.limitId ?? "default"]: single + }; + } + + return Object.fromEntries( + bucketEntries.map(([key, value]) => [key, summarizeRateLimit(value)]) + ); + } + + private async handleNotification(method: string, params: unknown): Promise { + switch (method) { + case "account/updated": { + const payload = params as { authMode: string | null; planType: IslandflowAiPlanType | null }; + this.state.account.authMode = payload.authMode as any; + this.state.account.planType = payload.planType; + this.emitState(); + return; + } + case "account/login/completed": { + const payload = params as { success: boolean; error: string | null }; + if (payload.success) { + this.state.account.login = { status: "idle", message: "Connected." }; + await this.refreshServerState(); + } else { + this.state.account.login = { + status: "error", + message: payload.error ?? "Login failed.", + loginId: null + }; + } + this.emitState(); + return; + } + case "account/rateLimits/updated": { + const payload = summarizeRateLimit((params as { rateLimits: unknown }).rateLimits); + this.state.rateLimitsByLimitId = { + ...this.state.rateLimitsByLimitId, + [payload.limitId ?? "default"]: payload + }; + this.emitState(); + return; + } + case "item/agentMessage/delta": { + const payload = params as { threadId: string; delta: string }; + const activeTask = this.activeTasksByThreadId.get(payload.threadId); + if (!activeTask) { + return; + } + const current = this.state.tasks.find((task) => task.taskId === activeTask.taskId); + if (!current) { + return; + } + this.patchTask(activeTask.taskId, { + text: current.text + payload.delta + }); + return; + } + case "item/completed": { + const payload = params as { + threadId: string; + item: { type: string; text?: string }; + }; + if (payload.item.type !== "agentMessage") { + return; + } + const activeTask = this.activeTasksByThreadId.get(payload.threadId); + if (!activeTask) { + return; + } + if (typeof payload.item.text === "string") { + this.patchTask(activeTask.taskId, { + text: payload.item.text + }); + } + return; + } + case "thread/tokenUsage/updated": { + const payload = params as { + threadId: string; + turnId: string; + tokenUsage: { + total: IslandflowAiTokenBreakdown; + last: IslandflowAiTokenBreakdown; + }; + }; + this.recordUsage(payload.threadId, payload.turnId, payload.tokenUsage.last); + return; + } + case "turn/completed": { + const payload = params as { + threadId: string; + turn: { + id: string; + status: string; + error: { message: string } | null; + }; + }; + const activeTask = this.activeTasksByThreadId.get(payload.threadId); + if (!activeTask) { + return; + } + const current = this.state.tasks.find((task) => task.taskId === activeTask.taskId); + if (!current) { + return; + } + + if (payload.turn.status === "failed") { + this.patchTask(activeTask.taskId, { + status: "failed", + error: payload.turn.error?.message ?? "The Copilot turn failed." + }); + } else { + let compiledScreen: IslandflowAiCompiledScreen | null = null; + let nextText = current.text; + if (current.kind === "screen-compile") { + compiledScreen = this.tryParseCompiledScreen(current.text); + if (compiledScreen) { + nextText = compiledScreen.rationale; + } + } + + this.patchTask(activeTask.taskId, { + status: "completed", + compiledScreen, + text: nextText, + error: null + }); + } + + this.activeTasksByThreadId.delete(payload.threadId); + return; + } + default: + return; + } + } + + private tryParseCompiledScreen(text: string): IslandflowAiCompiledScreen | null { + try { + return IslandflowAiCompiledScreenSchema.parse(JSON.parse(sanitizeJsonText(text))); + } catch { + return null; + } + } + + private recordUsage(threadId: string, turnId: string, rawBreakdown: IslandflowAiTokenBreakdown): void { + const activeTask = this.activeTasksByThreadId.get(threadId); + const currentTask = activeTask + ? this.state.tasks.find((task) => task.taskId === activeTask.taskId) + : null; + const breakdown = normalizeBreakdown(rawBreakdown); + const record: IslandflowAiUsageTurnRecord = { + threadId, + turnId, + taskId: currentTask?.taskId ?? null, + taskKind: currentTask?.kind ?? null, + taskTitle: currentTask?.title ?? null, + dayKey: isoDayKey(Date.now()), + profileId: activeTask?.profileId ?? this.state.selectedProfileId, + accountEmail: this.state.account.email, + planType: this.state.account.planType, + model: currentTask?.model ?? this.state.preferences.model, + breakdown, + normalizedCostUsd: estimateNormalizedCost(currentTask?.model ?? this.state.preferences.model, breakdown), + updatedAt: Date.now() + }; + + this.usageStore.turns[buildUsageKey(threadId, turnId)] = record; + void this.saveUsageStore(); + this.rebuildUsageDashboard(); + this.emitState(); + } + + private rebuildUsageDashboard(): void { + const records = Object.values(this.usageStore.turns).filter((record) => { + if (record.profileId !== this.state.selectedProfileId) { + return false; + } + if (this.state.account.email) { + return record.accountEmail === this.state.account.email; + } + return true; + }); + + const todayKey = isoDayKey(Date.now()); + this.state.usage = { + today: this.rollupUsage(records.filter((record) => record.dayKey === todayKey)), + lifetime: this.rollupUsage(records), + recentTurns: [...records].sort((left, right) => right.updatedAt - left.updatedAt).slice(0, 12) + }; + } + + private rollupUsage(records: IslandflowAiUsageTurnRecord[]): IslandflowAiUsageRollup { + const breakdown = records.reduce( + (accumulator, record) => addBreakdowns(accumulator, record.breakdown), + { ...EMPTY_BREAKDOWN } + ); + const normalizedCostUsd = records.reduce((accumulator, record) => accumulator + (record.normalizedCostUsd ?? 0), 0); + return { + breakdown, + normalizedCostUsd: Number(normalizedCostUsd.toFixed(6)), + turnCount: records.length, + activeDays: new Set(records.map((record) => record.dayKey)).size + }; + } + + private failActiveTasks(reason: string): void { + for (const activeTask of this.activeTasksByThreadId.values()) { + this.patchTask(activeTask.taskId, { + status: "failed", + error: reason + }); + } + this.activeTasksByThreadId.clear(); + } + + private patchTask(taskId: string, updates: Partial): void { + this.state.tasks = compactTaskList( + this.state.tasks.map((task) => + task.taskId === taskId + ? { + ...task, + ...updates, + updatedAt: Date.now() + } + : task + ) + ); + this.emitState(); + } + + private emitState(): void { + this.state.updatedAt = Date.now(); + this.publishState({ + ...this.state, + profiles: this.state.profiles.map((profile) => ({ + ...profile, + selected: profile.id === this.state.selectedProfileId + })), + tasks: compactTaskList(this.state.tasks) + }); + } + + private resolveSelectedProfileMode(): IslandflowAiProfileMode { + const selected = this.state.profiles.find((profile) => profile.id === this.state.selectedProfileId); + return IslandflowAiProfileModeSchema.parse(selected?.mode ?? "managed-chatgpt"); + } + + private async loadPreferences(): Promise { + try { + const raw = await readFile(this.preferencesPath, "utf8"); + const parsed = JSON.parse(raw) as PersistedPreferences; + this.state.preferences = { + model: typeof parsed.model === "string" ? parsed.model : null, + reasoningEffort: + parsed.reasoningEffort === null || parsed.reasoningEffort === undefined + ? DEFAULT_REASONING + : IslandflowAiReasoningEffortSchema.parse(parsed.reasoningEffort) + }; + } catch { + // Use defaults on first run or after malformed local state. + } + } + + private async savePreferences(): Promise { + const payload: PersistedPreferences = { + model: this.state.preferences.model, + reasoningEffort: this.state.preferences.reasoningEffort + }; + await writeFile(this.preferencesPath, JSON.stringify(payload, null, 2), "utf8"); + } + + private async loadUsageStore(): Promise { + try { + const raw = await readFile(this.usagePath, "utf8"); + const parsed = JSON.parse(raw) as PersistedUsageStore; + if (parsed.version === 1 && parsed.turns) { + this.usageStore = parsed; + } + } catch { + this.usageStore = createUsageStore(); + } + this.rebuildUsageDashboard(); + } + + private async saveUsageStore(): Promise { + await writeFile(this.usagePath, JSON.stringify(this.usageStore, null, 2), "utf8"); + } +} diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index e5006df..41d24a5 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -1,5 +1,6 @@ -import { app, BrowserWindow, shell } from "electron"; -import type { Event as ElectronEvent } from "electron"; +import { app, BrowserWindow, ipcMain, shell } from "electron"; +import type { Event as ElectronEvent, IpcMainInvokeEvent } from "electron"; +import { fileURLToPath } from "node:url"; import { DESKTOP_PRODUCTION_URL, @@ -7,11 +8,25 @@ import { isTrustedAppUrl, resolveDesktopStartUrl } from "./security.js"; +import { IslandflowDesktopAiService } from "./desktop-ai.js"; +import { + DESKTOP_AI_CANCEL_LOGIN, + DESKTOP_AI_GET_STATE, + DESKTOP_AI_LOGIN_BROWSER, + DESKTOP_AI_LOGIN_DEVICE, + DESKTOP_AI_LOGOUT, + DESKTOP_AI_RUN_TASK, + DESKTOP_AI_STATE_CHANNEL, + DESKTOP_AI_UPDATE_PREFERENCES +} from "./desktop-ai-ipc.js"; const WINDOW_BACKGROUND_COLOR = "#06080b"; const WINDOW_TITLE = "Islandflow"; let mainWindow: BrowserWindow | null = null; +let desktopAiService: IslandflowDesktopAiService | null = null; + +const PRELOAD_PATH = fileURLToPath(new URL("./preload.js", import.meta.url)); const canOpenExternalUrl = (sourceUrl: string, targetUrl: string): boolean => { return isTrustedAppUrl(sourceUrl) && isSafeExternalUrl(targetUrl); @@ -61,6 +76,7 @@ const createMainWindow = (): BrowserWindow => { title: WINDOW_TITLE, backgroundColor: WINDOW_BACKGROUND_COLOR, webPreferences: { + preload: PRELOAD_PATH, nodeIntegration: false, contextIsolation: true, sandbox: true, @@ -92,6 +108,68 @@ const createMainWindow = (): BrowserWindow => { return window; }; +const broadcastDesktopAiState = (): void => { + if (!desktopAiService) { + return; + } + + const state = desktopAiService.getState(); + for (const window of BrowserWindow.getAllWindows()) { + window.webContents.send(DESKTOP_AI_STATE_CHANNEL, state); + } +}; + +const getTrustedSenderUrl = (event: IpcMainInvokeEvent): string => { + const senderUrl = event.senderFrame?.url || event.sender.getURL(); + if (!isTrustedAppUrl(senderUrl)) { + throw new Error(`Rejected desktop AI IPC from untrusted origin: ${senderUrl || "unknown"}`); + } + + return senderUrl; +}; + +const registerDesktopAiIpc = (service: IslandflowDesktopAiService): void => { + const guard = (event: IpcMainInvokeEvent): void => { + getTrustedSenderUrl(event); + }; + + ipcMain.handle(DESKTOP_AI_GET_STATE, async (event) => { + guard(event); + await service.start(); + return service.getState(); + }); + + ipcMain.handle(DESKTOP_AI_LOGIN_BROWSER, async (event) => { + guard(event); + await service.loginWithBrowser(); + }); + + ipcMain.handle(DESKTOP_AI_LOGIN_DEVICE, async (event) => { + guard(event); + await service.loginWithDeviceCode(); + }); + + ipcMain.handle(DESKTOP_AI_CANCEL_LOGIN, async (event) => { + guard(event); + await service.cancelLogin(); + }); + + ipcMain.handle(DESKTOP_AI_LOGOUT, async (event) => { + guard(event); + await service.logout(); + }); + + ipcMain.handle(DESKTOP_AI_UPDATE_PREFERENCES, async (event, next) => { + guard(event); + await service.updatePreferences(next); + }); + + ipcMain.handle(DESKTOP_AI_RUN_TASK, async (event, request) => { + guard(event); + return service.runTask(request); + }); +}; + const ensureMainWindow = (): void => { if (mainWindow) { return; @@ -101,6 +179,20 @@ const ensureMainWindow = (): void => { }; app.whenReady().then(() => { + desktopAiService = new IslandflowDesktopAiService( + app.getPath("userData"), + async (url) => { + await shell.openExternal(url); + }, + () => { + broadcastDesktopAiState(); + } + ); + registerDesktopAiIpc(desktopAiService); + void desktopAiService.start().catch((error) => { + console.error("[desktop-ai] Failed to start Codex bridge:", error); + broadcastDesktopAiState(); + }); ensureMainWindow(); app.on("activate", () => { diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts new file mode 100644 index 0000000..ed6f8df --- /dev/null +++ b/apps/desktop/src/preload.ts @@ -0,0 +1,43 @@ +import { contextBridge, ipcRenderer } from "electron"; +import type { + IslandflowAiReasoningEffort, + IslandflowAiState, + IslandflowAiTaskRequest +} from "@islandflow/types"; +import { + DESKTOP_AI_CANCEL_LOGIN, + DESKTOP_AI_GET_STATE, + DESKTOP_AI_LOGIN_BROWSER, + DESKTOP_AI_LOGIN_DEVICE, + DESKTOP_AI_LOGOUT, + DESKTOP_AI_RUN_TASK, + DESKTOP_AI_STATE_CHANNEL, + DESKTOP_AI_UPDATE_PREFERENCES +} from "./desktop-ai-ipc.js"; + +const bridge = { + ai: { + getState: (): Promise => ipcRenderer.invoke(DESKTOP_AI_GET_STATE), + loginWithBrowser: (): Promise => ipcRenderer.invoke(DESKTOP_AI_LOGIN_BROWSER), + loginWithDeviceCode: (): Promise => ipcRenderer.invoke(DESKTOP_AI_LOGIN_DEVICE), + cancelLogin: (): Promise => ipcRenderer.invoke(DESKTOP_AI_CANCEL_LOGIN), + logout: (): Promise => ipcRenderer.invoke(DESKTOP_AI_LOGOUT), + updatePreferences: ( + next: Partial<{ model: string | null; reasoningEffort: IslandflowAiReasoningEffort | null }> + ): Promise => ipcRenderer.invoke(DESKTOP_AI_UPDATE_PREFERENCES, next), + runTask: (request: IslandflowAiTaskRequest): Promise<{ taskId: string }> => + ipcRenderer.invoke(DESKTOP_AI_RUN_TASK, request), + subscribe: (listener: (state: IslandflowAiState) => void): (() => void) => { + const handler = (_event: Electron.IpcRendererEvent, state: IslandflowAiState) => { + listener(state); + }; + + ipcRenderer.on(DESKTOP_AI_STATE_CHANNEL, handler); + return () => { + ipcRenderer.off(DESKTOP_AI_STATE_CHANNEL, handler); + }; + } + } +}; + +contextBridge.exposeInMainWorld("islandflowDesktop", bridge); diff --git a/apps/desktop/tsconfig.json b/apps/desktop/tsconfig.json index 5895037..de3fb15 100644 --- a/apps/desktop/tsconfig.json +++ b/apps/desktop/tsconfig.json @@ -2,8 +2,8 @@ "extends": "../../tsconfig.base.json", "compilerOptions": { "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", + "module": "ESNext", + "moduleResolution": "Bundler", "lib": ["ES2022"], "types": ["node"], "rootDir": "src", diff --git a/apps/web/app/charts/page.tsx b/apps/web/app/charts/page.tsx index 9d82bba..a2eb858 100644 --- a/apps/web/app/charts/page.tsx +++ b/apps/web/app/charts/page.tsx @@ -1,7 +1,7 @@ -import { redirect } from "next/navigation"; +import { ChartsRoute } from "../terminal"; export const dynamic = "force-dynamic"; export default function Page() { - redirect("/"); + return ; } diff --git a/apps/web/app/desktop-ai-panels.tsx b/apps/web/app/desktop-ai-panels.tsx new file mode 100644 index 0000000..65524dc --- /dev/null +++ b/apps/web/app/desktop-ai-panels.tsx @@ -0,0 +1,924 @@ +"use client"; + +import Link from "next/link"; +import { useMemo, useState, type ReactNode } from "react"; +import type { + AlertEvent, + ClassifierHitEvent, + FlowPacket, + IslandflowAiCompiledScreen, + IslandflowAiPlanType, + IslandflowAiRateLimitSnapshot, + IslandflowAiReasoningEffort, + IslandflowAiTaskKind, + OptionFlowFilters, + OptionPrint, + SmartMoneyEvent +} from "@islandflow/types"; +import { useDesktopAi } from "./desktop-ai"; + +const numberFormatter = new Intl.NumberFormat("en-US"); +const usdFormatter = new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + minimumFractionDigits: 2, + maximumFractionDigits: 4 +}); + +const humanizeValue = (value: string | null | undefined): string => { + if (!value) { + return "Unknown"; + } + return value + .replace(/_/g, " ") + .replace(/\b\w/g, (char) => char.toUpperCase()); +}; + +const formatTokens = (value: number): string => numberFormatter.format(value); + +const formatUsd = (value: number | null): string => (value === null ? "Unavailable" : usdFormatter.format(value)); + +const formatTimestamp = (value: number | null): string => { + if (!value) { + return "Not reported"; + } + return new Intl.DateTimeFormat("en-US", { + dateStyle: "medium", + timeStyle: "short" + }).format(value); +}; + +const formatPercent = (value: number): string => `${Math.round(value)}%`; + +const getTaskStatusLabel = (value: string): string => humanizeValue(value); + +const findTask = (tasks: T[], taskId: string | null): T | null => { + if (!taskId) { + return null; + } + return tasks.find((task) => task.taskId === taskId) ?? null; +}; + +const getCompiledScreenSummary = (compiled: IslandflowAiCompiledScreen): string[] => { + const filters = compiled.compiledFilters; + if (!filters) { + return []; + } + + const parts: string[] = []; + if (filters.view) { + parts.push(`View: ${filters.view}`); + } + if (filters.securityTypes?.length) { + parts.push(`Security: ${filters.securityTypes.join(", ")}`); + } + if (filters.optionTypes?.length) { + parts.push(`Options: ${filters.optionTypes.join(", ")}`); + } + if (filters.nbboSides?.length) { + parts.push(`NBBO: ${filters.nbboSides.join(", ")}`); + } + if (typeof filters.minNotional === "number") { + parts.push(`Min notional: $${numberFormatter.format(filters.minNotional)}`); + } + + return parts; +}; + +const CopilotPane = ({ + title, + eyebrow, + actions, + wide = false, + children +}: { + title: string; + eyebrow?: string; + actions?: ReactNode; + wide?: boolean; + children: ReactNode; +}) => { + return ( +
    +
    +
    +
    + {eyebrow ?
    {eyebrow}
    : null} +

    {title}

    +
    +
    + {actions ?
    {actions}
    : null} +
    +
    {children}
    +
    + ); +}; + +const UsageBreakdown = ({ + title, + breakdown, + normalizedCostUsd, + turnCount, + activeDays +}: { + title: string; + breakdown: { + totalTokens: number; + inputTokens: number; + cachedInputTokens: number; + outputTokens: number; + reasoningOutputTokens: number; + }; + normalizedCostUsd: number | null; + turnCount: number; + activeDays: number; +}) => { + return ( +
    +
    +

    {title}

    + {formatUsd(normalizedCostUsd)} +
    +
    +
    + Total tokens + {formatTokens(breakdown.totalTokens)} +
    +
    + Input + {formatTokens(breakdown.inputTokens)} +
    +
    + Cached input + {formatTokens(breakdown.cachedInputTokens)} +
    +
    + Output + {formatTokens(breakdown.outputTokens)} +
    +
    + Reasoning + {formatTokens(breakdown.reasoningOutputTokens)} +
    +
    + Turns + {formatTokens(turnCount)} +
    +
    + Active days + {formatTokens(activeDays)} +
    +
    +
    + ); +}; + +const RateLimitBoard = ({ limit }: { limit: IslandflowAiRateLimitSnapshot }) => { + return ( +
    +
    +
    + {limit.limitName ?? "Default rate window"} +

    + {limit.planType ? `Plan ${humanizeValue(limit.planType)}` : "Plan not reported"} +

    +
    + {limit.reachedType ? {humanizeValue(limit.reachedType)} : null} +
    +
    + {limit.primary ? ( +
    + Primary + {formatPercent(limit.primary.usedPercent)} +

    Resets {formatTimestamp(limit.primary.resetsAt)}

    +
    + ) : null} + {limit.secondary ? ( +
    + Secondary + {formatPercent(limit.secondary.usedPercent)} +

    Resets {formatTimestamp(limit.secondary.resetsAt)}

    +
    + ) : null} +
    + {limit.creditsBalance || limit.unlimitedCredits !== null ? ( +

    + Credits:{" "} + {limit.unlimitedCredits + ? "unlimited" + : limit.creditsBalance + ? limit.creditsBalance + : limit.hasCredits === false + ? "none" + : "not reported"} +

    + ) : null} +
    + ); +}; + +const TaskOutput = ({ + taskId, + emptyMessage +}: { + taskId: string | null; + emptyMessage: string; +}) => { + const { state } = useDesktopAi(); + const task = findTask(state.tasks, taskId); + + if (!task) { + return

    {emptyMessage}

    ; + } + + return ( +
    +
    +
    + {task.title} +

    + {task.subtitle} · {getTaskStatusLabel(task.status)} +

    +
    + {getTaskStatusLabel(task.status)} +
    + {task.error ?

    {task.error}

    : null} + {task.text ? 
    {task.text}
    : null} + {task.compiledScreen ? : null} +
    + ); +}; + +const CompiledScreenResult = ({ compiled }: { compiled: IslandflowAiCompiledScreen }) => { + const summary = getCompiledScreenSummary(compiled); + + return ( +
    + {summary.length > 0 ? ( +
    + {summary.map((item) => ( + + {item} + + ))} +
    + ) : ( +

    No filter fields were compiled from this prompt.

    + )} + {compiled.unhandledClauses.length > 0 ? ( +
    +
    Unhandled clauses
    + {compiled.unhandledClauses.map((item) => ( +
    + {item} +
    + ))} +
    + ) : null} +
    + ); +}; + +const AccountSummary = ({ + loggedIn, + email, + planType +}: { + loggedIn: boolean; + email: string | null; + planType: IslandflowAiPlanType | null; +}) => { + return ( +
    +
    +

    Desktop-only official Codex bridge

    +

    Analyst Copilot

    +

    + Managed ChatGPT login stays user-scoped, deterministic smart-money classification stays in charge, and every + AI turn is tracked with exact token telemetry from the app-server. +

    +
    +
    +
    + Account + {loggedIn ? email ?? "Connected" : "Disconnected"} +
    +
    + Plan + {loggedIn ? humanizeValue(planType) : "Not connected"} +
    +
    +
    + ); +}; + +const LoginStatePanel = () => { + const { state, loginWithBrowser, loginWithDeviceCode, cancelLogin, logout } = useDesktopAi(); + const [busyAction, setBusyAction] = useState(null); + const [actionError, setActionError] = useState(null); + const loginState = state.account.login; + const actionsDisabled = busyAction !== null || !state.desktopAvailable; + + const runAction = async (label: string, action: () => Promise) => { + setBusyAction(label); + setActionError(null); + try { + await action(); + } catch (error) { + setActionError(error instanceof Error ? error.message : String(error)); + } finally { + setBusyAction(null); + } + }; + + return ( + + {state.account.loggedIn ? ( + + ) : ( + <> + + + + )} + {(loginState.status === "browser_pending" || loginState.status === "device_code_pending") && !state.account.loggedIn ? ( + + ) : null} + + } + > + +
    +
    +
    Profile slots
    + {state.profiles.map((profile) => ( +
    +
    + {profile.label} +

    {profile.description}

    +
    + + {profile.selected ? "Selected" : profile.statusLabel} + +
    + ))} +
    +
    +
    Session status
    +
    + Transport + {humanizeValue(state.transportStatus)} +
    +
    + Auth mode + {humanizeValue(state.account.authMode)} +
    +
    + OpenAI auth required + {state.account.requiresOpenaiAuth ? "Yes" : "No"} +
    + {state.transportError ?

    {state.transportError}

    : null} + {loginState.message ?

    {loginState.message}

    : null} + {loginState.status === "browser_pending" ? ( +
    + Browser login in progress +

    Finish the ChatGPT sign-in flow in your browser. Islandflow will update automatically.

    +
    + ) : null} + {loginState.status === "device_code_pending" ? ( +
    + Device code + 
    {loginState.userCode}
    +

    Visit {loginState.verificationUrl} in any browser and enter the code above.

    +
    + ) : null} + {actionError ?

    {actionError}

    : null} +
    +
    +     + ); +}; + +export function DesktopAiSettingsRoute() { + const { state, updatePreferences } = useDesktopAi(); + const [busyPreference, setBusyPreference] = useState<"model" | "reasoning" | null>(null); + const [preferenceError, setPreferenceError] = useState(null); + const rateLimits = Object.values(state.rateLimitsByLimitId); + const selectedModel = state.preferences.model ?? ""; + const selectedReasoning = state.preferences.reasoningEffort ?? ""; + + const savePreference = async ( + key: "model" | "reasoning", + next: Partial<{ model: string | null; reasoningEffort: IslandflowAiReasoningEffort | null }> + ) => { + setBusyPreference(key); + setPreferenceError(null); + try { + await updatePreferences(next); + } catch (error) { + setPreferenceError(error instanceof Error ? error.message : String(error)); + } finally { + setBusyPreference(null); + } + }; + + return ( +
    + {!state.desktopAvailable ? ( + +
    +

    + AI controls are intentionally read-only in the browser build. Open Islandflow Desktop to use managed ChatGPT + login, structured Copilot turns, and app-server token telemetry. +

    +
    +     + ) : null} + + + +
    + +
    + + +
    +
    + {state.models.map((model) => ( +
    +
    + {model.displayName} +

    {model.description}

    +
    +
    + {model.model} + {model.pricing ? {formatUsd(model.pricing.inputUsdPer1MTokens)} / 1M input : null} +
    +
    + ))} +
    + {state.models.find((model) => model.model === state.preferences.model)?.pricing ? ( +

    + Normalized estimates use current API pricing for the selected model, not your literal ChatGPT subscription bill. +

    + ) : null} + {preferenceError ?

    {preferenceError}

    : null} +     + + + {rateLimits.length === 0 ? ( +

    No rate-limit snapshots have been reported yet.

    + ) : ( +
    + {rateLimits.map((limit) => ( + + ))} +
    + )} +     + + +
    + + +
    +     + + + {state.usage.recentTurns.length === 0 ? ( +

    No tracked turns yet.

    + ) : ( +
    + {state.usage.recentTurns.map((turn) => ( +
    +
    + {turn.taskTitle ?? "Ad hoc turn"} +

    + {turn.model ?? "default"} · {formatTimestamp(turn.updatedAt)} +

    +
    +
    + {formatTokens(turn.breakdown.totalTokens)} tok + {formatUsd(turn.normalizedCostUsd)} +
    +
    + ))} +
    + )} +     + + + {state.tasks.length === 0 ? ( +

    No Copilot tasks have been run yet.

    + ) : ( +
    + {state.tasks.map((task) => ( +
    +
    + {task.title} +

    + {task.subtitle} · {humanizeValue(task.model)} +

    +
    + {getTaskStatusLabel(task.status)} +
    + ))} +
    + )} +     +
    +
    + ); +} + +const requireDesktopActionCopy = (desktopAvailable: boolean, loggedIn: boolean): string => { + if (!desktopAvailable) { + return "This control is desktop-only. Open Islandflow Desktop to run Copilot tasks."; + } + if (!loggedIn) { + return "Connect a ChatGPT or Codex account in Settings before running Copilot analysis."; + } + return ""; +}; + +const SmartMoneyTaskButton = ({ + label, + kind, + symbol, + disabled, + busyKind, + onRun +}: { + label: string; + kind: IslandflowAiTaskKind; + symbol: string; + disabled: boolean; + busyKind: IslandflowAiTaskKind | null; + onRun: (kind: IslandflowAiTaskKind) => void; +}) => { + return ( + + ); +}; + +export function SmartMoneyCopilotPanel({ + event, + flowPacket, + evidencePrints, + relatedPackets +}: { + event: SmartMoneyEvent; + flowPacket: FlowPacket | null; + evidencePrints: OptionPrint[]; + relatedPackets: FlowPacket[]; +}) { + const { bridgeAvailable, state, runTask } = useDesktopAi(); + const [busyKind, setBusyKind] = useState(null); + const [activeTaskId, setActiveTaskId] = useState(null); + const [taskError, setTaskError] = useState(null); + const disabledCopy = requireDesktopActionCopy(bridgeAvailable, state.account.loggedIn); + const actionsDisabled = !bridgeAvailable || !state.account.loggedIn; + + const handleRun = async (kind: IslandflowAiTaskKind) => { + setBusyKind(kind); + setTaskError(null); + try { + const result = await runTask({ + kind: kind as + | "smart-money-explain" + | "smart-money-skeptic" + | "smart-money-burst-summary" + | "watchlist-synthesis", + context: { + event, + flowPacket, + evidencePrints, + relatedPackets + } + }); + setActiveTaskId(result.taskId); + } catch (error) { + setTaskError(error instanceof Error ? error.message : String(error)); + } finally { + setBusyKind(null); + } + }; + + return ( +
    +
    +
    +
    Analyst Copilot
    +

    Structured interpretation only, the deterministic classifier remains the source of truth.

    +
    + + AI settings + +
    +
    + void handleRun(kind)} + /> + void handleRun(kind)} + /> + void handleRun(kind)} + /> + void handleRun(kind)} + /> +
    + {disabledCopy ?

    {disabledCopy}

    : null} + {taskError ?

    {taskError}

    : null} + +
    + ); +} + +export function ReplayCopilotPanel({ + ticker, + flowFilters, + alerts, + smartMoneyEvents, + classifierHits, + flowPackets, + optionPrints +}: { + ticker: string | null; + flowFilters: OptionFlowFilters; + alerts: AlertEvent[]; + smartMoneyEvents: SmartMoneyEvent[]; + classifierHits: ClassifierHitEvent[]; + flowPackets: FlowPacket[]; + optionPrints: OptionPrint[]; +}) { + const { bridgeAvailable, state, runTask } = useDesktopAi(); + const [activeTaskId, setActiveTaskId] = useState(null); + const [busy, setBusy] = useState(false); + const [taskError, setTaskError] = useState(null); + const disabledCopy = requireDesktopActionCopy(bridgeAvailable, state.account.loggedIn); + const actionsDisabled = busy || !bridgeAvailable || !state.account.loggedIn; + + const handleRun = async () => { + setBusy(true); + setTaskError(null); + try { + const result = await runTask({ + kind: "replay-postmortem", + context: { + ticker, + flowFilters, + alerts, + smartMoneyEvents, + classifierHits, + flowPackets, + optionPrints + } + }); + setActiveTaskId(result.taskId); + } catch (error) { + setTaskError(error instanceof Error ? error.message : String(error)); + } finally { + setBusy(false); + } + }; + + return ( + + + AI settings + + + + } + > +

    + Copilot uses the current replay slice only: ticker scope, flow filters, visible alerts, classifier hits, packets, and option prints. +

    + {disabledCopy ?

    {disabledCopy}

    : null} + {taskError ?

    {taskError}

    : null} + +     + ); +} + +export function ScreenCompilerPanel({ + currentFilters, + onApplyFilters +}: { + currentFilters: OptionFlowFilters; + onApplyFilters: (next: OptionFlowFilters) => void; +}) { + const { bridgeAvailable, state, runTask } = useDesktopAi(); + const [prompt, setPrompt] = useState(""); + const [activeTaskId, setActiveTaskId] = useState(null); + const [busy, setBusy] = useState(false); + const [taskError, setTaskError] = useState(null); + const activeTask = useMemo(() => findTask(state.tasks, activeTaskId), [state.tasks, activeTaskId]); + const disabledCopy = requireDesktopActionCopy(bridgeAvailable, state.account.loggedIn); + const actionsDisabled = busy || !bridgeAvailable || !state.account.loggedIn; + + const handleCompile = async () => { + const trimmedPrompt = prompt.trim(); + if (!trimmedPrompt) { + setTaskError("Write a screen request first."); + return; + } + + setBusy(true); + setTaskError(null); + try { + const result = await runTask({ + kind: "screen-compile", + context: { + prompt: trimmedPrompt, + currentFilters + } + }); + setActiveTaskId(result.taskId); + } catch (error) { + setTaskError(error instanceof Error ? error.message : String(error)); + } finally { + setBusy(false); + } + }; + + const compiledFilters = activeTask?.compiledScreen?.compiledFilters ?? null; + + return ( + + + AI settings + + + + } + > +
    +