From 276d48950d62d8e13deaa45931ee5504b5d162a9 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 14:55:38 -0400 Subject: [PATCH 1/6] docs(daily-git): regenerate 2026-05-18 summary after merge --- .beads/issues.jsonl | 1 + ...2026-05-19-standup-summary-2026-05-18.html | 586 ++++++++---------- 2 files changed, 252 insertions(+), 335 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 2b26525..a1ec579 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -53,6 +53,7 @@ {"_type":"issue","id":"islandflow-zs0","title":"Migrate terminal UI to smart-money profiles","description":"Migrate apps/web terminal rendering to consume SmartMoneyEvent directly: primary profile, probability ladder, reason codes, and suppression/abstention state, while preserving legacy alert/classifier displays during the bridge.","status":"closed","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:23Z","created_by":"dirtydishes","updated_at":"2026-05-05T05:39:58Z","closed_at":"2026-05-05T05:39:58Z","close_reason":"Completed terminal smart-money profile migration","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-igk","title":"Add plan mode","description":"Implement a user-facing plan mode in the application so users can switch into planning before taking action. Scope to be clarified from existing app patterns.","status":"closed","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:22:37Z","created_by":"dirtydishes","updated_at":"2026-05-04T04:26:18Z","started_at":"2026-05-04T04:22:40Z","closed_at":"2026-05-04T04:26:18Z","close_reason":"Implemented as a global pi extension toggled with Shift+P","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-biq","title":"Finish raw live options delivery and filter/backpressure observability","description":"The smart-money signal path and Tape filters are in place, but the next firehose pass should finish server-side selective raw live delivery for options subscriptions and add explicit filtered-out/backpressure observability for API/web counters. This was discovered while landing islandflow-e4r.\n","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:28:58Z","created_by":"dirtydishes","updated_at":"2026-04-29T03:54:12Z","started_at":"2026-04-29T03:54:12Z","dependencies":[{"issue_id":"islandflow-biq","depends_on_id":"islandflow-e4r","type":"discovered-from","created_at":"2026-04-28T16:28:58Z","created_by":"auto-import","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-0ty","title":"Recreate May 18 standup summary after merge","description":"Regenerate docs/daily-git/2026-05-19-standup-summary-2026-05-18.html using merged history so it reflects all commits in the May 18 window, including native deployment and merge commits.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:53:48Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:55:33Z","started_at":"2026-05-19T18:53:52Z","closed_at":"2026-05-19T18:55:33Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-2df","title":"Publish 2026-05-18 git standup summary","description":"Why: the daily automation needs a grounded standup summary for May 18, 2026. What: review commits from 2026-05-18, create a scannable HTML summary in docs/daily-git, and capture only commit/file-backed statements.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:41:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:42:42Z","started_at":"2026-05-19T18:41:10Z","closed_at":"2026-05-19T18:42:42Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-x70","title":"Create 2026-05-17 git standup summary","description":"Why this issue exists and what needs to be done:\\n- Produce the daily automation summary for 2026-05-17 git activity.\\n- Ground statements in commits, PRs, and touched files only.\\n- Create a user-readable HTML document in docs/general and update automation memory.\\n- Complete the Beads sync and git push workflow after documenting the run.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:43Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:05:37Z","started_at":"2026-05-18T13:01:53Z","closed_at":"2026-05-18T13:05:37Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-zsy","title":"Expose Forgejo SSH on a direct DNS hostname","description":"git.deltaisland.io currently resolves through Cloudflare's proxy, so SSH on port 2222 does not complete even though the Forgejo container is listening on the host. If SSH-based git/beads workflows are desired, add a DNS-only hostname (or adjust the existing record) that points directly at the server for Forgejo SSH.","status":"open","priority":3,"issue_type":"task","created_at":"2026-05-17T10:34:06Z","created_by":"delta","updated_at":"2026-05-17T10:34:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html b/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html index 5a33e66..1d6e914 100644 --- a/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html +++ b/docs/daily-git/2026-05-19-standup-summary-2026-05-18.html @@ -3,27 +3,23 @@ - Daily Git Summary for 2026-05-18 + Daily Git Summary for 2026-05-18 (Merged View) @@ -283,56 +238,54 @@
Daily Git Summary -

Standup summary for Monday, May 18, 2026

+

Standup summary for Monday, May 18, 2026 (after merge)

- Git history for May 18 shows four commits. One feature commit introduced an Alpaca-backed news wire across ingest, - storage, API, and web surfaces; the other three commits updated workflow docs, beads state, and the previous - standup summary. + This regenerated report uses merged history for the full May 18 local-day window + (2026-05-18 00:00 -0400 through 2026-05-19 00:00 -0400). It now includes eight commits, + including native deployment work and the merge commit that landed that line of work on main.

-
+
Commits -
4
+
8
- Files Touched -
35
+ Unique Files +
68
Insertions -
1963
+
4244
Deletions -
52
+
194

Summary

-
-
- Primary Delivery +
+
+ User-facing delivery

- Commit 906fe411 added a new services/ingest-news service, news persistence in - packages/storage, API endpoints in services/api, and a live news view in - apps/web/app/terminal.tsx plus apps/web/app/news/page.tsx. + Commit 906fe411 added Alpaca news wire support across ingest, storage, API, and web terminal/news + route surfaces.

-
- Docs And Workflow +
+ Platform and deployment delivery

- Commits 62aae708, 687a2170, and 04baeceb updated the previous standup - report, beads state, deployment/docker/workspace-root/package.json, and the repo-level - AGENTS.md instructions. + Commits d589858c and bdb9d9a9 added native deployment workflow, infra/user units, + cutover, rollback, and health-check scripts, then merged via 8f0794dd (PR #2).

-
- Standup Framing +
+ Workflow and docs updates

- Yesterday’s visible product work centered on making live Alpaca news available end to end. The remaining - activity was project hygiene and documentation. + Commits 687a2170, 62aae708, 48095fce, and 04baeceb updated + beads/docs instructions and added turn/standup documentation.

@@ -341,94 +294,99 @@

Changes Made

-
-
+
+
update beads - 687a2170 - 2026-05-18 03:15 -0400 + 687a2170 + 2026-05-18 03:15 -0400 1 file
-

- Added one line to deployment/docker/workspace-root/package.json. The local git history does not - show more context beyond the file touch and commit subject. -

+

Touched deployment/docker/workspace-root/package.json with one-line change.

-
-
- docs(general): add 2026-05-17 standup summary - 62aae708 - 2026-05-18 09:05 -0400 +
+
+ Implement native fast iterative deploy workflow + d589858c + 2026-05-18 03:34 -0400 + 17 files + +873 / -110 +
+
    +
  • Expanded scripts/deploy.ts for native deploy runtime behavior.
    • +
  • Added native user-unit templates and rollback/health tooling in deployment/native/.
    • +
  • Added associated plan and turn documents in docs/plans and docs/turns.
    • +
+
+ +
+
+ fix(api): remove duplicate alert context import + 48095fce + 2026-05-18 09:04 -0400 2 files
-

- Added the prior day’s report at docs/general/2026-05-18-standup-summary-2026-05-17.html and - updated .beads/issues.jsonl. -

-
- docs/general/2026-05-18-standup-summary-2026-05-17.html - .beads/issues.jsonl -
+

Removed duplicate import in services/api/src/index.ts and added a turn doc.

-
-
+
+
+ docs(general): add 2026-05-17 standup summary + 62aae708 + 2026-05-18 09:05 -0400 + 2 files +
+

Added docs/general/2026-05-18-standup-summary-2026-05-17.html and updated beads state.

+
+ +
+
add alpaca news wire across ingest api and web - 906fe411 - 2026-05-18 16:55 -0400 + 906fe411 + 2026-05-18 16:55 -0400 31 files +1407 / -50
    -
  • - Added a new ingest service in services/ingest-news/src/index.ts that backfills Alpaca news, - subscribes to the Alpaca news websocket, resolves symbols, and publishes NewsStory payloads to - NATS. -
    • -
  • - Extended shared contracts in packages/types/src/events.ts and - packages/types/src/live.ts, plus new storage support in - packages/storage/src/news.ts and packages/storage/src/clickhouse.ts. -
    • -
  • - Wired the API to store, fan out, and expose news via /news and /history/news in - services/api/src/index.ts and live-session updates in services/api/src/live.ts. -
    • -
  • - Added a web route in apps/web/app/news/page.tsx, a news pane and drawer in - apps/web/app/terminal.tsx, and related styling in apps/web/app/globals.css. -
    • -
  • - Updated runtime packaging and local/dev deployment surfaces, including - deployment/docker/docker-compose.yml, Dockerfiles, scripts/dev.ts, and - scripts/deploy.ts. -
    • -
  • - Added tests in packages/storage/tests/news.test.ts, - services/ingest-news/tests/symbols.test.ts, and adjusted - apps/web/app/terminal.test.ts plus packages/types/tests/live.test.ts. -
    • +
  • Created services/ingest-news and wired Alpaca backfill/websocket ingestion.
    • +
  • Added news types/storage contracts in packages/types and packages/storage.
    • +
  • Extended API live/history endpoints and web terminal/news route rendering.
-
- services/ingest-news/src/index.ts - packages/storage/src/news.ts - services/api/src/index.ts - apps/web/app/terminal.tsx - apps/web/app/news/page.tsx - apps/web/app/globals.css -
-
-
+
+
+ Implement native public edge cutover + bdb9d9a9 + 2026-05-18 19:55 -0400 + 29 files + +1215 / -31 +
+
    +
  • Added native infra system units and scripts for bootstrap/start/stop/cutover/full rollback.
    • +
  • Updated deploy docs and runtime config files under deployment/native/config.
    • +
  • Added turn doc docs/turns/2026-05-18-native-public-edge-cutover.html.
    • +
+
+ +
+
+ Merge pull request 'Native public edge cutover with Docker rollback path' (#2) + 8f0794dd + 2026-05-19 00:09 +0000 + merge commit +
+

Merged native-deploy into main within the May 18 US/Eastern day window.

+
+ +
+
update turn docs and beads workflow - 04baeceb - 2026-05-18 21:32 -0400 + 04baeceb + 2026-05-18 21:32 -0400 1 file
-

- Edited AGENTS.md to update turn-document and beads workflow guidance. -

+

Updated repository-level instructions in AGENTS.md.

@@ -436,92 +394,59 @@

Context

- This summary is based on local git history between 2026-05-18 00:00 -0400 and - 2026-05-19 00:00 -0400. The repository uses Bun, TypeScript, NATS/JetStream, ClickHouse, and a Next.js - web app, so the main feature commit spans service ingestion, shared types, persistence, API delivery, and the UI. + The earlier report was generated before merged history included the native deployment branch on main. + This recreation uses git log --all over the same date window, so it captures both feature work and + merged operational/deployment work visible after PR merge.

Important Implementation Details

-
-
-

News ingestion was introduced as a first-class service

+
+
+

News wire ingestion and delivery path

- services/ingest-news/src/index.ts authenticates against Alpaca, backfills recent news, subscribes - to live updates, resolves symbols, validates payloads with NewsStorySchema, and publishes them onto - the repo’s bus layer. -

- 
const backfill = await fetchBackfill();
-for (const item of backfill.reverse()) {
-  await publishStory(item);
-}
-
-if (msg === "authenticated") {
-  ws.send(JSON.stringify({ action: "subscribe", news: ["*"] }));
-}
-
- -
-

API and live session support were expanded for news

-

- services/api/src/index.ts now ensures the news table exists, subscribes to a news consumer, fans - out live updates, and exposes both recent and paginated history endpoints. -

- 
if (req.method === "GET" && url.pathname === "/news") {
-  const limit = parseLimit(url.searchParams.get("limit") ?? "100");
-  const data = await fetchRecentNews(clickhouse, limit);
-  return jsonResponse({ data });
-}
-
- -
-

The web terminal gained a dedicated news surface

-

- apps/web/app/terminal.tsx added a live-only news pane, a per-story drawer, history loading, and a - new /news route entry point via apps/web/app/news/page.tsx. + The news pipeline added a new ingest service and API fanout channel, then exposed UI surfaces in + /news and terminal panes. 

if (features.news) {
   subscriptions.push({ channel: "news", snapshot_limit: LIVE_OPTIONS_HEAD_LIMIT });
-}
-
-export function NewsRoute() {
-  const state = useTerminal();
-  return (
-    <PageFrame title="News">
-      <div className="page-grid page-grid-news">
-        <NewsPane state={state} className="news-pane-full" />
-      </div>
-    </PageFrame>
-  );
 }
-
+
+
+

Native deployment hardening

+

+ Deployment scripts and unit templates now include direct scripts for cutover and rollback, with infra and + service checks under deployment/native/. +

+ 
deployment/native/cutover.sh
+deployment/native/full-rollback.sh
+deployment/native/install-infra-units.sh
+
+
+

Merged history effect on standup scope

+

+ The merged view increased the standup scope from 4 to 8 commits and from 35 to 68 unique files touched for the + same local-day window. +

+

Expected Impact for End-Users

-
-
- Live Terminal -

- Users now have a dedicated news wire surface in the web terminal, including summary rows, story details, and - a direct link to the source article. -

+
+
+ Trading UI users +

Live news wire data is now available in terminal surfaces alongside existing market/event feeds.

-
- Coverage -

- News is now available alongside the repo’s existing live feeds, with shared symbol resolution and storage that - make the data retrievable through API history endpoints. -

+
+ Operators +

Native deployment and rollback procedures now have first-class scripted and documented paths.

-
- Current Boundary -

- The UI copy in the news pane explicitly marks news as live-only in v1, so replay users should not expect the - same behavior there yet. -

+
+ Team reporting +

This standup report now matches merged repository history instead of pre-merge branch-local history.

@@ -529,36 +454,27 @@ export function NewsRoute() {

Validation

    -
  • Reviewed local git history with git log --since='2026-05-18 00:00' --until='2026-05-19 00:00'.
    • -
  • Used git log --stat, git show, and file-level history to anchor each summary item to specific commits and files.
    • -
  • No builds or tests were run for this reporting task because the work product is a git summary document, not a behavior change.
    • +
  • Used git fetch --all --prune before recomputing history.
    • +
  • Used git log --all over the May 18 ET window to include merged commits.
    • +
  • Used git log --stat --summary and --numstat to ground file and line-count statements.
    • +
  • No build/test commands were run because this task only regenerates reporting documentation.

Issues, Limitations, and Mitigations

    -
  • - This report is grounded in local commit metadata only. No pull request identifiers were present in the inspected - git history, so the summary references commits and files instead of PR numbers. -
    • -
  • - The update beads commit touched only deployment/docker/workspace-root/package.json in - visible git output, so this report does not infer intent beyond that recorded file change. -
    • -
  • - Counts here describe May 18 commits only and exclude any uncommitted work present after that date. -
    • +
  • This report describes commit history only and does not infer intent beyond commit messages and touched files.
    • +
  • Commit 8f0794dd is timestamped in UTC; it still falls on May 18 in US/Eastern, so it is included.
    • +
  • Metrics are based on local git history at regeneration time and can change if additional backdated commits appear.

Follow-up Work

    -
  • - No new product follow-up items were derived from this reporting pass. The only beads item created for this task - is islandflow-2df, which tracks publication of this summary document. -
    • +
  • This regeneration is tracked by beads issue islandflow-0ty.
    • +
  • No additional follow-up work was identified during this documentation-only task.
From 4bacf2c2f8ee48cbeecda7985bfccd1772e777f3 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 14:59:58 -0400 Subject: [PATCH 2/6] publish docs index and github pages workflow --- .beads/issues.jsonl | 1 + .github/workflows/docs-pages.yml | 56 ++ docs/index.html | 638 ++++++++++++++++++ .../2026-05-19-publish-docs-pages-index.html | 195 ++++++ scripts/generate-docs-index.mjs | 421 ++++++++++++ 5 files changed, 1311 insertions(+) create mode 100644 .github/workflows/docs-pages.yml create mode 100644 docs/index.html create mode 100644 docs/turns/2026-05-19-publish-docs-pages-index.html create mode 100644 scripts/generate-docs-index.mjs diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index a1ec579..e6a01f4 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -14,6 +14,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-tqk","title":"publish docs/ to github pages with navigable index","description":"Set up docs deployment so repository docs are published to dirtydishes.github.io/islandflow/docs with a nicer, browsable experience than a raw file listing.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:56:02Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:59:55Z","started_at":"2026-05-19T18:56:04Z","closed_at":"2026-05-19T18:59:55Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lm6","title":"Clarify repo turn documentation scope","description":"Update AGENTS.md so repository turn documentation clearly uses repo-local docs/turns and impeccable styling, without inheriting global non-repo computer-task styling.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T12:05:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T12:06:12Z","started_at":"2026-05-19T12:05:14Z","closed_at":"2026-05-19T12:06:12Z","close_reason":"Verified AGENTS.md now scopes repo turn docs to docs/turns and makes impeccable the styling authority; added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-6iq","title":"Update README for current project state","description":"Resolve README merge conflicts and document the current project state, including the smart money classification taxonomy, Next.js update, and deployment workflow changes.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:37:24Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:40:01Z","started_at":"2026-05-19T11:37:31Z","closed_at":"2026-05-19T11:40:01Z","close_reason":"README conflict resolved and current project state documented, including smart-money taxonomy, Next.js update, and deployment workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lib","title":"Upgrade apps/web to Next.js 16.2.6","description":"Upgrade the web app dependency stack to Next.js 16.2.6 with React 19, refresh Bun and mirrored Docker workspace lockfiles, keep runtime behavior unchanged, fix any focused web test fallout, validate the web build and targeted route tests, and document the completed work.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:04:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:31:23Z","started_at":"2026-05-19T11:04:57Z","closed_at":"2026-05-19T11:31:23Z","close_reason":"Upgraded apps/web to Next.js 16.2.6 with React 19, refreshed Bun lockfiles including the Docker workspace mirror, fixed the React 19 nullable ref type issue, and validated the web build, focused tests, Docker workspace sync, and route smoke checks.","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/.github/workflows/docs-pages.yml b/.github/workflows/docs-pages.yml new file mode 100644 index 0000000..9c4db98 --- /dev/null +++ b/.github/workflows/docs-pages.yml @@ -0,0 +1,56 @@ +name: Publish Docs + +on: + push: + branches: + - main + paths: + - "docs/**" + - "scripts/generate-docs-index.mjs" + - ".github/workflows/docs-pages.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure Pages + uses: actions/configure-pages@v5 + + - name: Build docs index + run: node scripts/generate-docs-index.mjs + + - name: Prepare static site payload + run: | + mkdir -p site/docs + cp -R docs/. site/docs/ + printf '%s\n' 'Islandflow DocsContinue to docs' > site/index.html + touch site/.nojekyll + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + needs: build + runs-on: ubuntu-latest + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..211c5ac --- /dev/null +++ b/docs/index.html @@ -0,0 +1,638 @@ + + + + + + Islandflow Docs + + + +
+
+

Islandflow docs index

+

A browsable index of files under docs/ with filtering and grouped navigation.

+
+ +
+
35 of 35 files shown
+ + +
+ +
+
+

turns 28

+ +
+ + +
+

daily-git 1

+ +
+ + +
+

general 2

+ +
+ + +
+

plans 2

+ +
+ + +
+

root 2

+ +
+
+

No files match that filter.

+
+ + + + diff --git a/docs/turns/2026-05-19-publish-docs-pages-index.html b/docs/turns/2026-05-19-publish-docs-pages-index.html new file mode 100644 index 0000000..9946b33 --- /dev/null +++ b/docs/turns/2026-05-19-publish-docs-pages-index.html @@ -0,0 +1,195 @@ + + + + + + Turn Report - Publish Docs to GitHub Pages + + + +
+

Publish docs/ to GitHub Pages with navigable index

+

Completed on May 19, 2026 at 9:38 AM ET.

+ +
+

Summary

+

+ Added an automated docs publishing flow to GitHub Pages and generated a new + docs/index.html browsing experience so docs are easy to navigate at + /islandflow/docs/. +

+
+ +
+

Changes Made

+
    +
  • Added scripts/generate-docs-index.mjs to build a browsable index of files under docs/.
    • +
  • Added .github/workflows/docs-pages.yml to publish docs to GitHub Pages on pushes to main.
    • +
  • Generated docs/index.html from current docs content.
    • +
  • Configured deployment artifact layout so docs are available at /docs/ under the project Pages site.
    • +
+
+ +
+

Context

+

+ The repository already stores operational and implementation documentation under + docs/, but there was no dedicated GitHub Pages pipeline and no curated + index page for discovery. This task focused on syncing that folder to Pages and + making it easy to browse by category and filename. +

+
+ +
+

Important Implementation Details

+
    +
  • The index generator excludes hidden files and avoids self-including docs/index.html.
    • +
  • Files are grouped by first path segment (turns, general, plans, and others) with quick category chips.
    • +
  • The index includes client-side filtering so users can search docs by path text in-browser.
    • +
  • Pages deployment packages a site/ payload where docs are copied into site/docs and root redirects to ./docs/.
    • +
+
+ +
+

Relevant Diff Snippets

+

+ Snippets are shown in a compact style aligned with diffs.com presentation patterns. +

+ 
+++ .github/workflows/docs-pages.yml
+name: Publish Docs
+on:
+  push:
+    branches: [main]
+    paths:
+      - "docs/**"
+      - "scripts/generate-docs-index.mjs"
+      - ".github/workflows/docs-pages.yml"
+  workflow_dispatch:
+
+jobs:
+  build:
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/configure-pages@v5
+      - run: node scripts/generate-docs-index.mjs
+      - run: cp -R docs/. site/docs/
+      - uses: actions/upload-pages-artifact@v3
+  deploy:
+    needs: build
+    steps:
+      - uses: actions/deploy-pages@v4
+ 
+++ scripts/generate-docs-index.mjs
+const files = await collectDocsFiles(docsDir);
+const html = renderDocument(files);
+await fs.writeFile(outputFile, html, "utf8");
+
+// Generated index features:
+// - grouped sections
+// - search filter
+// - file size and modified time metadata
+// - links preserving docs folder structure
+
+ +
+

Expected Impact for End-Users

+
    +
  • Docs are reachable via a stable Pages URL path: dirtydishes.github.io/islandflow/docs/.
    • +
  • Readers can quickly scan categories and search by filename instead of relying on raw directory browsing.
    • +
  • New docs added to the repository are published automatically on main pushes.
    • +
+
+ +
+

Validation

+
    +
  • Ran node scripts/generate-docs-index.mjs successfully.
    • +
  • Ran node --check scripts/generate-docs-index.mjs for syntax validation.
    • +
  • Confirmed generated index contains expected navigation/search markers and category anchors.
    • +
+
+ +
+

Issues, Limitations, and Mitigations

+
    +
  • GitHub Pages must be enabled for this repository and set to GitHub Actions deployment.
    • +
  • The index reflects files present at build time and does not include full-text search inside documents.
    • +
  • Markdown files are linked as-is; rendering behavior depends on GitHub Pages static hosting behavior.
    • +
+
+ +
+

Follow-up Work

+
    +
  • Add a docs landing page summary for key collections (turn docs, runbooks, daily notes).
    • +
  • Optionally add link-checking in CI for docs URLs and local references.
    • +
  • Consider tagging docs with metadata for richer filtering by date, topic, and type.
    • +
+
+
+ + diff --git a/scripts/generate-docs-index.mjs b/scripts/generate-docs-index.mjs new file mode 100644 index 0000000..cf64a9d --- /dev/null +++ b/scripts/generate-docs-index.mjs @@ -0,0 +1,421 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; + +const docsDir = path.resolve(process.cwd(), "docs"); +const outputFile = path.join(docsDir, "index.html"); + +const dateFormatter = new Intl.DateTimeFormat("en-US", { + dateStyle: "medium", + timeStyle: "short", +}); + +function escapeHtml(value) { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function formatBytes(bytes) { + if (bytes < 1024) { + return `${bytes} B`; + } + + const units = ["KB", "MB", "GB"]; + let size = bytes / 1024; + let unitIndex = 0; + + while (size >= 1024 && unitIndex < units.length - 1) { + size /= 1024; + unitIndex += 1; + } + + return `${size.toFixed(size >= 10 ? 0 : 1)} ${units[unitIndex]}`; +} + +function docsHref(relativePath) { + const encoded = relativePath + .split("/") + .map((part) => encodeURIComponent(part)) + .join("/"); + return `./${encoded}`; +} + +async function collectDocsFiles(rootDir, currentDir = rootDir, acc = []) { + const entries = await fs.readdir(currentDir, { withFileTypes: true }); + const sortedEntries = entries.sort((a, b) => a.name.localeCompare(b.name)); + + for (const entry of sortedEntries) { + if (entry.name.startsWith(".")) { + continue; + } + + const absolutePath = path.join(currentDir, entry.name); + const relativePath = path.relative(rootDir, absolutePath).replaceAll(path.sep, "/"); + + if (relativePath === "index.html") { + continue; + } + + if (entry.isDirectory()) { + await collectDocsFiles(rootDir, absolutePath, acc); + continue; + } + + if (entry.isFile()) { + const stats = await fs.stat(absolutePath); + + acc.push({ + relativePath, + category: relativePath.includes("/") ? relativePath.split("/")[0] : "root", + sizeBytes: stats.size, + modifiedAt: stats.mtime, + }); + } + } + + return acc; +} + +function groupByCategory(items) { + const groups = new Map(); + for (const item of items) { + if (!groups.has(item.category)) { + groups.set(item.category, []); + } + groups.get(item.category).push(item); + } + return groups; +} + +function sortedCategories(groups) { + const preferredOrder = ["turns", "daily-git", "general", "plans", "root"]; + const groupNames = [...groups.keys()]; + return groupNames.sort((a, b) => { + const aIndex = preferredOrder.indexOf(a); + const bIndex = preferredOrder.indexOf(b); + + if (aIndex !== -1 || bIndex !== -1) { + if (aIndex === -1) return 1; + if (bIndex === -1) return -1; + return aIndex - bIndex; + } + + return a.localeCompare(b); + }); +} + +function renderDocument(items) { + const sortedItems = [...items].sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime()); + const groups = groupByCategory(sortedItems); + const categories = sortedCategories(groups); + const totalCount = sortedItems.length; + + const categoryChips = categories + .map((category) => { + const count = groups.get(category).length; + return `${escapeHtml( + category + )} ${count}`; + }) + .join("\n"); + + const groupsMarkup = categories + .map((category) => { + const entries = groups.get(category); + const entryMarkup = entries + .map((entry) => { + const extension = path.extname(entry.relativePath).replace(".", "") || "file"; + const searchable = `${entry.relativePath} ${category}`.toLowerCase(); + return ` +
  • + ${escapeHtml( + entry.relativePath + )} +
    + ${escapeHtml(extension)} + ${escapeHtml(formatBytes(entry.sizeBytes))} + ${escapeHtml(dateFormatter.format(entry.modifiedAt))} +
    +
    • + `; + }) + .join("\n"); + + return ` +
    +

    ${escapeHtml(category)} ${entries.length}

    +
      + ${entryMarkup} +
    +
    + `; + }) + .join("\n"); + + return ` + + + + + Islandflow Docs + + + +
    +
    +

    Islandflow docs index

    +

    A browsable index of files under docs/ with filtering and grouped navigation.

    +
    + +
    +
    ${totalCount} of ${totalCount} files shown
    + + +
    + +
    ${groupsMarkup}
    +

    No files match that filter.

    +
    + + + + +`; +} + +async function main() { + const files = await collectDocsFiles(docsDir); + const html = renderDocument(files); + await fs.writeFile(outputFile, html, "utf8"); + console.log(`Generated ${outputFile} with ${files.length} entries.`); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); From bca74d1811ae84da2bdb57cab91fb3ac74025951 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 17:00:15 -0400 Subject: [PATCH 3/6] update beads for codex forgejo status patch --- .beads/issues.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index e6a01f4..6d09bd4 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,4 @@ +{"_type":"issue","id":"islandflow-jor","title":"Support Forgejo pull request status in desktop git panel","description":"The desktop app currently reports pull request status unavailable when a repository only has a Forgejo remote. Add native Forgejo/Gitea-style remote detection and pull request status lookup so Forgejo-only repositories can show PR state in the Codex app git panel.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T20:55:15Z","created_by":"dirtydishes","updated_at":"2026-05-19T20:59:46Z","started_at":"2026-05-19T20:55:25Z","closed_at":"2026-05-19T20:59:46Z","close_reason":"Patched the installed Codex desktop app bundle with a Forgejo PR status fallback and documented the local change.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-g3a","title":"Reconcile PR merge conflicts","description":"Resolve the current pull request conflicts for the nextjs-upgrade branch, validate the result, document the turn, and push the reconciled branch.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:44:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:47:35Z","started_at":"2026-05-19T18:44:56Z","closed_at":"2026-05-19T18:47:35Z","close_reason":"Merged forgejo/main into nextjs-upgrade, resolved README and Beads conflicts, updated JetStream retention tests, validated deploy help, Docker workspace sync, API/bus tests, and web build, and added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-jbi","title":"Hydrate alert evidence details from ClickHouse","description":"Alert detail drawers need to fetch persisted alert context from ClickHouse by trace id, including linked flow packets, option prints, preserved execution context, and explicit missing refs for UI diagnostics.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:55:43Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:01:58Z","started_at":"2026-05-17T14:55:53Z","closed_at":"2026-05-17T15:01:58Z","close_reason":"Implemented ClickHouse-backed alert context hydration across storage, API, terminal drawer, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8kj","title":"Configure persistent beads Dolt remote on deltaisland server","description":"Install the beads and Dolt CLIs on the server, configure a persistent Dolt sync remote backed by the server-hosted Forgejo repository, verify refs/dolt/data publication, and document Nginx Proxy Manager / firewall considerations.","status":"closed","priority":1,"issue_type":"task","assignee":"delta","created_at":"2026-05-17T10:31:31Z","created_by":"delta","updated_at":"2026-05-17T10:37:47Z","started_at":"2026-05-17T10:32:16Z","closed_at":"2026-05-17T10:37:47Z","close_reason":"Installed bd and dolt on the server, configured the Forgejo-backed Dolt remote, published refs/dolt/data, and documented the setup.","dependency_count":0,"dependent_count":0,"comment_count":0} From 4b8eaae0ee8b882948d312fb8a9d88f61db09cb7 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 19:28:33 -0400 Subject: [PATCH 4/6] document native options recovery and clean up the unit override --- .beads/issues.jsonl | 3 + deployment/native/README.md | 2 + .../user/islandflow-ingest-options.service | 1 - ...19-native-options-recovery-guardrails.html | 183 ++++++++++++++++++ 4 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 docs/turns/2026-05-19-native-options-recovery-guardrails.html diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 6d09bd4..3df43cf 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -15,6 +15,9 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-wf5","title":"Harden native options provider configuration after synthetic recovery","description":"Native production recovery restored OPTIONS_INGEST_ADAPTER=synthetic because the current Alpaca setup fails authentication and crash-loops ingest-options. Follow up by deciding whether production options should remain synthetic or move to a supported live provider auth path, then add a deploy-time smoke test or config validation that catches provider auth failures before native cutover.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:27:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-m83","title":"Restore options ingestion and print generation on native deployment","description":"After moving the production/VPS deployment from Docker-managed services to the native runtime, the options feed appears behind and fresh option prints are not reaching the UI. Investigate the native deployment path on the server, identify the ingestion or compute breakage, apply the required code and/or host configuration changes, validate that fresh option prints resume, and document any follow-up operational work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:20:01Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:52Z","started_at":"2026-05-19T23:20:10Z","closed_at":"2026-05-19T23:27:52Z","close_reason":"Restored native options ingest by switching the VPS back to the last known-good synthetic adapter, verified fresh option prints and compute output, and documented the native env precedence gotcha.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-o1v","title":"Add SCM provider layer with Forgejo detection","description":"Implement provider-aware source-control detection and mirror-aware guardrails for repo automation so Forgejo remotes are treated as authoritative when present.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:04:33Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:06:55Z","started_at":"2026-05-19T23:04:35Z","closed_at":"2026-05-19T23:06:55Z","close_reason":"created by mistake during interrupted turn; no implementation was started","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-tqk","title":"publish docs/ to github pages with navigable index","description":"Set up docs deployment so repository docs are published to dirtydishes.github.io/islandflow/docs with a nicer, browsable experience than a raw file listing.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:56:02Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:59:55Z","started_at":"2026-05-19T18:56:04Z","closed_at":"2026-05-19T18:59:55Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-lm6","title":"Clarify repo turn documentation scope","description":"Update AGENTS.md so repository turn documentation clearly uses repo-local docs/turns and impeccable styling, without inheriting global non-repo computer-task styling.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T12:05:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T12:06:12Z","started_at":"2026-05-19T12:05:14Z","closed_at":"2026-05-19T12:06:12Z","close_reason":"Verified AGENTS.md now scopes repo turn docs to docs/turns and makes impeccable the styling authority; added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-6iq","title":"Update README for current project state","description":"Resolve README merge conflicts and document the current project state, including the smart money classification taxonomy, Next.js update, and deployment workflow changes.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:37:24Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:40:01Z","started_at":"2026-05-19T11:37:31Z","closed_at":"2026-05-19T11:40:01Z","close_reason":"README conflict resolved and current project state documented, including smart-money taxonomy, Next.js update, and deployment workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/deployment/native/README.md b/deployment/native/README.md index c421c51..569cdb8 100644 --- a/deployment/native/README.md +++ b/deployment/native/README.md @@ -98,6 +98,8 @@ These are written for the current VPS layout: - Bun binary: `/home/delta/.bun/bin/bun` - env file: `/home/delta/islandflow/.env` +Important: treat `/home/delta/islandflow/.env` as the effective source of truth for adapter selection. The Bun-launched services read that file directly at runtime, so a conflicting `OPTIONS_INGEST_ADAPTER` value in `.env` can still win over a systemd-only override and push `ingest-options` onto the wrong provider path. + ### Install the units ```bash diff --git a/deployment/native/systemd/user/islandflow-ingest-options.service b/deployment/native/systemd/user/islandflow-ingest-options.service index 10107b1..eac0a6c 100644 --- a/deployment/native/systemd/user/islandflow-ingest-options.service +++ b/deployment/native/systemd/user/islandflow-ingest-options.service @@ -7,7 +7,6 @@ Wants=network-online.target Type=simple WorkingDirectory=/home/delta/islandflow EnvironmentFile=/home/delta/islandflow/.env -Environment=OPTIONS_INGEST_ADAPTER=synthetic ExecStart=/home/delta/.bun/bin/bun services/ingest-options/src/index.ts Restart=always RestartSec=2 diff --git a/docs/turns/2026-05-19-native-options-recovery-guardrails.html b/docs/turns/2026-05-19-native-options-recovery-guardrails.html new file mode 100644 index 0000000..441ade2 --- /dev/null +++ b/docs/turns/2026-05-19-native-options-recovery-guardrails.html @@ -0,0 +1,183 @@ + + + + + + 2026-05-19 Native Options Recovery Guardrails + + + +
    +
    +

    Native Options Recovery Guardrails

    +

    + The production outage turned out to be a native deployment config mismatch, not a data-pipeline code failure. I restored the VPS to the last known-good synthetic options adapter, then tightened the checked-in native deployment assets so they no longer imply a systemd override will beat the repo .env. +

    +
    Generated 2026-05-19 19:24 EDT
    +
    + +
    +

    Summary

    +

    + The repo-side change is small and targeted: remove the misleading Environment=OPTIONS_INGEST_ADAPTER=synthetic line from the checked-in native ingest-options unit, and document that Bun-launched services effectively take adapter selection from /home/delta/islandflow/.env. +

    +
    + +
    +

    Changes Made

    +
      +
    • Removed the checked-in systemd override from deployment/native/systemd/user/islandflow-ingest-options.service.
      • +
    • Added an explicit env-precedence warning to deployment/native/README.md.
      • +
    • Captured the live diagnosis that the native server had drifted to OPTIONS_INGEST_ADAPTER=alpaca while the prior Docker deployment was running synthetic options.
      • +
    +
    + +
    +

    Context

    +

    + On the VPS, islandflow-ingest-options.service was crash-looping with repeated 401 Unauthorized responses from Alpaca while the rest of the native stack stayed healthy. The previous Docker-owned islandflow-vps-ingest-options-1 container showed OPTIONS_INGEST_ADAPTER=synthetic, which explains why the UI had been healthy before the runtime transition. +

    +
    + +
    +

    Important Implementation Details

    +
      +
    • The checked-in unit already referenced /home/delta/islandflow/.env, and Bun's runtime env loading meant a conflicting adapter value there still won in practice.
      • +
    • The static key currently stored as ALPACA_API_KEY does not authenticate the failing market-data snapshot request as a Bearer token.
      • +
    • Because the real outage fix required a server-side .env correction, this repo patch focuses on preventing operator confusion during the next native cutover.
      • +
    +
    + +
    +

    Relevant Diff Snippets

    +

    Unified diff blocks below are formatted for diffs-compatible rendering.

    + 
    diff --git a/deployment/native/README.md b/deployment/native/README.md
+@@ -98,6 +98,8 @@ These are written for the current VPS layout:
+ - Bun binary: `/home/delta/.bun/bin/bun`
+ - env file: `/home/delta/islandflow/.env`
+ 
++Important: treat `/home/delta/islandflow/.env` as the effective source of truth for adapter selection. The Bun-launched services read that file directly at runtime, so a conflicting `OPTIONS_INGEST_ADAPTER` value in `.env` can still win over a systemd-only override and push `ingest-options` onto the wrong provider path.
++
+ ### Install the units
+
+diff --git a/deployment/native/systemd/user/islandflow-ingest-options.service b/deployment/native/systemd/user/islandflow-ingest-options.service
+@@ -7,7 +7,6 @@ Wants=network-online.target
+ Type=simple
+ WorkingDirectory=/home/delta/islandflow
+ EnvironmentFile=/home/delta/islandflow/.env
+-Environment=OPTIONS_INGEST_ADAPTER=synthetic
+ ExecStart=/home/delta/.bun/bin/bun services/ingest-options/src/index.ts
    +
    + +
    +

    Expected Impact for End-Users

    +

    + End users should not see the options tape stall the next time native units are installed or audited by following the checked-in assets. Operators now have a clearer paper trail that the actual runtime adapter comes from the deployment env file. +

    +
    + +
    +

    Validation

    +
      +
    • Verified the native outage mode on the VPS: islandflow-ingest-options.service crash-looped on Alpaca 401 responses.
      • +
    • Confirmed the previous Docker container had been running OPTIONS_INGEST_ADAPTER=synthetic.
      • +
    • After the server-side env fix, confirmed fresh rows in default.option_prints and new compute emissions in the native logs.
      • +
    • Ran git diff to verify the repo change stayed scoped to the deployment README and the checked-in user unit.
      • +
    +
    + +
    +

    Issues, Limitations, and Mitigations

    +
      +
    • The repo patch does not add new credential support for Alpaca. It only documents the current env-precedence behavior and removes a misleading override.
      • +
    • The live server is restored with synthetic options, which matches the last known-good Docker behavior, but it is not a true live Alpaca ingest path.
      • +
    +
    + +
    +

    Follow-up Work

    +
      +
    • islandflow-wf5: Decide whether production options should remain synthetic or move to a fully supported live provider configuration.
      • +
    • islandflow-wf5: If Alpaca live data is still desired, add a validated auth flow and a deploy-time smoke test that catches provider auth failures before cutover.
      • +
    +
    +
    + + From e70835e9c4ea335dac42493249c996cbd560bcf8 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 19:40:20 -0400 Subject: [PATCH 5/6] fix native deploy ssh assumptions --- .beads/issues.jsonl | 1 + ...05-19-harden-native-ssh-deploy-checks.html | 191 ++++++++++++++++++ scripts/deploy.ts | 14 ++ 3 files changed, 206 insertions(+) create mode 100644 docs/turns/2026-05-19-harden-native-ssh-deploy-checks.html diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 3df43cf..59c55f5 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -15,6 +15,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-fmg","title":"Fix native deploy SSH path and verification cwd assumptions","description":"Native deploys over SSH assumed bun was already on PATH and that remote verification would run from the repository root. On the live VPS, non-login SSH shells omitted /home/delta/.bun/bin and remote native verification could not find deployment/native/check-native-infra.sh because it ran from the home directory. Update the deploy helper to prepend /Users/kell/.bun/bin when present and cd into the repo before native verification checks run.","status":"open","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:38:32Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:38:32Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-wf5","title":"Harden native options provider configuration after synthetic recovery","description":"Native production recovery restored OPTIONS_INGEST_ADAPTER=synthetic because the current Alpaca setup fails authentication and crash-loops ingest-options. Follow up by deciding whether production options should remain synthetic or move to a supported live provider auth path, then add a deploy-time smoke test or config validation that catches provider auth failures before native cutover.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:27:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-m83","title":"Restore options ingestion and print generation on native deployment","description":"After moving the production/VPS deployment from Docker-managed services to the native runtime, the options feed appears behind and fresh option prints are not reaching the UI. Investigate the native deployment path on the server, identify the ingestion or compute breakage, apply the required code and/or host configuration changes, validate that fresh option prints resume, and document any follow-up operational work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:20:01Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:52Z","started_at":"2026-05-19T23:20:10Z","closed_at":"2026-05-19T23:27:52Z","close_reason":"Restored native options ingest by switching the VPS back to the last known-good synthetic adapter, verified fresh option prints and compute output, and documented the native env precedence gotcha.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-o1v","title":"Add SCM provider layer with Forgejo detection","description":"Implement provider-aware source-control detection and mirror-aware guardrails for repo automation so Forgejo remotes are treated as authoritative when present.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:04:33Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:06:55Z","started_at":"2026-05-19T23:04:35Z","closed_at":"2026-05-19T23:06:55Z","close_reason":"created by mistake during interrupted turn; no implementation was started","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/docs/turns/2026-05-19-harden-native-ssh-deploy-checks.html b/docs/turns/2026-05-19-harden-native-ssh-deploy-checks.html new file mode 100644 index 0000000..7cee829 --- /dev/null +++ b/docs/turns/2026-05-19-harden-native-ssh-deploy-checks.html @@ -0,0 +1,191 @@ + + + + + + 2026-05-19 Harden Native SSH Deploy Checks + + + +
    +
    +

    Harden Native SSH Deploy Checks

    +

    + Native deploys over SSH were failing for avoidable operator reasons: the remote shell did not inherit Bun's install path, and native verification assumed it was already running from the repository root before it called checked-in health scripts. This patch makes the SSH path more forgiving and fixes the verification working directory. +

    +
    Generated 2026-05-19 19:38 EDT
    +
    + +
    +

    Summary

    +

    + Updated scripts/deploy.ts so native SSH deploys prepend $HOME/.bun/bin when it exists, and native verification now explicitly cds into the remote repo before running the checked-in health helpers. +

    +
    + +
    +

    Changes Made

    +
      +
    • Prepended $HOME/.bun/bin during native remote precheck when available.
      • +
    • Prepended $HOME/.bun/bin during native remote rollout when available.
      • +
    • Changed native remote verification to run from /home/delta/islandflow before calling deployment/native/check-native-infra.sh.
      • +
    +
    + +
    +

    Context

    +

    + During a live native rollout, the deploy helper failed first because the non-login SSH shell could not find bun even though it was installed under the deploy user's home directory. After that was corrected on the host, worker rollout still reported failure because remote verification executed from the home directory and could not resolve the relative path to the checked-in infra check script. +

    +
    + +
    +

    Important Implementation Details

    +
      +
    • The fallback only adjusts PATH when $HOME/.bun/bin/bun exists, so it stays harmless on hosts that already expose Bun globally.
      • +
    • The repo-root cd keeps the existing relative helper calls intact instead of hardcoding every individual script path in multiple places.
      • +
    • This change improves SSH-based deploys without changing local-server deploy behavior.
      • +
    +
    + +
    +

    Relevant Diff Snippets

    +

    Unified diff blocks below are formatted for diffs-compatible rendering.

    + 
    diff --git a/scripts/deploy.ts b/scripts/deploy.ts
+@@ -754,6 +754,10 @@ set -euo pipefail
+ 
+ cd ${shellEscape(REMOTE_REPO)}
+ 
++if [[ -x "$HOME/.bun/bin/bun" ]]; then
++  export PATH="$HOME/.bun/bin:$PATH"
++fi
++
+ if ! command -v bun >/dev/null 2>&1; then
+
+@@ -855,6 +859,10 @@ set -euo pipefail
+ 
++if [[ -x "$HOME/.bun/bin/bun" ]]; then
++  export PATH="$HOME/.bun/bin:$PATH"
++fi
++
+ ${remoteGitUpdateScript(mode, remote, branch)}
+
+@@ -943,6 +951,12 @@ set -euo pipefail
+ 
++cd ${shellEscape(REMOTE_REPO)}
++
++if [[ -x "$HOME/.bun/bin/bun" ]]; then
++  export PATH="$HOME/.bun/bin:$PATH"
++fi
++
+ declare -a units=(${units})
    +
    + +
    +

    Expected Impact for End-Users

    +

    + End users should see fewer failed native deploy attempts and fewer partial restarts caused by tooling assumptions rather than application health. This lowers the odds of avoidable downtime during native rollouts. +

    +
    + +
    +

    Validation

    +
      +
    • Observed the original failures during live rollout: missing bun in SSH PATH and missing deployment/native/check-native-infra.sh during remote verification.
      • +
    • Used the patched operational path to complete native worker, API, and web rollouts successfully on the VPS.
      • +
    • Verified API health at http://127.0.0.1:4000/health and web health at both http://127.0.0.1:3000/ and https://flow.deltaisland.io.
      • +
    +
    + +
    +

    Issues, Limitations, and Mitigations

    +
      +
    • This patch does not solve the separate ingest-news credential problem. Full native deploys still need that unit and provider path to be made healthy before they are completely clean.
      • +
    • The VPS also needed a host-level Bun symlink during this recovery. The repo patch reduces dependence on that fix for future SSH deploys but does not remove it retroactively.
      • +
    +
    + +
    +

    Follow-up Work

    +
      +
    • islandflow-fmg: Keep the deploy helper aligned with the actual VPS runtime assumptions and add regression checks around native verification paths.
      • +
    • islandflow-wf5: Decide whether ingest-news and live options should stay provider-backed or remain intentionally synthetic until auth is hardened.
      • +
    +
    +
    + + diff --git a/scripts/deploy.ts b/scripts/deploy.ts index e703c49..169f7a9 100644 --- a/scripts/deploy.ts +++ b/scripts/deploy.ts @@ -754,6 +754,10 @@ set -euo pipefail cd ${shellEscape(REMOTE_REPO)} +if [[ -x "$HOME/.bun/bin/bun" ]]; then + export PATH="$HOME/.bun/bin:$PATH" +fi + if ! command -v bun >/dev/null 2>&1; then echo "Refusing native rollout: bun is not installed on the server." >&2 echo "The current supported VPS path remains --runtime docker." >&2 @@ -855,6 +859,10 @@ function remoteNativeRollout( `#!/usr/bin/env bash set -euo pipefail +if [[ -x "$HOME/.bun/bin/bun" ]]; then + export PATH="$HOME/.bun/bin:$PATH" +fi + ${remoteGitUpdateScript(mode, remote, branch)} cd ${shellEscape(REMOTE_REPO)} @@ -943,6 +951,12 @@ function remoteNativeVerification(scope: DeployScope, fast: boolean): void { `#!/usr/bin/env bash set -euo pipefail +cd ${shellEscape(REMOTE_REPO)} + +if [[ -x "$HOME/.bun/bin/bun" ]]; then + export PATH="$HOME/.bun/bin:$PATH" +fi + declare -a units=(${units}) for unit in "\${units[@]}"; do ${NATIVE_SYSTEMCTL_PREFIX} is-active --quiet "$unit" From e9739f5dc9a251407e933a42495b3fc33a9a39aa Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Tue, 19 May 2026 19:40:52 -0400 Subject: [PATCH 6/6] update beads for native deploy ssh fix --- .beads/issues.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 59c55f5..3ce8c65 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -15,7 +15,7 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"islandflow-fmg","title":"Fix native deploy SSH path and verification cwd assumptions","description":"Native deploys over SSH assumed bun was already on PATH and that remote verification would run from the repository root. On the live VPS, non-login SSH shells omitted /home/delta/.bun/bin and remote native verification could not find deployment/native/check-native-infra.sh because it ran from the home directory. Update the deploy helper to prepend /Users/kell/.bun/bin when present and cd into the repo before native verification checks run.","status":"open","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:38:32Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:38:32Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-fmg","title":"Fix native deploy SSH path and verification cwd assumptions","description":"Native deploys over SSH assumed bun was already on PATH and that remote verification would run from the repository root. On the live VPS, non-login SSH shells omitted /home/delta/.bun/bin and remote native verification could not find deployment/native/check-native-infra.sh because it ran from the home directory. Update the deploy helper to prepend /Users/kell/.bun/bin when present and cd into the repo before native verification checks run.","status":"closed","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:38:32Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:40:33Z","closed_at":"2026-05-19T23:40:33Z","close_reason":"Updated native SSH deploy flow to prepend Bun's home install path when present and run native verification from the repo root before health scripts.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-wf5","title":"Harden native options provider configuration after synthetic recovery","description":"Native production recovery restored OPTIONS_INGEST_ADAPTER=synthetic because the current Alpaca setup fails authentication and crash-loops ingest-options. Follow up by deciding whether production options should remain synthetic or move to a supported live provider auth path, then add a deploy-time smoke test or config validation that catches provider auth failures before native cutover.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:27:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-m83","title":"Restore options ingestion and print generation on native deployment","description":"After moving the production/VPS deployment from Docker-managed services to the native runtime, the options feed appears behind and fresh option prints are not reaching the UI. Investigate the native deployment path on the server, identify the ingestion or compute breakage, apply the required code and/or host configuration changes, validate that fresh option prints resume, and document any follow-up operational work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:20:01Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:52Z","started_at":"2026-05-19T23:20:10Z","closed_at":"2026-05-19T23:27:52Z","close_reason":"Restored native options ingest by switching the VPS back to the last known-good synthetic adapter, verified fresh option prints and compute output, and documented the native env precedence gotcha.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-o1v","title":"Add SCM provider layer with Forgejo detection","description":"Implement provider-aware source-control detection and mirror-aware guardrails for repo automation so Forgejo remotes are treated as authoritative when present.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:04:33Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:06:55Z","started_at":"2026-05-19T23:04:35Z","closed_at":"2026-05-19T23:06:55Z","close_reason":"created by mistake during interrupted turn; no implementation was started","dependency_count":0,"dependent_count":0,"comment_count":0}