diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index e025c4d..8653c73 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,6 @@ +{"_type":"issue","id":"islandflow-yza","title":"Persist historical flow packets for alert detail replay","description":"## Why\nAlert details can show a missing persisted flow packet when the packet is no longer present in the Redis hot cache, even though the associated historical alert and evidence were loaded from ClickHouse.\n\n## What needs to be done\nTrace the API path that resolves alert detail flow packets, compare Redis hot-cache lookups with ClickHouse historical fetches, and ensure historical flow packet payloads are treated as first-class persisted data with context preserved when replaying or loading older alerts.\n\n## Acceptance Criteria\n- Alert detail flow packets load for historical alerts even when the packet is absent from Redis hot cache\n- Historical ClickHouse-backed flow packet responses preserve the context required by the UI\n- Relevant automated tests cover the regression or the gap is explicitly documented","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-20T06:52:04Z","created_by":"dirtydishes","updated_at":"2026-05-20T06:59:26Z","started_at":"2026-05-20T06:52:09Z","closed_at":"2026-05-20T06:59:26Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-jor","title":"Support Forgejo pull request status in desktop git panel","description":"The desktop app currently reports pull request status unavailable when a repository only has a Forgejo remote. Add native Forgejo/Gitea-style remote detection and pull request status lookup so Forgejo-only repositories can show PR state in the Codex app git panel.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T20:55:15Z","created_by":"dirtydishes","updated_at":"2026-05-19T20:59:46Z","started_at":"2026-05-19T20:55:25Z","closed_at":"2026-05-19T20:59:46Z","close_reason":"Patched the installed Codex desktop app bundle with a Forgejo PR status fallback and documented the local change.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-g3a","title":"Reconcile PR merge conflicts","description":"Resolve the current pull request conflicts for the nextjs-upgrade branch, validate the result, document the turn, and push the reconciled branch.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:44:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:47:35Z","started_at":"2026-05-19T18:44:56Z","closed_at":"2026-05-19T18:47:35Z","close_reason":"Merged forgejo/main into nextjs-upgrade, resolved README and Beads conflicts, updated JetStream retention tests, validated deploy help, Docker workspace sync, API/bus tests, and web build, and added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-jbi","title":"Hydrate alert evidence details from ClickHouse","description":"Alert detail drawers need to fetch persisted alert context from ClickHouse by trace id, including linked flow packets, option prints, preserved execution context, and explicit missing refs for UI diagnostics.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-17T14:55:43Z","created_by":"dirtydishes","updated_at":"2026-05-17T15:01:58Z","started_at":"2026-05-17T14:55:53Z","closed_at":"2026-05-17T15:01:58Z","close_reason":"Implemented ClickHouse-backed alert context hydration across storage, API, terminal drawer, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-8kj","title":"Configure persistent beads Dolt remote on deltaisland server","description":"Install the beads and Dolt CLIs on the server, configure a persistent Dolt sync remote backed by the server-hosted Forgejo repository, verify refs/dolt/data publication, and document Nginx Proxy Manager / firewall considerations.","status":"closed","priority":1,"issue_type":"task","assignee":"delta","created_at":"2026-05-17T10:31:31Z","created_by":"delta","updated_at":"2026-05-17T10:37:47Z","started_at":"2026-05-17T10:32:16Z","closed_at":"2026-05-17T10:37:47Z","close_reason":"Installed bd and dolt on the server, configured the Forgejo-backed Dolt remote, published refs/dolt/data, and documented the setup.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-200","title":"Implement durable options tape history","description":"Implement the plan from docs/plans/2026-05-16-1711-durable-options-tape-history.html: durable ClickHouse-backed options history, signal/all prints view selection, preserved execution context, stale semantics limited to live health, reset runbook, tests, and turn documentation.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-16T21:21:30Z","created_by":"dirtydishes","updated_at":"2026-05-16T21:26:51Z","started_at":"2026-05-16T21:21:33Z","closed_at":"2026-05-16T21:26:51Z","close_reason":"Implemented durable options tape history, signal/raw view selection, reset runbook, tests, and turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -13,6 +16,19 @@ {"_type":"issue","id":"islandflow-ayo","title":"Drop stale backlog events from live fanout","description":"Follow-up to live freshness rollout: /ws/live was still fanning out stale backlog events for freshness-gated channels, which kept tape panes in Live feed behind despite active synthetic ingest. Gate fanout and cache ingest by freshness for options/nbbo/equities/flow.","status":"closed","priority":1,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:26:39Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:26:44Z","started_at":"2026-04-28T21:26:44Z","closed_at":"2026-04-28T21:26:44Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-0v6","title":"Fix tape freshness, NBBO coverage, pause controls, and filter popup","description":"Implement the tape fixes requested for synthetic options notional sizing, strict live freshness, live-mode pause/resume behavior, stronger NBBO snapshot coverage, and moving flow filters behind a popup. Includes server-side live cache changes, web terminal state/UI changes, and tests for synthetic pricing, live snapshot freshness/NBBO retention, and live pause/filter interactions.","status":"closed","priority":1,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T21:02:52Z","created_by":"dirtydishes","updated_at":"2026-04-28T21:13:38Z","started_at":"2026-04-28T21:02:57Z","closed_at":"2026-04-28T21:13:38Z","close_reason":"Completed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-e4r","title":"Implement smart-money flow filtering and synthetic firehose modes","description":"Implement the approved multi-surface plan for named synthetic market profiles, options raw-vs-signal filtering, live/API filter contracts, Tape page client-side flow filters, firehose-readiness improvements, tests, and README updates.","status":"closed","priority":1,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:10:49Z","created_by":"dirtydishes","updated_at":"2026-04-28T20:29:29Z","started_at":"2026-04-28T20:10:53Z","closed_at":"2026-04-28T20:29:29Z","close_reason":"Implemented synthetic market profiles, options signal-path filtering, signal-aware API/replay contracts, Tape page filters, tests, and README updates. Follow-up tracked in islandflow-biq.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-6tn","title":"Add Codex desktop login and usage bridge","description":"Implement a desktop-only Codex integration for the Islandflow Electron app using the official codex app-server with managed ChatGPT login, native IPC, settings UI, usage tracking, and clean web degradation.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-20T14:01:36Z","created_by":"dirtydishes","updated_at":"2026-05-20T14:40:49Z","started_at":"2026-05-20T14:01:48Z","closed_at":"2026-05-20T14:40:49Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-laq","title":"fix native alpaca news deploy and auth","description":"Why this issue exists and what needs to be done:\\n\\nNative Islandflow rollout is incomplete because services/ingest-news is not healthy on the VPS. The checked-in native user units and helper scripts do not fully include ingest-news, and the current service uses bearer-style auth that returns 401 against Alpaca news endpoints.\\n\\nThis task should verify the current Alpaca news auth requirements against official docs, update the repo code and native deployment assets as needed, install and enable the missing VPS unit, verify news events flow end-to-end, and document the work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:47:07Z","created_by":"dirtydishes","updated_at":"2026-05-20T00:05:20Z","started_at":"2026-05-19T23:47:12Z","closed_at":"2026-05-20T00:05:20Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-fmg","title":"Fix native deploy SSH path and verification cwd assumptions","description":"Native deploys over SSH assumed bun was already on PATH and that remote verification would run from the repository root. On the live VPS, non-login SSH shells omitted /home/delta/.bun/bin and remote native verification could not find deployment/native/check-native-infra.sh because it ran from the home directory. Update the deploy helper to prepend /Users/kell/.bun/bin when present and cd into the repo before native verification checks run.","status":"closed","priority":2,"issue_type":"bug","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:38:32Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:40:33Z","closed_at":"2026-05-19T23:40:33Z","close_reason":"Updated native SSH deploy flow to prepend Bun's home install path when present and run native verification from the repo root before health scripts.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-wf5","title":"Harden native options provider configuration after synthetic recovery","description":"Native production recovery restored OPTIONS_INGEST_ADAPTER=synthetic because the current Alpaca setup fails authentication and crash-loops ingest-options. Follow up by deciding whether production options should remain synthetic or move to a supported live provider auth path, then add a deploy-time smoke test or config validation that catches provider auth failures before native cutover.","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:27:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:51Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-m83","title":"Restore options ingestion and print generation on native deployment","description":"After moving the production/VPS deployment from Docker-managed services to the native runtime, the options feed appears behind and fresh option prints are not reaching the UI. Investigate the native deployment path on the server, identify the ingestion or compute breakage, apply the required code and/or host configuration changes, validate that fresh option prints resume, and document any follow-up operational work.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:20:01Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:27:52Z","started_at":"2026-05-19T23:20:10Z","closed_at":"2026-05-19T23:27:52Z","close_reason":"Restored native options ingest by switching the VPS back to the last known-good synthetic adapter, verified fresh option prints and compute output, and documented the native env precedence gotcha.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-o1v","title":"Add SCM provider layer with Forgejo detection","description":"Implement provider-aware source-control detection and mirror-aware guardrails for repo automation so Forgejo remotes are treated as authoritative when present.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T23:04:33Z","created_by":"dirtydishes","updated_at":"2026-05-19T23:06:55Z","started_at":"2026-05-19T23:04:35Z","closed_at":"2026-05-19T23:06:55Z","close_reason":"created by mistake during interrupted turn; no implementation was started","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-tqk","title":"publish docs/ to github pages with navigable index","description":"Set up docs deployment so repository docs are published to dirtydishes.github.io/islandflow/docs with a nicer, browsable experience than a raw file listing.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:56:02Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:59:55Z","started_at":"2026-05-19T18:56:04Z","closed_at":"2026-05-19T18:59:55Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-lm6","title":"Clarify repo turn documentation scope","description":"Update AGENTS.md so repository turn documentation clearly uses repo-local docs/turns and impeccable styling, without inheriting global non-repo computer-task styling.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T12:05:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T12:06:12Z","started_at":"2026-05-19T12:05:14Z","closed_at":"2026-05-19T12:06:12Z","close_reason":"Verified AGENTS.md now scopes repo turn docs to docs/turns and makes impeccable the styling authority; added turn documentation.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-6iq","title":"Update README for current project state","description":"Resolve README merge conflicts and document the current project state, including the smart money classification taxonomy, Next.js update, and deployment workflow changes.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:37:24Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:40:01Z","started_at":"2026-05-19T11:37:31Z","closed_at":"2026-05-19T11:40:01Z","close_reason":"README conflict resolved and current project state documented, including smart-money taxonomy, Next.js update, and deployment workflow.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-lib","title":"Upgrade apps/web to Next.js 16.2.6","description":"Upgrade the web app dependency stack to Next.js 16.2.6 with React 19, refresh Bun and mirrored Docker workspace lockfiles, keep runtime behavior unchanged, fix any focused web test fallout, validate the web build and targeted route tests, and document the completed work.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T11:04:51Z","created_by":"dirtydishes","updated_at":"2026-05-19T11:31:23Z","started_at":"2026-05-19T11:04:57Z","closed_at":"2026-05-19T11:31:23Z","close_reason":"Upgraded apps/web to Next.js 16.2.6 with React 19, refreshed Bun lockfiles including the Docker workspace mirror, fixed the React 19 nullable ref type issue, and validated the web build, focused tests, Docker workspace sync, and route smoke checks.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-8fn","title":"implement alpaca-backed news wire view","description":"Why this issue exists and what needs to be done:\\nAdd an Alpaca-powered live news pipeline, API, storage, and web experience, including a dedicated /news route, Home preview, live fanout, history pagination, ticker resolution, and replay-mode live-only empty states.\\n\\nAcceptance criteria:\\n- normalized NewsStory contract and live channel exist\\n- ingest-news service backfills and streams Alpaca news\\n- API persists, serves, and fans out news\\n- web app exposes /news plus Home preview and drawer\\n- tests cover types, storage, API, and key UI behaviors\\n- turn documentation is added\\n\\nDesign:\\nReuse Islandflow drawer, chips, panes, and terminal styling; keep news live-only in v1 replay mode.\\n\\nNotes:\\nImplement client-side ticker filtering in v1 and expose latest revision only per provider+story_id.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T20:37:13Z","created_by":"dirtydishes","updated_at":"2026-05-18T20:55:11Z","started_at":"2026-05-18T20:37:20Z","closed_at":"2026-05-18T20:55:11Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-k8i","title":"Fix duplicate alert context import in API entrypoint","description":"Recent alert-context work introduced a duplicate fetchAlertContextByTraceId import in services/api/src/index.ts, which risks breaking TypeScript compilation and API startup. Remove the duplicate import and validate the affected API/web tests.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:58Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:03:40Z","started_at":"2026-05-18T13:02:02Z","closed_at":"2026-05-18T13:03:40Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-lk9","title":"Fix PR creation workflow after Forgejo migration","description":"## Why\\nCreating pull requests with fails after the repository moved primary collaboration from GitHub to Forgejo. The current workflow still assumes GitHub GraphQL PR creation semantics, which do not work against the Forgejo remote.\\n\\n## What\\nInvestigate the current PR creation path, identify remaining GitHub-specific assumptions, and update the repo workflow/scripts/docs so contributors can reliably publish branches and open PRs in the Forgejo-based setup.\\n\\n## Acceptance Criteria\\n- The repo no longer instructs contributors to use a broken GitHub-specific PR creation path for Forgejo branches\\n- There is a documented and preferably scripted way to create the equivalent review request against Forgejo\\n- Validation demonstrates the new workflow behaves correctly or clearly documents any remaining platform limitation","status":"in_progress","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T10:26:47Z","created_by":"dirtydishes","updated_at":"2026-05-18T10:26:53Z","started_at":"2026-05-18T10:26:53Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-1ei","title":"Make deploy helper remote-aware for Forgejo","description":"Why: scripts/deploy.ts hardcodes git remote name origin for fetch/pull/push and branch verification, but this repository now uses forgejo/github remotes and may not have an origin remote. What: update deploy.ts to resolve the deploy git remote robustly (Forgejo-aware), use it across local prechecks, branch publish, and remote rollout git operations, and keep behavior explicit in output.","status":"closed","priority":2,"issue_type":"bug","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T03:20:12Z","created_by":"dirtydishes","updated_at":"2026-05-18T03:22:39Z","started_at":"2026-05-18T03:20:16Z","closed_at":"2026-05-18T03:22:39Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-xod","title":"Add --fast mode to deploy helper","description":"Why: full main deploys rebuild all images and run full verification, which is slow for routine rollouts. What: add a --fast flag to scripts/deploy.ts with explicit behavior that short-circuits slow steps while preserving basic safety checks; update help text/docs for discoverability.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T02:50:47Z","created_by":"dirtydishes","updated_at":"2026-05-18T02:53:41Z","started_at":"2026-05-18T02:50:50Z","closed_at":"2026-05-18T02:53:41Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-cif","title":"hydrate alert evidence context from clickhouse","description":"Implement alert detail hydration from ClickHouse with a new context endpoint and frontend drawer evidence resolution. Includes storage lookup by alert trace_id/evidence refs, unresolved refs diagnostics, API route GET /flow/alerts/:trace_id/context, terminal evidence hydration + loading states/copy updates, and tests across storage/api/web.","status":"closed","priority":2,"issue_type":"feature","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T00:15:55Z","created_by":"dirtydishes","updated_at":"2026-05-18T00:17:38Z","started_at":"2026-05-18T00:16:00Z","closed_at":"2026-05-18T00:17:38Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -46,6 +62,10 @@ {"_type":"issue","id":"islandflow-zs0","title":"Migrate terminal UI to smart-money profiles","description":"Migrate apps/web terminal rendering to consume SmartMoneyEvent directly: primary profile, probability ladder, reason codes, and suppression/abstention state, while preserving legacy alert/classifier displays during the bridge.","status":"closed","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-04T21:35:23Z","created_by":"dirtydishes","updated_at":"2026-05-05T05:39:58Z","closed_at":"2026-05-05T05:39:58Z","close_reason":"Completed terminal smart-money profile migration","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-igk","title":"Add plan mode","description":"Implement a user-facing plan mode in the application so users can switch into planning before taking action. Scope to be clarified from existing app patterns.","status":"closed","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-05-04T04:22:37Z","created_by":"dirtydishes","updated_at":"2026-05-04T04:26:18Z","started_at":"2026-05-04T04:22:40Z","closed_at":"2026-05-04T04:26:18Z","close_reason":"Implemented as a global pi extension toggled with Shift+P","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-biq","title":"Finish raw live options delivery and filter/backpressure observability","description":"The smart-money signal path and Tape filters are in place, but the next firehose pass should finish server-side selective raw live delivery for options subscriptions and add explicit filtered-out/backpressure observability for API/web counters. This was discovered while landing islandflow-e4r.\n","status":"in_progress","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-04-28T20:28:58Z","created_by":"dirtydishes","updated_at":"2026-04-29T03:54:12Z","started_at":"2026-04-29T03:54:12Z","dependencies":[{"issue_id":"islandflow-biq","depends_on_id":"islandflow-e4r","type":"discovered-from","created_at":"2026-04-28T16:28:58Z","created_by":"auto-import","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-8vr","title":"Summarize 2026-05-19 git activity for standup","description":"Create the daily git summary for 2026-05-19 in docs/general using yesterday's commits, touched files, and validation evidence only.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-20T13:02:41Z","created_by":"dirtydishes","updated_at":"2026-05-20T13:04:50Z","started_at":"2026-05-20T13:02:47Z","closed_at":"2026-05-20T13:04:50Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-0ty","title":"Recreate May 18 standup summary after merge","description":"Regenerate docs/daily-git/2026-05-19-standup-summary-2026-05-18.html using merged history so it reflects all commits in the May 18 window, including native deployment and merge commits.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:53:48Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:55:33Z","started_at":"2026-05-19T18:53:52Z","closed_at":"2026-05-19T18:55:33Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-2df","title":"Publish 2026-05-18 git standup summary","description":"Why: the daily automation needs a grounded standup summary for May 18, 2026. What: review commits from 2026-05-18, create a scannable HTML summary in docs/daily-git, and capture only commit/file-backed statements.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-19T18:41:07Z","created_by":"dirtydishes","updated_at":"2026-05-19T18:42:42Z","started_at":"2026-05-19T18:41:10Z","closed_at":"2026-05-19T18:42:42Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-x70","title":"Create 2026-05-17 git standup summary","description":"Why this issue exists and what needs to be done:\\n- Produce the daily automation summary for 2026-05-17 git activity.\\n- Ground statements in commits, PRs, and touched files only.\\n- Create a user-readable HTML document in docs/general and update automation memory.\\n- Complete the Beads sync and git push workflow after documenting the run.","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-18T13:01:43Z","created_by":"dirtydishes","updated_at":"2026-05-18T13:05:37Z","started_at":"2026-05-18T13:01:53Z","closed_at":"2026-05-18T13:05:37Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-zsy","title":"Expose Forgejo SSH on a direct DNS hostname","description":"git.deltaisland.io currently resolves through Cloudflare's proxy, so SSH on port 2222 does not complete even though the Forgejo container is listening on the host. If SSH-based git/beads workflows are desired, add a DNS-only hostname (or adjust the existing record) that points directly at the server for Forgejo SSH.","status":"open","priority":3,"issue_type":"task","created_at":"2026-05-17T10:34:06Z","created_by":"delta","updated_at":"2026-05-17T10:34:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-38p","title":"Add native deployment unit templates and rollback helpers","description":"The deploy helper now supports --runtime native, but the repo still relies on operator-managed systemd units and manual rollback. Add checked-in native deployment templates or provisioning guidance for the expected units, and consider lightweight rollback/smoke-test helpers once the host-native path is exercised on the real VPS.","status":"open","priority":3,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:46:42Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:46:42Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-575","title":"Document smart-money event calendar env","description":"Document smart-money event-calendar environment configuration in env examples and README.\n","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T06:57:14Z","created_by":"dirtydishes","updated_at":"2026-05-05T06:57:57Z","started_at":"2026-05-05T06:57:17Z","closed_at":"2026-05-05T06:57:57Z","close_reason":"Documented event-calendar env variables","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/.env.example b/.env.example index d42f715..be20b62 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,10 @@ REDIS_URL=redis://127.0.0.1:6379 # Options ingest OPTIONS_INGEST_ADAPTER=synthetic ALPACA_API_KEY= +ALPACA_API_KEY_ID= +ALPACA_KEY_ID= +ALPACA_API_SECRET_KEY= +ALPACA_SECRET_KEY= ALPACA_REST_URL=https://data.alpaca.markets ALPACA_WS_BASE_URL=wss://stream.data.alpaca.markets/v1beta1 ALPACA_FEED=indicative diff --git a/.github/workflows/docs-pages.yml b/.github/workflows/docs-pages.yml new file mode 100644 index 0000000..9c4db98 --- /dev/null +++ b/.github/workflows/docs-pages.yml @@ -0,0 +1,56 @@ +name: Publish Docs + +on: + push: + branches: + - main + paths: + - "docs/**" + - "scripts/generate-docs-index.mjs" + - ".github/workflows/docs-pages.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure Pages + uses: actions/configure-pages@v5 + + - name: Build docs index + run: node scripts/generate-docs-index.mjs + + - name: Prepare static site payload + run: | + mkdir -p site/docs + cp -R docs/. site/docs/ + printf '%s\n' 'Islandflow DocsContinue to docs' > site/index.html + touch site/.nojekyll + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + needs: build + runs-on: ubuntu-latest + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/AGENTS server.md b/AGENTS server.md new file mode 100644 index 0000000..08a484a --- /dev/null +++ b/AGENTS server.md @@ -0,0 +1,174 @@ + +## Beads Issue Tracker + +This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands. + +### Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work +bd close # Complete work +``` + +### Rules + +- Use `bd` for ALL task tracking — do NOT use TodoWrite, TaskCreate, or markdown TODO lists +- Run `bd prime` for detailed command reference and session close protocol +- Use `bd remember` for persistent knowledge — do NOT use MEMORY.md files + +## Session Completion + +**When ending a work session**, you MUST complete ALL steps below. Work is NOT complete until `git push` succeeds. + +**MANDATORY WORKFLOW:** + +1. **File issues for remaining work** - Create issues for anything that needs follow-up +2. **Run quality gates** (if code changed) - Tests, linters, builds +3. **Update issue status** - Close finished work, update in-progress items +4. **PUSH TO REMOTE** - This is MANDATORY: + ```bash + git pull --rebase + bd dolt push + git push + git status # MUST show "up to date with origin" + ``` +5. **Clean up** - Clear stashes, prune remote branches +6. **Verify** - All changes committed AND pushed +7. **Hand off** - Provide context for next session + +**CRITICAL RULES:** +- Work is NOT complete until `git push` succeeds +- NEVER stop before pushing - that leaves work stranded locally +- NEVER say "ready to push when you are" - YOU must push +- If push fails, resolve and retry until it succeeds + + +## Minimal Repo Operating Instructions + +This is a Bun + TypeScript monorepo for an event-sourced market-data pipeline: +- Flow: ingest services publish to NATS/JetStream, compute/candles derive events, API serves REST/WS, web consumes live/replay streams. +- Main folders: `services/*` (runtime services), `packages/*` (shared libs/types/storage), `apps/web` (Next.js UI). +- Infra dependency: local dev assumes Docker services (NATS, ClickHouse, Redis) are available. + +Use these repo-specific commands: +- Install deps: `bun install` +- Start full stack: `bun run dev` +- Start infra only: `bun run dev:infra` +- Start backend services only: `bun run dev:services` +- Start web only: `bun run dev:web` + +Testing and validation in this repo are Bun-first: +- Run tests: `bun test` +- Run scoped tests: `bun test services/compute/tests` (or another package/service path) +- Validate web production build when UI code changes: `bun --cwd=apps/web run build` + +Working style that avoids common problems here: +- Prefer editing in the touched workspace (`services/`, `packages/`, `apps/web`) and keep shared contract changes in `packages/types`. +- Keep `.env` aligned with `.env.example`; adapters default to synthetic modes for local development. +- Dev runners persist child PID state in `.tmp/`; if a previous run crashed, restart via the standard `bun run dev*` commands so stale processes are cleaned up. + +## Required Turn Documentation + +At the end of every completed implementation task, before final handoff, create a user-readable HTML document describing the work. + +This documentation is mandatory whenever code, configuration, tests, or project files were changed. + +### Location + +Save the document in: + +```text +docs/turns/ +``` +## Important: If you are not working inside a git repository, save the document to `~/dev/docs/turns/` + +Use a clear timestamped filename: + +```text +docs/turns/YYYY-MM-DD-short-task-name.html +``` + +Example: + +```text +docs/turns/2026-05-14-add-market-replay-controls.html +``` + +### Format + +Use the impeccable skill to structure the document as clean, readable HTML. + +If the impeccable skill is unavailable, still create a well-structured standalone HTML file with: + +- A concise summary at the top +- A detailed explanation of what changed +- Relevant context or background +- Specific code snippets or examples when helpful +- Issues, limitations, tradeoffs, or mitigations +- Validation performed, including tests, builds, linters, or manual checks +- Any remaining follow-up work, with corresponding Beads issue IDs when applicable + +### Required Sections + +Each turn document must include these sections: + +1. **Summary** +2. **Changes Made** +3. **Context** +4. **Important Implementation Details** +5. **Relevant Diff Snippets** +6. **Expected Impact for End-Users** +7. **Validation** +8. **Issues, Limitations, and Mitigations** +9. **Follow-up Work** + +### Completion Rule + +A task is not complete until: + +1. The Beads workflow is updated +2. The turn document is created in `docs/turns` +3. Relevant quality gates have passed or failures are documented +4. Changes are committed +5. `bd dolt push` succeeds +6. `git push` succeeds +7. `git status` shows the branch is up to date with origin + +For trivial changes, the document may be brief, but it must still exist and clearly explain what changed and how it was validated. + +## Plan Mode Documentation + +When working in plan mode, do not modify implementation files. + +At the end of plan mode, provide a concise summary of the plan and ask the user whether they want to proceed with implementation. + +If the user asks to save the plan, create a user-readable HTML plan document in: + +```text +docs/plans/ +``` + +Use a clear timestamped filename: + +```text +docs/plans/YYYY-MM-DD-short-plan-name.html +``` + +The plan document should be labeled clearly as a plan and should include: + +1. **Plan Summary** +2. **Goals** +3. **Proposed Changes** +4. **Relevant Context** +5. **Implementation Steps** +6. **Risks, Limitations, and Mitigations** +7. **Open Questions** + +Always do the following when you finish a task, finish the beads workflow and and make a commit: +- Document the changes in a user-readable format +- Use the impeccable skill to structure the document as HTML +- Create a clear, concise summary of the changes at the top, followed by a detailed description of the changes, including any relevant context or background as well as specific code snippets or examples. +- Note any relevant issues or limitations that were addressed or mitigated by the changes. +- The HTML file should be stored in the `docs/turns` directory. It should include the current date and time, as well as a brief explanation of changes. e.g. docs/turns/YYYY-MM-DD-{description}.html diff --git a/AGENTS.md b/AGENTS.md index 3ab1cf0..fe8ffca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -97,9 +97,11 @@ docs/turns/2026-05-14-add-market-replay-controls.html ### Format -Use the impeccable skill to structure the document as clean, readable HTML. +Use the `impeccable` skill to structure and style the document as clean, readable HTML. -If the impeccable skill is unavailable, still create a well-structured standalone HTML file with: +For this repository, `impeccable` is the styling and layout authority for turn documents when available. Do not apply global non-repo computer-task house styling to repository turn documents. + +If the `impeccable` skill is unavailable or blocked by an actual tool/file error, still create a well-structured standalone HTML file with: - A concise summary at the top - A detailed explanation of what changed @@ -117,10 +119,11 @@ Each turn document must include these sections: 2. **Changes Made** 3. **Context** 4. **Important Implementation Details** -5. **Expected Impact for End-Users** -5. **Validation** -6. **Issues, Limitations, and Mitigations** -7. **Follow-up Work** +5. **Relevant Diff Snippets** +6. **Expected Impact for End-Users** +7. **Validation** +8. **Issues, Limitations, and Mitigations** +9. **Follow-up Work** ### Completion Rule diff --git a/README.md b/README.md index 50063d9..6b3b7fc 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,12 @@ > **Pre-alpha warning** This project is in an early pre-alpha state. It will not perform consistently or as expected, and APIs, behavior, and data contracts may change without notice. -This repository contains a Bun + TypeScript monorepo for a personal-use, event-sourced market microstructure research platform focused on: +Islandflow is a Bun + TypeScript monorepo for a personal-use, event-sourced market microstructure research platform focused on: - options prints + NBBO, - off-exchange equity prints, -- explainable rule-based flow classification, +- market news context, +- explainable smart-money flow classification, - deterministic replay, - evidence-linked UI inspection. @@ -19,124 +20,176 @@ This repository contains a Bun + TypeScript monorepo for a personal-use, event-s Implemented now: - Bun workspaces with shared packages for schemas, bus, config, observability, and ClickHouse access. -- Infra orchestration via Docker Compose (NATS JetStream, ClickHouse, Redis). -- Options ingest service with adapters: - - synthetic stream, - - Alpaca options (dev-focused, bounded contracts), - - IBKR bridge (Python sidecar), - - Databento historical replay adapter (Python sidecar). -- Equities ingest service with adapters: - - synthetic stream, - - Alpaca equities trades/quotes. -- Compute service: - - deterministic option print clustering into `FlowPacket`s, - - NBBO join quality features and aggressor-mix metrics, - - rolling baselines in Redis, - - structure summarization and structure packet emission, - - rule-based classifiers + confidence-scored alert events, - - dark-style inferred events from equity prints/quotes, - - equity print-to-quote join events. -- Candles service: - - server-side equity candle aggregation, - - ClickHouse persistence, - - optional Redis hot cache, - - NATS publication. -- Replay service: - - deterministic republishing from ClickHouse to NATS, - - multi-stream merge with stable tie-break ordering, - - speed/start/end controls. -- API service: - - REST endpoints for recent + cursor pagination, - - REST range endpoints for chart windows, - - REST replay-oriented endpoints, - - WebSocket channels for options, NBBO, equities, quotes, joins, flow, classifier hits, alerts, inferred dark, and candles. -- Next.js web app: - - live tape/workspace views, - - replay controls and status, - - signals and chart-focused routes, - - evidence-centric terminal UI. -- Refdata + EOD enricher service entrypoints are present but currently scaffolds (lifecycle/logging only). +- Infra orchestration via Docker Compose for local NATS JetStream, ClickHouse, and Redis. +- Options ingest service with synthetic, Alpaca options, IBKR bridge, and Databento historical replay adapters. +- Equities ingest service with synthetic and Alpaca equities trades/quotes adapters. +- News ingest service for Alpaca news backfill and websocket publication. +- Compute service for deterministic parent-event reconstruction, flow packets, NBBO quality features, rolling baselines, smart-money profile scoring, compatibility classifier hits, alerts, inferred dark-style events, and equity print-to-quote joins. +- Candles service for server-side equity candle aggregation, ClickHouse persistence, optional Redis hot cache, and NATS publication. +- Replay service for deterministic ClickHouse-to-NATS republishing with multi-stream merge, stable tie-break ordering, speed, start, and end controls. +- API service with REST endpoints, cursor pagination, replay/history endpoints, live hot-cache hydration, and WebSocket channels for options, NBBO, equities, quotes, joins, flow, classifier hits, alerts, smart-money events, inferred dark, candles, and news. +- Next.js web app upgraded to Next.js `16.2.6`, React `19.2.0`, and React DOM `19.2.0`. +- Evidence-centric terminal UI, live/replay controls, chart-focused routes, news view, profile-aware smart-money display, and alert-context hydration. +- Thin Electron desktop shell in `apps/desktop` that can wrap the hosted app or local web UI. +- Refdata + EOD enricher service entrypoints are present, with refdata able to validate or refresh the event-calendar cache. Planned / not yet complete: - production-grade licensed feed integrations and entitlement workflow, - richer refdata/corp-action enrichment, - secure deployment/auth hardening, -- deeper structure + calibration workflows from `PLAN.md`. +- native deployment unit templates and rollback helpers, +- signed/notarized desktop distribution and richer desktop-native features, +- deeper calibration workflows from `PLAN.md` and `SMART_MONEY_REBUILD_PLAN.md`. ## Core Principles -- **Explainability first** — inferred outputs are evidence-backed and human-readable. -- **Event sourcing** — raw and derived events persist to support replay. -- **Determinism** — replay behavior tracks live pipeline logic. -- **Microstructure awareness** — bounded joins, confidence scoring, and explicit uncertainty. -- **Bun-first tooling** — runtime/package/scripts all use Bun. +- **Explainability first**: inferred outputs are evidence-backed and human-readable. +- **Event sourcing**: raw and derived events persist to support replay. +- **Determinism**: replay behavior tracks live pipeline logic. +- **Microstructure awareness**: bounded joins, confidence scoring, and explicit uncertainty. +- **Taxonomy over folklore**: "smart money" is modeled as participant-style hypotheses, not a single binary label. +- **Bun-first tooling**: runtime, package management, scripts, and tests use Bun. + +## Smart-Money Classification Taxonomy + +Islandflow now emits first-class `SmartMoneyEvent` records instead of treating old classifier hits as the final semantic object. `FlowPacket` remains the clustering bridge, while smart-money events carry typed features, profile scores, confidence bands, directions, reason codes, abstention state, and suppression reasons. + +Public profile IDs: + +| Profile ID | Meaning | Common evidence | +| --- | --- | --- | +| `institutional_directional` | Large directional parent flow with stronger institutional-style conviction. | premium, size, sweep/burst behavior, aggressor imbalance, quote quality, not short-dated retail-chase context | +| `retail_whale` | Large retail-style speculative bursts, often short-dated or attention-driven. | short-dated OTM concentration, burst prints, IV shock, lower premium than institutional blocks | +| `event_driven` | Flow aligned to known upcoming events. | event-calendar proximity, expiry after event, pre-event concentration, spread/IV pressure | +| `vol_seller` | Premium-selling or short-volatility structure evidence. | sell-side premium, straddles/strangles, neutral direction | +| `arbitrage` | Multi-leg or symmetric structures with low directional exposure. | matched leg symmetry, same-size legs, near-flat directional bias | +| `hedge_reactive` | Hedge or dealer-reaction style flow around short-dated ATM/gamma context. | 0-2 DTE, near-ATM contracts, underlying move linkage, size | + +Compatibility surfaces remain in place: + +- `ClassifierHitEvent` is derived from `SmartMoneyEvent.primary_profile_id`. +- `AlertEvent` may include `primary_profile_id` and `profile_scores`. +- Legacy classifier and alert endpoints still work. + +Primary smart-money access paths: + +```text +/flow/smart-money +/history/smart-money +/replay/smart-money +/ws/smart-money +``` + +The classifier intentionally abstains when evidence is weak or quote context is stale/missing. Suppression guards cover stale quotes, complex/special prints, retail-frenzy directional confusion, hedge-reactive short-dated ATM contexts, and arbitrage symmetry. ## Monorepo Layout - `apps/web` — Next.js UI shell/routes. -- `apps/desktop` — Electron desktop shell that loads the hosted Islandflow app. +- `apps/desktop` — Electron desktop shell that loads the hosted or local Islandflow app. - `services/ingest-options` — options print/NBBO ingest adapters. - `services/ingest-equities` — equity print/quote ingest adapters. -- `services/compute` — clustering, structures, classifiers, alerts, inferred dark. +- `services/ingest-news` — Alpaca news backfill and websocket ingest. +- `services/compute` — parent-event reconstruction, flow packets, smart-money scoring, alerts, inferred dark. - `services/candles` — server-side candle aggregation + cache. -- `services/replay` — ClickHouse → NATS replay streamer. +- `services/replay` — ClickHouse to NATS replay streamer. - `services/api` — REST + WebSocket gateway. -- `services/refdata` — scaffold service. +- `services/refdata` — event-calendar validation/provider refresh scaffolding. - `services/eod-enricher` — scaffold service. - `packages/types` — shared event schemas/types. - `packages/storage` — ClickHouse tables/queries. - `packages/bus` — NATS/JetStream helpers. - `packages/config` — env parsing. - `packages/observability` — logger + metrics facade. +- `deployment/docker` — supported VPS Docker Compose runtime. +- `deployment/native` — experimental host-native Bun + systemd deployment notes. ## Build and Run Install dependencies: -- `bun install` +```bash +bun install +``` Start infrastructure only: -- `docker compose up -d` +```bash +bun run dev:infra +``` Create env file: -- copy `.env.example` to `.env` and set provider credentials as needed. +```bash +cp .env.example .env +``` Start infra + all services + web: -- `bun run dev` +```bash +bun run dev +``` -Start services only (assumes infra is already running): +Start services only, assuming infra is already running: -- `bun run dev:services` +```bash +bun run dev:services +``` Start web only: -- `bun run dev:web` +```bash +bun run dev:web +``` Recommended fast iteration loop: -- `bun run dev:infra` for Docker-backed infra only -- `bun run dev:services` for native Bun backend services -- `bun run dev:web` for the local Next.js UI +```bash +bun run dev:infra +bun run dev:services +bun run dev:web +``` -This keeps Docker in the local workflow where it helps most (NATS, ClickHouse, Redis) without forcing the app services themselves into slower container rebuild/restart loops. +This keeps Docker in the local workflow where it helps most, for NATS, ClickHouse, and Redis, while keeping the app services in native Bun/Next.js loops. ## Deployment Workflow -- `./deploy main` keeps the current VPS Docker rollout path as the default and recommended path. -- Do not run the repo-root `docker-compose.yml` on the VPS. That file is for local infra only and can create duplicate exposed NATS, ClickHouse, and Redis containers on the server. -- `./deploy main --runtime native` targets an experimental host-native Bun + systemd deployment. -- `./deploy current-branch` and `./deploy current-branch --runtime native` keep branch deploys available during the transition, but Docker remains the supported path for the current VPS. -- Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, and `--no-build`. -- Docker runtime details live in `deployment/docker/README.md`. -- Native runtime expectations and prerequisites live in `deployment/native/README.md`. +Docker remains the supported and recommended path for the current VPS. + +```bash +./deploy main +./deploy main --runtime docker +./deploy current-branch +./deploy current-branch --runtime docker +``` + +Important deployment notes: + +- Run the deploy helper from the local repo checkout, not from the VPS shell. +- Do not run the repo-root `docker-compose.yml` on the VPS. It is local infra only and can create duplicate exposed NATS, ClickHouse, and Redis containers on the server. +- The Docker stack lives in `deployment/docker` and is separate from local development infra. +- Partial deploys are supported with `--web-only`, `--api-only`, `--services-only`, `--workers-only`, `--fast`, `--no-build`, and `--force-recreate`. +- `--fast` defaults to a services-only Docker rollout when no explicit scope is provided and trims public API route-suite verification while preserving remote service health checks. +- `./deploy current-branch` requires a clean local working tree and pushes the branch before moving the server checkout. +- The helper has Forgejo-aware remote resolution for deployments and branch pushes. +- When run from `/home/delta/islandflow` on the VPS itself, `./deploy` can execute locally instead of SSHing back into the same server. +- Native deployment is opt-in and experimental: + +```bash +./deploy main --runtime native +./deploy current-branch --runtime native +``` + +Native deployment expects Bun, systemd units, host-reachable infra, and deliberate reverse-proxy changes. Native deploys are intended primarily for worker-only fast iteration until the public edge is cut over deliberately. + +Read more: + +- `deployment/docker/README.md` +- `deployment/native/README.md` ## Desktop Shell -Islandflow also includes a thin Electron desktop shell in `apps/desktop`. +Islandflow includes a thin Electron desktop shell in `apps/desktop`. What it is: @@ -144,37 +197,35 @@ What it is: - a native app window plus packaging/distribution shell, - a way to run the existing web UI inside Electron without local backend services. -What it is not: +What it is not yet: - a bundled backend runtime, -- a packaged local Next.js frontend in v1, -- a desktop feature layer with notifications, preferences, or auto-updates yet. +- a packaged local Next.js frontend, +- a desktop feature layer with notifications, preferences, auto-updates, signing, or notarization. Run the desktop shell against a local web UI: -- `bun run dev:desktop` - -This starts the local Next.js app, defaults `NEXT_PUBLIC_API_URL` to `https://flow.deltaisland.io` unless you already set it, waits for port `3000`, and then launches Electron against `http://127.0.0.1:3000`. +```bash +bun run dev:desktop +``` Run the desktop shell directly against the hosted app: -- `bun run dev:desktop:remote` +```bash +bun run dev:desktop:remote +``` Package the desktop shell: -- `bun run package:desktop` -- `bun run make:desktop` +```bash +bun run package:desktop +bun run make:desktop +``` Desktop-specific environment: - `ISLANDFLOW_DESKTOP_START_URL` is only used by the Electron shell and is restricted to trusted Islandflow app origins. -- `NEXT_PUBLIC_API_URL` remains the web app's API/WebSocket origin control and should usually point at `https://flow.deltaisland.io` when developing the local UI inside Electron. - -Current desktop limitations: - -- v1 builds are unsigned internal macOS artifacts only, -- Forge currently makes a simple zip distributable for the current host architecture, -- signing, notarization, auto-updates, remembered window state, and richer native integrations are intentionally deferred. +- `NEXT_PUBLIC_API_URL` remains the web app API/WebSocket origin control and usually points at `https://flow.deltaisland.io` when developing local UI inside Electron. ## Environment Configuration @@ -196,32 +247,31 @@ All runtime configuration comes from `.env`. | `OPTIONS_INGEST_ADAPTER` | `synthetic` | Options ingest source: `synthetic`, `alpaca`, `ibkr`, or `databento`. | | `EQUITIES_INGEST_ADAPTER` | `synthetic` | Equities ingest source: `synthetic` or `alpaca`. | | `EMIT_INTERVAL_MS` | `1000` | Emit cadence for synthetic ingest adapters. | -| `SYNTHETIC_MARKET_MODE` | `realistic` | Shared synthetic profile (`realistic`, `active`, `firehose`) used when per-service override is unset. | -| `SYNTHETIC_OPTIONS_MODE` | empty | Options-only synthetic profile override; falls back to `SYNTHETIC_MARKET_MODE`. | -| `SYNTHETIC_EQUITIES_MODE` | empty | Equities-only synthetic profile override; falls back to `SYNTHETIC_MARKET_MODE`. | +| `SYNTHETIC_MARKET_MODE` | `realistic` | Shared synthetic profile: `realistic`, `active`, or `firehose`. | +| `SYNTHETIC_OPTIONS_MODE` | empty | Options-only synthetic profile override. | +| `SYNTHETIC_EQUITIES_MODE` | empty | Equities-only synthetic profile override. | -Synthetic profile intent: -- `realistic`: default local mode with lower synthetic burstiness/noise. -- `active`: busier demo flow while still readable. -- `firehose`: stress mode for throughput/backpressure/hot-window behavior. - -### Options ingest adapter configuration +### Alpaca and news configuration | Variable | Default | What it controls | | --- | --- | --- | -| `ALPACA_API_KEY` | empty | Single-token Alpaca API auth for options/equities adapters. Use this when your account provides one API key value. | -| `ALPACA_REST_URL` | `https://data.alpaca.markets` | Alpaca REST base URL for contract discovery/reference calls. | -| `ALPACA_WS_BASE_URL` | `wss://stream.data.alpaca.markets/v1beta1` (options), `wss://stream.data.alpaca.markets` (equities) | Alpaca websocket base URL. | -| `ALPACA_FEED` | `indicative` | Options feed tier for Alpaca options (`indicative` or `opra`). | +| `ALPACA_API_KEY` | empty | Legacy single-token fallback kept for older Alpaca setups. Prefer explicit key ID + secret vars for current Alpaca auth. | +| `ALPACA_API_KEY_ID` | empty | Preferred Alpaca key ID used for market-data REST and websocket auth. | +| `ALPACA_KEY_ID` | empty | Alternate name accepted for the Alpaca key ID. | +| `ALPACA_API_SECRET_KEY` | empty | Preferred Alpaca secret key paired with `ALPACA_API_KEY_ID`. | +| `ALPACA_SECRET_KEY` | empty | Alternate name accepted for the Alpaca secret key. | +| `ALPACA_REST_URL` | `https://data.alpaca.markets` | Alpaca REST base URL. | +| `ALPACA_WS_BASE_URL` | `wss://stream.data.alpaca.markets/v1beta1` for options, `wss://stream.data.alpaca.markets` for equities/news | Alpaca websocket base URL. | +| `ALPACA_FEED` | `indicative` | Options feed tier: `indicative` or `opra`. | | `ALPACA_UNDERLYINGS` | `SPY,NVDA,AAPL` | Comma-separated symbols targeted by Alpaca ingest. | | `ALPACA_STRIKES_PER_SIDE` | `8` | Contracts selected per side of spot for Alpaca options chain sampling. | | `ALPACA_MAX_DTE_DAYS` | `30` | Max days-to-expiry included for Alpaca options contract selection. | | `ALPACA_MONEYNESS_PCT` | `0.06` | Primary moneyness filter for Alpaca options contract selection. | | `ALPACA_MONEYNESS_FALLBACK_PCT` | `0.1` | Wider fallback moneyness filter if candidate set is too sparse. | | `ALPACA_MAX_QUOTES` | `200` | Upper bound on selected Alpaca options contracts/quotes per cycle. | -| `ALPACA_EQUITIES_FEED` | `iex` | Alpaca equities feed (`iex` free tier, `sip` paid consolidated feed). | - -For Alpaca adapters, configure `ALPACA_API_KEY`. +| `ALPACA_EQUITIES_FEED` | `iex` | Alpaca equities feed: `iex` or `sip`. | +| `ALPACA_NEWS_BACKFILL_LIMIT` | `50` | Alpaca news stories fetched on startup, capped at 50 by the Alpaca News API. | +| `ALPACA_NEWS_WEBSOCKET_PATH` | `/v1beta1/news` | Alpaca news websocket path. | ### Databento replay adapter configuration @@ -236,7 +286,7 @@ For Alpaca adapters, configure `ALPACA_API_KEY`. | `DATABENTO_SYMBOLS` | `ALL` | Symbol selection forwarded to Databento sidecar query. | | `DATABENTO_STYPE_IN` | `raw_symbol` | Databento input symbology type. | | `DATABENTO_STYPE_OUT` | `raw_symbol` | Databento output symbology type. | -| `DATABENTO_LIMIT` | `0` | Max Databento records (`0` means no explicit limit). | +| `DATABENTO_LIMIT` | `0` | Max Databento records, where `0` means no explicit limit. | | `DATABENTO_PRICE_SCALE` | `1` | Multiplier applied to decoded prices from sidecar output. | | `DATABENTO_PYTHON_BIN` | `python3` | Python executable used to run Databento sidecar script. | @@ -248,9 +298,9 @@ For Alpaca adapters, configure `ALPACA_API_KEY`. | `IBKR_PORT` | `7497` | TWS/Gateway port for IBKR bridge. | | `IBKR_CLIENT_ID` | `0` | IBKR client id used by the bridge connection. | | `IBKR_SYMBOL` | `SPY` | Underlying symbol requested from IBKR. | -| `IBKR_EXPIRY` | `20250117` | Option expiry (YYYYMMDD) requested from IBKR. | +| `IBKR_EXPIRY` | `20250117` | Option expiry requested from IBKR. | | `IBKR_STRIKE` | `450` | Strike requested from IBKR. | -| `IBKR_RIGHT` | `C` | Option side (`C` or `P`). | +| `IBKR_RIGHT` | `C` | Option side: `C` or `P`. | | `IBKR_EXCHANGE` | `SMART` | IBKR exchange routing code. | | `IBKR_CURRENCY` | `USD` | Contract currency. | | `IBKR_PYTHON_BIN` | `python3` | Python executable used for IBKR sidecar. | @@ -259,133 +309,77 @@ For Alpaca adapters, configure `ALPACA_API_KEY`. | Variable | Default | What it controls | | --- | --- | --- | -| `OPTIONS_SIGNAL_MODE` | `smart-money` | Signal pass policy (`smart-money`, `balanced`, `all`) for options prints. | +| `OPTIONS_SIGNAL_MODE` | `smart-money` | Signal pass policy: `smart-money`, `balanced`, or `all`. | | `OPTIONS_SIGNAL_MIN_NOTIONAL` | `10000` | Base minimum notional for most signal candidates. | | `OPTIONS_SIGNAL_ETF_MIN_NOTIONAL` | `50000` | ETF-specific minimum notional for signal inclusion. | -| `OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL` | `25000` | Minimum notional for bid-side (`B`/`BB`) or sweep/ISO thresholds. | +| `OPTIONS_SIGNAL_BID_SIDE_MIN_NOTIONAL` | `25000` | Minimum notional for bid-side or sweep/ISO thresholds. | | `OPTIONS_SIGNAL_MID_MIN_NOTIONAL` | `20000` | Minimum notional for non-sweep/non-ISO `MID` prints. | | `OPTIONS_SIGNAL_NBBO_MAX_AGE_MS` | `1500` | NBBO freshness threshold used during signal classification. | -| `OPTIONS_SIGNAL_ETF_UNDERLYINGS` | `SPY,QQQ,IWM,DIA,TLT,GLD,SLV,XLF,XLE,XLV,XLI,XLP,XLU,XLY,SMH,ARKK` | Comma-separated underlyings treated as ETFs by signal filters. | +| `OPTIONS_SIGNAL_ETF_UNDERLYINGS` | `SPY,QQQ,IWM,DIA,TLT,GLD,SLV,XLF,XLE,XLV,XLI,XLP,XLU,XLY,SMH,ARKK` | ETF underlyings treated specially by signal filters. | -Default `smart-money` policy rejects lower-information prints and keeps high-confidence/high-notional/sweep-style flow; `balanced` lowers thresholds; `all` bypasses filtering. +Default `smart-money` policy rejects lower-information prints and keeps higher-confidence, higher-notional, sweep-style flow. `balanced` lowers thresholds. `all` bypasses filtering. -### Compute/classifier/dark-inference configuration +### Compute, classifier, and dark-inference configuration | Variable | Default | What it controls | | --- | --- | --- | -| `CLUSTER_WINDOW_MS` | `500` | Time window used to cluster nearby option prints into a packet candidate. | -| `COMPUTE_DELIVER_POLICY` | `new` | Consumer start policy for compute stream subscriptions (`new`, `all`, `last`, `last_per_subject`). | -| `COMPUTE_CONSUMER_RESET` | `false` | If true, resets durable consumer position for compute on startup. | +| `CLUSTER_WINDOW_MS` | `500` | Time window used to cluster nearby option prints into packet candidates. | +| `COMPUTE_DELIVER_POLICY` | `new` | Consumer start policy for compute subscriptions. | +| `COMPUTE_CONSUMER_RESET` | `false` | Resets durable consumer position for compute on startup when true. | | `NBBO_MAX_AGE_MS` | `1000` | Max NBBO age accepted when enriching option prints in compute. | | `ROLLING_WINDOW_SIZE` | `50` | Number of observations retained per rolling metric key. | | `ROLLING_TTL_SEC` | `86400` | Redis TTL for rolling metric keys. | | `EQUITY_QUOTE_MAX_AGE_MS` | `1000` | Max quote staleness when joining equity prints for inference. | | `DARK_INFER_WINDOW_MS` | `60000` | Sliding window length for dark-style inference accumulation. | -| `DARK_INFER_COOLDOWN_MS` | `30000` | Cooldown before emitting repeated dark inferences for same symbol/pattern. | -| `DARK_INFER_MIN_BLOCK_SIZE` | `2000` | Minimum single-print size for block-style dark inference evidence. | -| `DARK_INFER_MIN_ACCUM_SIZE` | `3000` | Minimum aggregate size for accumulation-style dark inference evidence. | -| `DARK_INFER_MIN_ACCUM_COUNT` | `4` | Minimum print count for accumulation-style dark inference. | -| `DARK_INFER_MIN_PRINT_SIZE` | `200` | Minimum print size considered as dark inference evidence. | -| `DARK_INFER_MAX_EVIDENCE` | `20` | Max evidence items attached to one inferred dark event. | -| `DARK_INFER_MAX_SPREAD_PCT` | `0.005` | Maximum spread percentage allowed for dark inference confidence. | -| `CLASSIFIER_SWEEP_MIN_PREMIUM` | `40000` | Minimum premium to trigger sweep classifier logic. | -| `CLASSIFIER_SWEEP_MIN_COUNT` | `3` | Minimum child prints in cluster for sweep classifier hit. | -| `CLASSIFIER_SWEEP_MIN_PREMIUM_Z` | `2` | Min premium z-score for sweep classifier confirmation. | -| `CLASSIFIER_SPIKE_MIN_PREMIUM` | `20000` | Minimum premium for spike classifier logic. | -| `CLASSIFIER_SPIKE_MIN_SIZE` | `400` | Minimum total size for spike classifier logic. | -| `CLASSIFIER_SPIKE_MIN_PREMIUM_Z` | `2.5` | Min premium z-score for spike classifier confirmation. | -| `CLASSIFIER_SPIKE_MIN_SIZE_Z` | `2` | Min size z-score for spike classifier confirmation. | -| `CLASSIFIER_Z_MIN_SAMPLES` | `12` | Minimum rolling sample count before z-score gating applies. | -| `CLASSIFIER_MIN_NBBO_COVERAGE` | `0.5` | Required fraction of prints in cluster with valid NBBO context. | -| `CLASSIFIER_MIN_AGGRESSOR_RATIO` | `0.55` | Minimum aggressor-side ratio for classifier confidence. | -| `CLASSIFIER_0DTE_MAX_ATM_PCT` | `0.01` | Max distance-from-ATM to qualify as near-ATM 0DTE event. | -| `CLASSIFIER_0DTE_MIN_PREMIUM` | `20000` | Minimum premium for 0DTE classifier events. | -| `CLASSIFIER_0DTE_MIN_SIZE` | `400` | Minimum size for 0DTE classifier events. | -| `SMART_MONEY_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file used by compute to enrich event-driven smart-money profile features. | -| `REFDATA_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file for refdata service startup validation; falls back to `SMART_MONEY_EVENT_CALENDAR_PATH` when unset. | -| `REFDATA_EVENT_CALENDAR_PROVIDER` | empty | Set to `alpha_vantage` to have refdata refresh the calendar cache from Alpha Vantage. | -| `ALPHA_VANTAGE_API_KEY` | empty | Alpha Vantage key used when `REFDATA_EVENT_CALENDAR_PROVIDER=alpha_vantage`. | -| `ALPHA_VANTAGE_EARNINGS_HORIZON` | `3month` | Alpha Vantage earnings horizon: `3month`, `6month`, or `12month`. | -| `ALPHA_VANTAGE_EARNINGS_SYMBOL` | empty | Optional single-symbol Alpha Vantage earnings query; empty fetches the full scheduled earnings list. | -| `REFDATA_EVENT_CALENDAR_REFRESH_MS` | `86400000` | Refdata refresh cadence for provider-backed event-calendar cache writes. | +| `DARK_INFER_COOLDOWN_MS` | `30000` | Cooldown before repeated dark inferences for same symbol/pattern. | +| `SMART_MONEY_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar file used by compute. | +| `REFDATA_EVENT_CALENDAR_PATH` | empty | Optional JSON event-calendar path for refdata; falls back to `SMART_MONEY_EVENT_CALENDAR_PATH`. | +| `REFDATA_EVENT_CALENDAR_PROVIDER` | empty | Set to `alpha_vantage` to refresh event-calendar cache from Alpha Vantage. | +| `ALPHA_VANTAGE_API_KEY` | empty | Alpha Vantage key for provider-backed event-calendar refresh. | -Event-calendar rows may use `symbol`, `underlying`, or `underlying_id`; `event_date`, `event_time`, or `event_ts`; and `announced_ts`, `available_ts`, `as_of_ts`, or `created_ts`. Compute only uses events already available at the packet timestamp, so missing or unavailable rows leave event-alignment features as neutral `null` values. - -### Candle service configuration - -| Variable | Default | What it controls | -| --- | --- | --- | -| `CANDLE_INTERVALS_MS` | `60000,300000` | Comma-separated candle intervals generated from equity prints. | -| `CANDLE_MAX_LATE_MS` | `0` | Allowed lateness for out-of-order prints before candle rejection/roll policy applies. | -| `CANDLE_CACHE_LIMIT` | `2000` | Max cached candles per `(underlying, interval)` in Redis (`0` disables cache). | -| `CANDLE_DELIVER_POLICY` | `new` | Consumer start policy for candle service (`new`, `all`, `last`, `last_per_subject`). | -| `CANDLE_CONSUMER_RESET` | `false` | If true, resets candle durable consumer position on startup. | - -### API + live cache configuration +### API, live cache, and web client | Variable | Default | What it controls | | --- | --- | --- | | `API_PORT` | `4000` | API service listen port. | -| `REST_DEFAULT_LIMIT` | `200` | Default record count when a REST endpoint omits `limit`. | -| `API_DELIVER_POLICY` | `new` | JetStream consumer start policy used by API live subscribers (`new`, `all`, `last`, `last_per_subject`). | -| `API_CONSUMER_RESET` | `false` | If true, API resets/recreates its live durable consumers on startup. | -| `LIVE_LIMIT_OPTIONS` | `10000` | In-memory/Redis live cache depth for options channel (clamped `1..100000`). | -| `LIVE_LIMIT_NBBO` | `10000` | Live cache depth for options NBBO channel (clamped `1..100000`). | -| `LIVE_LIMIT_EQUITIES` | `10000` | Live cache depth for equities channel (clamped `1..100000`). | -| `LIVE_LIMIT_EQUITY_QUOTES` | `10000` | Live cache depth for equity quotes channel (clamped `1..100000`). | -| `LIVE_LIMIT_EQUITY_JOINS` | `10000` | Live cache depth for equity join channel (clamped `1..100000`). | -| `LIVE_LIMIT_FLOW` | `10000` | Live cache depth for flow packet channel (clamped `1..100000`). | -| `LIVE_LIMIT_CLASSIFIER_HITS` | `10000` | Live cache depth for classifier hits channel (clamped `1..100000`). | -| `LIVE_LIMIT_ALERTS` | `10000` | Live cache depth for alerts channel (clamped `1..100000`). | -| `LIVE_LIMIT_INFERRED_DARK` | `10000` | Live cache depth for inferred dark channel (clamped `1..100000`). | - -### Web client configuration (`NEXT_PUBLIC_*`) - -| Variable | Default | What it controls | -| --- | --- | --- | -| `NEXT_PUBLIC_API_URL` | auto-detected (`window.location.origin` in browser; `http://127.0.0.1:4000` fallback) | Explicit base URL for API/WS calls from the web app. | -| `NEXT_PUBLIC_LIVE_HOT_WINDOW` | `2000` | Max hot-window items retained for non-options live streams in UI state (`100..100000`). | -| `NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS` | `25000` | Dedicated max hot-window items retained for options prints (`100..100000`). | -| `NEXT_PUBLIC_NBBO_MAX_AGE_MS` | `1000` | Frontend NBBO staleness threshold used for UI status/placement logic. | -| `NEXT_PUBLIC_LIVE_EQUITIES_SILENT_WARNING_MS` | `25000` | Delay before warning when equities stream is quiet (`5000..300000`). | -| `NEXT_PUBLIC_PINNED_EVIDENCE_TTL_MS` | `1200000` | TTL for pinned evidence objects in UI (`60000..7200000`). | -| `NEXT_PUBLIC_PINNED_EVIDENCE_MAX_ITEMS` | `4000` | Maximum pinned evidence cache size in UI (`100..50000`). | -| `NEXT_PUBLIC_FLOW_FILTER_PRESET` | `smart-money` | Default flow filter preset applied on page load (`smart-money`, `balanced`, `all`). | +| `REST_DEFAULT_LIMIT` | `200` | Default REST record count. | +| `API_DELIVER_POLICY` | `new` | JetStream consumer start policy used by API live subscribers. | +| `API_CONSUMER_RESET` | `false` | Resets/recreates API live durable consumers on startup when true. | +| `LIVE_LIMIT_DEFAULT` | `1000` | Optional generic live cache depth default. | +| `LIVE_LIMIT_FLOW` | `500` | Live cache depth for flow packet events unless overridden. | +| `LIVE_LIMIT_SMART_MONEY` | `300` | Live cache depth for smart-money events unless overridden. | +| `LIVE_LIMIT_OPTIONS` | `1000` | Live cache depth for options channel unless overridden. | +| `LIVE_LIMIT_ALERTS` | `300` | Live cache depth for alerts channel unless overridden. | +| `LIVE_LIMIT_NEWS` | `100` | Live cache depth for news channel unless overridden. | +| `NEXT_PUBLIC_API_URL` | auto-detected in browser, `http://127.0.0.1:4000` fallback | Explicit base URL for API/WS calls from the web app. | +| `NEXT_PUBLIC_LIVE_HOT_WINDOW` | `600` | Max hot-window items retained for non-options live streams in UI state. | +| `NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS` | `1200` | Dedicated max hot-window items retained for options prints. | +| `NEXT_PUBLIC_NBBO_MAX_AGE_MS` | `1000` | Frontend NBBO staleness threshold. | +| `NEXT_PUBLIC_FLOW_FILTER_PRESET` | `smart-money` | Default flow filter preset: `smart-money`, `balanced`, or `all`. | ### Replay and testing controls | Variable | Default | What it controls | | --- | --- | --- | -| `REPLAY_ENABLED` | `false` | Dev-script toggle: starts replay service in `bun run dev` when truthy. | -| `REPLAY_STREAMS` | `options,nbbo,equities,equity-quotes` | Replay stream selection (`all` or comma list of supported aliases). | -| `REPLAY_START_TS` | `0` | Replay lower-bound timestamp; `0` means from earliest stored data. | -| `REPLAY_END_TS` | `0` | Replay upper-bound timestamp; `0` means no explicit end bound. | -| `REPLAY_SPEED` | `1` | Replay speed multiplier relative to original event timing. | -| `REPLAY_BATCH_SIZE` | `200` | Batch fetch size per replay stream pull. | -| `REPLAY_LOG_EVERY` | `1000` | Progress log interval (emitted event count). | +| `REPLAY_ENABLED` | `false` | Starts replay service in `bun run dev` when truthy. | +| `REPLAY_STREAMS` | `options,nbbo,equities,equity-quotes` | Replay stream selection. | +| `REPLAY_START_TS` | `0` | Replay lower-bound timestamp. | +| `REPLAY_END_TS` | `0` | Replay upper-bound timestamp. | +| `REPLAY_SPEED` | `1` | Replay speed multiplier. | +| `REPLAY_BATCH_SIZE` | `200` | Batch fetch size per stream. | +| `REPLAY_LOG_EVERY` | `1000` | Progress log interval. | | `TESTING_MODE` | `false` | Enables ingest publish throttling for deterministic/lower-volume test runs. | | `TESTING_THROTTLE_MS` | `200` | Minimum delay between emitted events while `TESTING_MODE=true`. | ## Quick Notes -- Python dependencies are required only for IBKR/Databento sidecars (`services/ingest-options/py/requirements.txt`). +- Python dependencies are required only for IBKR/Databento sidecars: `services/ingest-options/py/requirements.txt`. - Candle construction is server-side; the client consumes prebuilt OHLC events. -- Option prints now persist as enriched raw rows and can be queried as either: - - `view=signal` — default live/UI path and compute input. - - `view=raw` — audit/debug path that preserves every stored print. -- The default Tape page options/packets posture is now stock-only, hides `B` / `BB`, keeps calls and puts visible, and applies in-memory min-notional controls immediately. -- Live retention uses a two-tier model: - - ClickHouse is durable server history; Redis is a bounded hot cache per live generic channel. - - `LIVE_LIMIT_*` controls initial snapshot/hot-cache depth, not total persisted history. - - Browser state is only a rendering window and UI preferences, not a market-data database. - - Devices connected to the same API hydrate from the same server-seen history. - - UI keeps a bounded hot window for rendering performance around the signal view rather than raw noise. - - Options prints can use a deeper dedicated cap via `NEXT_PUBLIC_LIVE_HOT_WINDOW_OPTIONS` without raising every other feed. - - Alert/drawer evidence is pinned and hydrated by id/trace so details remain inspectable after hot-window eviction. -- Firehose-readiness strategy: - - preserve raw ingest for storage/replay, - - feed compute and default live UI from the filtered signal path, - - add filterable live subscription contracts now so selective delivery can move server-side without reshaping the protocol later. +- Option prints persist as enriched raw rows and can be queried as `view=signal` or `view=raw`. +- The default Tape page options/packets posture is stock-only, hides `B` / `BB`, keeps calls and puts visible, and applies in-memory min-notional controls immediately. +- Live retention uses ClickHouse for durable server history, Redis for bounded hot cache, and browser state for rendering windows/preferences. +- Alert and drawer evidence is pinned and hydrated by id/trace so details remain inspectable after hot-window eviction. +- Firehose readiness keeps raw ingest for storage/replay, routes default compute/UI through filtered signals, and keeps subscription contracts ready for server-side selective delivery. - This repository is for personal, non-redistributed usage. ## Useful Examples diff --git a/apps/desktop/package.json b/apps/desktop/package.json index c46915b..8a3c3e5 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -12,6 +12,9 @@ "package": "bun run build && electron-forge package", "make": "bun run build && electron-forge make" }, + "dependencies": { + "@islandflow/types": "workspace:*" + }, "devDependencies": { "@electron-forge/cli": "^7.8.1", "@electron-forge/core": "^7.11.1", diff --git a/apps/desktop/src/desktop-ai-ipc.ts b/apps/desktop/src/desktop-ai-ipc.ts new file mode 100644 index 0000000..25e53f6 --- /dev/null +++ b/apps/desktop/src/desktop-ai-ipc.ts @@ -0,0 +1,8 @@ +export const DESKTOP_AI_STATE_CHANNEL = "islandflow:desktop-ai:state"; +export const DESKTOP_AI_GET_STATE = "islandflow:desktop-ai:get-state"; +export const DESKTOP_AI_LOGIN_BROWSER = "islandflow:desktop-ai:login-browser"; +export const DESKTOP_AI_LOGIN_DEVICE = "islandflow:desktop-ai:login-device"; +export const DESKTOP_AI_CANCEL_LOGIN = "islandflow:desktop-ai:cancel-login"; +export const DESKTOP_AI_LOGOUT = "islandflow:desktop-ai:logout"; +export const DESKTOP_AI_UPDATE_PREFERENCES = "islandflow:desktop-ai:update-preferences"; +export const DESKTOP_AI_RUN_TASK = "islandflow:desktop-ai:run-task"; diff --git a/apps/desktop/src/desktop-ai.test.ts b/apps/desktop/src/desktop-ai.test.ts new file mode 100644 index 0000000..53f4822 --- /dev/null +++ b/apps/desktop/src/desktop-ai.test.ts @@ -0,0 +1,174 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { createAppServerChildEnv, IslandflowDesktopAiService, summarizeRateLimit } from "./desktop-ai.js"; + +const tempDirs: string[] = []; + +const makeTempDir = async (): Promise => { + const dir = await mkdtemp(path.join(tmpdir(), "islandflow-desktop-ai-")); + tempDirs.push(dir); + return dir; +}; + +afterEach(async () => { + await Promise.all( + tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })) + ); +}); + +describe("desktop ai auth environment", () => { + it("scrubs global OpenAI keys for managed ChatGPT sessions", () => { + const env = createAppServerChildEnv("managed-chatgpt", { + OPENAI_API_KEY: "openai-test", + CODEX_API_KEY: "codex-test", + HOME: "/tmp/home" + }); + + expect(env.OPENAI_API_KEY).toBeUndefined(); + expect(env.CODEX_API_KEY).toBeUndefined(); + expect(env.HOME).toBe("/tmp/home"); + }); + + it("preserves keys for api-key mode", () => { + const env = createAppServerChildEnv("api-key", { + OPENAI_API_KEY: "openai-test", + CODEX_API_KEY: "codex-test" + }); + + expect(env.OPENAI_API_KEY).toBe("openai-test"); + expect(env.CODEX_API_KEY).toBe("codex-test"); + }); +}); + +describe("desktop ai usage and state tracking", () => { + it("records exact token usage notifications into usage rollups", async () => { + const dir = await makeTempDir(); + const service = new IslandflowDesktopAiService(dir, async () => {}, () => {}); + const internal = service as any; + + internal.state.account.email = "analyst@example.com"; + internal.state.account.planType = "plus"; + internal.state.preferences.model = "gpt-5.4"; + internal.state.tasks = [ + { + taskId: "task-1", + kind: "smart-money-explain", + title: "Explain smart money event", + subtitle: "AAPL", + status: "running", + createdAt: Date.now(), + updatedAt: Date.now(), + threadId: "thread-1", + turnId: "turn-1", + model: "gpt-5.4", + reasoningEffort: "high", + text: "", + error: null, + compiledScreen: null + } + ]; + internal.activeTasksByThreadId.set("thread-1", { + taskId: "task-1", + taskKind: "smart-money-explain", + taskTitle: "Explain smart money event", + profileId: "managed-chatgpt" + }); + + await internal.handleNotification("thread/tokenUsage/updated", { + threadId: "thread-1", + turnId: "turn-1", + tokenUsage: { + total: { + totalTokens: 1800, + inputTokens: 1000, + cachedInputTokens: 500, + outputTokens: 250, + reasoningOutputTokens: 50 + }, + last: { + totalTokens: 1800, + inputTokens: 1000, + cachedInputTokens: 500, + outputTokens: 250, + reasoningOutputTokens: 50 + } + } + }); + + expect(service.getState().usage.today.breakdown).toEqual({ + totalTokens: 1800, + inputTokens: 1000, + cachedInputTokens: 500, + outputTokens: 250, + reasoningOutputTokens: 50 + }); + expect(service.getState().usage.today.turnCount).toBe(1); + expect(service.getState().usage.recentTurns[0]?.normalizedCostUsd).toBeCloseTo(0.007125, 6); + }); + + it("stores rate-limit snapshots with reset times", async () => { + const dir = await makeTempDir(); + const service = new IslandflowDesktopAiService(dir, async () => {}, () => {}); + const internal = service as any; + + await internal.handleNotification("account/rateLimits/updated", { + rateLimits: { + limitId: "chatgpt_plus", + limitName: "ChatGPT Plus", + primary: { + usedPercent: 38.4, + windowDurationMins: 180, + resetsAt: 1_710_000_000_000 + }, + secondary: { + usedPercent: 12.1, + windowDurationMins: 1440, + resetsAt: 1_710_003_600_000 + }, + planType: "plus" + } + }); + + expect(service.getState().rateLimitsByLimitId.chatgpt_plus).toEqual( + summarizeRateLimit({ + limitId: "chatgpt_plus", + limitName: "ChatGPT Plus", + primary: { + usedPercent: 38.4, + windowDurationMins: 180, + resetsAt: 1_710_000_000_000 + }, + secondary: { + usedPercent: 12.1, + windowDurationMins: 1440, + resetsAt: 1_710_003_600_000 + }, + planType: "plus" + }) + ); + }); + + it("clears local account state on logout", async () => { + const dir = await makeTempDir(); + const service = new IslandflowDesktopAiService(dir, async () => {}, () => {}); + const internal = service as any; + + internal.client = { + request: async () => ({}) + }; + internal.state.account.loggedIn = true; + internal.state.account.email = "analyst@example.com"; + internal.state.account.planType = "plus"; + internal.state.account.login = { status: "browser_pending", message: "Waiting", loginId: "login-1", authUrl: "https://example.com" }; + + await service.logout(); + + expect(service.getState().account.loggedIn).toBe(false); + expect(service.getState().account.email).toBeNull(); + expect(service.getState().account.planType).toBeNull(); + expect(service.getState().account.login).toEqual({ status: "idle", message: "Logged out." }); + }); +}); diff --git a/apps/desktop/src/desktop-ai.ts b/apps/desktop/src/desktop-ai.ts new file mode 100644 index 0000000..8d695a2 --- /dev/null +++ b/apps/desktop/src/desktop-ai.ts @@ -0,0 +1,1222 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { + IslandflowAiCompiledScreenSchema, + IslandflowAiProfileModeSchema, + IslandflowAiReasoningEffortSchema, + IslandflowAiTaskRequestSchema, + type IslandflowAiCompiledScreen, + type IslandflowAiModelSummary, + type IslandflowAiPlanType, + type IslandflowAiPricing, + type IslandflowAiProfileMode, + type IslandflowAiRateLimitSnapshot, + type IslandflowAiReasoningEffort, + type IslandflowAiState, + type IslandflowAiTaskKind, + type IslandflowAiTaskRequest, + type IslandflowAiTaskSnapshot, + type IslandflowAiTokenBreakdown, + type IslandflowAiUsageRollup, + type IslandflowAiUsageTurnRecord +} from "@islandflow/types"; + +const MANAGED_CHATGPT_PROFILE_ID = "managed-chatgpt"; +const WORKSPACE_PROVIDER_PROFILE_ID = "workspace-provider"; +const APP_SERVER_SERVICE_NAME = "Islandflow Analyst Copilot"; +const APP_SERVER_SANDBOX_CWD = "copilot-sandbox"; +const PREFERENCES_FILE = "copilot-preferences.json"; +const USAGE_FILE = "copilot-usage.json"; +const DEFAULT_REASONING = IslandflowAiReasoningEffortSchema.parse("high"); + +const EMPTY_BREAKDOWN: IslandflowAiTokenBreakdown = { + totalTokens: 0, + inputTokens: 0, + cachedInputTokens: 0, + outputTokens: 0, + reasoningOutputTokens: 0 +}; + +type JsonRpcSuccess = { + id: number; + result: unknown; +}; + +type JsonRpcFailure = { + id: number; + error: { + message?: string; + code?: number; + data?: unknown; + }; +}; + +type JsonRpcNotification = { + method: string; + params?: unknown; + id?: never; +}; + +type JsonRpcServerRequest = { + id: number; + method: string; + params?: unknown; +}; + +type JsonRpcMessage = JsonRpcSuccess | JsonRpcFailure | JsonRpcNotification | JsonRpcServerRequest; + +type CodexModelRecord = { + id: string; + model: string; + displayName: string; + description: string; + hidden: boolean; + isDefault: boolean; + supportedReasoningEfforts: Array<{ reasoningEffort: IslandflowAiReasoningEffort }>; + defaultReasoningEffort: IslandflowAiReasoningEffort | null; +}; + +type CodexThreadStartResult = { + thread: { + id: string; + }; + model: string; + reasoningEffort: IslandflowAiReasoningEffort | null; +}; + +type CodexTurnStartResult = { + turn: { + id: string; + }; +}; + +type PersistedUsageStore = { + version: 1; + turns: Record; +}; + +type PersistedPreferences = { + model: string | null; + reasoningEffort: IslandflowAiReasoningEffort | null; +}; + +type OpenExternalFn = (url: string) => Promise; + +type ActiveTaskContext = { + taskId: string; + taskKind: IslandflowAiTaskKind; + taskTitle: string; + profileId: string; +}; + +const MODEL_PRICING: Record = { + "gpt-5.5": { + inputUsdPer1MTokens: 5, + cachedInputUsdPer1MTokens: 0.5, + outputUsdPer1MTokens: 30, + sourceLabel: "OpenAI GPT-5.5 model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5.5" + }, + "gpt-5.4": { + inputUsdPer1MTokens: 2.5, + cachedInputUsdPer1MTokens: 0.25, + outputUsdPer1MTokens: 15, + sourceLabel: "OpenAI GPT-5.4 model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5.4" + }, + "gpt-5.4-mini": { + inputUsdPer1MTokens: 0.75, + cachedInputUsdPer1MTokens: 0.075, + outputUsdPer1MTokens: 4.5, + sourceLabel: "OpenAI GPT-5.4 mini model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5.4-mini" + }, + "gpt-5.3-codex": { + inputUsdPer1MTokens: 1.75, + cachedInputUsdPer1MTokens: 0.175, + outputUsdPer1MTokens: 14, + sourceLabel: "OpenAI GPT-5.3-Codex model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5.3-codex" + }, + "gpt-5.2": { + inputUsdPer1MTokens: 1.75, + cachedInputUsdPer1MTokens: 0.175, + outputUsdPer1MTokens: 14, + sourceLabel: "OpenAI GPT-5.2 model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5.2" + }, + "gpt-5.2-codex": { + inputUsdPer1MTokens: 1.75, + cachedInputUsdPer1MTokens: 0.175, + outputUsdPer1MTokens: 14, + sourceLabel: "OpenAI GPT-5.2-Codex model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5.2-codex" + }, + "gpt-5-codex": { + inputUsdPer1MTokens: 1.25, + cachedInputUsdPer1MTokens: 0.125, + outputUsdPer1MTokens: 10, + sourceLabel: "OpenAI GPT-5-Codex model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/gpt-5-codex" + }, + "codex-mini-latest": { + inputUsdPer1MTokens: 1.5, + cachedInputUsdPer1MTokens: 0.375, + outputUsdPer1MTokens: 6, + sourceLabel: "OpenAI codex-mini-latest model pricing", + sourceUrl: "https://developers.openai.com/api/docs/models/codex-mini-latest" + } +}; + +const createEmptyUsageRollup = (): IslandflowAiUsageRollup => ({ + breakdown: { ...EMPTY_BREAKDOWN }, + normalizedCostUsd: 0, + turnCount: 0, + activeDays: 0 +}); + +const createInitialState = (): IslandflowAiState => ({ + desktopAvailable: true, + transportStatus: "starting", + transportError: null, + profiles: [ + { + id: MANAGED_CHATGPT_PROFILE_ID, + label: "Managed ChatGPT login", + description: "User-scoped ChatGPT or Codex sign-in managed by the official app-server.", + mode: "managed-chatgpt", + enabled: true, + selected: true, + statusLabel: "Active" + }, + { + id: WORKSPACE_PROVIDER_PROFILE_ID, + label: "Workspace provider slot", + description: "Reserved for future shared API-key or enterprise access-token flows.", + mode: "workspace-provider", + enabled: false, + selected: false, + statusLabel: "Reserved" + } + ], + selectedProfileId: MANAGED_CHATGPT_PROFILE_ID, + account: { + loggedIn: false, + email: null, + planType: null, + authMode: null, + requiresOpenaiAuth: true, + login: { status: "idle", message: null } + }, + preferences: { + model: null, + reasoningEffort: DEFAULT_REASONING + }, + models: [], + rateLimitsByLimitId: {}, + usage: { + today: createEmptyUsageRollup(), + lifetime: createEmptyUsageRollup(), + recentTurns: [] + }, + tasks: [], + updatedAt: Date.now() +}); + +const buildUsageKey = (threadId: string, turnId: string): string => `${threadId}:${turnId}`; + +const normalizeBreakdown = (value: Partial | null | undefined): IslandflowAiTokenBreakdown => ({ + totalTokens: value?.totalTokens ?? 0, + inputTokens: value?.inputTokens ?? 0, + cachedInputTokens: value?.cachedInputTokens ?? 0, + outputTokens: value?.outputTokens ?? 0, + reasoningOutputTokens: value?.reasoningOutputTokens ?? 0 +}); + +const addBreakdowns = ( + left: IslandflowAiTokenBreakdown, + right: IslandflowAiTokenBreakdown +): IslandflowAiTokenBreakdown => ({ + totalTokens: left.totalTokens + right.totalTokens, + inputTokens: left.inputTokens + right.inputTokens, + cachedInputTokens: left.cachedInputTokens + right.cachedInputTokens, + outputTokens: left.outputTokens + right.outputTokens, + reasoningOutputTokens: left.reasoningOutputTokens + right.reasoningOutputTokens +}); + +const isoDayKey = (timestampMs: number): string => new Date(timestampMs).toISOString().slice(0, 10); + +const sanitizeJsonText = (value: string): string => { + const trimmed = value.trim(); + if (trimmed.startsWith("```")) { + return trimmed.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, ""); + } + return trimmed; +}; + +const estimateNormalizedCost = ( + model: string | null, + breakdown: IslandflowAiTokenBreakdown +): number | null => { + if (!model) { + return null; + } + const pricing = MODEL_PRICING[model]; + if (!pricing) { + return null; + } + const outputBillableTokens = breakdown.outputTokens + breakdown.reasoningOutputTokens; + const usd = + (breakdown.inputTokens / 1_000_000) * pricing.inputUsdPer1MTokens + + (breakdown.cachedInputTokens / 1_000_000) * pricing.cachedInputUsdPer1MTokens + + (outputBillableTokens / 1_000_000) * pricing.outputUsdPer1MTokens; + return Number(usd.toFixed(6)); +}; + +const compactTaskList = (tasks: IslandflowAiTaskSnapshot[]): IslandflowAiTaskSnapshot[] => + [...tasks].sort((left, right) => right.updatedAt - left.updatedAt).slice(0, 24); + +export const summarizeRateLimit = (snapshot: any): IslandflowAiRateLimitSnapshot => ({ + limitId: typeof snapshot?.limitId === "string" ? snapshot.limitId : null, + limitName: typeof snapshot?.limitName === "string" ? snapshot.limitName : null, + primary: snapshot?.primary + ? { + usedPercent: Number(snapshot.primary.usedPercent ?? 0), + windowDurationMins: + snapshot.primary.windowDurationMins === null || snapshot.primary.windowDurationMins === undefined + ? null + : Number(snapshot.primary.windowDurationMins), + resetsAt: + snapshot.primary.resetsAt === null || snapshot.primary.resetsAt === undefined + ? null + : Number(snapshot.primary.resetsAt) + } + : null, + secondary: snapshot?.secondary + ? { + usedPercent: Number(snapshot.secondary.usedPercent ?? 0), + windowDurationMins: + snapshot.secondary.windowDurationMins === null || snapshot.secondary.windowDurationMins === undefined + ? null + : Number(snapshot.secondary.windowDurationMins), + resetsAt: + snapshot.secondary.resetsAt === null || snapshot.secondary.resetsAt === undefined + ? null + : Number(snapshot.secondary.resetsAt) + } + : null, + planType: snapshot?.planType ?? null, + reachedType: snapshot?.rateLimitReachedType ?? null, + hasCredits: + snapshot?.credits?.hasCredits === undefined || snapshot?.credits?.hasCredits === null + ? null + : Boolean(snapshot.credits.hasCredits), + unlimitedCredits: + snapshot?.credits?.unlimited === undefined || snapshot?.credits?.unlimited === null + ? null + : Boolean(snapshot.credits.unlimited), + creditsBalance: typeof snapshot?.credits?.balance === "string" ? snapshot.credits.balance : null +}); + +export const createAppServerChildEnv = ( + profileMode: IslandflowAiProfileMode, + baseEnv: NodeJS.ProcessEnv = process.env +): NodeJS.ProcessEnv => { + const childEnv = { ...baseEnv }; + if (profileMode !== "api-key") { + delete childEnv.OPENAI_API_KEY; + delete childEnv.CODEX_API_KEY; + } + return childEnv; +}; + +const createTaskSnapshot = (request: IslandflowAiTaskRequest): Pick => { + switch (request.kind) { + case "smart-money-explain": + return { + kind: request.kind, + title: "Explain smart money event", + subtitle: `${request.context.event.underlying_id} · ${request.context.event.primary_direction}` + }; + case "smart-money-skeptic": + return { + kind: request.kind, + title: "Counter-thesis pass", + subtitle: `${request.context.event.underlying_id} · skepticism` + }; + case "smart-money-burst-summary": + return { + kind: request.kind, + title: "Burst summary", + subtitle: `${request.context.event.underlying_id} · related packets` + }; + case "watchlist-synthesis": + return { + kind: request.kind, + title: "Watchlist synthesis", + subtitle: `${request.context.event.underlying_id} · setups` + }; + case "replay-postmortem": + return { + kind: request.kind, + title: "Replay postmortem", + subtitle: `${request.context.ticker ?? "All symbols"} · replay session` + }; + case "screen-compile": + return { + kind: request.kind, + title: "Natural-language screen", + subtitle: request.context.prompt + }; + } + + throw new Error("Unsupported Copilot task kind."); +}; + +const SMART_MONEY_RUBRIC = [ + "Treat the deterministic classifier and event payload as the source of truth.", + "Act as an evidence interpreter, not the live classifier.", + "Use only the provided structured payloads, do not call tools or inspect the filesystem.", + "Lead with the clearest thesis, but include uncertainty, missing evidence, and alternate explanations.", + "Prefer practical market structure language: aggressor side, concentration, event timing, IV shock, NBBO quality, and packet construction.", + "Do not pretend to know price action or fundamentals beyond the supplied data.", + "When the data suggests retail frenzy, dealer hedging, volatility selling, or arbitrage, say so plainly.", + "Keep the answer terse, structured, and useful under pressure." +].join("\n"); + +const BASE_INSTRUCTIONS = [ + "You are Islandflow Analyst Copilot.", + "Work only from the structured Islandflow context provided in the user message.", + "Never call tools, never browse, and never inspect files.", + "If evidence is missing or ambiguous, say that directly." +].join("\n"); + +const buildUserPrompt = (request: IslandflowAiTaskRequest): string => { + switch (request.kind) { + case "smart-money-explain": + return [ + "Explain this selected smart-money event for a trader who wants the key evidence fast.", + "Output sections named Thesis, Evidence, Caveats, and What To Watch.", + JSON.stringify(request.context, null, 2) + ].join("\n\n"); + case "smart-money-skeptic": + return [ + "Run a skepticism pass on this selected smart-money event.", + "Output sections named Why It Might Be Wrong, Alternate Microstructure Explanations, Missing Evidence, and Confidence Check.", + JSON.stringify(request.context, null, 2) + ].join("\n\n"); + case "smart-money-burst-summary": + return [ + "Summarize the burst across the related packets for this selected smart-money event.", + "Output sections named Burst Read, Packet Relationships, Quality Flags, and Trading Relevance.", + JSON.stringify(request.context, null, 2) + ].join("\n\n"); + case "watchlist-synthesis": + return [ + "Turn this event into a practical watchlist and setup brief.", + "Output sections named Watchlist, Trigger Levels Or Conditions, Invalidations, and Session Notes.", + JSON.stringify(request.context, null, 2) + ].join("\n\n"); + case "replay-postmortem": + return [ + "Write a replay postmortem from this structured replay slice.", + "Output sections named Session Read, Best Evidence, What Was Noise, and Follow-up Questions.", + JSON.stringify(request.context, null, 2) + ].join("\n\n"); + case "screen-compile": + return [ + "Compile this natural-language screen into the existing Islandflow filter model where possible.", + "Return only valid JSON that matches the requested schema.", + JSON.stringify(request.context, null, 2) + ].join("\n\n"); + } + + throw new Error("Unsupported Copilot task kind."); +}; + +const buildScreenOutputSchema = () => ({ + type: "object", + additionalProperties: false, + required: ["compiledFilters", "rationale", "unhandledClauses", "sanitizedPrompt"], + properties: { + compiledFilters: { + anyOf: [ + { + type: "object", + additionalProperties: false, + properties: { + view: { type: "string", enum: ["signal", "raw"] }, + securityTypes: { + type: "array", + items: { type: "string", enum: ["stock", "etf"] } + }, + nbboSides: { + type: "array", + items: { type: "string", enum: ["AA", "A", "MID", "B", "BB", "MISSING", "STALE"] } + }, + optionTypes: { + type: "array", + items: { type: "string", enum: ["call", "put"] } + }, + minNotional: { type: "number", minimum: 0 } + } + }, + { type: "null" } + ] + }, + rationale: { type: "string" }, + unhandledClauses: { + type: "array", + items: { type: "string" } + }, + sanitizedPrompt: { type: "string" } + } +}); + +const createUsageStore = (): PersistedUsageStore => ({ + version: 1, + turns: {} +}); + +class CodexAppServerClient { + private child: ChildProcessWithoutNullStreams | null = null; + private readonly pending = new Map< + number, + { resolve: (value: any) => void; reject: (error: Error) => void; timeout: ReturnType } + >(); + private buffer = ""; + private nextId = 1; + + constructor( + private readonly sandboxCwd: string, + private readonly onNotification: (method: string, params: unknown) => Promise | void, + private readonly onExit: (reason: string) => Promise | void + ) {} + + async start(profileMode: IslandflowAiProfileMode): Promise { + if (this.child) { + return; + } + + await mkdir(this.sandboxCwd, { recursive: true }); + + this.child = spawn("codex", ["app-server"], { + stdio: ["pipe", "pipe", "pipe"], + env: createAppServerChildEnv(profileMode) + }); + + this.child.stdout.setEncoding("utf8"); + this.child.stderr.setEncoding("utf8"); + + this.child.stdout.on("data", (chunk: string) => { + this.buffer += chunk; + void this.flushBuffer(); + }); + + this.child.stderr.on("data", (chunk: string) => { + console.warn(`[desktop-ai] ${chunk.trim()}`); + }); + + this.child.once("exit", (code, signal) => { + this.child = null; + this.buffer = ""; + for (const [id, pending] of this.pending.entries()) { + clearTimeout(pending.timeout); + pending.reject(new Error(`Codex app-server exited before replying to request ${id}.`)); + } + this.pending.clear(); + void this.onExit(`app-server exited${code !== null ? ` (${code})` : ""}${signal ? ` via ${signal}` : ""}`); + }); + + await this.request("initialize", { + clientInfo: { + name: "islandflow-desktop", + title: "Islandflow Desktop", + version: "0.1.0" + }, + capabilities: { + experimentalApi: true, + requestAttestation: false, + optOutNotificationMethods: [ + "app/list/updated", + "remoteControl/status/changed", + "skills/changed", + "plugin/installed" + ] + } + }); + + this.notify("initialized"); + } + + async stop(): Promise { + if (!this.child) { + return; + } + this.child.kill("SIGTERM"); + this.child = null; + } + + async request(method: string, params: unknown): Promise { + if (!this.child) { + throw new Error("Codex app-server is not running."); + } + + const id = this.nextId++; + const payload = JSON.stringify({ id, method, params }) + "\n"; + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.pending.delete(id); + reject(new Error(`Timed out waiting for ${method}.`)); + }, 30_000); + + this.pending.set(id, { resolve, reject, timeout }); + this.child?.stdin.write(payload); + }); + } + + private notify(method: string, params?: unknown): void { + if (!this.child) { + return; + } + this.child.stdin.write(JSON.stringify(params === undefined ? { method } : { method, params }) + "\n"); + } + + private async flushBuffer(): Promise { + while (true) { + const newlineIndex = this.buffer.indexOf("\n"); + if (newlineIndex === -1) { + return; + } + + const line = this.buffer.slice(0, newlineIndex).trim(); + this.buffer = this.buffer.slice(newlineIndex + 1); + if (!line) { + continue; + } + + const message = JSON.parse(line) as JsonRpcMessage; + if ("id" in message && "result" in message) { + const pending = this.pending.get(message.id); + if (pending) { + clearTimeout(pending.timeout); + this.pending.delete(message.id); + pending.resolve(message.result); + } + continue; + } + + if ("id" in message && "error" in message) { + const pending = this.pending.get(message.id); + if (pending) { + clearTimeout(pending.timeout); + this.pending.delete(message.id); + pending.reject(new Error(message.error.message ?? `Request ${message.id} failed.`)); + } + continue; + } + + if (typeof (message as Partial).id === "number" && "method" in message) { + this.respondUnsupported(message as JsonRpcServerRequest); + continue; + } + + if ("method" in message) { + await this.onNotification(message.method, message.params); + } + } + } + + private respondUnsupported(message: JsonRpcServerRequest): void { + if (!this.child) { + return; + } + this.child.stdin.write( + JSON.stringify({ + id: message.id, + error: { + message: `Islandflow desktop does not support server request ${message.method}.` + } + }) + "\n" + ); + } +} + +export class IslandflowDesktopAiService { + private readonly preferencesPath: string; + private readonly usagePath: string; + private readonly sandboxCwd: string; + private readonly client: CodexAppServerClient; + private readonly activeTasksByThreadId = new Map(); + private usageStore: PersistedUsageStore = createUsageStore(); + private state: IslandflowAiState = createInitialState(); + private serviceTier: string | null = null; + private started = false; + + constructor( + userDataPath: string, + private readonly openExternalUrl: OpenExternalFn, + private readonly publishState: (state: IslandflowAiState) => void + ) { + this.preferencesPath = path.join(userDataPath, PREFERENCES_FILE); + this.usagePath = path.join(userDataPath, USAGE_FILE); + this.sandboxCwd = path.join(userDataPath, APP_SERVER_SANDBOX_CWD); + this.client = new CodexAppServerClient( + this.sandboxCwd, + async (method, params) => { + await this.handleNotification(method, params); + }, + async (reason) => { + this.state.transportStatus = "restarting"; + this.state.transportError = reason; + this.failActiveTasks(reason); + this.emitState(); + } + ); + } + + async start(): Promise { + if (this.started) { + return; + } + this.started = true; + await mkdir(path.dirname(this.preferencesPath), { recursive: true }); + await mkdir(this.sandboxCwd, { recursive: true }); + await this.loadPreferences(); + await this.loadUsageStore(); + await this.ensureClientReady(); + } + + getState(): IslandflowAiState { + return this.state; + } + + async loginWithBrowser(): Promise { + await this.start(); + await this.ensureClientReady(); + + const result = await this.client.request("account/login/start", { + type: "chatgpt", + codexStreamlinedLogin: true + }); + + this.state.account.login = { + status: "browser_pending", + message: "Waiting for browser sign-in to complete.", + loginId: String(result.loginId), + authUrl: String(result.authUrl) + }; + this.emitState(); + await this.openExternalUrl(String(result.authUrl)); + } + + async loginWithDeviceCode(): Promise { + await this.start(); + await this.ensureClientReady(); + + const result = await this.client.request("account/login/start", { + type: "chatgptDeviceCode" + }); + + this.state.account.login = { + status: "device_code_pending", + message: "Enter the device code in your browser to finish sign-in.", + loginId: String(result.loginId), + verificationUrl: String(result.verificationUrl), + userCode: String(result.userCode) + }; + this.emitState(); + await this.openExternalUrl(String(result.verificationUrl)); + } + + async cancelLogin(): Promise { + const login = this.state.account.login; + if (login.status !== "browser_pending" && login.status !== "device_code_pending") { + return; + } + await this.client.request("account/login/cancel", { loginId: login.loginId }); + this.state.account.login = { status: "idle", message: "Login cancelled." }; + this.emitState(); + } + + async logout(): Promise { + await this.client.request("account/logout", {}); + this.state.account.loggedIn = false; + this.state.account.email = null; + this.state.account.planType = null; + this.state.account.login = { status: "idle", message: "Logged out." }; + this.emitState(); + } + + async updatePreferences( + next: Partial<{ model: string | null; reasoningEffort: IslandflowAiReasoningEffort | null }> + ): Promise { + this.state.preferences = { + model: next.model === undefined ? this.state.preferences.model : next.model, + reasoningEffort: + next.reasoningEffort === undefined + ? this.state.preferences.reasoningEffort + : next.reasoningEffort + }; + await this.savePreferences(); + this.emitState(); + } + + async runTask(rawRequest: unknown): Promise<{ taskId: string }> { + await this.start(); + + if (!this.state.account.loggedIn) { + throw new Error("Log into a ChatGPT or Codex account first."); + } + + const request = IslandflowAiTaskRequestSchema.parse(rawRequest); + const meta = createTaskSnapshot(request); + const taskId = `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; + const task: IslandflowAiTaskSnapshot = { + taskId, + kind: meta.kind, + title: meta.title, + subtitle: meta.subtitle, + status: "queued", + createdAt: Date.now(), + updatedAt: Date.now(), + threadId: null, + turnId: null, + model: this.state.preferences.model, + reasoningEffort: this.state.preferences.reasoningEffort, + text: "", + error: null, + compiledScreen: null + }; + + this.state.tasks = compactTaskList([task, ...this.state.tasks]); + this.emitState(); + + try { + await this.ensureClientReady(); + + const thread = await this.client.request("thread/start", { + model: this.state.preferences.model ?? undefined, + cwd: this.sandboxCwd, + approvalPolicy: "never", + sandbox: "read-only", + serviceName: APP_SERVER_SERVICE_NAME, + baseInstructions: BASE_INSTRUCTIONS, + developerInstructions: SMART_MONEY_RUBRIC, + ephemeral: true, + serviceTier: this.serviceTier ?? undefined + }); + + this.activeTasksByThreadId.set(thread.thread.id, { + taskId, + taskKind: task.kind, + taskTitle: task.title, + profileId: this.state.selectedProfileId + }); + this.patchTask(taskId, { + status: "running", + threadId: thread.thread.id, + model: thread.model, + reasoningEffort: thread.reasoningEffort ?? this.state.preferences.reasoningEffort + }); + + const turn = await this.client.request("turn/start", { + threadId: thread.thread.id, + input: [ + { + type: "text", + text: buildUserPrompt(request), + text_elements: [] + } + ], + model: this.state.preferences.model ?? undefined, + effort: this.state.preferences.reasoningEffort ?? undefined, + serviceTier: this.serviceTier ?? undefined, + outputSchema: request.kind === "screen-compile" ? buildScreenOutputSchema() : undefined + }); + + this.patchTask(taskId, { + turnId: turn.turn.id + }); + + return { taskId }; + } catch (error) { + this.patchTask(taskId, { + status: "failed", + error: error instanceof Error ? error.message : String(error) + }); + throw error; + } + } + + private async ensureClientReady(): Promise { + const selectedProfile = this.resolveSelectedProfileMode(); + this.state.transportStatus = this.state.transportStatus === "restarting" ? "restarting" : "starting"; + this.emitState(); + + try { + await this.client.start(selectedProfile); + await this.refreshServerState(); + this.state.transportStatus = "ready"; + this.state.transportError = null; + this.emitState(); + } catch (error) { + this.state.transportStatus = "error"; + this.state.transportError = error instanceof Error ? error.message : String(error); + this.emitState(); + throw error; + } + } + + private async refreshServerState(): Promise { + const [config, models, account, auth, rateLimits] = await Promise.all([ + this.client.request("config/read", {}), + this.client.request("model/list", {}), + this.client.request("account/read", { refreshToken: false }), + this.client.request("getAuthStatus", {}), + this.client.request("account/rateLimits/read", {}) + ]); + + this.serviceTier = typeof config?.config?.service_tier === "string" ? config.config.service_tier : null; + const configModel = typeof config?.config?.model === "string" ? config.config.model : null; + const configReasoning = + config?.config?.model_reasoning_effort === null || config?.config?.model_reasoning_effort === undefined + ? null + : IslandflowAiReasoningEffortSchema.parse(config.config.model_reasoning_effort); + + if (!this.state.preferences.model) { + this.state.preferences.model = configModel; + } + if (!this.state.preferences.reasoningEffort) { + this.state.preferences.reasoningEffort = configReasoning ?? DEFAULT_REASONING; + } + + this.state.models = (Array.isArray(models?.data) ? models.data : []) + .filter((model: CodexModelRecord) => !model.hidden) + .map((model: CodexModelRecord): IslandflowAiModelSummary => ({ + id: model.id, + model: model.model, + displayName: model.displayName, + description: model.description, + isDefault: Boolean(model.isDefault), + supportedReasoningEfforts: model.supportedReasoningEfforts.map((entry) => entry.reasoningEffort), + defaultReasoningEffort: model.defaultReasoningEffort, + pricing: MODEL_PRICING[model.model] ?? null + })); + + this.state.account.loggedIn = Boolean(account?.account); + this.state.account.email = + account?.account?.type === "chatgpt" && typeof account.account.email === "string" + ? account.account.email + : null; + this.state.account.planType = + account?.account?.type === "chatgpt" ? (account.account.planType as IslandflowAiPlanType) : null; + this.state.account.authMode = auth?.authMethod ?? null; + this.state.account.requiresOpenaiAuth = Boolean(account?.requiresOpenaiAuth ?? auth?.requiresOpenaiAuth ?? true); + if (this.state.account.login.status === "idle") { + this.state.account.login = { + status: "idle", + message: this.state.account.loggedIn ? "Connected." : null + }; + } + + this.state.rateLimitsByLimitId = this.normalizeRateLimitBuckets(rateLimits); + this.rebuildUsageDashboard(); + } + + private normalizeRateLimitBuckets(payload: any): Record { + const bucketEntries = Object.entries(payload?.rateLimitsByLimitId ?? {}); + if (bucketEntries.length === 0 && payload?.rateLimits) { + const single = summarizeRateLimit(payload.rateLimits); + return { + [single.limitId ?? "default"]: single + }; + } + + return Object.fromEntries( + bucketEntries.map(([key, value]) => [key, summarizeRateLimit(value)]) + ); + } + + private async handleNotification(method: string, params: unknown): Promise { + switch (method) { + case "account/updated": { + const payload = params as { authMode: string | null; planType: IslandflowAiPlanType | null }; + this.state.account.authMode = payload.authMode as any; + this.state.account.planType = payload.planType; + this.emitState(); + return; + } + case "account/login/completed": { + const payload = params as { success: boolean; error: string | null }; + if (payload.success) { + this.state.account.login = { status: "idle", message: "Connected." }; + await this.refreshServerState(); + } else { + this.state.account.login = { + status: "error", + message: payload.error ?? "Login failed.", + loginId: null + }; + } + this.emitState(); + return; + } + case "account/rateLimits/updated": { + const payload = summarizeRateLimit((params as { rateLimits: unknown }).rateLimits); + this.state.rateLimitsByLimitId = { + ...this.state.rateLimitsByLimitId, + [payload.limitId ?? "default"]: payload + }; + this.emitState(); + return; + } + case "item/agentMessage/delta": { + const payload = params as { threadId: string; delta: string }; + const activeTask = this.activeTasksByThreadId.get(payload.threadId); + if (!activeTask) { + return; + } + const current = this.state.tasks.find((task) => task.taskId === activeTask.taskId); + if (!current) { + return; + } + this.patchTask(activeTask.taskId, { + text: current.text + payload.delta + }); + return; + } + case "item/completed": { + const payload = params as { + threadId: string; + item: { type: string; text?: string }; + }; + if (payload.item.type !== "agentMessage") { + return; + } + const activeTask = this.activeTasksByThreadId.get(payload.threadId); + if (!activeTask) { + return; + } + if (typeof payload.item.text === "string") { + this.patchTask(activeTask.taskId, { + text: payload.item.text + }); + } + return; + } + case "thread/tokenUsage/updated": { + const payload = params as { + threadId: string; + turnId: string; + tokenUsage: { + total: IslandflowAiTokenBreakdown; + last: IslandflowAiTokenBreakdown; + }; + }; + this.recordUsage(payload.threadId, payload.turnId, payload.tokenUsage.last); + return; + } + case "turn/completed": { + const payload = params as { + threadId: string; + turn: { + id: string; + status: string; + error: { message: string } | null; + }; + }; + const activeTask = this.activeTasksByThreadId.get(payload.threadId); + if (!activeTask) { + return; + } + const current = this.state.tasks.find((task) => task.taskId === activeTask.taskId); + if (!current) { + return; + } + + if (payload.turn.status === "failed") { + this.patchTask(activeTask.taskId, { + status: "failed", + error: payload.turn.error?.message ?? "The Copilot turn failed." + }); + } else { + let compiledScreen: IslandflowAiCompiledScreen | null = null; + let nextText = current.text; + if (current.kind === "screen-compile") { + compiledScreen = this.tryParseCompiledScreen(current.text); + if (compiledScreen) { + nextText = compiledScreen.rationale; + } + } + + this.patchTask(activeTask.taskId, { + status: "completed", + compiledScreen, + text: nextText, + error: null + }); + } + + this.activeTasksByThreadId.delete(payload.threadId); + return; + } + default: + return; + } + } + + private tryParseCompiledScreen(text: string): IslandflowAiCompiledScreen | null { + try { + return IslandflowAiCompiledScreenSchema.parse(JSON.parse(sanitizeJsonText(text))); + } catch { + return null; + } + } + + private recordUsage(threadId: string, turnId: string, rawBreakdown: IslandflowAiTokenBreakdown): void { + const activeTask = this.activeTasksByThreadId.get(threadId); + const currentTask = activeTask + ? this.state.tasks.find((task) => task.taskId === activeTask.taskId) + : null; + const breakdown = normalizeBreakdown(rawBreakdown); + const record: IslandflowAiUsageTurnRecord = { + threadId, + turnId, + taskId: currentTask?.taskId ?? null, + taskKind: currentTask?.kind ?? null, + taskTitle: currentTask?.title ?? null, + dayKey: isoDayKey(Date.now()), + profileId: activeTask?.profileId ?? this.state.selectedProfileId, + accountEmail: this.state.account.email, + planType: this.state.account.planType, + model: currentTask?.model ?? this.state.preferences.model, + breakdown, + normalizedCostUsd: estimateNormalizedCost(currentTask?.model ?? this.state.preferences.model, breakdown), + updatedAt: Date.now() + }; + + this.usageStore.turns[buildUsageKey(threadId, turnId)] = record; + void this.saveUsageStore(); + this.rebuildUsageDashboard(); + this.emitState(); + } + + private rebuildUsageDashboard(): void { + const records = Object.values(this.usageStore.turns).filter((record) => { + if (record.profileId !== this.state.selectedProfileId) { + return false; + } + if (this.state.account.email) { + return record.accountEmail === this.state.account.email; + } + return true; + }); + + const todayKey = isoDayKey(Date.now()); + this.state.usage = { + today: this.rollupUsage(records.filter((record) => record.dayKey === todayKey)), + lifetime: this.rollupUsage(records), + recentTurns: [...records].sort((left, right) => right.updatedAt - left.updatedAt).slice(0, 12) + }; + } + + private rollupUsage(records: IslandflowAiUsageTurnRecord[]): IslandflowAiUsageRollup { + const breakdown = records.reduce( + (accumulator, record) => addBreakdowns(accumulator, record.breakdown), + { ...EMPTY_BREAKDOWN } + ); + const normalizedCostUsd = records.reduce((accumulator, record) => accumulator + (record.normalizedCostUsd ?? 0), 0); + return { + breakdown, + normalizedCostUsd: Number(normalizedCostUsd.toFixed(6)), + turnCount: records.length, + activeDays: new Set(records.map((record) => record.dayKey)).size + }; + } + + private failActiveTasks(reason: string): void { + for (const activeTask of this.activeTasksByThreadId.values()) { + this.patchTask(activeTask.taskId, { + status: "failed", + error: reason + }); + } + this.activeTasksByThreadId.clear(); + } + + private patchTask(taskId: string, updates: Partial): void { + this.state.tasks = compactTaskList( + this.state.tasks.map((task) => + task.taskId === taskId + ? { + ...task, + ...updates, + updatedAt: Date.now() + } + : task + ) + ); + this.emitState(); + } + + private emitState(): void { + this.state.updatedAt = Date.now(); + this.publishState({ + ...this.state, + profiles: this.state.profiles.map((profile) => ({ + ...profile, + selected: profile.id === this.state.selectedProfileId + })), + tasks: compactTaskList(this.state.tasks) + }); + } + + private resolveSelectedProfileMode(): IslandflowAiProfileMode { + const selected = this.state.profiles.find((profile) => profile.id === this.state.selectedProfileId); + return IslandflowAiProfileModeSchema.parse(selected?.mode ?? "managed-chatgpt"); + } + + private async loadPreferences(): Promise { + try { + const raw = await readFile(this.preferencesPath, "utf8"); + const parsed = JSON.parse(raw) as PersistedPreferences; + this.state.preferences = { + model: typeof parsed.model === "string" ? parsed.model : null, + reasoningEffort: + parsed.reasoningEffort === null || parsed.reasoningEffort === undefined + ? DEFAULT_REASONING + : IslandflowAiReasoningEffortSchema.parse(parsed.reasoningEffort) + }; + } catch { + // Use defaults on first run or after malformed local state. + } + } + + private async savePreferences(): Promise { + const payload: PersistedPreferences = { + model: this.state.preferences.model, + reasoningEffort: this.state.preferences.reasoningEffort + }; + await writeFile(this.preferencesPath, JSON.stringify(payload, null, 2), "utf8"); + } + + private async loadUsageStore(): Promise { + try { + const raw = await readFile(this.usagePath, "utf8"); + const parsed = JSON.parse(raw) as PersistedUsageStore; + if (parsed.version === 1 && parsed.turns) { + this.usageStore = parsed; + } + } catch { + this.usageStore = createUsageStore(); + } + this.rebuildUsageDashboard(); + } + + private async saveUsageStore(): Promise { + await writeFile(this.usagePath, JSON.stringify(this.usageStore, null, 2), "utf8"); + } +} diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index e5006df..41d24a5 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -1,5 +1,6 @@ -import { app, BrowserWindow, shell } from "electron"; -import type { Event as ElectronEvent } from "electron"; +import { app, BrowserWindow, ipcMain, shell } from "electron"; +import type { Event as ElectronEvent, IpcMainInvokeEvent } from "electron"; +import { fileURLToPath } from "node:url"; import { DESKTOP_PRODUCTION_URL, @@ -7,11 +8,25 @@ import { isTrustedAppUrl, resolveDesktopStartUrl } from "./security.js"; +import { IslandflowDesktopAiService } from "./desktop-ai.js"; +import { + DESKTOP_AI_CANCEL_LOGIN, + DESKTOP_AI_GET_STATE, + DESKTOP_AI_LOGIN_BROWSER, + DESKTOP_AI_LOGIN_DEVICE, + DESKTOP_AI_LOGOUT, + DESKTOP_AI_RUN_TASK, + DESKTOP_AI_STATE_CHANNEL, + DESKTOP_AI_UPDATE_PREFERENCES +} from "./desktop-ai-ipc.js"; const WINDOW_BACKGROUND_COLOR = "#06080b"; const WINDOW_TITLE = "Islandflow"; let mainWindow: BrowserWindow | null = null; +let desktopAiService: IslandflowDesktopAiService | null = null; + +const PRELOAD_PATH = fileURLToPath(new URL("./preload.js", import.meta.url)); const canOpenExternalUrl = (sourceUrl: string, targetUrl: string): boolean => { return isTrustedAppUrl(sourceUrl) && isSafeExternalUrl(targetUrl); @@ -61,6 +76,7 @@ const createMainWindow = (): BrowserWindow => { title: WINDOW_TITLE, backgroundColor: WINDOW_BACKGROUND_COLOR, webPreferences: { + preload: PRELOAD_PATH, nodeIntegration: false, contextIsolation: true, sandbox: true, @@ -92,6 +108,68 @@ const createMainWindow = (): BrowserWindow => { return window; }; +const broadcastDesktopAiState = (): void => { + if (!desktopAiService) { + return; + } + + const state = desktopAiService.getState(); + for (const window of BrowserWindow.getAllWindows()) { + window.webContents.send(DESKTOP_AI_STATE_CHANNEL, state); + } +}; + +const getTrustedSenderUrl = (event: IpcMainInvokeEvent): string => { + const senderUrl = event.senderFrame?.url || event.sender.getURL(); + if (!isTrustedAppUrl(senderUrl)) { + throw new Error(`Rejected desktop AI IPC from untrusted origin: ${senderUrl || "unknown"}`); + } + + return senderUrl; +}; + +const registerDesktopAiIpc = (service: IslandflowDesktopAiService): void => { + const guard = (event: IpcMainInvokeEvent): void => { + getTrustedSenderUrl(event); + }; + + ipcMain.handle(DESKTOP_AI_GET_STATE, async (event) => { + guard(event); + await service.start(); + return service.getState(); + }); + + ipcMain.handle(DESKTOP_AI_LOGIN_BROWSER, async (event) => { + guard(event); + await service.loginWithBrowser(); + }); + + ipcMain.handle(DESKTOP_AI_LOGIN_DEVICE, async (event) => { + guard(event); + await service.loginWithDeviceCode(); + }); + + ipcMain.handle(DESKTOP_AI_CANCEL_LOGIN, async (event) => { + guard(event); + await service.cancelLogin(); + }); + + ipcMain.handle(DESKTOP_AI_LOGOUT, async (event) => { + guard(event); + await service.logout(); + }); + + ipcMain.handle(DESKTOP_AI_UPDATE_PREFERENCES, async (event, next) => { + guard(event); + await service.updatePreferences(next); + }); + + ipcMain.handle(DESKTOP_AI_RUN_TASK, async (event, request) => { + guard(event); + return service.runTask(request); + }); +}; + const ensureMainWindow = (): void => { if (mainWindow) { return; @@ -101,6 +179,20 @@ const ensureMainWindow = (): void => { }; app.whenReady().then(() => { + desktopAiService = new IslandflowDesktopAiService( + app.getPath("userData"), + async (url) => { + await shell.openExternal(url); + }, + () => { + broadcastDesktopAiState(); + } + ); + registerDesktopAiIpc(desktopAiService); + void desktopAiService.start().catch((error) => { + console.error("[desktop-ai] Failed to start Codex bridge:", error); + broadcastDesktopAiState(); + }); ensureMainWindow(); app.on("activate", () => { diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts new file mode 100644 index 0000000..ed6f8df --- /dev/null +++ b/apps/desktop/src/preload.ts @@ -0,0 +1,43 @@ +import { contextBridge, ipcRenderer } from "electron"; +import type { + IslandflowAiReasoningEffort, + IslandflowAiState, + IslandflowAiTaskRequest +} from "@islandflow/types"; +import { + DESKTOP_AI_CANCEL_LOGIN, + DESKTOP_AI_GET_STATE, + DESKTOP_AI_LOGIN_BROWSER, + DESKTOP_AI_LOGIN_DEVICE, + DESKTOP_AI_LOGOUT, + DESKTOP_AI_RUN_TASK, + DESKTOP_AI_STATE_CHANNEL, + DESKTOP_AI_UPDATE_PREFERENCES +} from "./desktop-ai-ipc.js"; + +const bridge = { + ai: { + getState: (): Promise => ipcRenderer.invoke(DESKTOP_AI_GET_STATE), + loginWithBrowser: (): Promise => ipcRenderer.invoke(DESKTOP_AI_LOGIN_BROWSER), + loginWithDeviceCode: (): Promise => ipcRenderer.invoke(DESKTOP_AI_LOGIN_DEVICE), + cancelLogin: (): Promise => ipcRenderer.invoke(DESKTOP_AI_CANCEL_LOGIN), + logout: (): Promise => ipcRenderer.invoke(DESKTOP_AI_LOGOUT), + updatePreferences: ( + next: Partial<{ model: string | null; reasoningEffort: IslandflowAiReasoningEffort | null }> + ): Promise => ipcRenderer.invoke(DESKTOP_AI_UPDATE_PREFERENCES, next), + runTask: (request: IslandflowAiTaskRequest): Promise<{ taskId: string }> => + ipcRenderer.invoke(DESKTOP_AI_RUN_TASK, request), + subscribe: (listener: (state: IslandflowAiState) => void): (() => void) => { + const handler = (_event: Electron.IpcRendererEvent, state: IslandflowAiState) => { + listener(state); + }; + + ipcRenderer.on(DESKTOP_AI_STATE_CHANNEL, handler); + return () => { + ipcRenderer.off(DESKTOP_AI_STATE_CHANNEL, handler); + }; + } + } +}; + +contextBridge.exposeInMainWorld("islandflowDesktop", bridge); diff --git a/apps/desktop/tsconfig.json b/apps/desktop/tsconfig.json index 5895037..de3fb15 100644 --- a/apps/desktop/tsconfig.json +++ b/apps/desktop/tsconfig.json @@ -2,8 +2,8 @@ "extends": "../../tsconfig.base.json", "compilerOptions": { "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", + "module": "ESNext", + "moduleResolution": "Bundler", "lib": ["ES2022"], "types": ["node"], "rootDir": "src", diff --git a/apps/web/app/charts/page.tsx b/apps/web/app/charts/page.tsx index 9d82bba..a2eb858 100644 --- a/apps/web/app/charts/page.tsx +++ b/apps/web/app/charts/page.tsx @@ -1,7 +1,7 @@ -import { redirect } from "next/navigation"; +import { ChartsRoute } from "../terminal"; export const dynamic = "force-dynamic"; export default function Page() { - redirect("/"); + return ; } diff --git a/apps/web/app/desktop-ai-panels.tsx b/apps/web/app/desktop-ai-panels.tsx new file mode 100644 index 0000000..65524dc --- /dev/null +++ b/apps/web/app/desktop-ai-panels.tsx @@ -0,0 +1,924 @@ +"use client"; + +import Link from "next/link"; +import { useMemo, useState, type ReactNode } from "react"; +import type { + AlertEvent, + ClassifierHitEvent, + FlowPacket, + IslandflowAiCompiledScreen, + IslandflowAiPlanType, + IslandflowAiRateLimitSnapshot, + IslandflowAiReasoningEffort, + IslandflowAiTaskKind, + OptionFlowFilters, + OptionPrint, + SmartMoneyEvent +} from "@islandflow/types"; +import { useDesktopAi } from "./desktop-ai"; + +const numberFormatter = new Intl.NumberFormat("en-US"); +const usdFormatter = new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + minimumFractionDigits: 2, + maximumFractionDigits: 4 +}); + +const humanizeValue = (value: string | null | undefined): string => { + if (!value) { + return "Unknown"; + } + return value + .replace(/_/g, " ") + .replace(/\b\w/g, (char) => char.toUpperCase()); +}; + +const formatTokens = (value: number): string => numberFormatter.format(value); + +const formatUsd = (value: number | null): string => (value === null ? "Unavailable" : usdFormatter.format(value)); + +const formatTimestamp = (value: number | null): string => { + if (!value) { + return "Not reported"; + } + return new Intl.DateTimeFormat("en-US", { + dateStyle: "medium", + timeStyle: "short" + }).format(value); +}; + +const formatPercent = (value: number): string => `${Math.round(value)}%`; + +const getTaskStatusLabel = (value: string): string => humanizeValue(value); + +const findTask = (tasks: T[], taskId: string | null): T | null => { + if (!taskId) { + return null; + } + return tasks.find((task) => task.taskId === taskId) ?? null; +}; + +const getCompiledScreenSummary = (compiled: IslandflowAiCompiledScreen): string[] => { + const filters = compiled.compiledFilters; + if (!filters) { + return []; + } + + const parts: string[] = []; + if (filters.view) { + parts.push(`View: ${filters.view}`); + } + if (filters.securityTypes?.length) { + parts.push(`Security: ${filters.securityTypes.join(", ")}`); + } + if (filters.optionTypes?.length) { + parts.push(`Options: ${filters.optionTypes.join(", ")}`); + } + if (filters.nbboSides?.length) { + parts.push(`NBBO: ${filters.nbboSides.join(", ")}`); + } + if (typeof filters.minNotional === "number") { + parts.push(`Min notional: $${numberFormatter.format(filters.minNotional)}`); + } + + return parts; +}; + +const CopilotPane = ({ + title, + eyebrow, + actions, + wide = false, + children +}: { + title: string; + eyebrow?: string; + actions?: ReactNode; + wide?: boolean; + children: ReactNode; +}) => { + return ( +
+
+
+
+ {eyebrow ?
{eyebrow}
: null} +

{title}

+
+
+ {actions ?
{actions}
: null} +
+
{children}
+
+ ); +}; + +const UsageBreakdown = ({ + title, + breakdown, + normalizedCostUsd, + turnCount, + activeDays +}: { + title: string; + breakdown: { + totalTokens: number; + inputTokens: number; + cachedInputTokens: number; + outputTokens: number; + reasoningOutputTokens: number; + }; + normalizedCostUsd: number | null; + turnCount: number; + activeDays: number; +}) => { + return ( +
+
+

{title}

+ {formatUsd(normalizedCostUsd)} +
+
+
+ Total tokens + {formatTokens(breakdown.totalTokens)} +
+
+ Input + {formatTokens(breakdown.inputTokens)} +
+
+ Cached input + {formatTokens(breakdown.cachedInputTokens)} +
+
+ Output + {formatTokens(breakdown.outputTokens)} +
+
+ Reasoning + {formatTokens(breakdown.reasoningOutputTokens)} +
+
+ Turns + {formatTokens(turnCount)} +
+
+ Active days + {formatTokens(activeDays)} +
+
+
+ ); +}; + +const RateLimitBoard = ({ limit }: { limit: IslandflowAiRateLimitSnapshot }) => { + return ( +
+
+
+ {limit.limitName ?? "Default rate window"} +

+ {limit.planType ? `Plan ${humanizeValue(limit.planType)}` : "Plan not reported"} +

+
+ {limit.reachedType ? {humanizeValue(limit.reachedType)} : null} +
+
+ {limit.primary ? ( +
+ Primary + {formatPercent(limit.primary.usedPercent)} +

Resets {formatTimestamp(limit.primary.resetsAt)}

+
+ ) : null} + {limit.secondary ? ( +
+ Secondary + {formatPercent(limit.secondary.usedPercent)} +

Resets {formatTimestamp(limit.secondary.resetsAt)}

+
+ ) : null} +
+ {limit.creditsBalance || limit.unlimitedCredits !== null ? ( +

+ Credits:{" "} + {limit.unlimitedCredits + ? "unlimited" + : limit.creditsBalance + ? limit.creditsBalance + : limit.hasCredits === false + ? "none" + : "not reported"} +

+ ) : null} +
+ ); +}; + +const TaskOutput = ({ + taskId, + emptyMessage +}: { + taskId: string | null; + emptyMessage: string; +}) => { + const { state } = useDesktopAi(); + const task = findTask(state.tasks, taskId); + + if (!task) { + return

{emptyMessage}

; + } + + return ( +
+
+
+ {task.title} +

+ {task.subtitle} · {getTaskStatusLabel(task.status)} +

+
+ {getTaskStatusLabel(task.status)} +
+ {task.error ?

{task.error}

: null} + {task.text ? 
{task.text}
: null} + {task.compiledScreen ? : null} +
+ ); +}; + +const CompiledScreenResult = ({ compiled }: { compiled: IslandflowAiCompiledScreen }) => { + const summary = getCompiledScreenSummary(compiled); + + return ( +
+ {summary.length > 0 ? ( +
+ {summary.map((item) => ( + + {item} + + ))} +
+ ) : ( +

No filter fields were compiled from this prompt.

+ )} + {compiled.unhandledClauses.length > 0 ? ( +
+
Unhandled clauses
+ {compiled.unhandledClauses.map((item) => ( +
+ {item} +
+ ))} +
+ ) : null} +
+ ); +}; + +const AccountSummary = ({ + loggedIn, + email, + planType +}: { + loggedIn: boolean; + email: string | null; + planType: IslandflowAiPlanType | null; +}) => { + return ( +
+
+

Desktop-only official Codex bridge

+

Analyst Copilot

+

+ Managed ChatGPT login stays user-scoped, deterministic smart-money classification stays in charge, and every + AI turn is tracked with exact token telemetry from the app-server. +

+
+
+
+ Account + {loggedIn ? email ?? "Connected" : "Disconnected"} +
+
+ Plan + {loggedIn ? humanizeValue(planType) : "Not connected"} +
+
+
+ ); +}; + +const LoginStatePanel = () => { + const { state, loginWithBrowser, loginWithDeviceCode, cancelLogin, logout } = useDesktopAi(); + const [busyAction, setBusyAction] = useState(null); + const [actionError, setActionError] = useState(null); + const loginState = state.account.login; + const actionsDisabled = busyAction !== null || !state.desktopAvailable; + + const runAction = async (label: string, action: () => Promise) => { + setBusyAction(label); + setActionError(null); + try { + await action(); + } catch (error) { + setActionError(error instanceof Error ? error.message : String(error)); + } finally { + setBusyAction(null); + } + }; + + return ( + + {state.account.loggedIn ? ( + + ) : ( + <> + + + + )} + {(loginState.status === "browser_pending" || loginState.status === "device_code_pending") && !state.account.loggedIn ? ( + + ) : null} + + } + > + +
+
+
Profile slots
+ {state.profiles.map((profile) => ( +
+
+ {profile.label} +

{profile.description}

+
+ + {profile.selected ? "Selected" : profile.statusLabel} + +
+ ))} +
+
+
Session status
+
+ Transport + {humanizeValue(state.transportStatus)} +
+
+ Auth mode + {humanizeValue(state.account.authMode)} +
+
+ OpenAI auth required + {state.account.requiresOpenaiAuth ? "Yes" : "No"} +
+ {state.transportError ?

{state.transportError}

: null} + {loginState.message ?

{loginState.message}

: null} + {loginState.status === "browser_pending" ? ( +
+ Browser login in progress +

Finish the ChatGPT sign-in flow in your browser. Islandflow will update automatically.

+
+ ) : null} + {loginState.status === "device_code_pending" ? ( +
+ Device code + 
{loginState.userCode}
+

Visit {loginState.verificationUrl} in any browser and enter the code above.

+
+ ) : null} + {actionError ?

{actionError}

: null} +
+
+ + ); +}; + +export function DesktopAiSettingsRoute() { + const { state, updatePreferences } = useDesktopAi(); + const [busyPreference, setBusyPreference] = useState<"model" | "reasoning" | null>(null); + const [preferenceError, setPreferenceError] = useState(null); + const rateLimits = Object.values(state.rateLimitsByLimitId); + const selectedModel = state.preferences.model ?? ""; + const selectedReasoning = state.preferences.reasoningEffort ?? ""; + + const savePreference = async ( + key: "model" | "reasoning", + next: Partial<{ model: string | null; reasoningEffort: IslandflowAiReasoningEffort | null }> + ) => { + setBusyPreference(key); + setPreferenceError(null); + try { + await updatePreferences(next); + } catch (error) { + setPreferenceError(error instanceof Error ? error.message : String(error)); + } finally { + setBusyPreference(null); + } + }; + + return ( +
+ {!state.desktopAvailable ? ( + +
+

+ AI controls are intentionally read-only in the browser build. Open Islandflow Desktop to use managed ChatGPT + login, structured Copilot turns, and app-server token telemetry. +

+
+ + ) : null} + + + +
+ +
+ + +
+
+ {state.models.map((model) => ( +
+
+ {model.displayName} +

{model.description}

+
+
+ {model.model} + {model.pricing ? {formatUsd(model.pricing.inputUsdPer1MTokens)} / 1M input : null} +
+
+ ))} +
+ {state.models.find((model) => model.model === state.preferences.model)?.pricing ? ( +

+ Normalized estimates use current API pricing for the selected model, not your literal ChatGPT subscription bill. +

+ ) : null} + {preferenceError ?

{preferenceError}

: null} + + + + {rateLimits.length === 0 ? ( +

No rate-limit snapshots have been reported yet.

+ ) : ( +
+ {rateLimits.map((limit) => ( + + ))} +
+ )} + + + +
+ + +
+ + + + {state.usage.recentTurns.length === 0 ? ( +

No tracked turns yet.

+ ) : ( +
+ {state.usage.recentTurns.map((turn) => ( +
+
+ {turn.taskTitle ?? "Ad hoc turn"} +

+ {turn.model ?? "default"} · {formatTimestamp(turn.updatedAt)} +

+
+
+ {formatTokens(turn.breakdown.totalTokens)} tok + {formatUsd(turn.normalizedCostUsd)} +
+
+ ))} +
+ )} + + + + {state.tasks.length === 0 ? ( +

No Copilot tasks have been run yet.

+ ) : ( +
+ {state.tasks.map((task) => ( +
+
+ {task.title} +

+ {task.subtitle} · {humanizeValue(task.model)} +

+
+ {getTaskStatusLabel(task.status)} +
+ ))} +
+ )} + +
+
+ ); +} + +const requireDesktopActionCopy = (desktopAvailable: boolean, loggedIn: boolean): string => { + if (!desktopAvailable) { + return "This control is desktop-only. Open Islandflow Desktop to run Copilot tasks."; + } + if (!loggedIn) { + return "Connect a ChatGPT or Codex account in Settings before running Copilot analysis."; + } + return ""; +}; + +const SmartMoneyTaskButton = ({ + label, + kind, + symbol, + disabled, + busyKind, + onRun +}: { + label: string; + kind: IslandflowAiTaskKind; + symbol: string; + disabled: boolean; + busyKind: IslandflowAiTaskKind | null; + onRun: (kind: IslandflowAiTaskKind) => void; +}) => { + return ( + + ); +}; + +export function SmartMoneyCopilotPanel({ + event, + flowPacket, + evidencePrints, + relatedPackets +}: { + event: SmartMoneyEvent; + flowPacket: FlowPacket | null; + evidencePrints: OptionPrint[]; + relatedPackets: FlowPacket[]; +}) { + const { bridgeAvailable, state, runTask } = useDesktopAi(); + const [busyKind, setBusyKind] = useState(null); + const [activeTaskId, setActiveTaskId] = useState(null); + const [taskError, setTaskError] = useState(null); + const disabledCopy = requireDesktopActionCopy(bridgeAvailable, state.account.loggedIn); + const actionsDisabled = !bridgeAvailable || !state.account.loggedIn; + + const handleRun = async (kind: IslandflowAiTaskKind) => { + setBusyKind(kind); + setTaskError(null); + try { + const result = await runTask({ + kind: kind as + | "smart-money-explain" + | "smart-money-skeptic" + | "smart-money-burst-summary" + | "watchlist-synthesis", + context: { + event, + flowPacket, + evidencePrints, + relatedPackets + } + }); + setActiveTaskId(result.taskId); + } catch (error) { + setTaskError(error instanceof Error ? error.message : String(error)); + } finally { + setBusyKind(null); + } + }; + + return ( +
+
+
+
Analyst Copilot
+

Structured interpretation only, the deterministic classifier remains the source of truth.

+
+ + AI settings + +
+
+ void handleRun(kind)} + /> + void handleRun(kind)} + /> + void handleRun(kind)} + /> + void handleRun(kind)} + /> +
+ {disabledCopy ?

{disabledCopy}

: null} + {taskError ?

{taskError}

: null} + +
+ ); +} + +export function ReplayCopilotPanel({ + ticker, + flowFilters, + alerts, + smartMoneyEvents, + classifierHits, + flowPackets, + optionPrints +}: { + ticker: string | null; + flowFilters: OptionFlowFilters; + alerts: AlertEvent[]; + smartMoneyEvents: SmartMoneyEvent[]; + classifierHits: ClassifierHitEvent[]; + flowPackets: FlowPacket[]; + optionPrints: OptionPrint[]; +}) { + const { bridgeAvailable, state, runTask } = useDesktopAi(); + const [activeTaskId, setActiveTaskId] = useState(null); + const [busy, setBusy] = useState(false); + const [taskError, setTaskError] = useState(null); + const disabledCopy = requireDesktopActionCopy(bridgeAvailable, state.account.loggedIn); + const actionsDisabled = busy || !bridgeAvailable || !state.account.loggedIn; + + const handleRun = async () => { + setBusy(true); + setTaskError(null); + try { + const result = await runTask({ + kind: "replay-postmortem", + context: { + ticker, + flowFilters, + alerts, + smartMoneyEvents, + classifierHits, + flowPackets, + optionPrints + } + }); + setActiveTaskId(result.taskId); + } catch (error) { + setTaskError(error instanceof Error ? error.message : String(error)); + } finally { + setBusy(false); + } + }; + + return ( + + + AI settings + + + + } + > +

+ Copilot uses the current replay slice only: ticker scope, flow filters, visible alerts, classifier hits, packets, and option prints. +

+ {disabledCopy ?

{disabledCopy}

: null} + {taskError ?

{taskError}

: null} + + + ); +} + +export function ScreenCompilerPanel({ + currentFilters, + onApplyFilters +}: { + currentFilters: OptionFlowFilters; + onApplyFilters: (next: OptionFlowFilters) => void; +}) { + const { bridgeAvailable, state, runTask } = useDesktopAi(); + const [prompt, setPrompt] = useState(""); + const [activeTaskId, setActiveTaskId] = useState(null); + const [busy, setBusy] = useState(false); + const [taskError, setTaskError] = useState(null); + const activeTask = useMemo(() => findTask(state.tasks, activeTaskId), [state.tasks, activeTaskId]); + const disabledCopy = requireDesktopActionCopy(bridgeAvailable, state.account.loggedIn); + const actionsDisabled = busy || !bridgeAvailable || !state.account.loggedIn; + + const handleCompile = async () => { + const trimmedPrompt = prompt.trim(); + if (!trimmedPrompt) { + setTaskError("Write a screen request first."); + return; + } + + setBusy(true); + setTaskError(null); + try { + const result = await runTask({ + kind: "screen-compile", + context: { + prompt: trimmedPrompt, + currentFilters + } + }); + setActiveTaskId(result.taskId); + } catch (error) { + setTaskError(error instanceof Error ? error.message : String(error)); + } finally { + setBusy(false); + } + }; + + const compiledFilters = activeTask?.compiledScreen?.compiledFilters ?? null; + + return ( + + + AI settings + + + + } + > +
+