From b66d2d1e34cb66e10acf4949152295dfe5fbbfbe Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Wed, 17 Jun 2026 13:08:44 -0400 Subject: [PATCH 1/2] add future market-data sample calibration bead --- .beads/issues.jsonl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 70a16df..458d00f 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -51,7 +51,7 @@ {"_type":"issue","id":"islandflow-259.3.2","title":"Split synthetic phase 03b: expected-output manifests","description":"PR-sized child of synthetic phase 03. Add expected-output manifest entries and comparison hooks for generated scenarios.","acceptance_criteria":"Expected-output manifests capture derived event expectations, alert/no-alert outcomes, evidence requirements, and false-positive penalties; comparison output is deterministic and focused for review.","spec_id":"docs/implementation/synthetic-market-data/03-scenarios-labels-expected-outputs.md","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-06-16T17:38:53Z","created_by":"dirtydishes","updated_at":"2026-06-16T17:38:53Z","labels":["phase","phase-split","planning","synthetic-market-data"],"dependencies":[{"issue_id":"islandflow-259.3.2","depends_on_id":"islandflow-259.3","type":"parent-child","created_at":"2026-06-16T13:38:52Z","created_by":"dirtydishes","metadata":"{}"},{"issue_id":"islandflow-259.3.2","depends_on_id":"islandflow-259.3.1","type":"blocks","created_at":"2026-06-16T13:39:15Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-259.3.1","title":"Split synthetic phase 03a: scenario catalog and labels","description":"PR-sized child of synthetic phase 03. Define the deterministic scenario catalog and separate label records without changing emitted market event contracts.","acceptance_criteria":"Scenario definitions are deterministic and named; labels are stored separately from market events; each label records run/scenario/event refs, expected class/direction/confidence band, and required or forbidden evidence.","spec_id":"docs/implementation/synthetic-market-data/03-scenarios-labels-expected-outputs.md","status":"open","priority":2,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-06-16T17:38:51Z","created_by":"dirtydishes","updated_at":"2026-06-16T17:38:51Z","labels":["phase","phase-split","planning","synthetic-market-data"],"dependencies":[{"issue_id":"islandflow-259.3.1","depends_on_id":"islandflow-259.3","type":"parent-child","created_at":"2026-06-16T13:38:51Z","created_by":"dirtydishes","metadata":"{}"},{"issue_id":"islandflow-259.3.1","depends_on_id":"islandflow-zxh.2","type":"blocks","created_at":"2026-06-16T13:39:14Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"islandflow-zxh.5","title":"Smart-flow phase 05: API/UI explainability","description":"Expose evidence-backed smart-flow results through API, websocket, and UI explainability surfaces as described in docs/implementation/smart-money/05-api-ui-explainability.md. This comes after replay-validated contracts and scoring behavior exist.","acceptance_criteria":"API/WS surfaces expose evidence, hypotheses, insights, alternatives, and abstention clearly; UI distinguishes evidence quality, confidence, conviction, and why-not signals; legacy endpoints remain aliases only where needed; replay/golden checks support changed projections.","spec_id":"docs/implementation/smart-money/05-api-ui-explainability.md","status":"open","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-06-16T17:38:47Z","created_by":"dirtydishes","updated_at":"2026-06-16T17:38:47Z","labels":["phase","planning","smart-flow","smart-money"],"dependencies":[{"issue_id":"islandflow-zxh.5","depends_on_id":"islandflow-259.5","type":"blocks","created_at":"2026-06-16T13:39:11Z","created_by":"dirtydishes","metadata":"{}"},{"issue_id":"islandflow-zxh.5","depends_on_id":"islandflow-zxh","type":"parent-child","created_at":"2026-06-16T13:38:47Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} -{"_type":"issue","id":"islandflow-259.5","title":"Synthetic market-data phase 05: demo and load profiles","description":"Implement named deterministic demo and load profiles as described in docs/implementation/synthetic-market-data/05-demo-load-profiles.md. This comes after foundations, manifests, scenarios, and replay are in place.","acceptance_criteria":"Demo profiles select named deterministic runs/scenarios instead of ambient randomness; load profiles scale rates without changing event semantics; live/demo emitters remain thin consumers of the synthetic package; UI/demo controls do not precede deterministic foundations.","spec_id":"docs/implementation/synthetic-market-data/05-demo-load-profiles.md","status":"open","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-06-16T17:38:46Z","created_by":"dirtydishes","updated_at":"2026-06-16T17:38:46Z","labels":["phase","planning","synthetic-market-data"],"dependencies":[{"issue_id":"islandflow-259.5","depends_on_id":"islandflow-259","type":"parent-child","created_at":"2026-06-16T13:38:45Z","created_by":"dirtydishes","metadata":"{}"},{"issue_id":"islandflow-259.5","depends_on_id":"islandflow-zxh.4","type":"blocks","created_at":"2026-06-16T13:39:10Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":1,"dependent_count":3,"comment_count":0} +{"_type":"issue","id":"islandflow-259.5","title":"Synthetic market-data phase 05: demo and load profiles","description":"Implement named deterministic demo and load profiles as described in docs/implementation/synthetic-market-data/05-demo-load-profiles.md. This comes after foundations, manifests, scenarios, and replay are in place.","acceptance_criteria":"Demo profiles select named deterministic runs/scenarios instead of ambient randomness; load profiles scale rates without changing event semantics; live/demo emitters remain thin consumers of the synthetic package; UI/demo controls do not precede deterministic foundations.","spec_id":"docs/implementation/synthetic-market-data/05-demo-load-profiles.md","status":"open","priority":2,"issue_type":"feature","owner":"dishes@dpdrm.com","created_at":"2026-06-16T17:38:46Z","created_by":"dirtydishes","updated_at":"2026-06-16T17:38:46Z","labels":["phase","planning","synthetic-market-data"],"dependencies":[{"issue_id":"islandflow-259.5","depends_on_id":"islandflow-259","type":"parent-child","created_at":"2026-06-16T13:38:45Z","created_by":"dirtydishes","metadata":"{}"},{"issue_id":"islandflow-259.5","depends_on_id":"islandflow-zxh.4","type":"blocks","created_at":"2026-06-16T13:39:10Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":1,"dependent_count":4,"comment_count":0} {"_type":"issue","id":"islandflow-v6f","title":"Virtualize dashboard priority board","description":"Improve the root dashboard Priority Board readability and scrolling. Remove the redundant packet column, show packet IDs as secondary evidence metadata without the flowpacket prefix, rename the confusing Decision column, and use TanStack virtual scrolling for the row list.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-06-15T20:58:29Z","created_by":"dirtydishes","updated_at":"2026-06-15T21:01:37Z","started_at":"2026-06-15T20:58:34Z","closed_at":"2026-06-15T21:01:37Z","close_reason":"Completed Priority Board virtualization, copy cleanup, and validation.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-h0k","title":"Polish dashboard route","description":"Final polish pass for the root dashboard route. Align the command surface to the existing Islandflow terminal design system, tighten visual and copy details, and validate responsive behavior and build quality.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-06-15T20:04:12Z","created_by":"dirtydishes","updated_at":"2026-06-15T20:09:14Z","started_at":"2026-06-15T20:04:13Z","closed_at":"2026-06-15T20:09:14Z","close_reason":"Dashboard route polish shipped on dedicated branch: route title hierarchy cleaned up, command rail isolated from legacy header styles, mobile overflow/touch behavior added, and build/tests passed.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-j30","title":"Polish dashboard command header","description":"Live design polish for the dashboard command header. Simplify the page header copy to Dashboard, keep status and scope in one compact line, and replace the vague empty-filter copy with a clearer all-flow state.","status":"closed","priority":2,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-06-15T19:29:23Z","created_by":"dirtydishes","updated_at":"2026-06-15T19:31:00Z","started_at":"2026-06-15T19:29:28Z","closed_at":"2026-06-15T19:31:00Z","close_reason":"Dashboard command header polish shipped: simplified title, clarified scope/filter language, compacted status rail, and validated with web build plus focused terminal tests.","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -170,6 +170,7 @@ {"_type":"issue","id":"islandflow-zsy","title":"Expose Forgejo SSH on a direct DNS hostname","description":"git.deltaisland.io currently resolves through Cloudflare's proxy, so SSH on port 2222 does not complete even though the Forgejo container is listening on the host. If SSH-based git/beads workflows are desired, add a DNS-only hostname (or adjust the existing record) that points directly at the server for Forgejo SSH.","status":"open","priority":3,"issue_type":"task","created_at":"2026-05-17T10:34:06Z","created_by":"delta","updated_at":"2026-05-17T10:34:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-38p","title":"Add native deployment unit templates and rollback helpers","description":"The deploy helper now supports --runtime native, but the repo still relies on operator-managed systemd units and manual rollback. Add checked-in native deployment templates or provisioning guidance for the expected units, and consider lightweight rollback/smoke-test helpers once the host-native path is exercised on the real VPS.","status":"open","priority":3,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-05-15T23:46:42Z","created_by":"dirtydishes","updated_at":"2026-05-15T23:46:42Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-575","title":"Document smart-money event calendar env","description":"Document smart-money event-calendar environment configuration in env examples and README.\n","status":"closed","priority":3,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-05-05T06:57:14Z","created_by":"dirtydishes","updated_at":"2026-05-05T06:57:57Z","started_at":"2026-05-05T06:57:17Z","closed_at":"2026-05-05T06:57:57Z","close_reason":"Documented event-calendar env variables","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"islandflow-259.6.1","title":"Future synthetic calibration: evaluate free historical market-data samples","description":"Evaluate whether free or low-cost historical market-data samples can improve synthetic generator calibration without making historical data an MVP dependency. Candidate sources to investigate include Stooq equity bars, Alpaca/IEX historical equities, Alpha Vantage small pulls, Cboe options volume summaries, SEC MIDAS aggregate market-structure data, and limited Databento OPRA samples or signup credits for true options trade/NBBO microstructure. The intended pattern is real sample data -\u003e derived priors/histograms/parameter snapshots -\u003e deterministic synthetic runs, not committing raw vendor data as fixtures.","acceptance_criteria":"Document candidate free or credit-backed sources with coverage, limits, and redistribution/licensing constraints; identify which sources can support equities bars, equity trade/quote shape, options volume summaries, and true options print/NBBO samples; recommend a first legal tiny-sample experiment; define what derived parameter snapshots can be committed safely; explicitly state that raw vendor data should not be committed unless licensing allows it; preserve canonical synthetic outputs and separate labels/manifests; keep this outside the MVP blocker chain.","notes":"Planning note from 2026-06-17 discussion: free equity samples are realistic, but free historical options prints/NBBO are difficult and licensing-sensitive. Use this to remember the calibration idea after the deterministic MVP path exists.","spec_id":"docs/implementation/synthetic-market-data/99-future-historical-calibration.md","status":"open","priority":4,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-06-17T17:08:22Z","created_by":"dirtydishes","updated_at":"2026-06-17T17:08:22Z","labels":["calibration","future","market-data","planning","synthetic-market-data"],"dependencies":[{"issue_id":"islandflow-259.6.1","depends_on_id":"islandflow-259.5","type":"blocks","created_at":"2026-06-17T13:08:22Z","created_by":"dirtydishes","metadata":"{}"},{"issue_id":"islandflow-259.6.1","depends_on_id":"islandflow-259.6","type":"parent-child","created_at":"2026-06-17T13:08:22Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-zxh.6","title":"Future smart-flow phase 99: calibration","description":"Track future smart-flow calibration and research-grade validation as described in docs/implementation/smart-money/99-future-calibration.md. This is explicitly not an MVP dependency.","acceptance_criteria":"Future calibration is separated from MVP scoring; calibration datasets, policy versioning, and evaluation metrics are specified before implementation; abstention and confidence semantics remain auditable.","spec_id":"docs/implementation/smart-money/99-future-calibration.md","status":"open","priority":4,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-06-16T17:38:50Z","created_by":"dirtydishes","updated_at":"2026-06-16T17:38:50Z","labels":["future","planning","smart-flow","smart-money"],"dependencies":[{"issue_id":"islandflow-zxh.6","depends_on_id":"islandflow-259.6","type":"blocks","created_at":"2026-06-16T13:39:13Z","created_by":"dirtydishes","metadata":"{}"},{"issue_id":"islandflow-zxh.6","depends_on_id":"islandflow-zxh","type":"parent-child","created_at":"2026-06-16T13:38:50Z","created_by":"dirtydishes","metadata":"{}"},{"issue_id":"islandflow-zxh.6","depends_on_id":"islandflow-zxh.5","type":"blocks","created_at":"2026-06-16T13:39:12Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":2,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"islandflow-259.6","title":"Future synthetic market-data phase 99: historical calibration","description":"Track future historical calibration for the synthetic generator as described in docs/implementation/synthetic-market-data/99-future-historical-calibration.md. This is explicitly not an MVP dependency.","acceptance_criteria":"Historical calibration remains outside the MVP blocker chain; calibration inputs, privacy/market-data constraints, and parameter fitting strategy are documented before implementation; deterministic synthetic foundations continue to work without historical data.","spec_id":"docs/implementation/synthetic-market-data/99-future-historical-calibration.md","status":"open","priority":4,"issue_type":"task","owner":"dishes@dpdrm.com","created_at":"2026-06-16T17:38:49Z","created_by":"dirtydishes","updated_at":"2026-06-16T17:38:49Z","labels":["future","planning","synthetic-market-data"],"dependencies":[{"issue_id":"islandflow-259.6","depends_on_id":"islandflow-259","type":"parent-child","created_at":"2026-06-16T13:38:48Z","created_by":"dirtydishes","metadata":"{}"},{"issue_id":"islandflow-259.6","depends_on_id":"islandflow-259.5","type":"blocks","created_at":"2026-06-16T13:39:12Z","created_by":"dirtydishes","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"islandflow-iwg","title":"Summarize June 9 git activity for standup","description":"Create the daily standup summary for git activity on 2026-06-09, grounded in commits and touched files, and save the report in docs/general.","status":"closed","priority":4,"issue_type":"task","assignee":"dirtydishes","owner":"dishes@dpdrm.com","created_at":"2026-06-10T13:02:19Z","created_by":"dirtydishes","updated_at":"2026-06-10T13:04:44Z","started_at":"2026-06-10T13:02:30Z","closed_at":"2026-06-10T13:04:44Z","close_reason":"Created docs/general/2026-06-10-0902-standup-summary-2026-06-09.html with a git-grounded summary of June 9 activity.","dependency_count":0,"dependent_count":0,"comment_count":0} From 4506ed7ffa47418f493e32cfc7c80cfee27923ec Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Wed, 17 Jun 2026 13:36:23 -0400 Subject: [PATCH 2/2] add readable implementation roadmap docs --- docs/implementation/index.html | 543 ++++++++++++ .../smart-money/00-roadmap.html | 502 +++++++++++ docs/implementation/smart-money/index.html | 3 + .../synthetic-market-data/00-roadmap.html | 481 +++++++++++ .../synthetic-market-data/index.html | 3 + .../plans/smart-flow-architecture-review.html | 814 ++++++++++++++++++ ...hetic-market-data-architecture-review.html | 2 + 7 files changed, 2348 insertions(+) create mode 100644 docs/implementation/index.html create mode 100644 docs/implementation/smart-money/00-roadmap.html create mode 100644 docs/implementation/synthetic-market-data/00-roadmap.html create mode 100644 docs/plans/smart-flow-architecture-review.html diff --git a/docs/implementation/index.html b/docs/implementation/index.html new file mode 100644 index 0000000..bfb9380 --- /dev/null +++ b/docs/implementation/index.html @@ -0,0 +1,543 @@ + + + + + + Implementation Phase Plans + + + +
+
+
+

Implementation Map

+

Implementation Phase Plans

+

+ The active planning layer for synthetic market-data and smart-money/smart-flow architecture work. + Architecture reviews and research reports are background; phase documents and Beads issues define execution scope. +

+
+ Active scope guide + Synthetic market data + Smart flow +
+
+ +
+ +
+

Jump to

+ +
+ +
+

Document Precedence

+
+
+
    +
  1. Current Beads issue
    2. +
  2. Referenced phase document under docs/implementation/
    3. +
  3. Architecture plan under docs/plans/
    4. +
  4. Research report under docs/research-docs/
    5. +
+

+ This repository uses docs/research-docs/ for research reports; docs/research/ + is not present. Research reports provide rationale and useful constraints, but they do not add active + implementation scope unless that scope is explicitly pulled into a phase document and Beads issue. +

+
+
+
+ +
+

Source Plans

+
+ + +
+
+ +
+

Planning Rules

+
+
+
    +
  • Prefer small, reviewable PRs.
    • +
  • Do not implement an entire architecture plan at once.
    • +
  • Use Beads issues for execution tracking and dependency management.
    • +
  • Keep durable architecture and phase detail in these docs, not in long Beads descriptions.
    • +
  • Synthetic data must emit canonical market event types, not synthetic-only pipeline event types.
    • +
  • Synthetic labels must remain separate from emitted market events.
    • +
  • Smart-flow logic must distinguish facts, evidence, hypotheses, confidence, and abstention.
    • +
  • Historical calibration is future work, not an MVP dependency.
    • +
  • Early synthetic tests must not require Docker, ClickHouse, NATS, or Redis.
    • +
  • Synthetic foundations should come before demos, UI controls, or live service work.
    • +
+
+
+
+ +
+

Beads Map

+
+ + + + + + + + + + + + + + + + + + + + +
StreamEpicRoadmap
Synthetic market dataislandflow-259 - Plan synthetic market-data implementation phasesdocs/implementation/synthetic-market-data/00-roadmap.html
Smart money / smart flowislandflow-zxh - Plan smart-money to smart-flow implementation phasesdocs/implementation/smart-money/00-roadmap.html
+
+
+ +
+

Dependency Order

+
+ + + + + + + + + + + + + + + + + + + + + +
OrderPhaseBeads issueBlocks next because
1ASynthetic deterministic spineislandflow-259.1Establishes seeded raw event generation and provenance assumptions for later synthetic work.
1BSmart-flow contracts and vocabularyislandflow-zxh.1Can safely run in parallel with synthetic phase 01; defines evidence/hypothesis language before scoring work.
2Synthetic manifests, fixtures, and CLIislandflow-259.2Evidence clustering needs deterministic fixtures before broad behavior changes.
3Smart-flow evidence clustering and featuresislandflow-zxh.2Scenario labels need the evidence vocabulary they are expected to exercise.
4Synthetic scenarios, labels, and expected outputsislandflow-259.3Hypothesis scoring needs labeled positive, negative, and abstention cases.
5Smart-flow hypothesis scoring and abstentionislandflow-zxh.3Synthetic replay integration should validate the derived hypothesis pipeline.
6Synthetic replay integrationislandflow-259.4Smart-flow golden tests need replayable synthetic runs.
7Smart-flow replay evaluation and golden testsislandflow-zxh.4Demos should wait until replay proves the semantics.
8Synthetic demo and load profilesislandflow-259.5API/UI explainability should show stable, named, deterministic runs.
9Smart-flow API/UI explainabilityislandflow-zxh.5Final MVP presentation layer after the evidence pipeline is validated.
+
+
+ +
+

Future Work

+
+
+
+

Synthetic historical calibration

+

islandflow-259.6 depends on synthetic phase 05, but is not required for MVP.

+
+
+
+
+

Smart-flow calibration

+

islandflow-zxh.6 depends on smart-flow phase 05 and synthetic future calibration, but is not required for MVP.

+
+
+
+
+ + + +
+ HTML companion for docs/implementation/README.md. Source Markdown remains the active editable document. +
+
+ + diff --git a/docs/implementation/smart-money/00-roadmap.html b/docs/implementation/smart-money/00-roadmap.html new file mode 100644 index 0000000..a783b79 --- /dev/null +++ b/docs/implementation/smart-money/00-roadmap.html @@ -0,0 +1,502 @@ + + + + + + Smart Money / Smart Flow Roadmap + + + +
+
+
+

Smart Flow Roadmap

+

Smart Money / Smart Flow Roadmap

+

+ Implementation-sized phases for turning smart-money detection into smart-flow inference: + observations, evidence clusters, cautious hypotheses, confidence, alternatives, abstention, + replay evaluation, and user-facing insight projections. +

+
+ Beads islandflow-zxh + Recommended: Option B + MVP before calibration +
+
+ +
+ +
+

Jump to

+ +
+ + + +
+

Core Constraints

+
+
+
    +
  • Do not treat "smart money" as a canonical fact emitted by the system.
    • +
  • Distinguish direct facts, evidence, hypotheses, confidence, alternatives, and abstention.
    • +
  • Preserve evidence and uncertainty in storage, API, websocket, and UI surfaces.
    • +
  • Keep Redis as hot cache only, not hidden baseline truth.
    • +
  • Make replay evaluation the acceptance gate before expanding UI confidence.
    • +
  • Keep historical or research-grade calibration as future work, not an MVP dependency.
    • +
+
+
+
+ +
+

Phase Sequence

+
+ + + + + + + + + + + + + + + + + +
PhaseBeads issueDepends onPurpose
01 - Contracts and vocabularyislandflow-zxh.1None; safe parallel with islandflow-259.1Define evidence/hypothesis/insight contracts and retire canonical overconfidence.
02 - Evidence clustering and featuresislandflow-zxh.2islandflow-259.2Extract eligibility, evidence facts, clusters, and traceable features.
03 - Hypothesis scoring and abstentionislandflow-zxh.3islandflow-259.3Score cautious hypotheses and represent abstention/alternatives.
04 - Replay evaluation and golden testsislandflow-zxh.4islandflow-259.4Validate derived outputs through deterministic replay and golden fixtures.
05 - API/UI explainabilityislandflow-zxh.5islandflow-259.5Expose evidence-backed insights and uncertainty to API, WS, and UI.
99 - Future calibrationislandflow-zxh.6islandflow-zxh.5, islandflow-259.6Calibrate confidence and policy behavior later with richer datasets.
+
+
+ +
+

PR Split Notes

+
+
+ Phase 02a +

islandflow-zxh.2.1 - Eligibility and evidence facts

+

Split out the direct fact and eligibility layer before clustering and feature vector work.

+
+
+ Phase 02b +

islandflow-zxh.2.2 - Clustering and feature vectors

+

Keep clustering and feature vector changes reviewable after the evidence vocabulary exists.

+
+
+ Phase 03a +

islandflow-zxh.3.1 - Hypothesis score vectors

+

Build scoring as a separate semantic layer, not as UI-ready certainty.

+
+
+ Phase 03b +

islandflow-zxh.3.2 - Abstention and insight projection

+

Represent alternatives, penalties, and abstention before exposing user-facing insight projections.

+
+
+ Phase 05a +

islandflow-zxh.5.1 - Evidence API and websocket surfaces

+

Expose evidence-backed contracts through transport before tuning the presentation layer.

+
+
+ Phase 05b +

islandflow-zxh.5.2 - UI explainability surfaces

+

Show evidence quality, confidence vs conviction, alternatives, abstention, and catalyst/noise context.

+
+
+

+ If an implementation PR crosses contracts, compute, storage, API, and UI in one change, stop and split it. +

+
+ +
+

Matching Beads Epic

+
+
+

islandflow-zxh - Plan smart-money to smart-flow implementation phases.

+
+
+
+ +
+ HTML companion for docs/implementation/smart-money/00-roadmap.md. Source Markdown remains the active editable document. +
+
+ + diff --git a/docs/implementation/smart-money/index.html b/docs/implementation/smart-money/index.html index 0c6ba0e..7e6bfda 100644 --- a/docs/implementation/smart-money/index.html +++ b/docs/implementation/smart-money/index.html @@ -411,6 +411,9 @@ tr:last-child td { border-bottom: 0; } Beads islandflow-zxh 7 source docs No app code + Implementation overview + Roadmap HTML + Architecture HTML diff --git a/docs/implementation/synthetic-market-data/00-roadmap.html b/docs/implementation/synthetic-market-data/00-roadmap.html new file mode 100644 index 0000000..71978eb --- /dev/null +++ b/docs/implementation/synthetic-market-data/00-roadmap.html @@ -0,0 +1,481 @@ + + + + + + Synthetic Market-Data Roadmap + + + +
+
+
+

Synthetic Roadmap

+

Synthetic Market-Data Roadmap

+

+ Implementation-sized phases for extracting deterministic synthetic generation into a first-class reusable engine + while keeping the useful NATS, ClickHouse, compute, API, replay, and web stack. +

+
+ Beads islandflow-259 + Recommended: Option B + Infra-free early gates +
+
+ +
+ +
+

Jump to

+ +
+ + + +
+

Core Constraints

+
+
+
    +
  • Emit canonical market event types: OptionPrint, OptionNBBO, EquityPrint, and EquityQuote.
    • +
  • Do not create synthetic-only market event types for the main pipeline.
    • +
  • Keep hidden ground-truth labels separate from emitted market events.
    • +
  • Keep early quality gates infra-free: bun test should not require Docker, ClickHouse, NATS, or Redis.
    • +
  • Build deterministic foundations before demos, UI controls, or live synthetic service behavior.
    • +
  • Treat historical calibration as future work, not as a dependency for the MVP synthetic generator.
    • +
+
+
+
+ +
+

Phase Sequence

+
+ + + + + + + + + + + + + + + + + +
PhaseBeads issueDepends onPurpose
01 - Deterministic spineislandflow-259.1NoneCreate the seeded generation foundation and canonical event output contract.
02 - Manifests, fixtures, CLIislandflow-259.2islandflow-zxh.1Turn deterministic generation into durable fixtures and manifests.
03 - Scenarios, labels, expected outputsislandflow-259.3islandflow-zxh.2Author named scenarios, separate labels, and expected derived outputs.
04 - Replay integrationislandflow-259.4islandflow-zxh.3Make replay consume synthetic runs with stable ordering and output comparison.
05 - Demo and load profilesislandflow-259.5islandflow-zxh.4Expose named deterministic demo/load profiles after replay validation.
99 - Future historical calibrationislandflow-259.6islandflow-259.5Calibrate parameters from historical data later, after the MVP is stable.
+
+
+ +
+

PR Split Notes

+
+
+ Phase 03a +

islandflow-259.3.1 - Scenario catalog and labels

+

Keep scenario authoring and ground-truth label shape focused before expected-output comparison grows around it.

+
+
+ Phase 03b +

islandflow-259.3.2 - Expected-output manifests

+

Store expected derived outputs as reviewable artifacts for downstream smart-flow validation.

+
+
+

+ If any other phase starts touching unrelated service, API, UI, and storage behavior in one PR, split it before implementation continues. +

+
+ +
+

Matching Beads Epic

+
+
+

islandflow-259 - Plan synthetic market-data implementation phases.

+
+
+
+ +
+ HTML companion for docs/implementation/synthetic-market-data/00-roadmap.md. Source Markdown remains the active editable document. +
+
+ + diff --git a/docs/implementation/synthetic-market-data/index.html b/docs/implementation/synthetic-market-data/index.html index dcb2262..e85f553 100644 --- a/docs/implementation/synthetic-market-data/index.html +++ b/docs/implementation/synthetic-market-data/index.html @@ -411,6 +411,9 @@ tr:last-child td { border-bottom: 0; } Beads islandflow-259 7 source docs No app code + Implementation overview + Roadmap HTML + Architecture HTML diff --git a/docs/plans/smart-flow-architecture-review.html b/docs/plans/smart-flow-architecture-review.html new file mode 100644 index 0000000..826fa34 --- /dev/null +++ b/docs/plans/smart-flow-architecture-review.html @@ -0,0 +1,814 @@ + + + + + + Smart Flow Architecture Review + + + +
+
+
+

Plan Document

+

Evidence-Backed Smart-Flow Detection

+

+ A readable architecture review for reshaping Islandflow's smart-flow system around direct observation, + evidence clusters, cautious hypotheses, preserved uncertainty, and replayable validation. +

+
+ Smart flow + Architecture review + Refactor, not rewrite + Recommended: Option B + Roadmap HTML + Phase dossier +
+
+ +
+ +
+

Jump to

+ +
+ +
+

Summary

+
+
+

+ No source code was modified as part of the architecture review. The conclusion is direct: + the current architecture is not suitable as-is, but it is close enough to refactor. + The stack is right; the domain language and pipeline shape are not. +

+

+ The research direction should be direct observation to inference to hypothesis, with preserved + evidence and visible uncertainty. The system should stop emitting "smart money" as if it is a + fact, and instead emit cautious, explainable smart-flow hypotheses. +

+
+
+
+ +
+

Source Documents

+
+ + + +
+
+ +
+

Area Classification

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AreaCallArchitecture Review
Domain modelrefactorGood bones, wrong center. Make evidence, hypotheses, scores, and alternatives first-class.
Event taxonomyrefactorRaw/derived split is good; smart_money, dark.inferred, and classifier_hits leak overconfident product language.
Service boundariesrefactorIngest does too much signal policy; compute is too broad. Split pipeline stages before adding more intelligence.
FlowPacketrefactorKeep concept, rename/reframe as FlowEvidenceCluster or FlowCandidate. Not a product domain object.
SmartMoneyEventredesignReplace canonical object with FlowHypothesisEvent; use SmartFlowInsight only as UI/API projection.
Classifier pipelineredesignCurrent rules mix evidence extraction, hypothesis scoring, narrative labels, and alerting. Needs staged outputs.
ClickHouse/storagerefactorRight datastore; raw tables are decent, derived evidence/hypotheses need typed/queryable columns plus JSON sidecars.
Redis baselines/cacherefactorRight hot-state role; wrong as hidden baseline truth. Baselines need replayable snapshots/versioning.
NATS/JetStream subjectsrefactorRight bus; subjects should express stage/version: observations, evidence, hypotheses, insights.
Replay determinismredesignPresent but not central enough. Replay must be the acceptance gate for derived outputs.
API/WebSocketrefactorMechanics are good; public surface should expose evidence bundles and hypotheses, not internal legacy names.
UI evidence modelrefactorDirectionally good, but still foregrounds profile/probability over evidence quality, alternatives, and uncertainty.
Test strategyredesignUnit tests are solid scaffolding; needs fixture replay, false-positive suites, calibration, and end-to-end determinism.
+
+
+ +
+

Direct Answers

+
+
+
    +
  1. 01

    Current suitability: no. Useful infrastructure, but not yet an evidence-backed smart-flow architecture.

    2. +
  2. 02

    SmartMoneyEvent: not a good canonical domain object. Use FlowHypothesisEvent. ParticipantHypothesisEvent implies participant identity too strongly. SmartFlowInsight should be a user-facing projection.

    3. +
  3. 03

    FlowPacket: not as named. Keep the abstraction as an internal evidence cluster, rename to FlowEvidenceCluster or FlowCandidate.

    4. +
  4. 04

    Service boundaries: not right. Ingest should normalize only; evidence quality, eligibility, clustering, hypothesis scoring, and insight projection should be separate stages.

    5. +
  5. 05

    ClickHouse/Redis/NATS roles: yes broadly. ClickHouse is the authoritative event/audit store. Redis is hot cache only. NATS is transport, not truth. All three need cleaner contracts.

    6. +
  6. 06

    Replay central enough: no. It should be how every detection change proves itself.

    7. +
  7. 07

    UI uncertainty: partially. It shows evidence refs, profile ladders, abstention, and suppression, but needs confidence vs conviction, alternative explanations, evidence quality, and why-not signals.

    8. +
  8. 08

    First-class domain objects: raw observations, execution context, quote join, eligibility decision, evidence cluster, structure hypothesis, evidence quality score, baseline snapshot, hypothesis score vector, false-positive penalty, catalyst context, flow hypothesis event, smart-flow insight, replay run.

    9. +
  9. 09

    Implementation details: Redis list layout, durable consumer names, current classifier thresholds, ClickHouse batch writer, adapter internals, legacy ClassifierHitEvent, alert severity math, UI cache mechanics.

    10. +
  10. 10

    Delete/defer: canonical smart-money naming, real-time dark-pool certainty, standalone whale-premium alerts, trade-level open/close claims, participant identity claims, simplistic premium alert score, ingest-time signal filtering, retail_whale as a canonical profile unless reframed as attention/lottery flow.

    11. +
+
+
+
+ +
+

Objects to Make First-Class

+
+ Raw Observation + Execution Context + Quote Join + Eligibility Decision + Flow Evidence Cluster + Structure Hypothesis + Evidence Quality Score + Baseline Snapshot + Hypothesis Score Vector + False-Positive Penalty + Catalyst Context + Flow Hypothesis Event + Smart-Flow Insight + Replay Run +
+
+ +
+

Options

+
+
+
+ Option A +

Conservative

+

Keep current objects and services; add evidence-quality fields, UI copy fixes, and replay tests.

+
+
+
    +
  • Pros

    Fastest, lowest migration risk, preserves current endpoints and UI.

    • +
  • Cons

    Leaves misleading canonical names and keeps inference tangled in compute.

    • +
  • Complexity

    Low.

    • +
  • Migration Risk

    Low.

    • +
+
    +
  1. Rename UI copy from smart money to smart flow candidate.
    2. +
  2. Add evidence-quality and alternative-explanation fields to existing event.
    3. +
  3. Add replay consistency tests around current outputs.
    4. +
  4. Add typed ClickHouse columns for high-value JSON fields.
    5. +
  5. Deprecate, but do not remove, legacy classifier hit display.
    6. +
+
+
+ +
+
+ Option B +

Refactor

+

Keep the stack and terminal UI, but rebuild the domain pipeline around evidence clusters and hypothesis events.

+
+
+
    +
  • Pros

    Fixes the product's epistemic spine without wasting useful infrastructure.

    • +
  • Cons

    Requires breaking contract migration across types, storage, compute, API, UI, and tests.

    • +
  • Complexity

    Medium-high.

    • +
  • Migration Risk

    Medium.

    • +
+
    +
  1. Introduce FlowEvidenceCluster, FlowHypothesisEvent, SmartFlowInsight, EvidenceQuality, and version fields with compatibility aliases.
    2. +
  2. Move signal eligibility out of ingest.
    3. +
  3. Split compute into evidence join, cluster/structure, hypothesis scoring, and insight/alert projection.
    4. +
  4. Replace derived JSON-only storage with typed query columns.
    5. +
  5. Add replay-run harness that recomputes derived outputs from raw streams.
    6. +
  6. Add /flow/evidence, /flow/hypotheses, /flow/insights, and WS equivalents.
    7. +
  7. Rework UI drawers/tables around evidence quality, confidence vs conviction, alternatives, abstention, and catalyst/noise context.
    8. +
  8. Add fixture suites for stale quotes, complex spreads, 0DTE/event noise, deep ITM, wide spreads, and off-exchange ambiguity.
    9. +
+
+
+ +
+
+ Option C +

Redesign

+

Start over with an event-sourced evidence engine and versioned, replayable policies.

+
+
+
    +
  • Pros

    Cleanest long-term architecture and strongest research discipline.

    • +
  • Cons

    Slowest, overkill before product fit, and discards too much working infrastructure.

    • +
  • Complexity

    Very high.

    • +
  • Migration Risk

    High.

    • +
+
    +
  1. Define new canonical event taxonomy and versioned policy registry.
    2. +
  2. Build raw observation lake and deterministic replay runner first.
    3. +
  3. Build evidence extraction and quote/condition eligibility services.
    4. +
  4. Build cluster and structure hypothesis services.
    5. +
  5. Build hypothesis scoring and calibration services.
    6. +
  6. Build insight projection API.
    7. +
  7. Rebuild terminal against new evidence/hypothesis contracts.
    8. +
  8. Backfill or discard old derived data.
    9. +
+
+
+
+
+ +
+

Recommendation

+
+

Choose Option B.

+

+ Option A is too timid for a pre-alpha product whose current names already fight the research. + Option C is intellectually clean but wastes too much working infrastructure. Option B keeps the + stack and terminal momentum while fixing the core mistake: treating smart money as a thing the + system emits, instead of treating smart flow as a cautious, evidence-backed hypothesis with alternatives. +

+

+ The first implementation move should be the contract/naming PR: introduce + FlowHypothesisEvent and FlowEvidenceCluster with compatibility aliases, + then make replay the gate before touching more classifier logic. +

+
+
+ +
+ HTML companion for docs/plans/smart-flow-architecture-review.md. Styled for the Islandflow product register. +
+
+ + diff --git a/docs/plans/synthetic-market-data-architecture-review.html b/docs/plans/synthetic-market-data-architecture-review.html index d1a40db..1823f52 100644 --- a/docs/plans/synthetic-market-data-architecture-review.html +++ b/docs/plans/synthetic-market-data-architecture-review.html @@ -521,6 +521,8 @@ Source: markdown review Mode: Plan Recommendation: Option B + Roadmap HTML + Phase dossier