From d2a09e095a744b0d9fbb75d4ef87fd4901a2e650 Mon Sep 17 00:00:00 2001 From: dirtydishes Date: Sat, 27 Dec 2025 18:45:26 -0500 Subject: [PATCH] Scaffold monorepo dev setup --- .gitignore | 16 +++ apps/web/app/globals.css | 44 +++++++ apps/web/app/layout.tsx | 19 +++ apps/web/app/page.tsx | 11 ++ apps/web/next-env.d.ts | 4 + apps/web/package.json | 15 +++ apps/web/tsconfig.json | 11 ++ bun.lock | 164 +++++++++++++++++++++++++ docker-compose.yml | 30 +++++ package.json | 17 +++ packages/config/package.json | 11 ++ packages/config/src/env.ts | 34 +++++ packages/config/src/index.ts | 1 + packages/config/tsconfig.json | 7 ++ packages/observability/package.json | 8 ++ packages/observability/src/index.ts | 2 + packages/observability/src/logger.ts | 54 ++++++++ packages/observability/src/metrics.ts | 51 ++++++++ packages/observability/tsconfig.json | 7 ++ packages/types/package.json | 11 ++ packages/types/src/events.ts | 105 ++++++++++++++++ packages/types/src/index.ts | 1 + packages/types/tsconfig.json | 7 ++ scripts/dev-services.ts | 68 ++++++++++ scripts/dev.ts | 70 +++++++++++ services/api/package.json | 12 ++ services/api/src/index.ts | 17 +++ services/api/tsconfig.json | 7 ++ services/candles/package.json | 12 ++ services/candles/src/index.ts | 17 +++ services/candles/tsconfig.json | 7 ++ services/compute/package.json | 12 ++ services/compute/src/index.ts | 17 +++ services/compute/tsconfig.json | 7 ++ services/eod-enricher/package.json | 12 ++ services/eod-enricher/src/index.ts | 17 +++ services/eod-enricher/tsconfig.json | 7 ++ services/ingest-equities/package.json | 12 ++ services/ingest-equities/src/index.ts | 17 +++ services/ingest-equities/tsconfig.json | 7 ++ services/ingest-options/package.json | 12 ++ services/ingest-options/src/index.ts | 17 +++ services/ingest-options/tsconfig.json | 7 ++ services/refdata/package.json | 12 ++ services/refdata/src/index.ts | 17 +++ services/refdata/tsconfig.json | 7 ++ tsconfig.base.json | 13 ++ 47 files changed, 1033 insertions(+) create mode 100644 .gitignore create mode 100644 apps/web/app/globals.css create mode 100644 apps/web/app/layout.tsx create mode 100644 apps/web/app/page.tsx create mode 100644 apps/web/next-env.d.ts create mode 100644 apps/web/package.json create mode 100644 apps/web/tsconfig.json create mode 100644 bun.lock create mode 100644 docker-compose.yml create mode 100644 package.json create mode 100644 packages/config/package.json create mode 100644 packages/config/src/env.ts create mode 100644 packages/config/src/index.ts create mode 100644 packages/config/tsconfig.json create mode 100644 packages/observability/package.json create mode 100644 packages/observability/src/index.ts create mode 100644 packages/observability/src/logger.ts create mode 100644 packages/observability/src/metrics.ts create mode 100644 packages/observability/tsconfig.json create mode 100644 packages/types/package.json create mode 100644 packages/types/src/events.ts create mode 100644 packages/types/src/index.ts create mode 100644 packages/types/tsconfig.json create mode 100644 scripts/dev-services.ts create mode 100644 scripts/dev.ts create mode 100644 services/api/package.json create mode 100644 services/api/src/index.ts create mode 100644 services/api/tsconfig.json create mode 100644 services/candles/package.json create mode 100644 services/candles/src/index.ts create mode 100644 services/candles/tsconfig.json create mode 100644 services/compute/package.json create mode 100644 services/compute/src/index.ts create mode 100644 services/compute/tsconfig.json create mode 100644 services/eod-enricher/package.json create mode 100644 services/eod-enricher/src/index.ts create mode 100644 services/eod-enricher/tsconfig.json create mode 100644 services/ingest-equities/package.json create mode 100644 services/ingest-equities/src/index.ts create mode 100644 services/ingest-equities/tsconfig.json create mode 100644 services/ingest-options/package.json create mode 100644 services/ingest-options/src/index.ts create mode 100644 services/ingest-options/tsconfig.json create mode 100644 services/refdata/package.json create mode 100644 services/refdata/src/index.ts create mode 100644 services/refdata/tsconfig.json create mode 100644 tsconfig.base.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8387591 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +.DS_Store +.node-version +.bun +bun.lockb +node_modules/ +dist/ +.env +.env.* +coverage/ +logs/ +.tmp/ +AGENTS.md +PLAN.md +CODING_STYLE.md +RESEARCH.md +README.md diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css new file mode 100644 index 0000000..8e0b786 --- /dev/null +++ b/apps/web/app/globals.css @@ -0,0 +1,44 @@ +:root { + color-scheme: light; + font-family: "IBM Plex Mono", "JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + background: #f4f3ef; + color: #1b1b1b; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + min-height: 100vh; +} + +.page { + display: grid; + place-items: center; + min-height: 100vh; + padding: 48px 24px; + background: radial-gradient(circle at top, #fef7e4, #f4f3ef 60%); +} + +.panel { + max-width: 520px; + padding: 32px 36px; + border: 1px solid #dad2c2; + border-radius: 18px; + background: #fff9ee; + box-shadow: 0 20px 40px rgba(48, 32, 12, 0.12); +} + +h1 { + margin: 0 0 12px; + font-size: 2.25rem; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +p { + margin: 8px 0; + line-height: 1.6; +} diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx new file mode 100644 index 0000000..c27753d --- /dev/null +++ b/apps/web/app/layout.tsx @@ -0,0 +1,19 @@ +import "./globals.css"; +import type { ReactNode } from "react"; + +export const metadata = { + title: "Islandflow", + description: "Realtime options flow & off-exchange analysis" +}; + +type RootLayoutProps = { + children: ReactNode; +}; + +export default function RootLayout({ children }: RootLayoutProps) { + return ( + + {children} + + ); +} diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx new file mode 100644 index 0000000..55c2a76 --- /dev/null +++ b/apps/web/app/page.tsx @@ -0,0 +1,11 @@ +export default function HomePage() { + return ( +
+
+

Islandflow

+

Realtime options flow + off-exchange analysis.

+

UI scaffold is up; live data wiring next.

+
+
+ ); +} diff --git a/apps/web/next-env.d.ts b/apps/web/next-env.d.ts new file mode 100644 index 0000000..8b0a849 --- /dev/null +++ b/apps/web/next-env.d.ts @@ -0,0 +1,4 @@ +/// +/// + +// Note: This file is normally generated by Next.js. diff --git a/apps/web/package.json b/apps/web/package.json new file mode 100644 index 0000000..e206b83 --- /dev/null +++ b/apps/web/package.json @@ -0,0 +1,15 @@ +{ + "name": "@islandflow/web", + "private": true, + "type": "module", + "scripts": { + "dev": "next dev -p 3000", + "build": "next build", + "start": "next start -p 3000" + }, + "dependencies": { + "next": "^14.2.4", + "react": "^18.3.1", + "react-dom": "^18.3.1" + } +} diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json new file mode 100644 index 0000000..f4c4210 --- /dev/null +++ b/apps/web/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "jsx": "preserve", + "lib": ["DOM", "DOM.Iterable", "ES2022"], + "incremental": true, + "noEmit": true + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], + "exclude": ["node_modules"] +} diff --git a/bun.lock b/bun.lock new file mode 100644 index 0000000..82ebb5e --- /dev/null +++ b/bun.lock @@ -0,0 +1,164 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "islandflow", + }, + "apps/web": { + "name": "@islandflow/web", + "dependencies": { + "next": "^14.2.4", + "react": "^18.3.1", + "react-dom": "^18.3.1", + }, + }, + "packages/config": { + "name": "@islandflow/config", + "dependencies": { + "zod": "^3.23.8", + }, + }, + "packages/observability": { + "name": "@islandflow/observability", + }, + "packages/types": { + "name": "@islandflow/types", + "dependencies": { + "zod": "^3.23.8", + }, + }, + "services/api": { + "name": "@islandflow/api", + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + }, + }, + "services/candles": { + "name": "@islandflow/candles", + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + }, + }, + "services/compute": { + "name": "@islandflow/compute", + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + }, + }, + "services/eod-enricher": { + "name": "@islandflow/eod-enricher", + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + }, + }, + "services/ingest-equities": { + "name": "@islandflow/ingest-equities", + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + }, + }, + "services/ingest-options": { + "name": "@islandflow/ingest-options", + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + }, + }, + "services/refdata": { + "name": "@islandflow/refdata", + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*", + }, + }, + }, + "packages": { + "@islandflow/api": ["@islandflow/api@workspace:services/api"], + + "@islandflow/candles": ["@islandflow/candles@workspace:services/candles"], + + "@islandflow/compute": ["@islandflow/compute@workspace:services/compute"], + + "@islandflow/config": ["@islandflow/config@workspace:packages/config"], + + "@islandflow/eod-enricher": ["@islandflow/eod-enricher@workspace:services/eod-enricher"], + + "@islandflow/ingest-equities": ["@islandflow/ingest-equities@workspace:services/ingest-equities"], + + "@islandflow/ingest-options": ["@islandflow/ingest-options@workspace:services/ingest-options"], + + "@islandflow/observability": ["@islandflow/observability@workspace:packages/observability"], + + "@islandflow/refdata": ["@islandflow/refdata@workspace:services/refdata"], + + "@islandflow/types": ["@islandflow/types@workspace:packages/types"], + + "@islandflow/web": ["@islandflow/web@workspace:apps/web"], + + "@next/env": ["@next/env@14.2.35", "", {}, "sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ=="], + + "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@14.2.33", "", { "os": "darwin", "cpu": "arm64" }, "sha512-HqYnb6pxlsshoSTubdXKu15g3iivcbsMXg4bYpjL2iS/V6aQot+iyF4BUc2qA/J/n55YtvE4PHMKWBKGCF/+wA=="], + + "@next/swc-darwin-x64": ["@next/swc-darwin-x64@14.2.33", "", { "os": "darwin", "cpu": "x64" }, "sha512-8HGBeAE5rX3jzKvF593XTTFg3gxeU4f+UWnswa6JPhzaR6+zblO5+fjltJWIZc4aUalqTclvN2QtTC37LxvZAA=="], + + "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@14.2.33", "", { "os": "linux", "cpu": "arm64" }, "sha512-JXMBka6lNNmqbkvcTtaX8Gu5by9547bukHQvPoLe9VRBx1gHwzf5tdt4AaezW85HAB3pikcvyqBToRTDA4DeLw=="], + + "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@14.2.33", "", { "os": "linux", "cpu": "arm64" }, "sha512-Bm+QulsAItD/x6Ih8wGIMfRJy4G73tu1HJsrccPW6AfqdZd0Sfm5Imhgkgq2+kly065rYMnCOxTBvmvFY1BKfg=="], + + "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@14.2.33", "", { "os": "linux", "cpu": "x64" }, "sha512-FnFn+ZBgsVMbGDsTqo8zsnRzydvsGV8vfiWwUo1LD8FTmPTdV+otGSWKc4LJec0oSexFnCYVO4hX8P8qQKaSlg=="], + + "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@14.2.33", "", { "os": "linux", "cpu": "x64" }, "sha512-345tsIWMzoXaQndUTDv1qypDRiebFxGYx9pYkhwY4hBRaOLt8UGfiWKr9FSSHs25dFIf8ZqIFaPdy5MljdoawA=="], + + "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@14.2.33", "", { "os": "win32", "cpu": "arm64" }, "sha512-nscpt0G6UCTkrT2ppnJnFsYbPDQwmum4GNXYTeoTIdsmMydSKFz9Iny2jpaRupTb+Wl298+Rh82WKzt9LCcqSQ=="], + + "@next/swc-win32-ia32-msvc": ["@next/swc-win32-ia32-msvc@14.2.33", "", { "os": "win32", "cpu": "ia32" }, "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q=="], + + "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@14.2.33", "", { "os": "win32", "cpu": "x64" }, "sha512-nOjfZMy8B94MdisuzZo9/57xuFVLHJaDj5e/xrduJp9CV2/HrfxTRH2fbyLe+K9QT41WBLUd4iXX3R7jBp0EUg=="], + + "@swc/counter": ["@swc/counter@0.1.3", "", {}, "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="], + + "@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="], + + "busboy": ["busboy@1.6.0", "", { "dependencies": { "streamsearch": "^1.1.0" } }, "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA=="], + + "caniuse-lite": ["caniuse-lite@1.0.30001761", "", {}, "sha512-JF9ptu1vP2coz98+5051jZ4PwQgd2ni8A+gYSN7EA7dPKIMf0pDlSUxhdmVOaV3/fYK5uWBkgSXJaRLr4+3A6g=="], + + "client-only": ["client-only@0.0.1", "", {}, "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="], + + "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], + + "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + + "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], + + "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], + + "next": ["next@14.2.35", "", { "dependencies": { "@next/env": "14.2.35", "@swc/helpers": "0.5.5", "busboy": "1.6.0", "caniuse-lite": "^1.0.30001579", "graceful-fs": "^4.2.11", "postcss": "8.4.31", "styled-jsx": "5.1.1" }, "optionalDependencies": { "@next/swc-darwin-arm64": "14.2.33", "@next/swc-darwin-x64": "14.2.33", "@next/swc-linux-arm64-gnu": "14.2.33", "@next/swc-linux-arm64-musl": "14.2.33", "@next/swc-linux-x64-gnu": "14.2.33", "@next/swc-linux-x64-musl": "14.2.33", "@next/swc-win32-arm64-msvc": "14.2.33", "@next/swc-win32-ia32-msvc": "14.2.33", "@next/swc-win32-x64-msvc": "14.2.33" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.41.2", "react": "^18.2.0", "react-dom": "^18.2.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-KhYd2Hjt/O1/1aZVX3dCwGXM1QmOV4eNM2UTacK5gipDdPN/oHHK/4oVGy7X8GMfPMsUTUEmGlsy0EY1YGAkig=="], + + "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], + + "postcss": ["postcss@8.4.31", "", { "dependencies": { "nanoid": "^3.3.6", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" } }, "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ=="], + + "react": ["react@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="], + + "react-dom": ["react-dom@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" }, "peerDependencies": { "react": "^18.3.1" } }, "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw=="], + + "scheduler": ["scheduler@0.23.2", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ=="], + + "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + + "streamsearch": ["streamsearch@1.1.0", "", {}, "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg=="], + + "styled-jsx": ["styled-jsx@5.1.1", "", { "dependencies": { "client-only": "0.0.1" }, "peerDependencies": { "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0" } }, "sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw=="], + + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d7a8c5c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +services: + clickhouse: + image: clickhouse/clickhouse-server:23.8 + ports: + - "8123:8123" + - "9000:9000" + volumes: + - clickhouse-data:/var/lib/clickhouse + ulimits: + nofile: + soft: 262144 + hard: 262144 + redis: + image: redis:7.2 + ports: + - "6379:6379" + volumes: + - redis-data:/data + nats: + image: nats:2.10 + command: ["-js"] + ports: + - "4222:4222" + - "8222:8222" + volumes: + - nats-data:/data +volumes: + clickhouse-data: + redis-data: + nats-data: diff --git a/package.json b/package.json new file mode 100644 index 0000000..bc6680d --- /dev/null +++ b/package.json @@ -0,0 +1,17 @@ +{ + "name": "islandflow", + "private": true, + "type": "module", + "workspaces": [ + "apps/*", + "services/*", + "packages/*" + ], + "scripts": { + "dev": "bun run scripts/dev.ts", + "dev:infra": "docker compose up", + "dev:infra:down": "docker compose down", + "dev:web": "bun --cwd apps/web run dev", + "dev:services": "bun run scripts/dev-services.ts" + } +} diff --git a/packages/config/package.json b/packages/config/package.json new file mode 100644 index 0000000..f977d82 --- /dev/null +++ b/packages/config/package.json @@ -0,0 +1,11 @@ +{ + "name": "@islandflow/config", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "dependencies": { + "zod": "^3.23.8" + } +} diff --git a/packages/config/src/env.ts b/packages/config/src/env.ts new file mode 100644 index 0000000..8ac9042 --- /dev/null +++ b/packages/config/src/env.ts @@ -0,0 +1,34 @@ +import { z } from "zod"; + +export class EnvError extends Error { + readonly issues: z.ZodIssue[]; + + constructor(message: string, issues: z.ZodIssue[]) { + super(message); + this.name = "EnvError"; + this.issues = issues; + } +} + +const formatIssues = (issues: z.ZodIssue[]): string => { + return issues + .map((issue) => { + const path = issue.path.length > 0 ? issue.path.join(".") : ""; + return `${path}: ${issue.message}`; + }) + .join("; "); +}; + +export const readEnv = ( + schema: T, + env: Record = Bun.env +): z.infer => { + const result = schema.safeParse(env); + + if (!result.success) { + const details = formatIssues(result.error.issues); + throw new EnvError(`Invalid environment: ${details}`, result.error.issues); + } + + return result.data; +}; diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts new file mode 100644 index 0000000..77b0d3c --- /dev/null +++ b/packages/config/src/index.ts @@ -0,0 +1 @@ +export * from "./env"; diff --git a/packages/config/tsconfig.json b/packages/config/tsconfig.json new file mode 100644 index 0000000..d8c6443 --- /dev/null +++ b/packages/config/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": [] + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/observability/package.json b/packages/observability/package.json new file mode 100644 index 0000000..392b376 --- /dev/null +++ b/packages/observability/package.json @@ -0,0 +1,8 @@ +{ + "name": "@islandflow/observability", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + } +} diff --git a/packages/observability/src/index.ts b/packages/observability/src/index.ts new file mode 100644 index 0000000..0837f53 --- /dev/null +++ b/packages/observability/src/index.ts @@ -0,0 +1,2 @@ +export * from "./logger"; +export * from "./metrics"; diff --git a/packages/observability/src/logger.ts b/packages/observability/src/logger.ts new file mode 100644 index 0000000..0c4b437 --- /dev/null +++ b/packages/observability/src/logger.ts @@ -0,0 +1,54 @@ +export type LogLevel = "debug" | "info" | "warn" | "error"; + +export type LogContext = Record; + +export type LogRecord = LogContext & { + level: LogLevel; + service: string; + msg: string; + ts: string; +}; + +export type LoggerFn = (msg: string, context?: LogContext) => void; + +export type Logger = { + debug: LoggerFn; + info: LoggerFn; + warn: LoggerFn; + error: LoggerFn; +}; + +export type LoggerOptions = { + service: string; + now?: () => string; + sink?: (record: LogRecord) => void; +}; + +const defaultSink = (record: LogRecord) => { + console.log(JSON.stringify(record)); +}; + +export const createLogger = ({ + service, + now = () => new Date().toISOString(), + sink = defaultSink +}: LoggerOptions): Logger => { + const write = (level: LogLevel, msg: string, context?: LogContext) => { + const record: LogRecord = { + level, + service, + msg, + ts: now(), + ...(context ?? {}) + }; + + sink(record); + }; + + return { + debug: (msg, context) => write("debug", msg, context), + info: (msg, context) => write("info", msg, context), + warn: (msg, context) => write("warn", msg, context), + error: (msg, context) => write("error", msg, context) + }; +}; diff --git a/packages/observability/src/metrics.ts b/packages/observability/src/metrics.ts new file mode 100644 index 0000000..1605ac6 --- /dev/null +++ b/packages/observability/src/metrics.ts @@ -0,0 +1,51 @@ +export type MetricType = "counter" | "gauge" | "timing"; + +export type MetricTags = Record; + +export type MetricRecord = { + name: string; + type: MetricType; + value: number; + ts: number; + service?: string; + tags?: MetricTags; +}; + +export type MetricsEmitter = (record: MetricRecord) => void; + +export type MetricsOptions = { + service?: string; + emit?: MetricsEmitter; + now?: () => number; +}; + +export type Metrics = { + count: (name: string, value?: number, tags?: MetricTags) => void; + gauge: (name: string, value: number, tags?: MetricTags) => void; + timing: (name: string, value: number, tags?: MetricTags) => void; +}; + +const noopEmit: MetricsEmitter = () => {}; + +export const createMetrics = ({ + service, + emit = noopEmit, + now = () => Date.now() +}: MetricsOptions = {}): Metrics => { + const write = (type: MetricType, name: string, value: number, tags?: MetricTags) => { + emit({ + name, + type, + value, + tags, + service, + ts: now() + }); + }; + + return { + count: (name, value = 1, tags) => write("counter", name, value, tags), + gauge: (name, value, tags) => write("gauge", name, value, tags), + timing: (name, value, tags) => write("timing", name, value, tags) + }; +}; diff --git a/packages/observability/tsconfig.json b/packages/observability/tsconfig.json new file mode 100644 index 0000000..d8c6443 --- /dev/null +++ b/packages/observability/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": [] + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/types/package.json b/packages/types/package.json new file mode 100644 index 0000000..dc4c5b1 --- /dev/null +++ b/packages/types/package.json @@ -0,0 +1,11 @@ +{ + "name": "@islandflow/types", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "dependencies": { + "zod": "^3.23.8" + } +} diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts new file mode 100644 index 0000000..1c3a4a6 --- /dev/null +++ b/packages/types/src/events.ts @@ -0,0 +1,105 @@ +import { z } from "zod"; + +export const EventMetaSchema = z.object({ + source_ts: z.number().int().nonnegative(), + ingest_ts: z.number().int().nonnegative(), + seq: z.number().int().nonnegative(), + trace_id: z.string().min(1) +}); + +export type EventMeta = z.infer; + +export const OptionPrintSchema = EventMetaSchema.merge( + z.object({ + ts: z.number().int().nonnegative(), + option_contract_id: z.string().min(1), + price: z.number().nonnegative(), + size: z.number().int().positive(), + exchange: z.string().min(1), + conditions: z.array(z.string().min(1)).optional() + }) +); + +export type OptionPrint = z.infer; + +export const OptionNBBOSchema = EventMetaSchema.merge( + z.object({ + ts: z.number().int().nonnegative(), + option_contract_id: z.string().min(1), + bid: z.number().nonnegative(), + ask: z.number().nonnegative(), + bidSize: z.number().int().nonnegative(), + askSize: z.number().int().nonnegative() + }) +); + +export type OptionNBBO = z.infer; + +export const EquityPrintSchema = EventMetaSchema.merge( + z.object({ + ts: z.number().int().nonnegative(), + underlying_id: z.string().min(1), + price: z.number().nonnegative(), + size: z.number().int().positive(), + exchange: z.string().min(1), + offExchangeFlag: z.boolean() + }) +); + +export type EquityPrint = z.infer; + +export const EquityQuoteSchema = EventMetaSchema.merge( + z.object({ + ts: z.number().int().nonnegative(), + underlying_id: z.string().min(1), + bid: z.number().nonnegative(), + ask: z.number().nonnegative() + }) +); + +export type EquityQuote = z.infer; + +export const FlowPacketSchema = EventMetaSchema.merge( + z.object({ + id: z.string().min(1), + members: z.array(z.string().min(1)), + features: z.record(z.union([z.string(), z.number(), z.boolean()])), + join_quality: z.record(z.number()) + }) +); + +export type FlowPacket = z.infer; + +export const ClassifierHitSchema = z.object({ + classifier_id: z.string().min(1), + confidence: z.number().min(0).max(1), + direction: z.string().min(1), + explanations: z.array(z.string().min(1)) +}); + +export type ClassifierHit = z.infer; + +export const ClassifierHitEventSchema = EventMetaSchema.merge(ClassifierHitSchema); + +export type ClassifierHitEvent = z.infer; + +export const AlertEventSchema = EventMetaSchema.merge( + z.object({ + score: z.number(), + severity: z.string().min(1), + hits: z.array(ClassifierHitSchema), + evidence_refs: z.array(z.string().min(1)) + }) +); + +export type AlertEvent = z.infer; + +export const InferredDarkEventSchema = EventMetaSchema.merge( + z.object({ + type: z.string().min(1), + confidence: z.number().min(0).max(1), + evidence_refs: z.array(z.string().min(1)) + }) +); + +export type InferredDarkEvent = z.infer; diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts new file mode 100644 index 0000000..1784004 --- /dev/null +++ b/packages/types/src/index.ts @@ -0,0 +1 @@ +export * from "./events"; diff --git a/packages/types/tsconfig.json b/packages/types/tsconfig.json new file mode 100644 index 0000000..d8c6443 --- /dev/null +++ b/packages/types/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": [] + }, + "include": ["src/**/*.ts"] +} diff --git a/scripts/dev-services.ts b/scripts/dev-services.ts new file mode 100644 index 0000000..f435237 --- /dev/null +++ b/scripts/dev-services.ts @@ -0,0 +1,68 @@ +type ChildSpec = { + name: string; + cmd: string[]; + cwd: string; +}; + +type Child = { + name: string; + process: Bun.Subprocess; +}; + +const children: Child[] = []; +let shuttingDown = false; + +const spawnChild = ({ name, cmd, cwd }: ChildSpec): void => { + const proc = Bun.spawn(cmd, { + cwd, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit" + }); + + children.push({ name, process: proc }); + + proc.exited.then((code) => { + if (shuttingDown) { + return; + } + + const exitCode = code ?? 0; + const statusLabel = exitCode === 0 ? "exited" : "failed"; + console.error(`[dev-services] ${name} ${statusLabel} (${exitCode})`); + shutdown(exitCode); + }); +}; + +const shutdown = (code: number): void => { + if (shuttingDown) { + return; + } + + shuttingDown = true; + + for (const child of children) { + child.process.kill(); + } + + process.exit(code); +}; + +process.on("SIGINT", () => shutdown(0)); +process.on("SIGTERM", () => shutdown(0)); + +const tasks: ChildSpec[] = [ + { name: "ingest-options", cmd: ["bun", "run", "dev"], cwd: "services/ingest-options" }, + { name: "ingest-equities", cmd: ["bun", "run", "dev"], cwd: "services/ingest-equities" }, + { name: "compute", cmd: ["bun", "run", "dev"], cwd: "services/compute" }, + { name: "candles", cmd: ["bun", "run", "dev"], cwd: "services/candles" }, + { name: "refdata", cmd: ["bun", "run", "dev"], cwd: "services/refdata" }, + { name: "eod-enricher", cmd: ["bun", "run", "dev"], cwd: "services/eod-enricher" }, + { name: "api", cmd: ["bun", "run", "dev"], cwd: "services/api" } +]; + +for (const task of tasks) { + spawnChild(task); +} + +await new Promise(() => {}); diff --git a/scripts/dev.ts b/scripts/dev.ts new file mode 100644 index 0000000..6da5b89 --- /dev/null +++ b/scripts/dev.ts @@ -0,0 +1,70 @@ +type ChildSpec = { + name: string; + cmd: string[]; + cwd?: string; +}; + +type Child = { + name: string; + process: Bun.Subprocess; +}; + +const children: Child[] = []; +let shuttingDown = false; + +const spawnChild = ({ name, cmd, cwd }: ChildSpec): void => { + const proc = Bun.spawn(cmd, { + cwd, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit" + }); + + children.push({ name, process: proc }); + + proc.exited.then((code) => { + if (shuttingDown) { + return; + } + + const exitCode = code ?? 0; + const statusLabel = exitCode === 0 ? "exited" : "failed"; + console.error(`[dev] ${name} ${statusLabel} (${exitCode})`); + shutdown(exitCode); + }); +}; + +const shutdown = (code: number): void => { + if (shuttingDown) { + return; + } + + shuttingDown = true; + + for (const child of children) { + child.process.kill(); + } + + process.exit(code); +}; + +process.on("SIGINT", () => shutdown(0)); +process.on("SIGTERM", () => shutdown(0)); + +const tasks: ChildSpec[] = [ + { name: "infra", cmd: ["docker", "compose", "up"] }, + { name: "web", cmd: ["bun", "run", "dev"], cwd: "apps/web" }, + { name: "ingest-options", cmd: ["bun", "run", "dev"], cwd: "services/ingest-options" }, + { name: "ingest-equities", cmd: ["bun", "run", "dev"], cwd: "services/ingest-equities" }, + { name: "compute", cmd: ["bun", "run", "dev"], cwd: "services/compute" }, + { name: "candles", cmd: ["bun", "run", "dev"], cwd: "services/candles" }, + { name: "refdata", cmd: ["bun", "run", "dev"], cwd: "services/refdata" }, + { name: "eod-enricher", cmd: ["bun", "run", "dev"], cwd: "services/eod-enricher" }, + { name: "api", cmd: ["bun", "run", "dev"], cwd: "services/api" } +]; + +for (const task of tasks) { + spawnChild(task); +} + +await new Promise(() => {}); diff --git a/services/api/package.json b/services/api/package.json new file mode 100644 index 0000000..91d3227 --- /dev/null +++ b/services/api/package.json @@ -0,0 +1,12 @@ +{ + "name": "@islandflow/api", + "private": true, + "type": "module", + "scripts": { + "dev": "bun run src/index.ts" + }, + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*" + } +} diff --git a/services/api/src/index.ts b/services/api/src/index.ts new file mode 100644 index 0000000..0a473e8 --- /dev/null +++ b/services/api/src/index.ts @@ -0,0 +1,17 @@ +import { createLogger } from "@islandflow/observability"; + +const service = "api"; +const logger = createLogger({ service }); + +logger.info("service starting"); + +const shutdown = (signal: string) => { + logger.info("service stopping", { signal }); + process.exit(0); +}; + +process.on("SIGINT", () => shutdown("SIGINT")); +process.on("SIGTERM", () => shutdown("SIGTERM")); + +// Keep the process alive until real listeners are wired. +setInterval(() => {}, 60_000); diff --git a/services/api/tsconfig.json b/services/api/tsconfig.json new file mode 100644 index 0000000..d8c6443 --- /dev/null +++ b/services/api/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": [] + }, + "include": ["src/**/*.ts"] +} diff --git a/services/candles/package.json b/services/candles/package.json new file mode 100644 index 0000000..d6cc269 --- /dev/null +++ b/services/candles/package.json @@ -0,0 +1,12 @@ +{ + "name": "@islandflow/candles", + "private": true, + "type": "module", + "scripts": { + "dev": "bun run src/index.ts" + }, + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*" + } +} diff --git a/services/candles/src/index.ts b/services/candles/src/index.ts new file mode 100644 index 0000000..1c777c3 --- /dev/null +++ b/services/candles/src/index.ts @@ -0,0 +1,17 @@ +import { createLogger } from "@islandflow/observability"; + +const service = "candles"; +const logger = createLogger({ service }); + +logger.info("service starting"); + +const shutdown = (signal: string) => { + logger.info("service stopping", { signal }); + process.exit(0); +}; + +process.on("SIGINT", () => shutdown("SIGINT")); +process.on("SIGTERM", () => shutdown("SIGTERM")); + +// Keep the process alive until real listeners are wired. +setInterval(() => {}, 60_000); diff --git a/services/candles/tsconfig.json b/services/candles/tsconfig.json new file mode 100644 index 0000000..d8c6443 --- /dev/null +++ b/services/candles/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": [] + }, + "include": ["src/**/*.ts"] +} diff --git a/services/compute/package.json b/services/compute/package.json new file mode 100644 index 0000000..0fa9583 --- /dev/null +++ b/services/compute/package.json @@ -0,0 +1,12 @@ +{ + "name": "@islandflow/compute", + "private": true, + "type": "module", + "scripts": { + "dev": "bun run src/index.ts" + }, + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*" + } +} diff --git a/services/compute/src/index.ts b/services/compute/src/index.ts new file mode 100644 index 0000000..128439a --- /dev/null +++ b/services/compute/src/index.ts @@ -0,0 +1,17 @@ +import { createLogger } from "@islandflow/observability"; + +const service = "compute"; +const logger = createLogger({ service }); + +logger.info("service starting"); + +const shutdown = (signal: string) => { + logger.info("service stopping", { signal }); + process.exit(0); +}; + +process.on("SIGINT", () => shutdown("SIGINT")); +process.on("SIGTERM", () => shutdown("SIGTERM")); + +// Keep the process alive until real listeners are wired. +setInterval(() => {}, 60_000); diff --git a/services/compute/tsconfig.json b/services/compute/tsconfig.json new file mode 100644 index 0000000..d8c6443 --- /dev/null +++ b/services/compute/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": [] + }, + "include": ["src/**/*.ts"] +} diff --git a/services/eod-enricher/package.json b/services/eod-enricher/package.json new file mode 100644 index 0000000..d5e18c4 --- /dev/null +++ b/services/eod-enricher/package.json @@ -0,0 +1,12 @@ +{ + "name": "@islandflow/eod-enricher", + "private": true, + "type": "module", + "scripts": { + "dev": "bun run src/index.ts" + }, + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*" + } +} diff --git a/services/eod-enricher/src/index.ts b/services/eod-enricher/src/index.ts new file mode 100644 index 0000000..909d1e2 --- /dev/null +++ b/services/eod-enricher/src/index.ts @@ -0,0 +1,17 @@ +import { createLogger } from "@islandflow/observability"; + +const service = "eod-enricher"; +const logger = createLogger({ service }); + +logger.info("service starting"); + +const shutdown = (signal: string) => { + logger.info("service stopping", { signal }); + process.exit(0); +}; + +process.on("SIGINT", () => shutdown("SIGINT")); +process.on("SIGTERM", () => shutdown("SIGTERM")); + +// Keep the process alive until real listeners are wired. +setInterval(() => {}, 60_000); diff --git a/services/eod-enricher/tsconfig.json b/services/eod-enricher/tsconfig.json new file mode 100644 index 0000000..d8c6443 --- /dev/null +++ b/services/eod-enricher/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": [] + }, + "include": ["src/**/*.ts"] +} diff --git a/services/ingest-equities/package.json b/services/ingest-equities/package.json new file mode 100644 index 0000000..b907ccf --- /dev/null +++ b/services/ingest-equities/package.json @@ -0,0 +1,12 @@ +{ + "name": "@islandflow/ingest-equities", + "private": true, + "type": "module", + "scripts": { + "dev": "bun run src/index.ts" + }, + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*" + } +} diff --git a/services/ingest-equities/src/index.ts b/services/ingest-equities/src/index.ts new file mode 100644 index 0000000..417922b --- /dev/null +++ b/services/ingest-equities/src/index.ts @@ -0,0 +1,17 @@ +import { createLogger } from "@islandflow/observability"; + +const service = "ingest-equities"; +const logger = createLogger({ service }); + +logger.info("service starting"); + +const shutdown = (signal: string) => { + logger.info("service stopping", { signal }); + process.exit(0); +}; + +process.on("SIGINT", () => shutdown("SIGINT")); +process.on("SIGTERM", () => shutdown("SIGTERM")); + +// Keep the process alive until real listeners are wired. +setInterval(() => {}, 60_000); diff --git a/services/ingest-equities/tsconfig.json b/services/ingest-equities/tsconfig.json new file mode 100644 index 0000000..d8c6443 --- /dev/null +++ b/services/ingest-equities/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": [] + }, + "include": ["src/**/*.ts"] +} diff --git a/services/ingest-options/package.json b/services/ingest-options/package.json new file mode 100644 index 0000000..eb0a0f0 --- /dev/null +++ b/services/ingest-options/package.json @@ -0,0 +1,12 @@ +{ + "name": "@islandflow/ingest-options", + "private": true, + "type": "module", + "scripts": { + "dev": "bun run src/index.ts" + }, + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*" + } +} diff --git a/services/ingest-options/src/index.ts b/services/ingest-options/src/index.ts new file mode 100644 index 0000000..a07d1cc --- /dev/null +++ b/services/ingest-options/src/index.ts @@ -0,0 +1,17 @@ +import { createLogger } from "@islandflow/observability"; + +const service = "ingest-options"; +const logger = createLogger({ service }); + +logger.info("service starting"); + +const shutdown = (signal: string) => { + logger.info("service stopping", { signal }); + process.exit(0); +}; + +process.on("SIGINT", () => shutdown("SIGINT")); +process.on("SIGTERM", () => shutdown("SIGTERM")); + +// Keep the process alive until real listeners are wired. +setInterval(() => {}, 60_000); diff --git a/services/ingest-options/tsconfig.json b/services/ingest-options/tsconfig.json new file mode 100644 index 0000000..d8c6443 --- /dev/null +++ b/services/ingest-options/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": [] + }, + "include": ["src/**/*.ts"] +} diff --git a/services/refdata/package.json b/services/refdata/package.json new file mode 100644 index 0000000..eb64122 --- /dev/null +++ b/services/refdata/package.json @@ -0,0 +1,12 @@ +{ + "name": "@islandflow/refdata", + "private": true, + "type": "module", + "scripts": { + "dev": "bun run src/index.ts" + }, + "dependencies": { + "@islandflow/config": "workspace:*", + "@islandflow/observability": "workspace:*" + } +} diff --git a/services/refdata/src/index.ts b/services/refdata/src/index.ts new file mode 100644 index 0000000..82bf816 --- /dev/null +++ b/services/refdata/src/index.ts @@ -0,0 +1,17 @@ +import { createLogger } from "@islandflow/observability"; + +const service = "refdata"; +const logger = createLogger({ service }); + +logger.info("service starting"); + +const shutdown = (signal: string) => { + logger.info("service stopping", { signal }); + process.exit(0); +}; + +process.on("SIGINT", () => shutdown("SIGINT")); +process.on("SIGTERM", () => shutdown("SIGTERM")); + +// Keep the process alive until real listeners are wired. +setInterval(() => {}, 60_000); diff --git a/services/refdata/tsconfig.json b/services/refdata/tsconfig.json new file mode 100644 index 0000000..d8c6443 --- /dev/null +++ b/services/refdata/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": [] + }, + "include": ["src/**/*.ts"] +} diff --git a/tsconfig.base.json b/tsconfig.base.json new file mode 100644 index 0000000..f98f46a --- /dev/null +++ b/tsconfig.base.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "lib": ["ES2022"], + "strict": true, + "isolatedModules": true, + "resolveJsonModule": true, + "skipLibCheck": true, + "noEmit": true + } +}