Compare commits

...
20 Commits
Author SHA1 Message Date
Jack e64e4d0fbb docs: add Space Bunny to V2 console docs (#50940) 2026-09-24 01:03:39 +08:00
opencode-agent[bot]andMrMushrooooom 7f39ae9ecb fix(stats): attribute Hy4 preview traffic to Tencent (#50935)
Co-authored-by: MrMushrooooom <19261047+MrMushrooooom@users.noreply.github.com>
2026-09-24 00:56:42 +08:00
James Long b3b030856b feat(tui): budget transcript mounting by rendered entries (#50936) 2026-09-23 12:36:10 -04:00
Aiden Cline a117ebb408 fix(client): include the detail in client error messages (#50929) 2026-09-23 11:33:49 -05:00
Aiden Cline f2d9ebb886 fix(ai): send PDFs as file parts on OpenAI Chat routes (#50933) 2026-09-23 11:27:33 -05:00
James Long 0332a26be6 fix(tui): load history until the oldest group is complete (#50930) 2026-09-23 11:58:44 -04:00
opencode-agent[bot] bb7876dfa8 chore: update nix node_modules hashes 2026-09-23 15:48:10 +00:00
Shoubhit Dash dbaa57a21b feat(ai): add transcription across inline, streamed, and queued routes (#50910) 2026-09-23 21:15:50 +05:30
James Long b6bc55764c refactor(tui): persist group expansion and exact scroll anchors (#48489) 2026-09-23 11:33:19 -04:00
Aiden Cline 14a3311a61 feat(codemode): add Uint8Array callback methods and coerce indexOf start indexes (#50829) 2026-09-23 10:27:40 -05:00
opencode-agent[bot]andnexxeln dddb5eb96f fix(util): update OpenTelemetry to patched core (#50867)
Co-authored-by: nexxeln <95541290+nexxeln@users.noreply.github.com>
2026-09-23 20:57:16 +05:30
Shoubhit Dash affa57e40f fix(core): relax websocket idle timeout and honor chunkTimeout (#50914) 2026-09-23 20:22:41 +05:30
opencode-agent[bot] 681ea07a92 chore(core): refresh bundled models.dev snapshot 2026-09-23 12:20:47 +00:00
Adam 6e89d9e2c3 feat(core): identify the client on Console requests (#50896) 2026-09-23 07:09:19 -05:00
Shoubhit Dash f526727178 feat(ai): add speech generation with streaming media routes (#50883) 2026-09-23 17:35:23 +05:30
Luke Parker 9c8a63e852 fix(app): keep composer focus after attaching files (#50864) 2026-09-23 18:04:39 +10:00
Luke Parker fabf56781c fix(app): keep timeline scroll when switching models (#50865) 2026-09-23 18:03:42 +10:00
opencode a25d304201 sync release versions for v2.0.15 2026-09-23 07:15:55 +00:00
Shoubhit Dash cc8886c8bb feat(ai): add video generation with queued media routes (#50703) 2026-09-23 12:27:38 +05:30
Luke Parker 8ce629be22 fix(cli): keep Windows upgrades and uninstalls from fighting the running binary (#50819) 2026-09-23 06:35:40 +00:00
204 changed files with 11862 additions and 895 deletions
+57 -71
View File
@@ -32,7 +32,7 @@
},
"packages/ai": {
"name": "@opencode/ai",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@aws-sdk/credential-providers": "3.1057.0",
"@opencode/schema": "workspace:*",
@@ -54,7 +54,7 @@
},
"packages/app": {
"name": "@opencode/app",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@corvu/drawer": "catalog:",
"@dnd-kit/abstract": "0.5.0",
@@ -112,8 +112,9 @@
},
"packages/cli": {
"name": "@opencode/cli",
"version": "2.0.14",
"version": "2.0.15",
"bin": {
"opencode": "./bin/opencode.cjs",
"opencode2": "./bin/opencode2.cjs",
},
"dependencies": {
@@ -176,7 +177,7 @@
},
"packages/client": {
"name": "@opencode/client",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@opencode/protocol": "workspace:*",
"@opencode/schema": "workspace:*",
@@ -202,7 +203,7 @@
},
"packages/codemode": {
"name": "@opencode/codemode",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"acorn": "8.15.0",
"effect": "catalog:",
@@ -215,7 +216,7 @@
},
"packages/console/app": {
"name": "@opencode/console-app",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@cloudflare/vite-plugin": "1.15.2",
"@ibm/plex": "6.4.1",
@@ -251,7 +252,7 @@
},
"packages/console/core": {
"name": "@opencode/console-core",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@aws-sdk/client-sts": "3.782.0",
"@jsx-email/render": "1.1.1",
@@ -278,7 +279,7 @@
},
"packages/console/function": {
"name": "@opencode/console-function",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@openauthjs/openauth": "0.0.0-20250322224806",
"@opencode/console-core": "workspace:*",
@@ -295,7 +296,7 @@
},
"packages/console/mail": {
"name": "@opencode/console-mail",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@jsx-email/all": "2.2.3",
"@jsx-email/cli": "1.4.3",
@@ -319,7 +320,7 @@
},
"packages/console/support": {
"name": "@opencode/console-support",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@cloudflare/vite-plugin": "1.15.2",
"@opencode/console-core": "workspace:*",
@@ -339,7 +340,7 @@
},
"packages/core": {
"name": "@opencode/core",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@ai-sdk/cohere": "3.0.27",
"@ai-sdk/gateway": "3.0.104",
@@ -407,7 +408,7 @@
},
"packages/desktop": {
"name": "@opencode/desktop",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@zip.js/zip.js": "2.7.62",
"electron-context-menu": "5.0.0",
@@ -456,7 +457,7 @@
},
"packages/enterprise": {
"name": "@opencode/enterprise",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@hono/standard-validator": "catalog:",
"@opencode-ai/sdk": "1.18.21",
@@ -493,7 +494,7 @@
},
"packages/function": {
"name": "@opencode/function",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@octokit/auth-app": "8.0.1",
"@octokit/rest": "catalog:",
@@ -509,7 +510,7 @@
},
"packages/http-recorder": {
"name": "@opencode/http-recorder",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@effect/platform-node-shared": "4.0.0-rc.112",
},
@@ -528,7 +529,7 @@
},
"packages/httpapi-codegen": {
"name": "@opencode/httpapi-codegen",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"effect": "catalog:",
"prettier": "3.6.2",
@@ -541,7 +542,7 @@
},
"packages/latex": {
"name": "@opencode/latex",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@opencode/plugin": "workspace:*",
"@opentui/core": "catalog:",
@@ -555,7 +556,7 @@
},
"packages/merman": {
"name": "@opencode/merman",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@opencode/plugin": "workspace:*",
"@opentui/core": "catalog:",
@@ -570,7 +571,7 @@
},
"packages/plugin": {
"name": "@opencode/plugin",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@ai-sdk/provider": "3.0.8",
"@opencode/ai": "workspace:*",
@@ -609,7 +610,7 @@
},
"packages/plugin-browser": {
"name": "@opencode/plugin-browser",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@opencode/plugin": "workspace:*",
"@opencode/schema": "workspace:*",
@@ -639,7 +640,7 @@
},
"packages/protocol": {
"name": "@opencode/protocol",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@opencode/schema": "workspace:*",
"effect": "catalog:",
@@ -654,7 +655,7 @@
},
"packages/schema": {
"name": "@opencode/schema",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@standard-schema/spec": "catalog:",
"effect": "catalog:",
@@ -678,7 +679,7 @@
},
"packages/sdk": {
"name": "@opencode/sdk",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@opencode/client": "workspace:*",
"@opencode/core": "workspace:*",
@@ -699,7 +700,7 @@
},
"packages/server": {
"name": "@opencode/server",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@effect/platform-node": "catalog:",
"@effect/platform-node-shared": "catalog:",
@@ -721,7 +722,7 @@
},
"packages/session-ui": {
"name": "@opencode/session-ui",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@kobalte/core": "catalog:",
"@opencode/client": "workspace:*",
@@ -756,7 +757,7 @@
},
"packages/simulation": {
"name": "@opencode/simulation",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@opencode/ai": "workspace:*",
"@opencode/core": "workspace:*",
@@ -776,7 +777,7 @@
},
"packages/stats/app": {
"name": "@opencode/stats-app",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@ibm/plex": "6.4.1",
"@kobalte/core": "catalog:",
@@ -810,7 +811,7 @@
},
"packages/stats/core": {
"name": "@opencode/stats-core",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@aws-sdk/client-athena": "3.933.0",
"@planetscale/database": "1.19.0",
@@ -829,7 +830,7 @@
},
"packages/stats/server": {
"name": "@opencode/stats-server",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@aws-sdk/client-firehose": "3.933.0",
"@effect/platform-node": "catalog:",
@@ -875,7 +876,7 @@
},
"packages/theme": {
"name": "@opencode/theme",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@opentui/core": "catalog:",
"effect": "catalog:",
@@ -889,7 +890,7 @@
},
"packages/tui": {
"name": "@opencode/tui",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@opencode/client": "workspace:*",
"@opencode/core": "workspace:*",
@@ -924,7 +925,7 @@
},
"packages/ui": {
"name": "@opencode/ui",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@kobalte/core": "catalog:",
"@pierre/diffs": "catalog:",
@@ -959,7 +960,7 @@
},
"packages/util": {
"name": "@opencode/util",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@effect/opentelemetry": "catalog:",
"@effect/platform-node": "catalog:",
@@ -967,10 +968,14 @@
"@npmcli/arborist": "catalog:",
"@npmcli/config": "10.8.1",
"@opentelemetry/api": "1.9.0",
"@opentelemetry/context-async-hooks": "2.6.1",
"@opentelemetry/exporter-trace-otlp-http": "0.214.0",
"@opentelemetry/sdk-trace-base": "2.6.1",
"@opentelemetry/sdk-trace-node": "2.6.1",
"@opentelemetry/api-logs": "0.219.0",
"@opentelemetry/context-async-hooks": "2.8.0",
"@opentelemetry/exporter-trace-otlp-http": "0.219.0",
"@opentelemetry/resources": "2.8.0",
"@opentelemetry/sdk-logs": "0.219.0",
"@opentelemetry/sdk-metrics": "2.8.0",
"@opentelemetry/sdk-trace-base": "2.8.0",
"@opentelemetry/sdk-trace-node": "2.8.0",
"cross-spawn": "catalog:",
"effect": "catalog:",
"glob": "13.0.5",
@@ -992,7 +997,7 @@
},
"packages/web": {
"name": "@opencode/web",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"@astrojs/cloudflare": "12.6.3",
"@astrojs/markdown-remark": "6.3.1",
@@ -1033,7 +1038,7 @@
},
"services/update": {
"name": "@opencode/update",
"version": "2.0.14",
"version": "2.0.15",
"dependencies": {
"jose": "6.0.11",
"semver": "catalog:",
@@ -1068,7 +1073,6 @@
"trustedDependencies": [
"electron",
"esbuild",
"protobufjs",
],
"patchedDependencies": {
"@pierre/trees@1.0.0-beta.4": "patches/@pierre%2Ftrees@1.0.0-beta.4.patch",
@@ -2220,31 +2224,31 @@
"@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="],
"@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.214.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-40lSJeqYO8Uz2Yj7u94/SJWE/wONa7rmMKjI1ZcIjgf3MHNHv1OZUCrCETGuaRF62d5pQD1wKIW+L4lmSMTzZA=="],
"@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.219.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-FFx7YnaYJlIjqWW/AG/yAZ0L/NEY724PipXXXQLdtZPbLwBGbUMTGL1i/esI56TWfTUXxhLfpgrnWJCG8aUJyg=="],
"@opentelemetry/context-async-hooks": ["@opentelemetry/context-async-hooks@2.6.1", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-XHzhwRNkBpeP8Fs/qjGrAf9r9PRv67wkJQ/7ZPaBQQ68DYlTBBx5MF9LvPx7mhuXcDessKK2b+DcxqwpgkcivQ=="],
"@opentelemetry/context-async-hooks": ["@opentelemetry/context-async-hooks@2.8.0", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-/3FIraneMcng67SUJCxvyInk/oxzwsxyadufk0wwfOBLf5wqtAGX4MoQASwSbndBPeARzBryUM9Azr5kHIdWLw=="],
"@opentelemetry/core": ["@opentelemetry/core@2.6.1", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-8xHSGWpJP9wBxgBpnqGL0R3PbdWQndL1Qp50qrg71+B28zK5OQmUgcDKLJgzyAAV38t4tOyLMGDD60LneR5W8g=="],
"@opentelemetry/core": ["@opentelemetry/core@2.8.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-hd1Lfh8p545nNz+jq1Ejfz+Mn1hyLuxYn1YzTfFNrxr8urEWMNQLPf1Th8kjOH+HxwawCrtgBp8JpBUR4ZSgww=="],
"@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/exporter-trace-otlp-http@0.214.0", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/otlp-exporter-base": "0.214.0", "@opentelemetry/otlp-transformer": "0.214.0", "@opentelemetry/resources": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-kIN8nTBMgV2hXzV/a20BCFilPZdAIMYYJGSgfMMRm/Xa+07y5hRDS2Vm12A/z8Cdu3Sq++ZvJfElokX2rkgGgw=="],
"@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/exporter-trace-otlp-http@0.219.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/otlp-exporter-base": "0.219.0", "@opentelemetry/otlp-transformer": "0.219.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-9t6SvBXXBEjOBcIzgozvBbd3jWrv3Gt3ngGhl1fhdZ/zRc7oZDVOFEqbi2zlBpW9BXhgDMKv422J0DL/3iQWfw=="],
"@opentelemetry/instrumentation": ["@opentelemetry/instrumentation@0.220.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "import-in-the-middle": "^3.0.0", "require-in-the-middle": "^8.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw=="],
"@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.214.0", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/otlp-transformer": "0.214.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-u1Gdv0/E9wP+apqWf7Wv2npXmgJtxsW2XL0TEv9FZloTZRuMBKmu8cYVXwS4Hm3q/f/3FuCnPTgiwYvIqRSpRg=="],
"@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.219.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/otlp-transformer": "0.219.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-zvIxQX/AZUVKDU+hCuYx+7UkiP7GRdnk1ZbFQRYzHvYp47cAWR4j3IhoPhV9KaeXEv2xdGq3IA6PnpzDmLcmSA=="],
"@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.214.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.214.0", "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", "@opentelemetry/sdk-logs": "0.214.0", "@opentelemetry/sdk-metrics": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1", "protobufjs": "^7.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-DSaYcuBRh6uozfsWN3R8HsN0yDhCuWP7tOFdkUOVaWD1KVJg8m4qiLUsg/tNhTLS9HUYUcwNpwL2eroLtsZZ/w=="],
"@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.219.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.219.0", "@opentelemetry/core": "2.8.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-logs": "0.219.0", "@opentelemetry/sdk-metrics": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-aaYKAyXhw9VchKZVGOopD3Gw/kPsyrX2c6IQ0AW32mTjqmZOh5Y6Gf5OYqTNqVktAeBjmFinhyFaCwW6GYK9YQ=="],
"@opentelemetry/resources": ["@opentelemetry/resources@2.6.1", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-lID/vxSuKWXM55XhAKNoYXu9Cutoq5hFdkbTdI/zDKQktXzcWBVhNsOkiZFTMU9UtEWuGRNe0HUgmsFldIdxVA=="],
"@opentelemetry/resources": ["@opentelemetry/resources@2.8.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-qmXQ27ilDbUK/vGMqwL8D4/rhn76C+sherM4wTbjlfknR8Nvfc/hCxjRJPhkzZzUsPiNg16SA31NxMabwttRjg=="],
"@opentelemetry/sdk-logs": ["@opentelemetry/sdk-logs@0.214.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.214.0", "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-zf6acnScjhsaBUU22zXZ/sLWim1dfhUAbGXdMmHmNG3LfBnQ3DKsOCITb2IZwoUsNNMTogqFKBnlIPPftUgGwA=="],
"@opentelemetry/sdk-logs": ["@opentelemetry/sdk-logs@0.219.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.219.0", "@opentelemetry/core": "2.8.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-s6lTKRakaPClvKoWHRChxnXjDMkM/TQ30ff78jN6EBGf7MI7VzANE5PU3f4z9qDUudWjvZjOLHG0rBnBKYvoXA=="],
"@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@2.6.1", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-9t9hJHX15meBy2NmTJxL+NJfXmnausR2xUDvE19XQce0Qi/GBtDGamU8nS1RMbdgDmhgpm3VaOu2+fiS/SfTpQ=="],
"@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@2.8.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/resources": "2.8.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-UDBGaj6W0Rgy5rTTaoxs8gVGF/aGkAKyjurJv7se6wjRxJu7FoquTLT/vt54DZfo4crbprYfhX/SOK9+BPw1qg=="],
"@opentelemetry/sdk-trace": ["@opentelemetry/sdk-trace@2.11.0", "", { "dependencies": { "@opentelemetry/core": "2.11.0", "@opentelemetry/resources": "2.11.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA=="],
"@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.6.1", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-r86ut4T1e8vNwB35CqCcKd45yzqH6/6Wzvpk2/cZB8PsPLlZFTvrh8yfOS3CYZYcUmAx4hHTZJ8AO8Dj8nrdhw=="],
"@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.8.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-mhU4jp+vW0mGbFRd+GeXHvmfA4aDqWjBjLC3pE5XMpLs0IE2ryYb019Ts2AQrOq67gaTF25D91+fgvEHDZEnuQ=="],
"@opentelemetry/sdk-trace-node": ["@opentelemetry/sdk-trace-node@2.6.1", "", { "dependencies": { "@opentelemetry/context-async-hooks": "2.6.1", "@opentelemetry/core": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-Hh2i4FwHWRFhnO2Q/p6svMxy8MPsNCG0uuzUY3glqm0rwM0nQvbTO1dXSp9OqQoTKXcQzaz9q1f65fsurmOhNw=="],
"@opentelemetry/sdk-trace-node": ["@opentelemetry/sdk-trace-node@2.8.0", "", { "dependencies": { "@opentelemetry/context-async-hooks": "2.8.0", "@opentelemetry/core": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-nZt9OGufioAc3AfoLTqA9bsAeaMJAictYDdI2VcNQ+PmT+3rfKjAZDZvgPfd8VPX0O5Bw1hdQF6kDK8VSpZiWg=="],
"@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.43.0", "", {}, "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg=="],
@@ -2554,24 +2558,6 @@
"@protobuf-ts/runtime-rpc": ["@protobuf-ts/runtime-rpc@2.11.1", "", { "dependencies": { "@protobuf-ts/runtime": "^2.11.1" } }, "sha512-4CqqUmNA+/uMz00+d3CYKgElXO9VrEbucjnBFEjqI4GuDrEQ32MaI3q+9qPBvIGOlL4PmHXrzM32vBPWRhQKWQ=="],
"@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="],
"@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="],
"@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="],
"@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.1", "", {}, "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg=="],
"@protobufjs/fetch": ["@protobufjs/fetch@1.1.1", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1" } }, "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw=="],
"@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="],
"@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="],
"@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="],
"@protobufjs/utf8": ["@protobufjs/utf8@1.1.2", "", {}, "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug=="],
"@puppeteer/browsers": ["@puppeteer/browsers@3.2.1", "", { "dependencies": { "modern-tar": "^0.8.0", "yargs": "^18.0.0" }, "peerDependencies": { "proxy-agent": ">=8.0.1", "yauzl": "^2.10.0 || ^3.4.0" }, "optionalPeers": ["proxy-agent", "yauzl"], "bin": { "browsers": "lib/main-cli.js" } }, "sha512-KDz+3qDRdBAlRlMjmKyj6dEs33YHTk/xRHEENSXq6TNnhgoU15ruSHtEBeVF6OZ9tBDY55Se4P0nFMNsipzU9A=="],
"@radix-ui/colors": ["@radix-ui/colors@1.0.1", "", {}, "sha512-xySw8f0ZVsAEP+e7iLl3EvcBXX7gsIlC1Zso/sPBW9gIWerBTgz6axrjU+MZ39wD+WFi5h5zdWpsg3+hwt2Qsg=="],
@@ -5048,8 +5034,6 @@
"proto-list": ["proto-list@1.2.4", "", {}, "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA=="],
"protobufjs": ["protobufjs@7.6.5", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw=="],
"proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="],
"pump": ["pump@3.0.4", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA=="],
@@ -6246,6 +6230,8 @@
"@opencode/www/wrangler": ["wrangler@4.110.0", "", { "dependencies": { "@cloudflare/kv-asset-handler": "0.5.0", "@cloudflare/unenv-preset": "2.16.1", "blake3-wasm": "2.1.5", "esbuild": "0.28.1", "miniflare": "4.20260708.1", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", "workerd": "1.20260708.1" }, "optionalDependencies": { "fsevents": "2.3.3" }, "peerDependencies": { "@cloudflare/workers-types": "^5.20260708.1" }, "optionalPeers": ["@cloudflare/workers-types"], "bin": { "wrangler": "bin/wrangler.js", "wrangler2": "bin/wrangler.js", "cf-wrangler": "bin/cf-wrangler.js" } }, "sha512-xZeXKYi7hxQRF5anL+v77RkufJNpF9f3Eqeyqq2QBsETpLZgh0Agj0jJ6JPtkbgn6ukZdh8OK5egsGPWIditgg=="],
"@opentelemetry/api-logs/@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="],
"@opentelemetry/instrumentation/@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.220.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w=="],
"@opentelemetry/sdk-trace/@opentelemetry/core": ["@opentelemetry/core@2.11.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA=="],
+4 -4
View File
@@ -1,8 +1,8 @@
{
"nodeModules": {
"x86_64-linux": "sha256-DVimx64BQEkFaYZkW56E5eEfkGxuW5j/9C0wCImBFl4=",
"aarch64-linux": "sha256-MhMl/ZkvcPXE31OvysutI6nKIJ6XtUa3c71+B4hnaQc=",
"aarch64-darwin": "sha256-+wMPYLBnw2zTZj0if8EPwWC9O3tVqSgnhYNfQq8UZ8Q=",
"x86_64-darwin": "sha256-l6LxVSXDbqVoXMVcGZAzRER0bxP6wDU4dfRDolx0JGM="
"x86_64-linux": "sha256-LQ1GAz1qF4R5P4j/kkgUygsQvxm7KStVlMf24nmyq44=",
"aarch64-linux": "sha256-PsNR3VaClA1O1vSE0z/Hb3XRT1pb6PvNRXuK+giIx6s=",
"aarch64-darwin": "sha256-7VCS+GT7tDYAMgqNCnLh1zvHAZkLTnJJ4vtfM88ruT8=",
"x86_64-darwin": "sha256-HyxXjiFVd8vcKvSb/BuHAY5Jj3MGgdXEvuhDuglp4ss="
}
}
+1 -1
View File
@@ -2,7 +2,7 @@
"$schema": "https://json.schemastore.org/package.json",
"name": "opencode",
"description": "AI-powered development tool",
"version": "2.0.14",
"version": "2.0.15",
"private": true,
"type": "module",
"packageManager": "bun@1.4.2",
+7 -1
View File
@@ -96,7 +96,13 @@ When a provider supports multiple physical transports, selection remains executi
### Media Routes
Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute.make(...)` (`src/route/media.ts`) composes a `MediaProtocol` kind with `Endpoint` and `Auth` and owns the transport plumbing: `http` option merging, URL/query rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol. `MediaProtocol.inline` (`src/route/media-protocol.ts`) is `body.from(request)` plus `response.decode(response, context)`; use `MediaProtocol.decodeJson` / `text` / `bytes` so decode failures retain the raw body and HTTP context. `Generation` (`src/generation.ts`) is the provider-neutral handle for a queued generation over a `GenerationRoute` (`status`, `result`, `cancel`, `pollHint`); the first video route implements it. Image protocol files follow the same section order as LLM protocols and declare unsupported common fields once through `MediaInput.rejectUnsupported`.
Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute.make(...)` (`src/route/media.ts`) composes a `MediaProtocol` kind with `Endpoint` and `Auth` and owns the transport plumbing: `http` option merging, URL/query rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol. `MediaProtocol.inline` (`src/route/media-protocol.ts`) is `body.from(request)` plus `response.decode(response, context)`; use `MediaProtocol.decodeJson` / `text` / `bytes` so decode failures retain the raw body and HTTP context. `Generation` (`src/generation.ts`) is the provider-neutral handle for a queued generation over a `GenerationRoute` (`status`, `result`, `cancel`, `pollHint`). Image protocol files follow the same section order as LLM protocols and declare unsupported common fields once through `MediaInput.rejectUnsupported`.
`MediaProtocol.queued` is the submit-then-poll kind every video route uses: `start` (body + decode into `{ token, snapshot }`), `status`, `result`, and optional `cancel`, each addressed by a route-owned `token` whose `Schema.Codec` makes it serializable. `MediaRoute.inline` and `MediaRoute.queued` compose the two kinds with `Endpoint` and `Auth`; the queued route decodes the token once at the boundary (`start` output or `resume` input) and closes over it in a token-free `GenerationRoute` (`status`/`result`/`cancel` are plain Effects), so `Generation` never sees the token's shape and only carries the encoded JSON for persistence. Polls reuse the route's auth and deployment headers plus the request's `http` overlay after `start`, and resolve relative paths against the route base URL (provider-issued absolute URLs such as fal's `status_url` pass through). `result` is always its own GET even when the provider returns output inside the status document, so `Generation.await` behaves the same after `start` and after `resume`. `PollContext.auth` carries only what `Auth` added so protocols can hand download credentials to output assets as transient `Media.Asset.headers` (Veo) — never part of `source` or JSON. Status strings map through a per-protocol `STATUS` table via `MediaProtocol.status`; terminal generations without output fail through `output.ended` / `output.contentPolicy` with the provider document on `reason.body`. `Generation.AwaitOptions` (`{ poll?: Poll }`) is the one options type for `await`, `events`, `Video.generate`, and `Video.stream`.
`MediaProtocol.stream` is the incremental kind every speech route uses, with the same discipline as LLM protocols. `MediaRoute.stream` submits the caller's request as `MediaProtocol.Addressed<Request>` (`{ ...request, mode }`, `mode: "generate" | "stream"`), so one provider stays one protocol: `body.from`, the endpoint path, and `frames` read `request.mode` to pick the body, path, and framing. `frames(bytes, context)` returns frames — `Framing.sse`, `Framing.lines`, `Framing.document` (a single-document response shaped like a streamed record), or the raw `bytes` for chunked audio. `initial()` is fresh per-response parser state; `step` folds each frame into it and emits modality events; `finish(state, context)` runs once after the last frame with the request, body, and observed `http` (header-only usage lives there) and emits exactly one terminal event or fails with `MediaProtocol.incomplete`. Keep parser state to real accumulators and derive anything the request or body determines in `finish`. `generate` runs the same stream and folds it with the modality's `collect`. Request-derived URL parameters go on the body's `query` (array values repeat the parameter), applied before route and caller `http.query`. Decode frames with `MediaProtocol.decodeFrame` and raise stream-time failures with `MediaProtocol.frameError` (the frame stays on `reason.body`); protocols never thread HTTP context, because the route fills `reason.http` on stream errors that lack it. Speech protocols share `protocols/utils/speech-stream.ts` for deltas, timestamps, voice ids, PCM and container descriptions, and the terminal asset.
Transcription uses all three kinds (OpenAI and Gemini stream, Deepgram is inline, AssemblyAI is queued): every route carries its `kind` and `TranscriptionClient` dispatches on it. Bodies are `json`, `multipart`, or `binary` (a raw upload), and a queued protocol that must upload media before submitting implements `start.prepare` (`MediaProtocol.Prepare`; AssemblyAI `/v2/upload`).
### URL Construction
+230 -1
View File
@@ -602,6 +602,233 @@ const program = Effect.gen(function* () {
The hosted result is represented as a provider-executed tool call and tool result, and the generated image is also emitted as a first-class `media` `LLMEvent` (`response.message` then carries a `media` part). Gemini image-capable models emit the same `media` event for inline image output. Retaining `response.message` preserves the generated image for continuation on both routes.
## Video generation
Video mirrors `Image` with one difference: every provider is asynchronous, so the route is a submit-then-poll
`Generation`. Models come from `.video(...)` selectors on the `Google` (Veo), `XAI`, `Fal`, and `Runway` facades.
Common fields (`frames`, `references`, `video`, `durationSeconds`, `aspectRatio`, `resolution`, `audio`, `n`, `seed`,
`negativePrompt`) lower natively or fail with a typed `AIError` before any network call; provider-native controls live
under `providerOptions`, inferred from the selected model.
```ts
import { Video, VideoClient } from "@opencode/ai"
import { Google } from "@opencode/ai/providers"
const google = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY })
// Simple: submit and wait.
const program = Effect.gen(function* () {
const response = yield* Video.generate(
{
model: google.video("veo-3.1-generate-preview"),
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
aspectRatio: "16:9",
resolution: "1080p",
durationSeconds: 8,
providerOptions: { personGeneration: "allow_adult" },
},
{ poll: { interval: "10 seconds", timeout: "10 minutes" } },
)
// Veo serves files for two days behind the API key. The asset knows the deadline (`expiresAt`) and carries the
// download credentials only on the live instance (`asset.headers`), never in `source` or JSON: materialize
// before persisting, or the persisted URL cannot be fetched again.
return yield* response.video.materialize()
})
// Explicit control: keep the handle, persist the token, resume elsewhere.
const controlled = Effect.gen(function* () {
const generation = yield* Video.start({ model: google.video("veo-3.1-generate-preview"), prompt })
generation.id // provider operation / task / request id
generation.status // "queued" | "running" | "completed" | "failed" | "cancelled" | "expired"
generation.token // route-owned JSON: `{ operation }`, `{ requestID }`, `{ taskID }`, or fal's follow-up URLs
const saved = JSON.stringify(generation.token)
const resumed = yield* Video.resume(google.video("veo-3.1-generate-preview"), JSON.parse(saved))
return yield* resumed.await({ poll: { interval: "10 seconds" } })
})
// Progress as a stream: generation-queued | generation-progress | video | finish.
const events = Video.stream({ model: Runway.configure({ apiKey }).video("gen4.5"), prompt }, { poll })
```
`VideoClient.layer` needs `RequestExecutor.Service`, and status polls, result fetches, cancels, and asset downloads
all run through the same executor with the route's auth. `Generation.await` and `Generation.events` fail with a
`Timeout` reason when `poll.timeout` (default 10 minutes) elapses. Failed,
cancelled, and expired generations fail typed with the provider's terminal document on `reason.body`; moderation
outcomes (Veo `raiMediaFilteredReasons`, xAI `respect_moderation`, Runway `SAFETY.*` codes) surface as `notices` when
a video is still returned and as a `ContentPolicy` reason when nothing is.
Provider notes:
- **Google Veo** takes inline bytes only (materialize `url` assets first); `frames.last` requires `frames.first`;
audio is always on, so `audio: false` fails typed; one video per request. Output URLs need the API key to
download, which the returned asset holds transiently (see above).
- **xAI** sends a `video` input to `/videos/edits`, or `/videos/extensions` with `providerOptions.mode: "extend"`.
`seed` and `negativePrompt` are not supported.
- **fal** endpoints are model-specific: `durationSeconds`, `references`, and `frames.last` fail typed and belong in
`providerOptions` under the model's own names (`duration: "8s"`, `end_image_url`, …). Auth is
`Authorization: Key <FAL_KEY>`.
- **Runway** expects pixel ratios in `aspectRatio` for most models (`"1280:720"`), pins `X-Runway-Version`, reports
`usage: { type: "credits" }`, and its output URLs expire after 2448 hours.
The promise client exposes the same surface: `ai.video.start(...)` resolves to a handle with `await`, `refresh`,
`cancel`, and `token`; `ai.video.generate`, `ai.video.resume(model, token)`, and `ai.video.stream` mirror the Effect
API.
```ts
import { ai } from "@opencode/ai/promise"
const generation = await ai.video.start({ model, prompt })
const video = await generation.await({ poll: { interval: 10_000 }, signal })
```
## Speech generation
Speech (text-to-speech) is one request whose response is parsed incrementally, so every route supports both
`Speech.generate` (the whole file) and `Speech.stream` (audio chunks as they arrive). Models come from `.speech(...)`
selectors on the `OpenAI`, `Google` (Gemini TTS), `ElevenLabs`, `Cartesia`, and `Deepgram` facades. Common fields
(`voice`, `format`, `speed`, `language`, `instructions`, `timestamps`) lower natively or fail with a typed `AIError`
before any network call; provider-native controls live under `providerOptions`, inferred from the selected model.
```ts
import { Media, Speech, SpeechClient, SpeechEvent } from "@opencode/ai"
import { ElevenLabs, OpenAI } from "@opencode/ai/providers"
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
// The whole file, written to disk.
const program = Effect.gen(function* () {
const response = yield* Speech.generate({
model: openai.speech("gpt-4o-mini-tts"),
text: "Hello from OpenCode.",
voice: "coral",
format: "mp3",
instructions: "Warm and unhurried.",
})
response.audio // Media.Asset with bytes; headerless PCM carries info.encoding / sampleRate / channels
response.usage // undefined: OpenAI reports tokens only on SSE streams (Gemini: tokens; ElevenLabs: credits; Deepgram: characters)
yield* Media.write(response.audio, "hello.mp3")
})
// Chunks as they arrive: audio-delta* (interleaved with timestamps) then one finish carrying the assembled asset.
const events = Speech.stream({
model: ElevenLabs.configure({ apiKey }).speech("eleven_flash_v2_5"),
text: "Hello from OpenCode.",
voice: "JBFqnCBsd6RMkjVDRZzb",
format: "pcm",
timestamps: true,
}).pipe(
Stream.tap((event) => {
if (SpeechEvent.is.audioDelta(event)) return play(event.chunk)
if (SpeechEvent.is.timestamps(event)) return highlight(event.items) // { text, startSeconds, endSeconds }[]
return Effect.void
}),
)
```
`voice` is the provider's own identifier — a name on OpenAI and Gemini (`"coral"`, `"Kore"`), a voice id on
ElevenLabs and Cartesia. `{ id }` selects an OpenAI custom voice (`{ id: "voice_1234" }`) and means the same as the
plain string elsewhere. There is no cross-provider voice catalog. `format` is the container-level word (`mp3`, `wav`,
`pcm`, `opus`, `aac`, `flac`); sample rates and bitrates live under `providerOptions`, and a value the route cannot
produce fails as `UnsupportedOperation`. Streams buffer every chunk so `finish` can carry the whole clip.
`SpeechClient.layer` needs `RequestExecutor.Service`.
Provider notes:
- **OpenAI** streams over SSE (`stream_format: "sse"`), which is also the only place it reports token usage; `tts-1`
and `tts-1-hd` do not support SSE and stream the raw audio body instead. `pcm` is 24 kHz 16-bit mono. `language`
and `timestamps` are not supported.
- **Gemini TTS** returns raw 16-bit PCM only (`audio/L16;codec=pcm;rate=24000`), so any `format` other than `pcm`
fails typed; wrap the samples yourself. Style is directed in the text, so `instructions` and `speed` fail typed.
Only `gemini-3.1-flash-tts-preview` and later support streaming. Two-speaker audio goes through
`providerOptions.speechConfig.multiSpeakerVoiceConfig`.
- **ElevenLabs** requires `voice` (the path voice id) and authenticates with `xi-api-key`. `format` maps to the
`output_format` query parameter (`mp3_44100_128`, `pcm_24000`, `wav_24000`, `opus_48000_64`);
`providerOptions.outputFormat` sets the exact string. WAV is only available from `generate`. `timestamps: true`
selects the `with-timestamps` endpoints and yields character-level alignment. `instructions` is not supported.
- **Cartesia** requires `voice` and pins `Cartesia-Version`. `generate` defaults to MP3 from `/tts/bytes`; streams
and `timestamps: true` (word-level) use `/tts/sse`, which only serves raw PCM. `providerOptions.sampleRate`,
`bitRate`, and `encoding` complete `output_format`. No usage is reported.
- **Deepgram** Aura's voice is the model id (`aura-2-thalia-en`), so `voice` and `language` fail typed. `format`
and `providerOptions` lower to query parameters (`encoding`, `container`, `sample_rate`, `bit_rate`); `pcm` is
`linear16` without a container. Auth is `Authorization: Token <DEEPGRAM_API_KEY>`.
The promise client mirrors the Effect API; `ai.speech.stream` is an `AsyncIterable`.
```ts
import { ai } from "@opencode/ai/promise"
const response = await ai.speech.generate({ model, text: "Hello from OpenCode.", voice: "coral" })
await Bun.write("hello.mp3", await ai.run(response.audio.bytes()))
for await (const event of ai.speech.stream({ model, text: "Hello from OpenCode.", voice: "coral" })) {
if (event.type === "audio-delta") player.write(event.chunk)
}
```
## Transcription
Transcription (speech-to-text) is the one modality whose providers use every route kind: OpenAI and Gemini stream,
Deepgram answers inline, and AssemblyAI is queued. `Transcription.generate` and `Transcription.stream` work on all of
them; `Transcription.start` / `resume` return a `Generation` on queued routes and fail with `UnsupportedOperation`
elsewhere. Models come from `.transcription(...)` selectors on the `OpenAI`, `Google`, `Deepgram`, and `AssemblyAI`
facades. Common fields (`language`, `prompt`, `timestamps: "none" | "segment" | "word"`, `diarize`, `speakers`) lower
natively or fail with a typed `AIError` before any network call; a route may return more than asked.
```ts
import { Media, Transcription, TranscriptionEvent } from "@opencode/ai"
import { AssemblyAI, Deepgram, OpenAI } from "@opencode/ai/providers"
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
const program = Effect.gen(function* () {
const audio = yield* Media.file("./call.mp3")
// Speaker-labelled segments; labels are provider-native strings ("A", "0", "spk:0").
const response = yield* Transcription.generate({
model: Deepgram.configure({ apiKey }).transcription("nova-3"),
audio,
diarize: true,
timestamps: "word",
})
response.text // "Hello from OpenCode."
response.segments // [{ text, startSeconds, endSeconds, speaker: "0" }]
response.words // [{ text, startSeconds, endSeconds, speaker, confidence }]
response.language // the provider's own value, lowercased ("en", "english", "en_us")
// Text deltas as the model transcribes, then one finish carrying the whole transcript.
yield* Transcription.stream({ model: openai.transcription("gpt-4o-mini-transcribe"), audio }).pipe(
Stream.tap((event) => (TranscriptionEvent.is.textDelta(event) ? Console.log(event.delta) : Effect.void)),
Stream.runDrain,
)
// Queued: persist the token, resume from another process, and await.
const model = AssemblyAI.configure({ apiKey }).transcription("universal-3-5-pro")
const generation = yield* Transcription.start({ model, audio })
const resumed = yield* Transcription.resume(model, JSON.parse(JSON.stringify(generation.token)))
const transcript = yield* resumed.await({ poll: { interval: "3 seconds" } })
})
```
Inline routes emit only `finish` from `stream` (no faked deltas); queued routes emit `generation-queued` /
`generation-progress` before it. `TranscriptionClient.layer` needs `RequestExecutor.Service`.
Provider notes:
- **OpenAI** takes inline audio only; `diarize` needs `gpt-4o-transcribe-diarize`, timestamps need `whisper-1`, and `whisper-1` does not stream.
- **Gemini** needs a transcribe model (`gemini-3.5-transcribe`); `prompt` and `speakers` fail typed.
- **Deepgram** detects the language unless `language` is set; vocabulary goes in `providerOptions.keyterm`.
- **AssemblyAI** uploads inline audio before submitting and is the only route that accepts `speakers`.
The promise client mirrors the Effect API:
```ts
const text = (await ai.transcription.generate({ model, audio })).text
for await (const event of ai.transcription.stream({ model, audio })) if (event.type === "text-delta") write(event.delta)
const generation = await ai.transcription.start({ model: assemblyai, audio })
const transcript = await generation.await({ poll: { interval: 3_000 } })
```
## Public API
- **`LLM.request({...})`** — build a provider-neutral `LLMRequest`. Accepts ergonomic inputs (`system: string`, `prompt: string`) that normalize into the canonical Schema classes.
@@ -613,7 +840,9 @@ The hosted result is represented as a provider-executed tool call and tool resul
- **`ImageClient`** — Effect service and layer for image execution, parallel to `LLMClient`.
- **`Media`** — the shared asset type (`Media.Asset`, `Media.Source`) and constructors used by messages, tool results, and media requests.
- **`Generation`** — provider-neutral handle for an in-flight media generation (`await`, `refresh`, `cancel`, `events`) used by queued media routes.
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm` and `image` as Promise / `AsyncIterable` APIs.
- **`Speech.request` / `Speech.generate` / `Speech.stream`** — text-to-speech through a provider-neutral request; `SpeechClient` is its Effect service and layer.
- **`Transcription.request` / `generate` / `stream` / `start` / `resume`** — speech-to-text over inline, streaming, and queued routes; `TranscriptionClient` is its Effect service and layer.
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm`, `image`, `video`, `speech`, and `transcription` as Promise / `AsyncIterable` APIs.
## Testing
+154 -38
View File
@@ -1,6 +1,6 @@
# Media generation in `@opencode/ai` — public API direction
Status: proposal. Branch `media-support`.
Status: phases 13 implemented (Speech and Transcription); phases 45 proposal.
## Goal
@@ -66,7 +66,7 @@ import { Media } from "@opencode/ai"
Media.Source =
| { type: "bytes"; data: Uint8Array; mediaType: string }
| { type: "base64"; data: string; mediaType: string }
| { type: "url"; url: string; mediaType?: string; expiresAt?: number; headers?: Record<string, string> }
| { type: "url"; url: string; mediaType?: string; expiresAt?: number }
| { type: "ref"; provider: ProviderID; id: string; mediaType?: string } // file_id, gs://, runway://, prior generation
class Media.Asset {
@@ -76,6 +76,7 @@ class Media.Asset {
readonly info?: { width?; height?; durationSeconds?; sampleRate?; channels?; encoding?; format? }
readonly expiresAt?: number
readonly providerMetadata?: ProviderMetadata
readonly headers?: Record<string, string> // transient download credentials (Veo); never in source/JSON
bytes(): Effect<Uint8Array, AIError, RequestExecutor.Service> // downloads/decodes lazily, cached
base64(): Effect<string, AIError, RequestExecutor.Service>
@@ -129,98 +130,210 @@ Editing is not a separate function; `images`/`mask` on the request select the ed
#### Video
Shipped in phase 2 (`src/video.ts`, `src/video-client.ts`, protocols `google-video`, `xai-video`, `fal-video`, `runway-video`).
```ts
const request = Video.request({
model: google.video("veo-3.1-generate-preview"),
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
frames: { first: Media.file("./start.png"), last: Media.file("./end.png") },
references: [Media.url("https://…/style.png")],
video: Media.ref("openai", "video_123"), // edit / extend / remix source
references: [Media.file("./style.png")],
video: Media.bytes(previous, "video/mp4"), // edit / extend source
durationSeconds: 8,
aspectRatio: "16:9",
resolution: "1080p",
audio: true,
n: 1,
providerOptions: { personGeneration: "dont_allow", negativePrompt: "text, watermark" },
seed: 7,
negativePrompt: "text, watermark", // common, not provider-native
providerOptions: { personGeneration: "allow_adult" },
})
// Simple: wait for it.
const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } })
response.video // Media.Asset (url with expiresAt, or bytes when the route downloads)
response.video // Media.Asset: url with expiresAt (+ transient `headers` for Veo downloads)
response.usage // credits on Runway; the other three report none
response.notices // Veo raiMediaFilteredReasons → filtered, xAI respect_moderation → moderated
yield* response.video.materialize() // pull bytes before the URL expires
// Explicit generation control.
const generation = yield* Video.start(request) // Generation<VideoResponse>
generation.id; generation.status; generation.progress; generation.token // token is serializable JSON
generation.id; generation.status; generation.progress; generation.position; generation.token
yield* generation.await({ poll }) // VideoResponse
yield* generation.cancel()
yield* generation.cancel() // fal PUT cancel_url, Runway DELETE /tasks/{id}; no-op for Veo and xAI
// Resume from another process.
const resumed = yield* Video.resume(model, token) // Generation<VideoResponse>
// Resume from another process. The token is validated against the route's codec and refreshed once.
const resumed = yield* Video.resume(model, JSON.parse(saved))
// Progress as a stream.
yield* Video.stream(request) // Stream<VideoEvent>: generation-queued { position } | generation-progress { progress, logs } | video { index, video } | finish
yield* Video.stream(request, { poll }) // Stream<VideoEvent>: generation-queued { id, position } | generation-progress { id, progress } | video { index, video } | finish { usage, notices }
```
Webhooks: `Video.complete(model, token, webhook)` finishes a generation from a webhook payload without polling. Token shape is route-owned and opaque (Veo operation name, fal `response_url`, Runway task id).
Tokens are route-owned JSON: Veo `{ operation }`, xAI `{ requestID }`, Runway `{ taskID }`, fal
`{ requestID, statusURL, responseURL, cancelURL }` (fal's follow-up URLs are authoritative and absolute). Common-field
lowering per provider: Veo takes inline media only and rejects `audio: false` and `n > 1`; xAI rejects `seed` and
`negativePrompt` and routes a `video` input to edits or (`providerOptions.mode: "extend"`) extensions; fal rejects
`durationSeconds`, `references`, and `frames.last` because the field names and enums differ per model; Runway passes
`aspectRatio` through as its pixel `ratio` and rejects `n`.
Deferred: `Video.complete(model, token, webhook)` (finish from a webhook payload without polling) and provider poll
hints (none of the four providers emit one). Later providers: Luma, Kling, MiniMax, Replicate.
#### Speech (TTS)
Shipped in phase 3 (`src/speech.ts`, `src/speech-client.ts`, protocols `openai-speech`, `google-speech`,
`elevenlabs-speech`, `cartesia-speech`, `deepgram-speech`; new `ElevenLabs`, `Cartesia`, and `Deepgram` facades).
```ts
const request = Speech.request({
model: elevenlabs.speech("eleven_v3"),
model: elevenlabs.speech("eleven_flash_v2_5"),
text: "Hello from OpenCode.",
voice: "JBFqnCBsd6RMkjVDRZzb", // name, uuid, or { id } — provider-normalized
voice: "JBFqnCBsd6RMkjVDRZzb", // provider-native identifier, or { id }
format: "mp3", // mp3 | wav | pcm | opus | aac | flac | (string & {})
speed: 1.0,
language: "en",
instructions: "Warm, unhurried.",
providerOptions: { stability: 0.5 },
instructions: "Warm, unhurried.", // only OpenAI; elsewhere fails typed
timestamps: true, // request alignment; routes without it fail typed
providerOptions: { voice_settings: { stability: 0.5 } },
})
const response = yield* Speech.generate(request) // SpeechResponse: audio: Media.Asset, timestamps?, usage
yield* Speech.stream(request) // Stream<SpeechEvent>: audio-delta { chunk } | timestamps { words } | finish
const response = yield* Speech.generate(request) // SpeechResponse: audio: Media.Asset, timestamps?, usage?, providerMetadata?
yield* Speech.stream(request) // Stream<SpeechEvent>: audio-delta { chunk } | timestamps { items } | finish { audio, usage? }
```
Streaming TTS is first-class on day one: OpenAI `stream_format: sse`, ElevenLabs `/stream`, Cartesia SSE, Deepgram chunked. Input-streaming TTS (WS, text arrives incrementally) is a later `Speech.session(...)` scoped resource, not part of `generate`.
Execution is `MediaProtocol.stream` for every provider: one request whose body is framed and folded by a `step`
state machine, with `generate` running the same stream and collecting it. The route submits the request with its
`mode` (`"generate" | "stream"`), which lets one provider stay one protocol — OpenAI adds `stream_format: "sse"` (except `tts-1`/`tts-1-hd`, which stream raw bytes), ElevenLabs appends
`/stream`, Cartesia switches `/tts/bytes` to `/tts/sse`, Gemini switches `generateContent` to
`streamGenerateContent`. The terminal `finish` event carries the assembled asset (every provider's stream is
concatenable chunks), so stream consumers also get the whole file and `generate` is just "take `finish`, gather
`timestamps`". The cost is memory: a stream holds every chunk until `finish`, so even a consumer that only plays deltas
keeps the whole clip in memory. That is bounded by the providers' input text limits (a few minutes of audio); a
long-form or session API would need an opt-out.
**Voice.** `voice?: string | { id: string }`. A string is passed through as the provider's native identifier — a
name on OpenAI and Gemini, a voice id on ElevenLabs (path segment) and Cartesia. `{ id }` selects an OpenAI custom
voice and is treated as the plain string on routes that do not distinguish custom from built-in. Deepgram's voice is
the model id (`aura-2-thalia-en`), so `voice` is `unsupported` there. There is no cross-provider voice catalog or
name→id resolution. Multi-speaker (Gemini `speechConfig.multiSpeakerVoiceConfig`) and per-voice settings
(ElevenLabs `voice_settings`) go through `providerOptions`.
**Format and PCM.** `format` is container-level; provider sample rates and bitrates live under `providerOptions`
(ElevenLabs `outputFormat`, Cartesia `sampleRate`/`bitRate`/`encoding`, Deepgram `encoding`/`container`/`sampleRate`/
`bitRate`). Each protocol maps `format` to its native value (ElevenLabs `mp3_44100_128`/`pcm_24000`/`wav_24000`/
`opus_48000_64`, Cartesia `{ container, encoding, sample_rate }`, Deepgram `encoding`+`container`) and declares the
asset's media type rather than sniffing, because headerless PCM can look like an MPEG frame sync. Headerless PCM
always carries `info.encoding`, `info.sampleRate`, and `info.channels`; its media type is the provider's declaration
(Gemini `audio/L16;codec=pcm;rate=24000`, Deepgram's `content-type`) or `audio/pcm`. Gemini returns PCM only, so any
other `format` is rejected rather than wrapped as WAV by the route. Every `format` value a route cannot produce (unknown
to it, a container on Cartesia SSE, WAV on an ElevenLabs stream, anything but PCM on Gemini) fails the same way as an
unsupported field: `UnsupportedOperation` with `operation: "media.format"`.
**Timestamps.** `timestamps: true` on the request asks for alignment. ElevenLabs selects the `with-timestamps`
endpoints (character-level, NDJSON when streaming); Cartesia sets `add_timestamps` on `/tts/sse` (word-level; a
`generate` with timestamps collects the SSE stream). OpenAI, Gemini, and Deepgram reject it.
Common-field lowering per provider:
| Provider | `voice` | `speed` | `language` | `instructions` | `timestamps` | Usage |
|---|---|---|---|---|---|---|
| OpenAI | `voice` (name or `{ id }`) | `speed` | unsupported | `instructions` | unsupported | `tokens` from SSE `speech.audio.done` only |
| Gemini | `prebuiltVoiceConfig.voiceName` | unsupported | `speechConfig.languageCode` | unsupported (direct in text) | unsupported | `tokens` from `usageMetadata` |
| ElevenLabs | path voice id (required) | `voice_settings.speed` | `language_code` | unsupported | `with-timestamps` | `credits` from `character-cost` header |
| Cartesia | `voice` (required) | `generation_config.speed` | `language` | unsupported | `add_timestamps` | none |
| Deepgram | unsupported (voice is the model) | `speed` query | unsupported | unsupported | unsupported | `characters` from `dg-char-count` header |
Deferred: `Speech.session(...)` — input-streaming TTS where text arrives incrementally over a WebSocket (ElevenLabs
`stream-input`, Cartesia WebSocket contexts, Deepgram WebSocket speak) — is a separate scoped resource, not part of
`generate`/`stream`, and ships with the realtime work in phase 5.
#### Transcription (STT)
Shipped as the second half of phase 3 (`src/transcription.ts`, `src/transcription-client.ts`, protocols
`openai-transcription`, `google-transcription`, `deepgram-transcription`, `assemblyai-transcription`; new `AssemblyAI`
facade).
```ts
const request = Transcription.request({
model: openai.transcription("gpt-4o-transcribe"),
audio: Media.file("./call.wav"),
language: "en",
prompt: "Names: Shoubhit, OpenCode.",
timestamps: "word", // none | segment | word
model: openai.transcription("gpt-4o-transcribe-diarize"),
audio: yield* Media.file("./call.wav"),
language: "en", // provider-native passthrough
timestamps: "segment", // none | segment | word
diarize: true,
providerOptions: { chunkingStrategy: "auto" },
speakers: 2, // expected count, hint only (AssemblyAI)
providerOptions: { known_speaker_names: ["agent"] },
})
const response = yield* Transcription.generate(request)
response.text; response.segments; response.words; response.language; response.durationSeconds
yield* Transcription.stream(request) // Stream<TranscriptionEvent>: text-delta | segment | finish
response.text; response.segments; response.words; response.language; response.durationSeconds; response.usage
yield* Transcription.stream(request) // Stream<TranscriptionEvent>: generation-queued | generation-progress | text-delta | segment | finish
const generation = yield* Transcription.start(request) // queued routes only
yield* Transcription.resume(model, token)
```
Realtime STT over WebSocket is the same future `session` shape as input-streaming TTS.
Transcription is the first modality whose providers span all three protocol kinds, and it needed no fourth kind.
Every `MediaRoute` now carries its `kind`; `TranscriptionRoute` is the union of the inline, stream, and queued routes;
`TranscriptionModel.fromRoute` is overloaded per protocol kind (arity picks the overload: `<Options>`,
`<Options, Frame, State>`, `<Options, Token>`) and composes through `MediaRoute.inline` / `stream` / `queued`; and
`TranscriptionClient` dispatches on `route.kind`. `generate` on a queued route is `start` then `await`; `stream` on an
inline route is the response as a single `finish`, and on a queued route it is the status observations followed by
`finish`. `start` / `resume` on a non-queued route fail with `UnsupportedOperation` (`transcription.start`). The
`finish` event carries the whole transcript (text, segments, words, language, duration, usage), so the stream route's
`collect` is just "take `finish`".
The route layer gained a `binary` body with array-valued `query` (Deepgram) and `Queued.start.prepare` (AssemblyAI's
upload); `packages/ai/AGENTS.md` (Media Routes) describes both.
Settled rules:
- **Timestamps.** A granularity the selected route or model cannot produce fails as `UnsupportedOperation`
(`media.timestamps`), following Speech; a route that returns more than asked (Deepgram and AssemblyAI always return
words) is not stripped. Segments always carry start and end times: Gemini times each transcription part from its
word offsets, so segment timestamps and diarization also request word offsets there.
- **Diarization.** `diarize` means segments (and words, where the provider labels them) carry `speaker`. Labels are
provider-native strings — OpenAI `A` or a known speaker name, Deepgram `0`, Gemini `spk:0`, AssemblyAI `A` — with no
cross-provider speaker model. `speakers` is a hint; only AssemblyAI (`speakers_expected`) accepts it.
- **Language** is passed through (`language`, OpenAI `gpt-transcribe` `languages[]`, Gemini `languageCodes`,
AssemblyAI `language_code`). `response.language` is the provider's own value, lowercased but not normalized: an
ISO code on most routes, `english` from whisper-1, `en_us` from AssemblyAI. Deepgram and AssemblyAI assume English
unless asked to detect, so a missing `language` enables their detection.
- **Gemini** requires a transcribe model; other model ids fail with `UnsupportedOperation` before the call, because
general models ignore `audioTranscriptionConfig` and answer conversationally. Streamed chunks carry whole speaker
turns (one part per turn), which join with a space.
- **Streaming inline providers** emit only `finish`; deltas are never faked.
- **Units.** AssemblyAI milliseconds and Gemini protobuf durations (`"0.400s"`) are normalized to seconds at the
protocol boundary.
| Provider | Kind | Audio input | `timestamps` | `diarize` | Unsupported | Usage |
|---|---|---|---|---|---|---|
| OpenAI | stream (`stream: true` in `stream` mode) | multipart `file` (inline only) | `whisper-1` (`verbose_json`); diarize model: `segment` | `gpt-4o-transcribe-diarize` (`diarized_json`) | `speakers`; `prompt` on the diarize model; streaming on `whisper-1` | `tokens` or `seconds` |
| Gemini | stream (`generateContent` / `streamGenerateContent`) | `inlineData` or Gemini Files `fileData` | `audioTranscriptionConfig.wordTimestamp` | `audioTranscriptionConfig.diarization` | `prompt`, `speakers` | `tokens` |
| Deepgram | inline | raw body, or JSON `{ url }` | words always; `segment``utterances` | `diarize_model=latest` + `utterances` | `prompt`, `speakers` | `seconds` (`metadata.duration`) |
| AssemblyAI | queued (upload → submit → poll) | `/v2/upload` then `audio_url`, or a URL | words always; `segment``speaker_labels` | `speaker_labels` | — | `seconds` (`audio_duration`) |
Deferred: `Transcription.session(...)` — realtime STT over WebSocket (Deepgram live, AssemblyAI streaming, ElevenLabs
realtime, OpenAI realtime transcription) — is the same future scoped `session` shape as input-streaming TTS and ships
with the realtime work in phase 5. ElevenLabs Scribe is not implemented yet.
### `Generation` — shared async execution
```ts
class Generation<Response> {
readonly id: string
readonly model: MediaModel
readonly route: GenerationRoute<Response> // token-free: { status, result, cancel?: Effect; pollHint? } closed over the decoded token
readonly token: unknown // route-owned serializable JSON
readonly status: "queued" | "running" | "completed" | "failed" | "cancelled" | "expired"
readonly progress?: number // 0..1, normalized
readonly position?: number
readonly expiresAt?: number
refresh(): Effect<Generation<Response>, AIError>
await(options?: { poll?: Poll }): Effect<Response, AIError>
result(): Effect<Response, AIError>
await(options?: AwaitOptions): Effect<Response, AIError>
cancel(): Effect<void, AIError>
events(options?): Stream<GenerationEvent, AIError>
events(options?: AwaitOptions): Stream<GenerationEvent, AIError> // fails with Timeout past poll.timeout, checked per observation
}
AwaitOptions = { poll?: Poll }
Poll = { interval?: Duration; timeout?: Duration; schedule?: Schedule } // route may override from provider hints (`openai-poll-after-ms`)
```
@@ -273,11 +386,14 @@ Existing facades gain per-modality selectors; the modality routes each facade pr
| Facade | llm | image | video | speech | transcription | other |
|---|---|---|---|---|---|---|
| `OpenAI` | responses (default), chat | Images API | Sora (deprecated 2026-09-24) | ✓ | ✓ | |
| `Google` | Gemini | Gemini-native (default), `imagen` | Veo | Gemini TTS | Gemini transcribe | |
| `Google` | Gemini | Gemini-native (default), `imagen` | Veo | Gemini TTS | `gemini-3.5-transcribe` | |
| `XAI` | ✓ | ✓ | ✓ | | | |
| `ElevenLabs` | | | | ✓ | Scribe | soundEffect, music |
| `Cartesia` | | | | ✓ | | |
| `Deepgram` | | | | Aura | ✓ | |
| `Fal` | | ✓ | ✓ | | | |
| `Replicate`, `Runway`, `Luma`, `Kling`, `MiniMax`, `Deepgram`, `Cartesia`, `AssemblyAI`, `BlackForestLabs`, `Stability` | | per provider | | | | |
| `AssemblyAI` | | | | | ✓ (queued) | |
| `Replicate`, `Runway`, `Luma`, `Kling`, `MiniMax`, `BlackForestLabs`, `Stability` | | per provider | | | | |
New facades follow the existing one-file-per-provider rule. Package entrypoints are modality-specific, such as `@opencode/ai/providers/openai/images`, and return the concrete model.
@@ -288,10 +404,10 @@ New facades follow the existing one-file-per-provider rule. Package entrypoints
Media does not fit the LLM four-axis route (SSE frames → event state machine) except for streaming TTS/STT. Reuse `Endpoint`, `Auth`, `Framing`, `RequestExecutor`, and add media protocol kinds:
- `MediaProtocol.inline``body.from(request)` (JSON, multipart, or query), `response.decode(response)` (JSON, or binary body → `Media.Asset`).
- `MediaProtocol.queued``start`, `status`, `result`, `cancel`, optional `download`, `pollHint`, `token` schema.
- `MediaProtocol.stream`framing + `step` state machine emitting modality events, same discipline as LLM protocols.
- `MediaProtocol.queued``start` (body + decode to `{ token, snapshot }`), `status`, `result`, optional `cancel`, `pollHint`, and a `token` codec. `result` is always a separate GET (against the status document for Veo/xAI/Runway, fal's `response_url` otherwise) so `await` after `start` and after `resume` share one path. `PollContext.auth` hands the auth headers the route sent to the protocol for output URLs that need them (Veo downloads); they become transient `Media.Asset.headers`, never part of `source`. There is no separate `download` step: `Media.Asset.bytes()` downloads through the executor with those headers. `MediaRoute.inline(...)` / `MediaRoute.queued(...)` compose each kind with endpoint and auth; the queued route decodes the token once and hands `Generation` a token-free `{ status, result, cancel? }`.
- `MediaProtocol.stream``body.from(request)` over the request plus its `mode`, `frames` (a function that picks the framing for the call: `Framing.sse`, `lines`, `document`, or the raw bytes), fresh per-response `initial()` state, `step` emitting modality events, and `finish(state, context)` — with the observed response for header-only usage — emitting exactly one terminal event or failing as an incomplete stream. The route fills `reason.http` on stream errors. `MediaRoute.stream(...)` exposes `stream` and `generate` (the same stream folded by the modality's `collect`).
`Route.make` for media composes one protocol kind with endpoint/auth. The existing `ImageRoute { generate(request, execute) }` is the ad-hoc version of `inline` and gets folded in.
`MediaRoute.inline` / `MediaRoute.queued` / `MediaRoute.stream` compose one protocol kind with endpoint/auth and tag the route with its `kind`; `ImageModel`/`VideoModel`/`SpeechModel`/`TranscriptionModel` share the `MediaModel` base (`src/media-model.ts`).
### LLM integration
@@ -320,8 +436,8 @@ Foundation + Image ship together as the reference implementation, serially. Vide
## Phasing
1. **Foundation** — per-modality selectors, `Media`, `Generation`, `Poll`, `Usage` union, `MediaProtocol` kinds, `@opencode/ai/promise` with `llm` + `image`. Port the five existing image protocols onto it. Unify `MediaPart` and add the `media` LLM event (fixes Gemini image output being dropped).
2. **Video** — Veo, xAI, fal, Runway first. Then Luma, Kling, MiniMax, Replicate.
3. **Speech + Transcription**OpenAI, ElevenLabs, Gemini TTS, Deepgram, Cartesia, AssemblyAI. Streaming TTS from the start.
2. **Video** Veo, xAI, fal, Runway shipped (`MediaProtocol.queued`, `Video.start/generate/resume/stream`, promise `ai.video`). Deferred: `Video.complete` (webhooks), Luma, Kling, MiniMax, Replicate.
3. **Speech + Transcription**✅ Speech: OpenAI, Gemini TTS, ElevenLabs, Cartesia, Deepgram shipped (`MediaProtocol.stream`, `Speech.generate/stream`, promise `ai.speech`). ✅ Transcription: OpenAI, Gemini, Deepgram, AssemblyAI shipped across all three route kinds (`Transcription.generate/stream/start/resume`, promise `ai.transcription`). Pending: ElevenLabs Scribe. Deferred: `Speech.session` and `Transcription.session` (WebSocket streaming).
4. **Image queued routes and partials** — BFL, fal, Replicate, Stability; OpenAI `partial_images` streaming.
5. **Later** — ElevenLabs music/SFX, Lyria, `Speech.session` / `Transcription.session`, realtime.
+1 -1
View File
@@ -1,6 +1,6 @@
{
"$schema": "https://json.schemastore.org/package.json",
"version": "2.0.14",
"version": "2.0.15",
"name": "@opencode/ai",
"type": "module",
"license": "MIT",
+77
View File
@@ -104,6 +104,83 @@ const PROVIDERS: ReadonlyArray<Provider> = [
vars: [{ name: "XAI_API_KEY" }],
validate: (env) => validateBearer("https://api.x.ai/v1/models", Redacted.make(env.XAI_API_KEY)),
},
{
id: "fal",
label: "fal",
tier: "canary",
note: "fal queue video recorded tests",
vars: [{ name: "FAL_KEY" }],
// fal has no free authenticated list endpoint; a 404 for an unknown request id proves the key was accepted.
validate: (env) =>
Effect.gen(function* () {
const http = yield* HttpClient.HttpClient
const response = yield* http.execute(
HttpClientRequest.get(
"https://queue.fal.run/fal-ai/veo3.1/requests/00000000-0000-0000-0000-000000000000/status",
).pipe(HttpClientRequest.setHeaders({ authorization: `Key ${Redacted.value(Redacted.make(env.FAL_KEY))}` })),
)
if (response.status === 404) return undefined
return yield* responseError(response)
}),
},
{
id: "runway",
label: "Runway",
tier: "canary",
note: "Runway task video recorded tests",
vars: [{ name: "RUNWAYML_API_SECRET" }],
validate: (env) =>
validateBearer("https://api.dev.runwayml.com/v1/organization", Redacted.make(env.RUNWAYML_API_SECRET), {
"X-Runway-Version": "2024-11-06",
}),
},
{
id: "elevenlabs",
label: "ElevenLabs",
tier: "canary",
note: "ElevenLabs text-to-speech recorded tests",
vars: [{ name: "ELEVENLABS_API_KEY" }],
validate: (env) =>
HttpClientRequest.get("https://api.elevenlabs.io/v1/models").pipe(
HttpClientRequest.setHeader("xi-api-key", Redacted.value(Redacted.make(env.ELEVENLABS_API_KEY))),
executeRequest,
),
},
{
id: "cartesia",
label: "Cartesia",
tier: "canary",
note: "Cartesia text-to-speech recorded tests",
vars: [{ name: "CARTESIA_API_KEY" }],
validate: (env) =>
validateBearer("https://api.cartesia.ai/voices?limit=1", Redacted.make(env.CARTESIA_API_KEY), {
"Cartesia-Version": "2026-08-14",
}),
},
{
id: "deepgram",
label: "Deepgram",
tier: "canary",
note: "Deepgram Aura text-to-speech and Nova transcription recorded tests",
vars: [{ name: "DEEPGRAM_API_KEY" }],
validate: (env) =>
HttpClientRequest.get("https://api.deepgram.com/v1/projects").pipe(
HttpClientRequest.setHeader("authorization", `Token ${Redacted.value(Redacted.make(env.DEEPGRAM_API_KEY))}`),
executeRequest,
),
},
{
id: "assemblyai",
label: "AssemblyAI",
tier: "canary",
note: "AssemblyAI queued transcription recorded tests",
vars: [{ name: "ASSEMBLYAI_API_KEY" }],
validate: (env) =>
HttpClientRequest.get("https://api.assemblyai.com/v2/transcript?limit=1").pipe(
HttpClientRequest.setHeader("authorization", Redacted.value(Redacted.make(env.ASSEMBLYAI_API_KEY))),
executeRequest,
),
},
{
id: "cloudflare-ai-gateway",
label: "Cloudflare AI Gateway",
+92 -38
View File
@@ -1,4 +1,4 @@
import { Duration, Effect, Schedule, Schema, Stream } from "effect"
import { Clock, Duration, Effect, Schedule, Schema, Stream } from "effect"
import { AIError, TimeoutError } from "./schema/errors.js"
export const Status = Schema.Literals(["queued", "running", "completed", "failed", "cancelled", "expired"])
@@ -15,13 +15,13 @@ export interface Snapshot {
}
/**
* Route-owned generation operations. `token` is the route's serializable handle (operation name, task id, response URL)
* so a generation can be resumed from another process; its shape is opaque to `Generation`.
* Route-owned generation operations for one generation. The media route decodes its serializable token once (from the
* submission response or a `resume` input) and closes over it, so `Generation` never sees the token's shape.
*/
export interface Route<Response> {
readonly status: (token: unknown) => Effect.Effect<Snapshot, AIError>
readonly result: (token: unknown) => Effect.Effect<Response, AIError>
readonly cancel?: (token: unknown) => Effect.Effect<void, AIError>
readonly status: Effect.Effect<Snapshot, AIError>
readonly result: Effect.Effect<Response, AIError>
readonly cancel?: Effect.Effect<void, AIError>
/** Provider polling hint (e.g. `openai-poll-after-ms`) that overrides the default interval for the next poll. */
readonly pollHint?: (snapshot: Snapshot) => Duration.Duration | undefined
}
@@ -33,13 +33,28 @@ export interface Poll {
readonly schedule?: Schedule.Schedule<unknown, Snapshot>
}
export interface AwaitOptions {
readonly poll?: Poll
}
export const DEFAULT_POLL_INTERVAL = Duration.seconds(5)
export const DEFAULT_POLL_TIMEOUT = Duration.minutes(10)
export type Event =
| { readonly type: "generation-queued"; readonly id: string; readonly position?: number }
| { readonly type: "generation-progress"; readonly id: string; readonly progress?: number }
| { readonly type: "generation-finished"; readonly id: string; readonly status: Status }
export const QueuedEvent = Schema.Struct({
type: Schema.tag("generation-queued"),
id: Schema.String,
position: Schema.optional(Schema.Number),
}).annotate({ identifier: "Generation.Event.Queued" })
export const ProgressEvent = Schema.Struct({
type: Schema.tag("generation-progress"),
id: Schema.String,
progress: Schema.optional(Schema.Number),
}).annotate({ identifier: "Generation.Event.Progress" })
export type Observation = Schema.Schema.Type<typeof QueuedEvent> | Schema.Schema.Type<typeof ProgressEvent>
export type Event = Observation | { readonly type: "generation-finished"; readonly id: string; readonly status: Status }
const TERMINAL: ReadonlySet<Status> = new Set(["completed", "failed", "cancelled", "expired"])
@@ -52,6 +67,7 @@ export class Generation<Response> {
constructor(
readonly route: Route<Response>,
/** Route-owned serializable JSON; pass it to the modality's `resume` from another process. */
readonly token: unknown,
snapshot: Snapshot,
) {
@@ -77,51 +93,79 @@ export class Generation<Response> {
}
refresh(): Effect.Effect<Generation<Response>, AIError> {
return this.route.status(this.token).pipe(Effect.map((snapshot) => new Generation(this.route, this.token, snapshot)))
return this.route.status.pipe(Effect.map((snapshot) => new Generation(this.route, this.token, snapshot)))
}
/** Fetch the result without polling; non-completed terminal generations fail with the provider's terminal body. */
result(): Effect.Effect<Response, AIError> {
return this.route.result
}
/** Poll until the generation reaches a terminal status, then fetch the result. Fails with a `Timeout` reason on deadline. */
await(options?: { readonly poll?: Poll }): Effect.Effect<Response, AIError> {
await(options?: AwaitOptions): Effect.Effect<Response, AIError> {
const timeout = Duration.fromInputUnsafe(options?.poll?.timeout ?? DEFAULT_POLL_TIMEOUT)
const settled = this.terminal ? Effect.succeed(this) : this.poll(options?.poll)
return settled.pipe(
// Non-completed terminal states also go through `result` so the route can surface its provider failure body.
Effect.flatMap((generation) => generation.route.result(generation.token)),
Effect.timeoutOrElse({
duration: timeout,
orElse: () =>
new AIError({
reason: new TimeoutError({
message: `Generation ${this.id} did not finish within ${Duration.format(timeout)}`,
timeoutMs: Duration.toMillis(timeout),
}),
}),
}),
Effect.flatMap((generation) => generation.result()),
Effect.timeoutOrElse({ duration: timeout, orElse: () => this.timeoutError(timeout) }),
)
}
cancel(): Effect.Effect<void, AIError> {
return this.route.cancel?.(this.token) ?? Effect.void
return this.route.cancel ?? Effect.void
}
/** Status observations as a stream, ending after the first terminal observation. */
events(options?: { readonly poll?: Poll }): Stream.Stream<Event, AIError> {
const observations = this.terminal
? Stream.make(this)
: Stream.fromEffectSchedule(this.refresh(), this.schedule(options?.poll)).pipe(
Stream.takeUntil((generation) => generation.terminal),
)
return observations.pipe(
Stream.map((generation): Event => {
if (generation.terminal) return { type: "generation-finished", id: generation.id, status: generation.status }
if (generation.status === "queued") return { type: "generation-queued", id: generation.id, position: generation.position }
return { type: "generation-progress", id: generation.id, progress: generation.progress }
}),
/**
* Status observations as a stream, ending after the first terminal observation. Each poll is bounded by the time
* remaining until `poll.timeout`, so a hung status request fails the stream instead of stalling it. (`Stream.interruptWhen`
* would express this directly but deadlocks under `TestClock` when the source completes while the timer sleeps.)
*/
events(options?: AwaitOptions): Stream.Stream<Event, AIError> {
if (this.terminal) return Stream.make(this.event())
const timeout = Duration.fromInputUnsafe(options?.poll?.timeout ?? DEFAULT_POLL_TIMEOUT)
return Stream.unwrap(
Clock.currentTimeMillis.pipe(
Effect.map((start) => {
const deadline = start + Duration.toMillis(timeout)
const refresh = Clock.currentTimeMillis.pipe(
Effect.flatMap((now) =>
this.refresh().pipe(
Effect.timeoutOrElse({
duration: Duration.millis(Math.max(0, deadline - now)),
orElse: () => this.timeoutError(timeout),
}),
),
),
)
return Stream.fromEffectSchedule(refresh, this.schedule(options?.poll)).pipe(
Stream.takeUntil((generation) => generation.terminal),
Stream.map((generation) => generation.event()),
)
}),
),
)
}
private event(): Event {
if (this.terminal) return { type: "generation-finished", id: this.id, status: this.status }
if (this.status === "queued") return { type: "generation-queued", id: this.id, position: this.position }
return { type: "generation-progress", id: this.id, progress: this.progress }
}
private timeoutError(timeout: Duration.Duration) {
return new AIError({
reason: new TimeoutError({
message: `Generation ${this.id} did not finish within ${Duration.format(timeout)}`,
timeoutMs: Duration.toMillis(timeout),
}),
})
}
private poll(poll: Poll | undefined) {
return this.refresh().pipe(Effect.repeat({ schedule: this.schedule(poll), until: (generation) => generation.terminal }))
return this.refresh().pipe(
Effect.repeat({ schedule: this.schedule(poll), until: (generation) => generation.terminal }),
)
}
private schedule(poll: Poll | undefined): Schedule.Schedule<unknown, Generation<Response>> {
@@ -135,3 +179,13 @@ export class Generation<Response> {
)
}
}
export const resultEvents = <Response, A>(
generation: Generation<Response>,
expand: (response: Response) => ReadonlyArray<A>,
options?: AwaitOptions,
): Stream.Stream<Observation | A, AIError> =>
generation.events(options).pipe(
Stream.filter((event): event is Observation => event.type !== "generation-finished"),
Stream.concat(Stream.fromIterableEffect(Effect.map(generation.result(), expand))),
)
+1 -1
View File
@@ -48,7 +48,7 @@ export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer
generate,
// Inline routes have no partial frames yet; the stream is the completed response expanded into events.
stream: (request) =>
Stream.unwrap(generate(request).pipe(Effect.map((response) => Stream.fromIterable(responseEvents(response))))),
Stream.fromIterableEffect(Effect.map(generate(request), responseEvents)),
})
}),
)
+15 -76
View File
@@ -1,17 +1,9 @@
import { Effect, Schema, Stream } from "effect"
import { Media } from "./media.js"
import { Endpoint } from "./route/endpoint.js"
import { MediaModel, composeRoute, tryRequest } from "./media-model.js"
import { MediaRoute } from "./route/media.js"
import type { MediaProtocol } from "./route/media-protocol.js"
import {
AIError,
HttpOptions,
InvalidRequestError,
MediaUsage,
ModelID,
ProviderID,
ProviderMetadata,
} from "./schema/index.js"
import { AIError, HttpOptions, MediaUsage, ProviderMetadata } from "./schema/index.js"
import { ImageClient, Service } from "./image-client.js"
// ---------------------------------------------------------------------------
@@ -25,27 +17,11 @@ export type ImageRoute<Options extends ImageOptions = ImageOptions> = MediaRoute
ImageResponse
>
export class ImageModel<Options extends ImageOptions = ImageOptions> {
declare protected readonly _Options: (options: Options) => Options
readonly id: ModelID
readonly provider: ProviderID
readonly route: ImageRoute<Options>
readonly http?: HttpOptions
export class ImageModel<Options extends ImageOptions = ImageOptions> extends MediaModel<ImageRoute<Options>, Options> {
declare protected readonly _ImageModel: void
constructor(input: ImageModel.Input<Options>) {
this.id = input.id
this.provider = input.provider
this.route = input.route
this.http = input.http
}
static make<Options extends ImageOptions = ImageOptions>(input: ImageModel.MakeInput<Options>) {
return new ImageModel<Options>({
id: ModelID.make(input.id),
provider: ProviderID.make(input.provider),
route: input.route,
http: input.http,
})
static make<Options extends ImageOptions = ImageOptions>(input: MediaModel.Input<ImageRoute<Options>>) {
return new ImageModel<Options>(input)
}
/** Compose an inline image protocol with its canonical path into a model for one deployment. */
@@ -53,44 +29,20 @@ export class ImageModel<Options extends ImageOptions = ImageOptions> {
route: ImageModel.RouteInput<Options>,
input: MediaRoute.ModelInput,
) {
return ImageModel.make<Options>({
return new ImageModel<Options>({
id: input.id,
provider: route.provider,
http: input.http,
route: MediaRoute.make({
id: route.id,
provider: route.provider,
protocol: route.protocol,
endpoint: Endpoint.path(route.path, { baseURL: input.baseURL ?? route.baseURL }),
auth: input.auth,
headers: input.headers,
}),
route: composeRoute(MediaRoute.inline, route, input),
})
}
}
export namespace ImageModel {
export interface Input<Options extends ImageOptions = ImageOptions> {
readonly id: ModelID
readonly provider: ProviderID
readonly route: ImageRoute<Options>
readonly http?: HttpOptions
}
export interface MakeInput<Options extends ImageOptions = ImageOptions>
extends Omit<Input<Options>, "id" | "provider"> {
readonly id: string | ModelID
readonly provider: string | ProviderID
}
export interface RouteInput<Options extends ImageOptions = ImageOptions> {
readonly id: string
readonly provider: string | ProviderID
readonly protocol: MediaProtocol.Inline<ImageRequestFor<Options>, ImageResponse>
readonly path: Endpoint.EndpointPart<MediaProtocol.Body, ImageRequestFor<Options>>
/** Canonical base URL; `ModelInput.baseURL` overrides it per deployment. */
readonly baseURL?: string
}
export type RouteInput<Options extends ImageOptions = ImageOptions> = MediaModel.RouteInput<
ImageRequestFor<Options>,
MediaProtocol.Inline<ImageRequestFor<Options>, ImageResponse>
>
}
export const ImageModelSchema = Schema.declare((value): value is ImageModel => value instanceof ImageModel, {
@@ -107,11 +59,8 @@ export const ImageSize = Schema.declare<ImageSize>(
{ title: "ImageSize" },
)
export type ImageAspectRatio = `${number}:${number}`
export const ImageAspectRatio = Schema.declare<ImageAspectRatio>(
(value): value is ImageAspectRatio => typeof value === "string" && /^\d+(?:\.\d+)?:\d+(?:\.\d+)?$/.test(value),
{ title: "ImageAspectRatio" },
)
export type ImageAspectRatio = Media.AspectRatio
export const ImageAspectRatio = Media.AspectRatio
export type ImageFormat = "png" | "jpeg" | "webp" | (string & {})
@@ -216,17 +165,7 @@ export function request(input: ImageRequest | ImageRequestInput) {
})
}
const requestEffect = (input: ImageRequest | ImageRequestInput) =>
Effect.try({
try: () => request(input),
catch: (error) =>
new AIError({
reason: new InvalidRequestError({
message: error instanceof Error ? error.message : String(error),
cause: error,
}),
}),
})
const requestEffect = (input: ImageRequest | ImageRequestInput) => tryRequest(() => request(input))
export function generate<const Model extends ImageModel>(
input: ImageRequestInput<Model>,
+65 -1
View File
@@ -29,9 +29,73 @@ export type {
ImageRoute,
} from "./image.js"
export { Image } from "./image.js"
export { VideoClient } from "./video-client.js"
export {
VideoAspectRatio,
VideoEvent,
VideoFrames,
VideoModel,
VideoModelSchema,
VideoRequest,
VideoResponse,
} from "./video.js"
export type {
VideoModelOptions,
VideoOptions,
VideoRequestFor,
VideoRequestInput,
VideoResolution,
VideoRoute,
} from "./video.js"
export { Video } from "./video.js"
export { SpeechClient } from "./speech-client.js"
export {
SpeechEvent,
SpeechModel,
SpeechModelSchema,
SpeechRequest,
SpeechResponse,
SpeechTimestamp,
SpeechVoice,
} from "./speech.js"
export type {
SpeechFormat,
SpeechModelOptions,
SpeechOptions,
SpeechRequestFor,
SpeechRequestInput,
SpeechRoute,
} from "./speech.js"
export { Speech } from "./speech.js"
export { TranscriptionClient } from "./transcription-client.js"
export {
TranscriptionEvent,
TranscriptionModel,
TranscriptionModelSchema,
TranscriptionRequest,
TranscriptionResponse,
TranscriptionSegment,
TranscriptionTimestamps,
TranscriptionWord,
} from "./transcription.js"
export type {
TranscriptionModelOptions,
TranscriptionOptions,
TranscriptionRequestFor,
TranscriptionRequestInput,
TranscriptionRoute,
} from "./transcription.js"
export { Transcription } from "./transcription.js"
export { Media } from "./media.js"
export { Generation } from "./generation.js"
export type { Event as GenerationEvent, Poll, Route as GenerationRoute, Snapshot as GenerationSnapshot, Status as GenerationStatus } from "./generation.js"
export type {
AwaitOptions as GenerationAwaitOptions,
Event as GenerationEvent,
Poll,
Route as GenerationRoute,
Snapshot as GenerationSnapshot,
Status as GenerationStatus,
} from "./generation.js"
export { Tool, ToolFailure, toDefinitions } from "./tool.js"
export { ToolRuntime } from "./tool-runtime.js"
export type { DispatchResult as ToolDispatchResult, ToolSettlement } from "./tool-runtime.js"
+74
View File
@@ -0,0 +1,74 @@
import { Effect } from "effect"
import { Endpoint } from "./route/endpoint.js"
import type { MediaRoute } from "./route/media.js"
import type { MediaProtocol } from "./route/media-protocol.js"
import { AIError, HttpOptions, InvalidRequestError, ModelID, ProviderID } from "./schema/index.js"
/**
* What every media model carries: ids, the configured route, and deployment `http` overlays. Modality classes
* (`ImageModel`, `VideoModel`, `SpeechModel`) extend it with their route type and a nominal marker so one cannot stand
* in for the other in requests.
*/
export class MediaModel<Route, Options> {
declare protected readonly _Options: (options: Options) => Options
readonly id: ModelID
readonly provider: ProviderID
readonly route: Route
readonly http?: HttpOptions
constructor(input: MediaModel.Input<Route>) {
this.id = ModelID.make(input.id)
this.provider = ProviderID.make(input.provider)
this.route = input.route
this.http = input.http
}
}
export namespace MediaModel {
export interface Input<Route> {
readonly id: string | ModelID
readonly provider: string | ProviderID
readonly route: Route
readonly http?: HttpOptions
}
/** A protocol plus its canonical start path; `ModelInput.baseURL` overrides `baseURL` per deployment. */
export interface RouteInput<Request extends MediaRoute.MediaRequest, Protocol> {
readonly id: string
readonly provider: string | ProviderID
readonly protocol: Protocol
readonly path: Endpoint.EndpointPart<MediaProtocol.Body, Request>
readonly baseURL?: string
/** Headers the protocol requires on every call, such as a pinned API version; deployment headers win. */
readonly headers?: Record<string, string>
}
}
/** Compose a protocol route input with one deployment through `MediaRoute.inline`, `queued`, or `stream`. */
export const composeRoute = <Request extends MediaRoute.MediaRequest, Protocol, Route>(
compose: (input: MediaRoute.Composition<Request> & { readonly protocol: Protocol }) => Route,
route: MediaModel.RouteInput<Request, Protocol>,
input: MediaRoute.ModelInput,
): Route =>
compose({
id: route.id,
provider: route.provider,
protocol: route.protocol,
endpoint: Endpoint.path(route.path, { baseURL: input.baseURL ?? route.baseURL }),
auth: input.auth,
headers:
route.headers === undefined && input.headers === undefined ? undefined : { ...route.headers, ...input.headers },
})
/** Lift a synchronous Schema-class constructor into a typed `InvalidRequest` failure. */
export const tryRequest = <A>(make: () => A): Effect.Effect<A, AIError> =>
Effect.try({
try: make,
catch: (error) =>
new AIError({
reason: new InvalidRequestError({
message: error instanceof Error ? error.message : String(error),
cause: error,
}),
}),
})
+23 -7
View File
@@ -34,8 +34,6 @@ const UrlSource = Schema.Struct({
mediaType: Schema.optional(Schema.String),
/** Epoch milliseconds after which the provider no longer serves the URL. */
expiresAt: Schema.optional(Schema.Number),
/** Headers required to fetch the URL, such as provider auth for Veo downloads. */
headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
})
/** A provider-side handle: OpenAI `file_id`, Gemini file URI, `gs://`, `runway://`, or a prior generation id. */
@@ -55,6 +53,12 @@ export type Source = Schema.Schema.Type<typeof Source>
// Kind, Info, Notice
// ---------------------------------------------------------------------------
export type AspectRatio = `${number}:${number}`
export const AspectRatio = Schema.declare<AspectRatio>(
(value): value is AspectRatio => typeof value === "string" && /^\d+(?:\.\d+)?:\d+(?:\.\d+)?$/.test(value),
{ title: "Media.AspectRatio" },
)
export const Kind = Schema.Literals(["image", "video", "audio", "document", "other"])
export type Kind = Schema.Schema.Type<typeof Kind>
@@ -110,6 +114,8 @@ export class Asset {
/** Epoch milliseconds after which a `url` source stops resolving. */
readonly expiresAt?: number
readonly providerMetadata?: ProviderMetadata
/** Transient download credentials for `url` sources; see `Asset.Input.headers`. */
readonly headers?: Record<string, string>
// Derived payload forms are cached on the instance because every protocol lowering re-reads the same payload. The
// cache is check-then-set (concurrent first reads of a `url` source may both download) and is never observable
@@ -127,6 +133,7 @@ export class Asset {
this.info = input.info
this.expiresAt = input.source.type === "url" ? input.source.expiresAt : undefined
this.providerMetadata = input.providerMetadata
this.headers = input.source.type === "url" ? input.headers : undefined
}
/** Inline payload without effects, for protocols that embed base64 or data URLs directly. */
@@ -151,7 +158,7 @@ export class Asset {
? Effect.fromResult(Encoding.decodeBase64(source.data)).pipe(
Effect.mapError((cause) => invalid(`Media asset contains invalid base64 data`, cause)),
)
: download(source)
: download(source, this.headers)
return decoded.pipe(Effect.tap((data) => Effect.sync(() => (this.#bytes = data))))
})
}
@@ -198,6 +205,12 @@ export namespace Asset {
readonly source: Source
readonly info?: Info
readonly providerMetadata?: ProviderMetadata
/**
* Headers required to download a `url` source, such as the provider API key Veo demands for its file URIs.
* They are runtime-only: never part of `source`, `toJSON()`, or `AssetSchema`, so a persisted asset cannot leak
* credentials and cannot be downloaded again after a round-trip. Call `materialize()` before persisting.
*/
readonly headers?: Record<string, string>
}
}
@@ -226,10 +239,13 @@ export const AssetSchema = AssetEncoded.pipe(
}),
)
const download = Effect.fn("Media.download")(function* (source: Extract<Source, { readonly type: "url" }>) {
const download = Effect.fn("Media.download")(function* (
source: Extract<Source, { readonly type: "url" }>,
headers: Record<string, string> | undefined,
) {
const executor = yield* Service
const response = yield* executor.execute(
HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(source.headers ?? {})),
HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(headers ?? {})),
)
const buffer = yield* response.arrayBuffer.pipe(
Effect.mapError(
@@ -264,8 +280,8 @@ export const url = (
value: string,
options?: AssetOptions & Omit<Extract<Source, { readonly type: "url" }>, "type" | "url">,
) => {
const { mediaType, expiresAt, headers, ...rest } = options ?? {}
return from({ type: "url", url: value, mediaType, expiresAt, headers }, rest)
const { mediaType, expiresAt, ...rest } = options ?? {}
return from({ type: "url", url: value, mediaType, expiresAt }, rest)
}
export const ref = (provider: string | ProviderID, id: string, mediaType?: string, options?: AssetOptions) =>
+98 -12
View File
@@ -1,4 +1,5 @@
import { Effect, Layer, ManagedRuntime, Stream } from "effect"
import type { AwaitOptions, Generation, Snapshot } from "./generation.js"
import { Image, ImageModel, ImageRequest, type ImageRequestInput } from "./image.js"
import { ImageClient } from "./image-client.js"
import { LLM } from "./index.js"
@@ -6,10 +7,23 @@ import { LLMClient } from "./route/client.js"
import { RequestExecutor } from "./route/executor.js"
import { LanguageModel, LLMRequest } from "./schema/index.js"
import type { RequestInput } from "./llm.js"
import { Speech, SpeechModel, SpeechRequest, type SpeechRequestInput } from "./speech.js"
import { SpeechClient } from "./speech-client.js"
import {
Transcription,
TranscriptionModel,
TranscriptionRequest,
type TranscriptionOptions,
type TranscriptionRequestInput,
} from "./transcription.js"
import { TranscriptionClient } from "./transcription-client.js"
import { Video, VideoModel, VideoRequest, type VideoOptions, type VideoRequestInput } from "./video.js"
import { VideoClient } from "./video-client.js"
/**
* Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM and image clients
* over a request executor; every method runs the corresponding Effect API and rethrows `AIError` unchanged.
* Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM, image, video, speech,
* and transcription clients over a request executor; every method runs the corresponding Effect API and rethrows
* `AIError` unchanged.
*/
export interface Options {
/** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or middleware here. */
@@ -23,8 +37,20 @@ export interface RunOptions {
export type Services =
| Layer.Success<typeof LLMClient.layer>
| Layer.Success<typeof ImageClient.layer>
| Layer.Success<typeof VideoClient.layer>
| Layer.Success<typeof SpeechClient.layer>
| Layer.Success<typeof TranscriptionClient.layer>
| RequestExecutor.Service
/** Promise view of a `Generation`: its snapshot plus `await`, `refresh`, and `cancel` returning promises. */
export type GenerationHandle<Response> = Snapshot & {
/** Serializable JSON; pass it back to `resume` from another process. */
readonly token: unknown
readonly await: (options?: AwaitOptions & RunOptions) => Promise<Response>
readonly refresh: (options?: RunOptions) => Promise<GenerationHandle<Response>>
readonly cancel: (options?: RunOptions) => Promise<void>
}
const abortEffect = (signal: AbortSignal | undefined) =>
signal === undefined
? Effect.never
@@ -40,9 +66,13 @@ const abortEffect = (signal: AbortSignal | undefined) =>
export const make = (options: Options = {}) => {
const runtime = ManagedRuntime.make(
Layer.mergeAll(LLMClient.layer, ImageClient.layer).pipe(
Layer.provideMerge(options.layer ?? RequestExecutor.fetchLayer),
),
Layer.mergeAll(
LLMClient.layer,
ImageClient.layer,
VideoClient.layer,
SpeechClient.layer,
TranscriptionClient.layer,
).pipe(Layer.provideMerge(options.layer ?? RequestExecutor.fetchLayer)),
)
/** Run any package Effect (for example `asset.bytes()`) inside this runtime. */
@@ -61,19 +91,31 @@ export const make = (options: Options = {}) => {
),
)
const handle = <Response>(generation: Generation<Response>): GenerationHandle<Response> => ({
...generation.snapshot,
token: generation.token,
await: (options) => run(generation.await({ poll: options?.poll }), options),
refresh: (options) => run(generation.refresh(), options).then(handle),
cancel: (options) => run(generation.cancel(), options),
})
// The typed `generate`/`stream` overloads take a concrete input or a request, not the union; normalize once here.
const llmRequest = (input: RequestInput | LLMRequest) => (input instanceof LLMRequest ? input : LLM.request(input))
const imageRequest = (input: ImageRequestInput | ImageRequest) =>
input instanceof ImageRequest ? input : Image.request(input)
const videoRequest = (input: VideoRequestInput | VideoRequest) =>
input instanceof VideoRequest ? input : Video.request(input)
const speechRequest = (input: SpeechRequestInput | SpeechRequest) =>
input instanceof SpeechRequest ? input : Speech.request(input)
const transcriptionRequest = (input: TranscriptionRequestInput | TranscriptionRequest) =>
input instanceof TranscriptionRequest ? input : Transcription.request(input)
return {
run,
llm: {
request: LLM.request,
generate: <const Model extends LanguageModel>(
input: RequestInput<Model> | LLMRequest,
options?: RunOptions,
) => run(LLM.generate(llmRequest(input)), options),
generate: <const Model extends LanguageModel>(input: RequestInput<Model> | LLMRequest, options?: RunOptions) =>
run(LLM.generate(llmRequest(input)), options),
stream: <const Model extends LanguageModel>(input: RequestInput<Model> | LLMRequest, options?: RunOptions) =>
iterate(LLM.stream(llmRequest(input)), options),
},
@@ -83,10 +125,54 @@ export const make = (options: Options = {}) => {
input: ImageRequestInput<Model> | ImageRequest,
options?: RunOptions,
) => run(Image.generate(imageRequest(input)), options),
stream: <const Model extends ImageModel>(
input: ImageRequestInput<Model> | ImageRequest,
stream: <const Model extends ImageModel>(input: ImageRequestInput<Model> | ImageRequest, options?: RunOptions) =>
iterate(Image.stream(imageRequest(input)), options),
},
video: {
request: Video.request,
start: <const Model extends VideoModel>(input: VideoRequestInput<Model> | VideoRequest, options?: RunOptions) =>
run(Video.start(videoRequest(input)), options).then(handle),
generate: <const Model extends VideoModel>(
input: VideoRequestInput<Model> | VideoRequest,
options?: AwaitOptions & RunOptions,
) => run(Video.generate(videoRequest(input), { poll: options?.poll }), options),
resume: <Options extends VideoOptions>(model: VideoModel<Options>, token: unknown, options?: RunOptions) =>
run(Video.resume(model, token), options).then(handle),
stream: <const Model extends VideoModel>(
input: VideoRequestInput<Model> | VideoRequest,
options?: AwaitOptions & RunOptions,
) => iterate(Video.stream(videoRequest(input), { poll: options?.poll }), options),
},
speech: {
request: Speech.request,
generate: <const Model extends SpeechModel>(
input: SpeechRequestInput<Model> | SpeechRequest,
options?: RunOptions,
) => iterate(Image.stream(imageRequest(input)), options),
) => run(Speech.generate(speechRequest(input)), options),
stream: <const Model extends SpeechModel>(
input: SpeechRequestInput<Model> | SpeechRequest,
options?: RunOptions,
) => iterate(Speech.stream(speechRequest(input)), options),
},
transcription: {
request: Transcription.request,
generate: <const Model extends TranscriptionModel>(
input: TranscriptionRequestInput<Model> | TranscriptionRequest,
options?: AwaitOptions & RunOptions,
) => run(Transcription.generate(transcriptionRequest(input), { poll: options?.poll }), options),
stream: <const Model extends TranscriptionModel>(
input: TranscriptionRequestInput<Model> | TranscriptionRequest,
options?: AwaitOptions & RunOptions,
) => iterate(Transcription.stream(transcriptionRequest(input), { poll: options?.poll }), options),
start: <const Model extends TranscriptionModel>(
input: TranscriptionRequestInput<Model> | TranscriptionRequest,
options?: RunOptions,
) => run(Transcription.start(transcriptionRequest(input)), options).then(handle),
resume: <Options extends TranscriptionOptions>(
model: TranscriptionModel<Options>,
token: unknown,
options?: RunOptions,
) => run(Transcription.resume(model, token), options).then(handle),
},
dispose: () => runtime.dispose(),
}
@@ -0,0 +1,211 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const ADAPTER = "assemblyai-transcription"
const NAME = "AssemblyAI"
const PROVIDER = ProviderID.make("assemblyai")
export const DEFAULT_BASE_URL = "https://api.assemblyai.com"
export const PATH = "/v2/transcript"
export const UPLOAD_PATH = "/v2/upload"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type AssemblyAITranscriptionOptions = {
readonly keyterms_prompt?: ReadonlyArray<string>
readonly punctuate?: boolean
readonly format_text?: boolean
readonly disfluencies?: boolean
readonly filter_profanity?: boolean
readonly temperature?: number
readonly speaker_options?: { readonly min_speakers_expected?: number; readonly max_speakers_expected?: number }
readonly language_detection_options?: {
readonly expected_languages?: ReadonlyArray<string>
readonly fallback_language?: string
readonly code_switching?: boolean
}
readonly speech_models?: ReadonlyArray<"universal-3-5-pro" | "universal-2" | (string & {})>
} & Record<string, unknown>
export type Request = TranscriptionRequestFor<AssemblyAITranscriptionOptions>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
export const Token = Schema.Struct({ transcriptID: Schema.String })
export type Token = Schema.Schema.Type<typeof Token>
const Upload = Schema.Struct({ upload_url: Schema.String })
/** Word and utterance times are milliseconds. */
const Transcript = Schema.Struct({
id: Schema.String,
status: Schema.String,
text: optionalNull(Schema.String),
words: optionalNull(
Schema.Array(
Schema.Struct({
text: Schema.String,
start: Schema.Number,
end: Schema.Number,
confidence: optionalNull(Schema.Number),
speaker: optionalNull(Schema.String),
}),
),
),
utterances: optionalNull(
Schema.Array(
Schema.Struct({
text: Schema.String,
start: Schema.Number,
end: Schema.Number,
speaker: optionalNull(Schema.String),
}),
),
),
language_code: optionalNull(Schema.String),
audio_duration: optionalNull(Schema.Number),
speech_model_used: optionalNull(Schema.String),
error: optionalNull(Schema.String),
})
const STATUS = {
queued: "queued",
processing: "running",
completed: "completed",
error: "failed",
} as const satisfies Record<string, Status>
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const decodeUpload = MediaProtocol.decodeJson(ADAPTER, NAME, Upload)
/** `/v2/transcript` only takes a URL, so inline audio is uploaded to `/v2/upload` first. */
const prepare = Effect.fn("AssemblyAITranscription.prepare")(function* (request: Request, send: MediaProtocol.Send) {
if (request.audio.source.type !== "bytes" && request.audio.source.type !== "base64") return request
const audio = yield* MediaInput.inlineBytes(ADAPTER, request.audio)
const uploaded = yield* send(UPLOAD_PATH, MediaProtocol.binary(audio, "application/octet-stream")).pipe(
Effect.flatMap(decodeUpload),
)
return { ...request, audio: Media.url(uploaded.value.upload_url, { mediaType: request.audio.mediaType }) }
})
const fromRequest = Effect.fn("AssemblyAITranscription.fromRequest")(function* (request: Request) {
const audio = yield* ProviderShared.mediaReference(request.audio, PROVIDER, NAME)
return MediaProtocol.json(
mergeJsonRecords(
{
audio_url: audio.value,
speech_models: [request.model.id],
language_code: request.language,
language_detection: request.language === undefined ? true : undefined,
prompt: request.prompt,
// Turn-level `utterances`, the only segments AssemblyAI returns, require speaker labels.
speaker_labels: request.diarize === true || request.timestamps === "segment" ? true : undefined,
speakers_expected: request.speakers,
},
request.providerOptions,
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeTranscript = MediaProtocol.decodeJson(ADAPTER, NAME, Transcript)
const decodeStart = Effect.fn("AssemblyAITranscription.decodeStart")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* decodeTranscript(response)
const status = yield* MediaProtocol.status(STATUS, output.value.status, output)
return { token: { transcriptID: output.value.id }, snapshot: { id: output.value.id, status } }
})
const decodeStatus = Effect.fn("AssemblyAITranscription.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeTranscript(response)
const status = yield* MediaProtocol.status(STATUS, output.value.status, output)
return { id: context.token.transcriptID, status }
})
const seconds = (milliseconds: number) => milliseconds / 1000
const decodeResult = Effect.fn("AssemblyAITranscription.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeTranscript(response)
const transcript = output.value
const status = yield* MediaProtocol.status(STATUS, transcript.status, output)
const error = transcript.error ?? undefined
if (status === "failed")
return yield* output.ended("failed", `${NAME} transcription failed${error === undefined ? "" : `: ${error}`}`)
if (status !== "completed")
return yield* output.invalid(`${NAME} transcript ${context.token.transcriptID} has not finished`)
const duration = transcript.audio_duration ?? undefined
return new TranscriptionResponse({
text: transcript.text ?? "",
segments: transcript.utterances?.map((utterance) => ({
text: utterance.text,
startSeconds: seconds(utterance.start),
endSeconds: seconds(utterance.end),
speaker: utterance.speaker ?? undefined,
})),
words: transcript.words?.map((word) => ({
text: word.text,
startSeconds: seconds(word.start),
endSeconds: seconds(word.end),
speaker: word.speaker ?? undefined,
confidence: word.confidence ?? undefined,
})),
language: transcript.language_code?.toLowerCase(),
durationSeconds: duration,
usage: duration === undefined ? undefined : { type: "seconds", seconds: duration },
providerMetadata: {
assemblyai: { transcriptId: transcript.id, speechModel: transcript.speech_model_used ?? undefined },
},
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
const transcriptPath = (token: Token) => `${PATH}/${token.transcriptID}`
export const protocol = MediaProtocol.queued<Request, TranscriptionResponse, Token>({
id: ADAPTER,
name: NAME,
token: Token,
start: { prepare, body: { from: fromRequest }, decode: decodeStart },
status: { path: transcriptPath, decode: decodeStatus },
result: { path: transcriptPath, decode: decodeResult },
})
export const model = (input: MediaRoute.ModelInput) =>
TranscriptionModel.fromRoute<AssemblyAITranscriptionOptions, Token>(
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
export const AssemblyAITranscription = {
protocol,
model,
} as const
@@ -28,6 +28,7 @@ import { Lifecycle } from "./utils/lifecycle.js"
import { MistralToolID } from "./utils/mistral-tool-id.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
import { ToolStream } from "./utils/tool-stream.js"
import { concatBytes } from "../utils/bytes.js"
const ADAPTER = "bedrock-converse"
@@ -524,14 +525,7 @@ interface ParserState {
readonly reasoningRedactedContent: Readonly<Record<number, ReadonlyArray<Uint8Array>>>
}
const encodeRedactedContent = (chunks: ReadonlyArray<Uint8Array>) => {
const bytes = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.length, 0))
chunks.reduce((offset, chunk) => {
bytes.set(chunk, offset)
return offset + chunk.length
}, 0)
return Encoding.encodeBase64(bytes)
}
const encodeRedactedContent = (chunks: ReadonlyArray<Uint8Array>) => Encoding.encodeBase64(concatBytes(chunks))
const step = (state: ParserState, event: BedrockEvent) =>
Effect.gen(function* () {
@@ -0,0 +1,195 @@
import { Effect, Schema } from "effect"
import { classifyProviderFailure } from "../provider-error.js"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { AIError, ProviderID, mergeJsonRecords } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { SpeechStream } from "./utils/speech-stream.js"
const ADAPTER = "cartesia-speech"
const NAME = "Cartesia"
const PROVIDER = ProviderID.make("cartesia")
export const DEFAULT_BASE_URL = "https://api.cartesia.ai"
export const API_VERSION = "2026-08-14"
export const BYTES_PATH = "/tts/bytes"
export const SSE_PATH = "/tts/sse"
const DEFAULT_SAMPLE_RATE = 44100
const DEFAULT_BIT_RATE = 128000
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type CartesiaSpeechString<Known extends string> = Known | (string & {})
export type CartesiaEncoding = SpeechStream.PcmEncoding
export type CartesiaSpeechOptions = {
readonly sampleRate?: 8000 | 16000 | 22050 | 24000 | 44100 | 48000
readonly bitRate?: 32000 | 64000 | 96000 | 128000 | 192000
readonly encoding?: CartesiaEncoding
readonly generation_config?: {
readonly volume?: number
readonly emotion?: CartesiaSpeechString<"neutral" | "calm" | "angry" | "content" | "sad" | "scared">
}
readonly pronunciation_dict_id?: string
} & Record<string, unknown>
export type Request = SpeechRequestFor<CartesiaSpeechOptions>
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
/** `phoneme_timestamps` and future record types are ignored. */
const SseEvent = Schema.Struct({
type: Schema.String,
data: Schema.optional(Schema.Uint8ArrayFromBase64),
word_timestamps: Schema.optional(
Schema.Struct({
words: Schema.Array(Schema.String),
start: Schema.Array(Schema.Number),
end: Schema.Array(Schema.Number),
}),
),
status_code: Schema.optional(Schema.Number),
title: Schema.optional(Schema.String),
message: Schema.optional(Schema.String),
error_code: optionalNull(Schema.String),
})
const decodeEvent = MediaProtocol.decodeFrame(ADAPTER, NAME, SseEvent)
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
interface State extends SpeechStream.Audio {
readonly done: boolean
}
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
/** Timestamps exist only on the SSE endpoint, so a `generate` that asks for them collects an SSE stream. */
const usesSse = (request: MediaProtocol.Addressed<Request>) => request.mode === "stream" || request.timestamps === true
const CONTAINERS: Readonly<Record<string, "raw" | "wav" | "mp3">> = { pcm: "raw", wav: "wav", mp3: "mp3" }
const outputFormat = Effect.fn("CartesiaSpeech.outputFormat")(function* (request: MediaProtocol.Addressed<Request>) {
const sse = usesSse(request)
const format = request.format ?? (sse ? "pcm" : "mp3")
const container = CONTAINERS[format]
if (container === undefined)
return yield* SpeechStream.unsupportedFormat(
PROVIDER,
ADAPTER,
`${NAME} supports the pcm, wav, and mp3 formats, not "${format}"`,
)
if (sse && container !== "raw")
return yield* SpeechStream.unsupportedFormat(
PROVIDER,
ADAPTER,
`${NAME} streams and timestamps only raw PCM; request format "pcm" instead of "${format}"`,
)
const sampleRate = request.providerOptions?.sampleRate ?? DEFAULT_SAMPLE_RATE
if (container === "mp3")
return { container, sample_rate: sampleRate, bit_rate: request.providerOptions?.bitRate ?? DEFAULT_BIT_RATE }
return { container, encoding: request.providerOptions?.encoding ?? "pcm_s16le", sample_rate: sampleRate }
})
const fromRequest = Effect.fn("CartesiaSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
const voice = SpeechStream.voiceID(request.voice)
if (voice === undefined)
return yield* ProviderShared.invalidRequest(`${NAME} requires a voice id; pass it as \`voice\``)
const { sampleRate: _sampleRate, bitRate: _bitRate, encoding: _encoding, ...native } = request.providerOptions ?? {}
return MediaProtocol.json(
mergeJsonRecords(
{
model_id: request.model.id,
transcript: request.text,
voice,
output_format: yield* outputFormat(request),
language: request.language,
generation_config: request.speed === undefined ? undefined : { speed: request.speed },
add_timestamps: request.timestamps === true ? true : undefined,
},
native,
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const onEvent = Effect.fn("CartesiaSpeech.onEvent")(function* (state: State, frame: string) {
const event = yield* decodeEvent(frame)
if (event.type === "chunk" && event.data !== undefined) return SpeechStream.delta(state, event.data)
if (event.type === "timestamps" && event.word_timestamps !== undefined) {
const words = event.word_timestamps
return [state, SpeechStream.timestamps(words.words, words.start, words.end)] as const
}
if (event.type === "done") return [{ ...state, done: true }, []] as const
if (event.type === "error")
return yield* new AIError({
reason: classifyProviderFailure({
message: `${NAME} stream failed${event.title === undefined ? "" : ` (${event.title})`}: ${event.message ?? "unknown error"}`,
status: event.status_code,
rawBody: frame,
}),
})
return [state, []] as const
})
const finish = Effect.fn("CartesiaSpeech.finish")(function* (
state: State,
context: MediaProtocol.ResponseContext<Request>,
) {
if (usesSse(context.request) && !state.done) return yield* MediaProtocol.incomplete(ADAPTER)
const format = yield* outputFormat(context.request)
return yield* SpeechStream.finish(
ADAPTER,
state,
format.container === "raw"
? SpeechStream.pcm(format.encoding, format.sample_rate)
: SpeechStream.container(format.container, format.sample_rate),
)
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>({
id: ADAPTER,
name: NAME,
unsupported: ["instructions"],
body: { from: fromRequest },
frames: (bytes, context) => (usesSse(context.request) ? Framing.sse.frame(bytes) : bytes),
initial: () => ({ chunks: [], done: false }),
step: SpeechStream.step(onEvent),
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<CartesiaSpeechOptions, string | Uint8Array, State>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
headers: { "Cartesia-Version": API_VERSION },
path: ({ request }) => (usesSse(request) ? SSE_PATH : BYTES_PATH),
},
input,
)
export const CartesiaSpeech = {
protocol,
model,
} as const
@@ -0,0 +1,144 @@
import { Effect } from "effect"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { MediaInput } from "./utils/media-input.js"
import { SpeechStream } from "./utils/speech-stream.js"
const ADAPTER = "deepgram-speech"
const NAME = "Deepgram"
const PROVIDER = ProviderID.make("deepgram")
export const DEFAULT_BASE_URL = "https://api.deepgram.com"
export const PATH = "/v1/speak"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type DeepgramSpeechString<Known extends string> = Known | (string & {})
export type DeepgramEncoding = DeepgramSpeechString<"linear16" | "mulaw" | "alaw" | "mp3" | "opus" | "flac" | "aac">
export type DeepgramSpeechOptions = {
readonly encoding?: DeepgramEncoding
readonly container?: DeepgramSpeechString<"wav" | "ogg" | "none">
readonly sampleRate?: number
readonly bitRate?: number
readonly mip_opt_out?: boolean
readonly tag?: string
} & Record<string, unknown>
export type Request = SpeechRequestFor<DeepgramSpeechOptions>
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
type State = SpeechStream.Audio
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const FORMATS: Readonly<Record<string, { readonly encoding: string; readonly container?: string }>> = {
mp3: { encoding: "mp3" },
wav: { encoding: "linear16", container: "wav" },
pcm: { encoding: "linear16", container: "none" },
opus: { encoding: "opus" },
flac: { encoding: "flac" },
aac: { encoding: "aac" },
}
const audioFormat = (request: Request) => {
const format = request.format === undefined ? undefined : FORMATS[request.format]
return {
encoding: request.providerOptions?.encoding ?? format?.encoding,
container: request.providerOptions?.container ?? format?.container,
}
}
const queryParameters = (request: Request) => {
const { encoding: _encoding, container: _container, sampleRate, bitRate, ...native } = request.providerOptions ?? {}
return MediaInput.query(ADAPTER, {
...native,
model: request.model.id,
...audioFormat(request),
sample_rate: sampleRate,
bit_rate: bitRate,
speed: request.speed,
})
}
const fromRequest = Effect.fn("DeepgramSpeech.fromRequest")(function* (request: Request) {
if (
request.format !== undefined &&
FORMATS[request.format] === undefined &&
request.providerOptions?.encoding === undefined
)
return yield* SpeechStream.unsupportedFormat(
PROVIDER,
ADAPTER,
`${NAME} has no encoding for format "${request.format}"; pass providerOptions.encoding`,
)
return MediaProtocol.json(
mergeJsonRecords({ text: request.text }, request.http?.body) ?? {},
yield* queryParameters(request),
)
})
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const HEADERLESS_ENCODINGS: Readonly<Record<string, SpeechStream.PcmEncoding>> = {
linear16: "pcm_s16le",
mulaw: "pcm_mulaw",
alaw: "pcm_alaw",
}
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
const headers = context.http.headers
const mediaType = headers["content-type"]
const format = audioFormat(context.request)
const encoding = HEADERLESS_ENCODINGS[format.encoding ?? ""]
const requestID = headers["dg-request-id"]
const modelName = headers["dg-model-name"]
return SpeechStream.finish(ADAPTER, state, {
...(format.container === "none" && encoding !== undefined
? SpeechStream.pcm(encoding, SpeechStream.sampleRate(mediaType), mediaType)
: // Deepgram's default encoding is MP3; WAV is a container around any encoding.
{ mediaType, info: { format: format.container === "wav" ? "wav" : (format.encoding ?? "mp3") } }),
usage: SpeechStream.headerUsage("characters", headers["dg-char-count"]),
providerMetadata:
requestID === undefined && modelName === undefined
? undefined
: { deepgram: { requestId: requestID, modelName } },
})
}
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, Uint8Array, State>({
id: ADAPTER,
name: NAME,
unsupported: ["voice", "language", "instructions", "timestamps"],
body: { from: fromRequest },
frames: (bytes) => bytes,
initial: () => ({ chunks: [] }),
step: (state, frame) => Effect.succeed(SpeechStream.delta(state, frame)),
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<DeepgramSpeechOptions, Uint8Array, State>(
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
export const DeepgramSpeech = {
protocol,
model,
} as const
@@ -0,0 +1,191 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const ADAPTER = "deepgram-transcription"
const NAME = "Deepgram"
const PROVIDER = ProviderID.make("deepgram")
export const DEFAULT_BASE_URL = "https://api.deepgram.com"
export const PATH = "/v1/listen"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type DeepgramTranscriptionOptions = {
readonly smart_format?: boolean
readonly punctuate?: boolean
readonly paragraphs?: boolean
readonly utterances?: boolean
readonly detect_language?: boolean | ReadonlyArray<string>
readonly keyterm?: ReadonlyArray<string>
readonly diarize_model?: "latest" | "v1" | "v2" | (string & {})
readonly filler_words?: boolean
readonly numerals?: boolean
readonly mip_opt_out?: boolean
readonly tag?: string | ReadonlyArray<string>
} & Record<string, unknown>
export type Request = TranscriptionRequestFor<DeepgramTranscriptionOptions>
// ---------------------------------------------------------------------------
// 2. Response schema
// ---------------------------------------------------------------------------
const Word = Schema.Struct({
word: Schema.String,
start: Schema.Number,
end: Schema.Number,
confidence: Schema.optional(Schema.Number),
speaker: Schema.optional(Schema.Number),
punctuated_word: Schema.optional(Schema.String),
})
const ListenResponse = Schema.Struct({
metadata: Schema.optional(
Schema.Struct({ request_id: Schema.optional(Schema.String), duration: Schema.optional(Schema.Number) }),
),
results: Schema.Struct({
channels: Schema.Array(
Schema.Struct({
alternatives: Schema.optional(
Schema.Array(Schema.Struct({ transcript: Schema.String, words: Schema.optional(Schema.Array(Word)) })),
),
detected_language: Schema.optional(Schema.String),
}),
),
utterances: Schema.optional(
Schema.Array(
Schema.Struct({
start: Schema.Number,
end: Schema.Number,
transcript: Schema.String,
speaker: Schema.optional(Schema.Number),
words: Schema.optional(Schema.Array(Word)),
}),
),
),
}),
})
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const query = (request: Request) =>
MediaInput.query(
ADAPTER,
mergeJsonRecords(
{
model: request.model.id,
smart_format: true,
language: request.language,
// Deepgram assumes English unless asked to detect, unlike the other routes' auto-detection.
detect_language: request.language === undefined ? true : undefined,
// `diarize=true` is deprecated in favor of choosing a diarization model.
diarize_model: request.diarize === true ? "latest" : undefined,
utterances: request.diarize === true || request.timestamps === "segment" ? true : undefined,
},
request.providerOptions,
) ?? {},
)
const fromRequest = Effect.fn("DeepgramTranscription.fromRequest")(function* (request: Request) {
const url = ProviderShared.mediaUrl(request.audio)
if (url !== undefined)
return MediaProtocol.json(mergeJsonRecords({ url }, request.http?.body) ?? {}, yield* query(request))
if (request.http?.body !== undefined)
return yield* ProviderShared.invalidRequest(`${NAME} sends inline audio as the raw body, so http.body cannot apply`)
const audio = yield* MediaInput.inlineBytes(ADAPTER, request.audio)
return MediaProtocol.binary(audio, request.audio.mediaType, yield* query(request))
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeListen = MediaProtocol.decodeJson(ADAPTER, NAME, ListenResponse)
const speaker = (value: number | undefined) => (value === undefined ? undefined : String(value))
const wordText = (word: typeof Word.Type) => word.punctuated_word ?? word.word
// Utterances split on pauses, not speakers: the v2 diarizer labels a whole utterance with one speaker even when its
// words change speaker, so segments split each utterance at speaker changes.
const speakerTurns = (words: ReadonlyArray<typeof Word.Type>) =>
words.reduce<Array<Array<typeof Word.Type>>>((turns, word) => {
const last = turns.at(-1)
if (last === undefined || last[0].speaker !== word.speaker) return [...turns, [word]]
last.push(word)
return turns
}, [])
const decodeResponse = Effect.fn("DeepgramTranscription.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* decodeListen(response)
const channel = output.value.results.channels[0]
const alternative = channel?.alternatives?.[0]
if (alternative === undefined) return yield* output.invalid(`${NAME} returned no transcript`)
const duration = output.value.metadata?.duration
const requestID = output.value.metadata?.request_id
return new TranscriptionResponse({
text: alternative.transcript,
segments: output.value.results.utterances?.flatMap((utterance) =>
utterance.words === undefined || utterance.words.length === 0
? [
{
text: utterance.transcript,
startSeconds: utterance.start,
endSeconds: utterance.end,
speaker: speaker(utterance.speaker),
},
]
: speakerTurns(utterance.words).map((turn) => ({
text: turn.map(wordText).join(" "),
startSeconds: turn[0].start,
endSeconds: turn[turn.length - 1].end,
speaker: speaker(turn[0].speaker),
})),
),
words: alternative.words?.map((word) => ({
text: wordText(word),
startSeconds: word.start,
endSeconds: word.end,
speaker: speaker(word.speaker),
confidence: word.confidence,
})),
language: channel?.detected_language?.toLowerCase(),
durationSeconds: duration,
usage: duration === undefined ? undefined : { type: "seconds", seconds: duration },
providerMetadata: requestID === undefined ? undefined : { deepgram: { requestId: requestID } },
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, TranscriptionResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["prompt", "speakers"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput) =>
TranscriptionModel.fromRoute<DeepgramTranscriptionOptions>(
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
export const DeepgramTranscription = {
protocol,
model,
} as const
@@ -0,0 +1,218 @@
import { Effect, Schema } from "effect"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { SpeechStream } from "./utils/speech-stream.js"
const ADAPTER = "elevenlabs-speech"
const NAME = "ElevenLabs"
const PROVIDER = ProviderID.make("elevenlabs")
export const DEFAULT_BASE_URL = "https://api.elevenlabs.io"
export const PATH = "/v1/text-to-speech"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type ElevenLabsSpeechString<Known extends string> = Known | (string & {})
export type ElevenLabsOutputFormat = ElevenLabsSpeechString<
| "mp3_22050_32"
| "mp3_24000_48"
| "mp3_44100_32"
| "mp3_44100_64"
| "mp3_44100_96"
| "mp3_44100_128"
| "mp3_44100_192"
| "pcm_8000"
| "pcm_16000"
| "pcm_22050"
| "pcm_24000"
| "pcm_32000"
| "pcm_44100"
| "pcm_48000"
| "wav_8000"
| "wav_16000"
| "wav_22050"
| "wav_24000"
| "wav_32000"
| "wav_44100"
| "wav_48000"
| "ulaw_8000"
| "alaw_8000"
| "opus_48000_32"
| "opus_48000_64"
| "opus_48000_96"
| "opus_48000_128"
| "opus_48000_192"
>
export type ElevenLabsSpeechOptions = {
readonly outputFormat?: ElevenLabsOutputFormat
readonly voice_settings?: {
readonly stability?: number
readonly similarity_boost?: number
readonly style?: number
readonly use_speaker_boost?: boolean
}
readonly seed?: number
readonly apply_text_normalization?: ElevenLabsSpeechString<"auto" | "on" | "off">
} & Record<string, unknown>
export type Request = SpeechRequestFor<ElevenLabsSpeechOptions>
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
const Alignment = Schema.Struct({
characters: Schema.Array(Schema.String),
character_start_times_seconds: Schema.Array(Schema.Number),
character_end_times_seconds: Schema.Array(Schema.Number),
})
const TimestampedAudio = Schema.Struct({
audio_base64: Schema.Uint8ArrayFromBase64,
alignment: optionalNull(Alignment),
})
const decodeRecord = MediaProtocol.decodeFrame(ADAPTER, NAME, TimestampedAudio)
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
type State = SpeechStream.Audio
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const OUTPUT_FORMATS: Readonly<Record<string, string>> = {
mp3: "mp3_44100_128",
pcm: "pcm_24000",
wav: "wav_24000",
opus: "opus_48000_64",
}
/** WAV is served only by the non-streaming endpoints. */
const outputFormat = Effect.fn("ElevenLabsSpeech.outputFormat")(function* (request: MediaProtocol.Addressed<Request>) {
const format = request.providerOptions?.outputFormat ?? OUTPUT_FORMATS[request.format ?? "mp3"]
if (format === undefined)
return yield* SpeechStream.unsupportedFormat(
PROVIDER,
ADAPTER,
`${NAME} has no default output format for "${request.format}"; pass providerOptions.outputFormat`,
)
if (request.mode === "stream" && format.startsWith("wav_"))
return yield* SpeechStream.unsupportedFormat(
PROVIDER,
ADAPTER,
`${NAME} streams mp3, pcm, opus, ulaw, and alaw but not "${format}"; use generate for WAV`,
)
return format
})
const fromRequest = Effect.fn("ElevenLabsSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
if (request.voice === undefined)
return yield* ProviderShared.invalidRequest(`${NAME} requires a voice id; pass it as \`voice\``)
const { outputFormat: _outputFormat, ...native } = request.providerOptions ?? {}
return MediaProtocol.json(
mergeJsonRecords(
{
text: request.text,
model_id: request.model.id,
language_code: request.language,
voice_settings: request.speed === undefined ? undefined : { speed: request.speed },
},
native,
request.http?.body,
) ?? {},
{ output_format: yield* outputFormat(request) },
)
})
const path = (request: MediaProtocol.Addressed<Request>) =>
`${PATH}/${encodeURIComponent(SpeechStream.voiceID(request.voice) ?? "")}${request.mode === "stream" ? "/stream" : ""}${
request.timestamps === true ? "/with-timestamps" : ""
}`
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const onRecord = Effect.fn("ElevenLabsSpeech.onRecord")(function* (state: State, frame: string) {
const record = yield* decodeRecord(frame)
const [next, events] = SpeechStream.delta(state, record.audio_base64)
const alignment = record.alignment
if (!alignment) return [next, events] as const
return [
next,
[
...events,
...SpeechStream.timestamps(
alignment.characters,
alignment.character_start_times_seconds,
alignment.character_end_times_seconds,
),
],
] as const
})
const PCM_CODECS: Readonly<Record<string, SpeechStream.PcmEncoding>> = {
pcm: "pcm_s16le",
ulaw: "pcm_mulaw",
alaw: "pcm_alaw",
}
const describeOutput = (format: string) => {
const [codec = format, rate] = format.split("_")
const sampleRate = rate === undefined ? undefined : Number(rate)
const encoding = PCM_CODECS[codec]
return encoding === undefined ? SpeechStream.container(codec, sampleRate) : SpeechStream.pcm(encoding, sampleRate)
}
const finish = Effect.fn("ElevenLabsSpeech.finish")(function* (
state: State,
context: MediaProtocol.ResponseContext<Request>,
) {
const requestID = context.http.headers["request-id"]
return yield* SpeechStream.finish(ADAPTER, state, {
...describeOutput(yield* outputFormat(context.request)),
// `character-cost` is billed credits, not a character count (3 for 20 characters on `eleven_flash_v2_5`).
usage: SpeechStream.headerUsage("credits", context.http.headers["character-cost"]),
providerMetadata: requestID === undefined ? undefined : { elevenlabs: { requestId: requestID } },
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>({
id: ADAPTER,
name: NAME,
unsupported: ["instructions"],
body: { from: fromRequest },
frames: (bytes, context) => {
if (context.request.timestamps !== true) return bytes
return context.request.mode === "stream" ? Framing.lines.frame(bytes) : Framing.document.frame(bytes)
},
initial: () => ({ chunks: [] }),
step: SpeechStream.step(onRecord),
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<ElevenLabsSpeechOptions, string | Uint8Array, State>(
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => path(request) },
input,
)
export const ElevenLabsSpeech = {
protocol,
model,
} as const
+198
View File
@@ -0,0 +1,198 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
import { ProviderShared, optionalNull } from "./shared.js"
const ADAPTER = "fal-video"
const NAME = "fal Video"
const PROVIDER = ProviderID.make("fal")
export const DEFAULT_BASE_URL = "https://queue.fal.run"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type FalVideoString<Known extends string> = Known | (string & {})
/**
* Provider-native input. fal video endpoints are model-specific: `duration` is a string enum whose values differ per
* model (`"8s"` for Veo, `"5"` for Kling), and last-frame fields are named per model (`end_image_url`,
* `last_frame_url`, `tail_image_url`), so those pass through here instead of lowering from common fields.
*/
export type FalVideoOptions = {
readonly duration?: FalVideoString<"4s" | "6s" | "8s" | "5" | "10">
} & Record<string, unknown>
export type Request = VideoRequestFor<FalVideoOptions>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
/** fal hands back absolute follow-up URLs on submit; they are authoritative for status, result, and cancel. */
export const Token = Schema.Struct({
requestID: Schema.String,
statusURL: Schema.String,
responseURL: Schema.String,
cancelURL: Schema.String,
})
export type Token = Schema.Schema.Type<typeof Token>
const StartResponse = Schema.Struct({
request_id: Schema.String,
status_url: Schema.String,
response_url: Schema.String,
cancel_url: Schema.String,
queue_position: optionalNull(Schema.Number),
})
const QueueStatus = Schema.Struct({
status: Schema.String,
queue_position: optionalNull(Schema.Number),
error: optionalNull(Schema.Unknown),
})
const QueueResult = Schema.StructWithRest(
Schema.Struct({
video: Schema.Struct({
url: Schema.String,
content_type: optionalNull(Schema.String),
file_name: optionalNull(Schema.String),
file_size: optionalNull(Schema.Number),
}),
seed: optionalNull(Schema.Number),
}),
[Schema.Record(Schema.String, Schema.Unknown)],
)
const STATUS = {
IN_QUEUE: "queued",
IN_PROGRESS: "running",
COMPLETED: "completed",
} as const satisfies Record<string, Status>
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
// fal accepts public URLs and data URIs; there is no provider file handle to forward.
const mediaUrl = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, undefined, NAME).pipe(Effect.map((reference) => reference.value))
const fromRequest = Effect.fn("FalVideo.fromRequest")(function* (request: Request) {
if (request.frames?.last !== undefined)
return yield* ProviderShared.unsupportedOperation({
operation: "video.frames.last",
provider: PROVIDER,
route: ADAPTER,
message: `${NAME} names the last frame per model; pass it through providerOptions (e.g. end_image_url) instead of frames.last`,
})
const imageUrl = request.frames?.first === undefined ? undefined : yield* mediaUrl(request.frames.first)
const videoUrl = request.video === undefined ? undefined : yield* mediaUrl(request.video)
return MediaProtocol.json(
mergeJsonRecords(
{
prompt: request.prompt,
negative_prompt: request.negativePrompt,
seed: request.seed,
aspect_ratio: request.aspectRatio,
resolution: request.resolution,
generate_audio: request.audio,
image_url: imageUrl,
video_url: videoUrl,
},
request.providerOptions,
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeStart = MediaProtocol.decodeStarted(ADAPTER, NAME, StartResponse, (value) => ({
token: {
requestID: value.request_id,
statusURL: value.status_url,
responseURL: value.response_url,
cancelURL: value.cancel_url,
},
snapshot: { id: value.request_id, status: "queued", position: value.queue_position ?? undefined },
}))
const decodeQueueStatus = MediaProtocol.decodeJson(ADAPTER, NAME, QueueStatus)
const decodeQueueResult = MediaProtocol.decodeJson(ADAPTER, NAME, QueueResult)
const decodeStatus = Effect.fn("FalVideo.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeQueueStatus(response)
const decoded = output.value
const status = yield* MediaProtocol.status(STATUS, decoded.status, output)
// fal reports request failures as COMPLETED with an `error`; the response endpoint carries the details.
const failed = status === "completed" && decoded.error !== undefined && decoded.error !== null
return {
id: context.token.requestID,
status: failed ? "failed" : status,
position: status === "queued" ? (decoded.queue_position ?? undefined) : undefined,
}
})
const decodeResult = Effect.fn("FalVideo.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeQueueResult(response)
const { video, seed, ...rest } = output.value
return new VideoResponse({
videos: [Media.url(video.url, { mediaType: video.content_type ?? "video/mp4" })],
providerMetadata: {
fal: {
requestId: context.token.requestID,
seed: seed ?? undefined,
fileName: video.file_name ?? undefined,
fileSize: video.file_size ?? undefined,
...rest,
},
},
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>({
id: ADAPTER,
name: NAME,
token: Token,
unsupported: ["n", "durationSeconds", "references"],
start: { body: { from: fromRequest }, decode: decodeStart },
status: { path: (token) => token.statusURL, decode: decodeStatus },
result: { path: (token) => token.responseURL, decode: decodeResult },
cancel: { method: "PUT", path: (token) => token.cancelURL },
})
export const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<FalVideoOptions, Token>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => `/${request.model.id}`,
},
input,
)
export const FalVideo = {
protocol,
model,
} as const
+2 -6
View File
@@ -22,6 +22,7 @@ import {
import { classifyProviderFailure } from "../provider-error.js"
import { Media } from "../media.js"
import { JsonObject, knownString, lenient, optionalArray, optionalNull, ProviderShared } from "./shared.js"
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
import { GeminiToolSchema } from "./utils/gemini-tool-schema.js"
import { Lifecycle } from "./utils/lifecycle.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
@@ -305,14 +306,9 @@ const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
tool: (name) => ({ functionCallingConfig: { mode: "ANY" as const, allowedFunctionNames: [name] } }),
})
// Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs.
const lowerContentPart = Effect.fn("Gemini.lowerContentPart")(function* (part: TextPart | MediaPart) {
if (part.type === "text") return { text: part.text }
const source = part.media.source
if (source.type === "ref" && source.provider === "google")
return { fileData: { mimeType: part.media.mediaType, fileUri: source.id } }
const media = yield* ProviderShared.requireInlineMedia("Gemini", part.media)
return { inlineData: { mimeType: media.mime, data: media.base64 } }
return yield* GeminiGenerateContent.mediaPart("Gemini", part.media)
})
const providerMetadata = (key: string, metadata: Record<string, unknown>): ProviderMetadata => ({ [key]: metadata })
+3 -22
View File
@@ -1,11 +1,11 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { ProviderShared } from "./shared.js"
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
import { MediaInput } from "./utils/media-input.js"
const ADAPTER = "google-images"
@@ -102,25 +102,6 @@ const generationConfig = (request: Request) => {
)
}
// Gemini does not fetch public URLs; inline payloads or Gemini Files references are the only accepted inputs.
const imagePart = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ inlineData: { mimeType: inline.mime, data: inline.base64 } })
const id = MediaInput.refID(asset, PROVIDER)
if (id) return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri: id } })
if (asset.source.type === "ref")
return Effect.fail(
ProviderShared.invalidRequest(
"Google generateContent requires Gemini file references rather than other providers' file IDs",
),
)
return Effect.fail(
ProviderShared.invalidRequest(
"Google generateContent does not fetch public image URLs; use bytes, a data URL, or a Gemini file reference",
),
)
}
const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request: Request) {
if (request.n !== undefined && request.n > 1)
return yield* ProviderShared.unsupportedOperation({
@@ -129,7 +110,7 @@ const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request: Re
route: ADAPTER,
message: `${NAME} generates one image per request; call it once per image instead of n=${request.n}`,
})
const parts = yield* Effect.forEach(request.images ?? [], imagePart)
const parts = yield* Effect.forEach(request.images ?? [], (image) => GeminiGenerateContent.mediaPart(NAME, image))
return MediaProtocol.json(
mergeJsonRecords(
{
+144
View File
@@ -0,0 +1,144 @@
import { Effect, Schema } from "effect"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
import { SpeechStream } from "./utils/speech-stream.js"
const ADAPTER = "google-speech"
const NAME = "Google Speech"
const PROVIDER = ProviderID.make("google")
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
const DEFAULT_SAMPLE_RATE = 24000
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
/** Style is directed in the text itself, and `speechConfig.multiSpeakerVoiceConfig` excludes `voice`. */
export type GoogleSpeechOptions = {
readonly temperature?: number
readonly seed?: number
readonly speechConfig?: {
readonly multiSpeakerVoiceConfig?: {
readonly speakerVoiceConfigs: ReadonlyArray<{
readonly speaker: string
readonly voiceConfig: { readonly prebuiltVoiceConfig: { readonly voiceName: string } }
}>
}
}
} & Record<string, unknown>
export type Request = SpeechRequestFor<GoogleSpeechOptions>
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
const GenerateContentChunk = GeminiGenerateContent.chunk(
Schema.Struct({
text: Schema.optional(Schema.String),
inlineData: Schema.optional(Schema.Struct({ mimeType: Schema.String, data: Schema.Uint8ArrayFromBase64 })),
}),
)
const decodeChunk = MediaProtocol.decodeFrame(ADAPTER, NAME, GenerateContentChunk)
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
interface State extends SpeechStream.Audio, GeminiGenerateContent.Metadata {
readonly mimeType?: string
}
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
if (request.format !== undefined && request.format !== "pcm")
return yield* SpeechStream.unsupportedFormat(
PROVIDER,
ADAPTER,
`${NAME} only returns raw PCM; request format "pcm" or omit it, then wrap the samples yourself`,
)
const voiceName = SpeechStream.voiceID(request.voice)
return MediaProtocol.json(
mergeJsonRecords(
{
contents: [{ role: "user", parts: [{ text: request.text }] }],
generationConfig: mergeJsonRecords(
{
responseModalities: ["AUDIO"],
speechConfig: {
voiceConfig: voiceName === undefined ? undefined : { prebuiltVoiceConfig: { voiceName } },
languageCode: request.language,
},
},
request.providerOptions,
),
},
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const step = Effect.fn("GoogleSpeech.step")(function* (state: State, frame: string) {
const chunk = yield* decodeChunk(frame)
const blocked = GeminiGenerateContent.blocked(NAME, chunk, frame)
if (blocked !== undefined) return yield* blocked
const audio = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) =>
part.inlineData === undefined ? [] : [part.inlineData],
)
const next: State = { ...GeminiGenerateContent.track(state, chunk), mimeType: state.mimeType ?? audio[0]?.mimeType }
return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])] as const
})
const finish = (state: State) => {
const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE
return SpeechStream.finish(ADAPTER, state, {
...SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`),
usage: GeminiGenerateContent.usage(state.usage),
providerMetadata: GeminiGenerateContent.providerMetadata(state),
detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`,
})
}
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string, State>({
id: ADAPTER,
name: NAME,
unsupported: ["instructions", "speed", "timestamps"],
body: { from: fromRequest },
frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode),
initial: () => ({ chunks: [] }),
step,
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<GoogleSpeechOptions, string, State>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
// Only `gemini-3.1-flash-tts-preview` and later stream; earlier TTS models reject `streamGenerateContent`.
path: ({ request }) => GeminiGenerateContent.path(request.model.id, request.mode),
},
input,
)
export const GoogleSpeech = {
protocol,
model,
} as const
@@ -0,0 +1,214 @@
import { Effect, Schema, SchemaGetter } from "effect"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import {
TranscriptionFinishEvent,
TranscriptionModel,
TranscriptionSegmentEvent,
TranscriptionTextDeltaEvent,
type TranscriptionRequestFor,
type TranscriptionSegment,
type TranscriptionWord,
type TranscriptionEvent,
} from "../transcription.js"
import { ProviderShared } from "./shared.js"
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
const ADAPTER = "google-transcription"
const NAME = "Google Transcription"
const PROVIDER = ProviderID.make("google")
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
/**
* Merged into `generationConfig`. The API rejects `customVocabulary` and `mode: "SMART"` alongside diarization or word
* timestamps.
*/
export type GoogleTranscriptionOptions = {
readonly audioTranscriptionConfig?: {
readonly mode?: "VERBATIM" | "SMART" | (string & {})
readonly customVocabulary?: ReadonlyArray<string>
readonly languageCodes?: ReadonlyArray<string>
}
} & Record<string, unknown>
export type Request = TranscriptionRequestFor<GoogleTranscriptionOptions>
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
const Seconds = Schema.String.check(Schema.isPattern(/^\d+(\.\d+)?s$/)).pipe(
Schema.decodeTo(Schema.Number, {
decode: SchemaGetter.transform((value) => Number.parseFloat(value)),
encode: SchemaGetter.transform((value) => `${value}s`),
}),
)
const AudioTranscription = Schema.Struct({
text: Schema.String,
speakerLabel: Schema.optional(Schema.String),
words: Schema.optional(
Schema.Array(
Schema.Struct({
word: Schema.String,
startOffset: Seconds,
endOffset: Seconds,
speakerLabel: Schema.optional(Schema.String),
}),
),
),
})
const decodeChunk = MediaProtocol.decodeFrame(
ADAPTER,
NAME,
GeminiGenerateContent.chunk(Schema.Struct({ audioTranscription: Schema.optional(AudioTranscription) })),
)
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
interface State extends GeminiGenerateContent.Metadata {
readonly text: string
readonly segments: Array<TranscriptionSegment>
readonly words: Array<TranscriptionWord>
}
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const fromRequest = Effect.fn("GoogleTranscription.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
// General Gemini models ignore `audioTranscriptionConfig` and answer the audio conversationally.
if (!request.model.id.includes("transcribe"))
return yield* ProviderShared.unsupportedOperation({
operation: "transcription.model",
provider: PROVIDER,
route: ADAPTER,
message: `${request.model.id} is not a transcription model; use a transcribe model such as gemini-3.5-transcribe`,
})
return MediaProtocol.json(
mergeJsonRecords(
{
contents: [{ role: "user", parts: [yield* GeminiGenerateContent.mediaPart(ADAPTER, request.audio)] }],
generationConfig: mergeJsonRecords(
{
audioTranscriptionConfig: {
languageCodes: request.language === undefined ? undefined : [request.language],
// Parts carry no offsets of their own, so segment times come from word offsets.
wordTimestamp:
request.diarize === true || request.timestamps === "word" || request.timestamps === "segment"
? true
: undefined,
diarization: request.diarize === true ? true : undefined,
},
},
request.providerOptions,
),
},
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const turn = (part: Schema.Schema.Type<typeof AudioTranscription>) => {
const words = (part.words ?? []).map((word) => ({
text: word.word,
startSeconds: word.startOffset,
endSeconds: word.endOffset,
speaker: word.speakerLabel ?? part.speakerLabel,
}))
const first = words[0]
const last = words.at(-1)
return {
text: part.text,
words,
segment:
first === undefined || last === undefined
? undefined
: {
text: part.text,
startSeconds: first.startSeconds,
endSeconds: last.endSeconds,
speaker: part.speakerLabel,
},
}
}
const step = Effect.fn("GoogleTranscription.step")(function* (state: State, frame: string) {
const chunk = yield* decodeChunk(frame)
const blocked = GeminiGenerateContent.blocked(NAME, chunk, frame)
if (blocked !== undefined) return yield* blocked
const turns = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) =>
part.audioTranscription === undefined ? [] : [turn(part.audioTranscription)],
)
const segments = turns.flatMap((item) => (item.segment === undefined ? [] : [item.segment]))
state.words.push(...turns.flatMap((item) => item.words))
state.segments.push(...segments)
// Each part is one whole speaker turn without surrounding whitespace, so turns join with a space.
const text = turns
.map((item) => item.text)
.filter((item) => item.length > 0)
.join(" ")
const delta = text.length === 0 || state.text.length === 0 ? text : ` ${text}`
const events: ReadonlyArray<TranscriptionEvent> = [
...(delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta })]),
...segments.map((segment) => TranscriptionSegmentEvent.make({ segment })),
]
return [{ ...GeminiGenerateContent.track(state, chunk), text: state.text + delta }, events] as const
})
const finish = (state: State) => {
if (state.finishReason === undefined) return Effect.fail(MediaProtocol.incomplete(ADAPTER))
return Effect.succeed([
TranscriptionFinishEvent.make({
text: state.text,
segments: state.segments.length === 0 ? undefined : state.segments,
words: state.words.length === 0 ? undefined : state.words,
usage: GeminiGenerateContent.usage(state.usage),
providerMetadata: GeminiGenerateContent.providerMetadata(state),
}),
])
}
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, string, State>({
id: ADAPTER,
name: NAME,
unsupported: ["prompt", "speakers"],
body: { from: fromRequest },
frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode),
initial: () => ({ text: "", segments: [], words: [] }),
step,
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
TranscriptionModel.fromRoute<GoogleTranscriptionOptions, string, State>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => GeminiGenerateContent.path(request.model.id, request.mode),
},
input,
)
export const GoogleTranscription = {
protocol,
model,
} as const
+233
View File
@@ -0,0 +1,233 @@
import { Duration, Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
import { ProviderShared, optionalArray } from "./shared.js"
const ADAPTER = "google-video"
const NAME = "Google Veo"
const PROVIDER = ProviderID.make("google")
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
/** Veo keeps generated files for two days; the asset carries that deadline so callers materialize in time. */
const FILE_RETENTION = Duration.days(2)
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type GoogleVideoString<Known extends string> = Known | (string & {})
/** Provider-native `parameters`. Common fields (`aspectRatio`, `resolution`, `durationSeconds`, `seed`) live on the request. */
export type GoogleVideoOptions = {
readonly personGeneration?: GoogleVideoString<"allow_all" | "allow_adult" | "dont_allow">
} & Record<string, unknown>
export type Request = VideoRequestFor<GoogleVideoOptions>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
/** The long-running operation name, e.g. `models/veo-3.1-generate-preview/operations/abc123`. */
export const Token = Schema.Struct({ operation: Schema.String })
export type Token = Schema.Schema.Type<typeof Token>
const StartResponse = Schema.Struct({ name: Schema.String })
const Operation = Schema.Struct({
done: Schema.optional(Schema.Boolean),
error: Schema.optional(Schema.Struct({ message: Schema.optional(Schema.String) })),
response: Schema.optional(
Schema.Struct({
generateVideoResponse: Schema.optional(
Schema.Struct({
generatedSamples: optionalArray(
Schema.Struct({
video: Schema.optional(
Schema.Struct({
uri: Schema.optional(Schema.String),
mimeType: Schema.optional(Schema.String),
}),
),
}),
),
raiMediaFilteredCount: Schema.optional(Schema.Number),
raiMediaFilteredReasons: optionalArray(Schema.String),
}),
),
}),
),
metadata: Schema.optional(Schema.Unknown),
})
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
// Veo takes inline media only; a prior Veo output is `Media.url` with transient auth, so materialize it first.
const inlineMedia = (asset: Media.Asset) =>
ProviderShared.requireInlineMedia(NAME, asset).pipe(
Effect.map((inline) => ({ inlineData: { mimeType: inline.mime, data: inline.base64 } })),
)
const fromRequest = Effect.fn("GoogleVideo.fromRequest")(function* (request: Request) {
if (request.n !== undefined && request.n > 1)
return yield* ProviderShared.unsupportedOperation({
operation: "video.n",
provider: PROVIDER,
route: ADAPTER,
message: `${NAME} generates one video per request; call it once per video instead of n=${request.n}`,
})
if (request.audio === false)
return yield* ProviderShared.unsupportedOperation({
operation: "video.audio",
provider: PROVIDER,
route: ADAPTER,
message: `${NAME} always generates audio; audio: false cannot be honored`,
})
if (request.frames?.last !== undefined && request.frames.first === undefined)
return yield* ProviderShared.invalidRequest(`${NAME} requires frames.first when frames.last is set`)
const image = request.frames?.first === undefined ? undefined : yield* inlineMedia(request.frames.first)
const lastFrame = request.frames?.last === undefined ? undefined : yield* inlineMedia(request.frames.last)
const video = request.video === undefined ? undefined : yield* inlineMedia(request.video)
const referenceImages = yield* Effect.forEach(request.references ?? [], (asset) =>
inlineMedia(asset).pipe(Effect.map((image) => ({ image, referenceType: "asset" }))),
)
return MediaProtocol.json(
mergeJsonRecords(
{
instances: [
{
prompt: request.prompt,
image,
lastFrame,
referenceImages: referenceImages.length === 0 ? undefined : referenceImages,
video,
},
],
parameters: mergeJsonRecords(
{
aspectRatio: request.aspectRatio,
resolution: request.resolution,
durationSeconds: request.durationSeconds,
negativePrompt: request.negativePrompt,
seed: request.seed,
},
request.providerOptions,
),
},
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeStart = MediaProtocol.decodeStarted(ADAPTER, NAME, StartResponse, (value) => ({
token: { operation: value.name },
snapshot: { id: value.name, status: "running" },
}))
// Operations carry no status string: not done is running, done with `error` failed, otherwise completed.
const statusOf = (operation: typeof Operation.Type): Status => {
if (operation.done !== true) return "running"
return operation.error === undefined ? "completed" : "failed"
}
const decodeOperation = MediaProtocol.decodeJson(ADAPTER, NAME, Operation)
const decodeStatus = Effect.fn("GoogleVideo.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeOperation(response)
return { id: context.token.operation, status: statusOf(output.value) }
})
const decodeResult = Effect.fn("GoogleVideo.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeOperation(response)
const operation = output.value
const status = statusOf(operation)
if (status === "running")
return yield* output.invalid(`${NAME} operation ${context.token.operation} has not finished`)
if (status === "failed")
return yield* output.ended(
"failed",
`${NAME} operation failed${operation.error?.message === undefined ? "" : `: ${operation.error.message}`}`,
)
const generated = operation.response?.generateVideoResponse
// Downloads require the same API key as the poll; the asset carries it transiently and follows the redirect.
const videos = yield* Effect.forEach(
(generated?.generatedSamples ?? []).flatMap((sample) =>
sample.video?.uri === undefined ? [] : [{ uri: sample.video.uri, mimeType: sample.video.mimeType }],
),
(video) =>
MediaProtocol.expiringUrl(video.uri, FILE_RETENTION, {
mediaType: video.mimeType ?? "video/mp4",
headers: context.auth,
}),
)
const reasons = generated?.raiMediaFilteredReasons ?? []
const notices = reasons.map((reason) => ({
type: "filtered" as const,
message: `${NAME} filtered media: ${reason}`,
providerMetadata: { google: { raiMediaFilteredReason: reason } },
}))
if (videos.length === 0 && (reasons.length > 0 || (generated?.raiMediaFilteredCount ?? 0) > 0))
return yield* output.contentPolicy(
`${NAME} filtered every video${reasons.length === 0 ? "" : `: ${reasons.join("; ")}`}`,
)
if (videos.length === 0) return yield* output.invalid(`${NAME} operation completed without any video`)
return new VideoResponse({
videos,
notices: notices.length === 0 ? undefined : notices,
providerMetadata: {
google: {
operation: context.token.operation,
raiMediaFilteredCount: generated?.raiMediaFilteredCount,
metadata: operation.metadata,
},
},
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
const operationPath = (token: Token) => `/${token.operation}`
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>({
id: ADAPTER,
name: NAME,
token: Token,
start: { body: { from: fromRequest }, decode: decodeStart },
status: { path: operationPath, decode: decodeStatus },
result: { path: operationPath, decode: decodeResult },
})
export const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<GoogleVideoOptions, Token>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => `/models/${request.model.id}:predictLongRunning`,
},
input,
)
export const GoogleVideo = {
protocol,
model,
} as const
+4 -8
View File
@@ -4,7 +4,7 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { JsonObject, ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
@@ -70,13 +70,9 @@ const Response = Schema.Struct({
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ image_url: inline.dataUrl })
const url = ProviderShared.mediaUrl(asset)
if (url) return Effect.succeed({ image_url: url })
return Effect.fail(ProviderShared.invalidRequest(`${NAME} accepts image bytes and URLs`))
}
// Meta has no file handles: refs are rejected even when they name this provider.
const reference = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, undefined, NAME).pipe(Effect.map((item) => ({ image_url: item.value })))
const fromRequest = Effect.fn("MetaImages.fromRequest")(function* (request: Request) {
const images = yield* Effect.forEach(request.images ?? [], reference)
+25 -12
View File
@@ -130,6 +130,10 @@ const OpenAIChatUserContent = Schema.Union([
type: Schema.Literal("image_url"),
image_url: Schema.Struct({ url: Schema.String }),
}),
Schema.Struct({
type: Schema.Literal("file"),
file: Schema.Struct({ filename: Schema.String, file_data: Schema.String }),
}),
])
const OpenAIChatMessage = Schema.Union([
@@ -360,6 +364,15 @@ const lowerToolCall = (part: ToolCallPart, options: LoweringOptions): OpenAIChat
})
const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart) {
// Chat Completions accepts PDFs, and no other documents, as inline `file` parts; file URLs are not supported.
if (part.media.mediaType.toLowerCase() === "application/pdf")
return {
type: "file" as const,
file: {
filename: part.filename ?? "document.pdf",
file_data: (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl,
},
}
if (part.media.kind !== "image")
return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.media.mediaType}`)
const url =
@@ -489,7 +502,7 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (
options: LoweringOptions,
) {
const messages: OpenAIChatMessage[] = []
const images: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
const attachments: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
for (const part of message.content) {
if (!ProviderShared.supportsContent(part, ["tool-result"]))
return yield* ProviderShared.unsupportedContent("OpenAI Chat", "tool", ["tool-result"])
@@ -511,9 +524,9 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (
cache_control: options.cacheControl?.(part.cache),
})
const files = content.filter((item) => item.type === "file")
images.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item)))))
attachments.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item)))))
}
return { messages, images }
return { messages, attachments }
})
const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (
@@ -574,21 +587,21 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request:
if (requireAssistantAfterTool && messages.at(-1)?.role === "tool")
messages.push({ role: "assistant", content: "Done." })
}
const pendingImages: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
const flushImages = () => {
if (pendingImages.length === 0) return
const pendingAttachments: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
const flushAttachments = () => {
if (pendingAttachments.length === 0) return
bridgeTools()
messages.push({ role: "user", content: pendingImages.splice(0) })
messages.push({ role: "user", content: pendingAttachments.splice(0) })
}
for (const message of request.messages) {
if (message.role === "user") bridgeTools()
if (message.role === "system") {
const part = yield* ProviderShared.wrappedSystemUpdate("OpenAI Chat", message)
if (pendingImages.length > 0) {
if (pendingAttachments.length > 0) {
messages.push({
role: "user",
content: [
...pendingImages.splice(0),
...pendingAttachments.splice(0),
{ type: "text", text: part.text, cache_control: options.cacheControl?.(part.cache) },
],
})
@@ -632,13 +645,13 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request:
if (message.role === "tool") {
const lowered = yield* lowerToolMessages(message, lowering)
messages.push(...lowered.messages)
pendingImages.push(...lowered.images)
pendingAttachments.push(...lowered.attachments)
continue
}
flushImages()
flushAttachments()
messages.push(...(yield* lowerMessage(message, reasoningField, requireReasoning, lowering)))
}
flushImages()
flushAttachments()
return messages
})
+9 -19
View File
@@ -4,7 +4,7 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
@@ -72,23 +72,10 @@ const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
const isInline = (asset: Media.Asset) => asset.inline() !== undefined
const blob = (data: Uint8Array, mediaType: string) => {
const buffer = new ArrayBuffer(data.byteLength)
new Uint8Array(buffer).set(data)
return new Blob([buffer], { type: mediaType })
}
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ image_url: inline.dataUrl })
const url = ProviderShared.mediaUrl(asset)
if (url) return Effect.succeed({ image_url: url })
const id = MediaInput.refID(asset, PROVIDER)
if (id) return Effect.succeed({ file_id: id })
return Effect.fail(
ProviderShared.invalidRequest("OpenAI Images accepts image URLs, data URLs, bytes, and OpenAI file IDs"),
const reference = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, PROVIDER, NAME).pipe(
Effect.map((item) => (item.type === "ref" ? { file_id: item.value } : { image_url: item.value })),
)
}
const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Request) {
const images = request.images ?? []
@@ -111,9 +98,11 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Re
form.append(key, typeof value === "string" ? value : ProviderShared.encodeJson(value))
})
const uploads = yield* Effect.forEach(images, (image) => MediaInput.inlineBytes(ADAPTER, image))
uploads.forEach((data, index) => form.append("image[]", blob(data, images[index].mediaType), `image-${index}`))
uploads.forEach((data, index) =>
form.append("image[]", MediaInput.blob(data, images[index].mediaType), `image-${index}`),
)
if (mask !== undefined)
form.append("mask", blob(yield* MediaInput.inlineBytes(ADAPTER, mask), mask.mediaType), "mask")
form.append("mask", MediaInput.blob(yield* MediaInput.inlineBytes(ADAPTER, mask), mask.mediaType), "mask")
return MediaProtocol.multipart(form)
}
@@ -137,6 +126,7 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Re
// ---------------------------------------------------------------------------
const requestedFormat = (body: MediaProtocol.Body) => {
if (body.type === "binary") return undefined
const value = body.type === "json" ? body.value.output_format : body.value.get("output_format")
return typeof value === "string" ? value : undefined
}
+141
View File
@@ -0,0 +1,141 @@
import { Effect, Schema } from "effect"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type MediaUsage } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { SpeechStream } from "./utils/speech-stream.js"
const ADAPTER = "openai-speech"
const NAME = "OpenAI Speech"
const PROVIDER = ProviderID.make("openai")
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/audio/speech"
/** `pcm` is raw 24 kHz, 16-bit signed little-endian mono samples without a header. */
const PCM_SAMPLE_RATE = 24000
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type OpenAISpeechOptions = Record<string, unknown>
export type Request = SpeechRequestFor<OpenAISpeechOptions>
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
const SpeechStreamEvent = Schema.Union([
Schema.Struct({ type: Schema.Literal("speech.audio.delta"), audio: Schema.Uint8ArrayFromBase64 }),
Schema.Struct({
type: Schema.Literal("speech.audio.done"),
usage: Schema.optional(
Schema.Struct({
input_tokens: Schema.optional(Schema.Number),
output_tokens: Schema.optional(Schema.Number),
total_tokens: Schema.optional(Schema.Number),
}),
),
}),
])
const decodeEvent = MediaProtocol.decodeFrame(ADAPTER, NAME, SpeechStreamEvent)
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
interface State extends SpeechStream.Audio {
readonly done: boolean
readonly usage?: MediaUsage
}
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
// `sse` is not supported for `tts-1` or `tts-1-hd`; those models stream the raw audio body instead.
const supportsSse = (model: string) => !/^tts-1(-hd)?(-|$)/.test(model)
const fromRequest = Effect.fn("OpenAISpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
return MediaProtocol.json(
mergeJsonRecords(
{
model: request.model.id,
input: request.text,
voice: request.voice,
instructions: request.instructions,
response_format: request.format,
speed: request.speed,
stream_format: request.mode === "stream" && supportsSse(request.model.id) ? "sse" : undefined,
},
request.providerOptions,
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const isSse = (body: MediaProtocol.Body) => body.type === "json" && body.value.stream_format === "sse"
const onEvent = Effect.fn("OpenAISpeech.onEvent")(function* (state: State, frame: string) {
const event = yield* decodeEvent(frame)
if (event.type === "speech.audio.delta") return SpeechStream.delta(state, event.audio)
const usage = event.usage
return [
{
...state,
done: true,
usage:
usage === undefined
? undefined
: {
type: "tokens" as const,
input: usage.input_tokens,
output: usage.output_tokens,
total: usage.total_tokens,
details: { openai: usage },
},
},
[],
] as const
})
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
if (isSse(context.body) && !state.done) return Effect.fail(MediaProtocol.incomplete(ADAPTER))
const format = context.request.format ?? "mp3"
return SpeechStream.finish(ADAPTER, state, {
...(format === "pcm" ? SpeechStream.pcm("pcm_s16le", PCM_SAMPLE_RATE) : SpeechStream.container(format)),
usage: state.usage,
})
}
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>({
id: ADAPTER,
name: NAME,
unsupported: ["language", "timestamps"],
body: { from: fromRequest },
frames: (bytes, context) => (isSse(context.body) ? Framing.sse.frame(bytes) : bytes),
initial: () => ({ chunks: [], done: false }),
step: SpeechStream.step(onEvent),
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<OpenAISpeechOptions, string | Uint8Array, State>(
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
export const OpenAISpeech = {
protocol,
model,
} as const
@@ -0,0 +1,277 @@
import { Effect, Schema, Stream } from "effect"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type MediaUsage } from "../schema/index.js"
import {
TranscriptionFinishEvent,
TranscriptionModel,
TranscriptionSegmentEvent,
TranscriptionTextDeltaEvent,
type TranscriptionEvent,
type TranscriptionRequestFor,
type TranscriptionSegment,
} from "../transcription.js"
import { mediaTypeExtension } from "../utils/media-type.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const ADAPTER = "openai-transcription"
const NAME = "OpenAI Transcription"
const PROVIDER = ProviderID.make("openai")
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/audio/transcriptions"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type OpenAITranscriptionOptions = {
readonly chunking_strategy?:
| "auto"
| {
readonly type: "server_vad"
readonly prefix_padding_ms?: number
readonly silence_duration_ms?: number
readonly threshold?: number
}
readonly include?: ReadonlyArray<"logprobs">
readonly keywords?: ReadonlyArray<string>
readonly known_speaker_names?: ReadonlyArray<string>
readonly known_speaker_references?: ReadonlyArray<string>
readonly temperature?: number
} & Record<string, unknown>
export type Request = TranscriptionRequestFor<OpenAITranscriptionOptions>
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
const Segment = Schema.Struct({
text: Schema.String,
start: Schema.Number,
end: Schema.Number,
speaker: Schema.optional(Schema.String),
})
const Usage = Schema.Union([
Schema.Struct({
type: Schema.Literal("tokens"),
input_tokens: Schema.optional(Schema.Number),
output_tokens: Schema.optional(Schema.Number),
total_tokens: Schema.optional(Schema.Number),
}),
Schema.Struct({ type: Schema.Literal("duration"), seconds: Schema.Number }),
])
const transcriptFields = {
text: Schema.String,
language: Schema.optional(Schema.String),
languages: Schema.optional(Schema.Array(Schema.Struct({ code: Schema.String }))),
duration: Schema.optional(Schema.Number),
segments: Schema.optional(Schema.Array(Segment)),
words: Schema.optional(
Schema.Array(Schema.Struct({ word: Schema.String, start: Schema.Number, end: Schema.Number })),
),
usage: Schema.optional(Usage),
}
const Event = Schema.Union([
Schema.Struct({ type: Schema.Literal("transcript.text.delta"), delta: Schema.String }),
Schema.Struct({ type: Schema.Literal("transcript.text.segment"), ...Segment.fields }),
Schema.Struct({ type: Schema.Literal("transcript.text.done"), ...transcriptFields }),
])
const Transcript = Schema.Struct(transcriptFields)
type Transcript = Schema.Schema.Type<typeof Transcript>
const decodeEvent = MediaProtocol.decodeFrame(ADAPTER, NAME, Event)
const decodeTranscript = MediaProtocol.decodeFrame(ADAPTER, NAME, Transcript)
type Frame = string | { readonly document: string }
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
interface State {
readonly segments: Array<TranscriptionSegment>
readonly transcript?: Transcript
}
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
interface Capabilities {
readonly stream: boolean
readonly timestamps: ReadonlyArray<"segment" | "word">
readonly diarize: boolean
readonly languageField: "language" | "languages"
}
const TRANSCRIBE: Capabilities = { stream: true, timestamps: [], diarize: false, languageField: "language" }
const capabilities = (model: string): Capabilities => {
if (model.startsWith("whisper")) return { ...TRANSCRIBE, stream: false, timestamps: ["segment", "word"] }
if (model.includes("diarize")) return { ...TRANSCRIBE, timestamps: ["segment"], diarize: true }
// `gpt-transcribe` replaces `language` with `languages[]` and rejects both together.
if (model.startsWith("gpt-transcribe")) return { ...TRANSCRIBE, languageField: "languages" }
return TRANSCRIBE
}
const unsupported = (operation: string, message: string) =>
Effect.fail(ProviderShared.unsupportedOperation({ operation, provider: PROVIDER, route: ADAPTER, message }))
const validate = (request: MediaProtocol.Addressed<Request>, model: Capabilities) => {
const id = request.model.id
if (request.mode === "stream" && !model.stream)
return unsupported("media.stream", `${id} does not stream; use Transcription.generate`)
if (request.diarize === true && !model.diarize)
return unsupported("media.diarize", `${id} does not diarize; use gpt-4o-transcribe-diarize`)
if (request.prompt !== undefined && model.diarize)
return unsupported("media.prompt", `${id} does not accept a prompt`)
if (
request.timestamps === undefined ||
request.timestamps === "none" ||
model.timestamps.includes(request.timestamps)
)
return Effect.void
return unsupported("media.timestamps", `${id} does not return ${request.timestamps} timestamps`)
}
const RESERVED_FORM_FIELDS = new Set([
"file",
"model",
"prompt",
"language",
"response_format",
"timestamp_granularities",
"stream",
])
const appendField = (form: FormData, key: string, value: unknown) => {
if (Array.isArray(value)) return value.forEach((item) => form.append(`${key}[]`, String(item)))
form.append(key, typeof value === "object" && value !== null ? ProviderShared.encodeJson(value) : String(value))
}
const fromRequest = Effect.fn("OpenAITranscription.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
const model = capabilities(request.model.id)
yield* validate(request, model)
// The API detects the audio format from the upload's filename extension.
const extension = mediaTypeExtension(request.audio.mediaType)
if (extension === undefined)
return yield* ProviderShared.invalidRequest(
`${NAME} cannot name a ${request.audio.mediaType} upload; send mp3, mp4, m4a, wav, webm, ogg, or flac audio`,
)
const audio = yield* MediaInput.inlineBytes(ADAPTER, request.audio)
const responseFormat = model.diarize
? "diarized_json"
: request.timestamps === undefined || request.timestamps === "none"
? undefined
: "verbose_json"
const native = Object.entries(mergeJsonRecords(request.providerOptions, request.http?.body) ?? {}).filter(
([key]) => !RESERVED_FORM_FIELDS.has(key),
)
const fields = mergeJsonRecords(
{
model: request.model.id,
language: model.languageField === "language" ? request.language : undefined,
languages: model.languageField === "languages" && request.language !== undefined ? [request.language] : undefined,
prompt: request.prompt,
response_format: responseFormat,
timestamp_granularities: responseFormat === "verbose_json" ? [request.timestamps] : undefined,
// Diarizing audio longer than 30 seconds requires a chunking strategy.
chunking_strategy: model.diarize ? "auto" : undefined,
stream: request.mode === "stream" ? true : undefined,
},
Object.fromEntries(native),
)
const form = new FormData()
form.append("file", MediaInput.blob(audio, request.audio.mediaType), `audio.${extension}`)
Object.entries(fields ?? {}).forEach(([key, value]) => appendField(form, key, value))
return MediaProtocol.multipart(form)
})
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const segment = (value: Schema.Schema.Type<typeof Segment>): TranscriptionSegment => ({
text: value.text.trim(),
startSeconds: value.start,
endSeconds: value.end,
speaker: value.speaker,
})
const onEvent = Effect.fn("OpenAITranscription.onEvent")(function* (state: State, frame: string) {
const event = yield* decodeEvent(frame)
if (event.type === "transcript.text.done") return [{ ...state, transcript: event }, []] as const
if (event.type === "transcript.text.delta")
return [state, event.delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta: event.delta })]] as const
const next = segment(event)
state.segments.push(next)
return [state, [TranscriptionSegmentEvent.make({ segment: next })]] as const
})
const step = (state: State, frame: Frame) =>
typeof frame === "string"
? onEvent(state, frame)
: decodeTranscript(frame.document).pipe(Effect.map((transcript) => [{ ...state, transcript }, []] as const))
const usage = (value: Transcript["usage"]): MediaUsage | undefined => {
if (value === undefined) return undefined
if (value.type === "duration") return { type: "seconds", seconds: value.seconds }
return {
type: "tokens",
input: value.input_tokens,
output: value.output_tokens,
total: value.total_tokens,
details: { openai: value },
}
}
const finish = (state: State) => {
const transcript = state.transcript
if (transcript === undefined) return Effect.fail(MediaProtocol.incomplete(ADAPTER))
const segments = transcript.segments?.map(segment) ?? state.segments
return Effect.succeed([
TranscriptionFinishEvent.make({
text: transcript.text,
segments: segments.length === 0 ? undefined : segments,
words: transcript.words?.map((word) => ({ text: word.word, startSeconds: word.start, endSeconds: word.end })),
language: (transcript.language ?? transcript.languages?.[0]?.code)?.toLowerCase(),
durationSeconds: transcript.duration,
usage: usage(transcript.usage),
}),
])
}
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, Frame, State>({
id: ADAPTER,
name: NAME,
unsupported: ["speakers"],
body: { from: fromRequest },
frames: (bytes, context) =>
context.request.mode === "stream"
? Framing.sse.frame(bytes)
: Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))),
initial: () => ({ segments: [] }),
step,
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
TranscriptionModel.fromRoute<OpenAITranscriptionOptions, Frame, State>(
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
export const OpenAITranscription = {
protocol,
model,
} as const
+204
View File
@@ -0,0 +1,204 @@
import { Duration, Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
import { ProviderShared, optionalArray, optionalNull } from "./shared.js"
const ADAPTER = "runway-video"
const NAME = "Runway"
const PROVIDER = ProviderID.make("runway")
export const DEFAULT_BASE_URL = "https://api.dev.runwayml.com/v1"
/** Every Runway request must pin the API version. */
export const API_VERSION = "2024-11-06"
export const TEXT_TO_VIDEO_PATH = "/text_to_video"
export const IMAGE_TO_VIDEO_PATH = "/image_to_video"
export const VIDEO_TO_VIDEO_PATH = "/video_to_video"
export const TASKS_PATH = "/tasks"
/** Output URLs are valid for 2448 hours; the asset carries the conservative bound. */
const OUTPUT_RETENTION = Duration.hours(24)
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type RunwayVideoString<Known extends string> = Known | (string & {})
/**
* Provider-native options. Common fields lower to Runway's names: `aspectRatio` → `ratio` (Runway expects pixel
* ratios such as `1280:720` for most models), `durationSeconds` → `duration`, `audio`, `negativePrompt`,
* `resolution`, `references`, and `frames` → `promptImage`.
*/
export type RunwayVideoOptions = {
readonly contentModeration?: { readonly publicFigureThreshold?: RunwayVideoString<"auto" | "low"> }
readonly outputFormat?: RunwayVideoString<"mp4" | "prores" | "png_sequence">
} & Record<string, unknown>
export type Request = VideoRequestFor<RunwayVideoOptions>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
export const Token = Schema.Struct({ taskID: Schema.String })
export type Token = Schema.Schema.Type<typeof Token>
const Cost = Schema.Struct({ credits: Schema.Number })
const StartResponse = Schema.Struct({ id: Schema.String })
const Task = Schema.Struct({
status: Schema.String,
progress: optionalNull(Schema.Number),
output: optionalArray(Schema.String),
failure: optionalNull(Schema.String),
failureCode: optionalNull(Schema.String),
cost: Schema.optional(Cost),
estimatedCost: Schema.optional(Cost),
})
const STATUS = {
PENDING: "queued",
THROTTLED: "queued",
RUNNING: "running",
SUCCEEDED: "completed",
FAILED: "failed",
CANCELLED: "cancelled",
} as const satisfies Record<string, Status>
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
// Runway accepts HTTPS URLs, `runway://` upload URIs, and data URIs, all as one string.
const mediaUri = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, PROVIDER, NAME).pipe(Effect.map((reference) => reference.value))
const fromRequest = Effect.fn("RunwayVideo.fromRequest")(function* (request: Request) {
const first = request.frames?.first === undefined ? undefined : yield* mediaUri(request.frames.first)
const last = request.frames?.last === undefined ? undefined : yield* mediaUri(request.frames.last)
const promptImage = [
...(first === undefined ? [] : [{ uri: first, position: "first" }]),
...(last === undefined ? [] : [{ uri: last, position: "last" }]),
]
const videoUri = request.video === undefined ? undefined : yield* mediaUri(request.video)
const references = yield* Effect.forEach(request.references ?? [], (asset) =>
mediaUri(asset).pipe(Effect.map((uri) => ({ uri }))),
)
return MediaProtocol.json(
mergeJsonRecords(
{
model: request.model.id,
promptText: request.prompt,
promptImage: promptImage.length === 0 ? undefined : promptImage,
videoUri,
references: references.length === 0 ? undefined : references,
ratio: request.aspectRatio,
duration: request.durationSeconds,
resolution: request.resolution,
audio: request.audio,
negativePrompt: request.negativePrompt,
seed: request.seed,
},
request.providerOptions,
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeStart = MediaProtocol.decodeStarted(ADAPTER, NAME, StartResponse, (value) => ({
token: { taskID: value.id },
snapshot: { id: value.id, status: "queued" },
}))
const decodeTask = MediaProtocol.decodeJson(ADAPTER, NAME, Task)
const decodeStatus = Effect.fn("RunwayVideo.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeTask(response)
const status = yield* MediaProtocol.status(STATUS, output.value.status, output)
return { id: context.token.taskID, status, progress: output.value.progress ?? undefined }
})
const decodeResult = Effect.fn("RunwayVideo.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeTask(response)
const task = output.value
const status = yield* MediaProtocol.status(STATUS, task.status, output)
if (status === "failed") {
const code = task.failureCode ?? undefined
const message = `${NAME} task failed${code === undefined ? "" : ` (${code})`}${task.failure ? `: ${task.failure}` : ""}`
// Runway failure codes are dotted paths; every moderation outcome carries a SAFETY segment.
if (code !== undefined && /(^|\.)SAFETY(\.|$)/.test(code)) return yield* output.contentPolicy(message)
return yield* output.ended("failed", message)
}
if (status === "cancelled")
return yield* output.ended("cancelled", `${NAME} task ${context.token.taskID} was cancelled`)
if (status !== "completed") return yield* output.invalid(`${NAME} task ${context.token.taskID} has not finished`)
const urls = task.output ?? []
if (urls.length === 0) return yield* output.invalid(`${NAME} task succeeded without any output`)
return new VideoResponse({
videos: yield* Effect.forEach(urls, (url) =>
MediaProtocol.expiringUrl(url, OUTPUT_RETENTION, { mediaType: "video/mp4" }),
),
usage: task.cost === undefined ? undefined : { type: "credits", credits: task.cost.credits },
providerMetadata: {
runway: {
taskId: context.token.taskID,
estimatedCredits: task.estimatedCost?.credits,
},
},
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
const taskPath = (token: Token) => `${TASKS_PATH}/${token.taskID}`
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>({
id: ADAPTER,
name: NAME,
token: Token,
unsupported: ["n"],
start: { body: { from: fromRequest }, decode: decodeStart },
status: { path: taskPath, decode: decodeStatus },
result: { path: taskPath, decode: decodeResult },
cancel: { method: "DELETE", path: taskPath },
})
const startPath = (request: Request) => {
if (request.video !== undefined) return VIDEO_TO_VIDEO_PATH
if (request.frames?.first !== undefined || request.frames?.last !== undefined) return IMAGE_TO_VIDEO_PATH
return TEXT_TO_VIDEO_PATH
}
export const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<RunwayVideoOptions, Token>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
headers: { "X-Runway-Version": API_VERSION },
path: ({ request }) => startPath(request),
},
input,
)
export const RunwayVideo = {
protocol,
model,
} as const
+22
View File
@@ -192,6 +192,28 @@ export const inlineRequired = (route: string, asset: Media.Asset) =>
/** The remote URL of a `url` asset, for protocols that accept `http(s)` references natively. */
export const mediaUrl = (asset: Media.Asset) => (asset.source.type === "url" ? asset.source.url : undefined)
export type MediaReference = { readonly type: "dataUrl" | "url" | "ref"; readonly value: string }
/**
* The one string a provider can address an asset by: inline payloads as a data URL, `url` sources as their URL, and
* this provider's own `ref` as its id. Other providers' refs are never forwarded and fail typed; omit `provider` for
* APIs with no file handles at all.
*/
export const mediaReference = (
asset: Media.Asset,
provider: ProviderID | undefined,
label: string,
): Effect.Effect<MediaReference, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ type: "dataUrl", value: inline.dataUrl })
const url = mediaUrl(asset)
if (url) return Effect.succeed({ type: "url", value: url })
if (provider !== undefined && asset.source.type === "ref" && asset.source.provider === provider)
return Effect.succeed({ type: "ref", value: asset.source.id })
const accepted = provider === undefined ? "" : `, and ${provider} references`
return Effect.fail(invalidRequest(`${label} accepts inline bytes, data URLs, http(s) URLs${accepted}`))
}
/**
* Lift a tool-result file into a `MediaPart`. Tool files carry either a data URL, an `http(s)` URL, or raw base64 in
* `uri`; the declared `mime` wins over any data-URL prefix so tool authors control the type the model sees.
@@ -0,0 +1,113 @@
import { Effect, Schema, type Stream } from "effect"
import type { Media } from "../../media.js"
import { Framing } from "../../route/framing.js"
import type { MediaProtocol } from "../../route/media-protocol.js"
import { AIError, ContentPolicyError, ProviderID, type MediaUsage, type ProviderMetadata } from "../../schema/index.js"
import { ProviderShared } from "../shared.js"
import { MediaInput } from "./media-input.js"
const PROVIDER = ProviderID.make("google")
const UsageMetadata = Schema.Struct({
promptTokenCount: Schema.optional(Schema.Number),
candidatesTokenCount: Schema.optional(Schema.Number),
totalTokenCount: Schema.optional(Schema.Number),
})
type UsageMetadata = Schema.Schema.Type<typeof UsageMetadata>
export const chunk = <const Part extends Schema.Top>(part: Part) =>
Schema.Struct({
candidates: Schema.optional(
Schema.Array(
Schema.Struct({
content: Schema.optional(Schema.Struct({ parts: Schema.optional(Schema.Array(part)) })),
finishReason: Schema.optional(Schema.String),
}),
),
),
promptFeedback: Schema.optional(
Schema.Struct({
blockReason: Schema.optional(Schema.String),
blockReasonMessage: Schema.optional(Schema.String),
}),
),
usageMetadata: Schema.optional(UsageMetadata),
modelVersion: Schema.optional(Schema.String),
responseId: Schema.optional(Schema.String),
})
interface Chunk {
readonly candidates?: ReadonlyArray<{ readonly finishReason?: string }>
readonly promptFeedback?: { readonly blockReason?: string; readonly blockReasonMessage?: string }
readonly usageMetadata?: UsageMetadata
readonly modelVersion?: string
readonly responseId?: string
}
export interface Metadata {
readonly usage?: UsageMetadata
readonly finishReason?: string
readonly modelVersion?: string
readonly responseId?: string
}
export const track = <State extends Metadata>(state: State, chunk: Chunk): State => ({
...state,
usage: chunk.usageMetadata ?? state.usage,
finishReason: chunk.candidates?.[0]?.finishReason ?? state.finishReason,
modelVersion: chunk.modelVersion ?? state.modelVersion,
responseId: chunk.responseId ?? state.responseId,
})
export const blocked = (name: string, chunk: Chunk, frame: string) => {
const feedback = chunk.promptFeedback
if (feedback?.blockReason === undefined) return undefined
return new AIError({
reason: new ContentPolicyError({
message: `${name} blocked the request (${feedback.blockReason})${
feedback.blockReasonMessage === undefined ? "" : `: ${feedback.blockReasonMessage}`
}`,
body: frame,
}),
})
}
export const usage = (usage: UsageMetadata | undefined): MediaUsage | undefined =>
usage === undefined
? undefined
: {
type: "tokens",
input: usage.promptTokenCount,
output: usage.candidatesTokenCount,
total: ProviderShared.totalTokens(usage.promptTokenCount, usage.candidatesTokenCount, usage.totalTokenCount),
details: { google: usage },
}
export const providerMetadata = (state: Metadata): ProviderMetadata => ({
google: { finishReason: state.finishReason, modelVersion: state.modelVersion, responseId: state.responseId },
})
export const path = (model: string, mode: MediaProtocol.Mode) =>
mode === "stream" ? `/models/${model}:streamGenerateContent?alt=sse` : `/models/${model}:generateContent`
// `generateContent` answers with one document shaped exactly like a streamed chunk, so it is a single frame.
export const frames = (bytes: Stream.Stream<Uint8Array, AIError>, mode: MediaProtocol.Mode) =>
mode === "stream" ? Framing.sse.frame(bytes) : Framing.document.frame(bytes)
// Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs.
export const mediaPart = (
route: string,
asset: Media.Asset,
): Effect.Effect<
| { readonly fileData: { readonly mimeType: string; readonly fileUri: string } }
| { readonly inlineData: { readonly mimeType: string; readonly data: string } },
AIError
> => {
const fileUri = MediaInput.refID(asset, PROVIDER)
if (fileUri !== undefined) return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri } })
return ProviderShared.requireInlineMedia(route, asset).pipe(
Effect.map((media) => ({ inlineData: { mimeType: media.mime, data: media.base64 } })),
)
}
export * as GeminiGenerateContent from "./gemini-generate-content.js"
@@ -1,5 +1,6 @@
import { Effect, Encoding } from "effect"
import { Media } from "../../media.js"
import type { MediaProtocol } from "../../route/media-protocol.js"
import type { AIError, ProviderID } from "../../schema/index.js"
import { ProviderShared } from "../shared.js"
@@ -13,6 +14,26 @@ export const inlineBytes = (route: string, asset: Media.Asset): Effect.Effect<Ui
)
}
/** Copied because `BlobPart` requires a plain `ArrayBuffer`. */
export const blob = (data: Uint8Array, mediaType: string) => {
const buffer = new ArrayBuffer(data.byteLength)
new Uint8Array(buffer).set(data)
return new Blob([buffer], { type: mediaType })
}
const isScalar = (value: unknown): value is string | number | boolean =>
typeof value === "string" || typeof value === "number" || typeof value === "boolean"
export const query = (route: string, values: Record<string, unknown>): Effect.Effect<MediaProtocol.Query, AIError> => {
const entries = Object.entries(values).filter(([, value]) => value !== undefined)
const invalid = entries.find(([, value]) => !isScalar(value) && !(Array.isArray(value) && value.every(isScalar)))
if (invalid !== undefined)
return Effect.fail(ProviderShared.invalidRequest(`${route} cannot send "${invalid[0]}" as a query parameter`))
return Effect.succeed(
Object.fromEntries(entries.map(([key, value]) => [key, Array.isArray(value) ? value.map(String) : String(value)])),
)
}
/** Provider file handle when the ref belongs to this provider; refs from other providers are never forwarded. */
export const refID = (asset: Media.Asset, provider: ProviderID) =>
asset.source.type === "ref" && asset.source.provider === provider ? asset.source.id : undefined
@@ -0,0 +1,118 @@
import { Effect } from "effect"
import { Media } from "../../media.js"
import { MediaProtocol } from "../../route/media-protocol.js"
import type { AIError, MediaUsage, ProviderID, ProviderMetadata } from "../../schema/index.js"
import {
SpeechAudioDeltaEvent,
SpeechFinishEvent,
SpeechTimestampsEvent,
type SpeechEvent,
type SpeechVoice,
} from "../../speech.js"
import { concatBytes } from "../../utils/bytes.js"
import { ProviderShared } from "../shared.js"
export interface Audio {
/** Appended in place: the route creates fresh state for each response through `initial`. */
readonly chunks: Array<Uint8Array>
}
export type StepResult<State> = readonly [State, ReadonlyArray<SpeechEvent>]
/** Empty chunks (keep-alive records) emit nothing. */
export const delta = <State extends Audio>(state: State, chunk: Uint8Array): StepResult<State> => {
if (chunk.length === 0) return [state, []]
state.chunks.push(chunk)
return [state, [SpeechAudioDeltaEvent.make({ chunk })]]
}
export const step =
<State extends Audio>(onRecord: (state: State, frame: string) => Effect.Effect<StepResult<State>, AIError>) =>
(state: State, frame: string | Uint8Array) =>
typeof frame === "string" ? onRecord(state, frame) : Effect.succeed(delta(state, frame))
export const timestamps = (
texts: ReadonlyArray<string>,
starts: ReadonlyArray<number>,
ends: ReadonlyArray<number>,
): ReadonlyArray<SpeechEvent> =>
texts.length === 0
? []
: [
SpeechTimestampsEvent.make({
items: texts.map((text, index) => ({ text, startSeconds: starts[index] ?? 0, endSeconds: ends[index] ?? 0 })),
}),
]
export const voiceID = (voice: SpeechVoice | undefined) => (typeof voice === "object" ? voice.id : voice)
const CONTAINER_MEDIA_TYPES: Readonly<Record<string, string>> = {
mp3: "audio/mpeg",
wav: "audio/wav",
opus: "audio/ogg",
aac: "audio/aac",
flac: "audio/flac",
}
export const container = (format: string, sampleRate?: number) => ({
mediaType: CONTAINER_MEDIA_TYPES[format],
info: { format, sampleRate },
})
const PCM_MEDIA_TYPES = {
pcm_s16le: "audio/pcm",
pcm_f32le: "audio/pcm",
pcm_mulaw: "audio/mulaw",
pcm_alaw: "audio/alaw",
} as const
export type PcmEncoding = keyof typeof PCM_MEDIA_TYPES
export const pcm = (encoding: PcmEncoding, sampleRate: number | undefined, mediaType?: string) => ({
mediaType: mediaType ?? PCM_MEDIA_TYPES[encoding],
info: { format: "pcm", encoding, sampleRate, channels: 1 },
})
export const sampleRate = (mediaType: string | undefined) => {
const rate = /rate=(\d+)/i.exec(mediaType ?? "")?.[1]
return rate === undefined ? undefined : Number(rate)
}
export const unsupportedFormat = (provider: ProviderID, route: string, message: string) =>
ProviderShared.unsupportedOperation({ operation: "media.format", provider, route, message })
/** A declared `mediaType` wins over sniffing: headerless PCM can start with bytes that look like an MPEG frame sync. */
export const finish = (
route: string,
state: Audio,
output: {
readonly mediaType: string | undefined
readonly info?: Media.Info
readonly usage?: MediaUsage
readonly providerMetadata?: ProviderMetadata
readonly detail?: string
},
): Effect.Effect<ReadonlyArray<SpeechEvent>, AIError> => {
if (state.chunks.length === 0)
return Effect.fail(
MediaProtocol.frameError(
route,
`The provider returned no audio${output.detail === undefined ? "" : ` (${output.detail})`}`,
),
)
return Effect.succeed([
SpeechFinishEvent.make({
audio: Media.bytes(concatBytes(state.chunks), output.mediaType, { info: output.info }),
usage: output.usage,
providerMetadata: output.providerMetadata,
}),
])
}
export const headerUsage = (type: "characters" | "credits", value: string | undefined): MediaUsage | undefined => {
const amount = Number(value)
if (!Number.isFinite(amount)) return undefined
return type === "credits" ? { type, credits: amount } : { type, characters: amount }
}
export * as SpeechStream from "./speech-stream.js"
+7 -10
View File
@@ -4,7 +4,7 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
@@ -58,15 +58,12 @@ const nativeOptions = (options: XAIImageOptions | undefined) => {
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ url: inline.dataUrl, type: "image_url" as const })
const url = ProviderShared.mediaUrl(asset)
if (url) return Effect.succeed({ url, type: "image_url" as const })
const id = MediaInput.refID(asset, PROVIDER)
if (id) return Effect.succeed({ file_id: id })
return Effect.fail(ProviderShared.invalidRequest(`${NAME} accepts image URLs, data URLs, bytes, and xAI file IDs`))
}
const reference = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, PROVIDER, NAME).pipe(
Effect.map((item) =>
item.type === "ref" ? { file_id: item.value } : { url: item.value, type: "image_url" as const },
),
)
const fromRequest = Effect.fn("XAIImages.fromRequest")(function* (request: Request) {
const references = yield* Effect.forEach(request.images ?? [], reference)
+211
View File
@@ -0,0 +1,211 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
import { ProviderShared, optionalNull } from "./shared.js"
const ADAPTER = "xai-video"
const NAME = "xAI Video"
const PROVIDER = ProviderID.make("xai")
export const DEFAULT_BASE_URL = "https://api.x.ai/v1"
export const PATH = "/videos/generations"
export const EDIT_PATH = "/videos/edits"
export const EXTEND_PATH = "/videos/extensions"
export const STATUS_PATH = "/videos"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
/**
* Provider-native options. Common fields (`frames`, `references`, `video`, `durationSeconds`, `aspectRatio`,
* `resolution`, `audio`) live on the request. `mode` selects the endpoint a `video` input is sent to.
*/
export type XAIVideoOptions = {
readonly mode?: "edit" | "extend"
readonly reference_audios?: ReadonlyArray<{ readonly voice_id: string }>
} & Record<string, unknown>
export type Request = VideoRequestFor<XAIVideoOptions>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
export const Token = Schema.Struct({ requestID: Schema.String })
export type Token = Schema.Schema.Type<typeof Token>
const StartResponse = Schema.Struct({ request_id: Schema.String })
const VideoStatus = Schema.Struct({
status: Schema.String,
progress: optionalNull(Schema.Number),
video: optionalNull(
Schema.Struct({
url: optionalNull(Schema.String),
duration: optionalNull(Schema.Number),
respect_moderation: optionalNull(Schema.Boolean),
}),
),
error: optionalNull(
Schema.Struct({
code: optionalNull(Schema.String),
message: optionalNull(Schema.String),
}),
),
model: optionalNull(Schema.String),
})
const STATUS = {
pending: "running",
done: "completed",
failed: "failed",
expired: "expired",
} as const satisfies Record<string, Status>
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const mediaInput = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, PROVIDER, NAME).pipe(
Effect.map((reference) => (reference.type === "ref" ? { file_id: reference.value } : { url: reference.value })),
)
const nativeOptions = (options: XAIVideoOptions | undefined) => {
if (!options) return undefined
const { mode: _mode, ...native } = options
return native
}
const fromRequest = Effect.fn("XAIVideo.fromRequest")(function* (request: Request) {
const image = request.frames?.first === undefined ? undefined : yield* mediaInput(request.frames.first)
const lastFrame = request.frames?.last === undefined ? undefined : yield* mediaInput(request.frames.last)
const video = request.video === undefined ? undefined : yield* mediaInput(request.video)
const references = yield* Effect.forEach(request.references ?? [], mediaInput)
return MediaProtocol.json(
mergeJsonRecords(
{
model: request.model.id,
prompt: request.prompt,
image,
last_frame: lastFrame,
reference_images: references.length === 0 ? undefined : references,
video,
duration: request.durationSeconds,
aspect_ratio: request.aspectRatio,
resolution: request.resolution,
generate_audio: request.audio,
},
nativeOptions(request.providerOptions),
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeStart = MediaProtocol.decodeStarted(ADAPTER, NAME, StartResponse, (value) => ({
token: { requestID: value.request_id },
snapshot: { id: value.request_id, status: "running" },
}))
// `progress` is undocumented but observed live as a 0..100 percentage (recorded cassette: 1 → 10 → 37 → 100).
const fraction = (progress: number | null | undefined) =>
progress !== undefined && progress !== null && progress >= 0 && progress <= 100 ? progress / 100 : undefined
const decodeVideoStatus = MediaProtocol.decodeJson(ADAPTER, NAME, VideoStatus)
const decodeStatus = Effect.fn("XAIVideo.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeVideoStatus(response)
const status = yield* MediaProtocol.status(STATUS, output.value.status, output)
return { id: context.token.requestID, status, progress: fraction(output.value.progress) }
})
const decodeResult = Effect.fn("XAIVideo.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeVideoStatus(response)
const decoded = output.value
const status = yield* MediaProtocol.status(STATUS, decoded.status, output)
if (status === "running") return yield* output.invalid(`${NAME} request ${context.token.requestID} has not finished`)
if (status === "failed") {
const code = decoded.error?.code ?? undefined
const message = decoded.error?.message ?? undefined
return yield* output.ended(
"failed",
`${NAME} generation failed${code === undefined ? "" : ` (${code})`}${message === undefined ? "" : `: ${message}`}`,
)
}
if (status !== "completed")
return yield* output.ended("expired", `${NAME} request ${context.token.requestID} expired`)
// `respect_moderation: false` marks a filtered result; a URL may still be present, so report it as a notice.
const notices =
decoded.video?.respect_moderation === false
? [{ type: "moderated" as const, message: `${NAME} flagged the generated video for moderation` }]
: undefined
const url = decoded.video?.url ?? undefined
if (url === undefined && notices !== undefined)
return yield* output.contentPolicy(`${NAME} withheld the video for moderation`)
if (url === undefined) return yield* output.invalid(`${NAME} completed without a video URL`)
const duration = decoded.video?.duration ?? undefined
return new VideoResponse({
videos: [
Media.url(url, {
mediaType: "video/mp4",
info: duration === undefined ? undefined : { durationSeconds: duration },
}),
],
notices,
providerMetadata: { xai: { requestId: context.token.requestID, model: decoded.model ?? undefined } },
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
const statusPath = (token: Token) => `${STATUS_PATH}/${token.requestID}`
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>({
id: ADAPTER,
name: NAME,
token: Token,
unsupported: ["n", "seed", "negativePrompt"],
start: { body: { from: fromRequest }, decode: decodeStart },
status: { path: statusPath, decode: decodeStatus },
result: { path: statusPath, decode: decodeResult },
})
// A source video goes to `/videos/edits` unless `providerOptions.mode` asks for an extension.
const startPath = (request: Request) => {
if (request.video === undefined) return PATH
return request.providerOptions?.mode === "extend" ? EXTEND_PATH : EDIT_PATH
}
export const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<XAIVideoOptions, Token>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => startPath(request),
},
input,
)
export const XAIVideo = {
protocol,
model,
} as const
+43
View File
@@ -0,0 +1,43 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { AssemblyAITranscription, DEFAULT_BASE_URL } from "../protocols/assemblyai-transcription.js"
export type { AssemblyAITranscriptionOptions } from "../protocols/assemblyai-transcription.js"
export const id = ProviderID.make("assemblyai")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
/** `https://api.eu.assemblyai.com` for the EU region. */
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
// The key is the whole `authorization` value, without a scheme.
const auth = (options: ProviderAuthOption<"optional">) => {
if ("auth" in options && options.auth) return options.auth
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
.orElse(Auth.config("ASSEMBLYAI_API_KEY"))
.pipe(Auth.header("authorization"))
}
export const configure = (input: Config = {}) => {
const transcription = (modelID: string | ModelID) =>
AssemblyAITranscription.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
transcription,
configure,
}
}
export const provider = configure()
export const transcription = provider.transcription
+35
View File
@@ -0,0 +1,35 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { CartesiaSpeech, DEFAULT_BASE_URL } from "../protocols/cartesia-speech.js"
export type { CartesiaEncoding, CartesiaSpeechOptions } from "../protocols/cartesia-speech.js"
export const id = ProviderID.make("cartesia")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "CARTESIA_API_KEY")
export const configure = (input: Config = {}) => {
const speech = (modelID: string | ModelID) =>
CartesiaSpeech.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
speech,
configure,
}
}
export const provider = configure()
export const speech = provider.speech
+44
View File
@@ -0,0 +1,44 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { DEFAULT_BASE_URL, DeepgramSpeech } from "../protocols/deepgram-speech.js"
import { DeepgramTranscription } from "../protocols/deepgram-transcription.js"
export type { DeepgramEncoding, DeepgramSpeechOptions } from "../protocols/deepgram-speech.js"
export type { DeepgramTranscriptionOptions } from "../protocols/deepgram-transcription.js"
export const id = ProviderID.make("deepgram")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
const auth = (options: ProviderAuthOption<"optional">) => {
if ("auth" in options && options.auth) return options.auth
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
.orElse(Auth.config("DEEPGRAM_API_KEY"))
.pipe(Auth.scheme("Token"))
}
export const configure = (input: Config = {}) => {
const media = (modelID: string | ModelID) => ({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
speech: (modelID: string | ModelID) => DeepgramSpeech.model(media(modelID)),
transcription: (modelID: string | ModelID) => DeepgramTranscription.model(media(modelID)),
configure,
}
}
export const provider = configure()
export const speech = provider.speech
export const transcription = provider.transcription
+41
View File
@@ -0,0 +1,41 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { DEFAULT_BASE_URL, ElevenLabsSpeech } from "../protocols/elevenlabs-speech.js"
export type { ElevenLabsOutputFormat, ElevenLabsSpeechOptions } from "../protocols/elevenlabs-speech.js"
export const id = ProviderID.make("elevenlabs")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
const auth = (options: ProviderAuthOption<"optional">) => {
if ("auth" in options && options.auth) return options.auth
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
.orElse(Auth.config("ELEVENLABS_API_KEY"))
.pipe(Auth.header("xi-api-key"))
}
export const configure = (input: Config = {}) => {
const speech = (modelID: string | ModelID) =>
ElevenLabsSpeech.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
speech,
configure,
}
}
export const provider = configure()
export const speech = provider.speech
+42
View File
@@ -0,0 +1,42 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { DEFAULT_BASE_URL, FalVideo } from "../protocols/fal-video.js"
export type { FalVideoOptions } from "../protocols/fal-video.js"
export const id = ProviderID.make("fal")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
// fal authenticates with `Authorization: Key <FAL_KEY>` rather than a bearer token.
const auth = (options: ProviderAuthOption<"optional">) => {
if ("auth" in options && options.auth) return options.auth
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
.orElse(Auth.config("FAL_KEY"))
.pipe(Auth.scheme("Key"))
}
export const configure = (input: Config = {}) => {
const video = (modelID: string | ModelID) =>
FalVideo.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
video,
configure,
}
}
export const provider = configure()
export const video = provider.video
+20 -9
View File
@@ -5,8 +5,14 @@ import type { ProviderPackage } from "../provider-package.js"
import { HttpOptions, ProviderID, mergeHttpOptions, type ModelID } from "../schema/index.js"
import { Gemini } from "../protocols/gemini.js"
import { GoogleImages } from "../protocols/google-images.js"
import { GoogleSpeech } from "../protocols/google-speech.js"
import { GoogleTranscription } from "../protocols/google-transcription.js"
import { GoogleVideo } from "../protocols/google-video.js"
export type { GoogleImageOptions } from "../protocols/google-images.js"
export type { GoogleSpeechOptions } from "../protocols/google-speech.js"
export type { GoogleTranscriptionOptions } from "../protocols/google-transcription.js"
export type { GoogleVideoOptions } from "../protocols/google-video.js"
export type GeminiOptionsInput = Gemini.OptionsInput
export type GeminiProviderOptionsInput = Gemini.ProviderOptionsInput
@@ -40,18 +46,20 @@ const configuredRoute = (input: Config) => {
export const configure = (input: Config = {}) => {
const route = configuredRoute(input)
const image = (modelID: string | ModelID) =>
GoogleImages.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL,
headers: input.headers,
http: mergeHttpOptions(input.http === undefined ? undefined : HttpOptions.make(input.http)),
})
const media = (modelID: string | ModelID) => ({
id: modelID,
auth: auth(input),
baseURL: input.baseURL,
headers: input.headers,
http: mergeHttpOptions(input.http === undefined ? undefined : HttpOptions.make(input.http)),
})
return {
id,
model: (modelID: string | ModelID) => route.model<Gemini.ProviderOptionsInput>({ id: modelID }),
image,
image: (modelID: string | ModelID) => GoogleImages.model(media(modelID)),
video: (modelID: string | ModelID) => GoogleVideo.model(media(modelID)),
speech: (modelID: string | ModelID) => GoogleSpeech.model(media(modelID)),
transcription: (modelID: string | ModelID) => GoogleTranscription.model(media(modelID)),
configure,
}
}
@@ -70,3 +78,6 @@ export const model: ProviderPackage.Definition<Settings, Gemini.ProviderOptionsI
}).model(modelID)
export const image = provider.image
export const video = provider.video
export const speech = provider.speech
export const transcription = provider.transcription
+6
View File
@@ -3,13 +3,18 @@ export * as Anthropic from "./anthropic.js"
export * as AnthropicCompatible from "./anthropic-compatible.js"
export * as AmazonBedrock from "./amazon-bedrock.js"
export * as AmazonBedrockMantle from "./amazon-bedrock-mantle.js"
export * as AssemblyAI from "./assemblyai.js"
export * as Azure from "./azure.js"
export * as Baseten from "./baseten.js"
export * as Cartesia from "./cartesia.js"
export * as Cerebras from "./cerebras.js"
export * as CloudflareAIGateway from "./cloudflare-ai-gateway.js"
export * as CloudflareWorkersAI from "./cloudflare-workers-ai.js"
export * as DeepInfra from "./deepinfra.js"
export * as Deepgram from "./deepgram.js"
export * as DeepSeek from "./deepseek.js"
export * as ElevenLabs from "./elevenlabs.js"
export * as Fal from "./fal.js"
export * as Fireworks from "./fireworks.js"
export * as Google from "./google.js"
export * as GoogleVertex from "./google-vertex.js"
@@ -26,6 +31,7 @@ export * as OpenAICompatible from "./openai-compatible.js"
export * as OpenAICompatibleResponses from "./openai-compatible-responses.js"
export * as OpenCodeZen from "./opencode-zen.js"
export * as OpenRouter from "./openrouter.js"
export * as Runway from "./runway.js"
export * as TogetherAI from "./togetherai.js"
export * as TypeSafeAI from "./typesafe-ai.js"
export * as VercelAIGateway from "./vercel-ai-gateway.js"
+21 -11
View File
@@ -6,9 +6,13 @@ import * as OpenAIChat from "../protocols/openai-chat.js"
import * as OpenAIResponses from "../protocols/openai-responses.js"
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options.js"
import { OpenAIImages, type OpenAIImageString } from "../protocols/openai-images.js"
import { OpenAISpeech } from "../protocols/openai-speech.js"
import { OpenAITranscription } from "../protocols/openai-transcription.js"
export type { OpenAIOptionsInput, OpenAIResponseIncludable } from "./openai-options.js"
export type { OpenAIImageOptions } from "../protocols/openai-images.js"
export type { OpenAISpeechOptions } from "../protocols/openai-speech.js"
export type { OpenAITranscriptionOptions } from "../protocols/openai-transcription.js"
export const id = ProviderID.make("openai")
@@ -95,17 +99,19 @@ export const configure = (input: Config = {}) => {
id,
compatibility: { supportsPromptCacheKey: true },
})
const image = (modelID: string | ModelID) =>
OpenAIImages.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL,
headers: input.headers,
http: mergeHttpOptions(
input.http === undefined ? undefined : HttpOptions.make(input.http),
input.queryParams === undefined ? undefined : new HttpOptions({ query: input.queryParams }),
),
})
const media = (modelID: string | ModelID) => ({
id: modelID,
auth: auth(input),
baseURL: input.baseURL,
headers: input.headers,
http: mergeHttpOptions(
input.http === undefined ? undefined : HttpOptions.make(input.http),
input.queryParams === undefined ? undefined : new HttpOptions({ query: input.queryParams }),
),
})
const image = (modelID: string | ModelID) => OpenAIImages.model(media(modelID))
const speech = (modelID: string | ModelID) => OpenAISpeech.model(media(modelID))
const transcription = (modelID: string | ModelID) => OpenAITranscription.model(media(modelID))
return {
id,
@@ -113,6 +119,8 @@ export const configure = (input: Config = {}) => {
responses,
chat,
image,
speech,
transcription,
configure,
}
}
@@ -159,3 +167,5 @@ export const chatModel: ProviderPackage.Definition<Settings, OpenAIProviderOptio
export const responses = provider.responses
export const chat = provider.chat
export const image = provider.image
export const speech = provider.speech
export const transcription = provider.transcription
+35
View File
@@ -0,0 +1,35 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { DEFAULT_BASE_URL, RunwayVideo } from "../protocols/runway-video.js"
export type { RunwayVideoOptions } from "../protocols/runway-video.js"
export const id = ProviderID.make("runway")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "RUNWAYML_API_SECRET")
export const configure = (input: Config = {}) => {
const video = (modelID: string | ModelID) =>
RunwayVideo.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
video,
configure,
}
}
export const provider = configure()
export const video = provider.video
+12 -9
View File
@@ -6,6 +6,7 @@ import { OpenAIChat } from "../protocols/openai-chat.js"
import { OpenResponsesChannel } from "../protocols/open-responses-channel.js"
import { XAIResponses } from "../protocols/xai-responses.js"
import { XAIImages } from "../protocols/xai-images.js"
import { XAIVideo } from "../protocols/xai-video.js"
import type { OpenAIOptionsInput } from "./openai-options.js"
import type { ProviderPackage } from "../provider-package.js"
@@ -27,6 +28,7 @@ export type Settings = ProviderPackage.Settings &
}
export type { XAIImageOptions } from "../protocols/xai-images.js"
export type { XAIVideoOptions } from "../protocols/xai-video.js"
const RESPONSES_WEBSOCKET_ROTATE_AFTER_MS = 24 * 60 * 1000
@@ -87,20 +89,20 @@ export const configure = (input: LanguageModelOptions = {}) => {
const chatRoute = configuredChatRoute(input)
const responses = (modelID: string | ModelID) => responsesRoute.model<XAIProviderOptionsInput>({ id: modelID })
const chat = (modelID: string | ModelID) => chatRoute.model<XAIProviderOptionsInput>({ id: modelID })
const image = (modelID: string | ModelID) =>
XAIImages.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
const media = (modelID: string | ModelID) => ({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
model: responses,
responses,
chat,
image,
image: (modelID: string | ModelID) => XAIImages.model(media(modelID)),
video: (modelID: string | ModelID) => XAIVideo.model(media(modelID)),
configure,
}
}
@@ -121,3 +123,4 @@ export const model: ProviderPackage.Definition<
export const responses = provider.responses
export const chat = provider.chat
export const image = provider.image
export const video = provider.video
+11 -1
View File
@@ -16,7 +16,7 @@ type Secret = string | Redacted.Redacted | Config.Config<string | Redacted.Redac
export interface AuthInput {
readonly request: { readonly http?: HttpOptions }
readonly method: "POST" | "GET"
readonly method: "POST" | "GET" | "PUT" | "DELETE"
readonly url: string
readonly body: string
readonly headers: Headers.Headers
@@ -134,6 +134,16 @@ export function bearerHeader(name: string, source?: Secret | Credential) {
return render(source)
}
/** `Authorization: <scheme> <secret>` for providers whose scheme is not `Bearer`, such as fal's `Key`. */
export function scheme(name: string): (source: Secret | Credential) => Definition
export function scheme(name: string, source: Secret | Credential): Definition
export function scheme(name: string, source?: Secret | Credential) {
const render = (input: Secret | Credential) =>
fromCredential(credentialInput(input), (secret) => ({ authorization: `${name} ${secret}` }))
if (source === undefined) return render
return render(source)
}
const toAIError = (error: AuthError): AIError => {
if (error instanceof MissingCredentialError || error instanceof Config.ConfigError) {
return new AIError({
+18 -1
View File
@@ -1,4 +1,4 @@
import type { Stream } from "effect"
import { Stream } from "effect"
import * as ProviderShared from "../protocols/shared.js"
import type { AIError } from "../schema/index.js"
@@ -12,6 +12,8 @@ import type { AIError } from "../schema/index.js"
* `[DONE]`; protocols that use it as a terminal select `sseWithDone`.
* - AWS event stream — length-prefixed binary frames with CRC checksums.
* Each emitted frame is one parsed binary event record.
* - Media streams — newline-delimited JSON (`lines`) or the whole body as one
* frame (`document`); chunked binary bodies need no framing.
*
* The frame type is opaque to this layer; the protocol's event schema decodes
* each frame before its state machine handles it.
@@ -38,4 +40,19 @@ export const sseEvents = (events: ReadonlySet<string>): Definition<string> => ({
frame: (bytes) => ProviderShared.sseFraming(bytes, events),
})
export const lines: Definition<string> = {
id: "lines",
frame: (bytes) =>
bytes.pipe(
Stream.decodeText(),
Stream.splitLines,
Stream.filter((line) => line.trim().length > 0),
),
}
export const document: Definition<string> = {
id: "document",
frame: (bytes) => Stream.fromEffect(Stream.mkString(bytes.pipe(Stream.decodeText()))),
}
export * as Framing from "./framing.js"
+207 -7
View File
@@ -1,17 +1,52 @@
import { Effect, Schema } from "effect"
import { Clock, Duration, Effect, Schema, type Stream } from "effect"
import { HttpClientResponse } from "effect/unstable/http"
import { AIError, HttpContext, InvalidProviderOutputError } from "../schema/index.js"
import type { Snapshot, Status } from "../generation.js"
import { Media } from "../media.js"
import type { AuthInput } from "./auth.js"
import {
AIError,
ContentPolicyError,
HttpContext,
InvalidProviderOutputError,
InvalidRequestError,
ProviderInternalError,
} from "../schema/index.js"
// ---------------------------------------------------------------------------
// Bodies
// ---------------------------------------------------------------------------
export type Body =
| { readonly type: "json"; readonly value: Record<string, unknown> }
| { readonly type: "multipart"; readonly value: FormData }
/** Array values become repeated parameters (`keyterm=a&keyterm=b`). */
export type Query = Readonly<Record<string, string | ReadonlyArray<string>>>
export const json = (value: Record<string, unknown>): Body => ({ type: "json", value })
/** `query` is appended to the endpoint URL before the route and caller `http.query` overlays. */
export type Body =
| { readonly type: "json"; readonly value: Record<string, unknown>; readonly query?: Query }
| { readonly type: "multipart"; readonly value: FormData }
| {
readonly type: "binary"
readonly value: Uint8Array
readonly contentType: string
readonly query?: Query
}
export const json = (value: Record<string, unknown>, query?: Query): Body => ({
type: "json",
value,
query,
})
export const multipart = (value: FormData): Body => ({ type: "multipart", value })
export const binary = (value: Uint8Array, contentType: string, query?: Query): Body => ({
type: "binary",
value,
contentType,
query,
})
export type Send = (path: string, body: Body) => Effect.Effect<HttpClientResponse.HttpClientResponse, AIError>
/** Runs after unsupported-field rejection and before `body.from`, for providers that need an upload first. */
export type Prepare<Request> = (request: Request, send: Send) => Effect.Effect<Request, AIError>
// ---------------------------------------------------------------------------
// Protocol kinds
@@ -45,6 +80,107 @@ export const inline = <Request, Response>(
...input,
})
/** What `start` learned from the submission response: the route-owned handle plus the first observation. */
export interface Started<Token> {
readonly token: Token
readonly snapshot: Snapshot
}
/**
* A follow-up call's inputs: the decoded token and the auth headers the route sent, so a protocol can attach them
* to output URLs that require the same credentials to download (Veo).
*/
export interface PollContext<Token> {
readonly token: Token
readonly auth: Record<string, string>
}
/**
* Submit, then poll. `start` posts the body to the route endpoint; `status`, `result`, and `cancel` are follow-up
* calls addressed by the token. Paths are relative to the route base URL unless the provider hands back absolute
* URLs (fal `status_url`), in which case they are used verbatim. `result` is always its own GET: providers that
* return the output inside the status body (Veo, xAI, Runway) point `result.path` at the status path and decode the
* same document, so `Generation.await` and `Video.resume(...).await()` behave identically everywhere.
*/
export interface Queued<Request, Response, Token> {
readonly kind: "queued"
readonly id: string
readonly name: string
/** Common request fields this protocol cannot lower; the route rejects them before `start.body.from` runs. */
readonly unsupported?: ReadonlyArray<keyof Request & string>
/** Serializable handle. `Generation.token` carries the encoded form so it can be persisted and resumed elsewhere. */
readonly token: Schema.Codec<Token, unknown>
readonly start: {
readonly prepare?: Prepare<Request>
readonly body: { readonly from: (request: Request) => Effect.Effect<Body, AIError> }
readonly decode: (
response: HttpClientResponse.HttpClientResponse,
context: DecodeContext<Request>,
) => Effect.Effect<Started<Token>, AIError>
}
readonly status: {
readonly path: (token: Token) => string
readonly decode: (
response: HttpClientResponse.HttpClientResponse,
context: PollContext<Token>,
) => Effect.Effect<Snapshot, AIError>
}
readonly result: {
readonly path: (token: Token) => string
readonly decode: (
response: HttpClientResponse.HttpClientResponse,
context: PollContext<Token>,
) => Effect.Effect<Response, AIError>
}
readonly cancel?: {
readonly method: AuthInput["method"]
readonly path: (token: Token) => string
}
}
export const queued = <Request, Response, Token>(
input: Omit<Queued<Request, Response, Token>, "kind">,
): Queued<Request, Response, Token> => ({
kind: "queued",
...input,
})
export type Mode = "generate" | "stream"
export type Addressed<Request> = Request & { readonly mode: Mode }
export interface ResponseContext<Request> extends DecodeContext<Addressed<Request>> {
readonly http: HttpContext
}
/**
* One request whose body is parsed incrementally, like LLM protocols: `frames` → `step`* → `finish`. `generate` and
* `stream` share this state machine; `request.mode` lets a protocol pick a different body, path, or framing.
*/
export interface Streamed<Request, Event, Frame, State> {
readonly kind: "stream"
readonly id: string
readonly name: string
/** Common request fields this protocol cannot lower; the route rejects them before `body.from` runs. */
readonly unsupported?: ReadonlyArray<keyof Request & string>
readonly body: { readonly from: (request: Addressed<Request>) => Effect.Effect<Body, AIError> }
readonly frames: (
bytes: Stream.Stream<Uint8Array, AIError>,
context: DecodeContext<Addressed<Request>>,
) => Stream.Stream<Frame, AIError>
readonly initial: () => State
readonly step: (state: State, frame: Frame) => Effect.Effect<readonly [State, ReadonlyArray<Event>], AIError>
/** Emit exactly one terminal event, or fail when the provider stopped before completing. */
readonly finish: (state: State, context: ResponseContext<Request>) => Effect.Effect<ReadonlyArray<Event>, AIError>
}
export const stream = <Request, Event, Frame, State>(
input: Omit<Streamed<Request, Event, Frame, State>, "kind">,
): Streamed<Request, Event, Frame, State> => ({
kind: "stream",
...input,
})
// ---------------------------------------------------------------------------
// Response helpers
// ---------------------------------------------------------------------------
@@ -52,7 +188,11 @@ export const inline = <Request, Response>(
const context = (response: HttpClientResponse.HttpClientResponse) =>
new HttpContext({ url: response.request.url, status: response.status, headers: response.headers })
/** Read a text body while retaining the original payload and HTTP context on every downstream error. */
/**
* Read a text body while retaining the original payload and HTTP context on every downstream error. `invalid` is a
* malformed provider document; `ended` is a generation that reached a terminal status without output (`failed` is
* provider-side, `cancelled`/`expired` mean the result will never exist); `contentPolicy` is a moderated result.
*/
export const text = Effect.fn("MediaProtocol.text")(function* (
route: string,
name: string,
@@ -77,9 +217,19 @@ export const text = Effect.fn("MediaProtocol.text")(function* (
http,
invalid: (message: string, cause?: unknown) =>
new AIError({ reason: new InvalidProviderOutputError({ route, message, body, http, cause }) }),
ended: (status: Exclude<Status, "queued" | "running" | "completed">, message: string) =>
new AIError({
reason:
status === "failed"
? new ProviderInternalError({ message, body, http })
: new InvalidRequestError({ message, body, http }),
}),
contentPolicy: (message: string) => new AIError({ reason: new ContentPolicyError({ message, body, http }) }),
}
})
export type Output = Effect.Success<ReturnType<typeof text>>
/** Read and Schema-decode a JSON body. Decode failures keep the raw body as `reason.body`. */
export const decodeJson = <A>(route: string, name: string, schema: Schema.Codec<A, unknown>) => {
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
@@ -92,4 +242,54 @@ export const decodeJson = <A>(route: string, name: string, schema: Schema.Codec<
})
}
/** Decode a submission response into the token and first snapshot. */
export const decodeStarted = <A, Token>(
route: string,
name: string,
schema: Schema.Codec<A, unknown>,
started: (value: A) => Started<Token>,
) => {
const decode = decodeJson(route, name, schema)
return (response: HttpClientResponse.HttpClientResponse) =>
decode(response).pipe(Effect.map((output) => started(output.value)))
}
/** Map a provider status string through the protocol's table; unknown values are an invalid provider document. */
export const status = <Table extends Record<string, Status>>(
table: Table,
raw: string,
output: Output,
): Effect.Effect<Status, AIError> => {
const normalized: Status | undefined = table[raw]
if (normalized === undefined) return Effect.fail(output.invalid(`Unknown generation status "${raw}"`))
return Effect.succeed(normalized)
}
export const frameError = (route: string, message: string, body?: string, cause?: unknown) =>
new AIError({ reason: new InvalidProviderOutputError({ route, message, body, cause }) })
export const incomplete = (route: string) =>
new AIError({
reason: new InvalidProviderOutputError({
route,
message: "The provider response ended unexpectedly.",
classification: "incomplete-stream",
}),
})
/** Schema-decode one JSON stream frame. Decode failures keep the frame as `reason.body`. */
export const decodeFrame = <A>(route: string, name: string, schema: Schema.Codec<A, unknown>) => {
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
return (frame: string) =>
decode(frame).pipe(
Effect.mapError((cause) => frameError(route, `${name} sent an invalid stream event`, frame, cause)),
)
}
/** A `url` asset whose provider-declared retention window starts now. */
export const expiringUrl = (url: string, retention: Duration.Duration, options?: Parameters<typeof Media.url>[1]) =>
Clock.currentTimeMillis.pipe(
Effect.map((now) => Media.url(url, { ...options, expiresAt: now + Duration.toMillis(retention) })),
)
export * as MediaProtocol from "./media-protocol.js"
+305 -36
View File
@@ -1,11 +1,20 @@
import { Effect } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { Auth } from "./auth.js"
import { Effect, Schema, Stream } from "effect"
import { Headers, HttpClientRequest, type HttpClientResponse } from "effect/unstable/http"
import { Auth, type AuthInput } from "./auth.js"
import { Endpoint } from "./endpoint.js"
import type { Interface } from "./executor-service.js"
import { RequestExecutor } from "./executor.js"
import { MediaProtocol } from "./media-protocol.js"
import { Generation, type Route as GenerationRoute } from "../generation.js"
import { ProviderShared } from "../protocols/shared.js"
import { AIError, HttpOptions, ProviderID, mergeHttpOptions } from "../schema/index.js"
import {
AIError,
AIErrorReason,
HttpOptions,
InvalidRequestError,
ProviderID,
mergeHttpOptions,
} from "../schema/index.js"
import { sanitizeSurrogates } from "../utils/sanitize.js"
export type Execute = Interface["execute"]
@@ -26,68 +35,328 @@ export interface ModelInput {
}
// ---------------------------------------------------------------------------
// Route
// Routes
// ---------------------------------------------------------------------------
/** One request, one response. */
export interface Route<Request extends MediaRequest, Response> {
readonly kind: "inline"
readonly id: string
readonly provider: ProviderID
readonly protocol: string
readonly generate: (request: Request, execute: Execute) => Effect.Effect<Response, AIError>
}
export interface MakeInput<Request extends MediaRequest, Response> {
/** Submit, then poll through the returned `Generation`. */
export interface QueuedRoute<Request extends MediaRequest, Response> {
readonly kind: "queued"
readonly id: string
readonly provider: ProviderID
readonly protocol: string
readonly start: (request: Request, execute: Execute) => Effect.Effect<Generation<Response>, AIError>
/** Rebuild a handle from a persisted `Generation.token`; fails typed when the token is not this route's. */
readonly resume: (
model: MediaRequest["model"],
token: unknown,
execute: Execute,
) => Effect.Effect<Generation<Response>, AIError>
}
/** One request whose response parses into events; `generate` runs the same stream and collects it. */
export interface StreamRoute<Request extends MediaRequest, Event, Response> {
readonly kind: "stream"
readonly id: string
readonly provider: ProviderID
readonly protocol: string
readonly stream: (request: Request, execute: Execute) => Stream.Stream<Event, AIError>
readonly generate: (request: Request, execute: Execute) => Effect.Effect<Response, AIError>
}
export interface Composition<Request extends MediaRequest> {
readonly id: string
readonly provider: string | ProviderID
readonly protocol: MediaProtocol.Inline<Request, Response>
readonly endpoint: Endpoint.Definition<MediaProtocol.Body, Request>
readonly auth: Auth.Definition
/** Deployment headers applied before transport authentication. */
readonly headers?: Record<string, string>
}
export interface InlineInput<Request extends MediaRequest, Response> extends Composition<Request> {
readonly protocol: MediaProtocol.Inline<Request, Response>
}
export interface QueuedInput<Request extends MediaRequest, Response, Token> extends Composition<Request> {
readonly protocol: MediaProtocol.Queued<Request, Response, Token>
}
export interface StreamInput<Request extends MediaRequest, Event, Response, Frame, State>
extends Composition<MediaProtocol.Addressed<Request>> {
readonly protocol: MediaProtocol.Streamed<Request, Event, Frame, State>
readonly collect: (events: ReadonlyArray<Event>) => Effect.Effect<Response, AIError>
}
/**
* Compose an inline media protocol with an endpoint and auth into a runnable route. The route owns the transport
* plumbing every media protocol would otherwise duplicate: option merging, surrogate sanitizing, unsupported-field rejection, URL and query
* rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol for decoding.
* plumbing every media protocol would otherwise duplicate: option merging, surrogate sanitizing, unsupported-field
* rejection, URL and query rendering, auth headers, JSON, multipart, or binary encoding, and handing responses back
* to the protocol.
*/
export const make = <Request extends MediaRequest, Response>(
input: MakeInput<Request, Response>,
export const inline = <Request extends MediaRequest, Response>(
input: InlineInput<Request, Response>,
): Route<Request, Response> => {
const transport = makeTransport(input)
return {
kind: "inline",
id: input.id,
provider: transport.provider,
protocol: input.protocol.id,
generate: Effect.fn(`MediaRoute.generate`)(function* (request: Request, execute: Execute) {
const submitted = yield* transport.submit(
request,
{ unsupported: input.protocol.unsupported, from: input.protocol.body.from },
execute,
)
return yield* input.protocol.response.decode(submitted.response, submitted.context)
}),
}
}
/**
* Compose a queued media protocol the same way, adding `start`/`resume` handles whose polls reuse the route's auth,
* deployment headers, and (for `start`) the request's `http` overlay. The token is decoded once at the boundary and
* closed over by the resulting `Generation.Route`.
*/
export const queued = <Request extends MediaRequest, Response, Token>(
input: QueuedInput<Request, Response, Token>,
): QueuedRoute<Request, Response> => {
const transport = makeTransport(input)
const protocol = input.protocol
const decodeToken = Schema.decodeUnknownEffect(protocol.token)
// A protocol producing a token its own codec rejects is a programmer defect, not a provider error.
const encodeToken = Schema.encodeSync(protocol.token)
const generationRoute = (token: Token, http: HttpOptions | undefined, execute: Execute) => {
const poll = <A>(operation: {
readonly path: (token: Token) => string
readonly decode: (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) => Effect.Effect<A, AIError>
}) =>
transport
.call("GET", operation.path(token), http, execute)
.pipe(Effect.flatMap((sent) => operation.decode(sent.response, { token, auth: sent.auth })))
const cancel = protocol.cancel
const route: GenerationRoute<Response> = {
status: poll(protocol.status),
result: poll(protocol.result),
cancel:
cancel === undefined
? undefined
: transport.call(cancel.method, cancel.path(token), http, execute).pipe(Effect.asVoid),
}
return route
}
const start = Effect.fn("MediaRoute.start")(function* (request: Request, execute: Execute) {
const submitted = yield* transport.submit(
request,
{ unsupported: protocol.unsupported, prepare: protocol.start.prepare, from: protocol.start.body.from },
execute,
)
const started = yield* protocol.start.decode(submitted.response, submitted.context)
const route = generationRoute(started.token, submitted.context.request.http, execute)
return new Generation(route, encodeToken(started.token), started.snapshot)
})
const resume = Effect.fn("MediaRoute.resume")(function* (
model: MediaRequest["model"],
raw: unknown,
execute: Execute,
) {
const token = yield* decodeToken(raw).pipe(
Effect.mapError(
(cause) =>
new AIError({
reason: new InvalidRequestError({
message: `${input.id} cannot resume a generation from this token`,
cause,
}),
}),
),
)
const route = generationRoute(token, transport.http(model), execute)
return new Generation(route, encodeToken(token), yield* route.status)
})
return { kind: "queued", id: input.id, provider: transport.provider, protocol: protocol.id, start, resume }
}
/** Compose a streaming media protocol; `generate` runs the same stream in `generate` mode and folds it with `collect`. */
export const stream = <Request extends MediaRequest, Event, Response, Frame, State>(
input: StreamInput<Request, Event, Response, Frame, State>,
): StreamRoute<Request, Event, Response> => {
const transport = makeTransport(input)
const protocol = input.protocol
const events = (request: Request, execute: Execute, mode: MediaProtocol.Mode) =>
Stream.unwrap(
Effect.gen(function* () {
const submitted = yield* transport.submit(
{ ...request, mode },
{ unsupported: protocol.unsupported, from: protocol.body.from },
execute,
)
const http = RequestExecutor.responseHttp(submitted.response)
return Stream.suspend(() => {
// Parser state is local to one response, exactly like `Route.make`'s LLM stream loop.
let state = protocol.initial()
return protocol.frames(RequestExecutor.responseStream(submitted.response), submitted.context).pipe(
Stream.mapEffect((frame) =>
protocol.step(state, frame).pipe(
Effect.map(([next, output]) => {
state = next
return output
}),
),
),
Stream.flattenIterable,
Stream.concat(
Stream.suspend(() => Stream.fromIterableEffect(protocol.finish(state, { ...submitted.context, http }))),
),
Stream.mapError((error) =>
error.reason.http !== undefined
? error
: new AIError({
reason: AIErrorReason.make({
...error.reason,
message: error.reason.message,
cause: error.reason.cause,
http,
}),
}),
),
)
})
}),
)
return {
kind: "stream",
id: input.id,
provider: transport.provider,
protocol: protocol.id,
stream: (request, execute) => events(request, execute, "stream"),
generate: (request, execute) =>
events(request, execute, "generate").pipe(Stream.runCollect, Effect.flatMap(input.collect)),
}
}
// ---------------------------------------------------------------------------
// Transport plumbing shared by every kind
// ---------------------------------------------------------------------------
const makeTransport = <Request extends MediaRequest>(input: Composition<Request>) => {
const provider = ProviderID.make(input.provider)
const routeHttp = input.headers === undefined ? undefined : new HttpOptions({ headers: input.headers })
const authorize = Auth.toEffect(input.auth)
const baseURL = (path: string) => new URL(`${ProviderShared.trimBaseUrl(input.endpoint.baseURL ?? "")}${path}`)
/** `auth` is only what `Auth` added, never deployment headers. */
const send = Effect.fn("MediaRoute.send")(function* (
call: {
readonly method: AuthInput["method"]
readonly url: URL
readonly headers: Headers.Headers
readonly request: AuthInput["request"]
readonly body?: MediaProtocol.Body
},
execute: Execute,
) {
const encoded = encode(call.body, call.headers)
const url = call.url.toString()
const headers = yield* authorize({
request: call.request,
method: call.method,
url,
body: encoded.text,
headers: encoded.headers,
})
const response = yield* execute(
encoded.apply(HttpClientRequest.make(call.method)(url).pipe(HttpClientRequest.setHeaders(headers))),
)
return { response, auth: Object.fromEntries(Object.entries(headers).filter(([key]) => !(key in call.headers))) }
})
return {
id: input.id,
provider,
protocol: input.protocol.id,
generate: Effect.fn(`MediaRoute.generate`)(function* (request: Request, execute: Execute) {
yield* rejectUnsupported(input.id, provider, request, input.protocol.unsupported)
/** Route and model overlays; `start` additionally merges the request's own `http`. */
http: (model: MediaRequest["model"]) => mergeHttpOptions(routeHttp, model.http),
/** POST the protocol body to the route endpoint. */
submit: Effect.fn("MediaRoute.submit")(function* (
request: Request,
protocol: {
readonly unsupported?: ReadonlyArray<keyof Request & string>
readonly prepare?: MediaProtocol.Prepare<Request>
readonly from: (request: Request) => Effect.Effect<MediaProtocol.Body, AIError>
},
execute: Execute,
) {
yield* rejectUnsupported(input.id, provider, request, protocol.unsupported)
const http = mergeHttpOptions(routeHttp, request.model.http, request.http)
const headers = Headers.fromInput(http?.headers)
const prepared =
protocol.prepare === undefined
? request
: yield* protocol.prepare(request, (path, body) =>
send({ method: "POST", url: baseURL(path), headers, request, body }, execute).pipe(
Effect.map((sent) => sent.response),
),
)
// Sanitize after merging so model-level overlays are covered; the model value is restored, not sanitized.
const resolved: Request = { ...sanitizeSurrogates({ ...request, http }), model: request.model }
const body = yield* input.protocol.body.from(resolved)
const url = Endpoint.render(input.endpoint, { request: resolved, body })
for (const [key, value] of Object.entries(http?.query ?? {})) url.searchParams.set(key, value)
const encoded = body.type === "json" ? ProviderShared.encodeJson(body.value) : "[multipart/form-data]"
const baseHeaders = Headers.fromInput(http?.headers)
const headers = yield* authorize({
request: resolved,
method: "POST",
url: url.toString(),
body: encoded,
// The HTTP client sets the multipart boundary; a caller-supplied content-type would corrupt it.
headers: body.type === "multipart" ? Headers.remove(baseHeaders, "content-type") : baseHeaders,
})
const transport = HttpClientRequest.post(url.toString()).pipe(
HttpClientRequest.setHeaders(headers),
body.type === "json"
? HttpClientRequest.bodyText(encoded, "application/json")
: HttpClientRequest.bodyFormData(body.value),
const resolved: Request = { ...sanitizeSurrogates({ ...prepared, http }), model: request.model }
const body = yield* protocol.from(resolved)
const url = withQuery(
withQuery(
Endpoint.render(input.endpoint, { request: resolved, body }),
body.type === "multipart" ? undefined : body.query,
),
http?.query,
)
const response = yield* execute(transport)
return yield* input.protocol.response.decode(response, { request: resolved, body })
const sent = yield* send({ method: "POST", url, headers, request: resolved, body }, execute)
return { response: sent.response, context: { request: resolved, body } }
}),
/** Bodiless follow-up call (status, result, cancel) with the same auth and headers as `submit`. */
call: (method: AuthInput["method"], path: string, http: HttpOptions | undefined, execute: Execute) => {
// Provider-issued absolute URLs (fal `status_url`) are used as-is; everything else resolves against the base.
const url = withQuery(/^https?:\/\//.test(path) ? new URL(path) : baseURL(path), http?.query)
for (const [key, value] of Object.entries(input.endpoint.query ?? {})) url.searchParams.set(key, value)
return send({ method, url, headers: Headers.fromInput(http?.headers), request: { http } }, execute)
},
}
}
const withQuery = (url: URL, query: MediaProtocol.Query | undefined) => {
for (const [key, value] of Object.entries(query ?? {})) {
url.searchParams.delete(key)
for (const item of typeof value === "string" ? [value] : value) url.searchParams.append(key, item)
}
return url
}
const encode = (body: MediaProtocol.Body | undefined, headers: Headers.Headers) => {
if (body === undefined) return { text: "", headers, apply: (request: HttpClientRequest.HttpClientRequest) => request }
if (body.type === "json") {
const text = ProviderShared.encodeJson(body.value)
return { text, headers, apply: HttpClientRequest.bodyText(text, "application/json") }
}
if (body.type === "binary")
return {
text: `[${body.contentType}]`,
headers,
apply: HttpClientRequest.bodyUint8Array(body.value, body.contentType),
}
return {
text: "[multipart/form-data]",
// The HTTP client sets the multipart boundary; a caller-supplied content-type would corrupt it.
headers: Headers.remove(headers, "content-type"),
apply: HttpClientRequest.bodyFormData(body.value),
}
}
+51
View File
@@ -0,0 +1,51 @@
import { Context, Effect, Layer, Stream } from "effect"
import { RequestExecutor } from "./route/executor.js"
import type { AIError } from "./schema/index.js"
import type { SpeechEvent, SpeechOptions, SpeechRequestFor, SpeechResponse } from "./speech.js"
export interface Interface {
readonly generate: <Options extends SpeechOptions>(
request: SpeechRequestFor<Options>,
) => Effect.Effect<SpeechResponse, AIError>
readonly stream: <Options extends SpeechOptions>(
request: SpeechRequestFor<Options>,
) => Stream.Stream<SpeechEvent, AIError>
}
export class Service extends Context.Service<Service, Interface>()("@opencode/SpeechClient") {}
export const generate = <Options extends SpeechOptions>(
request: SpeechRequestFor<Options>,
): Effect.Effect<SpeechResponse, AIError, Service> =>
Effect.gen(function* () {
const client = yield* Service
return yield* client.generate(request)
})
export const stream = <Options extends SpeechOptions>(
request: SpeechRequestFor<Options>,
): Stream.Stream<SpeechEvent, AIError, Service> =>
Stream.unwrap(
Effect.gen(function* () {
const client = yield* Service
return client.stream(request)
}),
)
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
Service,
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
return Service.of({
generate: (request) => request.model.route.generate(request, executor.execute),
stream: (request) => request.model.route.stream(request, executor.execute),
})
}),
)
export const SpeechClient = {
Service,
layer,
generate,
stream,
} as const
+217
View File
@@ -0,0 +1,217 @@
import { Effect, Schema, Stream } from "effect"
import { Media } from "./media.js"
import { MediaModel, composeRoute, tryRequest } from "./media-model.js"
import { MediaRoute } from "./route/media.js"
import type { MediaProtocol } from "./route/media-protocol.js"
import { AIError, HttpOptions, MediaUsage, ProviderMetadata } from "./schema/index.js"
import { SpeechClient, Service } from "./speech-client.js"
// ---------------------------------------------------------------------------
// Model
// ---------------------------------------------------------------------------
export type SpeechOptions = Record<string, unknown>
export type SpeechRoute<Options extends SpeechOptions = SpeechOptions> = MediaRoute.StreamRoute<
SpeechRequestFor<Options>,
SpeechEvent,
SpeechResponse
>
export class SpeechModel<Options extends SpeechOptions = SpeechOptions> extends MediaModel<
SpeechRoute<Options>,
Options
> {
declare protected readonly _SpeechModel: void
static make<Options extends SpeechOptions = SpeechOptions>(input: MediaModel.Input<SpeechRoute<Options>>) {
return new SpeechModel<Options>(input)
}
/** Compose a streaming speech protocol with its canonical path into a model for one deployment. */
static fromRoute<Options extends SpeechOptions = SpeechOptions, Frame = unknown, State = unknown>(
route: SpeechModel.RouteInput<Options, Frame, State>,
input: MediaRoute.ModelInput,
) {
return new SpeechModel<Options>({
id: input.id,
provider: route.provider,
http: input.http,
route: composeRoute(
(composition) => MediaRoute.stream({ ...composition, collect: collectResponse }),
route,
input,
),
})
}
}
export namespace SpeechModel {
export type RouteInput<
Options extends SpeechOptions = SpeechOptions,
Frame = unknown,
State = unknown,
> = MediaModel.RouteInput<
MediaProtocol.Addressed<SpeechRequestFor<Options>>,
MediaProtocol.Streamed<SpeechRequestFor<Options>, SpeechEvent, Frame, State>
>
}
export const SpeechModelSchema = Schema.declare((value): value is SpeechModel => value instanceof SpeechModel, {
expected: "Speech.Model",
})
// ---------------------------------------------------------------------------
// Request
// ---------------------------------------------------------------------------
/** Provider-native: a name on OpenAI and Gemini, a voice id on ElevenLabs and Cartesia; `{ id }` is an OpenAI custom voice. */
export const SpeechVoice = Schema.Union([Schema.String, Schema.Struct({ id: Schema.String })]).annotate({
identifier: "Speech.Voice",
})
export type SpeechVoice = Schema.Schema.Type<typeof SpeechVoice>
export type SpeechFormat = "mp3" | "wav" | "pcm" | "opus" | "aac" | "flac" | (string & {})
/** Granularity is provider-native: characters on ElevenLabs, words on Cartesia. */
export const SpeechTimestamp = Schema.Struct({
text: Schema.String,
startSeconds: Schema.Number,
endSeconds: Schema.Number,
}).annotate({ identifier: "Speech.Timestamp" })
export type SpeechTimestamp = Schema.Schema.Type<typeof SpeechTimestamp>
export class SpeechRequest extends Schema.Class<SpeechRequest>("Speech.Request")({
model: SpeechModelSchema,
text: Schema.String,
voice: Schema.optional(SpeechVoice),
format: Schema.optional(Schema.String),
speed: Schema.optional(Schema.Number),
language: Schema.optional(Schema.String),
instructions: Schema.optional(Schema.String),
timestamps: Schema.optional(Schema.Boolean),
providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
http: Schema.optional(HttpOptions),
}) {
declare protected readonly _SpeechRequest: void
}
export type SpeechRequestFor<Options extends SpeechOptions = SpeechOptions> = Omit<
SpeechRequest,
"model" | "providerOptions"
> & {
readonly model: SpeechModel<Options>
readonly providerOptions?: Options
}
export type SpeechModelOptions<Model> = Model extends SpeechModel<infer Options> ? Options : never
export type SpeechRequestInput<Model extends SpeechModel = SpeechModel> = Omit<
ConstructorParameters<typeof SpeechRequest>[0],
"model" | "providerOptions" | "http" | "format"
> & {
readonly model: Model
readonly format?: SpeechFormat
readonly providerOptions?: NoInfer<SpeechModelOptions<Model>>
readonly http?: HttpOptions.Input
}
// ---------------------------------------------------------------------------
// Response and events
// ---------------------------------------------------------------------------
export class SpeechResponse extends Schema.Class<SpeechResponse>("Speech.Response")({
/** The complete audio. Headerless PCM carries `info.encoding`, `info.sampleRate`, and `info.channels`. */
audio: Media.AssetSchema,
timestamps: Schema.optional(Schema.Array(SpeechTimestamp)),
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}) {}
export const SpeechAudioDeltaEvent = Schema.Struct({
type: Schema.tag("audio-delta"),
chunk: Schema.Uint8Array,
}).annotate({ identifier: "Speech.Event.AudioDelta" })
export const SpeechTimestampsEvent = Schema.Struct({
type: Schema.tag("timestamps"),
items: Schema.Array(SpeechTimestamp),
}).annotate({ identifier: "Speech.Event.Timestamps" })
/** `audio` is every `audio-delta` chunk concatenated, so the route holds the whole clip in memory until `finish`. */
export const SpeechFinishEvent = Schema.Struct({
type: Schema.tag("finish"),
audio: Media.AssetSchema,
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "Speech.Event.Finish" })
const speechEventTagged = Schema.Union([SpeechAudioDeltaEvent, SpeechTimestampsEvent, SpeechFinishEvent]).pipe(
Schema.toTaggedUnion("type"),
)
export const SpeechEvent = Object.assign(speechEventTagged, {
is: {
audioDelta: speechEventTagged.guards["audio-delta"],
timestamps: speechEventTagged.guards.timestamps,
finish: speechEventTagged.guards.finish,
},
})
export type SpeechEvent = Schema.Schema.Type<typeof speechEventTagged>
const collectResponse = (events: ReadonlyArray<SpeechEvent>): Effect.Effect<SpeechResponse> => {
const finish = events.find(SpeechEvent.is.finish)
// Every speech protocol's `finish` emits the terminal event or fails, so a completed stream always has one.
if (finish === undefined) return Effect.die(new Error("The speech stream completed without a finish event"))
const timestamps = events.filter(SpeechEvent.is.timestamps).flatMap((event) => event.items)
return Effect.succeed(
new SpeechResponse({
audio: finish.audio,
timestamps: timestamps.length === 0 ? undefined : timestamps,
usage: finish.usage,
notices: finish.notices,
providerMetadata: finish.providerMetadata,
}),
)
}
// ---------------------------------------------------------------------------
// Request-shaped call API
// ---------------------------------------------------------------------------
export function request<const Model extends SpeechModel>(
input: SpeechRequestInput<Model>,
): SpeechRequestFor<SpeechModelOptions<Model>>
export function request(input: SpeechRequest): SpeechRequest
export function request(input: SpeechRequest | SpeechRequestInput) {
if (input instanceof SpeechRequest) return input
return new SpeechRequest({
...input,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}
const requestEffect = (input: SpeechRequest | SpeechRequestInput) => tryRequest(() => request(input))
export function generate<const Model extends SpeechModel>(
input: SpeechRequestInput<Model>,
): Effect.Effect<SpeechResponse, AIError, Service>
export function generate(input: SpeechRequest): Effect.Effect<SpeechResponse, AIError, Service>
export function generate(input: SpeechRequest | SpeechRequestInput) {
return requestEffect(input).pipe(Effect.flatMap((request) => SpeechClient.generate(request)))
}
export function stream<const Model extends SpeechModel>(
input: SpeechRequestInput<Model>,
): Stream.Stream<SpeechEvent, AIError, Service>
export function stream(input: SpeechRequest): Stream.Stream<SpeechEvent, AIError, Service>
export function stream(input: SpeechRequest | SpeechRequestInput) {
return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => SpeechClient.stream(request))))
}
export const Speech = {
request,
generate,
stream,
} as const
+122
View File
@@ -0,0 +1,122 @@
import { Context, Effect, Layer, Stream } from "effect"
import { resultEvents, type AwaitOptions, type Generation } from "./generation.js"
import { ProviderShared } from "./protocols/shared.js"
import { RequestExecutor } from "./route/executor.js"
import type { AIError } from "./schema/index.js"
import {
responseEvents,
type TranscriptionEvent,
type TranscriptionModel,
type TranscriptionOptions,
type TranscriptionRequestFor,
type TranscriptionResponse,
type TranscriptionRoute,
} from "./transcription.js"
export interface Interface {
readonly generate: <Options extends TranscriptionOptions>(
request: TranscriptionRequestFor<Options>,
options?: AwaitOptions,
) => Effect.Effect<TranscriptionResponse, AIError>
readonly stream: <Options extends TranscriptionOptions>(
request: TranscriptionRequestFor<Options>,
options?: AwaitOptions,
) => Stream.Stream<TranscriptionEvent, AIError>
readonly start: <Options extends TranscriptionOptions>(
request: TranscriptionRequestFor<Options>,
) => Effect.Effect<Generation<TranscriptionResponse>, AIError>
readonly resume: <Options extends TranscriptionOptions>(
model: TranscriptionModel<Options>,
token: unknown,
) => Effect.Effect<Generation<TranscriptionResponse>, AIError>
}
export class Service extends Context.Service<Service, Interface>()("@opencode/TranscriptionClient") {}
export const generate = <Options extends TranscriptionOptions>(
request: TranscriptionRequestFor<Options>,
options?: AwaitOptions,
): Effect.Effect<TranscriptionResponse, AIError, Service> =>
Effect.gen(function* () {
const client = yield* Service
return yield* client.generate(request, options)
})
export const stream = <Options extends TranscriptionOptions>(
request: TranscriptionRequestFor<Options>,
options?: AwaitOptions,
): Stream.Stream<TranscriptionEvent, AIError, Service> =>
Stream.unwrap(
Effect.gen(function* () {
const client = yield* Service
return client.stream(request, options)
}),
)
export const start = <Options extends TranscriptionOptions>(
request: TranscriptionRequestFor<Options>,
): Effect.Effect<Generation<TranscriptionResponse>, AIError, Service> =>
Effect.gen(function* () {
const client = yield* Service
return yield* client.start(request)
})
export const resume = <Options extends TranscriptionOptions>(
model: TranscriptionModel<Options>,
token: unknown,
): Effect.Effect<Generation<TranscriptionResponse>, AIError, Service> =>
Effect.gen(function* () {
const client = yield* Service
return yield* client.resume(model, token)
})
const notQueued = <Options extends TranscriptionOptions>(route: TranscriptionRoute<Options>, operation: string) =>
ProviderShared.unsupportedOperation({
operation: `transcription.${operation}`,
provider: route.provider,
route: route.id,
message: `${route.provider}/${route.id} is not a queued route; use Transcription.generate or Transcription.stream`,
})
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
Service,
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
const start = <Options extends TranscriptionOptions>(request: TranscriptionRequestFor<Options>) => {
const route = request.model.route
if (route.kind !== "queued") return Effect.fail(notQueued(route, "start"))
return route.start(request, executor.execute)
}
return Service.of({
start,
resume: (model, token) => {
const route = model.route
if (route.kind !== "queued") return Effect.fail(notQueued(route, "resume"))
return route.resume(model, token, executor.execute)
},
generate: (request, options) => {
const route = request.model.route
if (route.kind !== "queued") return route.generate(request, executor.execute)
return start(request).pipe(Effect.flatMap((generation) => generation.await(options)))
},
stream: (request, options) => {
const route = request.model.route
if (route.kind === "stream") return route.stream(request, executor.execute)
if (route.kind === "queued")
return Stream.unwrap(
start(request).pipe(Effect.map((generation) => resultEvents(generation, responseEvents, options))),
)
return Stream.fromIterableEffect(Effect.map(route.generate(request, executor.execute), responseEvents))
},
})
}),
)
export const TranscriptionClient = {
Service,
layer,
generate,
stream,
start,
resume,
} as const
+294
View File
@@ -0,0 +1,294 @@
import { Effect, Schema, Stream } from "effect"
import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./generation.js"
import { Media } from "./media.js"
import { MediaModel, composeRoute, tryRequest } from "./media-model.js"
import { MediaRoute } from "./route/media.js"
import type { MediaProtocol } from "./route/media-protocol.js"
import { AIError, HttpOptions, MediaUsage, ProviderMetadata } from "./schema/index.js"
import { TranscriptionClient, Service } from "./transcription-client.js"
// ---------------------------------------------------------------------------
// Model
// ---------------------------------------------------------------------------
export type TranscriptionOptions = Record<string, unknown>
export type TranscriptionRoute<Options extends TranscriptionOptions = TranscriptionOptions> =
| MediaRoute.Route<TranscriptionRequestFor<Options>, TranscriptionResponse>
| MediaRoute.StreamRoute<TranscriptionRequestFor<Options>, TranscriptionEvent, TranscriptionResponse>
| MediaRoute.QueuedRoute<TranscriptionRequestFor<Options>, TranscriptionResponse>
export class TranscriptionModel<Options extends TranscriptionOptions = TranscriptionOptions> extends MediaModel<
TranscriptionRoute<Options>,
Options
> {
declare protected readonly _TranscriptionModel: void
static make<Options extends TranscriptionOptions = TranscriptionOptions>(
input: MediaModel.Input<TranscriptionRoute<Options>>,
) {
return new TranscriptionModel<Options>(input)
}
/** The number of type arguments selects the kind: `<Options>`, `<Options, Frame, State>`, or `<Options, Token>`. */
static fromRoute<Options extends TranscriptionOptions>(
route: TranscriptionModel.InlineRouteInput<Options>,
input: MediaRoute.ModelInput,
): TranscriptionModel<Options>
static fromRoute<Options extends TranscriptionOptions, Frame, State>(
route: TranscriptionModel.StreamRouteInput<Options, Frame, State>,
input: MediaRoute.ModelInput,
): TranscriptionModel<Options>
static fromRoute<Options extends TranscriptionOptions, Token>(
route: TranscriptionModel.QueuedRouteInput<Options, Token>,
input: MediaRoute.ModelInput,
): TranscriptionModel<Options>
static fromRoute<Options extends TranscriptionOptions, Frame, State, Token>(
route: TranscriptionModel.RouteInput<Options, Frame, State, Token>,
input: MediaRoute.ModelInput,
) {
const composed: TranscriptionRoute<Options> = isStreamInput(route)
? composeRoute((composition) => MediaRoute.stream({ ...composition, collect: collectResponse }), route, input)
: isQueuedInput(route)
? composeRoute(MediaRoute.queued, route, input)
: composeRoute(MediaRoute.inline, route, input)
return new TranscriptionModel<Options>({
id: input.id,
provider: route.provider,
http: input.http,
route: composed,
})
}
}
export namespace TranscriptionModel {
export type InlineRouteInput<Options extends TranscriptionOptions = TranscriptionOptions> = MediaModel.RouteInput<
TranscriptionRequestFor<Options>,
MediaProtocol.Inline<TranscriptionRequestFor<Options>, TranscriptionResponse>
>
export type StreamRouteInput<
Options extends TranscriptionOptions = TranscriptionOptions,
Frame = unknown,
State = unknown,
> = MediaModel.RouteInput<
MediaProtocol.Addressed<TranscriptionRequestFor<Options>>,
MediaProtocol.Streamed<TranscriptionRequestFor<Options>, TranscriptionEvent, Frame, State>
>
export type QueuedRouteInput<
Options extends TranscriptionOptions = TranscriptionOptions,
Token = unknown,
> = MediaModel.RouteInput<
TranscriptionRequestFor<Options>,
MediaProtocol.Queued<TranscriptionRequestFor<Options>, TranscriptionResponse, Token>
>
export type RouteInput<
Options extends TranscriptionOptions = TranscriptionOptions,
Frame = unknown,
State = unknown,
Token = unknown,
> = InlineRouteInput<Options> | StreamRouteInput<Options, Frame, State> | QueuedRouteInput<Options, Token>
}
const isStreamInput = <Options extends TranscriptionOptions, Frame, State, Token>(
route: TranscriptionModel.RouteInput<Options, Frame, State, Token>,
): route is TranscriptionModel.StreamRouteInput<Options, Frame, State> => route.protocol.kind === "stream"
const isQueuedInput = <Options extends TranscriptionOptions, Frame, State, Token>(
route: TranscriptionModel.RouteInput<Options, Frame, State, Token>,
): route is TranscriptionModel.QueuedRouteInput<Options, Token> => route.protocol.kind === "queued"
export const TranscriptionModelSchema = Schema.declare(
(value): value is TranscriptionModel => value instanceof TranscriptionModel,
{ expected: "Transcription.Model" },
)
// ---------------------------------------------------------------------------
// Request
// ---------------------------------------------------------------------------
export const TranscriptionTimestamps = Schema.Literals(["none", "segment", "word"])
export type TranscriptionTimestamps = Schema.Schema.Type<typeof TranscriptionTimestamps>
export class TranscriptionRequest extends Schema.Class<TranscriptionRequest>("Transcription.Request")({
model: TranscriptionModelSchema,
audio: Media.AssetSchema,
language: Schema.optional(Schema.String),
prompt: Schema.optional(Schema.String),
/** Routes that cannot produce the requested granularity fail typed; routes may return more than asked. */
timestamps: Schema.optional(TranscriptionTimestamps),
diarize: Schema.optional(Schema.Boolean),
speakers: Schema.optional(Schema.Int),
providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
http: Schema.optional(HttpOptions),
}) {
declare protected readonly _TranscriptionRequest: void
}
export type TranscriptionRequestFor<Options extends TranscriptionOptions = TranscriptionOptions> = Omit<
TranscriptionRequest,
"model" | "providerOptions"
> & {
readonly model: TranscriptionModel<Options>
readonly providerOptions?: Options
}
export type TranscriptionModelOptions<Model> = Model extends TranscriptionModel<infer Options> ? Options : never
export type TranscriptionRequestInput<Model extends TranscriptionModel = TranscriptionModel> = Omit<
ConstructorParameters<typeof TranscriptionRequest>[0],
"model" | "providerOptions" | "http"
> & {
readonly model: Model
readonly providerOptions?: NoInfer<TranscriptionModelOptions<Model>>
readonly http?: HttpOptions.Input
}
// ---------------------------------------------------------------------------
// Response and events
// ---------------------------------------------------------------------------
/** Speaker labels are provider-native (`A`, `0`, `spk:0`, or a known speaker name). */
export const TranscriptionSegment = Schema.Struct({
text: Schema.String,
startSeconds: Schema.Number,
endSeconds: Schema.Number,
speaker: Schema.optional(Schema.String),
}).annotate({ identifier: "Transcription.Segment" })
export type TranscriptionSegment = Schema.Schema.Type<typeof TranscriptionSegment>
export const TranscriptionWord = Schema.Struct({
text: Schema.String,
startSeconds: Schema.Number,
endSeconds: Schema.Number,
speaker: Schema.optional(Schema.String),
confidence: Schema.optional(Schema.Number),
}).annotate({ identifier: "Transcription.Word" })
export type TranscriptionWord = Schema.Schema.Type<typeof TranscriptionWord>
const transcriptFields = {
text: Schema.String,
segments: Schema.optional(Schema.Array(TranscriptionSegment)),
words: Schema.optional(Schema.Array(TranscriptionWord)),
/** Provider-native language as detected or echoed (`en`, `english`, `en_us`), lowercased but not normalized. */
language: Schema.optional(Schema.String),
durationSeconds: Schema.optional(Schema.Number),
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}
export class TranscriptionResponse extends Schema.Class<TranscriptionResponse>("Transcription.Response")(
transcriptFields,
) {}
export const TranscriptionTextDeltaEvent = Schema.Struct({
type: Schema.tag("text-delta"),
delta: Schema.String,
}).annotate({ identifier: "Transcription.Event.TextDelta" })
export const TranscriptionSegmentEvent = Schema.Struct({
type: Schema.tag("segment"),
segment: TranscriptionSegment,
}).annotate({ identifier: "Transcription.Event.Segment" })
export const TranscriptionFinishEvent = Schema.Struct({
type: Schema.tag("finish"),
...transcriptFields,
}).annotate({ identifier: "Transcription.Event.Finish" })
const transcriptionEventTagged = Schema.Union([
QueuedEvent,
ProgressEvent,
TranscriptionTextDeltaEvent,
TranscriptionSegmentEvent,
TranscriptionFinishEvent,
]).pipe(Schema.toTaggedUnion("type"))
export const TranscriptionEvent = Object.assign(transcriptionEventTagged, {
is: {
generationQueued: transcriptionEventTagged.guards["generation-queued"],
generationProgress: transcriptionEventTagged.guards["generation-progress"],
textDelta: transcriptionEventTagged.guards["text-delta"],
segment: transcriptionEventTagged.guards.segment,
finish: transcriptionEventTagged.guards.finish,
},
})
export type TranscriptionEvent = Schema.Schema.Type<typeof transcriptionEventTagged>
export const responseEvents = (response: TranscriptionResponse): ReadonlyArray<TranscriptionEvent> => [
TranscriptionFinishEvent.make({ ...response }),
]
const collectResponse = (events: ReadonlyArray<TranscriptionEvent>): Effect.Effect<TranscriptionResponse> => {
const finish = events.find(TranscriptionEvent.is.finish)
// Every transcription protocol's `finish` emits the terminal event or fails, so a completed stream always has one.
if (finish === undefined) return Effect.die(new Error("The transcription stream completed without a finish event"))
const { type: _type, ...transcript } = finish
return Effect.succeed(new TranscriptionResponse(transcript))
}
// ---------------------------------------------------------------------------
// Request-shaped call API
// ---------------------------------------------------------------------------
export function request<const Model extends TranscriptionModel>(
input: TranscriptionRequestInput<Model>,
): TranscriptionRequestFor<TranscriptionModelOptions<Model>>
export function request(input: TranscriptionRequest): TranscriptionRequest
export function request(input: TranscriptionRequest | TranscriptionRequestInput) {
if (input instanceof TranscriptionRequest) return input
return new TranscriptionRequest({
...input,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}
const requestEffect = (input: TranscriptionRequest | TranscriptionRequestInput) => tryRequest(() => request(input))
export function generate<const Model extends TranscriptionModel>(
input: TranscriptionRequestInput<Model>,
options?: AwaitOptions,
): Effect.Effect<TranscriptionResponse, AIError, Service>
export function generate(
input: TranscriptionRequest,
options?: AwaitOptions,
): Effect.Effect<TranscriptionResponse, AIError, Service>
export function generate(input: TranscriptionRequest | TranscriptionRequestInput, options?: AwaitOptions) {
return requestEffect(input).pipe(Effect.flatMap((request) => TranscriptionClient.generate(request, options)))
}
export function stream<const Model extends TranscriptionModel>(
input: TranscriptionRequestInput<Model>,
options?: AwaitOptions,
): Stream.Stream<TranscriptionEvent, AIError, Service>
export function stream(
input: TranscriptionRequest,
options?: AwaitOptions,
): Stream.Stream<TranscriptionEvent, AIError, Service>
export function stream(input: TranscriptionRequest | TranscriptionRequestInput, options?: AwaitOptions) {
return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => TranscriptionClient.stream(request, options))))
}
/** Inline and streaming routes fail with `UnsupportedOperation`. */
export function start<const Model extends TranscriptionModel>(
input: TranscriptionRequestInput<Model>,
): Effect.Effect<Generation<TranscriptionResponse>, AIError, Service>
export function start(input: TranscriptionRequest): Effect.Effect<Generation<TranscriptionResponse>, AIError, Service>
export function start(input: TranscriptionRequest | TranscriptionRequestInput) {
return requestEffect(input).pipe(Effect.flatMap((request) => TranscriptionClient.start(request)))
}
export const resume = <Options extends TranscriptionOptions>(
model: TranscriptionModel<Options>,
token: unknown,
): Effect.Effect<Generation<TranscriptionResponse>, AIError, Service> => TranscriptionClient.resume(model, token)
export const Transcription = {
request,
generate,
stream,
start,
resume,
} as const
+9
View File
@@ -0,0 +1,9 @@
export const concatBytes = (chunks: ReadonlyArray<Uint8Array>) => {
if (chunks.length === 1) return chunks[0]
const bytes = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.length, 0))
chunks.reduce((offset, chunk) => {
bytes.set(chunk, offset)
return offset + chunk.length
}, 0)
return bytes
}
+23 -2
View File
@@ -19,9 +19,12 @@ export const detectMediaType = (bytes: Uint8Array): string | undefined => {
if (startsWith(bytes, [0x25, 0x50, 0x44, 0x46])) return "application/pdf"
if (bytes.length >= 12 && ascii(bytes, 4, 8) === "ftyp") return "video/mp4"
if (startsWith(bytes, [0x1a, 0x45, 0xdf, 0xa3])) return "video/webm"
if (startsWith(bytes, [0x49, 0x44, 0x33]) || startsWith(bytes, [0xff, 0xfb]) || startsWith(bytes, [0xff, 0xf3]))
return "audio/mpeg"
if (startsWith(bytes, [0x49, 0x44, 0x33])) return "audio/mpeg"
// An 11-bit frame sync; layer bits `00` mark AAC ADTS, any other layer is MPEG audio.
if (bytes.length >= 2 && bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0)
return (bytes[1] & 0x06) === 0 ? "audio/aac" : "audio/mpeg"
if (startsWith(bytes, [0x4f, 0x67, 0x67, 0x53])) return "audio/ogg"
if (startsWith(bytes, [0x66, 0x4c, 0x61, 0x43])) return "audio/flac"
return undefined
}
@@ -35,8 +38,11 @@ const EXTENSIONS: Readonly<Record<string, string>> = {
mp4: "video/mp4",
webm: "video/webm",
mp3: "audio/mpeg",
m4a: "audio/mp4",
wav: "audio/wav",
ogg: "audio/ogg",
flac: "audio/flac",
aac: "audio/aac",
txt: "text/plain",
md: "text/markdown",
csv: "text/csv",
@@ -44,3 +50,18 @@ const EXTENSIONS: Readonly<Record<string, string>> = {
export const extensionMediaType = (path: string): string | undefined =>
EXTENSIONS[path.slice(path.lastIndexOf(".") + 1).toLowerCase()]
const EXTENSION_ALIASES: Readonly<Record<string, string>> = {
"audio/mp3": "mp3",
"audio/m4a": "m4a",
"audio/x-m4a": "m4a",
"audio/webm": "webm",
"audio/wave": "wav",
"audio/x-wav": "wav",
"audio/x-flac": "flac",
}
export const mediaTypeExtension = (mediaType: string): string | undefined => {
const type = mediaType.split(";", 1)[0].trim().toLowerCase()
return EXTENSION_ALIASES[type] ?? Object.entries(EXTENSIONS).find(([, known]) => known === type)?.[0]
}
+96
View File
@@ -0,0 +1,96 @@
import { Context, Effect, Layer, Stream } from "effect"
import { resultEvents, type AwaitOptions, type Generation } from "./generation.js"
import { RequestExecutor } from "./route/executor.js"
import type { AIError } from "./schema/index.js"
import {
responseEvents,
type VideoEvent,
type VideoModel,
type VideoOptions,
type VideoRequestFor,
type VideoResponse,
} from "./video.js"
export interface Interface {
readonly start: <Options extends VideoOptions>(
request: VideoRequestFor<Options>,
) => Effect.Effect<Generation<VideoResponse>, AIError>
readonly resume: <Options extends VideoOptions>(
model: VideoModel<Options>,
token: unknown,
) => Effect.Effect<Generation<VideoResponse>, AIError>
readonly generate: <Options extends VideoOptions>(
request: VideoRequestFor<Options>,
options?: AwaitOptions,
) => Effect.Effect<VideoResponse, AIError>
readonly stream: <Options extends VideoOptions>(
request: VideoRequestFor<Options>,
options?: AwaitOptions,
) => Stream.Stream<VideoEvent, AIError>
}
export class Service extends Context.Service<Service, Interface>()("@opencode/VideoClient") {}
export const start = <Options extends VideoOptions>(
request: VideoRequestFor<Options>,
): Effect.Effect<Generation<VideoResponse>, AIError, Service> =>
Effect.gen(function* () {
const client = yield* Service
return yield* client.start(request)
})
export const resume = <Options extends VideoOptions>(
model: VideoModel<Options>,
token: unknown,
): Effect.Effect<Generation<VideoResponse>, AIError, Service> =>
Effect.gen(function* () {
const client = yield* Service
return yield* client.resume(model, token)
})
export const generate = <Options extends VideoOptions>(
request: VideoRequestFor<Options>,
options?: AwaitOptions,
): Effect.Effect<VideoResponse, AIError, Service> =>
Effect.gen(function* () {
const client = yield* Service
return yield* client.generate(request, options)
})
export const stream = <Options extends VideoOptions>(
request: VideoRequestFor<Options>,
options?: AwaitOptions,
): Stream.Stream<VideoEvent, AIError, Service> =>
Stream.unwrap(
Effect.gen(function* () {
const client = yield* Service
return client.stream(request, options)
}),
)
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
Service,
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
const start = <Options extends VideoOptions>(request: VideoRequestFor<Options>) =>
request.model.route.start(request, executor.execute)
return Service.of({
start,
resume: (model, token) => model.route.resume(model, token, executor.execute),
generate: (request, options) => start(request).pipe(Effect.flatMap((generation) => generation.await(options))),
stream: (request, options) =>
Stream.unwrap(
start(request).pipe(Effect.map((generation) => resultEvents(generation, responseEvents, options))),
),
})
}),
)
export const VideoClient = {
Service,
layer,
start,
resume,
generate,
stream,
} as const
+218
View File
@@ -0,0 +1,218 @@
import { Effect, Schema, Stream } from "effect"
import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./generation.js"
import { Media } from "./media.js"
import { MediaModel, composeRoute, tryRequest } from "./media-model.js"
import { MediaRoute } from "./route/media.js"
import type { MediaProtocol } from "./route/media-protocol.js"
import { AIError, HttpOptions, MediaUsage, ProviderMetadata } from "./schema/index.js"
import { VideoClient, Service } from "./video-client.js"
// ---------------------------------------------------------------------------
// Model
// ---------------------------------------------------------------------------
export type VideoOptions = Record<string, unknown>
export type VideoRoute<Options extends VideoOptions = VideoOptions> = MediaRoute.QueuedRoute<
VideoRequestFor<Options>,
VideoResponse
>
export class VideoModel<Options extends VideoOptions = VideoOptions> extends MediaModel<VideoRoute<Options>, Options> {
declare protected readonly _VideoModel: void
static make<Options extends VideoOptions = VideoOptions>(input: MediaModel.Input<VideoRoute<Options>>) {
return new VideoModel<Options>(input)
}
/** Compose a queued video protocol with its canonical start path into a model for one deployment. */
static fromRoute<Options extends VideoOptions = VideoOptions, Token = unknown>(
route: VideoModel.RouteInput<Options, Token>,
input: MediaRoute.ModelInput,
) {
return new VideoModel<Options>({
id: input.id,
provider: route.provider,
http: input.http,
route: composeRoute(MediaRoute.queued, route, input),
})
}
}
export namespace VideoModel {
export type RouteInput<Options extends VideoOptions = VideoOptions, Token = unknown> = MediaModel.RouteInput<
VideoRequestFor<Options>,
MediaProtocol.Queued<VideoRequestFor<Options>, VideoResponse, Token>
>
}
export const VideoModelSchema = Schema.declare((value): value is VideoModel => value instanceof VideoModel, {
expected: "Video.Model",
})
// ---------------------------------------------------------------------------
// Request
// ---------------------------------------------------------------------------
export type VideoAspectRatio = Media.AspectRatio
export const VideoAspectRatio = Media.AspectRatio
export type VideoResolution = "480p" | "720p" | "1080p" | "4k" | (string & {})
/** Pinned frames. Routes that accept only a first frame fail typed when `last` is present. */
export const VideoFrames = Schema.Struct({
first: Schema.optional(Media.AssetSchema),
last: Schema.optional(Media.AssetSchema),
}).annotate({ identifier: "Video.Frames" })
export type VideoFrames = Schema.Schema.Type<typeof VideoFrames>
export class VideoRequest extends Schema.Class<VideoRequest>("Video.Request")({
model: VideoModelSchema,
prompt: Schema.String,
frames: Schema.optional(VideoFrames),
/** Style or subject references that guide the output without pinning a frame. */
references: Schema.optional(Schema.Array(Media.AssetSchema)),
/** Source video for edit or extension routes. */
video: Schema.optional(Media.AssetSchema),
durationSeconds: Schema.optional(Schema.Number),
aspectRatio: Schema.optional(VideoAspectRatio),
resolution: Schema.optional(Schema.String),
/** Whether to generate an audio track; routes whose audio is always on fail typed on `false`. */
audio: Schema.optional(Schema.Boolean),
n: Schema.optional(Schema.Int),
seed: Schema.optional(Schema.Number),
negativePrompt: Schema.optional(Schema.String),
providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
http: Schema.optional(HttpOptions),
}) {
declare protected readonly _VideoRequest: void
}
export type VideoRequestFor<Options extends VideoOptions = VideoOptions> = Omit<
VideoRequest,
"model" | "providerOptions"
> & {
readonly model: VideoModel<Options>
readonly providerOptions?: Options
}
export type VideoModelOptions<Model> = Model extends VideoModel<infer Options> ? Options : never
export type VideoRequestInput<Model extends VideoModel = VideoModel> = Omit<
ConstructorParameters<typeof VideoRequest>[0],
"model" | "providerOptions" | "http" | "resolution"
> & {
readonly model: Model
readonly resolution?: VideoResolution
readonly providerOptions?: NoInfer<VideoModelOptions<Model>>
readonly http?: HttpOptions.Input
}
// ---------------------------------------------------------------------------
// Response and events
// ---------------------------------------------------------------------------
export class VideoResponse extends Schema.Class<VideoResponse>("Video.Response")({
videos: Schema.Array(Media.AssetSchema),
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}) {
get video() {
return this.videos[0]
}
}
export const VideoOutputEvent = Schema.Struct({
type: Schema.tag("video"),
index: Schema.Number,
video: Media.AssetSchema,
}).annotate({ identifier: "Video.Event.Video" })
export const VideoFinishEvent = Schema.Struct({
type: Schema.tag("finish"),
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "Video.Event.Finish" })
const videoEventTagged = Schema.Union([QueuedEvent, ProgressEvent, VideoOutputEvent, VideoFinishEvent]).pipe(
Schema.toTaggedUnion("type"),
)
export const VideoEvent = Object.assign(videoEventTagged, {
is: {
generationQueued: videoEventTagged.guards["generation-queued"],
generationProgress: videoEventTagged.guards["generation-progress"],
video: videoEventTagged.guards.video,
finish: videoEventTagged.guards.finish,
},
})
export type VideoEvent = Schema.Schema.Type<typeof videoEventTagged>
/** A completed response expanded into the streaming event shape. */
export const responseEvents = (response: VideoResponse): ReadonlyArray<VideoEvent> => [
...response.videos.map((video, index) => VideoOutputEvent.make({ index, video })),
VideoFinishEvent.make({
usage: response.usage,
notices: response.notices,
providerMetadata: response.providerMetadata,
}),
]
// ---------------------------------------------------------------------------
// Request-shaped call API
// ---------------------------------------------------------------------------
export function request<const Model extends VideoModel>(
input: VideoRequestInput<Model>,
): VideoRequestFor<VideoModelOptions<Model>>
export function request(input: VideoRequest): VideoRequest
export function request(input: VideoRequest | VideoRequestInput) {
if (input instanceof VideoRequest) return input
return new VideoRequest({
...input,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}
const requestEffect = (input: VideoRequest | VideoRequestInput) => tryRequest(() => request(input))
export function start<const Model extends VideoModel>(
input: VideoRequestInput<Model>,
): Effect.Effect<Generation<VideoResponse>, AIError, Service>
export function start(input: VideoRequest): Effect.Effect<Generation<VideoResponse>, AIError, Service>
export function start(input: VideoRequest | VideoRequestInput) {
return requestEffect(input).pipe(Effect.flatMap((request) => VideoClient.start(request)))
}
export function generate<const Model extends VideoModel>(
input: VideoRequestInput<Model>,
options?: AwaitOptions,
): Effect.Effect<VideoResponse, AIError, Service>
export function generate(input: VideoRequest, options?: AwaitOptions): Effect.Effect<VideoResponse, AIError, Service>
export function generate(input: VideoRequest | VideoRequestInput, options?: AwaitOptions) {
return requestEffect(input).pipe(Effect.flatMap((request) => VideoClient.generate(request, options)))
}
/** Rebuild a generation handle from a persisted `Generation.token`, refreshing its status once. */
export const resume = <Options extends VideoOptions>(
model: VideoModel<Options>,
token: unknown,
): Effect.Effect<Generation<VideoResponse>, AIError, Service> => VideoClient.resume(model, token)
export function stream<const Model extends VideoModel>(
input: VideoRequestInput<Model>,
options?: AwaitOptions,
): Stream.Stream<VideoEvent, AIError, Service>
export function stream(input: VideoRequest, options?: AwaitOptions): Stream.Stream<VideoEvent, AIError, Service>
export function stream(input: VideoRequest | VideoRequestInput, options?: AwaitOptions) {
return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => VideoClient.stream(request, options))))
}
export const Video = {
request,
start,
generate,
resume,
stream,
} as const
+46 -1
View File
@@ -1,17 +1,38 @@
import { describe, expect, test } from "bun:test"
import { AIError, LanguageModel, LLM, LLMClient, Media, Provider } from "@opencode/ai"
import {
AIError,
Generation,
Image,
LanguageModel,
LLM,
LLMClient,
Media,
Provider,
Speech,
SpeechClient,
SpeechEvent,
Video,
VideoClient,
} from "@opencode/ai"
import { Route, Protocol, WebSocketTransport } from "@opencode/ai/route"
import { Provider as ProviderSubpath } from "@opencode/ai/provider"
import {
AssemblyAI,
Baseten,
Cartesia,
CloudflareAIGateway,
CloudflareWorkersAI,
Deepgram,
DeepSeek,
ElevenLabs,
Fal,
Fireworks,
Google,
OpenCodeZen,
OpenAI,
OpenAICompatible,
OpenRouter,
Runway,
TypeSafeAI,
VercelAIGateway,
XAI,
@@ -36,6 +57,15 @@ describe("public exports", () => {
expect(AIError).toBeFunction()
expect(LanguageModel.make).toBeFunction()
expect(Media.bytes).toBeFunction()
expect(Image.generate).toBeFunction()
expect(Video.start).toBeFunction()
expect(Video.resume).toBeFunction()
expect(VideoClient.layer).toBeDefined()
expect(Speech.generate).toBeFunction()
expect(Speech.stream).toBeFunction()
expect(SpeechClient.layer).toBeDefined()
expect(SpeechEvent.is.audioDelta).toBeFunction()
expect(Generation).toBeFunction()
expect(Provider.make).toBeFunction()
expect(ProviderSubpath.make).toBe(Provider.make)
expect(TestLLM.layer).toBeFunction()
@@ -82,6 +112,21 @@ describe("public exports", () => {
expect(XAI.provider.chat).toBe(XAI.chat)
expect(XAI.configure({ apiKey: "fixture" }).responses("grok-4.3").route.id).toBe("openai-responses")
expect(XAI.configure({ apiKey: "fixture" }).chat("grok-4.3").route.id).toBe("openai-compatible-chat")
expect(XAI.configure({ apiKey: "fixture" }).video("grok-imagine-video-1.5").route.id).toBe("xai-video")
expect(Fal.configure({ apiKey: "fixture" }).video("fal-ai/veo3.1").route.id).toBe("fal-video")
expect(Runway.configure({ apiKey: "fixture" }).video("gen4.5").route.id).toBe("runway-video")
expect(Runway.provider.video).toBe(Runway.video)
expect(OpenAI.configure({ apiKey: "fixture" }).speech("gpt-4o-mini-tts").route.id).toBe("openai-speech")
expect(Google.configure({ apiKey: "fixture" }).speech("gemini-2.5-flash-preview-tts").route.id).toBe(
"google-speech",
)
expect(ElevenLabs.configure({ apiKey: "fixture" }).speech("eleven_flash_v2_5").route.id).toBe("elevenlabs-speech")
expect(Cartesia.configure({ apiKey: "fixture" }).speech("sonic-3").route.id).toBe("cartesia-speech")
expect(Deepgram.configure({ apiKey: "fixture" }).speech("aura-2-thalia-en").route.id).toBe("deepgram-speech")
expect(OpenAI.configure({ apiKey: "fixture" }).transcription("gpt-transcribe").route.kind).toBe("stream")
expect(Google.configure({ apiKey: "fixture" }).transcription("gemini-3.5-transcribe").route.kind).toBe("stream")
expect(Deepgram.configure({ apiKey: "fixture" }).transcription("nova-3").route.kind).toBe("inline")
expect(AssemblyAI.configure({ apiKey: "fixture" }).transcription("universal-3-5-pro").route.kind).toBe("queued")
})
test("protocol barrels expose supported low-level routes", () => {
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,98 @@
{
"version": 1,
"metadata": {
"tags": [
"prefix:assemblyai-transcription",
"provider:assemblyai",
"protocol:assemblyai-transcription"
],
"name": "assemblyai-transcription/uploads-submits-and-polls-a-transcript",
"recordedAt": "2026-09-23T14:03:07.504Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.eu.assemblyai.com/v2/upload",
"headers": {
"content-type": "application/octet-stream"
},
"body": "[audio]"
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"upload_url\":\"https://cdn.assemblyai.com/upload/f057ab4f09d5f28527be4b830b3d92f8bf6a97e5f854837cf22efbf2d09b5182/ed6a04e2-e34c-4a70-89da-b1ec34d2fb5f\"}\n"
}
},
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.eu.assemblyai.com/v2/transcript",
"headers": {
"content-type": "application/json"
},
"body": "{\"audio_url\":\"https://cdn.assemblyai.com/upload/f057ab4f09d5f28527be4b830b3d92f8bf6a97e5f854837cf22efbf2d09b5182/ed6a04e2-e34c-4a70-89da-b1ec34d2fb5f\",\"speech_models\":[\"universal-3-5-pro\"],\"language_detection\":true,\"speaker_labels\":true}"
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json; charset=UTF-8"
},
"body": "{\"id\": \"a1de21ad-a178-4c28-b8f0-4b0334b72126\", \"language_model\": \"assemblyai_default\", \"acoustic_model\": \"assemblyai_default\", \"language_code\": null, \"speech_understanding\": null, \"translated_texts\": null, \"status\": \"processing\", \"audio_url\": \"https://cdn.assemblyai.com/upload/f057ab4f09d5f28527be4b830b3d92f8bf6a97e5f854837cf22efbf2d09b5182/ed6a04e2-e34c-4a70-89da-b1ec34d2fb5f\", \"text\": null, \"words\": null, \"utterances\": null, \"confidence\": null, \"audio_duration\": null, \"punctuate\": true, \"format_text\": true, \"dual_channel\": false, \"webhook_url\": null, \"webhook_status_code\": null, \"webhook_auth\": false, \"webhook_auth_header_name\": null, \"speed_boost\": false, \"auto_highlights_result\": null, \"auto_highlights\": false, \"audio_start_from\": null, \"audio_end_at\": null, \"word_boost\": [], \"boost_param\": null, \"prompt\": null, \"keyterms_prompt\": null, \"filter_profanity\": false, \"redact_pii\": false, \"redact_pii_audio\": false, \"redact_pii_audio_quality\": null, \"redact_pii_audio_options\": null, \"redact_pii_policies\": null, \"redact_pii_sub\": null, \"redact_static_entities\": null, \"speaker_labels\": true, \"speaker_options\": {\"min_speakers_expected\": null, \"max_speakers_expected\": null, \"advanced_speaker_segmentation\": true}, \"content_safety\": false, \"iab_categories\": false, \"content_safety_labels\": {}, \"iab_categories_result\": {}, \"language_detection\": true, \"language_detection_options\": {\"expected_languages\": null, \"fallback_language\": null, \"code_switching\": null, \"code_switching_confidence_threshold\": null, \"on_low_language_confidence\": null, \"on_no_speech_detected\": \"fallback\", \"swiss_german\": null, \"language_detection_model\": null, \"localization\": null}, \"language_detection_results\": null, \"language_confidence_threshold\": null, \"language_confidence\": null, \"custom_spelling\": null, \"throttled\": false, \"auto_chapters\": false, \"summarization\": false, \"summary_type\": null, \"summary_model\": null, \"custom_topics\": false, \"topics\": [], \"speech_threshold\": null, \"speech_model\": null, \"speech_models\": [\"universal-3-5-pro\"], \"speech_model_used\": null, \"temperature\": null, \"remove_audio_tags\": \"all\", \"chapters\": null, \"disfluencies\": false, \"entity_detection\": false, \"sentiment_analysis\": false, \"sentiment_analysis_results\": null, \"entities\": null, \"speakers_expected\": null, \"summary\": null, \"custom_topics_results\": null, \"is_deleted\": null, \"multichannel\": null, \"project_id\": 2040069, \"token_id\": 2077829}"
}
},
{
"transport": "http",
"request": {
"method": "GET",
"url": "https://api.eu.assemblyai.com/v2/transcript/a1de21ad-a178-4c28-b8f0-4b0334b72126",
"headers": {},
"body": ""
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json; charset=UTF-8"
},
"body": "{\"id\": \"a1de21ad-a178-4c28-b8f0-4b0334b72126\", \"language_model\": \"assemblyai_default\", \"acoustic_model\": \"assemblyai_default\", \"language_code\": null, \"speech_understanding\": null, \"translated_texts\": null, \"status\": \"processing\", \"audio_url\": \"https://cdn.assemblyai.com/upload/f057ab4f09d5f28527be4b830b3d92f8bf6a97e5f854837cf22efbf2d09b5182/ed6a04e2-e34c-4a70-89da-b1ec34d2fb5f\", \"text\": null, \"words\": null, \"utterances\": null, \"confidence\": null, \"audio_duration\": null, \"punctuate\": true, \"format_text\": true, \"dual_channel\": false, \"webhook_url\": null, \"webhook_status_code\": null, \"webhook_auth\": false, \"webhook_auth_header_name\": null, \"speed_boost\": false, \"auto_highlights_result\": null, \"auto_highlights\": false, \"audio_start_from\": null, \"audio_end_at\": null, \"word_boost\": [], \"boost_param\": null, \"prompt\": null, \"keyterms_prompt\": [], \"filter_profanity\": false, \"redact_pii\": false, \"redact_pii_audio\": false, \"redact_pii_audio_quality\": null, \"redact_pii_audio_options\": null, \"redact_pii_policies\": null, \"redact_pii_sub\": null, \"redact_static_entities\": null, \"speaker_labels\": true, \"speaker_options\": {\"min_speakers_expected\": null, \"max_speakers_expected\": null, \"advanced_speaker_segmentation\": true}, \"content_safety\": false, \"iab_categories\": false, \"content_safety_labels\": {}, \"iab_categories_result\": {}, \"language_detection\": true, \"language_detection_options\": {\"expected_languages\": null, \"fallback_language\": null, \"code_switching\": null, \"code_switching_confidence_threshold\": null, \"on_low_language_confidence\": null, \"on_no_speech_detected\": \"fallback\", \"swiss_german\": null, \"language_detection_model\": null, \"localization\": null}, \"language_detection_results\": null, \"language_confidence_threshold\": null, \"language_confidence\": null, \"custom_spelling\": null, \"throttled\": false, \"auto_chapters\": false, \"summarization\": false, \"summary_type\": null, \"summary_model\": null, \"custom_topics\": false, \"topics\": [], \"speech_threshold\": null, \"speech_model\": null, \"speech_models\": [\"universal-3-5-pro\"], \"speech_model_used\": null, \"temperature\": null, \"remove_audio_tags\": \"all\", \"chapters\": null, \"disfluencies\": false, \"entity_detection\": false, \"sentiment_analysis\": false, \"sentiment_analysis_results\": null, \"entities\": null, \"speakers_expected\": null, \"summary\": null, \"custom_topics_results\": null, \"is_deleted\": null, \"multichannel\": null, \"project_id\": 2040069, \"token_id\": 2077829}"
}
},
{
"transport": "http",
"request": {
"method": "GET",
"url": "https://api.eu.assemblyai.com/v2/transcript/a1de21ad-a178-4c28-b8f0-4b0334b72126",
"headers": {},
"body": ""
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json; charset=UTF-8"
},
"body": "{\"id\": \"a1de21ad-a178-4c28-b8f0-4b0334b72126\", \"language_model\": \"assemblyai_default\", \"acoustic_model\": \"assemblyai_default\", \"language_code\": \"en\", \"speech_understanding\": null, \"translated_texts\": null, \"status\": \"completed\", \"audio_url\": \"https://cdn.assemblyai.com/upload/f057ab4f09d5f28527be4b830b3d92f8bf6a97e5f854837cf22efbf2d09b5182/ed6a04e2-e34c-4a70-89da-b1ec34d2fb5f\", \"text\": \"Hello from Open Code.\", \"words\": [{\"text\": \"Hello\", \"start\": 445, \"end\": 633, \"confidence\": 0.8883743, \"speaker\": \"A\"}, {\"text\": \"from\", \"start\": 719, \"end\": 890, \"confidence\": 0.9978282, \"speaker\": \"A\"}, {\"text\": \"Open\", \"start\": 959, \"end\": 1147, \"confidence\": 0.9486656, \"speaker\": \"A\"}, {\"text\": \"Code.\", \"start\": 1215, \"end\": 1455, \"confidence\": 0.5913869, \"speaker\": \"A\"}], \"utterances\": [{\"speaker\": \"A\", \"text\": \"Hello from Open Code.\", \"confidence\": 0.85656375, \"start\": 445, \"end\": 1455, \"words\": [{\"text\": \"Hello\", \"start\": 445, \"end\": 633, \"confidence\": 0.8883743, \"speaker\": \"A\"}, {\"text\": \"from\", \"start\": 719, \"end\": 890, \"confidence\": 0.9978282, \"speaker\": \"A\"}, {\"text\": \"Open\", \"start\": 959, \"end\": 1147, \"confidence\": 0.9486656, \"speaker\": \"A\"}, {\"text\": \"Code.\", \"start\": 1215, \"end\": 1455, \"confidence\": 0.5913869, \"speaker\": \"A\"}]}], \"confidence\": 0.85656375, \"audio_duration\": 2, \"punctuate\": true, \"format_text\": true, \"dual_channel\": false, \"webhook_url\": null, \"webhook_status_code\": null, \"webhook_auth\": false, \"webhook_auth_header_name\": null, \"speed_boost\": false, \"auto_highlights_result\": null, \"auto_highlights\": false, \"audio_start_from\": null, \"audio_end_at\": null, \"word_boost\": [], \"boost_param\": null, \"prompt\": null, \"keyterms_prompt\": [], \"filter_profanity\": false, \"redact_pii\": false, \"redact_pii_audio\": false, \"redact_pii_audio_quality\": null, \"redact_pii_audio_options\": null, \"redact_pii_policies\": null, \"redact_pii_sub\": null, \"redact_static_entities\": null, \"speaker_labels\": true, \"speaker_options\": {\"min_speakers_expected\": null, \"max_speakers_expected\": null, \"advanced_speaker_segmentation\": true}, \"content_safety\": false, \"iab_categories\": false, \"content_safety_labels\": {\"status\": \"unavailable\", \"results\": [], \"summary\": {}}, \"iab_categories_result\": {\"status\": \"unavailable\", \"results\": [], \"summary\": {}}, \"language_detection\": true, \"language_detection_options\": {\"expected_languages\": null, \"fallback_language\": null, \"code_switching\": null, \"code_switching_confidence_threshold\": null, \"on_low_language_confidence\": null, \"on_no_speech_detected\": \"fallback\", \"swiss_german\": null, \"language_detection_model\": null, \"localization\": null}, \"language_detection_results\": null, \"language_confidence_threshold\": null, \"language_confidence\": 0.9934, \"custom_spelling\": null, \"throttled\": false, \"auto_chapters\": false, \"summarization\": false, \"summary_type\": null, \"summary_model\": null, \"custom_topics\": false, \"topics\": [], \"speech_threshold\": null, \"speech_model\": null, \"speech_models\": [\"universal-3-5-pro\"], \"speech_model_used\": \"universal-3-5-pro\", \"temperature\": null, \"remove_audio_tags\": \"all\", \"chapters\": null, \"disfluencies\": false, \"entity_detection\": false, \"sentiment_analysis\": false, \"sentiment_analysis_results\": null, \"entities\": null, \"speakers_expected\": null, \"summary\": null, \"custom_topics_results\": null, \"is_deleted\": null, \"multichannel\": null, \"project_id\": 2040069, \"token_id\": 2077829}"
}
},
{
"transport": "http",
"request": {
"method": "GET",
"url": "https://api.eu.assemblyai.com/v2/transcript/a1de21ad-a178-4c28-b8f0-4b0334b72126",
"headers": {},
"body": ""
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json; charset=UTF-8"
},
"body": "{\"id\": \"a1de21ad-a178-4c28-b8f0-4b0334b72126\", \"language_model\": \"assemblyai_default\", \"acoustic_model\": \"assemblyai_default\", \"language_code\": \"en\", \"speech_understanding\": null, \"translated_texts\": null, \"status\": \"completed\", \"audio_url\": \"https://cdn.assemblyai.com/upload/f057ab4f09d5f28527be4b830b3d92f8bf6a97e5f854837cf22efbf2d09b5182/ed6a04e2-e34c-4a70-89da-b1ec34d2fb5f\", \"text\": \"Hello from Open Code.\", \"words\": [{\"text\": \"Hello\", \"start\": 445, \"end\": 633, \"confidence\": 0.8883743, \"speaker\": \"A\"}, {\"text\": \"from\", \"start\": 719, \"end\": 890, \"confidence\": 0.9978282, \"speaker\": \"A\"}, {\"text\": \"Open\", \"start\": 959, \"end\": 1147, \"confidence\": 0.9486656, \"speaker\": \"A\"}, {\"text\": \"Code.\", \"start\": 1215, \"end\": 1455, \"confidence\": 0.5913869, \"speaker\": \"A\"}], \"utterances\": [{\"speaker\": \"A\", \"text\": \"Hello from Open Code.\", \"confidence\": 0.85656375, \"start\": 445, \"end\": 1455, \"words\": [{\"text\": \"Hello\", \"start\": 445, \"end\": 633, \"confidence\": 0.8883743, \"speaker\": \"A\"}, {\"text\": \"from\", \"start\": 719, \"end\": 890, \"confidence\": 0.9978282, \"speaker\": \"A\"}, {\"text\": \"Open\", \"start\": 959, \"end\": 1147, \"confidence\": 0.9486656, \"speaker\": \"A\"}, {\"text\": \"Code.\", \"start\": 1215, \"end\": 1455, \"confidence\": 0.5913869, \"speaker\": \"A\"}]}], \"confidence\": 0.85656375, \"audio_duration\": 2, \"punctuate\": true, \"format_text\": true, \"dual_channel\": false, \"webhook_url\": null, \"webhook_status_code\": null, \"webhook_auth\": false, \"webhook_auth_header_name\": null, \"speed_boost\": false, \"auto_highlights_result\": null, \"auto_highlights\": false, \"audio_start_from\": null, \"audio_end_at\": null, \"word_boost\": [], \"boost_param\": null, \"prompt\": null, \"keyterms_prompt\": [], \"filter_profanity\": false, \"redact_pii\": false, \"redact_pii_audio\": false, \"redact_pii_audio_quality\": null, \"redact_pii_audio_options\": null, \"redact_pii_policies\": null, \"redact_pii_sub\": null, \"redact_static_entities\": null, \"speaker_labels\": true, \"speaker_options\": {\"min_speakers_expected\": null, \"max_speakers_expected\": null, \"advanced_speaker_segmentation\": true}, \"content_safety\": false, \"iab_categories\": false, \"content_safety_labels\": {\"status\": \"unavailable\", \"results\": [], \"summary\": {}}, \"iab_categories_result\": {\"status\": \"unavailable\", \"results\": [], \"summary\": {}}, \"language_detection\": true, \"language_detection_options\": {\"expected_languages\": null, \"fallback_language\": null, \"code_switching\": null, \"code_switching_confidence_threshold\": null, \"on_low_language_confidence\": null, \"on_no_speech_detected\": \"fallback\", \"swiss_german\": null, \"language_detection_model\": null, \"localization\": null}, \"language_detection_results\": null, \"language_confidence_threshold\": null, \"language_confidence\": 0.9934, \"custom_spelling\": null, \"throttled\": false, \"auto_chapters\": false, \"summarization\": false, \"summary_type\": null, \"summary_model\": null, \"custom_topics\": false, \"topics\": [], \"speech_threshold\": null, \"speech_model\": null, \"speech_models\": [\"universal-3-5-pro\"], \"speech_model_used\": \"universal-3-5-pro\", \"temperature\": null, \"remove_audio_tags\": \"all\", \"chapters\": null, \"disfluencies\": false, \"entity_detection\": false, \"sentiment_analysis\": false, \"sentiment_analysis_results\": null, \"entities\": null, \"speakers_expected\": null, \"summary\": null, \"custom_topics_results\": null, \"is_deleted\": null, \"multichannel\": null, \"project_id\": 2040069, \"token_id\": 2077829}"
}
}
]
}
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,32 @@
{
"version": 1,
"metadata": {
"tags": [
"prefix:deepgram-transcription",
"provider:deepgram",
"protocol:deepgram-transcription"
],
"name": "deepgram-transcription/splits-utterances-at-speaker-changes",
"recordedAt": "2026-09-23T13:46:35.886Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&detect_language=true&diarize_model=latest&utterances=true",
"headers": {
"content-type": "audio/mpeg"
},
"body": "[audio]"
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"metadata\":{\"transaction_key\":\"deprecated\",\"request_id\":\"01a0ce84-a81d-7161-8505-c08d5e0c25bb\",\"sha256\":\"12e9ccb9d346c28b071c2fab7ce48556d0cc115af6850aacb338e7548807cb80\",\"created\":\"2026-09-23T13:46:35.566Z\",\"duration\":3.552625,\"channels\":1,\"models\":[\"2187e11a-3532-4498-b076-81fa530bdd49\"],\"model_info\":{\"2187e11a-3532-4498-b076-81fa530bdd49\":{\"name\":\"general-nova-3\",\"version\":\"2025-07-31.0\",\"arch\":\"nova-3\"}},\"diarize_info\":{\"model_uuid\":\"6946b038-8264-4505-acf7-9822c009fecb\",\"arch\":\"v2\"}},\"results\":{\"channels\":[{\"alternatives\":[{\"transcript\":\"Did the release ship? Yes. It shipped this morning.\",\"confidence\":1.0,\"words\":[{\"word\":\"did\",\"start\":0.08,\"end\":0.48,\"confidence\":1.0,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"Did\"},{\"word\":\"the\",\"start\":0.48,\"end\":0.64,\"confidence\":1.0,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"the\"},{\"word\":\"release\",\"start\":0.64,\"end\":0.96,\"confidence\":0.99902344,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"release\"},{\"word\":\"ship\",\"start\":0.96,\"end\":1.52,\"confidence\":1.0,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"ship?\"},{\"word\":\"yes\",\"start\":1.5999999,\"end\":2.32,\"confidence\":0.9980469,\"speaker\":1,\"speaker_confidence\":0.98,\"punctuated_word\":\"Yes.\"},{\"word\":\"it\",\"start\":2.32,\"end\":2.56,\"confidence\":1.0,\"speaker\":1,\"speaker_confidence\":0.98,\"punctuated_word\":\"It\"},{\"word\":\"shipped\",\"start\":2.56,\"end\":2.8799999,\"confidence\":0.9707031,\"speaker\":1,\"speaker_confidence\":0.98,\"punctuated_word\":\"shipped\"},{\"word\":\"this\",\"start\":2.8799999,\"end\":3.1999998,\"confidence\":1.0,\"speaker\":1,\"speaker_confidence\":0.98,\"punctuated_word\":\"this\"},{\"word\":\"morning\",\"start\":3.1999998,\"end\":3.52,\"confidence\":1.0,\"speaker\":1,\"speaker_confidence\":0.98,\"punctuated_word\":\"morning.\"}],\"paragraphs\":{\"transcript\":\"\\nSpeaker 0: Did the release ship?\\n\\nSpeaker 1: Yes. It shipped this morning.\",\"paragraphs\":[{\"sentences\":[{\"text\":\"Did the release ship?\",\"start\":0.08,\"end\":1.52}],\"speaker\":0,\"num_words\":4,\"start\":0.08,\"end\":1.52},{\"sentences\":[{\"text\":\"Yes.\",\"start\":1.5999999,\"end\":2.32},{\"text\":\"It shipped this morning.\",\"start\":2.32,\"end\":3.52}],\"speaker\":1,\"num_words\":5,\"start\":1.5999999,\"end\":3.52}]}}],\"detected_language\":\"en\",\"language_confidence\":0.917855}],\"utterances\":[{\"start\":0.08,\"end\":3.52,\"confidence\":0.99641925,\"channel\":0,\"transcript\":\"Did the release ship? Yes. It shipped this morning.\",\"words\":[{\"word\":\"did\",\"start\":0.08,\"end\":0.48,\"confidence\":1.0,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"Did\"},{\"word\":\"the\",\"start\":0.48,\"end\":0.64,\"confidence\":1.0,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"the\"},{\"word\":\"release\",\"start\":0.64,\"end\":0.96,\"confidence\":0.99902344,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"release\"},{\"word\":\"ship\",\"start\":0.96,\"end\":1.52,\"confidence\":1.0,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"ship?\"},{\"word\":\"yes\",\"start\":1.5999999,\"end\":2.32,\"confidence\":0.9980469,\"speaker\":1,\"speaker_confidence\":0.98,\"punctuated_word\":\"Yes.\"},{\"word\":\"it\",\"start\":2.32,\"end\":2.56,\"confidence\":1.0,\"speaker\":1,\"speaker_confidence\":0.98,\"punctuated_word\":\"It\"},{\"word\":\"shipped\",\"start\":2.56,\"end\":2.8799999,\"confidence\":0.9707031,\"speaker\":1,\"speaker_confidence\":0.98,\"punctuated_word\":\"shipped\"},{\"word\":\"this\",\"start\":2.8799999,\"end\":3.1999998,\"confidence\":1.0,\"speaker\":1,\"speaker_confidence\":0.98,\"punctuated_word\":\"this\"},{\"word\":\"morning\",\"start\":3.1999998,\"end\":3.52,\"confidence\":1.0,\"speaker\":1,\"speaker_confidence\":0.98,\"punctuated_word\":\"morning.\"}],\"speaker\":1,\"id\":\"67ba8055-e541-4089-abfa-0163558083cc\"}]}}"
}
}
]
}
@@ -0,0 +1,50 @@
{
"version": 1,
"metadata": {
"tags": [
"prefix:deepgram-transcription",
"provider:deepgram",
"protocol:deepgram-transcription"
],
"name": "deepgram-transcription/transcribes-raw-audio-with-speakers-and-word-timestamps",
"recordedAt": "2026-09-23T12:32:18.578Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&detect_language=true&diarize_model=latest&utterances=true",
"headers": {
"content-type": "audio/mpeg"
},
"body": "[audio]"
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"metadata\":{\"transaction_key\":\"deprecated\",\"request_id\":\"01a0ce40-a44f-73b0-9d99-cdfc4f184268\",\"sha256\":\"95495edaebf6b10b2877de8c7b9eb7372de315c105fa8e73ed656ef2ec28ca02\",\"created\":\"2026-09-23T12:32:18.036Z\",\"duration\":1.5411875,\"channels\":1,\"models\":[\"2187e11a-3532-4498-b076-81fa530bdd49\"],\"model_info\":{\"2187e11a-3532-4498-b076-81fa530bdd49\":{\"name\":\"general-nova-3\",\"version\":\"2025-07-31.0\",\"arch\":\"nova-3\"}},\"diarize_info\":{\"model_uuid\":\"6946b038-8264-4505-acf7-9822c009fecb\",\"arch\":\"v2\"}},\"results\":{\"channels\":[{\"alternatives\":[{\"transcript\":\"Hello from OpenCode.\",\"confidence\":0.96972656,\"words\":[{\"word\":\"hello\",\"start\":0.16,\"end\":0.64,\"confidence\":0.9892578,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"Hello\"},{\"word\":\"from\",\"start\":0.64,\"end\":0.88,\"confidence\":0.96972656,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"from\"},{\"word\":\"opencode\",\"start\":0.88,\"end\":1.4399999,\"confidence\":0.9472656,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"OpenCode.\"}],\"paragraphs\":{\"transcript\":\"\\nSpeaker 0: Hello from OpenCode.\",\"paragraphs\":[{\"sentences\":[{\"text\":\"Hello from OpenCode.\",\"start\":0.16,\"end\":1.4399999}],\"speaker\":0,\"num_words\":3,\"start\":0.16,\"end\":1.4399999}]}}],\"detected_language\":\"en\",\"language_confidence\":0.9998003}],\"utterances\":[{\"start\":0.16,\"end\":1.4399999,\"confidence\":0.96875,\"channel\":0,\"transcript\":\"Hello from OpenCode.\",\"words\":[{\"word\":\"hello\",\"start\":0.16,\"end\":0.64,\"confidence\":0.9892578,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"Hello\"},{\"word\":\"from\",\"start\":0.64,\"end\":0.88,\"confidence\":0.96972656,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"from\"},{\"word\":\"opencode\",\"start\":0.88,\"end\":1.4399999,\"confidence\":0.9472656,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"OpenCode.\"}],\"speaker\":0,\"id\":\"43d717d5-ca00-48c1-a39c-381be0b1dad2\"}]}}"
}
},
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&detect_language=true&diarize_model=latest&utterances=true",
"headers": {
"content-type": "audio/mpeg"
},
"body": "[audio]"
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"metadata\":{\"transaction_key\":\"deprecated\",\"request_id\":\"01a0ce40-a609-78c2-8a9d-806e3ff8db0b\",\"sha256\":\"95495edaebf6b10b2877de8c7b9eb7372de315c105fa8e73ed656ef2ec28ca02\",\"created\":\"2026-09-23T12:32:18.478Z\",\"duration\":1.5411875,\"channels\":1,\"models\":[\"2187e11a-3532-4498-b076-81fa530bdd49\"],\"model_info\":{\"2187e11a-3532-4498-b076-81fa530bdd49\":{\"name\":\"general-nova-3\",\"version\":\"2025-07-31.0\",\"arch\":\"nova-3\"}},\"diarize_info\":{\"model_uuid\":\"6946b038-8264-4505-acf7-9822c009fecb\",\"arch\":\"v2\"}},\"results\":{\"channels\":[{\"alternatives\":[{\"transcript\":\"Hello from OpenCode.\",\"confidence\":0.96972656,\"words\":[{\"word\":\"hello\",\"start\":0.16,\"end\":0.64,\"confidence\":0.9892578,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"Hello\"},{\"word\":\"from\",\"start\":0.64,\"end\":0.88,\"confidence\":0.96972656,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"from\"},{\"word\":\"opencode\",\"start\":0.88,\"end\":1.4399999,\"confidence\":0.9472656,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"OpenCode.\"}],\"paragraphs\":{\"transcript\":\"\\nSpeaker 0: Hello from OpenCode.\",\"paragraphs\":[{\"sentences\":[{\"text\":\"Hello from OpenCode.\",\"start\":0.16,\"end\":1.4399999}],\"speaker\":0,\"num_words\":3,\"start\":0.16,\"end\":1.4399999}]}}],\"detected_language\":\"en\",\"language_confidence\":0.9998003}],\"utterances\":[{\"start\":0.16,\"end\":1.4399999,\"confidence\":0.96875,\"channel\":0,\"transcript\":\"Hello from OpenCode.\",\"words\":[{\"word\":\"hello\",\"start\":0.16,\"end\":0.64,\"confidence\":0.9892578,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"Hello\"},{\"word\":\"from\",\"start\":0.64,\"end\":0.88,\"confidence\":0.96972656,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"from\"},{\"word\":\"opencode\",\"start\":0.88,\"end\":1.4399999,\"confidence\":0.9472656,\"speaker\":0,\"speaker_confidence\":0.98,\"punctuated_word\":\"OpenCode.\"}],\"speaker\":0,\"id\":\"6f46ea75-616b-4209-b644-aae3e8f1e3c0\"}]}}"
}
}
]
}
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,32 @@
{
"version": 1,
"metadata": {
"tags": [
"prefix:openai-transcription",
"provider:openai",
"protocol:openai-transcription"
],
"name": "openai-transcription/diarizes-speakers-into-segments",
"recordedAt": "2026-09-23T12:32:01.274Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.openai.com/v1/audio/transcriptions",
"headers": {
"content-type": "multipart/form-data; boundary=----WebKitFormBoundaryf476753a62264fa88f2d753a7e85c3a4"
},
"body": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.mp3\"\r\nContent-Type: audio/mpeg\r\n\r\n[audio]\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\ngpt-4o-transcribe-diarize\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"response_format\"\r\n\r\ndiarized_json\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"chunking_strategy\"\r\n\r\nauto\r\n--BOUNDARY--\r\n"
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"text\":\"Hello from OpenCode!\",\"task\":\"transcribe\",\"duration\":1.54125,\"segments\":[{\"type\":\"transcript.text.segment\",\"text\":\" Hello from OpenCode!\",\"speaker\":\"A\",\"start\":0.30000000000000004,\"end\":1.3499999999999999,\"id\":\"seg_0\"}],\"usage\":{\"type\":\"tokens\",\"total_tokens\":107,\"input_tokens\":15,\"input_token_details\":{\"text_tokens\":0,\"audio_tokens\":15},\"output_tokens\":92}}"
}
}
]
}
@@ -0,0 +1,32 @@
{
"version": 1,
"metadata": {
"tags": [
"prefix:openai-transcription",
"provider:openai",
"protocol:openai-transcription"
],
"name": "openai-transcription/returns-whisper-word-timestamps",
"recordedAt": "2026-09-23T12:32:02.972Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.openai.com/v1/audio/transcriptions",
"headers": {
"content-type": "multipart/form-data; boundary=----WebKitFormBoundarydfe85869a7d6448994e94f2f41d9ce55"
},
"body": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.mp3\"\r\nContent-Type: audio/mpeg\r\n\r\n[audio]\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nwhisper-1\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"response_format\"\r\n\r\nverbose_json\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"timestamp_granularities[]\"\r\n\r\nword\r\n--BOUNDARY--\r\n"
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"task\":\"transcribe\",\"language\":\"english\",\"duration\":1.5399999618530273,\"text\":\"Hello from OpenCode!\",\"words\":[{\"word\":\"Hello\",\"start\":0.0,\"end\":0.6200000047683716},{\"word\":\"from\",\"start\":0.6200000047683716,\"end\":0.8799999952316284},{\"word\":\"OpenCode\",\"start\":0.8799999952316284,\"end\":1.399999976158142}],\"usage\":{\"type\":\"duration\",\"seconds\":2}}"
}
}
]
}
@@ -0,0 +1,32 @@
{
"version": 1,
"metadata": {
"tags": [
"prefix:openai-transcription",
"provider:openai",
"protocol:openai-transcription"
],
"name": "openai-transcription/streams-text-deltas",
"recordedAt": "2026-09-23T12:31:59.149Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.openai.com/v1/audio/transcriptions",
"headers": {
"content-type": "multipart/form-data; boundary=----WebKitFormBoundarya44da5204f47452ea1ef8dec145321dc"
},
"body": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.mp3\"\r\nContent-Type: audio/mpeg\r\n\r\n[audio]\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\ngpt-4o-mini-transcribe\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"stream\"\r\n\r\ntrue\r\n--BOUNDARY--\r\n"
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream; charset=utf-8"
},
"body": "data: {\"type\":\"transcript.text.delta\",\"delta\":\"Hello\"}\r\n\r\ndata: {\"type\":\"transcript.text.delta\",\"delta\":\" from\"}\r\n\r\ndata: {\"type\":\"transcript.text.delta\",\"delta\":\" Open\"}\r\n\r\ndata: {\"type\":\"transcript.text.delta\",\"delta\":\"Code\"}\r\n\r\ndata: {\"type\":\"transcript.text.delta\",\"delta\":\".\"}\r\n\r\ndata: {\"type\":\"transcript.text.done\",\"text\":\"Hello from OpenCode.\",\"usage\":{\"type\":\"tokens\",\"total_tokens\":22,\"input_tokens\":15,\"input_token_details\":{\"text_tokens\":0,\"audio_tokens\":15},\"output_tokens\":7}}\r\n\r\ndata: [DONE]\r\n\r\n"
}
}
]
}
@@ -0,0 +1,32 @@
{
"version": 1,
"metadata": {
"tags": [
"prefix:openai-transcription",
"provider:openai",
"protocol:openai-transcription"
],
"name": "openai-transcription/transcribes-with-a-language-hint",
"recordedAt": "2026-09-23T12:31:56.657Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.openai.com/v1/audio/transcriptions",
"headers": {
"content-type": "multipart/form-data; boundary=----WebKitFormBoundary484bf529b0064c43ac4a965c37433c9e"
},
"body": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.mp3\"\r\nContent-Type: audio/mpeg\r\n\r\n[audio]\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\ngpt-transcribe\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"languages[]\"\r\n\r\nen\r\n--BOUNDARY--\r\n"
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"text\":\"Hello from OpenCode.\",\"languages\":[{\"code\":\"en\"}],\"usage\":{\"type\":\"duration\",\"seconds\":2}}"
}
}
]
}
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+31 -12
View File
@@ -23,15 +23,14 @@ const scriptedRoute = (statuses: ReadonlyArray<GenerationStatus>, result: string
progress: count / statuses.length,
})
const route: GenerationRoute<string> = {
status: () => Ref.updateAndGet(polls, (count) => count + 1).pipe(Effect.map(snapshot)),
result: (token) =>
Effect.gen(function* () {
const count = yield* Ref.get(polls)
const status = snapshot(count).status
if (status === "completed") return `${result}:${String(token)}`
return yield* new AIError({ reason: new InvalidProviderOutputError({ message: `Generation ended ${status}` }) })
}),
cancel: () => Ref.set(cancelled, true),
status: Ref.updateAndGet(polls, (count) => count + 1).pipe(Effect.map(snapshot)),
result: Effect.gen(function* () {
const count = yield* Ref.get(polls)
const status = snapshot(count).status
if (status === "completed") return result
return yield* new AIError({ reason: new InvalidProviderOutputError({ message: `Generation ended ${status}` }) })
}),
cancel: Ref.set(cancelled, true),
}
return { route, polls, cancelled }
})
@@ -47,7 +46,7 @@ describe("Generation", () => {
yield* TestClock.adjust("3 seconds")
const result = yield* Fiber.join(fiber)
expect(result).toBe("done:[object Object]")
expect(result).toBe("done")
expect(yield* Ref.get(scripted.polls)).toBe(3)
}),
)
@@ -57,7 +56,7 @@ describe("Generation", () => {
const scripted = yield* scriptedRoute(["completed"], "done")
yield* Ref.set(scripted.polls, 1)
const generation = new Generation(scripted.route, "t", { id: "gen_1", status: "completed" })
expect(yield* generation.await()).toBe("done:t")
expect(yield* generation.await()).toBe("done")
expect(yield* Ref.get(scripted.polls)).toBe(1)
}),
)
@@ -80,6 +79,24 @@ describe("Generation", () => {
}),
)
it.effect("fails an event stream at the deadline even when a status poll hangs", () =>
Effect.gen(function* () {
const route: GenerationRoute<string> = {
status: Effect.never,
result: Effect.succeed("never"),
}
const generation = new Generation(route, "t", { id: "gen_1", status: "queued" })
const fiber = yield* Effect.forkChild(
generation.events({ poll: { interval: "1 second", timeout: "5 seconds" } }).pipe(Stream.runCollect, Effect.flip),
)
yield* TestClock.adjust("6 seconds")
const error = yield* Fiber.join(fiber)
expect(error.reason._tag).toBe("Timeout")
}),
)
it.effect("surfaces the route failure body for failed generations", () =>
Effect.gen(function* () {
const scripted = yield* scriptedRoute(["running", "failed"], "unused")
@@ -99,7 +116,9 @@ describe("Generation", () => {
const scripted = yield* scriptedRoute(["queued", "running", "completed"], "done")
const generation = new Generation(scripted.route, "t", { id: "gen_1", status: "queued" })
const fiber = yield* Effect.forkChild(generation.events({ poll: { interval: "1 second" } }).pipe(Stream.runCollect))
const fiber = yield* Effect.forkChild(
generation.events({ poll: { interval: "1 second" } }).pipe(Stream.runCollect),
)
yield* TestClock.adjust("3 seconds")
const events = Array.from(yield* Fiber.join(fiber))
+7
View File
@@ -63,6 +63,13 @@ describe("Media", () => {
expect(Media.detectMediaType(WEBP)).toBe("image/webp")
expect(Media.detectMediaType(PDF)).toBe("application/pdf")
expect(Media.detectMediaType(Uint8Array.from([1, 2, 3]))).toBeUndefined()
expect(Media.detectMediaType(new TextEncoder().encode("ID3\x04"))).toBe("audio/mpeg")
expect(Media.detectMediaType(Uint8Array.from([0xff, 0xfb, 0x90, 0x64]))).toBe("audio/mpeg")
expect(Media.detectMediaType(Uint8Array.from([0xff, 0xf3, 0x90, 0x64]))).toBe("audio/mpeg")
expect(Media.detectMediaType(Uint8Array.from([0xff, 0xf1, 0x50, 0x80]))).toBe("audio/aac")
expect(Media.detectMediaType(new TextEncoder().encode("RIFF\0\0\0\0WAVEfmt "))).toBe("audio/wav")
expect(Media.detectMediaType(new TextEncoder().encode("OggS\0\x02"))).toBe("audio/ogg")
expect(Media.detectMediaType(new TextEncoder().encode("fLaC\0\0\0\x22"))).toBe("audio/flac")
expect(Media.bytes(PNG).mediaType).toBe("image/png")
expect(Media.bytes(Uint8Array.from([1, 2, 3])).mediaType).toBe("application/octet-stream")
expect(Media.bytes(Uint8Array.from([1, 2, 3]), "image/x-custom").mediaType).toBe("image/x-custom")
+129 -3
View File
@@ -1,10 +1,10 @@
import { describe, expect, test } from "bun:test"
import { Effect, Layer } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { AIError, LLMEvent, Media } from "../src/index.js"
import { AIError, LLMEvent, Media, SpeechEvent, TranscriptionEvent } from "../src/index.js"
import { RequestExecutor } from "../src/route.js"
import { AI } from "../src/promise.js"
import { OpenAI } from "../src/providers.js"
import { AssemblyAI, OpenAI, Runway } from "../src/providers.js"
import { handlerLayer } from "./lib/http.js"
import { sseEvents } from "./lib/sse.js"
@@ -16,7 +16,11 @@ const chatBody = sseEvents(
{ choices: [{ delta: {}, finish_reason: "stop" }] },
)
/** Executor layer that answers chat completions with SSE text and image generations with one base64 PNG. */
/**
* Executor layer that answers chat completions with SSE text, image generations with one base64 PNG, Runway video
* tasks with a queued submission that succeeds on the second poll, speech with raw audio or SSE audio deltas, OpenAI
* transcription with JSON or SSE text deltas, and AssemblyAI transcripts that complete on the first poll.
*/
const executor = (seen: Array<string>) =>
RequestExecutor.layer.pipe(
Layer.provide(
@@ -30,6 +34,51 @@ const executor = (seen: Array<string>) =>
})
if (web.url.endsWith("/chat/completions"))
return input.respond(chatBody, { headers: { "content-type": "text/event-stream" } })
if (web.url.endsWith("/audio/speech"))
return JSON.parse(input.text).stream_format === "sse"
? input.respond(
sseEvents(
{ type: "speech.audio.delta", audio: "AQI=" },
{ type: "speech.audio.delta", audio: "Aw==" },
{ type: "speech.audio.done", usage: { input_tokens: 4, output_tokens: 8, total_tokens: 12 } },
),
{ headers: { "content-type": "text/event-stream" } },
)
: input.respond(Uint8Array.from([1, 2, 3]), { headers: { "content-type": "audio/pcm" } })
if (web.url.endsWith("/audio/transcriptions"))
return input.text.includes('name="stream"')
? input.respond(
sseEvents(
{ type: "transcript.text.delta", delta: "Hello" },
{ type: "transcript.text.delta", delta: " there." },
{ type: "transcript.text.done", text: "Hello there." },
),
{ headers: { "content-type": "text/event-stream" } },
)
: input.respond(JSON.stringify({ text: "Hello there." }), {
headers: { "content-type": "application/json" },
})
if (web.url.endsWith("/v2/transcript"))
return input.respond(JSON.stringify({ id: "tr_1", status: "queued" }), {
headers: { "content-type": "application/json" },
})
if (web.url.endsWith("/v2/transcript/tr_1"))
return input.respond(JSON.stringify({ id: "tr_1", status: "completed", text: "Hello there." }), {
headers: { "content-type": "application/json" },
})
if (web.url.endsWith("/text_to_video"))
return input.respond(JSON.stringify({ id: "task_1" }), { headers: { "content-type": "application/json" } })
if (web.url.endsWith("/tasks/task_1")) {
const polls = seen.filter((url) => url.endsWith("/tasks/task_1")).length
return input.respond(
JSON.stringify(
polls === 1
? { status: "RUNNING", progress: 0.5 }
: { status: "SUCCEEDED", output: ["https://runway.test/out.mp4"], cost: { credits: 5 } },
),
{ headers: { "content-type": "application/json" } },
)
}
return input.respond(JSON.stringify({ error: { message: "not found" } }), {
status: 404,
headers: { "content-type": "application/json" },
@@ -73,6 +122,81 @@ describe("AI promise client", () => {
await ai.dispose()
})
test("starts, awaits, and resumes video generations over the same runtime", async () => {
const seen: Array<string> = []
const ai = AI.make({ layer: executor(seen) })
const model = Runway.configure({ apiKey: "test", baseURL: "https://runway.test/v1" }).video("gen4.5")
const generation = await ai.video.start({ model, prompt: "A kite" })
expect(generation.id).toBe("task_1")
expect(generation.status).toBe("queued")
expect(generation.token).toEqual({ taskID: "task_1" })
const refreshed = await generation.refresh()
expect(refreshed.status).toBe("running")
expect(refreshed.progress).toBe(0.5)
const response = await refreshed.await({ poll: { interval: 10 } })
expect(response.video).toBeInstanceOf(Media.Asset)
expect(response.video.source).toMatchObject({ type: "url", url: "https://runway.test/out.mp4" })
expect(response.usage).toEqual({ type: "credits", credits: 5 })
const resumed = await ai.video.resume(model, JSON.parse(JSON.stringify(generation.token)))
expect(resumed.status).toBe("completed")
expect((await resumed.await()).videos).toHaveLength(1)
const events: Array<string> = []
for await (const event of ai.video.stream({ model, prompt: "A kite" }, { poll: { interval: 10 } })) {
events.push(event.type)
}
expect(events).toEqual(["video", "finish"])
expect(seen[0]).toBe("https://runway.test/v1/text_to_video")
expect(seen.filter((url) => url.endsWith("/tasks/task_1")).length).toBeGreaterThanOrEqual(5)
await ai.dispose()
})
test("generates, streams, and starts transcriptions over the same runtime", async () => {
const ai = AI.make({ layer: executor([]) })
const audio = Media.url("https://audio.test/hello.mp3")
const bytes = Media.bytes(Uint8Array.from([0x49, 0x44, 0x33]), "audio/mpeg")
expect(
(await ai.transcription.generate({ model: openai.transcription("gpt-4o-mini-transcribe"), audio: bytes })).text,
).toBe("Hello there.")
const deltas: Array<string> = []
for await (const event of ai.transcription.stream({
model: openai.transcription("gpt-4o-mini-transcribe"),
audio: bytes,
}))
if (TranscriptionEvent.is.textDelta(event)) deltas.push(event.delta)
expect(deltas).toEqual(["Hello", " there."])
const model = AssemblyAI.configure({ apiKey: "test", baseURL: "https://assemblyai.test" }).transcription(
"universal-2",
)
const generation = await ai.transcription.start({ model, audio })
expect(generation.token).toEqual({ transcriptID: "tr_1" })
expect((await generation.await({ poll: { interval: 10 } })).text).toBe("Hello there.")
await ai.dispose()
})
test("generates and streams speech over the same runtime", async () => {
const ai = AI.make({ layer: executor([]) })
const model = openai.speech("gpt-4o-mini-tts")
const response = await ai.speech.generate({ model, text: "Hello", voice: "coral", format: "pcm" })
expect(await ai.run(response.audio.bytes())).toEqual(Uint8Array.from([1, 2, 3]))
const events: Array<SpeechEvent> = []
for await (const event of ai.speech.stream({ model, text: "Hello", voice: "coral" })) events.push(event)
expect(events.map((event) => event.type)).toEqual(["audio-delta", "audio-delta", "finish"])
const finish = events.find(SpeechEvent.is.finish)
expect(await ai.run(finish!.audio.bytes())).toEqual(Uint8Array.from([1, 2, 3]))
await ai.dispose()
})
test("rethrows AIError unchanged and honors abort signals", async () => {
const ai = AI.make({ layer: executor([]) })
@@ -97,6 +221,8 @@ describe("AI promise client", () => {
test("the default client is created lazily and can be disposed", async () => {
expect(typeof AI.ai.llm.generate).toBe("function")
expect(typeof AI.ai.image.generate).toBe("function")
expect(typeof AI.ai.video.start).toBe("function")
expect(typeof AI.ai.speech.generate).toBe("function")
await AI.ai.dispose()
})
})
@@ -0,0 +1,39 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Transcription } from "../../src/index.js"
import { AssemblyAI } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
import { TRANSCRIPT, audio, audioRecording } from "./transcription-recording.js"
import { queuedPoll } from "./queued-recording.js"
// Recorded against the EU region: the US endpoint was unreachable from the recording network. Same API and shapes.
const model = AssemblyAI.configure({
apiKey: process.env.ASSEMBLYAI_API_KEY ?? "fixture",
baseURL: "https://api.eu.assemblyai.com",
}).transcription("universal-3-5-pro")
const recorded = recordedTests({
prefix: "assemblyai-transcription",
provider: "assemblyai",
protocol: "assemblyai-transcription",
requires: ["ASSEMBLYAI_API_KEY"],
options: audioRecording,
})
describe("AssemblyAI Transcription recorded", () => {
recorded.effect(
"uploads, submits, and polls a transcript",
() =>
Effect.gen(function* () {
const generation = yield* Transcription.start({ model, audio: yield* audio, diarize: true })
const response = yield* generation.await({ poll: queuedPoll })
expect(response.text).toMatch(TRANSCRIPT)
expect(response.words?.[0]).toMatchObject({ text: "Hello", speaker: "A" })
expect(response.words?.every((word) => word.startSeconds < 5 && word.endSeconds < 5)).toBe(true)
expect(response.segments?.[0]?.speaker).toBe("A")
expect(response.usage).toEqual({ type: "seconds", seconds: expect.any(Number) })
}),
{ timeout: 15 * 60 * 1000 },
)
})
@@ -0,0 +1,47 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Speech, SpeechEvent } from "../../src/index.js"
import { Cartesia } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
import { TEXT, collectSpeech } from "./speech-recording.js"
const model = Cartesia.configure({
apiKey: process.env.CARTESIA_API_KEY ?? "fixture",
}).speech("sonic-3")
// "Katie", the voice from Cartesia's quickstart.
const voice = "f786b574-daa5-4673-aa0c-cbe3e8534c02"
const recorded = recordedTests({
prefix: "cartesia-speech",
provider: "cartesia",
protocol: "cartesia-speech",
requires: ["CARTESIA_API_KEY"],
options: { redact: { allowRequestHeaders: ["cartesia-version"] } },
})
describe("Cartesia Speech recorded", () => {
recorded.effect("generates speech", () =>
Effect.gen(function* () {
const response = yield* Speech.generate({ model, text: TEXT, voice, format: "mp3" })
expect(response.audio.mediaType).toBe("audio/mpeg")
expect(response.audio.info).toMatchObject({ format: "mp3", sampleRate: 44100 })
expect((yield* response.audio.bytes()).length).toBeGreaterThan(0)
}),
)
recorded.effect("streams speech with timestamps", () =>
Effect.gen(function* () {
const { events, finish } = yield* collectSpeech(
Speech.stream({ model, text: TEXT, voice, timestamps: true, providerOptions: { sampleRate: 24000 } }),
)
const words = events.filter(SpeechEvent.is.timestamps).flatMap((event) => event.items)
expect(finish.audio.mediaType).toBe("audio/pcm")
expect(finish.audio.info).toMatchObject({ encoding: "pcm_s16le", sampleRate: 24000, channels: 1 })
expect(words.length).toBeGreaterThan(0)
expect(words.every((item) => item.endSeconds >= item.startSeconds)).toBe(true)
}),
)
})
@@ -0,0 +1,41 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Speech } from "../../src/index.js"
import { Deepgram } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
import { TEXT, collectSpeech } from "./speech-recording.js"
const model = Deepgram.configure({
apiKey: process.env.DEEPGRAM_API_KEY ?? "fixture",
}).speech("aura-2-thalia-en")
const recorded = recordedTests({
prefix: "deepgram-speech",
provider: "deepgram",
protocol: "deepgram-speech",
requires: ["DEEPGRAM_API_KEY"],
// Usage and the request id arrive only as response headers.
options: { redact: { allowResponseHeaders: ["dg-char-count", "dg-request-id", "dg-model-name"] } },
})
describe("Deepgram Speech recorded", () => {
recorded.effect("generates speech", () =>
Effect.gen(function* () {
const response = yield* Speech.generate({ model, text: TEXT })
expect(response.audio.mediaType).toBe("audio/mpeg")
expect(response.audio.info).toEqual({ format: "mp3" })
expect((yield* response.audio.bytes()).length).toBeGreaterThan(0)
expect(response.usage).toEqual({ type: "characters", characters: expect.any(Number) })
}),
)
recorded.effect("streams speech", () =>
Effect.gen(function* () {
const { finish } = yield* collectSpeech(Speech.stream({ model, text: TEXT, format: "pcm" }))
expect(finish.audio.info).toMatchObject({ format: "pcm", encoding: "pcm_s16le", channels: 1 })
expect(finish.usage).toEqual({ type: "characters", characters: expect.any(Number) })
}),
)
})
@@ -0,0 +1,47 @@
import { describe, expect } from "bun:test"
import { Effect, Stream } from "effect"
import { Transcription } from "../../src/index.js"
import { Deepgram } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
import { TRANSCRIPT, audio, audioRecording, dialog } from "./transcription-recording.js"
const model = Deepgram.configure({ apiKey: process.env.DEEPGRAM_API_KEY ?? "fixture" }).transcription("nova-3")
const recorded = recordedTests({
prefix: "deepgram-transcription",
provider: "deepgram",
protocol: "deepgram-transcription",
requires: ["DEEPGRAM_API_KEY"],
options: audioRecording,
})
describe("Deepgram Transcription recorded", () => {
recorded.effect("transcribes raw audio with speakers and word timestamps", () =>
Effect.gen(function* () {
const request = Transcription.request({ model, audio: yield* audio, diarize: true, timestamps: "word" })
const response = yield* Transcription.generate(request)
expect(response.text).toMatch(TRANSCRIPT)
expect(response.words?.map((word) => word.text)).toEqual(["Hello", "from", "OpenCode."])
expect(response.words?.every((word) => word.speaker === "0" && word.confidence !== undefined)).toBe(true)
expect(response.segments).toEqual([
{ text: response.text, startSeconds: expect.any(Number), endSeconds: expect.any(Number), speaker: "0" },
])
expect(response.language).toBe("en")
expect(response.usage).toEqual({ type: "seconds", seconds: response.durationSeconds })
const events = Array.from(yield* Stream.runCollect(Transcription.stream(request)))
expect(events.map((event) => event.type)).toEqual(["finish"])
}),
)
recorded.effect("splits utterances at speaker changes", () =>
Effect.gen(function* () {
const response = yield* Transcription.generate({ model, audio: yield* dialog, diarize: true })
expect(response.segments?.map((segment) => segment.speaker)).toEqual(["0", "1"])
expect(response.segments?.[0].text).toMatch(/release ship\?$/)
expect(response.segments?.[1].text).toMatch(/shipped this morning\.$/)
}),
)
})
@@ -0,0 +1,54 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Speech, SpeechEvent } from "../../src/index.js"
import { ElevenLabs } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
import { TEXT, collectSpeech } from "./speech-recording.js"
const model = ElevenLabs.configure({
apiKey: process.env.ELEVENLABS_API_KEY ?? "fixture",
}).speech("eleven_flash_v2_5")
// "George", an ElevenLabs premade voice.
const voice = "JBFqnCBsd6RMkjVDRZzb"
const recorded = recordedTests({
prefix: "elevenlabs-speech",
provider: "elevenlabs",
protocol: "elevenlabs-speech",
requires: ["ELEVENLABS_API_KEY"],
// Usage and the request id arrive only as response headers.
options: { redact: { allowResponseHeaders: ["character-cost", "request-id"] } },
})
describe("ElevenLabs Speech recorded", () => {
recorded.effect("generates speech", () =>
Effect.gen(function* () {
const response = yield* Speech.generate({ model, text: TEXT, voice })
expect(response.audio.mediaType).toBe("audio/mpeg")
expect(response.audio.info).toMatchObject({ format: "mp3", sampleRate: 44100 })
expect((yield* response.audio.bytes()).length).toBeGreaterThan(0)
expect(response.usage).toEqual({ type: "credits", credits: 3 })
}),
)
recorded.effect("streams speech with timestamps", () =>
Effect.gen(function* () {
const { events, finish } = yield* collectSpeech(
Speech.stream({ model, text: TEXT, voice, format: "pcm", timestamps: true }),
)
const timestamps = events.filter(SpeechEvent.is.timestamps).flatMap((event) => event.items)
expect(finish.audio.mediaType).toBe("audio/pcm")
expect(finish.audio.info).toMatchObject({ encoding: "pcm_s16le", sampleRate: 24000, channels: 1 })
expect(
timestamps
.map((item) => item.text)
.join("")
.trim(),
).toBe(TEXT)
expect(timestamps.every((item) => item.endSeconds >= item.startSeconds)).toBe(true)
}),
)
})
@@ -0,0 +1,41 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Video } from "../../src/index.js"
import { Fal } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
import { queuedPoll } from "./queued-recording.js"
const model = Fal.configure({
apiKey: process.env.FAL_KEY ?? "fixture",
}).video("fal-ai/veo3.1/fast")
const recorded = recordedTests({
prefix: "fal-video",
provider: "fal",
protocol: "fal-video",
requires: ["FAL_KEY"],
})
describe("fal Video recorded", () => {
recorded.effect(
"generates a video",
() =>
Effect.gen(function* () {
const response = yield* Video.generate(
{
model,
prompt: "A single red balloon drifting slowly upward against a clear blue sky.",
aspectRatio: "16:9",
resolution: "720p",
providerOptions: { duration: "4s" },
},
{ poll: queuedPoll },
)
expect(response.videos).toHaveLength(1)
expect(response.video.source.type).toBe("url")
expect((yield* response.video.bytes()).length).toBeGreaterThan(0)
}),
{ timeout: 15 * 60 * 1000 },
)
})
@@ -0,0 +1,45 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Speech } from "../../src/index.js"
import { Google } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
import { TEXT, collectSpeech } from "./speech-recording.js"
const google = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture" })
const recorded = recordedTests({
prefix: "google-speech",
provider: "google",
protocol: "google-speech",
requires: ["GOOGLE_GENERATIVE_AI_API_KEY"],
})
describe("Google Speech recorded", () => {
recorded.effect("generates speech", () =>
Effect.gen(function* () {
const response = yield* Speech.generate({
model: google.speech("gemini-2.5-flash-preview-tts"),
text: `Say: ${TEXT}`,
voice: "Kore",
})
expect(response.audio.mediaType).toMatch(/^audio\/l16/i)
expect(response.audio.info).toMatchObject({ encoding: "pcm_s16le", sampleRate: 24000, channels: 1 })
expect((yield* response.audio.bytes()).length).toBeGreaterThan(0)
expect(response.usage?.type).toBe("tokens")
}),
)
// Only Gemini 3.1 TTS and later support `streamGenerateContent`.
recorded.effect("streams speech", () =>
Effect.gen(function* () {
const { finish } = yield* collectSpeech(
Speech.stream({ model: google.speech("gemini-3.1-flash-tts-preview"), text: `Say: ${TEXT}`, voice: "Kore" }),
)
expect(finish.audio.mediaType).toMatch(/^audio\/l16/i)
expect(finish.audio.info).toMatchObject({ encoding: "pcm_s16le", sampleRate: 24000, channels: 1 })
expect(finish.usage?.type).toBe("tokens")
}),
)
})
@@ -0,0 +1,54 @@
import { describe, expect } from "bun:test"
import { Effect, Stream } from "effect"
import { Transcription, TranscriptionEvent } from "../../src/index.js"
import { Google } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
import { TRANSCRIPT, audio, audioRecording, dialog } from "./transcription-recording.js"
const model = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture" }).transcription(
"gemini-3.5-transcribe",
)
const recorded = recordedTests({
prefix: "google-transcription",
provider: "google",
protocol: "google-transcription",
requires: ["GOOGLE_GENERATIVE_AI_API_KEY"],
options: audioRecording,
})
describe("Google Transcription recorded", () => {
recorded.effect("transcribes with speakers and word timestamps", () =>
Effect.gen(function* () {
const response = yield* Transcription.generate({ model, audio: yield* audio, diarize: true, language: "en-US" })
expect(response.text).toMatch(TRANSCRIPT)
expect(response.words?.length).toBeGreaterThan(2)
expect(response.words?.every((word) => word.endSeconds > word.startSeconds)).toBe(true)
expect(response.segments).toEqual([
{
text: response.text,
startSeconds: response.words?.[0]?.startSeconds,
endSeconds: response.words?.at(-1)?.endSeconds,
speaker: expect.any(String),
},
])
expect(response.usage?.type).toBe("tokens")
}),
)
recorded.effect("streams speaker turns", () =>
Effect.gen(function* () {
const events = Array.from(
yield* Stream.runCollect(Transcription.stream({ model, audio: yield* dialog, diarize: true })),
)
const finish = events.at(-1)
if (finish === undefined || !TranscriptionEvent.is.finish(finish)) throw new Error("Expected a finish event")
expect(finish.text).toMatch(/release ship\? Yes, it shipped this morning\.$/)
const speakers = events.filter(TranscriptionEvent.is.segment).map((event) => event.segment.speaker)
expect(speakers).toHaveLength(2)
expect(new Set(speakers).size).toBe(2)
}),
)
})
@@ -0,0 +1,41 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Video } from "../../src/index.js"
import { Google } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
import { queuedPoll } from "./queued-recording.js"
const model = Google.configure({
apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture",
}).video("veo-3.1-fast-generate-preview")
const recorded = recordedTests({
prefix: "google-video",
provider: "google",
protocol: "google-video",
requires: ["GOOGLE_GENERATIVE_AI_API_KEY"],
})
describe("Google Veo recorded", () => {
recorded.effect(
"generates a video",
() =>
Effect.gen(function* () {
const response = yield* Video.generate(
{
model,
prompt: "A single red balloon drifting slowly upward against a clear blue sky.",
aspectRatio: "16:9",
durationSeconds: 4,
},
{ poll: queuedPoll },
)
expect(response.videos).toHaveLength(1)
expect(response.video.source.type).toBe("url")
expect(response.video.headers?.["x-goog-api-key"]).toBeString()
expect((yield* response.video.bytes()).length).toBeGreaterThan(0)
}),
{ timeout: 15 * 60 * 1000 },
)
})
@@ -791,6 +791,89 @@ describe("OpenAI Chat route", () => {
}),
)
it.effect("lowers inline PDFs as file parts", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model,
messages: [
Message.user([
{ type: "text", text: "Summarize these." },
{ type: "media", media: Media.base64("JVBERi0=", "application/pdf"), filename: "report.pdf" },
{ type: "media", media: Media.fromDataUrl("data:application/pdf;base64,JVBERi0=") },
]),
],
}),
)
expect(prepared.body.messages).toEqual([
{
role: "user",
content: [
{ type: "text", text: "Summarize these." },
{ type: "file", file: { filename: "report.pdf", file_data: "data:application/pdf;base64,JVBERi0=" } },
{ type: "file", file: { filename: "document.pdf", file_data: "data:application/pdf;base64,JVBERi0=" } },
],
},
])
}),
)
it.effect("moves PDFs from tool results into a follow-up user message", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model,
messages: [
Message.user("Read the report."),
Message.assistant([ToolCallPart.make({ id: "call_pdf", name: "read", input: {} })]),
Message.tool({
id: "call_pdf",
name: "read",
resultType: "content",
result: [
{ type: "text", text: "PDF read successfully" },
{
type: "file",
mime: "application/pdf",
uri: "data:application/pdf;base64,JVBERi0=",
name: "report.pdf",
},
],
}),
],
}),
)
expect(prepared.body.messages).toContainEqual({
role: "tool",
tool_call_id: "call_pdf",
content: "PDF read successfully",
})
expect(prepared.body.messages.at(-1)).toEqual({
role: "user",
content: [
{ type: "file", file: { filename: "report.pdf", file_data: "data:application/pdf;base64,JVBERi0=" } },
],
})
}),
)
it.effect("requires inline data for PDF files", () =>
Effect.gen(function* () {
const error = yield* compileRequest(
LLM.request({
model,
messages: [
Message.user({
type: "media",
media: Media.url("https://example.com/report.pdf", { mediaType: "application/pdf" }),
}),
],
}),
).pipe(Effect.flip)
expect(error.message).toContain("OpenAI Chat requires inline media")
}),
)
it.effect("prepares raw and data URL image media as vision input", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(

Some files were not shown because too many files have changed in this diff Show More