mirror of
https://github.com/anomalyco/opencode.git
synced 2026-09-25 18:17:35 +00:00
Compare commits
157
Commits
sync-console-docs
...
v2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6585bb7105 | ||
|
|
f954688fbb | ||
|
|
1463dabde9 | ||
|
|
29ea6ee05b | ||
|
|
b170904731 | ||
|
|
88e1fa9304 | ||
|
|
ff1bf315ed | ||
|
|
144ce00e00 | ||
|
|
ad504094f0 | ||
|
|
bad6834a3e | ||
|
|
0c4bbc3cd1 | ||
|
|
ae7dd82126 | ||
|
|
65d5123ead | ||
|
|
4eb46a8885 | ||
|
|
1986e92842 | ||
|
|
14fc63ba9e | ||
|
|
c34ffa117e | ||
|
|
beeb14e910 | ||
|
|
aae42e2e75 | ||
|
|
cc9011c1ae | ||
|
|
6cd938e1e9 | ||
|
|
7de6b3fc15 | ||
|
|
c1c9a13993 | ||
|
|
917d904f18 | ||
|
|
ee5b67eb84 | ||
|
|
048a47e89e | ||
|
|
5335347e80 | ||
|
|
16b18dff13 | ||
|
|
61c2349cef | ||
|
|
684721efb8 | ||
|
|
962c14a49c | ||
|
|
85b98e7da4 | ||
|
|
8061220b08 | ||
|
|
b02cc35f13 | ||
|
|
e23d89c9a9 | ||
|
|
e8b3e19e85 | ||
|
|
5256f30957 | ||
|
|
61ecf404b9 | ||
|
|
92d2b1700f | ||
|
|
56262121ee | ||
|
|
e3b588e7d2 | ||
|
|
1de648cb13 | ||
|
|
03be7f385b | ||
|
|
8118690839 | ||
|
|
a16eedfed7 | ||
|
|
e796f2f9a5 | ||
|
|
20610e6645 | ||
|
|
7f245b0968 | ||
|
|
7013e925f5 | ||
|
|
499c2feaa3 | ||
|
|
03af821aa5 | ||
|
|
c903774556 | ||
|
|
14aaf91e65 | ||
|
|
c832432d89 | ||
|
|
1d431a80df | ||
|
|
19a9e41c28 | ||
|
|
d932cad09c | ||
|
|
e22c1622e0 | ||
|
|
0e4ad0ce76 | ||
|
|
6afcc887c8 | ||
|
|
c1f50659a7 | ||
|
|
32d3535f66 | ||
|
|
9810d98bc2 | ||
|
|
2caba90a63 | ||
|
|
eccf0b3b7b | ||
|
|
808588e9b9 | ||
|
|
c35c211460 | ||
|
|
efb8e23dcc | ||
|
|
ad53e39d2e | ||
|
|
3ddfc15df5 | ||
|
|
ac2426e103 | ||
|
|
0bc8b8dbeb | ||
|
|
dca73ba9e3 | ||
|
|
318a8c1aba | ||
|
|
bee5014f89 | ||
|
|
8652729fdb | ||
|
|
bfcaf1b0cd | ||
|
|
f24fdeca70 | ||
|
|
dc48655743 | ||
|
|
ab52135b4e | ||
|
|
e31f78af78 | ||
|
|
bc3669b97f | ||
|
|
7460d855ac | ||
|
|
683d470fe4 | ||
|
|
ef4938d344 | ||
|
|
b098b0e06f | ||
|
|
27dfac3d36 | ||
|
|
5253661d36 | ||
|
|
ff6b8c21e7 | ||
|
|
8e39040eed | ||
|
|
95cfca73b0 | ||
|
|
f0aff6cfe8 | ||
|
|
b7d707cd64 | ||
|
|
b486652067 | ||
|
|
7d1e212fb5 | ||
|
|
2c369a21c9 | ||
|
|
f2e73dd03f | ||
|
|
88a9688c9c | ||
|
|
8d5eb3bf9c | ||
|
|
6e0857cc0e | ||
|
|
757e565c23 | ||
|
|
f53cc41657 | ||
|
|
1b6acc3347 | ||
|
|
7d126eb1e5 | ||
|
|
bbbac2507b | ||
|
|
7f93d0018c | ||
|
|
33b686feb9 | ||
|
|
fb4422ef62 | ||
|
|
d31ce33ca5 | ||
|
|
56db9053e8 | ||
|
|
c98d9c65dc | ||
|
|
b47824fb63 | ||
|
|
f51dc62721 | ||
|
|
ad15173aa7 | ||
|
|
e64e4d0fbb | ||
|
|
7f39ae9ecb | ||
|
|
b3b030856b | ||
|
|
a117ebb408 | ||
|
|
f2d9ebb886 | ||
|
|
0332a26be6 | ||
|
|
bb7876dfa8 | ||
|
|
dbaa57a21b | ||
|
|
b6bc55764c | ||
|
|
14a3311a61 | ||
|
|
dddb5eb96f | ||
|
|
affa57e40f | ||
|
|
681ea07a92 | ||
|
|
6e89d9e2c3 | ||
|
|
f526727178 | ||
|
|
9c8a63e852 | ||
|
|
fabf56781c | ||
|
|
a25d304201 | ||
|
|
cc8886c8bb | ||
|
|
8ce629be22 | ||
|
|
68b28bdb98 | ||
|
|
150dc69e4b | ||
|
|
d5d4461e67 | ||
|
|
d56ce74373 | ||
|
|
17abc5906b | ||
|
|
8656838a5b | ||
|
|
53179daefa | ||
|
|
bab26d63ea | ||
|
|
f0381e5da3 | ||
|
|
740072694d | ||
|
|
2e4abeb25d | ||
|
|
43f1dad8e1 | ||
|
|
cf4b4c2312 | ||
|
|
3bf8a5a8cf | ||
|
|
ddeb19790a | ||
|
|
fe0d1682ca | ||
|
|
1746672c42 | ||
|
|
3a2203eaac | ||
|
|
38c320ea4c | ||
|
|
5c53cfc342 | ||
|
|
8683406690 | ||
|
|
126294a322 | ||
|
|
d2bbefbac8 |
@@ -53,8 +53,6 @@ runs:
|
||||
with:
|
||||
path: ${{ steps.cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-bun-
|
||||
|
||||
- name: Install setuptools for distutils compatibility
|
||||
run: python3 -m pip install setuptools || pip install setuptools || true
|
||||
@@ -66,9 +64,9 @@ runs:
|
||||
# e.g. ./patches/ for standard-openapi
|
||||
# https://github.com/oven-sh/bun/issues/28147
|
||||
if [ "$RUNNER_OS" = "Windows" ]; then
|
||||
bun install --linker hoisted ${{ inputs.install-flags }}
|
||||
bun install --frozen-lockfile --linker hoisted ${{ inputs.install-flags }}
|
||||
else
|
||||
bun install ${{ inputs.install-flags }}
|
||||
bun install --frozen-lockfile ${{ inputs.install-flags }}
|
||||
fi
|
||||
shell: bash
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
},
|
||||
"packages/ai": {
|
||||
"name": "@opencode/ai",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@aws-sdk/credential-providers": "3.1057.0",
|
||||
"@opencode/schema": "workspace:*",
|
||||
@@ -54,7 +54,7 @@
|
||||
},
|
||||
"packages/app": {
|
||||
"name": "@opencode/app",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@corvu/drawer": "catalog:",
|
||||
"@dnd-kit/abstract": "0.5.0",
|
||||
@@ -112,8 +112,9 @@
|
||||
},
|
||||
"packages/cli": {
|
||||
"name": "@opencode/cli",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"bin": {
|
||||
"opencode": "./bin/opencode.cjs",
|
||||
"opencode2": "./bin/opencode2.cjs",
|
||||
},
|
||||
"dependencies": {
|
||||
@@ -176,7 +177,7 @@
|
||||
},
|
||||
"packages/client": {
|
||||
"name": "@opencode/client",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@opencode/protocol": "workspace:*",
|
||||
"@opencode/schema": "workspace:*",
|
||||
@@ -202,7 +203,7 @@
|
||||
},
|
||||
"packages/codemode": {
|
||||
"name": "@opencode/codemode",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"acorn": "8.15.0",
|
||||
"effect": "catalog:",
|
||||
@@ -215,7 +216,7 @@
|
||||
},
|
||||
"packages/console/app": {
|
||||
"name": "@opencode/console-app",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@cloudflare/vite-plugin": "1.15.2",
|
||||
"@ibm/plex": "6.4.1",
|
||||
@@ -251,7 +252,7 @@
|
||||
},
|
||||
"packages/console/core": {
|
||||
"name": "@opencode/console-core",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-sts": "3.782.0",
|
||||
"@jsx-email/render": "1.1.1",
|
||||
@@ -278,7 +279,7 @@
|
||||
},
|
||||
"packages/console/function": {
|
||||
"name": "@opencode/console-function",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@openauthjs/openauth": "0.0.0-20250322224806",
|
||||
"@opencode/console-core": "workspace:*",
|
||||
@@ -295,7 +296,7 @@
|
||||
},
|
||||
"packages/console/mail": {
|
||||
"name": "@opencode/console-mail",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@jsx-email/all": "2.2.3",
|
||||
"@jsx-email/cli": "1.4.3",
|
||||
@@ -319,7 +320,7 @@
|
||||
},
|
||||
"packages/console/support": {
|
||||
"name": "@opencode/console-support",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@cloudflare/vite-plugin": "1.15.2",
|
||||
"@opencode/console-core": "workspace:*",
|
||||
@@ -339,7 +340,7 @@
|
||||
},
|
||||
"packages/core": {
|
||||
"name": "@opencode/core",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@ai-sdk/cohere": "3.0.27",
|
||||
"@ai-sdk/gateway": "3.0.104",
|
||||
@@ -367,7 +368,7 @@
|
||||
"drizzle-orm": "catalog:",
|
||||
"effect": "catalog:",
|
||||
"fuzzysort": "3.1.0",
|
||||
"gitlab-ai-provider": "6.12.1",
|
||||
"gitlab-ai-provider": "6.18.0",
|
||||
"google-auth-library": "10.5.0",
|
||||
"gray-matter": "4.0.3",
|
||||
"htmlparser2": "8.0.2",
|
||||
@@ -407,7 +408,7 @@
|
||||
},
|
||||
"packages/desktop": {
|
||||
"name": "@opencode/desktop",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@zip.js/zip.js": "2.7.62",
|
||||
"electron-context-menu": "5.0.0",
|
||||
@@ -456,7 +457,7 @@
|
||||
},
|
||||
"packages/enterprise": {
|
||||
"name": "@opencode/enterprise",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@hono/standard-validator": "catalog:",
|
||||
"@opencode-ai/sdk": "1.18.21",
|
||||
@@ -493,7 +494,7 @@
|
||||
},
|
||||
"packages/function": {
|
||||
"name": "@opencode/function",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@octokit/auth-app": "8.0.1",
|
||||
"@octokit/rest": "catalog:",
|
||||
@@ -509,7 +510,7 @@
|
||||
},
|
||||
"packages/http-recorder": {
|
||||
"name": "@opencode/http-recorder",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@effect/platform-node-shared": "4.0.0-rc.112",
|
||||
},
|
||||
@@ -528,7 +529,7 @@
|
||||
},
|
||||
"packages/httpapi-codegen": {
|
||||
"name": "@opencode/httpapi-codegen",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"effect": "catalog:",
|
||||
"prettier": "3.6.2",
|
||||
@@ -541,7 +542,7 @@
|
||||
},
|
||||
"packages/latex": {
|
||||
"name": "@opencode/latex",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@opencode/plugin": "workspace:*",
|
||||
"@opentui/core": "catalog:",
|
||||
@@ -555,7 +556,7 @@
|
||||
},
|
||||
"packages/merman": {
|
||||
"name": "@opencode/merman",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@opencode/plugin": "workspace:*",
|
||||
"@opentui/core": "catalog:",
|
||||
@@ -570,7 +571,7 @@
|
||||
},
|
||||
"packages/plugin": {
|
||||
"name": "@opencode/plugin",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@ai-sdk/provider": "3.0.8",
|
||||
"@opencode/ai": "workspace:*",
|
||||
@@ -596,8 +597,8 @@
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opencode/theme": "workspace:*",
|
||||
"@opentui/core": ">=0.5.10",
|
||||
"@opentui/solid": ">=0.5.10",
|
||||
"@opentui/core": ">=0.5.12",
|
||||
"@opentui/solid": ">=0.5.12",
|
||||
"solid-js": ">=1.9.0",
|
||||
},
|
||||
"optionalPeers": [
|
||||
@@ -609,7 +610,7 @@
|
||||
},
|
||||
"packages/plugin-browser": {
|
||||
"name": "@opencode/plugin-browser",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@opencode/plugin": "workspace:*",
|
||||
"@opencode/schema": "workspace:*",
|
||||
@@ -639,7 +640,7 @@
|
||||
},
|
||||
"packages/protocol": {
|
||||
"name": "@opencode/protocol",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@opencode/schema": "workspace:*",
|
||||
"effect": "catalog:",
|
||||
@@ -654,7 +655,7 @@
|
||||
},
|
||||
"packages/schema": {
|
||||
"name": "@opencode/schema",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@standard-schema/spec": "catalog:",
|
||||
"effect": "catalog:",
|
||||
@@ -678,7 +679,7 @@
|
||||
},
|
||||
"packages/sdk": {
|
||||
"name": "@opencode/sdk",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@opencode/client": "workspace:*",
|
||||
"@opencode/core": "workspace:*",
|
||||
@@ -699,7 +700,7 @@
|
||||
},
|
||||
"packages/server": {
|
||||
"name": "@opencode/server",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@effect/platform-node": "catalog:",
|
||||
"@effect/platform-node-shared": "catalog:",
|
||||
@@ -721,7 +722,7 @@
|
||||
},
|
||||
"packages/session-ui": {
|
||||
"name": "@opencode/session-ui",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@kobalte/core": "catalog:",
|
||||
"@opencode/client": "workspace:*",
|
||||
@@ -756,7 +757,7 @@
|
||||
},
|
||||
"packages/simulation": {
|
||||
"name": "@opencode/simulation",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@opencode/ai": "workspace:*",
|
||||
"@opencode/core": "workspace:*",
|
||||
@@ -776,7 +777,7 @@
|
||||
},
|
||||
"packages/stats/app": {
|
||||
"name": "@opencode/stats-app",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@ibm/plex": "6.4.1",
|
||||
"@kobalte/core": "catalog:",
|
||||
@@ -810,7 +811,7 @@
|
||||
},
|
||||
"packages/stats/core": {
|
||||
"name": "@opencode/stats-core",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-athena": "3.933.0",
|
||||
"@planetscale/database": "1.19.0",
|
||||
@@ -829,7 +830,7 @@
|
||||
},
|
||||
"packages/stats/server": {
|
||||
"name": "@opencode/stats-server",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-firehose": "3.933.0",
|
||||
"@effect/platform-node": "catalog:",
|
||||
@@ -875,7 +876,7 @@
|
||||
},
|
||||
"packages/theme": {
|
||||
"name": "@opencode/theme",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@opentui/core": "catalog:",
|
||||
"effect": "catalog:",
|
||||
@@ -889,7 +890,7 @@
|
||||
},
|
||||
"packages/tui": {
|
||||
"name": "@opencode/tui",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@opencode/client": "workspace:*",
|
||||
"@opencode/core": "workspace:*",
|
||||
@@ -924,7 +925,7 @@
|
||||
},
|
||||
"packages/ui": {
|
||||
"name": "@opencode/ui",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@kobalte/core": "catalog:",
|
||||
"@pierre/diffs": "catalog:",
|
||||
@@ -959,7 +960,7 @@
|
||||
},
|
||||
"packages/util": {
|
||||
"name": "@opencode/util",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@effect/opentelemetry": "catalog:",
|
||||
"@effect/platform-node": "catalog:",
|
||||
@@ -967,10 +968,14 @@
|
||||
"@npmcli/arborist": "catalog:",
|
||||
"@npmcli/config": "10.8.1",
|
||||
"@opentelemetry/api": "1.9.0",
|
||||
"@opentelemetry/context-async-hooks": "2.6.1",
|
||||
"@opentelemetry/exporter-trace-otlp-http": "0.214.0",
|
||||
"@opentelemetry/sdk-trace-base": "2.6.1",
|
||||
"@opentelemetry/sdk-trace-node": "2.6.1",
|
||||
"@opentelemetry/api-logs": "0.219.0",
|
||||
"@opentelemetry/context-async-hooks": "2.8.0",
|
||||
"@opentelemetry/exporter-trace-otlp-http": "0.219.0",
|
||||
"@opentelemetry/resources": "2.8.0",
|
||||
"@opentelemetry/sdk-logs": "0.219.0",
|
||||
"@opentelemetry/sdk-metrics": "2.8.0",
|
||||
"@opentelemetry/sdk-trace-base": "2.8.0",
|
||||
"@opentelemetry/sdk-trace-node": "2.8.0",
|
||||
"cross-spawn": "catalog:",
|
||||
"effect": "catalog:",
|
||||
"glob": "13.0.5",
|
||||
@@ -992,7 +997,7 @@
|
||||
},
|
||||
"packages/web": {
|
||||
"name": "@opencode/web",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"@astrojs/cloudflare": "12.6.3",
|
||||
"@astrojs/markdown-remark": "6.3.1",
|
||||
@@ -1033,7 +1038,7 @@
|
||||
},
|
||||
"services/update": {
|
||||
"name": "@opencode/update",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"dependencies": {
|
||||
"jose": "6.0.11",
|
||||
"semver": "catalog:",
|
||||
@@ -1068,13 +1073,13 @@
|
||||
"trustedDependencies": [
|
||||
"electron",
|
||||
"esbuild",
|
||||
"protobufjs",
|
||||
],
|
||||
"patchedDependencies": {
|
||||
"@pierre/trees@1.0.0-beta.4": "patches/@pierre%2Ftrees@1.0.0-beta.4.patch",
|
||||
"@tanstack/virtual-core@3.17.8": "patches/@tanstack%2Fvirtual-core@3.17.8.patch",
|
||||
"ghostty-web@github:anomalyco/ghostty-web#83c0a07": "patches/ghostty-web@0.3.0.patch",
|
||||
"@modelcontextprotocol/client@2.0.0": "patches/@modelcontextprotocol%2Fclient@2.0.0.patch",
|
||||
"pacote@21.5.1": "patches/pacote@21.5.1.patch",
|
||||
"@standard-community/standard-openapi@0.2.9": "patches/@standard-community%2Fstandard-openapi@0.2.9.patch",
|
||||
"@npmcli/agent@4.0.2": "patches/@npmcli%2Fagent@4.0.2.patch",
|
||||
"@silvia-odwyer/photon-node@0.3.4": "patches/@silvia-odwyer%2Fphoton-node@0.3.4.patch",
|
||||
@@ -1109,9 +1114,9 @@
|
||||
"@npmcli/arborist": "9.4.0",
|
||||
"@octokit/rest": "22.0.0",
|
||||
"@openauthjs/openauth": "0.0.0-20250322224806",
|
||||
"@opentui/core": "0.5.10",
|
||||
"@opentui/keymap": "0.5.10",
|
||||
"@opentui/solid": "0.5.10",
|
||||
"@opentui/core": "0.5.12",
|
||||
"@opentui/keymap": "0.5.12",
|
||||
"@opentui/solid": "0.5.12",
|
||||
"@pierre/diffs": "1.2.10",
|
||||
"@playwright/test": "1.59.1",
|
||||
"@sentry/solid": "10.71.0",
|
||||
@@ -2219,55 +2224,55 @@
|
||||
|
||||
"@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="],
|
||||
|
||||
"@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.214.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-40lSJeqYO8Uz2Yj7u94/SJWE/wONa7rmMKjI1ZcIjgf3MHNHv1OZUCrCETGuaRF62d5pQD1wKIW+L4lmSMTzZA=="],
|
||||
"@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.219.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-FFx7YnaYJlIjqWW/AG/yAZ0L/NEY724PipXXXQLdtZPbLwBGbUMTGL1i/esI56TWfTUXxhLfpgrnWJCG8aUJyg=="],
|
||||
|
||||
"@opentelemetry/context-async-hooks": ["@opentelemetry/context-async-hooks@2.6.1", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-XHzhwRNkBpeP8Fs/qjGrAf9r9PRv67wkJQ/7ZPaBQQ68DYlTBBx5MF9LvPx7mhuXcDessKK2b+DcxqwpgkcivQ=="],
|
||||
"@opentelemetry/context-async-hooks": ["@opentelemetry/context-async-hooks@2.8.0", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-/3FIraneMcng67SUJCxvyInk/oxzwsxyadufk0wwfOBLf5wqtAGX4MoQASwSbndBPeARzBryUM9Azr5kHIdWLw=="],
|
||||
|
||||
"@opentelemetry/core": ["@opentelemetry/core@2.6.1", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-8xHSGWpJP9wBxgBpnqGL0R3PbdWQndL1Qp50qrg71+B28zK5OQmUgcDKLJgzyAAV38t4tOyLMGDD60LneR5W8g=="],
|
||||
"@opentelemetry/core": ["@opentelemetry/core@2.8.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-hd1Lfh8p545nNz+jq1Ejfz+Mn1hyLuxYn1YzTfFNrxr8urEWMNQLPf1Th8kjOH+HxwawCrtgBp8JpBUR4ZSgww=="],
|
||||
|
||||
"@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/exporter-trace-otlp-http@0.214.0", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/otlp-exporter-base": "0.214.0", "@opentelemetry/otlp-transformer": "0.214.0", "@opentelemetry/resources": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-kIN8nTBMgV2hXzV/a20BCFilPZdAIMYYJGSgfMMRm/Xa+07y5hRDS2Vm12A/z8Cdu3Sq++ZvJfElokX2rkgGgw=="],
|
||||
"@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/exporter-trace-otlp-http@0.219.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/otlp-exporter-base": "0.219.0", "@opentelemetry/otlp-transformer": "0.219.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-9t6SvBXXBEjOBcIzgozvBbd3jWrv3Gt3ngGhl1fhdZ/zRc7oZDVOFEqbi2zlBpW9BXhgDMKv422J0DL/3iQWfw=="],
|
||||
|
||||
"@opentelemetry/instrumentation": ["@opentelemetry/instrumentation@0.220.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "import-in-the-middle": "^3.0.0", "require-in-the-middle": "^8.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw=="],
|
||||
|
||||
"@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.214.0", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/otlp-transformer": "0.214.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-u1Gdv0/E9wP+apqWf7Wv2npXmgJtxsW2XL0TEv9FZloTZRuMBKmu8cYVXwS4Hm3q/f/3FuCnPTgiwYvIqRSpRg=="],
|
||||
"@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.219.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/otlp-transformer": "0.219.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-zvIxQX/AZUVKDU+hCuYx+7UkiP7GRdnk1ZbFQRYzHvYp47cAWR4j3IhoPhV9KaeXEv2xdGq3IA6PnpzDmLcmSA=="],
|
||||
|
||||
"@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.214.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.214.0", "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", "@opentelemetry/sdk-logs": "0.214.0", "@opentelemetry/sdk-metrics": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1", "protobufjs": "^7.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-DSaYcuBRh6uozfsWN3R8HsN0yDhCuWP7tOFdkUOVaWD1KVJg8m4qiLUsg/tNhTLS9HUYUcwNpwL2eroLtsZZ/w=="],
|
||||
"@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.219.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.219.0", "@opentelemetry/core": "2.8.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-logs": "0.219.0", "@opentelemetry/sdk-metrics": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-aaYKAyXhw9VchKZVGOopD3Gw/kPsyrX2c6IQ0AW32mTjqmZOh5Y6Gf5OYqTNqVktAeBjmFinhyFaCwW6GYK9YQ=="],
|
||||
|
||||
"@opentelemetry/resources": ["@opentelemetry/resources@2.6.1", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-lID/vxSuKWXM55XhAKNoYXu9Cutoq5hFdkbTdI/zDKQktXzcWBVhNsOkiZFTMU9UtEWuGRNe0HUgmsFldIdxVA=="],
|
||||
"@opentelemetry/resources": ["@opentelemetry/resources@2.8.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-qmXQ27ilDbUK/vGMqwL8D4/rhn76C+sherM4wTbjlfknR8Nvfc/hCxjRJPhkzZzUsPiNg16SA31NxMabwttRjg=="],
|
||||
|
||||
"@opentelemetry/sdk-logs": ["@opentelemetry/sdk-logs@0.214.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.214.0", "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-zf6acnScjhsaBUU22zXZ/sLWim1dfhUAbGXdMmHmNG3LfBnQ3DKsOCITb2IZwoUsNNMTogqFKBnlIPPftUgGwA=="],
|
||||
"@opentelemetry/sdk-logs": ["@opentelemetry/sdk-logs@0.219.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.219.0", "@opentelemetry/core": "2.8.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-s6lTKRakaPClvKoWHRChxnXjDMkM/TQ30ff78jN6EBGf7MI7VzANE5PU3f4z9qDUudWjvZjOLHG0rBnBKYvoXA=="],
|
||||
|
||||
"@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@2.6.1", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-9t9hJHX15meBy2NmTJxL+NJfXmnausR2xUDvE19XQce0Qi/GBtDGamU8nS1RMbdgDmhgpm3VaOu2+fiS/SfTpQ=="],
|
||||
"@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@2.8.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/resources": "2.8.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-UDBGaj6W0Rgy5rTTaoxs8gVGF/aGkAKyjurJv7se6wjRxJu7FoquTLT/vt54DZfo4crbprYfhX/SOK9+BPw1qg=="],
|
||||
|
||||
"@opentelemetry/sdk-trace": ["@opentelemetry/sdk-trace@2.11.0", "", { "dependencies": { "@opentelemetry/core": "2.11.0", "@opentelemetry/resources": "2.11.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA=="],
|
||||
|
||||
"@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.6.1", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-r86ut4T1e8vNwB35CqCcKd45yzqH6/6Wzvpk2/cZB8PsPLlZFTvrh8yfOS3CYZYcUmAx4hHTZJ8AO8Dj8nrdhw=="],
|
||||
"@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.8.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-mhU4jp+vW0mGbFRd+GeXHvmfA4aDqWjBjLC3pE5XMpLs0IE2ryYb019Ts2AQrOq67gaTF25D91+fgvEHDZEnuQ=="],
|
||||
|
||||
"@opentelemetry/sdk-trace-node": ["@opentelemetry/sdk-trace-node@2.6.1", "", { "dependencies": { "@opentelemetry/context-async-hooks": "2.6.1", "@opentelemetry/core": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-Hh2i4FwHWRFhnO2Q/p6svMxy8MPsNCG0uuzUY3glqm0rwM0nQvbTO1dXSp9OqQoTKXcQzaz9q1f65fsurmOhNw=="],
|
||||
"@opentelemetry/sdk-trace-node": ["@opentelemetry/sdk-trace-node@2.8.0", "", { "dependencies": { "@opentelemetry/context-async-hooks": "2.8.0", "@opentelemetry/core": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-nZt9OGufioAc3AfoLTqA9bsAeaMJAictYDdI2VcNQ+PmT+3rfKjAZDZvgPfd8VPX0O5Bw1hdQF6kDK8VSpZiWg=="],
|
||||
|
||||
"@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.43.0", "", {}, "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg=="],
|
||||
|
||||
"@opentui/core": ["@opentui/core@0.5.10", "", { "dependencies": { "bun-ffi-structs": "0.3.1", "diff": "9.0.0", "marked": "17.0.1", "string-width": "7.2.0", "strip-ansi": "7.1.2" }, "optionalDependencies": { "@opentui/core-darwin-arm64": "0.5.10", "@opentui/core-darwin-x64": "0.5.10", "@opentui/core-linux-arm64": "0.5.10", "@opentui/core-linux-arm64-musl": "0.5.10", "@opentui/core-linux-x64": "0.5.10", "@opentui/core-linux-x64-musl": "0.5.10", "@opentui/core-win32-arm64": "0.5.10", "@opentui/core-win32-x64": "0.5.10" }, "peerDependencies": { "web-tree-sitter": "0.25.10" } }, "sha512-C3a2UbmefeAjIxAgm4BqjuSxKT4oqutfvYFwVvUgMxmGRHkNbBc/s7sukV0JgwcxFcV3uMFrXxo+E+BQtvuOiw=="],
|
||||
"@opentui/core": ["@opentui/core@0.5.12", "", { "dependencies": { "bun-ffi-structs": "0.3.1", "diff": "9.0.0", "marked": "17.0.1", "string-width": "7.2.0", "strip-ansi": "7.1.2" }, "optionalDependencies": { "@opentui/core-darwin-arm64": "0.5.12", "@opentui/core-darwin-x64": "0.5.12", "@opentui/core-linux-arm64": "0.5.12", "@opentui/core-linux-arm64-musl": "0.5.12", "@opentui/core-linux-x64": "0.5.12", "@opentui/core-linux-x64-musl": "0.5.12", "@opentui/core-win32-arm64": "0.5.12", "@opentui/core-win32-x64": "0.5.12" }, "peerDependencies": { "web-tree-sitter": "0.25.10" } }, "sha512-ZXBE5gmvdovmV8zJQrOQf6E44v1tJRDEgrM2MYhEglzgXZ+smIUp95O8zeRYGsuIzQIiMPMgQqKtTJuzvAb7BQ=="],
|
||||
|
||||
"@opentui/core-darwin-arm64": ["@opentui/core-darwin-arm64@0.5.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Vyb+nTbhab8ZcRy5gg1loEEGwRcIbjAeVRIBfHBcbFDqmITBOg7x2gqJ+x/TnoOy4uwMhCmICUN2wiyREw3r1Q=="],
|
||||
"@opentui/core-darwin-arm64": ["@opentui/core-darwin-arm64@0.5.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-YdVnP0tAyerBNl0mIcmQEOotPeZzW1VnSXKBl5cyZ5e6nDd2Y+ui/8eRPpn1oqcamf1NCnzS4ohMgejOvna8Zg=="],
|
||||
|
||||
"@opentui/core-darwin-x64": ["@opentui/core-darwin-x64@0.5.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-tTFLcM7Oj1gTyhm/bUdAt3C6grZdCxPk6+/g2azcZBUlI3/62LwbeRS6HbQKFFmm+1fUmX8cq6kWrtul885mVg=="],
|
||||
"@opentui/core-darwin-x64": ["@opentui/core-darwin-x64@0.5.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-uRrQJdHmLUSj3PV23QPi3WSimYTTxcXnVouxF6U4xMXlOv4N3SxnHfVwMRQkPqbGOfvVWHeLE6FdK4C+ubU0sQ=="],
|
||||
|
||||
"@opentui/core-linux-arm64": ["@opentui/core-linux-arm64@0.5.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-ncJXcgudhBf2GdJyF3xVQN/Ec+1F7GOL+pRrURmgBYSj2v1w6EyoDQFAACtPTK2c3R38W6fvZwL4JSLlm4EFXQ=="],
|
||||
"@opentui/core-linux-arm64": ["@opentui/core-linux-arm64@0.5.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-XeKhuIaEtgipvuPHbl4qPOBj+Ut+2zObmsxMVM1jDcjz/FatG9PGeGQPx1G1SnvH2AgpT4K+eCu7DUF0+yIqoQ=="],
|
||||
|
||||
"@opentui/core-linux-arm64-musl": ["@opentui/core-linux-arm64-musl@0.5.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-dGMphDKexSdeYqwl0wgoFBP88Ta/cdi1Zc1mk29/ENkSCGz+74zlCHgqTHRNGLmI8W5TfuUtCyktQH11/Z+TBQ=="],
|
||||
"@opentui/core-linux-arm64-musl": ["@opentui/core-linux-arm64-musl@0.5.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-VZ2sNMw1d/r1SLPjUbOP9LKscKz1CQjID8adTL6gG8Lrrq+mYcIUxutyB+P/eG0J/7oRZLPR6OMt7dUOap6RTg=="],
|
||||
|
||||
"@opentui/core-linux-x64": ["@opentui/core-linux-x64@0.5.10", "", { "os": "linux", "cpu": "x64" }, "sha512-5qtYaOgwVycZD1GaGshTRsi0rXPAmVExO03N1JQaHu+NYxK/vXSOc7Bu4QW0sPXx3Sp0SpzpP+FHjXABfoK66g=="],
|
||||
"@opentui/core-linux-x64": ["@opentui/core-linux-x64@0.5.12", "", { "os": "linux", "cpu": "x64" }, "sha512-eZiCjEzwbb6qClPPfk32Nha9xmr9obt69Xj0+9SKsXxWLBKkjQEGOMRoh/R9ObaQF4aq8If1xV3VEY0sD9W9vg=="],
|
||||
|
||||
"@opentui/core-linux-x64-musl": ["@opentui/core-linux-x64-musl@0.5.10", "", { "os": "linux", "cpu": "x64" }, "sha512-Oj4H9hApuvuTKPWxh4SoZAgGJorR7vbvnrZA/cAkSMAk2VGSoHRRcqeXQbcH8IcdjVZ0KFpv8Zkl/D5Ye+2mew=="],
|
||||
"@opentui/core-linux-x64-musl": ["@opentui/core-linux-x64-musl@0.5.12", "", { "os": "linux", "cpu": "x64" }, "sha512-WWW0hVBoSYZ3D6AgZ4u2Y5/u/IyIq2pDb+4yI3WgJ70Wyt6ofHy+6kRGRgbXFn1p+rPInAHjCXD2v6C7iEKSrA=="],
|
||||
|
||||
"@opentui/core-win32-arm64": ["@opentui/core-win32-arm64@0.5.10", "", { "os": "win32", "cpu": "arm64" }, "sha512-A9VhgvTxQoUdZ+8LmUumEng1sQNbj9QQQT3NYG9mSxI54qTANi7vOWNSphMiY6RMVsr22pgm6nUvSSvJXv7Jog=="],
|
||||
"@opentui/core-win32-arm64": ["@opentui/core-win32-arm64@0.5.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-aLbm6870Ybls6CYL4zMOCImTBPLZHZMUXJFGqMI44lIWxitkAtT6zg5lYA4oRqFRzzryDclxr29+hDgT3p3Blw=="],
|
||||
|
||||
"@opentui/core-win32-x64": ["@opentui/core-win32-x64@0.5.10", "", { "os": "win32", "cpu": "x64" }, "sha512-u3KHa7kEeWrmKVDRJYpxSGO+g5E9cMGlrmTsPN3GVPHUmQMiREUawLXUvsU8+IHaQnqG3Q5nuE1yf4fPBzS+Qw=="],
|
||||
"@opentui/core-win32-x64": ["@opentui/core-win32-x64@0.5.12", "", { "os": "win32", "cpu": "x64" }, "sha512-KTwtwpfd2zF9opVh3SyRJYDd1o3Xv4XL8OZb8Zi+CqWUel6Y2IDCiVivCv8fGJt3J7wOIXXtuZI9ZUkLyKJCiQ=="],
|
||||
|
||||
"@opentui/keymap": ["@opentui/keymap@0.5.10", "", { "dependencies": { "@opentui/core": "0.5.10" }, "peerDependencies": { "@opentui/react": "0.5.10", "@opentui/solid": "0.5.10", "react": ">=19.2.0", "solid-js": "1.9.12" }, "optionalPeers": ["@opentui/react", "@opentui/solid", "react", "solid-js"] }, "sha512-8vDJF+ltXscSnLEv3rgCa4m7PcoYZeUT9BngugpFCmVoNevbaRtYijjdfiUuLmXfT61lO5QbR6nEhn2RZMK8ow=="],
|
||||
"@opentui/keymap": ["@opentui/keymap@0.5.12", "", { "dependencies": { "@opentui/core": "0.5.12" }, "peerDependencies": { "@opentui/react": "0.5.12", "@opentui/solid": "0.5.12", "react": ">=19.2.0", "solid-js": "1.9.12" }, "optionalPeers": ["@opentui/react", "@opentui/solid", "react", "solid-js"] }, "sha512-yWPvJjRhJTRoRSUucQq9Ua8ZW7n/2YQ/j6JxWq5Qekm4WuFiTplEkebR/Aj2/xA8tX68NOE5qv1LrY0Jk3NLNQ=="],
|
||||
|
||||
"@opentui/solid": ["@opentui/solid@0.5.10", "", { "dependencies": { "@babel/core": "7.28.0", "@babel/preset-typescript": "7.27.1", "@opentui/core": "0.5.10", "babel-plugin-module-resolver": "5.0.2", "babel-preset-solid": "1.9.12", "entities": "7.0.1", "s-js": "^0.4.9" }, "peerDependencies": { "solid-js": "1.9.12" } }, "sha512-KrmMIsHiKBHOABTC0brOwqWm+sGq1ZX2sGCAx6WgtBbE3STMup9n8TAy/6gUYhwcjC9zugT53ytfSVwCwVWZUg=="],
|
||||
"@opentui/solid": ["@opentui/solid@0.5.12", "", { "dependencies": { "@babel/core": "7.28.0", "@babel/preset-typescript": "7.27.1", "@opentui/core": "0.5.12", "babel-plugin-module-resolver": "5.0.2", "babel-preset-solid": "1.9.12", "entities": "7.0.1", "s-js": "^0.4.9" }, "peerDependencies": { "solid-js": "1.9.12" } }, "sha512-hAiVlVMtT7AkHGblKwcW1YAuXtxkSy1XSf/RRc4j3IlG3mTNX0bhJdnGOo3Xw14EqeZMp41Mcp5WzHAzMm/DzA=="],
|
||||
|
||||
"@oslojs/asn1": ["@oslojs/asn1@1.0.0", "", { "dependencies": { "@oslojs/binary": "1.0.0" } }, "sha512-zw/wn0sj0j0QKbIXfIlnEcTviaCzYOY3V5rAyjR6YtOByFtJiT574+8p9Wlach0lZH9fddD4yb9laEAIl4vXQA=="],
|
||||
|
||||
@@ -2553,24 +2558,6 @@
|
||||
|
||||
"@protobuf-ts/runtime-rpc": ["@protobuf-ts/runtime-rpc@2.11.1", "", { "dependencies": { "@protobuf-ts/runtime": "^2.11.1" } }, "sha512-4CqqUmNA+/uMz00+d3CYKgElXO9VrEbucjnBFEjqI4GuDrEQ32MaI3q+9qPBvIGOlL4PmHXrzM32vBPWRhQKWQ=="],
|
||||
|
||||
"@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="],
|
||||
|
||||
"@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="],
|
||||
|
||||
"@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="],
|
||||
|
||||
"@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.1", "", {}, "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg=="],
|
||||
|
||||
"@protobufjs/fetch": ["@protobufjs/fetch@1.1.1", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1" } }, "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw=="],
|
||||
|
||||
"@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="],
|
||||
|
||||
"@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="],
|
||||
|
||||
"@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="],
|
||||
|
||||
"@protobufjs/utf8": ["@protobufjs/utf8@1.1.2", "", {}, "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug=="],
|
||||
|
||||
"@puppeteer/browsers": ["@puppeteer/browsers@3.2.1", "", { "dependencies": { "modern-tar": "^0.8.0", "yargs": "^18.0.0" }, "peerDependencies": { "proxy-agent": ">=8.0.1", "yauzl": "^2.10.0 || ^3.4.0" }, "optionalPeers": ["proxy-agent", "yauzl"], "bin": { "browsers": "lib/main-cli.js" } }, "sha512-KDz+3qDRdBAlRlMjmKyj6dEs33YHTk/xRHEENSXq6TNnhgoU15ruSHtEBeVF6OZ9tBDY55Se4P0nFMNsipzU9A=="],
|
||||
|
||||
"@radix-ui/colors": ["@radix-ui/colors@1.0.1", "", {}, "sha512-xySw8f0ZVsAEP+e7iLl3EvcBXX7gsIlC1Zso/sPBW9gIWerBTgz6axrjU+MZ39wD+WFi5h5zdWpsg3+hwt2Qsg=="],
|
||||
@@ -4127,7 +4114,7 @@
|
||||
|
||||
"github-slugger": ["github-slugger@2.0.0", "", {}, "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw=="],
|
||||
|
||||
"gitlab-ai-provider": ["gitlab-ai-provider@6.12.1", "", { "dependencies": { "@anthropic-ai/sdk": "^0.71.0", "@anycable/core": "^0.9.2", "graphql-request": "^6.1.0", "isomorphic-ws": "^5.0.0", "openai": "^6.16.0", "socket.io-client": "^4.8.1", "vscode-jsonrpc": "^8.2.1", "zod": "^3.25.76" }, "peerDependencies": { "@ai-sdk/provider": ">=3.0.0", "@ai-sdk/provider-utils": ">=4.0.0" } }, "sha512-Qn5iHqvjG8yktI5MWaUgdRR94l7O4WtYW0CAbhsCh1Tj0Fei/DeprOYPVyf4Nht1Ix6U2PXSYM32QOHI6Z2TDw=="],
|
||||
"gitlab-ai-provider": ["gitlab-ai-provider@6.18.0", "", { "dependencies": { "@anthropic-ai/sdk": "^0.71.0", "@anycable/core": "^0.9.2", "graphql-request": "^6.1.0", "isomorphic-ws": "^5.0.0", "openai": "^6.16.0", "socket.io-client": "^4.8.1", "vscode-jsonrpc": "^8.2.1", "zod": "^3.25.76" }, "peerDependencies": { "@ai-sdk/provider": ">=3.0.0", "@ai-sdk/provider-utils": ">=4.0.0" } }, "sha512-dXTXkNt1SFCL7jGlqazHL6iUJEug21qq0tx2s+Tui7jWNUqIAHEzY9WY1+PnZlhxvB56jPPBcFAT0attpjnopg=="],
|
||||
|
||||
"glob": ["glob@13.0.5", "", { "dependencies": { "minimatch": "^10.2.1", "minipass": "^7.1.2", "path-scurry": "^2.0.0" } }, "sha512-BzXxZg24Ibra1pbQ/zE7Kys4Ua1ks7Bn6pKLkVPZ9FZe4JQS6/Q7ef3LG1H+k7lUf5l4T3PLSyYyYJVYUvfgTw=="],
|
||||
|
||||
@@ -5047,8 +5034,6 @@
|
||||
|
||||
"proto-list": ["proto-list@1.2.4", "", {}, "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA=="],
|
||||
|
||||
"protobufjs": ["protobufjs@7.6.5", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw=="],
|
||||
|
||||
"proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="],
|
||||
|
||||
"pump": ["pump@3.0.4", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA=="],
|
||||
@@ -6245,6 +6230,8 @@
|
||||
|
||||
"@opencode/www/wrangler": ["wrangler@4.110.0", "", { "dependencies": { "@cloudflare/kv-asset-handler": "0.5.0", "@cloudflare/unenv-preset": "2.16.1", "blake3-wasm": "2.1.5", "esbuild": "0.28.1", "miniflare": "4.20260708.1", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", "workerd": "1.20260708.1" }, "optionalDependencies": { "fsevents": "2.3.3" }, "peerDependencies": { "@cloudflare/workers-types": "^5.20260708.1" }, "optionalPeers": ["@cloudflare/workers-types"], "bin": { "wrangler": "bin/wrangler.js", "wrangler2": "bin/wrangler.js", "cf-wrangler": "bin/cf-wrangler.js" } }, "sha512-xZeXKYi7hxQRF5anL+v77RkufJNpF9f3Eqeyqq2QBsETpLZgh0Agj0jJ6JPtkbgn6ukZdh8OK5egsGPWIditgg=="],
|
||||
|
||||
"@opentelemetry/api-logs/@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="],
|
||||
|
||||
"@opentelemetry/instrumentation/@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.220.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w=="],
|
||||
|
||||
"@opentelemetry/sdk-trace/@opentelemetry/core": ["@opentelemetry/core@2.11.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA=="],
|
||||
|
||||
+4
-4
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"nodeModules": {
|
||||
"x86_64-linux": "sha256-WBGj4wU5PQs+0zbDbScFhJs0GL8qWnswIsJb/Kee5Qc=",
|
||||
"aarch64-linux": "sha256-Z8hkyNMdqCpuv24q50QhO2RECINvENjBhLDY5FKcnFc=",
|
||||
"aarch64-darwin": "sha256-HZHb9HcXMOc7bKR7eADQSyfWxX37GAcoRi/EMSpRiP4=",
|
||||
"x86_64-darwin": "sha256-CcWCOAcW4t9ADs48fMhumSbDcQWdk91LpkG6fe/gWlk="
|
||||
"x86_64-linux": "sha256-aQQQhaUlAhpfqzH0vNi0IJ1cg7FQHIKYzxeq5d8PZoU=",
|
||||
"aarch64-linux": "sha256-r9aDFu3UYmudmmYPhzCrpFvQlaejXc8V1IzLtG3jZPc=",
|
||||
"aarch64-darwin": "sha256-B0m41LelD7d61vPHGIZZSO/cU7gbjHDJHt6oxNRRM8Q=",
|
||||
"x86_64-darwin": "sha256-9TWJsyI3Y6BMomtGSgqA1th9LpxpqP4F5Tl/GyexVYw="
|
||||
}
|
||||
}
|
||||
|
||||
+5
-5
@@ -2,7 +2,7 @@
|
||||
"$schema": "https://json.schemastore.org/package.json",
|
||||
"name": "opencode",
|
||||
"description": "AI-powered development tool",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"packageManager": "bun@1.4.2",
|
||||
@@ -52,9 +52,9 @@
|
||||
"@octokit/rest": "22.0.0",
|
||||
"@hono/standard-validator": "0.2.0",
|
||||
"@hono/zod-validator": "0.4.2",
|
||||
"@opentui/core": "0.5.10",
|
||||
"@opentui/keymap": "0.5.10",
|
||||
"@opentui/solid": "0.5.10",
|
||||
"@opentui/core": "0.5.12",
|
||||
"@opentui/keymap": "0.5.12",
|
||||
"@opentui/solid": "0.5.12",
|
||||
"@tanstack/solid-virtual": "3.13.37",
|
||||
"@shikijs/stream": "4.4.3",
|
||||
"@standard-schema/spec": "1.1.0",
|
||||
@@ -173,7 +173,7 @@
|
||||
"solid-js@1.9.15": "patches/solid-js@1.9.15.patch",
|
||||
"@ai-sdk/mistral@3.0.51": "patches/@ai-sdk%2Fmistral@3.0.51.patch",
|
||||
"gcp-metadata@8.1.2": "patches/gcp-metadata@8.1.2.patch",
|
||||
"pacote@21.5.0": "patches/pacote@21.5.0.patch",
|
||||
"pacote@21.5.1": "patches/pacote@21.5.1.patch",
|
||||
"@ai-sdk/google@3.0.73": "patches/@ai-sdk%2Fgoogle@3.0.73.patch",
|
||||
"@pierre/trees@1.0.0-beta.4": "patches/@pierre%2Ftrees@1.0.0-beta.4.patch",
|
||||
"@tanstack/virtual-core@3.17.8": "patches/@tanstack%2Fvirtual-core@3.17.8.patch",
|
||||
|
||||
+13
-6
@@ -10,11 +10,11 @@
|
||||
|
||||
## Conventions
|
||||
|
||||
Per-type constructors live on the type, not as top-level re-exports. Use `Message.system(...)`, `Message.user(...)`, `Message.assistant(...)`, `Message.tool(...)`, `Message.media(...)`, `LanguageModel.make(...)`, `ToolDefinition.make(...)`, `ToolCallPart.make(...)`, `ToolResultPart.make(...)`, `ToolChoice.make(...)`, `ToolChoice.named(...)`, `SystemPart.make(...)`, and `GenerationOptions.make(...)` directly. The top-level `LLM` namespace is reserved for request-shaped call APIs: `LLM.request`, `LLM.generate`, `LLM.stream`, and `LLM.generateObject`. Use `LLMRequest.update(...)` when deriving canonical request data; do not add a duplicate `LLM.updateRequest(...)` path. Two ways to construct the same thing is one too many.
|
||||
Per-type constructors live on the type, not as top-level re-exports. Use `Message.system(...)`, `Message.user(...)`, `Message.assistant(...)`, `Message.tool(...)`, `Message.media(...)`, `LanguageModel.make(...)`, `ToolDefinition.make(...)`, `ToolCallPart.make(...)`, `ToolResultPart.make(...)`, `ToolChoice.make(...)`, `ToolChoice.named(...)`, `SystemPart.make(...)`, and `GenerationOptions.make(...)` directly. The top-level `LLM` namespace is reserved for request-shaped call APIs: `LLM.request`, `LLM.generate`, `LLM.stream`, and `LLM.generateObject`. `LLM.generate`/`LLM.stream` and Promise `ai.llm.generate`/`ai.llm.stream` accept ergonomic input or an `LLMRequest`; both paths use the same canonical request. Core still builds, logs, replays, and updates that durable `LLMRequest` boundary. Use `LLMRequest.update(...)` when deriving canonical request data; do not add a duplicate `LLM.updateRequest(...)` path.
|
||||
|
||||
Modality namespaces mirror `LLM` exactly: `Image.request`, `Image.generate`, `Image.stream` (later `Video`, `Speech`, `Transcription`). Common request fields (`images`, `mask`, `n`, `size`, `aspectRatio`, `seed`, `format`) lower natively or fail with a typed `AIError`; provider-native controls always live under `providerOptions`, never under a modality-specific `options` key.
|
||||
Modality namespaces mirror `LLM` exactly: `Image.request`, `Image.generate`, `Image.stream`, and the same for `Video`, `Speech`, and `Transcription`. Common request fields (`images`, `mask`, `n`, `size`, `aspectRatio`, `seed`, `format`) lower natively or fail with a typed `AIError`; provider-native controls always live under `providerOptions`, never under a modality-specific `options` key.
|
||||
|
||||
Media payloads are always `Media.Asset` (`src/media.ts`). Construct them with `Media.bytes`, `Media.base64`, `Media.url`, `Media.ref`, `Media.fromDataUrl`, or `Media.file`; never introduce a parallel `data: string | Uint8Array` shape. `MediaPart.media`, `ImageRequest.images`/`mask`, `ImageResponse.images`, and the `media` `LLMEvent` all share it. Protocols branch on `asset.source.type` and `asset.kind` and use `ProviderShared.inlineMedia` / `requireInlineMedia` / `mediaUrl` / `MediaInput.refID` rather than re-deriving base64 or URL handling.
|
||||
Media payloads are always `Media.Asset` (`src/media.ts`). Construct them with `Media.bytes`, `Media.base64`, `Media.url`, `Media.ref`, `Media.fromDataUrl`, or `Media.file`; never introduce a parallel `data: string | Uint8Array` shape. `MediaPart.media`, `ImageRequest.images`/`mask`, `ImageResponse.images`, and the `media` `LLMEvent` all share it. Protocols branch on `asset.source.type` and `asset.kind` and use `ProviderShared.requireInlineMedia` / `inlineRequired` / `mediaUrl` / `mediaReference` and `MediaInput.inlineBytes` / `refID` rather than re-deriving base64 or URL handling.
|
||||
|
||||
`schema/messages.ts → media.ts → route/executor-service.ts` is an accepted runtime dependency from the schema layer on the executor service tag: `Media.Asset.bytes()` must be able to download `url` sources, and the tag lives in that leaf module precisely so the schema barrel never imports the executor implementation (which imports the schema barrel back). Do not move the tag into `route/executor.ts` or import `route/executor.ts` from `src/schema/*` or `src/media.ts`.
|
||||
|
||||
@@ -55,7 +55,7 @@ const request = LLM.request({
|
||||
prompt: "Say hello.",
|
||||
})
|
||||
|
||||
const response = yield * LLMClient.generate(request)
|
||||
const response = yield * LLMClient.generate(request) // inside Effect.gen
|
||||
```
|
||||
|
||||
`LLM.request(...)` builds an `LLMRequest`. `LLMClient.generate(...)` reads the executable route carried by `request.model.route`, builds the provider-native body, asks the route's transport for a real `HttpClientRequest.HttpClientRequest`, sends it through `RequestExecutor.Service`, parses the provider stream into common `LLMEvent`s, and finally returns an `LLMResponse`.
|
||||
@@ -96,7 +96,13 @@ When a provider supports multiple physical transports, selection remains executi
|
||||
|
||||
### Media Routes
|
||||
|
||||
Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute.make(...)` (`src/route/media.ts`) composes a `MediaProtocol` kind with `Endpoint` and `Auth` and owns the transport plumbing: `http` option merging, URL/query rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol. `MediaProtocol.inline` (`src/route/media-protocol.ts`) is `body.from(request)` plus `response.decode(response, context)`; use `MediaProtocol.decodeJson` / `text` / `bytes` so decode failures retain the raw body and HTTP context. `Generation` (`src/generation.ts`) is the provider-neutral handle for a queued generation over a `GenerationRoute` (`status`, `result`, `cancel`, `pollHint`); the first video route implements it. Image protocol files follow the same section order as LLM protocols and declare unsupported common fields once through `MediaInput.rejectUnsupported`.
|
||||
Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute.inline(...)` / `queued(...)` / `stream(...)` (`src/route/media.ts`) compose a `MediaProtocol` kind with `Endpoint` and `Auth` and own the transport plumbing: `http` option merging, URL/query rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol. `MediaProtocol.inline` (`src/route/media-protocol.ts`) is `body.from(request)` plus `response.decode(response, context)`; each protocol declares `const route = MediaProtocol.identity({ id, name, provider })` once and decodes through `route.decodeJson` / `route.text` / `route.decodeStarted` so decode failures retain the raw body and HTTP context, raising `route.unsupported(operation, message)` for requests it cannot lower, and passes `route` as the first argument to `MediaProtocol.inline` / `queued` / `stream`. `Generation` (`src/generation.ts`) is the provider-neutral handle for a queued generation over a `GenerationRoute` (`status`, `result`, `cancel`). Image protocol files follow the same section order as LLM protocols and declare unsupported common fields once through the protocol's `unsupported` list.
|
||||
|
||||
`MediaProtocol.queued` is the submit-then-poll kind every video route uses: `start` (body + decode into `{ token, snapshot }`), `status`, `result`, and optional `cancel` (with `activeOnly` when the provider's cancel endpoint deletes finished work, as Runway's does: the route refreshes status first and skips terminal generations), each addressed by a route-owned `token` whose `Schema.Codec` makes it serializable. `MediaRoute.inline` and `MediaRoute.queued` compose the two kinds with `Endpoint` and `Auth`; the queued route decodes the token once at the boundary (`start` output or `resume` input) and closes over it in a token-free `GenerationRoute` (`status`/`result`/`cancel` are plain Effects), so `Generation` never sees the token's shape and only carries the encoded JSON for persistence. Polls reuse the route's auth and deployment headers plus the request's `http` overlay after `start`, and resolve relative paths against the route base URL (provider-issued absolute URLs such as fal's `status_url` pass through). `result` is always its own GET even when the provider returns output inside the status document, so `Generation.await` behaves the same after `start` and after `resume`. `PollContext.auth` carries only what `Auth` added or changed so protocols can hand download credentials to output assets as transient `Media.Asset.headers` (Veo) — never part of `source` or JSON. Status strings map through a per-protocol `STATUS` table via `MediaProtocol.status`; terminal generations without output fail through `output.ended` / `output.contentPolicy` with the provider document on `reason.body`. `GenerationAwaitOptions` (`AwaitOptions` in `src/generation.ts`, `{ poll?: Poll }`) is the one options type for `await`, `events`, `Video.generate`, and `Video.stream`.
|
||||
|
||||
`MediaProtocol.stream` is the incremental kind every speech route uses, with the same discipline as LLM protocols. `MediaRoute.stream` submits the caller's request as `MediaProtocol.Addressed<Request>` (`{ ...request, mode }`, `mode: "generate" | "stream"`), so one provider stays one protocol: `body.from`, the endpoint path, and `frames` read `request.mode` to pick the body, path, and framing. `frames(bytes, context)` returns frames — `Framing.sse`, `Framing.lines`, `Framing.document` (a single-document response shaped like a streamed record), or the raw `bytes` for chunked audio. `initial()` is fresh per-response parser state; `step` folds each frame into it and emits modality events; `finish(state, context)` runs once after the last frame with the request, body, and observed `http` (header-only usage lives there) and emits exactly one terminal event or fails with `route.incomplete()`. Keep parser state to real accumulators and derive anything the request or body determines in `finish`. `generate` runs the same stream and folds it with the modality's `collect`. Request-derived URL parameters go on the body's `query` (array values repeat the parameter), applied before route and caller `http.query`. Decode frames with `route.decodeFrame` and raise stream-time failures with `route.frameError` (the frame stays on `reason.body`); protocols never thread HTTP context, because the route fills `reason.http` on stream errors that lack it. Speech protocols share `protocols/utils/speech-stream.ts` for deltas, timestamps, voice ids, PCM and container descriptions, and the terminal asset.
|
||||
|
||||
Every modality route is the inline | stream | queued union (transcription uses all three: OpenAI and Gemini stream, Deepgram is inline, AssemblyAI is queued), every client is `MediaClient.make(Service, { modality, responseEvents })` (`src/media-client.ts`), which dispatches on the route's `kind`, and every model composes through `composeRoute`. fal queue protocols come from `protocols/utils/fal-queue.ts`, bodies are `json`, `multipart`, or `binary` (a raw upload), and a queued protocol that must upload media before submitting implements `start.prepare` (`MediaProtocol.Prepare`; AssemblyAI `/v2/upload`).
|
||||
|
||||
### URL Construction
|
||||
|
||||
@@ -106,7 +112,7 @@ For providers where the URL is derived from typed inputs (Azure resource name, B
|
||||
|
||||
### Provider Facades
|
||||
|
||||
Provider-facing APIs are configured facades over route values. Endpoint/auth/resource/API-version setup happens before model selection, and model selectors accept only a model or deployment id. Media models use per-modality selectors on the same facade (`openai.image(id)`, later `.video` / `.speech` / `.transcription`) that mirror `openai.responses(id)`; the one-word overlap with the request namespace is accepted over a second construction path:
|
||||
Provider-facing APIs are configured facades over route values. Endpoint/auth/resource/API-version setup happens before model selection, and model selectors accept only a model or deployment id. Media models use per-modality selectors on the same facade (`openai.image(id)`, `.speech(id)`, `.transcription(id)`, `google.video(id)`) that mirror `openai.responses(id)`; the one-word overlap with the request namespace is accepted over a second construction path:
|
||||
|
||||
```ts
|
||||
const openai = OpenAI.configure({ apiKey, baseURL })
|
||||
@@ -269,6 +275,7 @@ Use this order for every protocol module:
|
||||
### Rules
|
||||
|
||||
- Keep protocol files focused on the protocol. Move provider-specific projection, signing, media normalization, or other bulky transformations into `src/protocols/utils/*`.
|
||||
- Send `tool.inputSchema` as given. `prepareRequest` applies the tool schema rules (`ToolSchemaProjection.tools`) once per request, including tools in namespaces. A protocol whose API needs a model family's rules for every model declares `sanitizer` instead of transforming schemas itself.
|
||||
- Use `Effect.fn("Provider.fromRequest")` for request body construction entrypoints. Use `Effect.fn(...)` for event handlers that yield effects; keep purely synchronous handlers as plain functions returning a `StepResult` that the dispatcher lifts via `Effect.succeed(...)`.
|
||||
- Parser state owns terminal information. The state machine records finish reason, usage, and pending tool calls; emit one terminal `finish` event (or `provider-error`) for each completed response. If a provider splits reason and usage across events, merge them in parser state before flushing.
|
||||
- Emit exactly one terminal `finish` event for a completed response, normally after a matching `step-finish`. Use `stream.terminal` to stop reading when the provider has a completion sentinel; use `stream.onHalt` when the final event must be flushed after the framed stream ends.
|
||||
|
||||
+442
-100
@@ -1,40 +1,38 @@
|
||||
# @opencode/ai
|
||||
|
||||
Schema-first language model and image-generation APIs built with Effect.
|
||||
Schema-first APIs for text, images, video, speech, and transcription, built with Effect.
|
||||
|
||||
```ts
|
||||
import { Effect, Layer } from "effect"
|
||||
import { LLM, LLMClient } from "@opencode/ai"
|
||||
import { RequestExecutor } from "@opencode/ai/route"
|
||||
import { Effect } from "effect"
|
||||
import { AIClient, LLM } from "@opencode/ai"
|
||||
import { OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
|
||||
|
||||
const request = LLM.request({
|
||||
model: openai.responses("gpt-4o-mini"), // `.chat(...)` selects the Chat Completions API instead
|
||||
system: "You are concise.",
|
||||
prompt: "Say hello in one short sentence.",
|
||||
generation: { maxTokens: 40 },
|
||||
})
|
||||
|
||||
const program = Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request)
|
||||
const response = yield* LLM.generate({
|
||||
model: openai.responses("gpt-4o-mini"), // `.chat(...)` selects the Chat Completions API instead
|
||||
system: "You are concise.",
|
||||
prompt: "Say hello in one short sentence.",
|
||||
generation: { maxTokens: 40 },
|
||||
})
|
||||
console.log(response.text)
|
||||
})
|
||||
|
||||
const llmLayer = LLMClient.layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
|
||||
|
||||
await Effect.runPromise(program.pipe(Effect.provide(llmLayer)))
|
||||
// Every modality client plus the HTTP request executor; `AIClient.layerWith(executor)` swaps the executor.
|
||||
await Effect.runPromise(program.pipe(Effect.provide(AIClient.layer)))
|
||||
```
|
||||
|
||||
Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
|
||||
Run `LLM.stream(...)` instead of `generate` when you want incremental `LLMEvent`s. Both accept input or a prebuilt
|
||||
`LLM.request(...)`. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses,
|
||||
Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
|
||||
|
||||
The same configured facade names image models. `Image.request` resolves the provider's image route from the ref and
|
||||
returns `Media.Asset`s with lazily decoded bytes:
|
||||
The same configured facade names image, video, speech, and transcription models. `Image.generate` resolves the
|
||||
provider's image route from the model and returns `Media.Asset`s with lazily decoded bytes:
|
||||
|
||||
```ts
|
||||
import { NodeFileSystem } from "@effect/platform-node"
|
||||
import { Image, ImageClient, Media } from "@opencode/ai"
|
||||
import { Image, Media } from "@opencode/ai"
|
||||
|
||||
const image = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
@@ -46,21 +44,38 @@ const image = Effect.gen(function* () {
|
||||
yield* Media.write(response.image, "./garden.png")
|
||||
})
|
||||
|
||||
// `asset.bytes()` / `Media.write` also need the executor, so merge it into the environment instead of hiding it.
|
||||
const imageLayer = ImageClient.layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer))
|
||||
|
||||
await Effect.runPromise(image.pipe(Effect.provide(imageLayer), Effect.provide(NodeFileSystem.layer)))
|
||||
// `Media.file` / `Media.write` use the Effect `FileSystem` service; provide your platform's layer.
|
||||
await Effect.runPromise(image.pipe(Effect.provide(AIClient.layer), Effect.provide(NodeFileSystem.layer)))
|
||||
```
|
||||
|
||||
Prefer promises? `@opencode/ai/promise` exposes the same LLM and image APIs over one managed runtime:
|
||||
Advanced: each client also has its own `layer`, which requires `RequestExecutor.Service`. Compose client layers with
|
||||
`Layer.provideMerge`, not `Layer.provide`: `asset.bytes()`, `Media.write`, and Gemini's `media` output parts need the
|
||||
executor too, and hiding it fails type-checking with `RequestExecutorService` left in the requirements.
|
||||
|
||||
To share a policy such as logging across every client, wrap the executor once with `RequestExecutor.middleware`:
|
||||
|
||||
```ts
|
||||
import { RequestExecutor } from "@opencode/ai/route"
|
||||
|
||||
const logged = RequestExecutor.middleware((request, next) =>
|
||||
Effect.log(`${request.method} ${request.url}`).pipe(Effect.andThen(next(request))),
|
||||
)
|
||||
|
||||
const everything = AIClient.layerWith(logged) // or AI.make({ layer: logged })
|
||||
```
|
||||
|
||||
Prefer promises? `@opencode/ai/promise` exposes the same LLM and media APIs over one managed runtime, plus asset
|
||||
helpers; `ai.file` and `ai.write` load `node:fs/promises` on first use, so no Effect `FileSystem` is needed:
|
||||
|
||||
```ts
|
||||
import { AI } from "@opencode/ai/promise"
|
||||
|
||||
const ai = AI.make()
|
||||
const text = await ai.llm.generate({ model: openai.responses("gpt-4o-mini"), prompt: "Say hello." })
|
||||
const input = { model: openai.responses("gpt-4o-mini"), prompt: "Say hello." }
|
||||
const text = await ai.llm.generate(input)
|
||||
const generated = await ai.image.generate({ model: openai.image("gpt-image-2"), prompt: "A lighthouse" })
|
||||
for await (const event of ai.llm.stream({ model: openai.responses("gpt-4o-mini"), prompt: "Stream hello." })) {
|
||||
await ai.write(generated.image, "./lighthouse.png") // also ai.file(path), ai.bytes(asset), ai.base64(asset), ai.materialize(asset)
|
||||
for await (const event of ai.llm.stream(ai.llm.request(input))) {
|
||||
// LLMEvent
|
||||
}
|
||||
await ai.dispose()
|
||||
@@ -322,10 +337,9 @@ and `moonshot/responses`; each exports `model(modelID, settings)`.
|
||||
MiniMax defaults to its Messages API and reads `MINIMAX_API_KEY` when `apiKey` is omitted:
|
||||
|
||||
```ts
|
||||
import { Effect, Layer } from "effect"
|
||||
import { LLM, LLMClient } from "@opencode/ai"
|
||||
import { Effect } from "effect"
|
||||
import { AIClient, LLM } from "@opencode/ai"
|
||||
import { MiniMax } from "@opencode/ai/providers"
|
||||
import { RequestExecutor } from "@opencode/ai/route"
|
||||
|
||||
const minimax = MiniMax.configure({ apiKey: process.env.MINIMAX_API_KEY })
|
||||
const request = LLM.request({
|
||||
@@ -335,8 +349,7 @@ const request = LLM.request({
|
||||
generation: { maxTokens: 1536 },
|
||||
})
|
||||
|
||||
const layer = LLMClient.layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
|
||||
const response = await Effect.runPromise(LLMClient.generate(request).pipe(Effect.provide(layer)))
|
||||
const response = await Effect.runPromise(LLM.generate(request).pipe(Effect.provide(AIClient.layer)))
|
||||
console.log(response.text)
|
||||
```
|
||||
|
||||
@@ -405,14 +418,14 @@ Use `Image.generate` for one-off generation or editing:
|
||||
import { Image, Media } from "@opencode/ai"
|
||||
|
||||
const generation = Image.generate({
|
||||
model: meta("muse-image-1.0"),
|
||||
model: meta.image("muse-image-1.0"),
|
||||
prompt: "A flat black square on a white background.",
|
||||
n: 1,
|
||||
providerOptions: { reasoningStrength: "low" },
|
||||
})
|
||||
|
||||
const edit = Image.generate({
|
||||
model: meta("muse-image-1.0"),
|
||||
model: meta.image("muse-image-1.0"),
|
||||
prompt: "Make the square purple.",
|
||||
images: [Media.bytes(imageBytes, "image/webp")],
|
||||
format: "png",
|
||||
@@ -459,6 +472,25 @@ const program = Effect.gen(function* () {
|
||||
})
|
||||
```
|
||||
|
||||
Common fields are portable in shape, not in support. Unsupported fields fail with a typed `AIError` before any network
|
||||
call rather than being dropped, so check this table before swapping only the `model`:
|
||||
|
||||
| Provider | `n` | `size` | `aspectRatio` | `seed` | `format` | `images` | `mask` |
|
||||
| --------------------- | --- | --------- | ------------- | ------ | -------- | -------------------------------- | ------------------- |
|
||||
| OpenAI | ✓¹ | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ |
|
||||
| Google (Gemini) | 1 | ✗ | ✓ | ✓ | ✗ | ✓ (no public URLs) | ✗ |
|
||||
| xAI | ✓ | ✗ | ✓ | ✗ | ✗ | ✓ | ✗ |
|
||||
| Z.ai | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ |
|
||||
| Meta | ✓ | ✓ (hint) | ✗ | ✗ | ✓ | ✓ | ✗ |
|
||||
| Black Forest Labs | 1 | per model | per model | ✓ | ✓ | per model (1–8) | `flux-pro-1.0-fill` |
|
||||
| fal | ✓ | per model | per model | ✓ | ✓ | 1 (several on `/edit`, `/multi`) | ✓ |
|
||||
| Replicate | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ (use `providerOptions`) | ✗ |
|
||||
| Stability `image` | 1 | ✗ | ✓ | ✓ | ✓ | 1 (not on `core`) | ✗ |
|
||||
| Stability `upscale()` | ✗ | ✗ | ✗ | ✓ | ✓ | exactly 1 (required) | ✗ |
|
||||
|
||||
✓ lowers natively; ✗ fails whenever the field is set (including `n: 1`); `1` means `n > 1` fails. ¹ `Image.stream` on OpenAI generates one image. fal
|
||||
rejects `size` and `aspectRatio` together; which one a fal or BFL model takes depends on the model.
|
||||
|
||||
`Media.Asset` is the one asset type shared by image requests, image responses, LLM messages, and tool results.
|
||||
`asset.source` is the serializable `Media.Source` (`bytes`, `base64`, `url`, or `ref`); `asset.bytes()`,
|
||||
`asset.base64()`, and `asset.dataUrl()` decode or download lazily and cache; `asset.materialize()` pulls a `url`
|
||||
@@ -468,9 +500,8 @@ asset into owned bytes before the provider URL expires. Construct assets with `M
|
||||
Pass ordered image inputs to the same method for editing, composition, or image-conditioned generation:
|
||||
|
||||
```ts
|
||||
const response =
|
||||
yield *
|
||||
Image.generate({
|
||||
const composed = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model,
|
||||
prompt: "Combine these product photos into one studio scene",
|
||||
images: [
|
||||
@@ -481,23 +512,25 @@ const response =
|
||||
providerOptions,
|
||||
http,
|
||||
})
|
||||
return response.images
|
||||
})
|
||||
```
|
||||
|
||||
`Media.ref(provider, id)` represents provider file handles such as OpenAI file IDs or Gemini Files URIs; routes
|
||||
only forward refs that belong to their own provider. Raw strings are not accepted as image inputs, avoiding
|
||||
ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a
|
||||
non-empty array selects the provider's edit behavior without enforcing provider image-count limits locally. OpenAI
|
||||
only forward refs that belong to their own provider (OpenAI, xAI, and Gemini images accept them). No shipped route
|
||||
returns a ref yet, and `asset.bytes()` / `materialize()` on a ref fail by design. Raw strings are not accepted as
|
||||
image inputs, avoiding ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a
|
||||
non-empty array selects the provider's edit behavior (see the table above for routes that limit the count). OpenAI
|
||||
uses multipart for byte/data-URL edits and its JSON reference body for URL or file-ID edits. The common `mask`
|
||||
field selects inpainting; routes that cannot honor it fail with `UnsupportedOperation`:
|
||||
|
||||
```ts
|
||||
yield *
|
||||
Image.generate({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt,
|
||||
images: [Media.bytes(sourceBytes, "image/png")],
|
||||
mask: Media.bytes(maskBytes, "image/png"),
|
||||
})
|
||||
const inpainted = Image.generate({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt,
|
||||
images: [Media.bytes(sourceBytes, "image/png")],
|
||||
mask: Media.bytes(maskBytes, "image/png"),
|
||||
})
|
||||
```
|
||||
|
||||
On multipart requests, `http.body` can override option fields but not structural `model`, `prompt`, `image[]`,
|
||||
@@ -508,31 +541,31 @@ not accept image inputs. These cases fail with a typed `AIError` before network
|
||||
Provider-native image options belong to each request. Raw `http.body` fields have final precedence over them:
|
||||
|
||||
```ts
|
||||
yield *
|
||||
Image.generate({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt,
|
||||
providerOptions: { quality: "medium" },
|
||||
http,
|
||||
})
|
||||
const medium = Image.generate({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt,
|
||||
providerOptions: { quality: "medium" },
|
||||
http,
|
||||
})
|
||||
```
|
||||
|
||||
xAI image models use the same request API with xAI-native controls:
|
||||
|
||||
```ts
|
||||
yield *
|
||||
Image.generate({
|
||||
model: XAI.configure({ apiKey })("any-model-id"),
|
||||
prompt,
|
||||
n: 2,
|
||||
aspectRatio: "16:9",
|
||||
providerOptions: {
|
||||
resolution: "1k",
|
||||
responseFormat: "b64_json",
|
||||
future_option: true,
|
||||
},
|
||||
http,
|
||||
})
|
||||
import { XAI } from "@opencode/ai/providers"
|
||||
|
||||
const xai = Image.generate({
|
||||
model: XAI.configure({ apiKey }).image("any-model-id"),
|
||||
prompt,
|
||||
n: 2,
|
||||
aspectRatio: "16:9",
|
||||
providerOptions: {
|
||||
resolution: "1k",
|
||||
responseFormat: "b64_json",
|
||||
future_option: true,
|
||||
},
|
||||
http,
|
||||
})
|
||||
```
|
||||
|
||||
Google's current Gemini image models use the same direct API:
|
||||
@@ -542,7 +575,7 @@ import { Google } from "@opencode/ai/providers"
|
||||
|
||||
const googleProgram = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: Google.configure({ apiKey })("any-model-id"),
|
||||
model: Google.configure({ apiKey }).image("any-model-id"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
aspectRatio: "16:9",
|
||||
seed: 42,
|
||||
@@ -567,23 +600,88 @@ their mapped aliases, and `http.body` is the final deep overlay. The selected mo
|
||||
Z.ai image models infer open Z.ai-native options from the selected model:
|
||||
|
||||
```ts
|
||||
yield *
|
||||
Image.generate({
|
||||
model: ZAI.configure({ apiKey })("any-model-id"),
|
||||
prompt,
|
||||
providerOptions: {
|
||||
quality: "hd",
|
||||
userID: "user-123",
|
||||
future_option: true,
|
||||
},
|
||||
http,
|
||||
})
|
||||
import { ZAI } from "@opencode/ai/providers"
|
||||
|
||||
const zai = Image.generate({
|
||||
model: ZAI.configure({ apiKey }).image("any-model-id"),
|
||||
prompt,
|
||||
providerOptions: {
|
||||
quality: "hd",
|
||||
userID: "user-123",
|
||||
future_option: true,
|
||||
},
|
||||
http,
|
||||
})
|
||||
```
|
||||
|
||||
Z.ai does not include trustworthy MIME metadata for output URLs, so generated images use
|
||||
`application/octet-stream` until materialized. Output URLs expire after 30 days; call `asset.materialize()` and
|
||||
persist the bytes promptly if they must remain available.
|
||||
|
||||
### Partial images
|
||||
|
||||
OpenAI's GPT image models stream previews. `Image.stream` sends `stream: true` with `partialImages` (0–3, default 2)
|
||||
and emits `image-partial` events before each final `image`; `Image.generate` keeps the plain JSON request:
|
||||
|
||||
```ts
|
||||
import { Stream } from "effect"
|
||||
import { ImageEvent } from "@opencode/ai"
|
||||
|
||||
const previews = Image.stream({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt: "A lighthouse at dusk",
|
||||
providerOptions: { partialImages: 2 },
|
||||
}).pipe(Stream.runForEach((event) => (ImageEvent.is.imagePartial(event) ? showPreview(event.image) : Effect.void)))
|
||||
```
|
||||
|
||||
The provider may send fewer previews than requested when the final image is ready first.
|
||||
|
||||
### Queued image providers
|
||||
|
||||
Black Forest Labs, fal, Replicate, and Stability's creative upscaler are submit-then-poll routes. `Image.generate`
|
||||
and `Image.stream` poll for you (pass `{ poll }` to tune the interval and timeout); `Image.start` returns a
|
||||
`Generation` whose `token` is serializable JSON for `Image.resume` in another process:
|
||||
|
||||
```ts
|
||||
import { BlackForestLabs, Stability } from "@opencode/ai/providers"
|
||||
|
||||
const bfl = BlackForestLabs.configure({ apiKey: process.env.BFL_API_KEY })
|
||||
|
||||
const submit = Effect.gen(function* () {
|
||||
const generation = yield* Image.start({ model: bfl.image("flux-2-pro"), prompt, size: "1024x768" })
|
||||
persist({ provider: "black-forest-labs", modelID: "flux-2-pro", token: generation.token })
|
||||
})
|
||||
|
||||
const finish = Effect.gen(function* () {
|
||||
const saved = load()
|
||||
const resumed = yield* Image.resume(bfl.image(saved.modelID), saved.token)
|
||||
return yield* resumed.await({ poll: { interval: "2 seconds" } })
|
||||
})
|
||||
```
|
||||
|
||||
The token carries no route identity, so persist the provider and model ID alongside it: `resume` needs the model.
|
||||
|
||||
- **Black Forest Labs** — results are downloaded before returning, because `result.sample` expires in 10 minutes.
|
||||
- **Replicate** — inputs are model-defined, so only `prompt` lowers: sizing, count, seed, format, and files go in
|
||||
`providerOptions` under the model's names, with files as `Media.Asset` (data URLs up to 256 KB, larger by URL).
|
||||
Outputs are removed an hour after the prediction completes. `Prefer: wait=60` in `headers` or `http.headers` holds
|
||||
the submission open so a fast prediction costs one result read.
|
||||
- **Stability** — `stability.image(id)` generates inline; `stability.upscale()` is the creative upscaler, queued:
|
||||
|
||||
```ts
|
||||
const stability = Stability.configure({ apiKey: process.env.STABILITY_API_KEY })
|
||||
const upscaled = Effect.gen(function* () {
|
||||
const small = yield* Media.file("./small.png")
|
||||
return yield* Image.generate(
|
||||
{ model: stability.upscale(), prompt: "A lighthouse", images: [small] },
|
||||
{ poll: { interval: "5 seconds" } },
|
||||
)
|
||||
})
|
||||
```
|
||||
|
||||
Imagen is not available: Google shut it down on the Gemini API, and Vertex discontinued the Imagen 4 models on
|
||||
2026-06-30. `Google.image(...)` uses Gemini-native image models.
|
||||
|
||||
Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool:
|
||||
|
||||
```ts
|
||||
@@ -600,20 +698,257 @@ const program = Effect.gen(function* () {
|
||||
})
|
||||
```
|
||||
|
||||
The hosted result is represented as a provider-executed tool call and tool result, and the generated image is also emitted as a first-class `media` `LLMEvent` (`response.message` then carries a `media` part). Gemini image-capable models emit the same `media` event for inline image output. Retaining `response.message` preserves the generated image for continuation on both routes.
|
||||
The hosted result is represented as a provider-executed tool call and a tool result whose content carries the generated image as a file. Gemini image-capable models instead emit a first-class `media` `LLMEvent` for inline image output (`response.message` then carries a `media` part). Retaining `response.message` preserves the generated image for continuation on both routes.
|
||||
|
||||
## Video generation
|
||||
|
||||
Video mirrors `Image` with one difference: every provider is asynchronous, so the route is a submit-then-poll
|
||||
`Generation`. Models come from `.video(...)` selectors on the `Google` (Veo), `XAI`, `Fal`, and `Runway` facades.
|
||||
Common fields (`frames`, `references`, `video`, `durationSeconds`, `aspectRatio`, `resolution`, `audio`, `n`, `seed`,
|
||||
`negativePrompt`) lower natively or fail with a typed `AIError` before any network call; provider-native controls live
|
||||
under `providerOptions`, inferred from the selected model.
|
||||
|
||||
```ts
|
||||
import { Video } from "@opencode/ai"
|
||||
import { Google, Runway } from "@opencode/ai/providers"
|
||||
|
||||
const google = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY })
|
||||
|
||||
// Simple: submit and wait.
|
||||
const program = Effect.gen(function* () {
|
||||
const response = yield* Video.generate(
|
||||
{
|
||||
model: google.video("veo-3.1-generate-preview"),
|
||||
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
|
||||
aspectRatio: "16:9",
|
||||
resolution: "1080p",
|
||||
durationSeconds: 8,
|
||||
providerOptions: { personGeneration: "allow_adult" },
|
||||
},
|
||||
{ poll: { interval: "10 seconds", timeout: "10 minutes" } },
|
||||
)
|
||||
// Veo serves files for two days behind the API key. The asset knows the deadline (`expiresAt`) and carries the
|
||||
// download credentials only on the live instance (`asset.headers`), never in `source` or JSON: materialize
|
||||
// before persisting, or the persisted URL cannot be fetched again.
|
||||
return yield* response.video.materialize()
|
||||
})
|
||||
|
||||
// Explicit control: keep the handle, persist the token, resume elsewhere.
|
||||
const controlled = Effect.gen(function* () {
|
||||
const generation = yield* Video.start({ model: google.video("veo-3.1-generate-preview"), prompt })
|
||||
generation.id // provider operation / task / request id
|
||||
generation.status // "queued" | "running" | "completed" | "failed" | "cancelled" | "expired"
|
||||
generation.token // route-owned JSON: `{ operation }`, `{ requestID }`, `{ taskID }`, or fal's follow-up URLs
|
||||
// The token carries no route identity: persist the provider and model ID alongside it, since `resume` needs the model.
|
||||
const saved = JSON.stringify(generation.token)
|
||||
|
||||
const resumed = yield* Video.resume(google.video("veo-3.1-generate-preview"), JSON.parse(saved))
|
||||
return yield* resumed.await({ poll: { interval: "10 seconds" } })
|
||||
})
|
||||
|
||||
// Progress as a stream: generation-queued | generation-progress | video | finish.
|
||||
const events = Video.stream({ model: Runway.configure({ apiKey }).video("gen4.5"), prompt }, { poll })
|
||||
```
|
||||
|
||||
Status polls, result fetches, cancels, and asset downloads all run through the same request executor with the route's
|
||||
auth. `Generation.await` and `Generation.events` fail with a
|
||||
`Timeout` reason when `poll.timeout` (default 10 minutes) elapses. Failed,
|
||||
cancelled, and expired generations fail typed with the provider's terminal document on `reason.body`; moderation
|
||||
outcomes (Veo `raiMediaFilteredReasons`, xAI `respect_moderation`, Runway `SAFETY.*` codes) surface as `notices` when
|
||||
a video is still returned and as a `ContentPolicy` reason when nothing is.
|
||||
|
||||
Provider notes:
|
||||
|
||||
- **Google Veo** takes inline bytes only (materialize `url` assets first); `frames.last` requires `frames.first`;
|
||||
audio is always on, so `audio: false` fails typed; one video per request. Output URLs need the API key to
|
||||
download, which the returned asset holds transiently (see above).
|
||||
- **xAI** sends a `video` input to `/videos/edits`, or `/videos/extensions` with `providerOptions.mode: "extend"`.
|
||||
`seed` and `negativePrompt` are not supported.
|
||||
- **fal** endpoints are model-specific: `durationSeconds`, `references`, and `frames.last` fail typed and belong in
|
||||
`providerOptions` under the model's own names (`duration: "8s"`, `end_image_url`, …). Auth is
|
||||
`Authorization: Key <FAL_KEY>`.
|
||||
- **Runway** expects pixel ratios in `aspectRatio` for most models (`"1280:720"`), pins `X-Runway-Version`, reports
|
||||
`usage: { type: "credits" }`, and its output URLs expire after 24–48 hours.
|
||||
|
||||
The promise client exposes the same surface: `ai.video.start(...)` resolves to a handle with `await`, `events`,
|
||||
`result`, `refresh`, `cancel`, and `token`; `ai.video.generate`, `ai.video.resume(model, token)`, and
|
||||
`ai.video.stream` mirror the Effect API. The handle's `status` and `progress` are a snapshot from when it was
|
||||
created; `refresh()` resolves to a new handle.
|
||||
|
||||
```ts
|
||||
import { ai } from "@opencode/ai/promise"
|
||||
|
||||
const generation = await ai.video.start({ model, prompt })
|
||||
for await (const event of generation.events({ poll: { interval: 10_000 } })) console.log(event.type)
|
||||
const video = await generation.result({ signal })
|
||||
await ai.write(video.video, "./kite.mp4")
|
||||
```
|
||||
|
||||
## Speech generation
|
||||
|
||||
Speech (text-to-speech) is one request whose response is parsed incrementally, so every route supports both
|
||||
`Speech.generate` (the whole file) and `Speech.stream` (audio chunks as they arrive). Models come from `.speech(...)`
|
||||
selectors on the `OpenAI`, `Google` (Gemini TTS), `ElevenLabs`, `Cartesia`, and `Deepgram` facades. Common fields
|
||||
(`voice`, `format`, `speed`, `language`, `instructions`, `timestamps`) lower natively or fail with a typed `AIError`
|
||||
before any network call; provider-native controls live under `providerOptions`, inferred from the selected model.
|
||||
|
||||
```ts
|
||||
import { Media, Speech, SpeechClient, SpeechEvent } from "@opencode/ai"
|
||||
import { ElevenLabs, OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
|
||||
|
||||
// The whole file, written to disk.
|
||||
const program = Effect.gen(function* () {
|
||||
const response = yield* Speech.generate({
|
||||
model: openai.speech("gpt-4o-mini-tts"),
|
||||
text: "Hello from OpenCode.",
|
||||
voice: "coral",
|
||||
format: "mp3",
|
||||
instructions: "Warm and unhurried.",
|
||||
})
|
||||
response.audio // Media.Asset with bytes; headerless PCM carries info.encoding / sampleRate / channels
|
||||
response.usage // undefined: OpenAI reports tokens only on SSE streams (Gemini: tokens; ElevenLabs: credits; Deepgram: characters)
|
||||
yield* Media.write(response.audio, "hello.mp3")
|
||||
})
|
||||
|
||||
// Chunks as they arrive: audio-delta* (interleaved with timestamps) then one finish carrying the assembled asset.
|
||||
const events = Speech.stream({
|
||||
model: ElevenLabs.configure({ apiKey }).speech("eleven_flash_v2_5"),
|
||||
text: "Hello from OpenCode.",
|
||||
voice: "JBFqnCBsd6RMkjVDRZzb",
|
||||
format: "pcm",
|
||||
timestamps: true,
|
||||
}).pipe(
|
||||
Stream.tap((event) => {
|
||||
if (SpeechEvent.is.audioDelta(event)) return play(event.chunk)
|
||||
if (SpeechEvent.is.timestamps(event)) return highlight(event.items) // { text, startSeconds, endSeconds }[]
|
||||
return Effect.void
|
||||
}),
|
||||
)
|
||||
```
|
||||
|
||||
`voice` is the provider's own identifier — a name on OpenAI and Gemini (`"coral"`, `"Kore"`), a voice id on
|
||||
ElevenLabs and Cartesia. `{ id }` selects an OpenAI custom voice (`{ id: "voice_1234" }`) and means the same as the
|
||||
plain string elsewhere. There is no cross-provider voice catalog. `format` is the container-level word (`mp3`, `wav`,
|
||||
`pcm`, `opus`, `aac`, `flac`); sample rates and bitrates live under `providerOptions`, and a value the route cannot
|
||||
produce fails as `UnsupportedOperation`. Streams buffer every chunk so `finish` can carry the whole clip.
|
||||
|
||||
Provider notes:
|
||||
|
||||
- **OpenAI** streams over SSE (`stream_format: "sse"`), which is also the only place it reports token usage; `tts-1`
|
||||
and `tts-1-hd` do not support SSE and stream the raw audio body instead. `pcm` is 24 kHz 16-bit mono. `language`
|
||||
and `timestamps` are not supported.
|
||||
- **Gemini TTS** returns the provider's default output: WAV for Gemini 3.8 TTS `generate`, raw 16-bit PCM
|
||||
(`audio/L16;codec=pcm;rate=24000`) otherwise. `pcm` is the only explicit `format` it accepts, and it fails typed on
|
||||
Gemini 3.8 `generate`; the route never wraps PCM as WAV. Style is directed in the text, so `instructions` and
|
||||
`speed` fail typed. Only `gemini-3.1-flash-tts-preview` and later support streaming. Two-speaker audio goes through
|
||||
`providerOptions.speechConfig.multiSpeakerVoiceConfig`.
|
||||
- **ElevenLabs** requires `voice` (the path voice id) and authenticates with `xi-api-key`. `format` maps to the
|
||||
`output_format` query parameter (`mp3_44100_128`, `pcm_24000`, `wav_24000`, `opus_48000_64`);
|
||||
`providerOptions.outputFormat` sets the exact string. WAV is only available from `generate`. `timestamps: true`
|
||||
selects the `with-timestamps` endpoints and yields character-level alignment. `instructions` is not supported.
|
||||
- **Cartesia** requires `voice` and pins `Cartesia-Version`. `generate` defaults to MP3 from `/tts/bytes`; streams
|
||||
and `timestamps: true` (word-level) use `/tts/sse`, which only serves raw PCM. `providerOptions.sampleRate`,
|
||||
`bitRate`, and `encoding` complete `output_format`. No usage is reported.
|
||||
- **Deepgram** Aura's voice is the model id (`aura-2-thalia-en`), so `voice` and `language` fail typed. `format`
|
||||
and `providerOptions` lower to query parameters (`encoding`, `container`, `sample_rate`, `bit_rate`); `pcm` is
|
||||
`linear16` without a container. Auth is `Authorization: Token <DEEPGRAM_API_KEY>`.
|
||||
|
||||
The promise client mirrors the Effect API; `ai.speech.stream` is an `AsyncIterable`.
|
||||
|
||||
```ts
|
||||
import { ai } from "@opencode/ai/promise"
|
||||
|
||||
const response = await ai.speech.generate({ model, text: "Hello from OpenCode.", voice: "coral" })
|
||||
await ai.write(response.audio, "hello.mp3")
|
||||
|
||||
for await (const event of ai.speech.stream({ model, text: "Hello from OpenCode.", voice: "coral" })) {
|
||||
if (event.type === "audio-delta") player.write(event.chunk)
|
||||
}
|
||||
```
|
||||
|
||||
## Transcription
|
||||
|
||||
Transcription (speech-to-text) is the one modality whose providers use every route kind: OpenAI and Gemini stream,
|
||||
Deepgram answers inline, and AssemblyAI is queued. `Transcription.generate` and `Transcription.stream` work on all of
|
||||
them; `Transcription.start` / `resume` return a `Generation` on queued routes and fail with `UnsupportedOperation`
|
||||
elsewhere. Models come from `.transcription(...)` selectors on the `OpenAI`, `Google`, `Deepgram`, and `AssemblyAI`
|
||||
facades. Common fields (`language`, `prompt`, `timestamps: "none" | "segment" | "word"`, `diarize`, `speakers`) lower
|
||||
natively or fail with a typed `AIError` before any network call; a route may return more than asked.
|
||||
|
||||
```ts
|
||||
import { Console, Effect, Stream } from "effect"
|
||||
import { Media, Transcription, TranscriptionEvent } from "@opencode/ai"
|
||||
import { AssemblyAI, Deepgram, OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
|
||||
|
||||
const program = Effect.gen(function* () {
|
||||
const audio = yield* Media.file("./call.mp3")
|
||||
|
||||
// Speaker-labelled segments; labels are provider-native strings ("A", "0", "spk:0").
|
||||
const response = yield* Transcription.generate({
|
||||
model: Deepgram.configure({ apiKey }).transcription("nova-3"),
|
||||
audio,
|
||||
diarize: true,
|
||||
timestamps: "word",
|
||||
})
|
||||
response.text // "Hello from OpenCode."
|
||||
response.segments // [{ text, startSeconds, endSeconds, speaker: "0" }]
|
||||
response.words // [{ text, startSeconds, endSeconds, speaker, confidence }]
|
||||
response.language // the provider's own value, lowercased ("en", "english", "en_us")
|
||||
|
||||
// Text deltas as the model transcribes, then one finish carrying the whole transcript.
|
||||
yield* Transcription.stream({ model: openai.transcription("gpt-4o-mini-transcribe"), audio }).pipe(
|
||||
Stream.tap((event) => (TranscriptionEvent.is.textDelta(event) ? Console.log(event.delta) : Effect.void)),
|
||||
Stream.runDrain,
|
||||
)
|
||||
|
||||
// Queued: persist the token with the provider and model ID (the token alone cannot pick the model), resume, and await.
|
||||
const model = AssemblyAI.configure({ apiKey }).transcription("universal-3-5-pro")
|
||||
const generation = yield* Transcription.start({ model, audio })
|
||||
const resumed = yield* Transcription.resume(model, JSON.parse(JSON.stringify(generation.token)))
|
||||
const transcript = yield* resumed.await({ poll: { interval: "3 seconds" } })
|
||||
})
|
||||
```
|
||||
|
||||
Inline routes emit only `finish` from `stream` (no faked deltas); queued routes emit `generation-queued` /
|
||||
`generation-progress` before it.
|
||||
|
||||
Provider notes:
|
||||
|
||||
- **OpenAI** takes inline audio only; `diarize` needs `gpt-4o-transcribe-diarize`, timestamps need `whisper-1`, and `whisper-1` does not stream.
|
||||
- **Gemini** needs a transcribe model (`gemini-3.5-transcribe`); `prompt` and `speakers` fail typed.
|
||||
- **Deepgram** detects the language unless `language` is set; vocabulary goes in `providerOptions.keyterm`.
|
||||
- **AssemblyAI** uploads inline audio before submitting and is the only route that accepts `speakers`.
|
||||
|
||||
The promise client mirrors the Effect API:
|
||||
|
||||
```ts
|
||||
const audio = await ai.file("./call.mp3")
|
||||
const text = (await ai.transcription.generate({ model, audio })).text
|
||||
for await (const event of ai.transcription.stream({ model, audio })) if (event.type === "text-delta") write(event.delta)
|
||||
const generation = await ai.transcription.start({ model: assemblyai, audio })
|
||||
const transcript = await generation.await({ poll: { interval: 3_000 } })
|
||||
```
|
||||
|
||||
## Public API
|
||||
|
||||
- **`LLM.request({...})`** — build a provider-neutral `LLMRequest`. Accepts ergonomic inputs (`system: string`, `prompt: string`) that normalize into the canonical Schema classes.
|
||||
- **`LLM.generate` / `LLM.stream`** — re-exported from `LLMClient` for one-import use.
|
||||
- **`LLM.generate` / `LLM.stream`** — run direct input or an `LLMRequest` through `LLMClient` for one-import use.
|
||||
- **`Message.user(...)` / `Message.assistant(...)` / `Message.tool(...)`** — message constructors from the canonical schema model.
|
||||
- **`LanguageModel.make(...)` / `ToolCallPart.make(...)` / `ToolResultPart.make(...)` / `ToolDefinition.make(...)`** — model and tool-related constructors from the canonical schema model.
|
||||
- **`LLMEvent.is.*`** — typed guards (`is.textDelta`, `is.toolCall`, `is.finish`, …) for filtering streams.
|
||||
- **`Image.request` / `Image.generate` / `Image.stream`** — generate images through a provider-neutral image request and response model.
|
||||
- **`Image.request` / `generate` / `stream` / `start` / `resume`** — images over inline, streaming (partial previews), and queued routes through a provider-neutral request and response model.
|
||||
- **`ImageClient`** — Effect service and layer for image execution, parallel to `LLMClient`.
|
||||
- **`Media`** — the shared asset type (`Media.Asset`, `Media.Source`) and constructors used by messages, tool results, and media requests.
|
||||
- **`Generation`** — provider-neutral handle for an in-flight media generation (`await`, `refresh`, `cancel`, `events`) used by queued media routes.
|
||||
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm` and `image` as Promise / `AsyncIterable` APIs.
|
||||
- **`Video.request` / `generate` / `stream` / `start` / `resume`** — queued video generation through a provider-neutral request; `VideoClient` is its Effect service and layer.
|
||||
- **`Speech.request` / `Speech.generate` / `Speech.stream`** — text-to-speech through a provider-neutral request; `SpeechClient` is its Effect service and layer.
|
||||
- **`Transcription.request` / `generate` / `stream` / `start` / `resume`** — speech-to-text over inline, streaming, and queued routes; `TranscriptionClient` is its Effect service and layer.
|
||||
- **`AIClient.layer` / `AIClient.layerWith(executor)`** — every modality client plus the request executor in one layer.
|
||||
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm`, `image`, `video`, `speech`, and `transcription` as Promise / `AsyncIterable` APIs, plus `file`, `write`, `bytes`, `base64`, and `materialize` for assets.
|
||||
|
||||
## Testing
|
||||
|
||||
@@ -676,11 +1011,13 @@ This is different from prompt caching, server-side history storage, or truncatio
|
||||
Prefer this operation, where supported, when the application owns compaction policy and durable context updates.
|
||||
|
||||
```ts
|
||||
const result = yield * LLMClient.compact(request)
|
||||
const next = LLMRequest.update(request, {
|
||||
messages: result.replacement,
|
||||
const compacted = Effect.gen(function* () {
|
||||
const result = yield* LLMClient.compact(request)
|
||||
const next = LLMRequest.update(request, {
|
||||
messages: result.replacement,
|
||||
})
|
||||
return yield* LLMClient.generate(next)
|
||||
})
|
||||
const response = yield * LLMClient.generate(next)
|
||||
```
|
||||
|
||||
`replacement` replaces the complete input window. Do not append it to the original transcript or extract only the encrypted item: the provider may retain additional messages in its output. Retained user and assistant messages remain ordinary messages with typed text, media, or reasoning parts, in their original order. Provider-specific message IDs, status, and phase use `providerMetadata`, not a raw output array hidden in an assistant message. Unsupported returned item types fail explicitly.
|
||||
@@ -696,16 +1033,16 @@ The input must still fit the model's context window. Explicit compaction is not
|
||||
OpenAI Responses also exposes a separate, explicitly selected mechanism:
|
||||
|
||||
```ts
|
||||
const result =
|
||||
yield *
|
||||
LLMClient.compact(request, {
|
||||
const checkpoint = Effect.gen(function* () {
|
||||
const result = yield* LLMClient.compact(request, {
|
||||
mechanism: "trigger",
|
||||
webSocket, // Optional: without it, the request uses HTTP/SSE.
|
||||
})
|
||||
|
||||
result.checkpoint // Successful encrypted CompactionPart.
|
||||
result.responseID
|
||||
result.usage
|
||||
result.checkpoint // Successful encrypted CompactionPart.
|
||||
result.responseID
|
||||
result.usage
|
||||
})
|
||||
```
|
||||
|
||||
This appends a native `compaction_trigger` control item to the full input and sends a normal Responses request, with tools and instructions retained, `stream: true`, `store: false`, and parallel tool calls enabled. It removes normal-answer text/output-format controls, forced tool choices, output-token/tool-call limits, and automatic `context_management`. Body overlays cannot replace `input` or supply `previous_response_id`/`conversation`; the complete canonical history is required for safe stateless replay. Request metadata, auth, headers, query parameters, service tier, and supported prompt-cache settings are preserved.
|
||||
@@ -719,9 +1056,11 @@ The supplied WebSocket executor can reuse a compatible append baseline for the c
|
||||
Trigger support is separate from endpoint support. Only the OpenAI Responses route advertises it; Azure, xAI, Chat, and compatible Responses routes do not inherit it. Untyped calls still fail before sending: missing route capabilities return `UnsupportedOperation`, while unknown mechanism names and invalid inputs return `InvalidRequest`. Dynamic callers must narrow for the selected mechanism:
|
||||
|
||||
```ts
|
||||
if (LLMClient.canCompact(request, { mechanism: "trigger" })) {
|
||||
const result = yield * LLMClient.compact(request, { mechanism: "trigger" })
|
||||
}
|
||||
const narrowed = Effect.gen(function* () {
|
||||
if (LLMClient.canCompact(request, { mechanism: "trigger" })) {
|
||||
const result = yield* LLMClient.compact(request, { mechanism: "trigger" })
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
This capability describes protocol implementation, **not universal availability on OpenAI API deployments**. The host application owns subscription/deployment eligibility, OAuth, endpoint selection, and deployment-specific headers. Local protocol/socket tests do not establish live provider support.
|
||||
@@ -730,9 +1069,10 @@ This capability describes protocol implementation, **not universal availability
|
||||
|
||||
`providerOptions.contextManagement` lets the provider decide when to compact during an ordinary `generate` or `stream` call. This is an advanced option for callers that own persistence and recovery: persist the complete assistant message, including its checkpoint, before continuing. Enabling the option does not provide durable checkpoint storage, interruption recovery, or model-switch policy. Keep the prior context until a successful checkpoint has been persisted.
|
||||
|
||||
Inside an `Effect.gen`, enable OpenAI compaction with typed provider options:
|
||||
Enable OpenAI compaction with typed provider options:
|
||||
|
||||
```ts
|
||||
import { Effect } from "effect"
|
||||
import { LLM, LLMClient, LLMRequest, Message } from "@opencode/ai"
|
||||
import { OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
@@ -743,9 +1083,11 @@ const request = LLM.request({
|
||||
contextManagement: [{ type: "compaction", compactThreshold: 200_000 }],
|
||||
},
|
||||
})
|
||||
const response = yield * LLMClient.generate(request)
|
||||
const next = LLMRequest.update(request, {
|
||||
messages: [...request.messages, response.message, Message.user("Continue")],
|
||||
const continued = Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request)
|
||||
return LLMRequest.update(request, {
|
||||
messages: [...request.messages, response.message, Message.user("Continue")],
|
||||
})
|
||||
})
|
||||
```
|
||||
|
||||
@@ -973,7 +1315,7 @@ Compose a route with `Route.make({ protocol, endpoint, auth, framing, ... })`. T
|
||||
|
||||
## Effect
|
||||
|
||||
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `LLMClient.layer` for LLM dispatch and `ImageClient.layer` for image dispatch, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
|
||||
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `AIClient.layer` (or `AIClient.layerWith(executor)`) for every modality, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
|
||||
|
||||
## See also
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# Media generation in `@opencode/ai` — public API direction
|
||||
|
||||
Status: proposal. Branch `media-support`.
|
||||
Status: phases 1–4 implemented (through Image queued routes and partial images; ElevenLabs Scribe transcription
|
||||
pending); phase 5 proposal.
|
||||
|
||||
## Goal
|
||||
|
||||
@@ -11,11 +12,11 @@ The design below is derived from a survey of the raw provider APIs (OpenAI, Gemi
|
||||
## What the survey forces
|
||||
|
||||
1. **Three execution shapes, everywhere.** Inline sync (OpenAI images, all TTS, Gemini), async job with polling or webhook (every video provider, BFL, fal, Replicate, AssemblyAI), and bidirectional streams (ElevenLabs/Cartesia/Deepgram WS, realtime). Video has no sync provider at all.
|
||||
2. **Output is never just bytes.** base64, signed URLs with TTLs from 10 minutes (BFL) to 2 days (Veo), URLs that need auth plus redirect (Veo), separate download endpoints (Sora `/content?variant=`), raw bodies (Stability, TTS). Multi-output is the norm.
|
||||
2. **Output is never just bytes.** base64, signed URLs with TTLs from 10 minutes (BFL, so its route downloads before returning via `PollContext.materialize`) to 2 days (Veo), URLs that need auth plus redirect (Veo), separate download endpoints (Sora `/content?variant=`), raw bodies (Stability, TTS). Multi-output is the norm.
|
||||
3. **Inputs have roles.** First/last frame, mask, style/subject reference, source video for edit/extend, reference audio, prior generation id, provider-side file handles (`file_id`, `gs://`, `runway://`, `mm_file://`).
|
||||
4. **Partial streaming is modality-specific.** Images: a few whole partial frames. Audio: ordered chunks plus timestamp events. Jobs: status/progress/logs. Video: none.
|
||||
5. **Usage is a union**: tokens, seconds, characters (often only in headers), credits, compute time.
|
||||
6. **Moderation can be partial success** (Veo strips audio but returns video). Deprecations are constant (Sora API shuts down 2026-09-24, Imagen on Gemini API 2026-08-17).
|
||||
6. **Moderation can be partial success** (Veo strips audio but returns video). Deprecations are constant (Sora API shuts down 2026-09-24; Imagen is shut down on the Gemini API and past its 2026-06-30 discontinuation date on Vertex).
|
||||
|
||||
## Where existing SDKs are weak and we should not be
|
||||
|
||||
@@ -40,7 +41,7 @@ The design below is derived from a survey of the raw provider APIs (OpenAI, Gemi
|
||||
|
||||
### Model selection
|
||||
|
||||
A model value is built as `OpenAI.configure({ apiKey }).responses("gpt-5")` or `.image("gpt-image-2")`: `configure` fixes credentials, endpoint, and defaults; the selector fixes which of the provider's APIs to hit and binds the typed `providerOptions` generic. Media follows the same shape with one selector per modality — `openai.image(id)` today, `.video(id)` / `.speech(id)` / `.transcription(id)` as those modalities land — mirroring `openai.responses(id)`. `Image.request` accepts `ImageModel` only, exactly as `LLM.request` accepts `LanguageModel`.
|
||||
A model value is built as `OpenAI.configure({ apiKey }).responses("gpt-5")` or `.image("gpt-image-2")`: `configure` fixes credentials, endpoint, and defaults; the selector fixes which of the provider's APIs to hit and binds the typed `providerOptions` generic. Media follows the same shape with one selector per modality — `.image(id)`, `.video(id)`, `.speech(id)`, `.transcription(id)` on the facades that offer each — mirroring `openai.responses(id)`. `Image.request` accepts `ImageModel` only, exactly as `LLM.request` accepts `LanguageModel`.
|
||||
|
||||
```ts
|
||||
import { OpenAI, Google } from "@opencode/ai/providers"
|
||||
@@ -54,7 +55,7 @@ Speech.request({ model: openai.speech("gpt-4o-mini-tts"), text })
|
||||
Transcription.request({ model: openai.transcription("gpt-4o-transcribe"), audio })
|
||||
```
|
||||
|
||||
The request namespace and the selector share one word (`Image.request` + `.image(...)`). That redundancy is accepted: a callable facade returning a lazily resolved ref would be a second way to construct the same model, and the type machinery to infer `providerOptions` through it is not worth one word. Where a provider has two APIs for one modality, the selectors stay explicit (`openai.chat`, a future `google.imagen`), and one default per modality per provider is part of the facade definition (OpenAI image → Images API, Google image → Gemini-native since Imagen on the Gemini API shuts down 2026-08-17). Provider package entrypoints keep `model(modelID, settings)` per modality-specific path, e.g. `@opencode/ai/providers/openai/responses`.
|
||||
The request namespace and the selector share one word (`Image.request` + `.image(...)`). That redundancy is accepted: a callable facade returning a lazily resolved ref would be a second way to construct the same model, and the type machinery to infer `providerOptions` through it is not worth one word. Where a provider has two routes for one modality, the selectors stay explicit (`openai.chat`, `stability.image` inline vs `stability.upscale()` queued), and one default per modality per provider is part of the facade definition (OpenAI image → Images API, Google image → Gemini-native; Imagen is shut down, so there is no `google.imagen`). The facade selector (`openai.image(id)`) is the public path for media models. Modality-specific package entrypoints (`model(modelID, settings)` beside today's LLM paths such as `@opencode/ai/providers/openai/responses`) are deferred until Core has a modality-aware model resolver; Core's resolver accepts only `LanguageModel` today.
|
||||
|
||||
### `Media` — the asset type
|
||||
|
||||
@@ -66,7 +67,7 @@ import { Media } from "@opencode/ai"
|
||||
Media.Source =
|
||||
| { type: "bytes"; data: Uint8Array; mediaType: string }
|
||||
| { type: "base64"; data: string; mediaType: string }
|
||||
| { type: "url"; url: string; mediaType?: string; expiresAt?: number; headers?: Record<string, string> }
|
||||
| { type: "url"; url: string; mediaType?: string; expiresAt?: number }
|
||||
| { type: "ref"; provider: ProviderID; id: string; mediaType?: string } // file_id, gs://, runway://, prior generation
|
||||
|
||||
class Media.Asset {
|
||||
@@ -76,6 +77,7 @@ class Media.Asset {
|
||||
readonly info?: { width?; height?; durationSeconds?; sampleRate?; channels?; encoding?; format? }
|
||||
readonly expiresAt?: number
|
||||
readonly providerMetadata?: ProviderMetadata
|
||||
readonly headers?: Record<string, string> // transient download credentials (Veo); never in source/JSON
|
||||
|
||||
bytes(): Effect<Uint8Array, AIError, RequestExecutor.Service> // downloads/decodes lazily, cached
|
||||
base64(): Effect<string, AIError, RequestExecutor.Service>
|
||||
@@ -85,10 +87,18 @@ class Media.Asset {
|
||||
|
||||
Media.bytes(data, mediaType?) Media.base64(data, mediaType?)
|
||||
Media.url(url, options?) Media.ref(provider, id)
|
||||
Media.file(path) // Bun/Node: reads + sniffs; Effect FileSystem variant for layers
|
||||
Media.write(asset, path) // convenience, uses FileSystem
|
||||
Media.file(path) // Effect<Asset, AIError, FileSystem>: reads + sniffs
|
||||
Media.write(asset, path) // Effect<void, AIError, FileSystem | RequestExecutor.Service>
|
||||
```
|
||||
|
||||
`Media.file` and `Media.write` stay Effect-only: bring your platform's `FileSystem` layer. The Promise client owns the
|
||||
runtime path: `ai.file(path)` and `ai.write(asset, path)` read and write through `node:fs/promises` (loaded on first
|
||||
use) with the same media-type sniffing and `InvalidRequest` failures, and `ai.bytes`, `ai.base64`, and
|
||||
`ai.materialize` run the asset methods in its runtime.
|
||||
|
||||
A `ref` source is accepted as input only by routes whose provider issues file handles. No shipped route produces one
|
||||
yet, so `bytes()` and `materialize()` on a ref fail by design until a producer exists.
|
||||
|
||||
Raw-PCM outputs (Gemini TTS, Cartesia raw, Deepgram WS) carry `info.encoding/sampleRate/channels` because there is no container header.
|
||||
|
||||
### Modality namespaces
|
||||
@@ -103,128 +113,253 @@ import { OpenAI, Google, ElevenLabs, Fal } from "@opencode/ai/providers"
|
||||
#### Image
|
||||
|
||||
```ts
|
||||
const request = Image.request({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
images: [Media.file("./ref.png")], // references / edit sources
|
||||
mask: Media.file("./mask.png"),
|
||||
n: 2,
|
||||
size: "1536x1024", // or aspectRatio: "3:2"
|
||||
seed: 7,
|
||||
format: "webp",
|
||||
providerOptions: { quality: "high", background: "transparent" }, // typed per model
|
||||
Effect.gen(function* () {
|
||||
const request = Image.request({
|
||||
model: openai.image("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
images: [yield* Media.file("./ref.png")], // references / edit sources
|
||||
mask: yield* Media.file("./mask.png"),
|
||||
n: 2,
|
||||
size: "1536x1024", // OpenAI sizes by pixels; Gemini/xAI take aspectRatio instead
|
||||
format: "webp",
|
||||
providerOptions: { quality: "high", background: "transparent" }, // typed per model
|
||||
})
|
||||
|
||||
const response = yield* Image.generate(request) // ImageResponse
|
||||
response.image // Media.Asset (first)
|
||||
response.images // Media.Asset[]
|
||||
response.usage // Usage union (see below)
|
||||
response.notices // moderation / partial-result notices
|
||||
|
||||
Image.stream(request) // Stream<ImageEvent>
|
||||
// ImageEvent: generation-queued | generation-progress | image-partial { index, image } | image { index, image } | finish { usage }
|
||||
})
|
||||
|
||||
const response = yield* Image.generate(request) // ImageResponse
|
||||
response.image // Media.Asset (first)
|
||||
response.images // Media.Asset[]
|
||||
response.usage // Usage union (see below)
|
||||
response.notices // moderation / partial-result notices
|
||||
|
||||
yield* Image.stream(request) // Stream<ImageEvent>
|
||||
// ImageEvent: generation-queued | generation-progress | image-partial { index, image } | image { index, image } | finish { usage }
|
||||
```
|
||||
|
||||
`size` and `aspectRatio` are not interchangeable; each route rejects fields it cannot lower — see the portability table
|
||||
in the README's Image generation section.
|
||||
|
||||
Editing is not a separate function; `images`/`mask` on the request select the edit path in the route (OpenAI `/images/edits`, Gemini multimodal parts, xAI `/images/edits`). Routes that cannot honor `mask` fail with `Unsupported`.
|
||||
|
||||
`ImageRoute` is the inline | stream | queued union, dispatched on `route.kind`, like every modality route. `Image.stream` on a streaming route emits `image-partial` previews before each `image`; on a queued route it emits `generation-queued` / `generation-progress` observations, then the result's `image` and `finish` events.
|
||||
|
||||
#### Video
|
||||
|
||||
Shipped in phase 2 (`src/video.ts`, `src/video-client.ts`, protocols `google-video`, `xai-video`, `fal-video`, `runway-video`).
|
||||
|
||||
```ts
|
||||
const request = Video.request({
|
||||
model: google.video("veo-3.1-generate-preview"),
|
||||
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
|
||||
frames: { first: Media.file("./start.png"), last: Media.file("./end.png") },
|
||||
references: [Media.url("https://…/style.png")],
|
||||
video: Media.ref("openai", "video_123"), // edit / extend / remix source
|
||||
durationSeconds: 8,
|
||||
aspectRatio: "16:9",
|
||||
resolution: "1080p",
|
||||
audio: true,
|
||||
n: 1,
|
||||
providerOptions: { personGeneration: "dont_allow", negativePrompt: "text, watermark" },
|
||||
Effect.gen(function* () {
|
||||
const request = Video.request({
|
||||
model: google.video("veo-3.1-generate-preview"),
|
||||
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
|
||||
frames: { first: yield* Media.file("./start.png"), last: yield* Media.file("./end.png") },
|
||||
references: [yield* Media.file("./style.png")],
|
||||
video: Media.bytes(previous, "video/mp4"), // edit / extend source
|
||||
durationSeconds: 8,
|
||||
aspectRatio: "16:9",
|
||||
resolution: "1080p",
|
||||
audio: true,
|
||||
n: 1,
|
||||
seed: 7,
|
||||
negativePrompt: "text, watermark", // common, not provider-native
|
||||
providerOptions: { personGeneration: "allow_adult" },
|
||||
})
|
||||
|
||||
// Simple: wait for it.
|
||||
const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } })
|
||||
response.video // Media.Asset: url (expiresAt on Veo and Runway; transient `headers` for Veo downloads)
|
||||
response.usage // credits on Runway; the other three report none (xAI's usage.cost_in_usd_ticks is not decoded)
|
||||
response.notices // Veo raiMediaFilteredReasons → filtered, xAI respect_moderation → moderated
|
||||
yield* response.video.materialize() // pull bytes before the URL expires
|
||||
|
||||
// Explicit generation control.
|
||||
const generation = yield* Video.start(request) // Generation<VideoResponse>
|
||||
generation.id; generation.status; generation.progress; generation.position; generation.token
|
||||
yield* generation.await({ poll }) // VideoResponse
|
||||
yield* generation.cancel() // fal PUT cancel_url, Runway DELETE /tasks/{id}; Veo and xAI succeed without a request
|
||||
|
||||
// Resume from another process. The token is validated against the route's codec and refreshed once. It carries no
|
||||
// route identity, so persist the provider and model ID alongside it: `resume` needs the model.
|
||||
const resumed = yield* Video.resume(model, JSON.parse(saved))
|
||||
|
||||
// Progress as a stream.
|
||||
Video.stream(request, { poll }) // Stream<VideoEvent>: generation-queued { id, position } | generation-progress { id, progress } | video { index, video } | finish { usage, notices }
|
||||
})
|
||||
|
||||
// Simple: wait for it.
|
||||
const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } })
|
||||
response.video // Media.Asset (url with expiresAt, or bytes when the route downloads)
|
||||
yield* response.video.materialize() // pull bytes before the URL expires
|
||||
|
||||
// Explicit generation control.
|
||||
const generation = yield* Video.start(request) // Generation<VideoResponse>
|
||||
generation.id; generation.status; generation.progress; generation.token // token is serializable JSON
|
||||
yield* generation.await({ poll }) // VideoResponse
|
||||
yield* generation.cancel()
|
||||
|
||||
// Resume from another process.
|
||||
const resumed = yield* Video.resume(model, token) // Generation<VideoResponse>
|
||||
|
||||
// Progress as a stream.
|
||||
yield* Video.stream(request) // Stream<VideoEvent>: generation-queued { position } | generation-progress { progress, logs } | video { index, video } | finish
|
||||
```
|
||||
|
||||
Webhooks: `Video.complete(model, token, webhook)` finishes a generation from a webhook payload without polling. Token shape is route-owned and opaque (Veo operation name, fal `response_url`, Runway task id).
|
||||
Tokens are route-owned JSON: Veo `{ operation }`, xAI `{ requestID }`, Runway `{ taskID }`, fal
|
||||
`{ requestID, statusURL, responseURL, cancelURL }` (fal's follow-up URLs are authoritative and absolute). Common-field
|
||||
lowering per provider: Veo takes inline media only, rejects `audio: false` and `n > 1`, and requires `frames.first`
|
||||
when `frames.last` is set; xAI rejects `n`, `seed`, and `negativePrompt` and routes a `video` input to edits or
|
||||
(`providerOptions.mode: "extend"`) extensions; fal rejects `n`, plus `durationSeconds`, `references`, and `frames.last`
|
||||
because the field names and enums differ per model; Runway passes `aspectRatio` through as its pixel `ratio` and
|
||||
rejects `n`.
|
||||
|
||||
Deferred: `Video.complete(model, token, webhook)` (finish from a webhook payload without polling) and provider poll
|
||||
hints (none of the four providers emit one). Later providers: Luma, Kling, MiniMax, Replicate.
|
||||
|
||||
#### Speech (TTS)
|
||||
|
||||
Shipped in phase 3 (`src/speech.ts`, `src/speech-client.ts`, protocols `openai-speech`, `google-speech`,
|
||||
`elevenlabs-speech`, `cartesia-speech`, `deepgram-speech`; new `ElevenLabs`, `Cartesia`, and `Deepgram` facades).
|
||||
|
||||
```ts
|
||||
const request = Speech.request({
|
||||
model: elevenlabs.speech("eleven_v3"),
|
||||
model: elevenlabs.speech("eleven_flash_v2_5"),
|
||||
text: "Hello from OpenCode.",
|
||||
voice: "JBFqnCBsd6RMkjVDRZzb", // name, uuid, or { id } — provider-normalized
|
||||
voice: "JBFqnCBsd6RMkjVDRZzb", // provider-native identifier, or { id }
|
||||
format: "mp3", // mp3 | wav | pcm | opus | aac | flac | (string & {})
|
||||
speed: 1.0,
|
||||
language: "en",
|
||||
instructions: "Warm, unhurried.",
|
||||
providerOptions: { stability: 0.5 },
|
||||
instructions: "Warm, unhurried.", // only OpenAI; elsewhere fails typed
|
||||
timestamps: true, // request alignment; routes without it fail typed
|
||||
providerOptions: { voice_settings: { stability: 0.5 } },
|
||||
})
|
||||
|
||||
const response = yield* Speech.generate(request) // SpeechResponse: audio: Media.Asset, timestamps?, usage
|
||||
yield* Speech.stream(request) // Stream<SpeechEvent>: audio-delta { chunk } | timestamps { words } | finish
|
||||
const response = yield* Speech.generate(request) // SpeechResponse: audio: Media.Asset, timestamps?, usage?, providerMetadata?
|
||||
yield* Speech.stream(request) // Stream<SpeechEvent>: generation-queued | generation-progress | audio-delta { chunk } | timestamps { items } | finish { audio, usage? }
|
||||
```
|
||||
|
||||
Streaming TTS is first-class on day one: OpenAI `stream_format: sse`, ElevenLabs `/stream`, Cartesia SSE, Deepgram chunked. Input-streaming TTS (WS, text arrives incrementally) is a later `Speech.session(...)` scoped resource, not part of `generate`.
|
||||
Execution is `MediaProtocol.stream` for every provider: one request whose body is framed and folded by a `step`
|
||||
state machine, with `generate` running the same stream and collecting it. The route submits the request with its
|
||||
`mode` (`"generate" | "stream"`), which lets one provider stay one protocol — OpenAI adds `stream_format: "sse"` (except `tts-1`/`tts-1-hd`, which stream raw bytes), ElevenLabs appends
|
||||
`/stream`, Cartesia switches `/tts/bytes` to `/tts/sse`, Gemini switches `generateContent` to
|
||||
`streamGenerateContent`. The terminal `finish` event carries the assembled asset (every provider's stream is
|
||||
concatenable chunks), so stream consumers also get the whole file and `generate` is just "take `finish`, gather
|
||||
`timestamps`". The cost is memory: a stream holds every chunk until `finish`, so even a consumer that only plays deltas
|
||||
keeps the whole clip in memory. That is bounded by the providers' input text limits (a few minutes of audio); a
|
||||
long-form or session API would need an opt-out.
|
||||
|
||||
**Voice.** `voice?: string | { id: string }`. A string is passed through as the provider's native identifier — a
|
||||
name on OpenAI and Gemini, a voice id on ElevenLabs (path segment) and Cartesia. `{ id }` selects an OpenAI custom
|
||||
voice and is treated as the plain string on routes that do not distinguish custom from built-in. Deepgram's voice is
|
||||
the model id (`aura-2-thalia-en`), so `voice` is `unsupported` there. There is no cross-provider voice catalog or
|
||||
name→id resolution. Multi-speaker (Gemini `speechConfig.multiSpeakerVoiceConfig`) and per-voice settings
|
||||
(ElevenLabs `voice_settings`) go through `providerOptions`.
|
||||
|
||||
**Format and PCM.** `format` is container-level; provider sample rates and bitrates live under `providerOptions`
|
||||
(ElevenLabs `outputFormat`, Cartesia `sampleRate`/`bitRate`/`encoding`, Deepgram `encoding`/`container`/`sampleRate`/
|
||||
`bitRate`). Each protocol maps `format` to its native value (ElevenLabs `mp3_44100_128`/`pcm_24000`/`wav_24000`/
|
||||
`opus_48000_64`, Cartesia `{ container, encoding, sample_rate }`, Deepgram `encoding`+`container`) and declares the
|
||||
asset's media type rather than sniffing, because headerless PCM can look like an MPEG frame sync. Headerless PCM
|
||||
always carries `info.encoding`, `info.sampleRate`, and `info.channels`; its media type is the provider's declaration
|
||||
(Gemini `audio/L16;codec=pcm;rate=24000`, Deepgram's `content-type`) or `audio/pcm`. Gemini's asset follows the
|
||||
provider's declared type: WAV for Gemini 3.8 TTS `generate`, headerless PCM otherwise. The route never wraps PCM as WAV,
|
||||
so `pcm` is the only explicit `format` it accepts, and not on Gemini 3.8 `generate`. Every `format` value a route cannot
|
||||
produce (unknown to it, a container on Cartesia SSE, WAV on an ElevenLabs stream, anything but `pcm` on Gemini, `pcm` on
|
||||
Gemini 3.8 `generate`) fails the same way as an unsupported field: `UnsupportedOperation` with
|
||||
`operation: "media.format"`.
|
||||
|
||||
**Timestamps.** `timestamps: true` on the request asks for alignment. ElevenLabs selects the `with-timestamps`
|
||||
endpoints (character-level, NDJSON when streaming); Cartesia sets `add_timestamps` on `/tts/sse` (word-level; a
|
||||
`generate` with timestamps collects the SSE stream). OpenAI, Gemini, and Deepgram reject it.
|
||||
|
||||
Common-field lowering per provider:
|
||||
|
||||
| Provider | `voice` | `speed` | `language` | `instructions` | `timestamps` | Usage |
|
||||
|---|---|---|---|---|---|---|
|
||||
| OpenAI | `voice` (name or `{ id }`) | `speed` | unsupported | `instructions` | unsupported | `tokens` from SSE `speech.audio.done` only |
|
||||
| Gemini | `prebuiltVoiceConfig.voiceName` | unsupported | `speechConfig.languageCode` | unsupported (direct in text) | unsupported | `tokens` from `usageMetadata` |
|
||||
| ElevenLabs | path voice id (required) | `voice_settings.speed` | `language_code` | unsupported | `with-timestamps` | `credits` from `character-cost` header |
|
||||
| Cartesia | `voice` (required) | `generation_config.speed` | `language` | unsupported | `add_timestamps` | none |
|
||||
| Deepgram | unsupported (voice is the model) | `speed` query | unsupported | unsupported | unsupported | `characters` from `dg-char-count` header |
|
||||
|
||||
Deferred: `Speech.session(...)` — input-streaming TTS where text arrives incrementally over a WebSocket (ElevenLabs
|
||||
`stream-input`, Cartesia WebSocket contexts, Deepgram WebSocket speak) — is a separate scoped resource, not part of
|
||||
`generate`/`stream`, and ships with the realtime work in phase 5.
|
||||
|
||||
#### Transcription (STT)
|
||||
|
||||
Shipped as the second half of phase 3 (`src/transcription.ts`, `src/transcription-client.ts`, protocols
|
||||
`openai-transcription`, `google-transcription`, `deepgram-transcription`, `assemblyai-transcription`; new `AssemblyAI`
|
||||
facade).
|
||||
|
||||
```ts
|
||||
const request = Transcription.request({
|
||||
model: openai.transcription("gpt-4o-transcribe"),
|
||||
audio: Media.file("./call.wav"),
|
||||
language: "en",
|
||||
prompt: "Names: Shoubhit, OpenCode.",
|
||||
timestamps: "word", // none | segment | word
|
||||
model: openai.transcription("gpt-4o-transcribe-diarize"),
|
||||
audio: yield* Media.file("./call.wav"),
|
||||
language: "en", // provider-native passthrough
|
||||
timestamps: "segment", // none | segment | word
|
||||
diarize: true,
|
||||
providerOptions: { chunkingStrategy: "auto" },
|
||||
speakers: 2, // exact speaker count (AssemblyAI only)
|
||||
providerOptions: { known_speaker_names: ["agent"] },
|
||||
})
|
||||
|
||||
const response = yield* Transcription.generate(request)
|
||||
response.text; response.segments; response.words; response.language; response.durationSeconds
|
||||
yield* Transcription.stream(request) // Stream<TranscriptionEvent>: text-delta | segment | finish
|
||||
response.text; response.segments; response.words; response.language; response.durationSeconds; response.usage
|
||||
yield* Transcription.stream(request) // Stream<TranscriptionEvent>: generation-queued | generation-progress | text-delta | segment | finish
|
||||
const generation = yield* Transcription.start(request) // queued routes only
|
||||
yield* Transcription.resume(model, token)
|
||||
```
|
||||
|
||||
Realtime STT over WebSocket is the same future `session` shape as input-streaming TTS.
|
||||
Transcription is the first modality whose providers span all three protocol kinds, and it needed no fourth kind.
|
||||
Every `MediaRoute` now carries its `kind`; `TranscriptionRoute` is the union of the inline, stream, and queued routes;
|
||||
`TranscriptionModel.fromRoute` is overloaded per protocol kind (arity picks the overload: `<Options>`,
|
||||
`<Options, Frame, State>`, `<Options, Token>`) and composes through the shared `composeRoute` (`src/media-model.ts`),
|
||||
which picks `MediaRoute.inline` / `stream` / `queued`; and `TranscriptionClient`, like every modality client, is
|
||||
`MediaClient.make` (`src/media-client.ts`), which dispatches on `route.kind`. `generate` on a queued route is `start`
|
||||
then `await`; `stream` on an inline route is the response as a single `finish`, and on a queued route it is the status
|
||||
observations followed by `finish`. `start` / `resume` on a non-queued route fail with `UnsupportedOperation` (`transcription.start`). The
|
||||
`finish` event carries the whole transcript (text, segments, words, language, duration, usage), so the stream route's
|
||||
`collect` is just "take `finish`".
|
||||
|
||||
The route layer gained a `binary` body with array-valued `query` (Deepgram) and `Queued.start.prepare` (AssemblyAI's
|
||||
upload); `packages/ai/AGENTS.md` (Media Routes) describes both.
|
||||
|
||||
Settled rules:
|
||||
|
||||
- **Timestamps.** A granularity the selected route or model cannot produce fails as `UnsupportedOperation`
|
||||
(`media.timestamps`), following Speech; a route that returns more than asked (Deepgram and AssemblyAI always return
|
||||
words) is not stripped. Segments always carry start and end times: Gemini times each transcription part from its
|
||||
word offsets, so segment timestamps and diarization also request word offsets there.
|
||||
- **Diarization.** `diarize` means segments (and words, where the provider labels them) carry `speaker`. Labels are
|
||||
provider-native strings — OpenAI `A` or a known speaker name, Deepgram `0`, Gemini `spk:0`, AssemblyAI `A` — with no
|
||||
cross-provider speaker model. `speakers` is the exact number of speakers to label, which AssemblyAI (`speakers_expected`, the only route that
|
||||
accepts it) treats as a constraint rather than a hint.
|
||||
- **Language** is passed through (`language`, OpenAI `gpt-transcribe` `languages[]`, Gemini `languageCodes`,
|
||||
AssemblyAI `language_code`). `response.language` is the provider's own value, lowercased but not normalized: an
|
||||
ISO code on most routes (AssemblyAI's detection returns `en`), `english` from whisper-1. Deepgram and AssemblyAI
|
||||
assume English unless asked to detect, so a missing `language` enables their detection.
|
||||
- **Gemini** requires a transcribe model; other model ids fail with `UnsupportedOperation` before the call, because
|
||||
general models ignore `audioTranscriptionConfig` and answer conversationally. Streamed chunks carry whole speaker
|
||||
turns (one part per turn), which join with a space.
|
||||
- **Streaming inline providers** emit only `finish`; deltas are never faked.
|
||||
- **Units.** AssemblyAI milliseconds and Gemini protobuf durations (`"0.400s"`) are normalized to seconds at the
|
||||
protocol boundary.
|
||||
|
||||
| Provider | Kind | Audio input | `timestamps` | `diarize` | Unsupported | Usage |
|
||||
|---|---|---|---|---|---|---|
|
||||
| OpenAI | stream (`stream: true` in `stream` mode; `whisper-1` ignores `stream`, so it emits only `finish`) | multipart `file` (inline only) | `whisper-1` (`verbose_json`); diarize model: `segment` | `gpt-4o-transcribe-diarize` (`diarized_json`) | `speakers`; `prompt` on the diarize model | `tokens` or `seconds` |
|
||||
| Gemini | stream (`generateContent` / `streamGenerateContent`) | `inlineData` or Gemini Files `fileData` | `audioTranscriptionConfig.wordTimestamp` | `audioTranscriptionConfig.diarization` | `prompt`, `speakers` | `tokens` |
|
||||
| Deepgram | inline | raw body, or JSON `{ url }` | words always; `segment` → `utterances` | `diarize_model=latest` + `utterances` | `prompt`, `speakers` | `seconds` (`metadata.duration`) |
|
||||
| AssemblyAI | queued (upload → submit → poll) | `/v2/upload` then `audio_url`, or a URL | words always; `segment` → `speaker_labels` | `speaker_labels` | — | `seconds` (`audio_duration`) |
|
||||
|
||||
Deferred: `Transcription.session(...)` — realtime STT over WebSocket (Deepgram live, AssemblyAI streaming, ElevenLabs
|
||||
realtime, OpenAI realtime transcription) — is the same future scoped `session` shape as input-streaming TTS and ships
|
||||
with the realtime work in phase 5. ElevenLabs Scribe is not implemented yet.
|
||||
|
||||
### `Generation` — shared async execution
|
||||
|
||||
```ts
|
||||
class Generation<Response> {
|
||||
readonly id: string
|
||||
readonly model: MediaModel
|
||||
readonly route: GenerationRoute<Response> // token-free: { status, result, cancel?: Effect } closed over the decoded token
|
||||
readonly token: unknown // route-owned serializable JSON
|
||||
readonly status: "queued" | "running" | "completed" | "failed" | "cancelled" | "expired"
|
||||
readonly progress?: number // 0..1, normalized
|
||||
readonly position?: number
|
||||
readonly expiresAt?: number
|
||||
refresh(): Effect<Generation<Response>, AIError>
|
||||
await(options?: { poll?: Poll }): Effect<Response, AIError>
|
||||
result(): Effect<Response, AIError>
|
||||
await(options?: GenerationAwaitOptions): Effect<Response, AIError>
|
||||
cancel(): Effect<void, AIError>
|
||||
events(options?): Stream<GenerationEvent, AIError>
|
||||
events(options?: GenerationAwaitOptions): Stream<GenerationEvent, AIError> // fails with Timeout past poll.timeout, checked per observation
|
||||
}
|
||||
|
||||
Poll = { interval?: Duration; timeout?: Duration; schedule?: Schedule } // route may override from provider hints (`openai-poll-after-ms`)
|
||||
GenerationAwaitOptions = { poll?: Poll }
|
||||
Poll = { interval?: Duration; timeout?: Duration }
|
||||
```
|
||||
|
||||
`Generation` is not video-specific. Image routes on BFL, fal, and Replicate are queued; `Image.start` exists for them. A route declares itself `inline` or `queued`; `generate` on a queued route is `start` then `await`.
|
||||
`Generation` is not video-specific. Image routes on BFL, fal, Replicate, and Stability `upscale()` are queued; `Image.start` exists for them. A route declares itself `inline` or `queued`; `generate` on a queued route is `start` then `await`.
|
||||
|
||||
### Usage
|
||||
|
||||
@@ -250,15 +385,18 @@ const ai = AI.make() // ManagedRuntime over Reque
|
||||
// AI.make({ layer }) to inject a custom executor / recorder / middleware
|
||||
|
||||
const image = await ai.image.generate({ model, prompt })
|
||||
await image.image.bytes()
|
||||
await ai.bytes(image.image) // also ai.base64, ai.materialize, ai.write(asset, path)
|
||||
const reference = await ai.file("./ref.png")
|
||||
|
||||
for await (const event of ai.speech.stream({ model, text, voice })) { … }
|
||||
|
||||
const generation = await ai.video.start({ model, prompt })
|
||||
const generation = await ai.video.start({ model, prompt }) // snapshot handle; refresh() returns a new one
|
||||
for await (const event of generation.events({ poll: { interval: 10_000 } })) { … }
|
||||
const video = await generation.await({ poll: { interval: 10_000 }, signal })
|
||||
const resumed = ai.video.resume(model, JSON.parse(saved))
|
||||
const resumed = await ai.video.resume(model, JSON.parse(saved)) // persist provider + model ID with the token
|
||||
|
||||
const text = await ai.llm.generate({ model, prompt }) // closes today's gap: LLM has no promise API either
|
||||
const request = ai.llm.request({ model, prompt })
|
||||
const text = await ai.llm.generate(request)
|
||||
for await (const event of ai.llm.stream(request)) { … }
|
||||
|
||||
await ai.dispose()
|
||||
@@ -268,30 +406,38 @@ Streams become `AsyncIterable` via `Stream.toAsyncIterable`. `AIError` is thrown
|
||||
|
||||
### Providers
|
||||
|
||||
Existing facades gain per-modality selectors; the modality routes each facade provides:
|
||||
Existing facades gain per-modality selectors; the modality routes each facade provides (*italics* are not
|
||||
implemented):
|
||||
|
||||
| Facade | llm | image | video | speech | transcription | other |
|
||||
|---|---|---|---|---|---|---|
|
||||
| `OpenAI` | responses (default), chat | Images API | Sora (deprecated 2026-09-24) | ✓ | ✓ | |
|
||||
| `Google` | Gemini | Gemini-native (default), `imagen` | Veo | Gemini TTS | Gemini transcribe | |
|
||||
| `OpenAI` | responses (default), chat | Images API (stream) | *Sora skipped (decision 8)* | ✓ | ✓ | |
|
||||
| `Google` | Gemini | Gemini-native | Veo | Gemini TTS | `gemini-3.5-transcribe` | |
|
||||
| `XAI` | ✓ | ✓ | ✓ | | | |
|
||||
| `ElevenLabs` | | | | ✓ | Scribe | soundEffect, music |
|
||||
| `Fal` | | ✓ | ✓ | | | |
|
||||
| `Replicate`, `Runway`, `Luma`, `Kling`, `MiniMax`, `Deepgram`, `Cartesia`, `AssemblyAI`, `BlackForestLabs`, `Stability` | | per provider | | | | |
|
||||
| `ElevenLabs` | | | | ✓ | *Scribe (pending)* | *soundEffect, music (phase 5)* |
|
||||
| `Cartesia` | | | | ✓ | | |
|
||||
| `Deepgram` | | | | Aura | ✓ | |
|
||||
| `Fal` | | ✓ (queued) | ✓ | | | |
|
||||
| `AssemblyAI` | | | | | ✓ (queued) | |
|
||||
| `BlackForestLabs` | | ✓ (queued) | | | | |
|
||||
| `Replicate` | | ✓ (queued) | | | | |
|
||||
| `Stability` | | `image` (inline), `upscale()` (queued) | | | | |
|
||||
| `Runway` | | | ✓ | | | |
|
||||
| `Luma`, `Kling`, `MiniMax` | | *deferred* | *deferred* | | | |
|
||||
|
||||
New facades follow the existing one-file-per-provider rule. Package entrypoints are modality-specific, such as `@opencode/ai/providers/openai/images`, and return the concrete model.
|
||||
New facades follow the existing one-file-per-provider rule. The facade selector is the public path for media models; modality-specific package entrypoints (for example `@opencode/ai/providers/openai/images`) are deferred until Core has a modality-aware model resolver.
|
||||
|
||||
`ImageModel<Options>` already gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, `TranscriptionModel` follow the same generic. A shared `MediaModel` union is what `Generation` and the promise client key on.
|
||||
`ImageModel<Options>` gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, and `TranscriptionModel` follow the same generic. As with `LanguageModel`, the route type does not carry `Options`, so `ImageModel<OpenAIImageOptions>` is an `ImageModel` and client methods take plain `ImageRequestFor`. They share an internal `MediaModel` base class (ids, route, `http` overlays) that is not part of the public exports; `Generation` and the promise client work with the concrete modality models.
|
||||
|
||||
### Routes and protocols
|
||||
|
||||
Media does not fit the LLM four-axis route (SSE frames → event state machine) except for streaming TTS/STT. Reuse `Endpoint`, `Auth`, `Framing`, `RequestExecutor`, and add media protocol kinds:
|
||||
|
||||
- `MediaProtocol.inline` — `body.from(request)` (JSON, multipart, or query), `response.decode(response)` (JSON, or binary body → `Media.Asset`).
|
||||
- `MediaProtocol.queued` — `start`, `status`, `result`, `cancel`, optional `download`, `pollHint`, `token` schema.
|
||||
- `MediaProtocol.stream` — framing + `step` state machine emitting modality events, same discipline as LLM protocols.
|
||||
- `MediaProtocol.queued` — `start` (body + decode to `{ token, snapshot }`), `status`, `result`, optional `cancel`, and a `token` codec. `result` is always a separate GET (against the status document for Veo/xAI/Runway, fal's `response_url` otherwise) so `await` after `start` and after `resume` share one path. `PollContext.auth` hands the auth headers the route sent to the protocol for output URLs that need them (Veo downloads); they become transient `Media.Asset.headers`, never part of `source`. There is no separate `download` step: `Media.Asset.bytes()` downloads through the executor with those headers. `MediaRoute.inline(...)` / `MediaRoute.queued(...)` compose each kind with endpoint and auth; the queued route decodes the token once and hands `Generation` a token-free `{ status, result, cancel? }`.
|
||||
- `MediaProtocol.stream` — `body.from(request)` over the request plus its `mode`, `frames` (a function that picks the framing for the call: `Framing.sse`, `lines`, `document`, or the raw bytes), fresh per-response `initial()` state, `step` emitting modality events, and `finish(state, context)` — with the observed response for header-only usage — emitting exactly one terminal event or failing as an incomplete stream. The route fills `reason.http` on stream errors. `MediaRoute.stream(...)` exposes `stream` and `generate` (the same stream folded by the modality's `collect`).
|
||||
|
||||
`Route.make` for media composes one protocol kind with endpoint/auth. The existing `ImageRoute { generate(request, execute) }` is the ad-hoc version of `inline` and gets folded in.
|
||||
`MediaRoute.inline` / `MediaRoute.queued` / `MediaRoute.stream` compose one protocol kind with endpoint/auth and tag the route with its `kind`; `ImageModel`/`VideoModel`/`SpeechModel`/`TranscriptionModel` share the `MediaModel` base (`src/media-model.ts`).
|
||||
|
||||
### LLM integration
|
||||
|
||||
@@ -320,9 +466,9 @@ Foundation + Image ship together as the reference implementation, serially. Vide
|
||||
## Phasing
|
||||
|
||||
1. **Foundation** — per-modality selectors, `Media`, `Generation`, `Poll`, `Usage` union, `MediaProtocol` kinds, `@opencode/ai/promise` with `llm` + `image`. Port the five existing image protocols onto it. Unify `MediaPart` and add the `media` LLM event (fixes Gemini image output being dropped).
|
||||
2. **Video** — Veo, xAI, fal, Runway first. Then Luma, Kling, MiniMax, Replicate.
|
||||
3. **Speech + Transcription** — OpenAI, ElevenLabs, Gemini TTS, Deepgram, Cartesia, AssemblyAI. Streaming TTS from the start.
|
||||
4. **Image queued routes and partials** — BFL, fal, Replicate, Stability; OpenAI `partial_images` streaming.
|
||||
2. **Video** — ✅ Veo, xAI, fal, Runway shipped (`MediaProtocol.queued`, `Video.start/generate/resume/stream`, promise `ai.video`). Deferred: `Video.complete` (webhooks), Luma, Kling, MiniMax, Replicate.
|
||||
3. **Speech + Transcription** — ✅ Speech: OpenAI, Gemini TTS, ElevenLabs, Cartesia, Deepgram shipped (`MediaProtocol.stream`, `Speech.generate/stream`, promise `ai.speech`). ✅ Transcription: OpenAI, Gemini, Deepgram, AssemblyAI shipped across all three route kinds (`Transcription.generate/stream/start/resume`, promise `ai.transcription`). Pending: ElevenLabs Scribe. Deferred: `Speech.session` and `Transcription.session` (WebSocket streaming).
|
||||
4. **Image queued routes and partials** — ✅ BFL, fal, Replicate, and Stability creative upscale queued; Stability generate inline; OpenAI `partial_images` streaming (`image-partial` restored). Imagen dropped: shut down on the Gemini API and discontinued on Vertex (2026-06-30). Deferred: Stability's synchronous edit and fast/conservative upscale endpoints.
|
||||
5. **Later** — ElevenLabs music/SFX, Lyria, `Speech.session` / `Transcription.session`, realtime.
|
||||
|
||||
Core adoption (session attachments beyond png/jpeg/gif/webp/pdf, image-generation tool, TUI rendering) comes after phase 1 and is a Core concern.
|
||||
|
||||
@@ -1,18 +1,7 @@
|
||||
import { Config, Effect, Formatter, Layer, Schema, Stream } from "effect"
|
||||
import { Config, Effect, Formatter, Schema, Stream } from "effect"
|
||||
import { NodeFileSystem } from "@effect/platform-node"
|
||||
import {
|
||||
Image,
|
||||
ImageClient,
|
||||
LLM,
|
||||
LLMClient,
|
||||
LLMRequest,
|
||||
Media,
|
||||
Message,
|
||||
ProviderID,
|
||||
Tool,
|
||||
ToolRuntime,
|
||||
} from "@opencode/ai"
|
||||
import { Route, Auth, Endpoint, Framing, Protocol, RequestExecutor } from "@opencode/ai/route"
|
||||
import { AIClient, Image, LLM, LLMRequest, Media, Message, ProviderID, Tool, ToolRuntime } from "@opencode/ai"
|
||||
import { Route, Auth, Endpoint, Framing, Protocol } from "@opencode/ai/route"
|
||||
import { OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
/**
|
||||
@@ -243,12 +232,10 @@ const generateImage = Effect.gen(function* () {
|
||||
yield* Media.write(response.image, "tutorial-image.jpg").pipe(Effect.provide(NodeFileSystem.layer))
|
||||
})
|
||||
|
||||
// Provide the LLM runtime and the HTTP request executor once. Keep one path
|
||||
// enabled at a time so the tutorial can demonstrate generate, stream, or
|
||||
// Provide every modality client and the HTTP request executor once with
|
||||
// `AIClient.layer` (`AIClient.layerWith(executor)` swaps the executor). Keep one
|
||||
// path enabled at a time so the tutorial can demonstrate generate, stream, or
|
||||
// tool-loop behavior without spending tokens on every example.
|
||||
const requestExecutorLayer = RequestExecutor.fetchLayer
|
||||
const llmClientLayer = LLMClient.layer.pipe(Layer.provide(requestExecutorLayer))
|
||||
const imageClientLayer = ImageClient.layer.pipe(Layer.provide(requestExecutorLayer))
|
||||
|
||||
const program = Effect.gen(function* () {
|
||||
// yield* generateOnce
|
||||
@@ -257,6 +244,6 @@ const program = Effect.gen(function* () {
|
||||
// yield* generateDynamicObject.pipe(Effect.andThen((response) => Effect.sync(() => console.log(response.object))))
|
||||
// yield* generateImage
|
||||
yield* streamWithTools
|
||||
}).pipe(Effect.provide(Layer.mergeAll(requestExecutorLayer, llmClientLayer, imageClientLayer)))
|
||||
}).pipe(Effect.provide(AIClient.layer))
|
||||
|
||||
Effect.runPromise(program)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/package.json",
|
||||
"version": "2.0.14",
|
||||
"version": "2.0.16",
|
||||
"name": "@opencode/ai",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
|
||||
@@ -104,6 +104,111 @@ const PROVIDERS: ReadonlyArray<Provider> = [
|
||||
vars: [{ name: "XAI_API_KEY" }],
|
||||
validate: (env) => validateBearer("https://api.x.ai/v1/models", Redacted.make(env.XAI_API_KEY)),
|
||||
},
|
||||
{
|
||||
id: "fal",
|
||||
label: "fal",
|
||||
tier: "canary",
|
||||
note: "fal queue image and video recorded tests",
|
||||
vars: [{ name: "FAL_KEY" }],
|
||||
// fal has no free authenticated list endpoint; a 404 for an unknown request id proves the key was accepted.
|
||||
validate: (env) =>
|
||||
Effect.gen(function* () {
|
||||
const http = yield* HttpClient.HttpClient
|
||||
const response = yield* http.execute(
|
||||
HttpClientRequest.get(
|
||||
"https://queue.fal.run/fal-ai/veo3.1/requests/00000000-0000-0000-0000-000000000000/status",
|
||||
).pipe(HttpClientRequest.setHeaders({ authorization: `Key ${Redacted.value(Redacted.make(env.FAL_KEY))}` })),
|
||||
)
|
||||
if (response.status === 404) return undefined
|
||||
return yield* responseError(response)
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "black-forest-labs",
|
||||
label: "Black Forest Labs",
|
||||
tier: "canary",
|
||||
note: "BFL FLUX queued image recorded tests",
|
||||
vars: [{ name: "BFL_API_KEY" }],
|
||||
validate: (env) =>
|
||||
HttpClientRequest.get("https://api.bfl.ai/v1/credits").pipe(
|
||||
HttpClientRequest.setHeader("x-key", Redacted.value(Redacted.make(env.BFL_API_KEY))),
|
||||
executeRequest,
|
||||
),
|
||||
},
|
||||
{
|
||||
id: "replicate",
|
||||
label: "Replicate",
|
||||
tier: "canary",
|
||||
note: "Replicate prediction image recorded tests",
|
||||
vars: [{ name: "REPLICATE_API_TOKEN" }],
|
||||
validate: (env) => validateBearer("https://api.replicate.com/v1/account", Redacted.make(env.REPLICATE_API_TOKEN)),
|
||||
},
|
||||
{
|
||||
id: "stability",
|
||||
label: "Stability AI",
|
||||
tier: "canary",
|
||||
note: "Stability inline generate and queued upscale recorded tests",
|
||||
vars: [{ name: "STABILITY_API_KEY" }],
|
||||
validate: (env) => validateBearer("https://api.stability.ai/v1/user/balance", Redacted.make(env.STABILITY_API_KEY)),
|
||||
},
|
||||
{
|
||||
id: "runway",
|
||||
label: "Runway",
|
||||
tier: "canary",
|
||||
note: "Runway task video recorded tests",
|
||||
vars: [{ name: "RUNWAYML_API_SECRET" }],
|
||||
validate: (env) =>
|
||||
validateBearer("https://api.dev.runwayml.com/v1/organization", Redacted.make(env.RUNWAYML_API_SECRET), {
|
||||
"X-Runway-Version": "2024-11-06",
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "elevenlabs",
|
||||
label: "ElevenLabs",
|
||||
tier: "canary",
|
||||
note: "ElevenLabs text-to-speech recorded tests",
|
||||
vars: [{ name: "ELEVENLABS_API_KEY" }],
|
||||
validate: (env) =>
|
||||
HttpClientRequest.get("https://api.elevenlabs.io/v1/models").pipe(
|
||||
HttpClientRequest.setHeader("xi-api-key", Redacted.value(Redacted.make(env.ELEVENLABS_API_KEY))),
|
||||
executeRequest,
|
||||
),
|
||||
},
|
||||
{
|
||||
id: "cartesia",
|
||||
label: "Cartesia",
|
||||
tier: "canary",
|
||||
note: "Cartesia text-to-speech recorded tests",
|
||||
vars: [{ name: "CARTESIA_API_KEY" }],
|
||||
validate: (env) =>
|
||||
validateBearer("https://api.cartesia.ai/voices?limit=1", Redacted.make(env.CARTESIA_API_KEY), {
|
||||
"Cartesia-Version": "2026-08-14",
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "deepgram",
|
||||
label: "Deepgram",
|
||||
tier: "canary",
|
||||
note: "Deepgram Aura text-to-speech and Nova transcription recorded tests",
|
||||
vars: [{ name: "DEEPGRAM_API_KEY" }],
|
||||
validate: (env) =>
|
||||
HttpClientRequest.get("https://api.deepgram.com/v1/projects").pipe(
|
||||
HttpClientRequest.setHeader("authorization", `Token ${Redacted.value(Redacted.make(env.DEEPGRAM_API_KEY))}`),
|
||||
executeRequest,
|
||||
),
|
||||
},
|
||||
{
|
||||
id: "assemblyai",
|
||||
label: "AssemblyAI",
|
||||
tier: "canary",
|
||||
note: "AssemblyAI queued transcription recorded tests",
|
||||
vars: [{ name: "ASSEMBLYAI_API_KEY" }],
|
||||
validate: (env) =>
|
||||
HttpClientRequest.get("https://api.assemblyai.com/v2/transcript?limit=1").pipe(
|
||||
HttpClientRequest.setHeader("authorization", Redacted.value(Redacted.make(env.ASSEMBLYAI_API_KEY))),
|
||||
executeRequest,
|
||||
),
|
||||
},
|
||||
{
|
||||
id: "cloudflare-ai-gateway",
|
||||
label: "Cloudflare AI Gateway",
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { Layer } from "effect"
|
||||
import { ImageClient } from "./image-client.js"
|
||||
import { LLMClient } from "./route/client.js"
|
||||
import { RequestExecutor } from "./route/executor.js"
|
||||
import { SpeechClient } from "./speech-client.js"
|
||||
import { TranscriptionClient } from "./transcription-client.js"
|
||||
import { VideoClient } from "./video-client.js"
|
||||
|
||||
/** Every modality client over `executor`, which stays in the output so `asset.bytes()` and `Media.write` resolve. */
|
||||
export const layerWith = <E, R>(executor: Layer.Layer<RequestExecutor.Service, E, R>) =>
|
||||
Layer.mergeAll(
|
||||
LLMClient.layer,
|
||||
ImageClient.layer,
|
||||
VideoClient.layer,
|
||||
SpeechClient.layer,
|
||||
TranscriptionClient.layer,
|
||||
).pipe(Layer.provideMerge(executor))
|
||||
|
||||
/** Every modality client plus the executor over `RequestExecutor.fetchLayer`: the one layer most programs need. */
|
||||
export const layer = layerWith(RequestExecutor.fetchLayer)
|
||||
|
||||
export type Services = Layer.Success<typeof layer>
|
||||
|
||||
export * as AIClient from "./ai-client.js"
|
||||
@@ -86,7 +86,7 @@ export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer
|
||||
})
|
||||
}),
|
||||
)
|
||||
export const fetchLayer = layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
|
||||
export const fetchLayer = layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer))
|
||||
|
||||
export const EvaluationClient = {
|
||||
Service,
|
||||
|
||||
@@ -214,7 +214,7 @@ export function request(input: EvaluationRequest | EvaluationRequestInput) {
|
||||
return new EvaluationRequest({
|
||||
...input,
|
||||
model: input.model as unknown as EvaluationModel,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
http: HttpOptions.make(input.http),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+108
-57
@@ -1,4 +1,4 @@
|
||||
import { Duration, Effect, Schedule, Schema, Stream } from "effect"
|
||||
import { Clock, Duration, Effect, Schedule, Schema, Stream } from "effect"
|
||||
import { AIError, TimeoutError } from "./schema/errors.js"
|
||||
|
||||
export const Status = Schema.Literals(["queued", "running", "completed", "failed", "cancelled", "expired"])
|
||||
@@ -11,47 +11,59 @@ export interface Snapshot {
|
||||
/** Normalized 0..1 when the provider reports progress. */
|
||||
readonly progress?: number
|
||||
readonly position?: number
|
||||
readonly expiresAt?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Route-owned generation operations. `token` is the route's serializable handle (operation name, task id, response URL)
|
||||
* so a generation can be resumed from another process; its shape is opaque to `Generation`.
|
||||
* Route-owned generation operations for one generation. The media route decodes its serializable token once (from the
|
||||
* submission response or a `resume` input) and closes over it, so `Generation` never sees the token's shape.
|
||||
*/
|
||||
export interface Route<Response> {
|
||||
readonly status: (token: unknown) => Effect.Effect<Snapshot, AIError>
|
||||
readonly result: (token: unknown) => Effect.Effect<Response, AIError>
|
||||
readonly cancel?: (token: unknown) => Effect.Effect<void, AIError>
|
||||
/** Provider polling hint (e.g. `openai-poll-after-ms`) that overrides the default interval for the next poll. */
|
||||
readonly pollHint?: (snapshot: Snapshot) => Duration.Duration | undefined
|
||||
readonly status: Effect.Effect<Snapshot, AIError>
|
||||
readonly result: Effect.Effect<Response, AIError>
|
||||
readonly cancel?: Effect.Effect<void, AIError>
|
||||
}
|
||||
|
||||
export interface Poll {
|
||||
readonly interval?: Duration.Input
|
||||
readonly timeout?: Duration.Input
|
||||
/** Full override of the polling schedule; `interval` and `pollHint` are ignored when supplied. */
|
||||
readonly schedule?: Schedule.Schedule<unknown, Snapshot>
|
||||
}
|
||||
|
||||
export interface AwaitOptions {
|
||||
readonly poll?: Poll
|
||||
}
|
||||
|
||||
export const DEFAULT_POLL_INTERVAL = Duration.seconds(5)
|
||||
export const DEFAULT_POLL_TIMEOUT = Duration.minutes(10)
|
||||
|
||||
export type Event =
|
||||
| { readonly type: "generation-queued"; readonly id: string; readonly position?: number }
|
||||
| { readonly type: "generation-progress"; readonly id: string; readonly progress?: number }
|
||||
| { readonly type: "generation-finished"; readonly id: string; readonly status: Status }
|
||||
export const QueuedEvent = Schema.Struct({
|
||||
type: Schema.tag("generation-queued"),
|
||||
id: Schema.String,
|
||||
position: Schema.optional(Schema.Number),
|
||||
}).annotate({ identifier: "Generation.Event.Queued" })
|
||||
|
||||
export const ProgressEvent = Schema.Struct({
|
||||
type: Schema.tag("generation-progress"),
|
||||
id: Schema.String,
|
||||
progress: Schema.optional(Schema.Number),
|
||||
}).annotate({ identifier: "Generation.Event.Progress" })
|
||||
|
||||
export type Observation = Schema.Schema.Type<typeof QueuedEvent> | Schema.Schema.Type<typeof ProgressEvent>
|
||||
|
||||
export type Event = Observation | { readonly type: "generation-finished"; readonly id: string; readonly status: Status }
|
||||
|
||||
const TERMINAL: ReadonlySet<Status> = new Set(["completed", "failed", "cancelled", "expired"])
|
||||
|
||||
export const isTerminal = (status: Status) => TERMINAL.has(status)
|
||||
|
||||
export class Generation<Response> {
|
||||
readonly id: string
|
||||
readonly status: Status
|
||||
readonly progress?: number
|
||||
readonly position?: number
|
||||
readonly expiresAt?: number
|
||||
|
||||
constructor(
|
||||
readonly route: Route<Response>,
|
||||
/** Route-owned serializable JSON; pass it to the modality's `resume` from another process. */
|
||||
readonly token: unknown,
|
||||
snapshot: Snapshot,
|
||||
) {
|
||||
@@ -59,7 +71,6 @@ export class Generation<Response> {
|
||||
this.status = snapshot.status
|
||||
this.progress = snapshot.progress
|
||||
this.position = snapshot.position
|
||||
this.expiresAt = snapshot.expiresAt
|
||||
}
|
||||
|
||||
get snapshot(): Snapshot {
|
||||
@@ -68,70 +79,110 @@ export class Generation<Response> {
|
||||
status: this.status,
|
||||
progress: this.progress,
|
||||
position: this.position,
|
||||
expiresAt: this.expiresAt,
|
||||
}
|
||||
}
|
||||
|
||||
get terminal() {
|
||||
return TERMINAL.has(this.status)
|
||||
return isTerminal(this.status)
|
||||
}
|
||||
|
||||
refresh(): Effect.Effect<Generation<Response>, AIError> {
|
||||
return this.route.status(this.token).pipe(Effect.map((snapshot) => new Generation(this.route, this.token, snapshot)))
|
||||
return this.route.status.pipe(Effect.map((snapshot) => new Generation(this.route, this.token, snapshot)))
|
||||
}
|
||||
|
||||
/** Fetch the result without polling; non-completed terminal generations fail with the provider's terminal body. */
|
||||
result(): Effect.Effect<Response, AIError> {
|
||||
return this.route.result
|
||||
}
|
||||
|
||||
/** Poll until the generation reaches a terminal status, then fetch the result. Fails with a `Timeout` reason on deadline. */
|
||||
await(options?: { readonly poll?: Poll }): Effect.Effect<Response, AIError> {
|
||||
await(options?: AwaitOptions): Effect.Effect<Response, AIError> {
|
||||
const timeout = Duration.fromInputUnsafe(options?.poll?.timeout ?? DEFAULT_POLL_TIMEOUT)
|
||||
const settled = this.terminal ? Effect.succeed(this) : this.poll(options?.poll)
|
||||
return settled.pipe(
|
||||
// Non-completed terminal states also go through `result` so the route can surface its provider failure body.
|
||||
Effect.flatMap((generation) => generation.route.result(generation.token)),
|
||||
Effect.timeoutOrElse({
|
||||
duration: timeout,
|
||||
orElse: () =>
|
||||
new AIError({
|
||||
reason: new TimeoutError({
|
||||
message: `Generation ${this.id} did not finish within ${Duration.format(timeout)}`,
|
||||
timeoutMs: Duration.toMillis(timeout),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
Effect.flatMap((generation) => generation.result()),
|
||||
Effect.timeoutOrElse({ duration: timeout, orElse: () => this.timeoutError(timeout) }),
|
||||
)
|
||||
}
|
||||
|
||||
cancel(): Effect.Effect<void, AIError> {
|
||||
return this.route.cancel?.(this.token) ?? Effect.void
|
||||
return this.route.cancel ?? Effect.void
|
||||
}
|
||||
|
||||
/** Status observations as a stream, ending after the first terminal observation. */
|
||||
events(options?: { readonly poll?: Poll }): Stream.Stream<Event, AIError> {
|
||||
const observations = this.terminal
|
||||
? Stream.make(this)
|
||||
: Stream.fromEffectSchedule(this.refresh(), this.schedule(options?.poll)).pipe(
|
||||
Stream.takeUntil((generation) => generation.terminal),
|
||||
)
|
||||
return observations.pipe(
|
||||
Stream.map((generation): Event => {
|
||||
if (generation.terminal) return { type: "generation-finished", id: generation.id, status: generation.status }
|
||||
if (generation.status === "queued") return { type: "generation-queued", id: generation.id, position: generation.position }
|
||||
return { type: "generation-progress", id: generation.id, progress: generation.progress }
|
||||
}),
|
||||
/**
|
||||
* Status observations as a stream, ending after the first terminal observation. Each poll and each sleep between polls
|
||||
* is bounded by the time remaining until `poll.timeout`, so a hung status request or a long interval fails the stream at
|
||||
* the deadline instead of stalling it. (`Stream.interruptWhen` would express this directly but deadlocks under
|
||||
* `TestClock` when the source completes while the timer sleeps.)
|
||||
*/
|
||||
events(options?: AwaitOptions): Stream.Stream<Event, AIError> {
|
||||
if (this.terminal) return Stream.make(this.event())
|
||||
const timeout = Duration.fromInputUnsafe(options?.poll?.timeout ?? DEFAULT_POLL_TIMEOUT)
|
||||
return Stream.unwrap(
|
||||
Clock.currentTimeMillis.pipe(
|
||||
Effect.map((start) => {
|
||||
const deadline = start + Duration.toMillis(timeout)
|
||||
// Fail before polling once the deadline has passed: a fast status request could otherwise win the zero-budget
|
||||
// race and schedule another zero-delay poll.
|
||||
const refresh = Clock.currentTimeMillis.pipe(
|
||||
Effect.flatMap((now) =>
|
||||
now >= deadline
|
||||
? this.timeoutError(timeout)
|
||||
: this.refresh().pipe(
|
||||
Effect.timeoutOrElse({
|
||||
duration: Duration.millis(deadline - now),
|
||||
orElse: () => this.timeoutError(timeout),
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
const schedule = this.schedule(options?.poll).pipe(
|
||||
Schedule.modifyDelay((meta) =>
|
||||
Effect.succeed(Duration.min(meta.duration, Duration.millis(Math.max(0, deadline - meta.now)))),
|
||||
),
|
||||
)
|
||||
return Stream.fromEffectSchedule(refresh, schedule).pipe(
|
||||
Stream.takeUntil((generation) => generation.terminal),
|
||||
Stream.map((generation) => generation.event()),
|
||||
)
|
||||
}),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
private event(): Event {
|
||||
if (this.terminal) return { type: "generation-finished", id: this.id, status: this.status }
|
||||
if (this.status === "queued") return { type: "generation-queued", id: this.id, position: this.position }
|
||||
return { type: "generation-progress", id: this.id, progress: this.progress }
|
||||
}
|
||||
|
||||
private timeoutError(timeout: Duration.Duration) {
|
||||
return new AIError({
|
||||
reason: new TimeoutError({
|
||||
message: `Generation ${this.id} did not finish within ${Duration.format(timeout)}`,
|
||||
timeoutMs: Duration.toMillis(timeout),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
private poll(poll: Poll | undefined) {
|
||||
return this.refresh().pipe(Effect.repeat({ schedule: this.schedule(poll), until: (generation) => generation.terminal }))
|
||||
}
|
||||
|
||||
private schedule(poll: Poll | undefined): Schedule.Schedule<unknown, Generation<Response>> {
|
||||
if (poll?.schedule) return poll.schedule.pipe(Schedule.setInputType<Generation<Response>>())
|
||||
const interval = poll?.interval ?? DEFAULT_POLL_INTERVAL
|
||||
const pollHint = this.route.pollHint
|
||||
const spaced = Schedule.spaced(interval).pipe(Schedule.setInputType<Generation<Response>>())
|
||||
if (!pollHint) return spaced
|
||||
return spaced.pipe(
|
||||
Schedule.modifyDelay((metadata) => Effect.succeed(pollHint(metadata.input.snapshot) ?? interval)),
|
||||
return this.refresh().pipe(
|
||||
Effect.repeat({ schedule: this.schedule(poll), until: (generation) => generation.terminal }),
|
||||
)
|
||||
}
|
||||
|
||||
private schedule(poll: Poll | undefined) {
|
||||
return Schedule.spaced(poll?.interval ?? DEFAULT_POLL_INTERVAL)
|
||||
}
|
||||
}
|
||||
|
||||
export const resultEvents = <Response, A>(
|
||||
generation: Generation<Response>,
|
||||
expand: (response: Response) => ReadonlyArray<A>,
|
||||
options?: AwaitOptions,
|
||||
): Stream.Stream<Observation | A, AIError> =>
|
||||
generation.events(options).pipe(
|
||||
Stream.filter((event): event is Observation => event.type !== "generation-finished"),
|
||||
Stream.concat(Stream.fromIterableEffect(Effect.map(generation.result(), expand))),
|
||||
)
|
||||
|
||||
@@ -1,61 +1,30 @@
|
||||
import { Context, Effect, Layer, Stream } from "effect"
|
||||
import { RequestExecutor } from "./route/executor.js"
|
||||
import type { AIError } from "./schema/index.js"
|
||||
import { Context } from "effect"
|
||||
import { MediaClient } from "./media-client.js"
|
||||
import {
|
||||
responseEvents,
|
||||
ImageOutputEvent,
|
||||
ImageFinishEvent,
|
||||
type ImageEvent,
|
||||
type ImageOptions,
|
||||
type ImageRequestFor,
|
||||
type ImageResponse,
|
||||
} from "./image.js"
|
||||
|
||||
export interface Interface {
|
||||
readonly generate: <Options extends ImageOptions>(
|
||||
request: ImageRequestFor<Options>,
|
||||
) => Effect.Effect<ImageResponse, AIError>
|
||||
readonly stream: <Options extends ImageOptions>(
|
||||
request: ImageRequestFor<Options>,
|
||||
) => Stream.Stream<ImageEvent, AIError>
|
||||
}
|
||||
export type Interface = MediaClient.Interface<ImageRequestFor, ImageEvent, ImageResponse>
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/ImageClient") {}
|
||||
|
||||
export const generate = <Options extends ImageOptions>(
|
||||
request: ImageRequestFor<Options>,
|
||||
): Effect.Effect<ImageResponse, AIError, Service> =>
|
||||
Effect.gen(function* () {
|
||||
const client = yield* Service
|
||||
return yield* client.generate(request)
|
||||
})
|
||||
|
||||
export const stream = <Options extends ImageOptions>(
|
||||
request: ImageRequestFor<Options>,
|
||||
): Stream.Stream<ImageEvent, AIError, Service> =>
|
||||
Stream.unwrap(
|
||||
Effect.gen(function* () {
|
||||
const client = yield* Service
|
||||
return client.stream(request)
|
||||
}),
|
||||
)
|
||||
|
||||
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
const generate = <Options extends ImageOptions>(request: ImageRequestFor<Options>) =>
|
||||
request.model.route.generate(request, executor.execute)
|
||||
return Service.of({
|
||||
generate,
|
||||
// Inline routes have no partial frames yet; the stream is the completed response expanded into events.
|
||||
stream: (request) =>
|
||||
Stream.unwrap(generate(request).pipe(Effect.map((response) => Stream.fromIterable(responseEvents(response))))),
|
||||
})
|
||||
}),
|
||||
)
|
||||
export class ImageClientService extends Context.Service<ImageClientService, Interface>()("@opencode/ImageClient") {}
|
||||
export const Service = ImageClientService
|
||||
export type Service = ImageClientService
|
||||
|
||||
export const ImageClient = {
|
||||
Service,
|
||||
layer,
|
||||
generate,
|
||||
stream,
|
||||
...MediaClient.make(Service, {
|
||||
modality: "image",
|
||||
responseEvents: (response: ImageResponse) => [
|
||||
...response.images.map((image, index) => ImageOutputEvent.make({ index, image })),
|
||||
ImageFinishEvent.make({
|
||||
usage: response.usage,
|
||||
notices: response.notices,
|
||||
providerMetadata: response.providerMetadata,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
} as const
|
||||
|
||||
+81
-111
@@ -1,98 +1,48 @@
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./generation.js"
|
||||
import { Media } from "./media.js"
|
||||
import { Endpoint } from "./route/endpoint.js"
|
||||
import { MediaModel, composeRoute, tryRequest } from "./media-model.js"
|
||||
import { MediaRoute } from "./route/media.js"
|
||||
import type { MediaProtocol } from "./route/media-protocol.js"
|
||||
import {
|
||||
AIError,
|
||||
HttpOptions,
|
||||
InvalidRequestError,
|
||||
MediaUsage,
|
||||
ModelID,
|
||||
ProviderID,
|
||||
ProviderMetadata,
|
||||
} from "./schema/index.js"
|
||||
import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js"
|
||||
import { ImageClient, Service } from "./image-client.js"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Model
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type ImageOptions = Record<string, unknown>
|
||||
export type ImageOptions = MediaModel.Options
|
||||
|
||||
export type ImageRoute<Options extends ImageOptions = ImageOptions> = MediaRoute.Route<
|
||||
ImageRequestFor<Options>,
|
||||
ImageResponse
|
||||
>
|
||||
export type ImageRoute = MediaRoute.AnyRoute<ImageRequestFor, ImageEvent, ImageResponse>
|
||||
|
||||
export class ImageModel<Options extends ImageOptions = ImageOptions> {
|
||||
declare protected readonly _Options: (options: Options) => Options
|
||||
readonly id: ModelID
|
||||
readonly provider: ProviderID
|
||||
readonly route: ImageRoute<Options>
|
||||
readonly http?: HttpOptions
|
||||
export class ImageModel<Options extends ImageOptions = ImageOptions> extends MediaModel<ImageRoute, Options> {
|
||||
declare protected readonly _ImageModel: void
|
||||
|
||||
constructor(input: ImageModel.Input<Options>) {
|
||||
this.id = input.id
|
||||
this.provider = input.provider
|
||||
this.route = input.route
|
||||
this.http = input.http
|
||||
}
|
||||
|
||||
static make<Options extends ImageOptions = ImageOptions>(input: ImageModel.MakeInput<Options>) {
|
||||
return new ImageModel<Options>({
|
||||
id: ModelID.make(input.id),
|
||||
provider: ProviderID.make(input.provider),
|
||||
route: input.route,
|
||||
http: input.http,
|
||||
})
|
||||
}
|
||||
|
||||
/** Compose an inline image protocol with its canonical path into a model for one deployment. */
|
||||
static fromRoute<Options extends ImageOptions = ImageOptions>(
|
||||
route: ImageModel.RouteInput<Options>,
|
||||
/** The number of type arguments selects the kind: `<Options>`, `<Options, Frame, State>`, or `<Options, Token>`. */
|
||||
static fromRoute<Options extends ImageOptions>(
|
||||
route: MediaModel.InlineRouteInput<ImageRequestFor<Options>, ImageResponse>,
|
||||
input: MediaRoute.ModelInput,
|
||||
): ImageModel<Options>
|
||||
static fromRoute<Options extends ImageOptions, Frame, State>(
|
||||
route: MediaModel.StreamRouteInput<ImageRequestFor<Options>, ImageEvent, Frame, State>,
|
||||
input: MediaRoute.ModelInput,
|
||||
): ImageModel<Options>
|
||||
static fromRoute<Options extends ImageOptions, Token>(
|
||||
route: MediaModel.QueuedRouteInput<ImageRequestFor<Options>, ImageResponse, Token>,
|
||||
input: MediaRoute.ModelInput,
|
||||
): ImageModel<Options>
|
||||
static fromRoute<Options extends ImageOptions, Frame, State, Token>(
|
||||
route: MediaModel.AnyRouteInput<ImageRequestFor<Options>, ImageEvent, ImageResponse, Frame, State, Token>,
|
||||
input: MediaRoute.ModelInput,
|
||||
) {
|
||||
return ImageModel.make<Options>({
|
||||
return new ImageModel<Options>({
|
||||
id: input.id,
|
||||
provider: route.provider,
|
||||
provider: route.protocol.provider,
|
||||
http: input.http,
|
||||
route: MediaRoute.make({
|
||||
id: route.id,
|
||||
provider: route.provider,
|
||||
protocol: route.protocol,
|
||||
endpoint: Endpoint.path(route.path, { baseURL: input.baseURL ?? route.baseURL }),
|
||||
auth: input.auth,
|
||||
headers: input.headers,
|
||||
}),
|
||||
route: composeRoute(route, input, collectResponse) as ImageRoute,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export namespace ImageModel {
|
||||
export interface Input<Options extends ImageOptions = ImageOptions> {
|
||||
readonly id: ModelID
|
||||
readonly provider: ProviderID
|
||||
readonly route: ImageRoute<Options>
|
||||
readonly http?: HttpOptions
|
||||
}
|
||||
|
||||
export interface MakeInput<Options extends ImageOptions = ImageOptions>
|
||||
extends Omit<Input<Options>, "id" | "provider"> {
|
||||
readonly id: string | ModelID
|
||||
readonly provider: string | ProviderID
|
||||
}
|
||||
|
||||
export interface RouteInput<Options extends ImageOptions = ImageOptions> {
|
||||
readonly id: string
|
||||
readonly provider: string | ProviderID
|
||||
readonly protocol: MediaProtocol.Inline<ImageRequestFor<Options>, ImageResponse>
|
||||
readonly path: Endpoint.EndpointPart<MediaProtocol.Body, ImageRequestFor<Options>>
|
||||
/** Canonical base URL; `ModelInput.baseURL` overrides it per deployment. */
|
||||
readonly baseURL?: string
|
||||
}
|
||||
}
|
||||
|
||||
export const ImageModelSchema = Schema.declare((value): value is ImageModel => value instanceof ImageModel, {
|
||||
expected: "Image.Model",
|
||||
})
|
||||
@@ -107,13 +57,10 @@ export const ImageSize = Schema.declare<ImageSize>(
|
||||
{ title: "ImageSize" },
|
||||
)
|
||||
|
||||
export type ImageAspectRatio = `${number}:${number}`
|
||||
export const ImageAspectRatio = Schema.declare<ImageAspectRatio>(
|
||||
(value): value is ImageAspectRatio => typeof value === "string" && /^\d+(?:\.\d+)?:\d+(?:\.\d+)?$/.test(value),
|
||||
{ title: "ImageAspectRatio" },
|
||||
)
|
||||
export type ImageAspectRatio = Media.AspectRatio
|
||||
export const ImageAspectRatio = Media.AspectRatio
|
||||
|
||||
export type ImageFormat = "png" | "jpeg" | "webp" | (string & {})
|
||||
export type ImageFormat = OpenString<"png" | "jpeg" | "webp">
|
||||
|
||||
export class ImageRequest extends Schema.Class<ImageRequest>("Image.Request")({
|
||||
model: ImageModelSchema,
|
||||
@@ -168,6 +115,13 @@ export class ImageResponse extends Schema.Class<ImageResponse>("Image.Response")
|
||||
}
|
||||
}
|
||||
|
||||
/** The `index`-th lower-fidelity preview of an image still generating; the `image` event supersedes it. */
|
||||
export const ImagePartialEvent = Schema.Struct({
|
||||
type: Schema.tag("image-partial"),
|
||||
index: Schema.Number,
|
||||
image: Media.AssetSchema,
|
||||
}).annotate({ identifier: "Image.Event.Partial" })
|
||||
|
||||
export const ImageOutputEvent = Schema.Struct({
|
||||
type: Schema.tag("image"),
|
||||
index: Schema.Number,
|
||||
@@ -181,24 +135,37 @@ export const ImageFinishEvent = Schema.Struct({
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}).annotate({ identifier: "Image.Event.Finish" })
|
||||
|
||||
const imageEventTagged = Schema.Union([ImageOutputEvent, ImageFinishEvent]).pipe(Schema.toTaggedUnion("type"))
|
||||
const imageEventTagged = Schema.Union([
|
||||
QueuedEvent,
|
||||
ProgressEvent,
|
||||
ImagePartialEvent,
|
||||
ImageOutputEvent,
|
||||
ImageFinishEvent,
|
||||
]).pipe(Schema.toTaggedUnion("type"))
|
||||
export const ImageEvent = Object.assign(imageEventTagged, {
|
||||
is: {
|
||||
generationQueued: imageEventTagged.guards["generation-queued"],
|
||||
generationProgress: imageEventTagged.guards["generation-progress"],
|
||||
imagePartial: imageEventTagged.guards["image-partial"],
|
||||
image: imageEventTagged.guards.image,
|
||||
finish: imageEventTagged.guards.finish,
|
||||
},
|
||||
})
|
||||
export type ImageEvent = Schema.Schema.Type<typeof imageEventTagged>
|
||||
|
||||
/** Inline routes produce every image at once; expand the response into the streaming event shape. */
|
||||
export const responseEvents = (response: ImageResponse): ReadonlyArray<ImageEvent> => [
|
||||
...response.images.map((image, index) => ImageOutputEvent.make({ index, image })),
|
||||
ImageFinishEvent.make({
|
||||
usage: response.usage,
|
||||
notices: response.notices,
|
||||
providerMetadata: response.providerMetadata,
|
||||
}),
|
||||
]
|
||||
const collectResponse = (events: ReadonlyArray<ImageEvent>): Effect.Effect<ImageResponse> => {
|
||||
const finish = events.find(ImageEvent.is.finish)
|
||||
// Every image protocol's `finish` emits the terminal event or fails, so a completed stream always has one.
|
||||
if (finish === undefined) return Effect.die(new Error("The image stream completed without a finish event"))
|
||||
return Effect.succeed(
|
||||
new ImageResponse({
|
||||
images: events.filter(ImageEvent.is.image).map((event) => event.image),
|
||||
usage: finish.usage,
|
||||
notices: finish.notices,
|
||||
providerMetadata: finish.providerMetadata,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Request-shaped call API
|
||||
@@ -212,40 +179,43 @@ export function request(input: ImageRequest | ImageRequestInput) {
|
||||
if (input instanceof ImageRequest) return input
|
||||
return new ImageRequest({
|
||||
...input,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
http: HttpOptions.make(input.http),
|
||||
})
|
||||
}
|
||||
|
||||
const requestEffect = (input: ImageRequest | ImageRequestInput) =>
|
||||
Effect.try({
|
||||
try: () => request(input),
|
||||
catch: (error) =>
|
||||
new AIError({
|
||||
reason: new InvalidRequestError({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
cause: error,
|
||||
}),
|
||||
}),
|
||||
})
|
||||
const requestEffect = (input: ImageRequest | ImageRequestInput) => tryRequest(() => request(input))
|
||||
|
||||
export function generate<const Model extends ImageModel>(
|
||||
input: ImageRequestInput<Model>,
|
||||
input: ImageRequest | ImageRequestInput<Model>,
|
||||
options?: AwaitOptions,
|
||||
): Effect.Effect<ImageResponse, AIError, Service>
|
||||
export function generate(input: ImageRequest): Effect.Effect<ImageResponse, AIError, Service>
|
||||
export function generate(input: ImageRequest | ImageRequestInput) {
|
||||
return requestEffect(input).pipe(Effect.flatMap((request) => ImageClient.generate(request)))
|
||||
export function generate(input: ImageRequest | ImageRequestInput, options?: AwaitOptions) {
|
||||
return requestEffect(input).pipe(Effect.flatMap((request) => ImageClient.generate(request, options)))
|
||||
}
|
||||
|
||||
export function stream<const Model extends ImageModel>(
|
||||
input: ImageRequestInput<Model>,
|
||||
input: ImageRequest | ImageRequestInput<Model>,
|
||||
options?: AwaitOptions,
|
||||
): Stream.Stream<ImageEvent, AIError, Service>
|
||||
export function stream(input: ImageRequest): Stream.Stream<ImageEvent, AIError, Service>
|
||||
export function stream(input: ImageRequest | ImageRequestInput) {
|
||||
return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => ImageClient.stream(request))))
|
||||
export function stream(input: ImageRequest | ImageRequestInput, options?: AwaitOptions) {
|
||||
return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => ImageClient.stream(request, options))))
|
||||
}
|
||||
|
||||
/** Inline and streaming routes fail with `UnsupportedOperation`. */
|
||||
export function start<const Model extends ImageModel>(
|
||||
input: ImageRequest | ImageRequestInput<Model>,
|
||||
): Effect.Effect<Generation<ImageResponse>, AIError, Service>
|
||||
export function start(input: ImageRequest | ImageRequestInput) {
|
||||
return requestEffect(input).pipe(Effect.flatMap((request) => ImageClient.start(request)))
|
||||
}
|
||||
|
||||
export const resume = (model: ImageModel, token: unknown): Effect.Effect<Generation<ImageResponse>, AIError, Service> =>
|
||||
ImageClient.resume(model, token)
|
||||
|
||||
export const Image = {
|
||||
request,
|
||||
generate,
|
||||
stream,
|
||||
start,
|
||||
resume,
|
||||
} as const
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
export { AIClient } from "./ai-client.js"
|
||||
export { LLMClient } from "./route/client.js"
|
||||
export { ImageClient } from "./image-client.js"
|
||||
export { Auth } from "./route/auth.js"
|
||||
@@ -8,7 +9,7 @@ export type {
|
||||
RouteLanguageModelInput,
|
||||
RouteRoutedLanguageModelInput,
|
||||
Interface as LLMClientShape,
|
||||
Service as LLMClientService,
|
||||
LLMClientService,
|
||||
} from "./route/client.js"
|
||||
export * from "./schema/index.js"
|
||||
export {
|
||||
@@ -29,9 +30,73 @@ export type {
|
||||
ImageRoute,
|
||||
} from "./image.js"
|
||||
export { Image } from "./image.js"
|
||||
export { VideoClient } from "./video-client.js"
|
||||
export {
|
||||
VideoAspectRatio,
|
||||
VideoEvent,
|
||||
VideoFrames,
|
||||
VideoModel,
|
||||
VideoModelSchema,
|
||||
VideoRequest,
|
||||
VideoResponse,
|
||||
} from "./video.js"
|
||||
export type {
|
||||
VideoModelOptions,
|
||||
VideoOptions,
|
||||
VideoRequestFor,
|
||||
VideoRequestInput,
|
||||
VideoResolution,
|
||||
VideoRoute,
|
||||
} from "./video.js"
|
||||
export { Video } from "./video.js"
|
||||
export { SpeechClient } from "./speech-client.js"
|
||||
export {
|
||||
SpeechEvent,
|
||||
SpeechModel,
|
||||
SpeechModelSchema,
|
||||
SpeechRequest,
|
||||
SpeechResponse,
|
||||
SpeechTimestamp,
|
||||
SpeechVoice,
|
||||
} from "./speech.js"
|
||||
export type {
|
||||
SpeechFormat,
|
||||
SpeechModelOptions,
|
||||
SpeechOptions,
|
||||
SpeechRequestFor,
|
||||
SpeechRequestInput,
|
||||
SpeechRoute,
|
||||
} from "./speech.js"
|
||||
export { Speech } from "./speech.js"
|
||||
export { TranscriptionClient } from "./transcription-client.js"
|
||||
export {
|
||||
TranscriptionEvent,
|
||||
TranscriptionModel,
|
||||
TranscriptionModelSchema,
|
||||
TranscriptionRequest,
|
||||
TranscriptionResponse,
|
||||
TranscriptionSegment,
|
||||
TranscriptionTimestamps,
|
||||
TranscriptionWord,
|
||||
} from "./transcription.js"
|
||||
export type {
|
||||
TranscriptionModelOptions,
|
||||
TranscriptionOptions,
|
||||
TranscriptionRequestFor,
|
||||
TranscriptionRequestInput,
|
||||
TranscriptionRoute,
|
||||
} from "./transcription.js"
|
||||
export { Transcription } from "./transcription.js"
|
||||
export { Media } from "./media.js"
|
||||
export { Generation } from "./generation.js"
|
||||
export type { Event as GenerationEvent, Poll, Route as GenerationRoute, Snapshot as GenerationSnapshot, Status as GenerationStatus } from "./generation.js"
|
||||
export type {
|
||||
AwaitOptions as GenerationAwaitOptions,
|
||||
Event as GenerationEvent,
|
||||
Poll,
|
||||
Route as GenerationRoute,
|
||||
Snapshot as GenerationSnapshot,
|
||||
Status as GenerationStatus,
|
||||
} from "./generation.js"
|
||||
export { Tool, ToolFailure, toDefinitions } from "./tool.js"
|
||||
export { ToolRuntime } from "./tool-runtime.js"
|
||||
export type { DispatchResult as ToolDispatchResult, ToolSettlement } from "./tool-runtime.js"
|
||||
|
||||
+23
-5
@@ -1,5 +1,6 @@
|
||||
import { Effect, JsonSchema, Schema } from "effect"
|
||||
import { LLMClient, Service } from "./route/client.js"
|
||||
import { Effect, JsonSchema, Schema, Stream } from "effect"
|
||||
import { tryRequest } from "./media-model.js"
|
||||
import { LLMClient, Service, type StreamOptions } from "./route/client.js"
|
||||
import {
|
||||
GenerationOptions,
|
||||
HttpOptions,
|
||||
@@ -35,9 +36,26 @@ export type RequestInput<SelectedLanguageModel extends LanguageModel = LanguageM
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
export const generate = LLMClient.generate
|
||||
export function generate<const Model extends LanguageModel>(
|
||||
input: RequestInput<Model>,
|
||||
options?: StreamOptions,
|
||||
): Effect.Effect<LLMResponse, AIError, Service>
|
||||
export function generate(input: LLMRequest, options?: StreamOptions): Effect.Effect<LLMResponse, AIError, Service>
|
||||
export function generate(input: RequestInput | LLMRequest, options?: StreamOptions) {
|
||||
return requestEffect(input).pipe(Effect.flatMap((request) => LLMClient.generate(request, options)))
|
||||
}
|
||||
|
||||
export const stream = LLMClient.stream
|
||||
export function stream<const Model extends LanguageModel>(
|
||||
input: RequestInput<Model>,
|
||||
options?: StreamOptions,
|
||||
): Stream.Stream<LLMEvent, AIError, Service>
|
||||
export function stream(input: LLMRequest, options?: StreamOptions): Stream.Stream<LLMEvent, AIError, Service>
|
||||
export function stream(input: RequestInput | LLMRequest, options?: StreamOptions) {
|
||||
return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => LLMClient.stream(request, options))))
|
||||
}
|
||||
|
||||
const requestEffect = (input: RequestInput | LLMRequest) =>
|
||||
input instanceof LLMRequest ? Effect.succeed(input) : tryRequest(() => request(input))
|
||||
|
||||
export const request = <const SelectedLanguageModel extends LanguageModel>(
|
||||
input: RequestInput<SelectedLanguageModel>,
|
||||
@@ -61,7 +79,7 @@ export const request = <const SelectedLanguageModel extends LanguageModel>(
|
||||
toolChoice: requestToolChoice ? ToolChoice.make(requestToolChoice) : undefined,
|
||||
generation: requestGeneration === undefined ? undefined : GenerationOptions.make(requestGeneration),
|
||||
providerOptions: requestProviderOptions,
|
||||
http: requestHttp === undefined ? undefined : HttpOptions.make(requestHttp),
|
||||
http: HttpOptions.make(requestHttp),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { type Context, Effect, Layer, Stream } from "effect"
|
||||
import { resultEvents, type AwaitOptions, type Generation, type Observation } from "./generation.js"
|
||||
import { RequestExecutor } from "./route/executor.js"
|
||||
import type { MediaRoute } from "./route/media.js"
|
||||
import { AIError, UnsupportedOperationError } from "./schema/index.js"
|
||||
|
||||
/** A media request whose model carries the route that executes it. */
|
||||
export interface RoutedRequest<Self extends MediaRoute.MediaRequest, Event, Response> extends MediaRoute.MediaRequest {
|
||||
readonly model: MediaRoute.MediaRequest["model"] & { readonly route: MediaRoute.AnyRoute<Self, Event, Response> }
|
||||
}
|
||||
|
||||
/** `start` and `resume` fail with `UnsupportedOperation` on inline and stream routes. */
|
||||
export interface Interface<Req extends RoutedRequest<Req, Event, Response>, Event, Response> {
|
||||
readonly generate: (request: Req, options?: AwaitOptions) => Effect.Effect<Response, AIError>
|
||||
readonly stream: (request: Req, options?: AwaitOptions) => Stream.Stream<Event | Observation, AIError>
|
||||
readonly start: (request: Req) => Effect.Effect<Generation<Response>, AIError>
|
||||
readonly resume: (model: Req["model"], token: unknown) => Effect.Effect<Generation<Response>, AIError>
|
||||
}
|
||||
|
||||
/** One modality's layer and service accessors, dispatching each request on its route's `kind`. */
|
||||
export const make = <Self, Req extends RoutedRequest<Req, Event, Response>, Event, Response>(
|
||||
service: Context.Service<Self, Interface<Req, Event, Response>>,
|
||||
input: {
|
||||
readonly modality: string
|
||||
/** A completed response expanded into the streaming event shape. */
|
||||
readonly responseEvents: (response: Response) => ReadonlyArray<Event>
|
||||
},
|
||||
) => ({
|
||||
layer: Layer.effect(
|
||||
service,
|
||||
Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
const notQueued = (route: MediaRoute.AnyRoute<Req, Event, Response>, operation: string) =>
|
||||
new AIError({
|
||||
reason: new UnsupportedOperationError({
|
||||
operation: `${input.modality}.${operation}`,
|
||||
provider: route.provider,
|
||||
route: route.id,
|
||||
message: `${route.provider}/${route.id} is not a queued route; use generate or stream`,
|
||||
}),
|
||||
})
|
||||
const start = (request: Req) => {
|
||||
const route = request.model.route
|
||||
if (route.kind !== "queued") return Effect.fail(notQueued(route, "start"))
|
||||
return route.start(request, executor.execute)
|
||||
}
|
||||
return service.of({
|
||||
start,
|
||||
resume: (model, token) => {
|
||||
if (model.route.kind !== "queued") return Effect.fail(notQueued(model.route, "resume"))
|
||||
return model.route.resume(model, token, executor.execute)
|
||||
},
|
||||
generate: (request, options) => {
|
||||
const route = request.model.route
|
||||
if (route.kind !== "queued") return route.generate(request, executor.execute)
|
||||
return start(request).pipe(Effect.flatMap((generation) => generation.await(options)))
|
||||
},
|
||||
stream: (request, options) => {
|
||||
const route = request.model.route
|
||||
if (route.kind === "stream") return route.stream(request, executor.execute)
|
||||
if (route.kind === "queued")
|
||||
return Stream.unwrap(
|
||||
start(request).pipe(Effect.map((generation) => resultEvents(generation, input.responseEvents, options))),
|
||||
)
|
||||
return Stream.fromIterableEffect(Effect.map(route.generate(request, executor.execute), input.responseEvents))
|
||||
},
|
||||
})
|
||||
}),
|
||||
),
|
||||
generate: (request: Req, options?: AwaitOptions) => service.use((client) => client.generate(request, options)),
|
||||
stream: (request: Req, options?: AwaitOptions) =>
|
||||
Stream.unwrap(service.useSync((client) => client.stream(request, options))),
|
||||
start: (request: Req) => service.use((client) => client.start(request)),
|
||||
resume: (model: Req["model"], token: unknown) => service.use((client) => client.resume(model, token)),
|
||||
})
|
||||
|
||||
export * as MediaClient from "./media-client.js"
|
||||
@@ -0,0 +1,110 @@
|
||||
import { Effect } from "effect"
|
||||
import { Endpoint } from "./route/endpoint.js"
|
||||
import { MediaRoute } from "./route/media.js"
|
||||
import type { MediaProtocol } from "./route/media-protocol.js"
|
||||
import { AIError, HttpOptions, InvalidRequestError, ModelID, ProviderID } from "./schema/index.js"
|
||||
|
||||
/**
|
||||
* What every media model carries: ids, the configured route, and deployment `http` overlays. Modality classes
|
||||
* (`ImageModel`, `VideoModel`, `SpeechModel`, `TranscriptionModel`) extend it with their route type and a nominal
|
||||
* marker so one cannot stand in for the other in requests.
|
||||
*/
|
||||
export class MediaModel<Route, Options> {
|
||||
// As with `LanguageModel`, the route type is erased over `Options`; `fromRoute` and the constructor trust that the
|
||||
// route accepts every request this model's `Options` admit.
|
||||
declare protected readonly _Options: Options
|
||||
readonly id: ModelID
|
||||
readonly provider: ProviderID
|
||||
readonly route: Route
|
||||
readonly http?: HttpOptions
|
||||
|
||||
constructor(input: MediaModel.Input<Route>) {
|
||||
this.id = ModelID.make(input.id)
|
||||
this.provider = ProviderID.make(input.provider)
|
||||
this.route = input.route
|
||||
this.http = input.http
|
||||
}
|
||||
}
|
||||
|
||||
export namespace MediaModel {
|
||||
export type Options = Record<string, unknown>
|
||||
|
||||
export interface Input<Route> {
|
||||
readonly id: string | ModelID
|
||||
readonly provider: string | ProviderID
|
||||
readonly route: Route
|
||||
readonly http?: HttpOptions
|
||||
}
|
||||
|
||||
/** A protocol plus its canonical start path; `ModelInput.baseURL` overrides `baseURL` per deployment. */
|
||||
export interface RouteInput<Request extends MediaRoute.MediaRequest, Protocol> {
|
||||
readonly protocol: Protocol
|
||||
readonly path: Endpoint.EndpointPart<MediaProtocol.Body, Request>
|
||||
readonly baseURL?: string
|
||||
/** Headers the protocol requires on every call, such as a pinned API version; deployment headers win. */
|
||||
readonly headers?: Record<string, string>
|
||||
}
|
||||
|
||||
export type InlineRouteInput<Request extends MediaRoute.MediaRequest, Response> = RouteInput<
|
||||
Request,
|
||||
MediaProtocol.Inline<Request, Response>
|
||||
>
|
||||
|
||||
export type StreamRouteInput<Request extends MediaRoute.MediaRequest, Event, Frame, State> = RouteInput<
|
||||
MediaProtocol.Addressed<Request>,
|
||||
MediaProtocol.Streamed<Request, Event, Frame, State>
|
||||
>
|
||||
|
||||
export type QueuedRouteInput<Request extends MediaRoute.MediaRequest, Response, Token> = RouteInput<
|
||||
Request,
|
||||
MediaProtocol.Queued<Request, Response, Token>
|
||||
>
|
||||
|
||||
export type AnyRouteInput<Request extends MediaRoute.MediaRequest, Event, Response, Frame, State, Token> =
|
||||
| InlineRouteInput<Request, Response>
|
||||
| StreamRouteInput<Request, Event, Frame, State>
|
||||
| QueuedRouteInput<Request, Response, Token>
|
||||
}
|
||||
|
||||
/** Compose a protocol route input with one deployment through `MediaRoute.inline`, `queued`, or `stream`. */
|
||||
export const composeRoute = <Request extends MediaRoute.MediaRequest, Event, Response, Frame, State, Token>(
|
||||
route: MediaModel.AnyRouteInput<Request, Event, Response, Frame, State, Token>,
|
||||
input: MediaRoute.ModelInput,
|
||||
collect: (events: ReadonlyArray<Event>) => Effect.Effect<Response, AIError>,
|
||||
): MediaRoute.AnyRoute<Request, Event, Response> => {
|
||||
if (isStreamInput(route)) return MediaRoute.stream({ ...composition(route, input), collect })
|
||||
if (isQueuedInput(route)) return MediaRoute.queued(composition(route, input))
|
||||
return MediaRoute.inline(composition(route, input))
|
||||
}
|
||||
|
||||
const composition = <Request extends MediaRoute.MediaRequest, Protocol>(
|
||||
route: MediaModel.RouteInput<Request, Protocol>,
|
||||
input: MediaRoute.ModelInput,
|
||||
): MediaRoute.Composition<Request> & { readonly protocol: Protocol } => ({
|
||||
protocol: route.protocol,
|
||||
endpoint: Endpoint.path(route.path, { baseURL: input.baseURL ?? route.baseURL }),
|
||||
auth: input.auth,
|
||||
headers:
|
||||
route.headers === undefined && input.headers === undefined ? undefined : { ...route.headers, ...input.headers },
|
||||
})
|
||||
|
||||
const isStreamInput = <Request extends MediaRoute.MediaRequest, Event, Response, Frame, State, Token>(
|
||||
route: MediaModel.AnyRouteInput<Request, Event, Response, Frame, State, Token>,
|
||||
): route is MediaModel.StreamRouteInput<Request, Event, Frame, State> => route.protocol.kind === "stream"
|
||||
|
||||
const isQueuedInput = <Request extends MediaRoute.MediaRequest, Event, Response, Frame, State, Token>(
|
||||
route: MediaModel.AnyRouteInput<Request, Event, Response, Frame, State, Token>,
|
||||
): route is MediaModel.QueuedRouteInput<Request, Response, Token> => route.protocol.kind === "queued"
|
||||
|
||||
/** Lift a synchronous Schema-class constructor into a typed `InvalidRequest` failure. */
|
||||
export const tryRequest = <A>(make: () => A): Effect.Effect<A, AIError> =>
|
||||
Effect.try({
|
||||
try: make,
|
||||
catch: (error) =>
|
||||
new AIError({
|
||||
reason: new InvalidRequestError({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
cause: error,
|
||||
}),
|
||||
}),
|
||||
})
|
||||
@@ -6,7 +6,7 @@ import { ProviderID } from "./schema/ids.js"
|
||||
import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "./schema/errors.js"
|
||||
import { ProviderMetadata } from "./schema/options.js"
|
||||
import { Service } from "./route/executor-service.js"
|
||||
import { detectMediaType, extensionMediaType } from "./utils/media-type.js"
|
||||
import { detectMediaType, fileMediaType } from "./utils/media-type.js"
|
||||
|
||||
export { detectMediaType } from "./utils/media-type.js"
|
||||
|
||||
@@ -34,8 +34,6 @@ const UrlSource = Schema.Struct({
|
||||
mediaType: Schema.optional(Schema.String),
|
||||
/** Epoch milliseconds after which the provider no longer serves the URL. */
|
||||
expiresAt: Schema.optional(Schema.Number),
|
||||
/** Headers required to fetch the URL, such as provider auth for Veo downloads. */
|
||||
headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
|
||||
})
|
||||
|
||||
/** A provider-side handle: OpenAI `file_id`, Gemini file URI, `gs://`, `runway://`, or a prior generation id. */
|
||||
@@ -55,6 +53,12 @@ export type Source = Schema.Schema.Type<typeof Source>
|
||||
// Kind, Info, Notice
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type AspectRatio = `${number}:${number}`
|
||||
export const AspectRatio = Schema.declare<AspectRatio>(
|
||||
(value): value is AspectRatio => typeof value === "string" && /^\d+(?:\.\d+)?:\d+(?:\.\d+)?$/.test(value),
|
||||
{ title: "Media.AspectRatio" },
|
||||
)
|
||||
|
||||
export const Kind = Schema.Literals(["image", "video", "audio", "document", "other"])
|
||||
export type Kind = Schema.Schema.Type<typeof Kind>
|
||||
|
||||
@@ -110,6 +114,8 @@ export class Asset {
|
||||
/** Epoch milliseconds after which a `url` source stops resolving. */
|
||||
readonly expiresAt?: number
|
||||
readonly providerMetadata?: ProviderMetadata
|
||||
/** Transient download credentials for `url` sources; see `Asset.Input.headers`. */
|
||||
readonly headers?: Record<string, string>
|
||||
|
||||
// Derived payload forms are cached on the instance because every protocol lowering re-reads the same payload. The
|
||||
// cache is check-then-set (concurrent first reads of a `url` source may both download) and is never observable
|
||||
@@ -127,6 +133,7 @@ export class Asset {
|
||||
this.info = input.info
|
||||
this.expiresAt = input.source.type === "url" ? input.source.expiresAt : undefined
|
||||
this.providerMetadata = input.providerMetadata
|
||||
this.headers = input.source.type === "url" ? input.headers : undefined
|
||||
}
|
||||
|
||||
/** Inline payload without effects, for protocols that embed base64 or data URLs directly. */
|
||||
@@ -151,7 +158,7 @@ export class Asset {
|
||||
? Effect.fromResult(Encoding.decodeBase64(source.data)).pipe(
|
||||
Effect.mapError((cause) => invalid(`Media asset contains invalid base64 data`, cause)),
|
||||
)
|
||||
: download(source)
|
||||
: download(source, this.headers)
|
||||
return decoded.pipe(Effect.tap((data) => Effect.sync(() => (this.#bytes = data))))
|
||||
})
|
||||
}
|
||||
@@ -198,6 +205,12 @@ export namespace Asset {
|
||||
readonly source: Source
|
||||
readonly info?: Info
|
||||
readonly providerMetadata?: ProviderMetadata
|
||||
/**
|
||||
* Headers required to download a `url` source, such as the provider API key Veo demands for its file URIs.
|
||||
* They are runtime-only: never part of `source`, `toJSON()`, or `AssetSchema`, so a persisted asset cannot leak
|
||||
* credentials and cannot be downloaded again after a round-trip. Call `materialize()` before persisting.
|
||||
*/
|
||||
readonly headers?: Record<string, string>
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,10 +239,13 @@ export const AssetSchema = AssetEncoded.pipe(
|
||||
}),
|
||||
)
|
||||
|
||||
const download = Effect.fn("Media.download")(function* (source: Extract<Source, { readonly type: "url" }>) {
|
||||
const download = Effect.fn("Media.download")(function* (
|
||||
source: Extract<Source, { readonly type: "url" }>,
|
||||
headers: Record<string, string> | undefined,
|
||||
) {
|
||||
const executor = yield* Service
|
||||
const response = yield* executor.execute(
|
||||
HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(source.headers ?? {})),
|
||||
HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(headers ?? {})),
|
||||
)
|
||||
const buffer = yield* response.arrayBuffer.pipe(
|
||||
Effect.mapError(
|
||||
@@ -264,8 +280,8 @@ export const url = (
|
||||
value: string,
|
||||
options?: AssetOptions & Omit<Extract<Source, { readonly type: "url" }>, "type" | "url">,
|
||||
) => {
|
||||
const { mediaType, expiresAt, headers, ...rest } = options ?? {}
|
||||
return from({ type: "url", url: value, mediaType, expiresAt, headers }, rest)
|
||||
const { mediaType, expiresAt, ...rest } = options ?? {}
|
||||
return from({ type: "url", url: value, mediaType, expiresAt }, rest)
|
||||
}
|
||||
|
||||
export const ref = (provider: string | ProviderID, id: string, mediaType?: string, options?: AssetOptions) =>
|
||||
@@ -293,7 +309,7 @@ export const file = (path: string, options?: AssetOptions): Effect.Effect<Asset,
|
||||
const data = yield* fs
|
||||
.readFile(path)
|
||||
.pipe(Effect.mapError((cause) => invalid(`Failed to read media file ${path}`, cause)))
|
||||
return bytes(data, detectMediaType(data) ?? extensionMediaType(path), options)
|
||||
return bytes(data, fileMediaType(data, path), options)
|
||||
})
|
||||
|
||||
/** Materialize an asset and write its bytes through `FileSystem`. */
|
||||
|
||||
+142
-29
@@ -1,18 +1,30 @@
|
||||
import { Effect, Layer, ManagedRuntime, Stream } from "effect"
|
||||
import { Image, ImageModel, ImageRequest, type ImageRequestInput } from "./image.js"
|
||||
import { ImageClient } from "./image-client.js"
|
||||
import { AIClient } from "./ai-client.js"
|
||||
import type { AwaitOptions, Event, Generation, Snapshot } from "./generation.js"
|
||||
import { Image, type ImageModel, type ImageRequest, type ImageRequestInput } from "./image.js"
|
||||
import { LLM } from "./index.js"
|
||||
import { LLMClient } from "./route/client.js"
|
||||
import { Media } from "./media.js"
|
||||
import { tryRequest } from "./media-model.js"
|
||||
import { RequestExecutor } from "./route/executor.js"
|
||||
import { LanguageModel, LLMRequest } from "./schema/index.js"
|
||||
import { AIError, InvalidRequestError, LanguageModel, LLMRequest } from "./schema/index.js"
|
||||
import type { RequestInput } from "./llm.js"
|
||||
import { Speech, type SpeechModel, type SpeechRequest, type SpeechRequestInput } from "./speech.js"
|
||||
import {
|
||||
Transcription,
|
||||
type TranscriptionModel,
|
||||
type TranscriptionRequest,
|
||||
type TranscriptionRequestInput,
|
||||
} from "./transcription.js"
|
||||
import { fileMediaType } from "./utils/media-type.js"
|
||||
import { Video, type VideoModel, type VideoRequest, type VideoRequestInput } from "./video.js"
|
||||
|
||||
/**
|
||||
* Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM and image clients
|
||||
* over a request executor; every method runs the corresponding Effect API and rethrows `AIError` unchanged.
|
||||
* Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM, image, video, speech,
|
||||
* and transcription clients over a request executor; every method runs the corresponding Effect API and rethrows
|
||||
* `AIError` unchanged. `file` and `write` load `node:fs/promises` on first use, so importing this module does not.
|
||||
*/
|
||||
export interface Options {
|
||||
/** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or middleware here. */
|
||||
/** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or `RequestExecutor.middleware(fn)` here. */
|
||||
readonly layer?: Layer.Layer<RequestExecutor.Service>
|
||||
}
|
||||
|
||||
@@ -20,10 +32,23 @@ export interface RunOptions {
|
||||
readonly signal?: AbortSignal
|
||||
}
|
||||
|
||||
export type Services =
|
||||
| Layer.Success<typeof LLMClient.layer>
|
||||
| Layer.Success<typeof ImageClient.layer>
|
||||
| RequestExecutor.Service
|
||||
export type Services = AIClient.Services
|
||||
|
||||
/**
|
||||
* Promise view of a `Generation`. Its fields are a snapshot taken when the handle was created; `refresh()` resolves to a
|
||||
* new handle rather than updating this one.
|
||||
*/
|
||||
export type GenerationHandle<Response> = Snapshot & {
|
||||
/** Serializable JSON; pass it back to `resume` from another process. */
|
||||
readonly token: unknown
|
||||
readonly await: (options?: AwaitOptions & RunOptions) => Promise<Response>
|
||||
/** Status observations until the first terminal one, polling like `await`; abort ends iteration without throwing. */
|
||||
readonly events: (options?: AwaitOptions & RunOptions) => AsyncIterable<Event>
|
||||
/** The result without polling; fails when the generation has not completed. */
|
||||
readonly result: (options?: RunOptions) => Promise<Response>
|
||||
readonly refresh: (options?: RunOptions) => Promise<GenerationHandle<Response>>
|
||||
readonly cancel: (options?: RunOptions) => Promise<void>
|
||||
}
|
||||
|
||||
const abortEffect = (signal: AbortSignal | undefined) =>
|
||||
signal === undefined
|
||||
@@ -39,13 +64,9 @@ const abortEffect = (signal: AbortSignal | undefined) =>
|
||||
})
|
||||
|
||||
export const make = (options: Options = {}) => {
|
||||
const runtime = ManagedRuntime.make(
|
||||
Layer.mergeAll(LLMClient.layer, ImageClient.layer).pipe(
|
||||
Layer.provideMerge(options.layer ?? RequestExecutor.fetchLayer),
|
||||
),
|
||||
)
|
||||
const runtime = ManagedRuntime.make(AIClient.layerWith(options.layer ?? RequestExecutor.fetchLayer))
|
||||
|
||||
/** Run any package Effect (for example `asset.bytes()`) inside this runtime. */
|
||||
/** Run any package Effect (for example `LLMClient.compact(...)`) inside this runtime. */
|
||||
const run = <A, E>(effect: Effect.Effect<A, E, Services>, options?: RunOptions) =>
|
||||
runtime.runPromise(effect, { signal: options?.signal })
|
||||
|
||||
@@ -61,32 +82,121 @@ export const make = (options: Options = {}) => {
|
||||
),
|
||||
)
|
||||
|
||||
// The typed `generate`/`stream` overloads take a concrete input or a request, not the union; normalize once here.
|
||||
const llmRequest = (input: RequestInput | LLMRequest) => (input instanceof LLMRequest ? input : LLM.request(input))
|
||||
const imageRequest = (input: ImageRequestInput | ImageRequest) =>
|
||||
input instanceof ImageRequest ? input : Image.request(input)
|
||||
const handle = <Response>(generation: Generation<Response>): GenerationHandle<Response> => ({
|
||||
...generation.snapshot,
|
||||
token: generation.token,
|
||||
await: (options) => run(generation.await({ poll: options?.poll }), options),
|
||||
events: (options) => iterate(generation.events({ poll: options?.poll }), options),
|
||||
result: (options) => run(generation.result(), options),
|
||||
refresh: (options) => run(generation.refresh(), options).then(handle),
|
||||
cancel: (options) => run(generation.cancel(), options),
|
||||
})
|
||||
|
||||
const llmRequest = (input: RequestInput | LLMRequest) =>
|
||||
input instanceof LLMRequest ? Effect.succeed(input) : tryRequest(() => LLM.request(input))
|
||||
|
||||
return {
|
||||
run,
|
||||
/** Decoded asset bytes, downloading `url` sources through the executor. */
|
||||
bytes: (asset: Media.Asset, options?: RunOptions) => run(asset.bytes(), options),
|
||||
base64: (asset: Media.Asset, options?: RunOptions) => run(asset.base64(), options),
|
||||
/** Pull a `url` asset into owned bytes before the provider URL expires. */
|
||||
materialize: (asset: Media.Asset, options?: RunOptions) => run(asset.materialize(), options),
|
||||
/** Read a file into an asset like `Media.file`: sniffed media type, then the extension's. */
|
||||
file: async (path: string, options?: Media.AssetOptions & RunOptions) => {
|
||||
const { readFile } = await import("node:fs/promises")
|
||||
return run(
|
||||
Effect.tryPromise({
|
||||
try: (signal) => readFile(path, { signal }),
|
||||
catch: (cause) => fileError(`Failed to read media file ${path}`, cause),
|
||||
}).pipe(
|
||||
Effect.map((buffer) => {
|
||||
const data = new Uint8Array(buffer)
|
||||
return Media.bytes(data, fileMediaType(data, path), options)
|
||||
}),
|
||||
),
|
||||
options,
|
||||
)
|
||||
},
|
||||
/** Write an asset's bytes like `Media.write`, downloading `url` sources through the executor. */
|
||||
write: async (asset: Media.Asset, path: string, options?: RunOptions) => {
|
||||
const { writeFile } = await import("node:fs/promises")
|
||||
return run(
|
||||
asset.bytes().pipe(
|
||||
Effect.flatMap((data) =>
|
||||
Effect.tryPromise({
|
||||
try: (signal) => writeFile(path, data, { signal }),
|
||||
catch: (cause) => fileError(`Failed to write media file ${path}`, cause),
|
||||
}),
|
||||
),
|
||||
),
|
||||
options,
|
||||
)
|
||||
},
|
||||
llm: {
|
||||
request: LLM.request,
|
||||
generate: <const Model extends LanguageModel>(
|
||||
input: RequestInput<Model> | LLMRequest,
|
||||
options?: RunOptions,
|
||||
) => run(LLM.generate(llmRequest(input)), options),
|
||||
generate: <const Model extends LanguageModel>(input: RequestInput<Model> | LLMRequest, options?: RunOptions) =>
|
||||
run(Effect.flatMap(llmRequest(input), LLM.generate), options),
|
||||
stream: <const Model extends LanguageModel>(input: RequestInput<Model> | LLMRequest, options?: RunOptions) =>
|
||||
iterate(LLM.stream(llmRequest(input)), options),
|
||||
iterate(Stream.unwrap(Effect.map(llmRequest(input), LLM.stream)), options),
|
||||
},
|
||||
image: {
|
||||
request: Image.request,
|
||||
generate: <const Model extends ImageModel>(
|
||||
input: ImageRequestInput<Model> | ImageRequest,
|
||||
options?: RunOptions,
|
||||
) => run(Image.generate(imageRequest(input)), options),
|
||||
options?: AwaitOptions & RunOptions,
|
||||
) => run(Image.generate(input, { poll: options?.poll }), options),
|
||||
stream: <const Model extends ImageModel>(
|
||||
input: ImageRequestInput<Model> | ImageRequest,
|
||||
options?: AwaitOptions & RunOptions,
|
||||
) => iterate(Image.stream(input, { poll: options?.poll }), options),
|
||||
start: <const Model extends ImageModel>(input: ImageRequestInput<Model> | ImageRequest, options?: RunOptions) =>
|
||||
run(Image.start(input), options).then(handle),
|
||||
resume: (model: ImageModel, token: unknown, options?: RunOptions) =>
|
||||
run(Image.resume(model, token), options).then(handle),
|
||||
},
|
||||
video: {
|
||||
request: Video.request,
|
||||
start: <const Model extends VideoModel>(input: VideoRequestInput<Model> | VideoRequest, options?: RunOptions) =>
|
||||
run(Video.start(input), options).then(handle),
|
||||
generate: <const Model extends VideoModel>(
|
||||
input: VideoRequestInput<Model> | VideoRequest,
|
||||
options?: AwaitOptions & RunOptions,
|
||||
) => run(Video.generate(input, { poll: options?.poll }), options),
|
||||
resume: (model: VideoModel, token: unknown, options?: RunOptions) =>
|
||||
run(Video.resume(model, token), options).then(handle),
|
||||
stream: <const Model extends VideoModel>(
|
||||
input: VideoRequestInput<Model> | VideoRequest,
|
||||
options?: AwaitOptions & RunOptions,
|
||||
) => iterate(Video.stream(input, { poll: options?.poll }), options),
|
||||
},
|
||||
speech: {
|
||||
request: Speech.request,
|
||||
generate: <const Model extends SpeechModel>(
|
||||
input: SpeechRequestInput<Model> | SpeechRequest,
|
||||
options?: RunOptions,
|
||||
) => iterate(Image.stream(imageRequest(input)), options),
|
||||
) => run(Speech.generate(input), options),
|
||||
stream: <const Model extends SpeechModel>(
|
||||
input: SpeechRequestInput<Model> | SpeechRequest,
|
||||
options?: RunOptions,
|
||||
) => iterate(Speech.stream(input), options),
|
||||
},
|
||||
transcription: {
|
||||
request: Transcription.request,
|
||||
generate: <const Model extends TranscriptionModel>(
|
||||
input: TranscriptionRequestInput<Model> | TranscriptionRequest,
|
||||
options?: AwaitOptions & RunOptions,
|
||||
) => run(Transcription.generate(input, { poll: options?.poll }), options),
|
||||
stream: <const Model extends TranscriptionModel>(
|
||||
input: TranscriptionRequestInput<Model> | TranscriptionRequest,
|
||||
options?: AwaitOptions & RunOptions,
|
||||
) => iterate(Transcription.stream(input, { poll: options?.poll }), options),
|
||||
start: <const Model extends TranscriptionModel>(
|
||||
input: TranscriptionRequestInput<Model> | TranscriptionRequest,
|
||||
options?: RunOptions,
|
||||
) => run(Transcription.start(input), options).then(handle),
|
||||
resume: (model: TranscriptionModel, token: unknown, options?: RunOptions) =>
|
||||
run(Transcription.resume(model, token), options).then(handle),
|
||||
},
|
||||
dispose: () => runtime.dispose(),
|
||||
}
|
||||
@@ -94,6 +204,9 @@ export const make = (options: Options = {}) => {
|
||||
|
||||
export type Client = ReturnType<typeof make>
|
||||
|
||||
const fileError = (message: string, cause: unknown) =>
|
||||
new AIError({ reason: new InvalidRequestError({ message, cause }) })
|
||||
|
||||
/** Default client over `RequestExecutor.fetchLayer` for scripts; the runtime builds its layer on first use. */
|
||||
export const ai = make()
|
||||
|
||||
|
||||
@@ -70,7 +70,11 @@ export const protocol = Protocol.make({
|
||||
return {
|
||||
...(yield* OpenAIChat.protocol.body.from(req)),
|
||||
enable_thinking: opts.enableThinking,
|
||||
thinking_budget: opts.thinkingBudget,
|
||||
// Alibaba also rejects an explicit budget that is not below `max_completion_tokens`.
|
||||
thinking_budget:
|
||||
opts.thinkingBudget === undefined
|
||||
? undefined
|
||||
: ProviderShared.fitThinkingBudget(opts.thinkingBudget, req.generation?.maxTokens),
|
||||
preserve_thinking: opts.preserveThinking,
|
||||
clear_thinking: opts.clearThinking,
|
||||
thinking: opts.thinking,
|
||||
|
||||
@@ -26,18 +26,21 @@ export const protocol = Protocol.make({
|
||||
from: Effect.fn("AlibabaMessages.fromRequest")(function* (req) {
|
||||
const opts = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(Options))(req.providerOptions ?? {})
|
||||
// Model Studio accepts enabled thinking without Anthropic's mandatory token budget.
|
||||
const body = yield* AnthropicMessages.protocol.body.from(
|
||||
LLMRequest.update(req, {
|
||||
providerOptions: { ...req.providerOptions, thinking: undefined },
|
||||
}),
|
||||
)
|
||||
const budget = opts.thinking?.budgetTokens ?? opts.thinking?.budget_tokens
|
||||
return {
|
||||
...(yield* AnthropicMessages.protocol.body.from(
|
||||
LLMRequest.update(req, {
|
||||
providerOptions: { ...req.providerOptions, thinking: undefined },
|
||||
}),
|
||||
)),
|
||||
...body,
|
||||
thinking:
|
||||
opts.thinking === undefined
|
||||
? undefined
|
||||
: {
|
||||
type: opts.thinking.type,
|
||||
budget_tokens: opts.thinking.budgetTokens ?? opts.thinking.budget_tokens,
|
||||
budget_tokens:
|
||||
budget === undefined ? undefined : ProviderShared.fitThinkingBudget(budget, body.max_tokens),
|
||||
},
|
||||
}
|
||||
}),
|
||||
|
||||
@@ -18,7 +18,6 @@ import {
|
||||
type CacheHint,
|
||||
type FinishReasonDetails,
|
||||
type FinishReason,
|
||||
type JsonSchema,
|
||||
type MediaPart,
|
||||
type ProviderMetadata,
|
||||
type ProviderOptions,
|
||||
@@ -31,13 +30,13 @@ import { classifyProviderFailure } from "../provider-error.js"
|
||||
import { effortUpdate, resolveEffortUpdates } from "../effort-updates.js"
|
||||
import * as Cache from "./utils/cache.js"
|
||||
import { Lifecycle } from "./utils/lifecycle.js"
|
||||
import { ToolSchemaProjection } from "./utils/tool-schema.js"
|
||||
import { ToolStream } from "./utils/tool-stream.js"
|
||||
|
||||
const ADAPTER = "anthropic-messages"
|
||||
export const DEFAULT_BASE_URL = "https://api.anthropic.com/v1"
|
||||
export const PATH = "/messages"
|
||||
export const DEFAULT_MAX_TOKENS = 32_000
|
||||
const MIN_THINKING_BUDGET = 1_024
|
||||
const DEFAULT_EFFORT = "high"
|
||||
|
||||
const SSE_EVENTS = new Set([
|
||||
@@ -524,10 +523,10 @@ const redactedDataFromMetadata = (metadata: ProviderMetadata | undefined, key: s
|
||||
return typeof provider.redactedData === "string" ? provider.redactedData : undefined
|
||||
}
|
||||
|
||||
const lowerTool = (breakpoints: Cache.Breakpoints, tool: ToolDefinition, inputSchema: JsonSchema): AnthropicTool => ({
|
||||
const lowerTool = (breakpoints: Cache.Breakpoints, tool: ToolDefinition): AnthropicTool => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
input_schema: inputSchema,
|
||||
input_schema: tool.inputSchema,
|
||||
cache_control: cacheControl(breakpoints, tool.cache),
|
||||
})
|
||||
|
||||
@@ -1027,6 +1026,15 @@ const applyThinkingBindingDefault = (model: LLMRequest["model"], thinking: Anthr
|
||||
}
|
||||
}
|
||||
|
||||
// Anthropic also requires an explicit thinking budget below `max_tokens` and at or above its minimum.
|
||||
const fitThinking = (thinking: AnthropicThinking | undefined, maxTokens: number) =>
|
||||
thinking?.type === "enabled"
|
||||
? {
|
||||
...thinking,
|
||||
budget_tokens: ProviderShared.fitThinkingBudget(thinking.budget_tokens, maxTokens, MIN_THINKING_BUDGET),
|
||||
}
|
||||
: thinking
|
||||
|
||||
const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (request: LLMRequest) {
|
||||
const options = yield* decodeOptions(request.providerOptions ?? {})
|
||||
const management = options.contextManagement
|
||||
@@ -1034,22 +1042,12 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
|
||||
const format = outputConfig?.format ?? undefined
|
||||
const updates = resolveEffortUpdates(request, options.effort ?? outputConfig?.effort ?? undefined)
|
||||
const generation = request.generation
|
||||
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
|
||||
// Allocate the 4-breakpoint budget in invalidation order: tools → system →
|
||||
// messages. Tools live highest in the cache hierarchy, so when callers
|
||||
// over-mark we keep their tool hints and shed the message-tail ones first.
|
||||
const breakpoints = Cache.newBreakpoints(ANTHROPIC_BREAKPOINT_CAP)
|
||||
const flattened = ProviderShared.flattenToolRequest(updates.request)
|
||||
const tools =
|
||||
flattened.tools.length === 0
|
||||
? undefined
|
||||
: flattened.tools.map((tool) =>
|
||||
lowerTool(
|
||||
breakpoints,
|
||||
tool,
|
||||
ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility),
|
||||
),
|
||||
)
|
||||
const tools = flattened.tools.length === 0 ? undefined : flattened.tools.map((tool) => lowerTool(breakpoints, tool))
|
||||
// Anthropic rejects tool_choice when tools are absent; "none" is only meaningful with tools present.
|
||||
const toolChoice = tools === undefined || !request.toolChoice ? undefined : yield* lowerToolChoice(request.toolChoice)
|
||||
const systemParts = request.system.filter((part) => part.text.length > 0)
|
||||
@@ -1069,6 +1067,7 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
|
||||
}
|
||||
const output_config =
|
||||
updates.effort === undefined && format === undefined ? undefined : { effort: updates.effort, format }
|
||||
const maxTokens = generation?.maxTokens ?? DEFAULT_MAX_TOKENS
|
||||
const body = {
|
||||
model: request.model.id,
|
||||
system,
|
||||
@@ -1076,12 +1075,12 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
|
||||
tools,
|
||||
tool_choice: toolChoice,
|
||||
stream: true as const,
|
||||
max_tokens: generation?.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||
max_tokens: maxTokens,
|
||||
temperature: generation?.temperature,
|
||||
top_p: generation?.topP,
|
||||
top_k: generation?.topK,
|
||||
stop_sequences: generation?.stop,
|
||||
thinking: applyThinkingBindingDefault(request.model, options.thinking),
|
||||
thinking: applyThinkingBindingDefault(request.model, fitThinking(options.thinking, maxTokens)),
|
||||
output_config,
|
||||
// top-level passthrough per SDK MessageCreateParamsBase:4638,4643,4649,4654,4670
|
||||
cache_control: options.cache_control ?? options.cacheControl,
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import type { Status } from "../generation.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js"
|
||||
import { ProviderShared, optionalNull } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "assemblyai-transcription", name: "AssemblyAI", provider: "assemblyai" })
|
||||
export const DEFAULT_BASE_URL = "https://api.assemblyai.com"
|
||||
export const PATH = "/v2/transcript"
|
||||
export const UPLOAD_PATH = "/v2/upload"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type AssemblyAITranscriptionOptions = {
|
||||
readonly keyterms_prompt?: ReadonlyArray<string>
|
||||
readonly punctuate?: boolean
|
||||
readonly format_text?: boolean
|
||||
readonly disfluencies?: boolean
|
||||
readonly filter_profanity?: boolean
|
||||
readonly temperature?: number
|
||||
readonly speaker_options?: { readonly min_speakers_expected?: number; readonly max_speakers_expected?: number }
|
||||
readonly language_detection_options?: {
|
||||
readonly expected_languages?: ReadonlyArray<string>
|
||||
readonly fallback_language?: string
|
||||
readonly code_switching?: boolean
|
||||
}
|
||||
readonly speech_models?: ReadonlyArray<OpenString<"universal-3-5-pro" | "universal-2">>
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = TranscriptionRequestFor<AssemblyAITranscriptionOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Token and response schemas
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const Token = Schema.Struct({ transcriptID: Schema.String })
|
||||
export type Token = Schema.Schema.Type<typeof Token>
|
||||
|
||||
const Upload = Schema.Struct({ upload_url: Schema.String })
|
||||
|
||||
/** Word and utterance times are milliseconds. */
|
||||
const Transcript = Schema.Struct({
|
||||
id: Schema.String,
|
||||
status: Schema.String,
|
||||
text: optionalNull(Schema.String),
|
||||
words: optionalNull(
|
||||
Schema.Array(
|
||||
Schema.Struct({
|
||||
text: Schema.String,
|
||||
start: Schema.Number,
|
||||
end: Schema.Number,
|
||||
confidence: optionalNull(Schema.Number),
|
||||
speaker: optionalNull(Schema.String),
|
||||
}),
|
||||
),
|
||||
),
|
||||
utterances: optionalNull(
|
||||
Schema.Array(
|
||||
Schema.Struct({
|
||||
text: Schema.String,
|
||||
start: Schema.Number,
|
||||
end: Schema.Number,
|
||||
speaker: optionalNull(Schema.String),
|
||||
}),
|
||||
),
|
||||
),
|
||||
language_code: optionalNull(Schema.String),
|
||||
audio_duration: optionalNull(Schema.Number),
|
||||
speech_model_used: optionalNull(Schema.String),
|
||||
error: optionalNull(Schema.String),
|
||||
})
|
||||
|
||||
const STATUS = {
|
||||
queued: "queued",
|
||||
processing: "running",
|
||||
completed: "completed",
|
||||
error: "failed",
|
||||
} as const satisfies Record<string, Status>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeUpload = route.decodeJson(Upload)
|
||||
|
||||
/** `/v2/transcript` only takes a URL, so inline audio is uploaded to `/v2/upload` first. */
|
||||
const prepare = Effect.fn("AssemblyAITranscription.prepare")(function* (request: Request, send: MediaProtocol.Send) {
|
||||
if (request.audio.source.type !== "bytes" && request.audio.source.type !== "base64") return request
|
||||
const audio = yield* MediaInput.inlineBytes(route.id, request.audio)
|
||||
const uploaded = yield* send(UPLOAD_PATH, MediaProtocol.binary(audio, "application/octet-stream")).pipe(
|
||||
Effect.flatMap(decodeUpload),
|
||||
)
|
||||
return { ...request, audio: Media.url(uploaded.value.upload_url, { mediaType: request.audio.mediaType }) }
|
||||
})
|
||||
|
||||
const fromRequest = Effect.fn("AssemblyAITranscription.fromRequest")(function* (request: Request) {
|
||||
const audio = yield* ProviderShared.mediaReference(request.audio, route.provider, route.name)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
audio_url: audio.value,
|
||||
speech_models: [request.model.id],
|
||||
language_code: request.language,
|
||||
language_detection: request.language === undefined ? true : undefined,
|
||||
prompt: request.prompt,
|
||||
// Turn-level `utterances`, the only segments AssemblyAI returns, and `speakers_expected` require speaker labels.
|
||||
speaker_labels:
|
||||
request.diarize === true || request.timestamps === "segment" || request.speakers !== undefined
|
||||
? true
|
||||
: undefined,
|
||||
speakers_expected: request.speakers,
|
||||
},
|
||||
request.providerOptions,
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeTranscript = route.decodeJson(Transcript)
|
||||
|
||||
const decodeStart = Effect.fn("AssemblyAITranscription.decodeStart")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const output = yield* decodeTranscript(response)
|
||||
const status = yield* MediaProtocol.status(STATUS, output.value.status, output)
|
||||
return { token: { transcriptID: output.value.id }, snapshot: { id: output.value.id, status } }
|
||||
})
|
||||
|
||||
const decodeStatus = Effect.fn("AssemblyAITranscription.decodeStatus")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
const output = yield* decodeTranscript(response)
|
||||
const status = yield* MediaProtocol.status(STATUS, output.value.status, output)
|
||||
return { id: context.token.transcriptID, status }
|
||||
})
|
||||
|
||||
const seconds = (milliseconds: number) => milliseconds / 1000
|
||||
|
||||
const decodeResult = Effect.fn("AssemblyAITranscription.decodeResult")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
const output = yield* decodeTranscript(response)
|
||||
const transcript = output.value
|
||||
const status = yield* MediaProtocol.status(STATUS, transcript.status, output)
|
||||
const error = transcript.error ?? undefined
|
||||
if (status === "failed")
|
||||
return yield* output.ended("failed", `${route.name} transcription failed${error === undefined ? "" : `: ${error}`}`)
|
||||
if (status !== "completed") return yield* output.pending(context.token.transcriptID)
|
||||
const duration = transcript.audio_duration ?? undefined
|
||||
return new TranscriptionResponse({
|
||||
text: transcript.text ?? "",
|
||||
segments: transcript.utterances?.map((utterance) => ({
|
||||
text: utterance.text,
|
||||
startSeconds: seconds(utterance.start),
|
||||
endSeconds: seconds(utterance.end),
|
||||
speaker: utterance.speaker ?? undefined,
|
||||
})),
|
||||
words: transcript.words?.map((word) => ({
|
||||
text: word.text,
|
||||
startSeconds: seconds(word.start),
|
||||
endSeconds: seconds(word.end),
|
||||
speaker: word.speaker ?? undefined,
|
||||
confidence: word.confidence ?? undefined,
|
||||
})),
|
||||
language: transcript.language_code?.toLowerCase(),
|
||||
durationSeconds: duration,
|
||||
usage: duration === undefined ? undefined : { type: "seconds", seconds: duration },
|
||||
providerMetadata: {
|
||||
assemblyai: { transcriptId: transcript.id, speechModel: transcript.speech_model_used ?? undefined },
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const transcriptPath = (token: Token) => `${PATH}/${token.transcriptID}`
|
||||
|
||||
export const protocol = MediaProtocol.queued<Request, TranscriptionResponse, Token>(route, {
|
||||
token: Token,
|
||||
start: { prepare, body: { from: fromRequest }, decode: decodeStart },
|
||||
status: { path: transcriptPath, decode: decodeStatus },
|
||||
result: { path: transcriptPath, decode: decodeResult },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
TranscriptionModel.fromRoute<AssemblyAITranscriptionOptions, Token>(
|
||||
{ protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
|
||||
input,
|
||||
)
|
||||
|
||||
export const AssemblyAITranscription = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -9,9 +9,8 @@ import {
|
||||
type CacheHint,
|
||||
type FinishReason,
|
||||
type FinishReasonDetails,
|
||||
type JsonSchema,
|
||||
type LLMRequest,
|
||||
type LanguageModelToolSchemaCompatibility,
|
||||
type LanguageModel,
|
||||
type ProviderMetadata,
|
||||
type ReasoningPart,
|
||||
type ToolCallPart,
|
||||
@@ -26,8 +25,8 @@ import { BedrockCache } from "./utils/bedrock-cache.js"
|
||||
import { BedrockMedia } from "./utils/bedrock-media.js"
|
||||
import { Lifecycle } from "./utils/lifecycle.js"
|
||||
import { MistralToolID } from "./utils/mistral-tool-id.js"
|
||||
import { ToolSchemaProjection } from "./utils/tool-schema.js"
|
||||
import { ToolStream } from "./utils/tool-stream.js"
|
||||
import { concatBytes } from "../utils/bytes.js"
|
||||
|
||||
const ADAPTER = "bedrock-converse"
|
||||
|
||||
@@ -220,22 +219,18 @@ type BedrockEvent = Schema.Schema.Type<typeof BedrockEvent>
|
||||
// =============================================================================
|
||||
// Request Lowering
|
||||
// =============================================================================
|
||||
const lowerToolSpec = (tool: ToolDefinition, inputSchema: JsonSchema): BedrockToolSpec => ({
|
||||
const lowerToolSpec = (tool: ToolDefinition): BedrockToolSpec => ({
|
||||
toolSpec: {
|
||||
name: tool.name,
|
||||
...(tool.description.trim().length > 0 ? { description: tool.description } : {}),
|
||||
inputSchema: { json: inputSchema },
|
||||
inputSchema: { json: tool.inputSchema },
|
||||
},
|
||||
})
|
||||
|
||||
const lowerTools = (
|
||||
compatibility: LanguageModelToolSchemaCompatibility | undefined,
|
||||
breakpoints: BedrockCache.Breakpoints,
|
||||
tools: ReadonlyArray<ToolDefinition>,
|
||||
): BedrockTool[] => {
|
||||
const lowerTools = (breakpoints: BedrockCache.Breakpoints, tools: ReadonlyArray<ToolDefinition>): BedrockTool[] => {
|
||||
const result: BedrockTool[] = []
|
||||
for (const tool of tools) {
|
||||
result.push(lowerToolSpec(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, compatibility)))
|
||||
result.push(lowerToolSpec(tool))
|
||||
const cachePoint = BedrockCache.block(breakpoints, tool.cache)
|
||||
if (cachePoint) result.push(cachePoint)
|
||||
}
|
||||
@@ -429,17 +424,50 @@ const lowerSystem = (breakpoints: BedrockCache.Breakpoints, system: ReadonlyArra
|
||||
return content.length === 0 ? undefined : content
|
||||
}
|
||||
|
||||
// Nova 2 rejects `maxTokens` at high reasoning effort, where its output can exceed the field's maximum. Other models
|
||||
// that take `reasoningConfig`, such as Grok on Bedrock, accept it.
|
||||
const isNova2 = (model: LanguageModel) => /\bamazon\.nova-2-/.test(model.id)
|
||||
const isHighReasoningEffort = Schema.is(
|
||||
Schema.Struct({
|
||||
additionalModelRequestFields: Schema.Struct({
|
||||
reasoningConfig: Schema.Struct({ maxReasoningEffort: Schema.Literal("high") }),
|
||||
}),
|
||||
}),
|
||||
)
|
||||
|
||||
const Options = Schema.Struct({
|
||||
thinking: Schema.optional(Schema.Struct({ type: Schema.Literal("enabled"), budgetTokens: Schema.Number })),
|
||||
})
|
||||
export type OptionsInput = typeof Options.Type
|
||||
const decodeOptions = ProviderShared.validateWith(Schema.decodeUnknownEffect(Options))
|
||||
// Claude on Bedrock requires the thinking budget below `maxTokens`, with a minimum of 1,024.
|
||||
const MIN_THINKING_BUDGET = 1_024
|
||||
|
||||
const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request: LLMRequest) {
|
||||
const toolChoice = request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined
|
||||
const flattened = ProviderShared.flattenToolRequest(request)
|
||||
const generation = request.generation
|
||||
const options = yield* decodeOptions(request.providerOptions ?? {})
|
||||
const maxTokens =
|
||||
isNova2(request.model) && isHighReasoningEffort(request.http?.body) ? undefined : generation?.maxTokens
|
||||
const thinking =
|
||||
options.thinking === undefined
|
||||
? undefined
|
||||
: {
|
||||
type: "enabled",
|
||||
budget_tokens: ProviderShared.fitThinkingBudget(
|
||||
options.thinking.budgetTokens,
|
||||
maxTokens,
|
||||
MIN_THINKING_BUDGET,
|
||||
),
|
||||
}
|
||||
// Bedrock-Claude shares Anthropic's 4-breakpoint cap. Spend the budget in
|
||||
// tools → system → messages order to favour the highest-impact prefixes.
|
||||
const breakpoints = BedrockCache.breakpoints(request.model.id)
|
||||
const toolConfig = (() => {
|
||||
if (flattened.tools.length === 0) return undefined
|
||||
return {
|
||||
tools: lowerTools(request.model.compatibility?.toolSchema, breakpoints, flattened.tools),
|
||||
tools: lowerTools(breakpoints, flattened.tools),
|
||||
// Converse has no native "none". Keep definitions stable for prompt
|
||||
// caching and omit only the unsupported choice.
|
||||
toolChoice,
|
||||
@@ -454,14 +482,14 @@ const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request:
|
||||
}
|
||||
const inferenceConfig = (() => {
|
||||
if (
|
||||
generation?.maxTokens === undefined &&
|
||||
maxTokens === undefined &&
|
||||
generation?.temperature === undefined &&
|
||||
generation?.topP === undefined &&
|
||||
(generation?.stop === undefined || generation.stop.length === 0)
|
||||
)
|
||||
return undefined
|
||||
return {
|
||||
maxTokens: generation?.maxTokens,
|
||||
maxTokens,
|
||||
temperature: generation?.temperature,
|
||||
topP: generation?.topP,
|
||||
stopSequences: generation?.stop,
|
||||
@@ -473,9 +501,15 @@ const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request:
|
||||
system,
|
||||
inferenceConfig,
|
||||
toolConfig,
|
||||
// Converse's base inferenceConfig has no topK; Anthropic/Nova accept it
|
||||
// as a model-specific field, so it goes through additionalModelRequestFields.
|
||||
additionalModelRequestFields: generation?.topK === undefined ? undefined : { top_k: generation.topK },
|
||||
// Converse's base inferenceConfig has no topK or thinking; Anthropic/Nova accept them
|
||||
// as model-specific fields, so they go through additionalModelRequestFields.
|
||||
additionalModelRequestFields:
|
||||
generation?.topK === undefined && thinking === undefined
|
||||
? undefined
|
||||
: {
|
||||
...(generation?.topK === undefined ? {} : { top_k: generation.topK }),
|
||||
...(thinking === undefined ? {} : { thinking }),
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
@@ -524,14 +558,7 @@ interface ParserState {
|
||||
readonly reasoningRedactedContent: Readonly<Record<number, ReadonlyArray<Uint8Array>>>
|
||||
}
|
||||
|
||||
const encodeRedactedContent = (chunks: ReadonlyArray<Uint8Array>) => {
|
||||
const bytes = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.length, 0))
|
||||
chunks.reduce((offset, chunk) => {
|
||||
bytes.set(chunk, offset)
|
||||
return offset + chunk.length
|
||||
}, 0)
|
||||
return Encoding.encodeBase64(bytes)
|
||||
}
|
||||
const encodeRedactedContent = (chunks: ReadonlyArray<Uint8Array>) => Encoding.encodeBase64(concatBytes(chunks))
|
||||
|
||||
const step = (state: ParserState, event: BedrockEvent) =>
|
||||
Effect.gen(function* () {
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import type { Status } from "../generation.js"
|
||||
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords } from "../schema/index.js"
|
||||
import { ProviderShared, optionalNull } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "bfl-images", name: "Black Forest Labs", provider: "black-forest-labs" })
|
||||
export const DEFAULT_BASE_URL = "https://api.bfl.ai"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type BlackForestLabsImageOptions = {
|
||||
readonly safety_tolerance?: number
|
||||
readonly prompt_upsampling?: boolean
|
||||
readonly disable_pup?: boolean
|
||||
readonly raw?: boolean
|
||||
readonly guidance?: number
|
||||
readonly steps?: number
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = ImageRequestFor<BlackForestLabsImageOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Token and response schemas
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Regional clusters answer on different hosts, so the returned `polling_url` is followed verbatim. BFL reports the
|
||||
* credit cost on submit, so it rides on the token; it is optional so tokens persisted before it existed still decode.
|
||||
*/
|
||||
export const Token = Schema.Struct({
|
||||
id: Schema.String,
|
||||
pollingURL: Schema.String,
|
||||
cost: Schema.optionalKey(Schema.Number),
|
||||
})
|
||||
export type Token = Schema.Schema.Type<typeof Token>
|
||||
|
||||
const StartResponse = Schema.Struct({
|
||||
id: Schema.String,
|
||||
polling_url: Schema.String,
|
||||
cost: optionalNull(Schema.Number),
|
||||
})
|
||||
|
||||
const Result = Schema.Struct({
|
||||
id: Schema.String,
|
||||
status: Schema.String,
|
||||
result: optionalNull(
|
||||
Schema.StructWithRest(
|
||||
Schema.Struct({ sample: Schema.String, seed: optionalNull(Schema.Number), prompt: optionalNull(Schema.String) }),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
),
|
||||
),
|
||||
cost: optionalNull(Schema.Number),
|
||||
})
|
||||
|
||||
const STATUS = {
|
||||
Pending: "running",
|
||||
Reasoning: "running",
|
||||
Generating: "running",
|
||||
Ready: "completed",
|
||||
Error: "failed",
|
||||
// Moderation is terminal; `decodeResult` reports it as a content-policy failure.
|
||||
"Content Moderated": "failed",
|
||||
"Request Moderated": "failed",
|
||||
"Task not found": "expired",
|
||||
} as const satisfies Record<string, Status>
|
||||
|
||||
const isModerated = (status: string) => status === "Content Moderated" || status === "Request Moderated"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface Capabilities {
|
||||
readonly sizing: "dimensions" | "aspectRatio" | "none"
|
||||
/** `input_image` numbers extra references `input_image_2`…; the other fields take one image. */
|
||||
readonly imageField: "input_image" | "image_prompt" | "image"
|
||||
readonly maxImages: number
|
||||
readonly mask: boolean
|
||||
}
|
||||
|
||||
const capabilities = (model: string): Capabilities => {
|
||||
if (model.startsWith("flux-pro-1.0-fill")) return { sizing: "none", imageField: "image", maxImages: 1, mask: true }
|
||||
if (model.startsWith("flux-pro-1.0-expand")) return { sizing: "none", imageField: "image", maxImages: 1, mask: false }
|
||||
if (model.startsWith("flux-kontext"))
|
||||
return { sizing: "aspectRatio", imageField: "input_image", maxImages: 4, mask: false }
|
||||
if (model.startsWith("flux-pro-1.1-ultra"))
|
||||
return { sizing: "aspectRatio", imageField: "image_prompt", maxImages: 1, mask: false }
|
||||
if (model.startsWith("flux-pro-1.1") || model.startsWith("flux-dev"))
|
||||
return { sizing: "dimensions", imageField: "image_prompt", maxImages: 1, mask: false }
|
||||
if (model.startsWith("flux-2-klein"))
|
||||
return { sizing: "dimensions", imageField: "input_image", maxImages: 4, mask: false }
|
||||
return { sizing: "dimensions", imageField: "input_image", maxImages: 8, mask: false }
|
||||
}
|
||||
|
||||
const validate = (request: Request, model: Capabilities) => {
|
||||
const id = request.model.id
|
||||
const images = request.images?.length ?? 0
|
||||
if (request.n !== undefined && request.n > 1)
|
||||
return Effect.fail(route.unsupported("media.n", `${id} generates one image per request; call it once per image`))
|
||||
if (request.size !== undefined && model.sizing !== "dimensions")
|
||||
return Effect.fail(route.unsupported("media.size", `${id} does not take size (width and height)`))
|
||||
if (request.aspectRatio !== undefined && model.sizing !== "aspectRatio")
|
||||
return Effect.fail(route.unsupported("media.aspectRatio", `${id} does not take aspectRatio`))
|
||||
if (images > model.maxImages)
|
||||
return Effect.fail(route.unsupported("media.images", `${id} takes at most ${model.maxImages} images`))
|
||||
if (request.mask !== undefined && !model.mask)
|
||||
return Effect.fail(route.unsupported("media.mask", `${id} does not inpaint; use flux-pro-1.0-fill`))
|
||||
return Effect.void
|
||||
}
|
||||
|
||||
const imageInput = (asset: Media.Asset) => {
|
||||
const value = asset.inline()?.base64 ?? ProviderShared.mediaUrl(asset)
|
||||
if (value === undefined)
|
||||
return Effect.fail(ProviderShared.invalidRequest(`${route.name} accepts inline images or https URLs`))
|
||||
return Effect.succeed(value)
|
||||
}
|
||||
|
||||
const fromRequest = Effect.fn("BlackForestLabsImages.fromRequest")(function* (request: Request) {
|
||||
const model = capabilities(request.model.id)
|
||||
yield* validate(request, model)
|
||||
const images = yield* Effect.forEach(request.images ?? [], imageInput)
|
||||
const fields = images.map((image, index) => [
|
||||
index === 0 ? model.imageField : `${model.imageField}_${index + 1}`,
|
||||
image,
|
||||
])
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
prompt: request.prompt,
|
||||
...(request.size === undefined ? {} : MediaInput.dimensions(request.size)),
|
||||
aspect_ratio: request.aspectRatio,
|
||||
seed: request.seed,
|
||||
output_format: request.format,
|
||||
mask: request.mask === undefined ? undefined : yield* imageInput(request.mask),
|
||||
...Object.fromEntries(fields),
|
||||
},
|
||||
request.providerOptions,
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
|
||||
token: {
|
||||
id: value.id,
|
||||
pollingURL: value.polling_url,
|
||||
...(value.cost === undefined || value.cost === null ? {} : { cost: value.cost }),
|
||||
},
|
||||
snapshot: { id: value.id, status: "queued" },
|
||||
}))
|
||||
|
||||
const decodeDocument = route.decodeJson(Result)
|
||||
|
||||
const decodeStatus = Effect.fn("BlackForestLabsImages.decodeStatus")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
const output = yield* decodeDocument(response)
|
||||
return { id: context.token.id, status: yield* MediaProtocol.status(STATUS, output.value.status, output) }
|
||||
})
|
||||
|
||||
const decodeResult = Effect.fn("BlackForestLabsImages.decodeResult")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
const output = yield* decodeDocument(response)
|
||||
const document = output.value
|
||||
const status = yield* MediaProtocol.status(STATUS, document.status, output)
|
||||
if (isModerated(document.status)) return yield* output.contentPolicy(`${route.name} moderated the generation`)
|
||||
if (status === "failed" || status === "expired")
|
||||
return yield* output.ended(status, `${route.name} generation ${context.token.id} ended with ${document.status}`)
|
||||
if (status !== "completed") return yield* output.pending(context.token.id)
|
||||
if (document.result === undefined || document.result === null)
|
||||
return yield* output.invalid(`${route.name} generation ${context.token.id} has no result`)
|
||||
const { sample, seed, prompt, ...rest } = document.result
|
||||
// A settled `cost` on the result supersedes the submit-time cost carried on the token.
|
||||
const cost = document.cost ?? context.token.cost
|
||||
return new ImageResponse({
|
||||
// `sample` is a signed URL that expires 10 minutes after the result is ready, so it is downloaded now.
|
||||
images: [yield* context.materialize(Media.url(sample))],
|
||||
usage: cost === undefined ? undefined : { type: "credits", credits: cost },
|
||||
providerMetadata: {
|
||||
bfl: { id: context.token.id, seed: seed ?? undefined, prompt: prompt ?? undefined, ...rest },
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.queued<Request, ImageResponse, Token>(route, {
|
||||
token: Token,
|
||||
start: { body: { from: fromRequest }, decode: decodeStart },
|
||||
status: { path: (token) => token.pollingURL, decode: decodeStatus },
|
||||
result: { path: (token) => token.pollingURL, decode: decodeResult },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<BlackForestLabsImageOptions, Token>(
|
||||
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => `/v1/${request.model.id}` },
|
||||
input,
|
||||
)
|
||||
|
||||
export const BlackForestLabsImages = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -0,0 +1,185 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { classifyProviderFailure } from "../provider-error.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { AIError, mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
|
||||
import { ProviderShared, optionalNull } from "./shared.js"
|
||||
import { SpeechStream } from "./utils/speech-stream.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "cartesia-speech", name: "Cartesia", provider: "cartesia" })
|
||||
export const DEFAULT_BASE_URL = "https://api.cartesia.ai"
|
||||
export const API_VERSION = "2026-08-14"
|
||||
export const BYTES_PATH = "/tts/bytes"
|
||||
export const SSE_PATH = "/tts/sse"
|
||||
const DEFAULT_SAMPLE_RATE = 44100
|
||||
const DEFAULT_BIT_RATE = 128000
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type CartesiaEncoding = SpeechStream.PcmEncoding
|
||||
|
||||
export type CartesiaSpeechOptions = {
|
||||
readonly sampleRate?: 8000 | 16000 | 22050 | 24000 | 44100 | 48000
|
||||
readonly bitRate?: 32000 | 64000 | 96000 | 128000 | 192000
|
||||
readonly encoding?: CartesiaEncoding
|
||||
readonly generation_config?: {
|
||||
readonly volume?: number
|
||||
readonly emotion?: OpenString<"neutral" | "calm" | "angry" | "content" | "sad" | "scared">
|
||||
}
|
||||
readonly pronunciation_dict_id?: string
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = SpeechRequestFor<CartesiaSpeechOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3. Streaming event schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** `phoneme_timestamps` and future record types are ignored. */
|
||||
const SseEvent = Schema.Struct({
|
||||
type: Schema.String,
|
||||
data: Schema.optional(Schema.Uint8ArrayFromBase64),
|
||||
word_timestamps: Schema.optional(
|
||||
Schema.Struct({
|
||||
words: Schema.Array(Schema.String),
|
||||
start: Schema.Array(Schema.Number),
|
||||
end: Schema.Array(Schema.Number),
|
||||
}),
|
||||
),
|
||||
status_code: Schema.optional(Schema.Number),
|
||||
title: Schema.optional(Schema.String),
|
||||
message: Schema.optional(Schema.String),
|
||||
error_code: optionalNull(Schema.String),
|
||||
})
|
||||
|
||||
const decodeEvent = route.decodeFrame(SseEvent)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. Parser state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface State extends SpeechStream.Audio {
|
||||
readonly done: boolean
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Timestamps exist only on the SSE endpoint, so a `generate` that asks for them collects an SSE stream. */
|
||||
const usesSse = (request: MediaProtocol.Addressed<Request>) => request.mode === "stream" || request.timestamps === true
|
||||
|
||||
const CONTAINERS: Readonly<Record<string, "raw" | "wav" | "mp3">> = { pcm: "raw", wav: "wav", mp3: "mp3" }
|
||||
|
||||
const outputFormat = Effect.fn("CartesiaSpeech.outputFormat")(function* (request: MediaProtocol.Addressed<Request>) {
|
||||
const sse = usesSse(request)
|
||||
const format = request.format ?? (sse ? "pcm" : "mp3")
|
||||
const container = CONTAINERS[format]
|
||||
if (container === undefined)
|
||||
return yield* route.unsupported(
|
||||
"media.format",
|
||||
`${route.name} supports the pcm, wav, and mp3 formats, not "${format}"`,
|
||||
)
|
||||
if (sse && container !== "raw")
|
||||
return yield* route.unsupported(
|
||||
"media.format",
|
||||
`${route.name} streams and timestamps only raw PCM; request format "pcm" instead of "${format}"`,
|
||||
)
|
||||
const sampleRate = request.providerOptions?.sampleRate ?? DEFAULT_SAMPLE_RATE
|
||||
if (container === "mp3")
|
||||
return { container, sample_rate: sampleRate, bit_rate: request.providerOptions?.bitRate ?? DEFAULT_BIT_RATE }
|
||||
return { container, encoding: request.providerOptions?.encoding ?? "pcm_s16le", sample_rate: sampleRate }
|
||||
})
|
||||
|
||||
const fromRequest = Effect.fn("CartesiaSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
|
||||
const voice = SpeechStream.voiceID(request.voice)
|
||||
if (voice === undefined)
|
||||
return yield* ProviderShared.invalidRequest(`${route.name} requires a voice id; pass it as \`voice\``)
|
||||
const { sampleRate: _sampleRate, bitRate: _bitRate, encoding: _encoding, ...native } = request.providerOptions ?? {}
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
model_id: request.model.id,
|
||||
transcript: request.text,
|
||||
voice,
|
||||
output_format: yield* outputFormat(request),
|
||||
language: request.language,
|
||||
generation_config: request.speed === undefined ? undefined : { speed: request.speed },
|
||||
add_timestamps: request.timestamps === true ? true : undefined,
|
||||
},
|
||||
native,
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Stream parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const onEvent = Effect.fn("CartesiaSpeech.onEvent")(function* (state: State, frame: string) {
|
||||
const event = yield* decodeEvent(frame)
|
||||
if (event.type === "chunk" && event.data !== undefined) return SpeechStream.delta(state, event.data)
|
||||
if (event.type === "timestamps" && event.word_timestamps !== undefined) {
|
||||
const words = event.word_timestamps
|
||||
return [state, SpeechStream.timestamps(words.words, words.start, words.end)] as const
|
||||
}
|
||||
if (event.type === "done") return [{ ...state, done: true }, []] as const
|
||||
if (event.type === "error")
|
||||
return yield* new AIError({
|
||||
reason: classifyProviderFailure({
|
||||
message: `${route.name} stream failed${event.title === undefined ? "" : ` (${event.title})`}: ${event.message ?? "unknown error"}`,
|
||||
status: event.status_code,
|
||||
rawBody: frame,
|
||||
}),
|
||||
})
|
||||
return [state, []] as const
|
||||
})
|
||||
|
||||
const finish = Effect.fn("CartesiaSpeech.finish")(function* (
|
||||
state: State,
|
||||
context: MediaProtocol.ResponseContext<Request>,
|
||||
) {
|
||||
if (usesSse(context.request) && !state.done) return yield* route.incomplete()
|
||||
const format = yield* outputFormat(context.request)
|
||||
return yield* SpeechStream.finish(
|
||||
route,
|
||||
state,
|
||||
format.container === "raw"
|
||||
? SpeechStream.pcm(format.encoding, format.sample_rate)
|
||||
: SpeechStream.container(format.container, format.sample_rate),
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>(route, {
|
||||
unsupported: ["instructions"],
|
||||
body: { from: fromRequest },
|
||||
frames: (bytes, context) => (usesSse(context.request) ? Framing.sse.frame(bytes) : bytes),
|
||||
initial: () => ({ chunks: [], done: false }),
|
||||
step: SpeechStream.step(onEvent),
|
||||
finish,
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
SpeechModel.fromRoute<CartesiaSpeechOptions, string | Uint8Array, State>(
|
||||
{
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
headers: { "Cartesia-Version": API_VERSION },
|
||||
path: ({ request }) => (usesSse(request) ? SSE_PATH : BYTES_PATH),
|
||||
},
|
||||
input,
|
||||
)
|
||||
|
||||
export const CartesiaSpeech = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -0,0 +1,140 @@
|
||||
import { Effect } from "effect"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
import { SpeechStream } from "./utils/speech-stream.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "deepgram-speech", name: "Deepgram", provider: "deepgram" })
|
||||
export const DEFAULT_BASE_URL = "https://api.deepgram.com"
|
||||
export const PATH = "/v1/speak"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type DeepgramEncoding = OpenString<"linear16" | "mulaw" | "alaw" | "mp3" | "opus" | "flac" | "aac">
|
||||
|
||||
export type DeepgramSpeechOptions = {
|
||||
readonly encoding?: DeepgramEncoding
|
||||
readonly container?: OpenString<"wav" | "ogg" | "none">
|
||||
readonly sampleRate?: number
|
||||
readonly bitRate?: number
|
||||
readonly mip_opt_out?: boolean
|
||||
readonly tag?: string
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = SpeechRequestFor<DeepgramSpeechOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. Parser state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type State = SpeechStream.Audio
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const FORMATS: Readonly<Record<string, { readonly encoding: string; readonly container?: string }>> = {
|
||||
mp3: { encoding: "mp3" },
|
||||
wav: { encoding: "linear16", container: "wav" },
|
||||
pcm: { encoding: "linear16", container: "none" },
|
||||
opus: { encoding: "opus" },
|
||||
flac: { encoding: "flac" },
|
||||
aac: { encoding: "aac" },
|
||||
}
|
||||
|
||||
const audioFormat = (request: Request) => {
|
||||
const format = request.format === undefined ? undefined : FORMATS[request.format]
|
||||
return {
|
||||
encoding: request.providerOptions?.encoding ?? format?.encoding,
|
||||
container: request.providerOptions?.container ?? format?.container,
|
||||
}
|
||||
}
|
||||
|
||||
const queryParameters = (request: Request) => {
|
||||
const { encoding: _encoding, container: _container, sampleRate, bitRate, ...native } = request.providerOptions ?? {}
|
||||
return MediaInput.query(route.id, {
|
||||
...native,
|
||||
model: request.model.id,
|
||||
...audioFormat(request),
|
||||
sample_rate: sampleRate,
|
||||
bit_rate: bitRate,
|
||||
speed: request.speed,
|
||||
})
|
||||
}
|
||||
|
||||
const fromRequest = Effect.fn("DeepgramSpeech.fromRequest")(function* (request: Request) {
|
||||
// Not in `unsupported`: that list would also reject `timestamps: false`, which asks for nothing.
|
||||
if (request.timestamps === true)
|
||||
return yield* route.unsupported("media.timestamps", `${route.name} does not return timestamps`)
|
||||
if (
|
||||
request.format !== undefined &&
|
||||
FORMATS[request.format] === undefined &&
|
||||
request.providerOptions?.encoding === undefined
|
||||
)
|
||||
return yield* route.unsupported(
|
||||
"media.format",
|
||||
`${route.name} has no encoding for format "${request.format}"; pass providerOptions.encoding`,
|
||||
)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords({ text: request.text }, request.http?.body) ?? {},
|
||||
yield* queryParameters(request),
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Stream parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const HEADERLESS_ENCODINGS: Readonly<Record<string, SpeechStream.PcmEncoding>> = {
|
||||
linear16: "pcm_s16le",
|
||||
mulaw: "pcm_mulaw",
|
||||
alaw: "pcm_alaw",
|
||||
}
|
||||
|
||||
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
|
||||
const headers = context.http.headers
|
||||
const mediaType = headers["content-type"]
|
||||
const format = audioFormat(context.request)
|
||||
const encoding = HEADERLESS_ENCODINGS[format.encoding ?? ""]
|
||||
const requestID = headers["dg-request-id"]
|
||||
const modelName = headers["dg-model-name"]
|
||||
return SpeechStream.finish(route, state, {
|
||||
...(format.container === "none" && encoding !== undefined
|
||||
? SpeechStream.pcm(encoding, SpeechStream.sampleRate(mediaType), mediaType)
|
||||
: // Deepgram's default encoding is MP3; WAV is a container around any encoding.
|
||||
{ mediaType, info: { format: format.container === "wav" ? "wav" : (format.encoding ?? "mp3") } }),
|
||||
usage: SpeechStream.headerUsage("characters", headers["dg-char-count"]),
|
||||
providerMetadata:
|
||||
requestID === undefined && modelName === undefined
|
||||
? undefined
|
||||
: { deepgram: { requestId: requestID, modelName } },
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.stream<Request, SpeechEvent, Uint8Array, State>(route, {
|
||||
unsupported: ["voice", "language", "instructions"],
|
||||
body: { from: fromRequest },
|
||||
frames: (bytes) => bytes,
|
||||
initial: () => ({ chunks: [] }),
|
||||
step: (state, frame) => Effect.succeed(SpeechStream.delta(state, frame)),
|
||||
finish,
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
SpeechModel.fromRoute<DeepgramSpeechOptions, Uint8Array, State>(
|
||||
{ protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
|
||||
input,
|
||||
)
|
||||
|
||||
export const DeepgramSpeech = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -0,0 +1,186 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js"
|
||||
import { ProviderShared } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "deepgram-transcription", name: "Deepgram", provider: "deepgram" })
|
||||
export const DEFAULT_BASE_URL = "https://api.deepgram.com"
|
||||
export const PATH = "/v1/listen"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type DeepgramTranscriptionOptions = {
|
||||
readonly smart_format?: boolean
|
||||
readonly punctuate?: boolean
|
||||
readonly paragraphs?: boolean
|
||||
readonly utterances?: boolean
|
||||
readonly detect_language?: boolean | ReadonlyArray<string>
|
||||
readonly keyterm?: ReadonlyArray<string>
|
||||
readonly diarize_model?: OpenString<"latest" | "v1" | "v2">
|
||||
readonly filler_words?: boolean
|
||||
readonly numerals?: boolean
|
||||
readonly mip_opt_out?: boolean
|
||||
readonly tag?: string | ReadonlyArray<string>
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = TranscriptionRequestFor<DeepgramTranscriptionOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Response schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const Word = Schema.Struct({
|
||||
word: Schema.String,
|
||||
start: Schema.Number,
|
||||
end: Schema.Number,
|
||||
confidence: Schema.optional(Schema.Number),
|
||||
speaker: Schema.optional(Schema.Number),
|
||||
punctuated_word: Schema.optional(Schema.String),
|
||||
})
|
||||
|
||||
const ListenResponse = Schema.Struct({
|
||||
metadata: Schema.optional(
|
||||
Schema.Struct({ request_id: Schema.optional(Schema.String), duration: Schema.optional(Schema.Number) }),
|
||||
),
|
||||
results: Schema.Struct({
|
||||
channels: Schema.Array(
|
||||
Schema.Struct({
|
||||
alternatives: Schema.optional(
|
||||
Schema.Array(Schema.Struct({ transcript: Schema.String, words: Schema.optional(Schema.Array(Word)) })),
|
||||
),
|
||||
detected_language: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
utterances: Schema.optional(
|
||||
Schema.Array(
|
||||
Schema.Struct({
|
||||
start: Schema.Number,
|
||||
end: Schema.Number,
|
||||
transcript: Schema.String,
|
||||
speaker: Schema.optional(Schema.Number),
|
||||
words: Schema.optional(Schema.Array(Word)),
|
||||
}),
|
||||
),
|
||||
),
|
||||
}),
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const query = (request: Request) =>
|
||||
MediaInput.query(
|
||||
route.id,
|
||||
mergeJsonRecords(
|
||||
{
|
||||
model: request.model.id,
|
||||
smart_format: true,
|
||||
language: request.language,
|
||||
// Deepgram assumes English unless asked to detect, unlike the other routes' auto-detection.
|
||||
detect_language: request.language === undefined ? true : undefined,
|
||||
// `diarize=true` is deprecated in favor of choosing a diarization model.
|
||||
diarize_model: request.diarize === true ? "latest" : undefined,
|
||||
utterances: request.diarize === true || request.timestamps === "segment" ? true : undefined,
|
||||
},
|
||||
request.providerOptions,
|
||||
) ?? {},
|
||||
)
|
||||
|
||||
const fromRequest = Effect.fn("DeepgramTranscription.fromRequest")(function* (request: Request) {
|
||||
const url = ProviderShared.mediaUrl(request.audio)
|
||||
if (url !== undefined)
|
||||
return MediaProtocol.json(mergeJsonRecords({ url }, request.http?.body) ?? {}, yield* query(request))
|
||||
if (request.http?.body !== undefined)
|
||||
return yield* ProviderShared.invalidRequest(
|
||||
`${route.name} sends inline audio as the raw body, so http.body cannot apply`,
|
||||
)
|
||||
const audio = yield* MediaInput.inlineBytes(route.id, request.audio)
|
||||
return MediaProtocol.binary(audio, request.audio.mediaType, yield* query(request))
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeListen = route.decodeJson(ListenResponse)
|
||||
|
||||
const speaker = (value: number | undefined) => (value === undefined ? undefined : String(value))
|
||||
|
||||
const wordText = (word: typeof Word.Type) => word.punctuated_word ?? word.word
|
||||
|
||||
// Utterances split on pauses, not speakers: the v2 diarizer labels a whole utterance with one speaker even when its
|
||||
// words change speaker, so segments split each utterance at speaker changes.
|
||||
const speakerTurns = (words: ReadonlyArray<typeof Word.Type>) =>
|
||||
words.reduce<Array<Array<typeof Word.Type>>>((turns, word) => {
|
||||
const last = turns.at(-1)
|
||||
if (last === undefined || last[0].speaker !== word.speaker) return [...turns, [word]]
|
||||
last.push(word)
|
||||
return turns
|
||||
}, [])
|
||||
|
||||
const decodeResponse = Effect.fn("DeepgramTranscription.decodeResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const output = yield* decodeListen(response)
|
||||
const channel = output.value.results.channels[0]
|
||||
const alternative = channel?.alternatives?.[0]
|
||||
if (alternative === undefined) return yield* output.invalid(`${route.name} returned no transcript`)
|
||||
const duration = output.value.metadata?.duration
|
||||
const requestID = output.value.metadata?.request_id
|
||||
return new TranscriptionResponse({
|
||||
text: alternative.transcript,
|
||||
segments: output.value.results.utterances?.flatMap((utterance) =>
|
||||
utterance.words === undefined || utterance.words.length === 0
|
||||
? [
|
||||
{
|
||||
text: utterance.transcript,
|
||||
startSeconds: utterance.start,
|
||||
endSeconds: utterance.end,
|
||||
speaker: speaker(utterance.speaker),
|
||||
},
|
||||
]
|
||||
: speakerTurns(utterance.words).map((turn) => ({
|
||||
text: turn.map(wordText).join(" "),
|
||||
startSeconds: turn[0].start,
|
||||
endSeconds: turn[turn.length - 1].end,
|
||||
speaker: speaker(turn[0].speaker),
|
||||
})),
|
||||
),
|
||||
words: alternative.words?.map((word) => ({
|
||||
text: wordText(word),
|
||||
startSeconds: word.start,
|
||||
endSeconds: word.end,
|
||||
speaker: speaker(word.speaker),
|
||||
confidence: word.confidence,
|
||||
})),
|
||||
language: channel?.detected_language?.toLowerCase(),
|
||||
durationSeconds: duration,
|
||||
usage: duration === undefined ? undefined : { type: "seconds", seconds: duration },
|
||||
providerMetadata: requestID === undefined ? undefined : { deepgram: { requestId: requestID } },
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, TranscriptionResponse>(route, {
|
||||
unsupported: ["prompt", "speakers"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
TranscriptionModel.fromRoute<DeepgramTranscriptionOptions>({ protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, input)
|
||||
|
||||
export const DeepgramTranscription = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -0,0 +1,210 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
|
||||
import { ProviderShared, optionalNull } from "./shared.js"
|
||||
import { SpeechStream } from "./utils/speech-stream.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "elevenlabs-speech", name: "ElevenLabs", provider: "elevenlabs" })
|
||||
export const DEFAULT_BASE_URL = "https://api.elevenlabs.io"
|
||||
export const PATH = "/v1/text-to-speech"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type ElevenLabsOutputFormat = OpenString<
|
||||
| "mp3_22050_32"
|
||||
| "mp3_24000_48"
|
||||
| "mp3_44100_32"
|
||||
| "mp3_44100_64"
|
||||
| "mp3_44100_96"
|
||||
| "mp3_44100_128"
|
||||
| "mp3_44100_192"
|
||||
| "pcm_8000"
|
||||
| "pcm_16000"
|
||||
| "pcm_22050"
|
||||
| "pcm_24000"
|
||||
| "pcm_32000"
|
||||
| "pcm_44100"
|
||||
| "pcm_48000"
|
||||
| "wav_8000"
|
||||
| "wav_16000"
|
||||
| "wav_22050"
|
||||
| "wav_24000"
|
||||
| "wav_32000"
|
||||
| "wav_44100"
|
||||
| "wav_48000"
|
||||
| "ulaw_8000"
|
||||
| "alaw_8000"
|
||||
| "opus_48000_32"
|
||||
| "opus_48000_64"
|
||||
| "opus_48000_96"
|
||||
| "opus_48000_128"
|
||||
| "opus_48000_192"
|
||||
>
|
||||
|
||||
export type ElevenLabsSpeechOptions = {
|
||||
readonly outputFormat?: ElevenLabsOutputFormat
|
||||
readonly voice_settings?: {
|
||||
readonly stability?: number
|
||||
readonly similarity_boost?: number
|
||||
readonly style?: number
|
||||
readonly use_speaker_boost?: boolean
|
||||
}
|
||||
readonly seed?: number
|
||||
readonly apply_text_normalization?: OpenString<"auto" | "on" | "off">
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = SpeechRequestFor<ElevenLabsSpeechOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3. Streaming event schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const Alignment = Schema.Struct({
|
||||
characters: Schema.Array(Schema.String),
|
||||
character_start_times_seconds: Schema.Array(Schema.Number),
|
||||
character_end_times_seconds: Schema.Array(Schema.Number),
|
||||
})
|
||||
|
||||
const TimestampedAudio = Schema.Struct({
|
||||
audio_base64: Schema.Uint8ArrayFromBase64,
|
||||
alignment: optionalNull(Alignment),
|
||||
})
|
||||
|
||||
const decodeRecord = route.decodeFrame(TimestampedAudio)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. Parser state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type State = SpeechStream.Audio
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const OUTPUT_FORMATS: Readonly<Record<string, string>> = {
|
||||
mp3: "mp3_44100_128",
|
||||
pcm: "pcm_24000",
|
||||
wav: "wav_24000",
|
||||
opus: "opus_48000_64",
|
||||
}
|
||||
|
||||
/** WAV is served only by the non-streaming endpoints. */
|
||||
const outputFormat = Effect.fn("ElevenLabsSpeech.outputFormat")(function* (request: MediaProtocol.Addressed<Request>) {
|
||||
const format = request.providerOptions?.outputFormat ?? OUTPUT_FORMATS[request.format ?? "mp3"]
|
||||
if (format === undefined)
|
||||
return yield* route.unsupported(
|
||||
"media.format",
|
||||
`${route.name} has no default output format for "${request.format}"; pass providerOptions.outputFormat`,
|
||||
)
|
||||
if (request.mode === "stream" && format.startsWith("wav_"))
|
||||
return yield* route.unsupported(
|
||||
"media.format",
|
||||
`${route.name} streams mp3, pcm, opus, ulaw, and alaw but not "${format}"; use generate for WAV`,
|
||||
)
|
||||
return format
|
||||
})
|
||||
|
||||
const fromRequest = Effect.fn("ElevenLabsSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
|
||||
if (request.voice === undefined)
|
||||
return yield* ProviderShared.invalidRequest(`${route.name} requires a voice id; pass it as \`voice\``)
|
||||
const { outputFormat: _outputFormat, ...native } = request.providerOptions ?? {}
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
text: request.text,
|
||||
model_id: request.model.id,
|
||||
language_code: request.language,
|
||||
voice_settings: request.speed === undefined ? undefined : { speed: request.speed },
|
||||
},
|
||||
native,
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
{ output_format: yield* outputFormat(request) },
|
||||
)
|
||||
})
|
||||
|
||||
const path = (request: MediaProtocol.Addressed<Request>) =>
|
||||
`${PATH}/${encodeURIComponent(SpeechStream.voiceID(request.voice) ?? "")}${request.mode === "stream" ? "/stream" : ""}${
|
||||
request.timestamps === true ? "/with-timestamps" : ""
|
||||
}`
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Stream parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const onRecord = Effect.fn("ElevenLabsSpeech.onRecord")(function* (state: State, frame: string) {
|
||||
const record = yield* decodeRecord(frame)
|
||||
const [next, events] = SpeechStream.delta(state, record.audio_base64)
|
||||
const alignment = record.alignment
|
||||
if (!alignment) return [next, events] as const
|
||||
return [
|
||||
next,
|
||||
[
|
||||
...events,
|
||||
...SpeechStream.timestamps(
|
||||
alignment.characters,
|
||||
alignment.character_start_times_seconds,
|
||||
alignment.character_end_times_seconds,
|
||||
),
|
||||
],
|
||||
] as const
|
||||
})
|
||||
|
||||
const PCM_CODECS: Readonly<Record<string, SpeechStream.PcmEncoding>> = {
|
||||
pcm: "pcm_s16le",
|
||||
ulaw: "pcm_mulaw",
|
||||
alaw: "pcm_alaw",
|
||||
}
|
||||
|
||||
const describeOutput = (format: string) => {
|
||||
const [codec = format, rate] = format.split("_")
|
||||
const sampleRate = rate === undefined ? undefined : Number(rate)
|
||||
const encoding = PCM_CODECS[codec]
|
||||
return encoding === undefined ? SpeechStream.container(codec, sampleRate) : SpeechStream.pcm(encoding, sampleRate)
|
||||
}
|
||||
|
||||
const finish = Effect.fn("ElevenLabsSpeech.finish")(function* (
|
||||
state: State,
|
||||
context: MediaProtocol.ResponseContext<Request>,
|
||||
) {
|
||||
const requestID = context.http.headers["request-id"]
|
||||
return yield* SpeechStream.finish(route, state, {
|
||||
...describeOutput(yield* outputFormat(context.request)),
|
||||
// `character-cost` is billed credits, not a character count (3 for 20 characters on `eleven_flash_v2_5`).
|
||||
usage: SpeechStream.headerUsage("credits", context.http.headers["character-cost"]),
|
||||
providerMetadata: requestID === undefined ? undefined : { elevenlabs: { requestId: requestID } },
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>(route, {
|
||||
unsupported: ["instructions"],
|
||||
body: { from: fromRequest },
|
||||
frames: (bytes, context) => {
|
||||
if (context.request.timestamps !== true) return bytes
|
||||
return context.request.mode === "stream" ? Framing.lines.frame(bytes) : Framing.document.frame(bytes)
|
||||
},
|
||||
initial: () => ({ chunks: [] }),
|
||||
step: SpeechStream.step(onRecord),
|
||||
finish,
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
SpeechModel.fromRoute<ElevenLabsSpeechOptions, string | Uint8Array, State>(
|
||||
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => path(request) },
|
||||
input,
|
||||
)
|
||||
|
||||
export const ElevenLabsSpeech = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -0,0 +1,156 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { ProviderShared, optionalNull } from "./shared.js"
|
||||
import { FalQueue } from "./utils/fal-queue.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "fal-images", name: "fal Images", provider: "fal" })
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type FalImageOptions = {
|
||||
readonly image_size?: OpenString<
|
||||
"square_hd" | "square" | "portrait_4_3" | "portrait_16_9" | "landscape_4_3" | "landscape_16_9"
|
||||
>
|
||||
readonly enable_safety_checker?: boolean
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = ImageRequestFor<FalImageOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Response schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const QueueResult = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
images: Schema.Array(
|
||||
Schema.Struct({
|
||||
url: Schema.String,
|
||||
width: optionalNull(Schema.Number),
|
||||
height: optionalNull(Schema.Number),
|
||||
content_type: optionalNull(Schema.String),
|
||||
}),
|
||||
),
|
||||
seed: optionalNull(Schema.Number),
|
||||
has_nsfw_concepts: optionalNull(Schema.Array(Schema.Boolean)),
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const sizing = (model: string) => {
|
||||
if (/^fal-ai\/(nano-banana|flux-pro\/(v1\.1-ultra|kontext))/.test(model)) return "aspect_ratio"
|
||||
if (model.startsWith("fal-ai/flux")) return "image_size"
|
||||
return undefined
|
||||
}
|
||||
|
||||
const validate = (request: Request) => {
|
||||
const id = request.model.id
|
||||
const field = sizing(id)
|
||||
if (request.size !== undefined && request.aspectRatio !== undefined)
|
||||
return Effect.fail(ProviderShared.invalidRequest(`${route.name} accepts either size or aspectRatio, not both`))
|
||||
if (request.size !== undefined && field === "aspect_ratio")
|
||||
return Effect.fail(route.unsupported("media.size", `${id} sizes by aspectRatio`))
|
||||
if (request.aspectRatio !== undefined && field === "image_size")
|
||||
return Effect.fail(route.unsupported("media.aspectRatio", `${id} sizes by size (image_size)`))
|
||||
if ((request.images?.length ?? 0) > 1 && !takesImageList(id))
|
||||
return Effect.fail(
|
||||
route.unsupported(
|
||||
"media.images",
|
||||
`${id} takes one image_url; use an /edit or /multi endpoint for several images`,
|
||||
),
|
||||
)
|
||||
return Effect.void
|
||||
}
|
||||
|
||||
// `/edit` and `/multi` (Kontext) endpoints take an `image_urls` list; image-to-image, fill, and Ultra take one
|
||||
// `image_url` (beside `mask_url`).
|
||||
const takesImageList = (model: string) => model.endsWith("/edit") || model.endsWith("/multi")
|
||||
|
||||
const fromRequest = Effect.fn("FalImages.fromRequest")(function* (request: Request) {
|
||||
yield* validate(request)
|
||||
const images = yield* Effect.forEach(request.images ?? [], (image) => FalQueue.mediaUrl(image, route.name))
|
||||
const list = takesImageList(request.model.id)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
prompt: request.prompt,
|
||||
num_images: request.n,
|
||||
seed: request.seed,
|
||||
image_size: request.size === undefined ? undefined : MediaInput.dimensions(request.size),
|
||||
aspect_ratio: request.aspectRatio,
|
||||
output_format: request.format,
|
||||
image_urls: list && images.length > 0 ? images : undefined,
|
||||
image_url: list ? undefined : images[0],
|
||||
mask_url: request.mask === undefined ? undefined : yield* FalQueue.mediaUrl(request.mask, route.name),
|
||||
},
|
||||
request.providerOptions,
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeQueueResult = route.decodeJson(QueueResult)
|
||||
|
||||
const decodeResult = Effect.fn("FalImages.decodeResult")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<FalQueue.Token>,
|
||||
) {
|
||||
const output = yield* decodeQueueResult(response)
|
||||
const { images, seed, has_nsfw_concepts, ...rest } = output.value
|
||||
if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`)
|
||||
// With the safety checker on, flagged images come back blacked out rather than omitted.
|
||||
const flagged = (has_nsfw_concepts ?? []).flatMap((value, index) => (value ? [index] : []))
|
||||
return new ImageResponse({
|
||||
images: images.map((image) => {
|
||||
const info = { width: image.width ?? undefined, height: image.height ?? undefined }
|
||||
// `sync_mode: true` returns data URIs instead of hosted URLs.
|
||||
return (
|
||||
Media.parseDataUrl(image.url, { info }) ??
|
||||
Media.url(image.url, { mediaType: image.content_type ?? undefined, info })
|
||||
)
|
||||
}),
|
||||
notices:
|
||||
flagged.length === 0
|
||||
? undefined
|
||||
: flagged.map((index) => ({
|
||||
type: "moderated" as const,
|
||||
message: `${route.name} flagged image ${index} as NSFW`,
|
||||
})),
|
||||
providerMetadata: { fal: { requestId: context.token.requestID, seed: seed ?? undefined, ...rest } },
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = FalQueue.protocol<Request, ImageResponse>(route, {
|
||||
from: fromRequest,
|
||||
decodeResult,
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<FalImageOptions, FalQueue.Token>(
|
||||
{ protocol, baseURL: FalQueue.DEFAULT_BASE_URL, path: ({ request }) => `/${request.model.id}` },
|
||||
input,
|
||||
)
|
||||
|
||||
export const FalImages = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -0,0 +1,121 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
|
||||
import { optionalNull } from "./shared.js"
|
||||
import { FalQueue } from "./utils/fal-queue.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "fal-video", name: "fal Video", provider: "fal" })
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Provider-native input. fal video endpoints are model-specific: `duration` is a string enum whose values differ per
|
||||
* model (`"8s"` for Veo, `"5"` for Kling), and last-frame fields are named per model (`end_image_url`,
|
||||
* `last_frame_url`, `tail_image_url`), so those pass through here instead of lowering from common fields.
|
||||
*/
|
||||
export type FalVideoOptions = {
|
||||
readonly duration?: OpenString<"4s" | "6s" | "8s" | "5" | "10">
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = VideoRequestFor<FalVideoOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Response schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const QueueResult = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
video: Schema.Struct({
|
||||
url: Schema.String,
|
||||
content_type: optionalNull(Schema.String),
|
||||
file_name: optionalNull(Schema.String),
|
||||
file_size: optionalNull(Schema.Number),
|
||||
}),
|
||||
seed: optionalNull(Schema.Number),
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const fromRequest = Effect.fn("FalVideo.fromRequest")(function* (request: Request) {
|
||||
if (request.frames?.last !== undefined)
|
||||
return yield* route.unsupported(
|
||||
"video.frames.last",
|
||||
`${route.name} names the last frame per model; pass it through providerOptions (e.g. end_image_url) instead of frames.last`,
|
||||
)
|
||||
const imageUrl =
|
||||
request.frames?.first === undefined ? undefined : yield* FalQueue.mediaUrl(request.frames.first, route.name)
|
||||
const videoUrl = request.video === undefined ? undefined : yield* FalQueue.mediaUrl(request.video, route.name)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
prompt: request.prompt,
|
||||
negative_prompt: request.negativePrompt,
|
||||
seed: request.seed,
|
||||
aspect_ratio: request.aspectRatio,
|
||||
resolution: request.resolution,
|
||||
generate_audio: request.audio,
|
||||
image_url: imageUrl,
|
||||
video_url: videoUrl,
|
||||
},
|
||||
request.providerOptions,
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeQueueResult = route.decodeJson(QueueResult)
|
||||
|
||||
const decodeResult = Effect.fn("FalVideo.decodeResult")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<FalQueue.Token>,
|
||||
) {
|
||||
const output = yield* decodeQueueResult(response)
|
||||
const { video, seed, ...rest } = output.value
|
||||
return new VideoResponse({
|
||||
videos: [Media.url(video.url, { mediaType: video.content_type ?? "video/mp4" })],
|
||||
providerMetadata: {
|
||||
fal: {
|
||||
requestId: context.token.requestID,
|
||||
seed: seed ?? undefined,
|
||||
fileName: video.file_name ?? undefined,
|
||||
fileSize: video.file_size ?? undefined,
|
||||
...rest,
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = FalQueue.protocol<Request, VideoResponse>(route, {
|
||||
unsupported: ["n", "durationSeconds", "references"],
|
||||
from: fromRequest,
|
||||
decodeResult,
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
VideoModel.fromRoute<FalVideoOptions, FalQueue.Token>(
|
||||
{ protocol, baseURL: FalQueue.DEFAULT_BASE_URL, path: ({ request }) => `/${request.model.id}` },
|
||||
input,
|
||||
)
|
||||
|
||||
export const FalVideo = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -10,7 +10,6 @@ import {
|
||||
LLMEvent,
|
||||
Usage,
|
||||
type FinishReason,
|
||||
type JsonSchema,
|
||||
type LLMRequest,
|
||||
type MediaPart,
|
||||
type ProviderMetadata,
|
||||
@@ -22,13 +21,14 @@ import {
|
||||
import { classifyProviderFailure } from "../provider-error.js"
|
||||
import { Media } from "../media.js"
|
||||
import { JsonObject, knownString, lenient, optionalArray, optionalNull, ProviderShared } from "./shared.js"
|
||||
import { GeminiToolSchema } from "./utils/gemini-tool-schema.js"
|
||||
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
|
||||
import { Lifecycle } from "./utils/lifecycle.js"
|
||||
import { ToolSchemaProjection } from "./utils/tool-schema.js"
|
||||
|
||||
const ADAPTER = "gemini"
|
||||
// Google documents this sentinel for replaying Gemini 3 function calls after their original signature was lost.
|
||||
const SKIP_THOUGHT_SIGNATURE_VALIDATOR = "skip_thought_signature_validator"
|
||||
// Gemini 2.5 rejects a budget under the model's minimum: 512 on Flash-Lite, the highest, and 128 on Pro.
|
||||
const MIN_THINKING_BUDGET = 512
|
||||
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
||||
|
||||
// Gemini 3 rejects replayed function calls without a thought signature. Google's SDKs avoid that in normal chats by
|
||||
@@ -132,7 +132,7 @@ const GeminiSystemInstruction = Schema.Struct({
|
||||
const GeminiFunctionDeclaration = Schema.Struct({
|
||||
name: Schema.String,
|
||||
description: Schema.String,
|
||||
parameters: Schema.optional(JsonObject),
|
||||
parametersJsonSchema: JsonObject,
|
||||
})
|
||||
|
||||
const GeminiTool = Schema.Struct({
|
||||
@@ -265,36 +265,14 @@ interface ParserState {
|
||||
readonly seenCallIds?: ReadonlySet<string>
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Tool Schema Conversion
|
||||
// =============================================================================
|
||||
// Tool-schema conversion has two distinct concerns:
|
||||
//
|
||||
// 1. Sanitize — fix common authoring mistakes Gemini rejects: integer/number
|
||||
// enums (must be strings), `required` entries that don't match a property,
|
||||
// untyped arrays (`items` must be present), and `properties`/`required`
|
||||
// keys on non-object scalars. Mirrors OpenCode's historical Gemini rules.
|
||||
//
|
||||
// 2. Project — lossy mapping from JSON Schema to Gemini's schema dialect:
|
||||
// drop empty root parameter schemas while preserving nested empty objects,
|
||||
// expand type arrays into `anyOf`, derive `nullable: true` from null members,
|
||||
// coerce `const` to `[const]` enum, recurse properties/items, and propagate
|
||||
// only an allowlisted set of keys (description, required, format, type,
|
||||
// nullable, enum, properties, items, allOf, anyOf, oneOf, minLength).
|
||||
// Anything outside the allowlist (e.g. `additionalProperties`, `$ref`) is
|
||||
// silently dropped.
|
||||
//
|
||||
// Sanitize runs first, then project. The implementation lives in
|
||||
// `utils/gemini-tool-schema` so this protocol keeps the same shape as the other
|
||||
// provider protocols.
|
||||
|
||||
// =============================================================================
|
||||
// Request Lowering
|
||||
// =============================================================================
|
||||
const lowerTool = (tool: ToolDefinition, inputSchema: JsonSchema) => ({
|
||||
// Tool schemas go in `parametersJsonSchema`, which accepts standard JSON Schema.
|
||||
const lowerTool = (tool: ToolDefinition) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: GeminiToolSchema.convert(inputSchema),
|
||||
parametersJsonSchema: tool.inputSchema,
|
||||
})
|
||||
|
||||
const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
|
||||
@@ -305,14 +283,9 @@ const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
|
||||
tool: (name) => ({ functionCallingConfig: { mode: "ANY" as const, allowedFunctionNames: [name] } }),
|
||||
})
|
||||
|
||||
// Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs.
|
||||
const lowerContentPart = Effect.fn("Gemini.lowerContentPart")(function* (part: TextPart | MediaPart) {
|
||||
if (part.type === "text") return { text: part.text }
|
||||
const source = part.media.source
|
||||
if (source.type === "ref" && source.provider === "google")
|
||||
return { fileData: { mimeType: part.media.mediaType, fileUri: source.id } }
|
||||
const media = yield* ProviderShared.requireInlineMedia("Gemini", part.media)
|
||||
return { inlineData: { mimeType: media.mime, data: media.base64 } }
|
||||
return yield* GeminiGenerateContent.mediaPart("Gemini", part.media)
|
||||
})
|
||||
|
||||
const providerMetadata = (key: string, metadata: Record<string, unknown>): ProviderMetadata => ({ [key]: metadata })
|
||||
@@ -469,7 +442,6 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
|
||||
const hasTools = flattened.tools.length > 0
|
||||
const generation = request.generation
|
||||
const options = yield* decodeOptions(request.providerOptions ?? {})
|
||||
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
|
||||
const generationConfig = {
|
||||
maxOutputTokens: generation?.maxTokens,
|
||||
temperature: generation?.temperature,
|
||||
@@ -479,10 +451,22 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
|
||||
presencePenalty: generation?.presencePenalty,
|
||||
seed: generation?.seed,
|
||||
stopSequences: generation?.stop,
|
||||
// Gemini accepts a budget above `maxOutputTokens`, but thinking then leaves the answer empty.
|
||||
thinkingConfig:
|
||||
options.thinkingConfig === undefined
|
||||
? undefined
|
||||
: { ...options.thinkingConfig, includeThoughts: options.thinkingConfig.includeThoughts ?? true },
|
||||
: {
|
||||
...options.thinkingConfig,
|
||||
includeThoughts: options.thinkingConfig.includeThoughts ?? true,
|
||||
thinkingBudget:
|
||||
options.thinkingConfig.thinkingBudget === undefined
|
||||
? undefined
|
||||
: ProviderShared.fitThinkingBudget(
|
||||
options.thinkingConfig.thinkingBudget,
|
||||
generation?.maxTokens,
|
||||
MIN_THINKING_BUDGET,
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -495,9 +479,7 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
|
||||
tools: hasTools
|
||||
? [
|
||||
{
|
||||
functionDeclarations: flattened.tools.map((tool) =>
|
||||
lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility)),
|
||||
),
|
||||
functionDeclarations: flattened.tools.map(lowerTool),
|
||||
},
|
||||
]
|
||||
: undefined,
|
||||
@@ -833,6 +815,8 @@ export const protocol = Protocol.make({
|
||||
schema: GeminiBody,
|
||||
from: fromRequest,
|
||||
},
|
||||
// Gemini's schema rules are this API's default, including for tuned endpoints whose IDs do not name Gemini.
|
||||
sanitizer: "gemini",
|
||||
stream: {
|
||||
event: Protocol.jsonEvent(GeminiEvent),
|
||||
initial: (request) => ({
|
||||
|
||||
@@ -1,28 +1,24 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { ProviderShared } from "./shared.js"
|
||||
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
const ADAPTER = "google-images"
|
||||
const NAME = "Google Images"
|
||||
const PROVIDER = ProviderID.make("google")
|
||||
const route = MediaProtocol.identity({ id: "google-images", name: "Google Images", provider: "google" })
|
||||
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type GoogleImageString<Known extends string> = Known | (string & {})
|
||||
|
||||
/** Provider-native options. Common fields (`aspectRatio`, `seed`, `images`) live on the request. */
|
||||
export type GoogleImageOptions = {
|
||||
readonly imageSize?: GoogleImageString<"1K" | "2K" | "4K">
|
||||
readonly thinkingLevel?: GoogleImageString<"MINIMAL" | "LOW" | "MEDIUM" | "HIGH">
|
||||
readonly imageSize?: OpenString<"1K" | "2K" | "4K">
|
||||
readonly thinkingLevel?: OpenString<"MINIMAL" | "LOW" | "MEDIUM" | "HIGH">
|
||||
readonly includeThoughts?: boolean
|
||||
} & Record<string, unknown>
|
||||
|
||||
@@ -102,34 +98,15 @@ const generationConfig = (request: Request) => {
|
||||
)
|
||||
}
|
||||
|
||||
// Gemini does not fetch public URLs; inline payloads or Gemini Files references are the only accepted inputs.
|
||||
const imagePart = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
|
||||
const inline = asset.inline()
|
||||
if (inline) return Effect.succeed({ inlineData: { mimeType: inline.mime, data: inline.base64 } })
|
||||
const id = MediaInput.refID(asset, PROVIDER)
|
||||
if (id) return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri: id } })
|
||||
if (asset.source.type === "ref")
|
||||
return Effect.fail(
|
||||
ProviderShared.invalidRequest(
|
||||
"Google generateContent requires Gemini file references rather than other providers' file IDs",
|
||||
),
|
||||
)
|
||||
return Effect.fail(
|
||||
ProviderShared.invalidRequest(
|
||||
"Google generateContent does not fetch public image URLs; use bytes, a data URL, or a Gemini file reference",
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request: Request) {
|
||||
if (request.n !== undefined && request.n > 1)
|
||||
return yield* ProviderShared.unsupportedOperation({
|
||||
operation: "image.n",
|
||||
provider: PROVIDER,
|
||||
route: ADAPTER,
|
||||
message: `${NAME} generates one image per request; call it once per image instead of n=${request.n}`,
|
||||
})
|
||||
const parts = yield* Effect.forEach(request.images ?? [], imagePart)
|
||||
return yield* route.unsupported(
|
||||
"media.n",
|
||||
`${route.name} generates one image per request; call it once per image instead of n=${request.n}`,
|
||||
)
|
||||
const parts = yield* Effect.forEach(request.images ?? [], (image) =>
|
||||
GeminiGenerateContent.mediaPart(route.name, image),
|
||||
)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
@@ -145,10 +122,12 @@ const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request: Re
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeDocument = route.decodeJson(GoogleImageResponse)
|
||||
|
||||
const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, GoogleImageResponse)(response)
|
||||
const output = yield* decodeDocument(response)
|
||||
const decoded = output.value
|
||||
const candidates = decoded.candidates ?? []
|
||||
const candidateMetadata = candidates.map((candidate, candidateIndex) => ({
|
||||
@@ -188,7 +167,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
|
||||
const images = yield* Effect.forEach(encoded, (item) =>
|
||||
MediaInput.decodedAsset(
|
||||
output.invalid,
|
||||
`${NAME} candidate ${item.candidateIndex} part ${item.partIndex}`,
|
||||
`${route.name} candidate ${item.candidateIndex} part ${item.partIndex}`,
|
||||
item.inlineData.data,
|
||||
item.inlineData.mimeType,
|
||||
{
|
||||
@@ -211,7 +190,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
|
||||
candidate.finishReason === undefined ? [] : [candidate.finishReason],
|
||||
)
|
||||
return yield* output.invalid(
|
||||
`${NAME} returned no final images${
|
||||
`${route.name} returned no final images${
|
||||
finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})`
|
||||
}; inspect body for prompt feedback and candidate details`,
|
||||
)
|
||||
@@ -223,7 +202,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
|
||||
: [
|
||||
{
|
||||
type: "filtered" as const,
|
||||
message: `${NAME} reported prompt feedback`,
|
||||
message: `${route.name} reported prompt feedback`,
|
||||
providerMetadata: { google: { promptFeedback: decoded.promptFeedback } },
|
||||
},
|
||||
]),
|
||||
@@ -233,7 +212,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
|
||||
: [
|
||||
{
|
||||
type: "filtered" as const,
|
||||
message: `${NAME} candidate ${candidate.index ?? index} finished with ${candidate.finishReason}${
|
||||
message: `${route.name} candidate ${candidate.index ?? index} finished with ${candidate.finishReason}${
|
||||
candidate.finishMessage === undefined ? "" : `: ${candidate.finishMessage}`
|
||||
}`,
|
||||
providerMetadata: {
|
||||
@@ -283,9 +262,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
|
||||
id: ADAPTER,
|
||||
name: NAME,
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
|
||||
unsupported: ["mask", "size", "format"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
@@ -294,8 +271,6 @@ export const protocol = MediaProtocol.inline<Request, ImageResponse>({
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<GoogleImageOptions>(
|
||||
{
|
||||
id: ADAPTER,
|
||||
provider: PROVIDER,
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) => `/models/${request.model.id}:generateContent`,
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords } from "../schema/index.js"
|
||||
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
|
||||
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
|
||||
import { SpeechStream } from "./utils/speech-stream.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "google-speech", name: "Google Speech", provider: "google" })
|
||||
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
||||
const DEFAULT_SAMPLE_RATE = 24000
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Style is directed in the text itself, and `speechConfig.multiSpeakerVoiceConfig` excludes `voice`. */
|
||||
export type GoogleSpeechOptions = {
|
||||
readonly temperature?: number
|
||||
readonly seed?: number
|
||||
readonly speechConfig?: {
|
||||
readonly multiSpeakerVoiceConfig?: {
|
||||
readonly speakerVoiceConfigs: ReadonlyArray<{
|
||||
readonly speaker: string
|
||||
readonly voiceConfig: { readonly prebuiltVoiceConfig: { readonly voiceName: string } }
|
||||
}>
|
||||
}
|
||||
}
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = SpeechRequestFor<GoogleSpeechOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3. Streaming event schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const GenerateContentChunk = GeminiGenerateContent.chunk(
|
||||
Schema.Struct({
|
||||
text: Schema.optional(Schema.String),
|
||||
inlineData: Schema.optional(Schema.Struct({ mimeType: Schema.String, data: Schema.Uint8ArrayFromBase64 })),
|
||||
}),
|
||||
)
|
||||
|
||||
const decodeChunk = route.decodeFrame(GenerateContentChunk)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. Parser state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface State extends SpeechStream.Audio, GeminiGenerateContent.Metadata {
|
||||
readonly mimeType?: string
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
|
||||
// Not in `unsupported`: that list would also reject `timestamps: false`, which asks for nothing.
|
||||
if (request.timestamps === true)
|
||||
return yield* route.unsupported("media.timestamps", `${route.name} does not return timestamps`)
|
||||
if (request.format === "pcm" && request.mode === "generate" && /^gemini-3\.8-.*-tts(?:-|$)/.test(request.model.id))
|
||||
return yield* route.unsupported(
|
||||
"media.format",
|
||||
`${route.name} returns WAV by default for Gemini 3.8 TTS unary requests; omit the format to accept it`,
|
||||
)
|
||||
if (request.format !== undefined && request.format !== "pcm")
|
||||
return yield* route.unsupported(
|
||||
"media.format",
|
||||
`${route.name} only accepts raw PCM as an explicit format; omit it to accept the provider's default output`,
|
||||
)
|
||||
const voiceName = SpeechStream.voiceID(request.voice)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
contents: [{ role: "user", parts: [{ text: request.text }] }],
|
||||
generationConfig: mergeJsonRecords(
|
||||
{
|
||||
responseModalities: ["AUDIO"],
|
||||
speechConfig: {
|
||||
voiceConfig: voiceName === undefined ? undefined : { prebuiltVoiceConfig: { voiceName } },
|
||||
languageCode: request.language,
|
||||
},
|
||||
},
|
||||
request.providerOptions,
|
||||
),
|
||||
},
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Stream parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const step = Effect.fn("GoogleSpeech.step")(function* (state: State, frame: string) {
|
||||
const chunk = yield* decodeChunk(frame)
|
||||
const blocked = GeminiGenerateContent.blocked(route.name, chunk, frame)
|
||||
if (blocked !== undefined) return yield* blocked
|
||||
const audio = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) =>
|
||||
part.inlineData === undefined ? [] : [part.inlineData],
|
||||
)
|
||||
const next: State = { ...GeminiGenerateContent.track(state, chunk), mimeType: state.mimeType ?? audio[0]?.mimeType }
|
||||
return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])] as const
|
||||
})
|
||||
|
||||
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
|
||||
const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE
|
||||
const output =
|
||||
state.mimeType?.split(";")[0]?.toLowerCase() === "audio/wav"
|
||||
? SpeechStream.container("wav", sampleRate)
|
||||
: SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`)
|
||||
if (context.request.format === "pcm" && output.info.format !== "pcm")
|
||||
return Effect.fail(
|
||||
route.frameError(`Google Speech returned ${output.info.format} instead of the requested raw PCM`),
|
||||
)
|
||||
return SpeechStream.finish(route, state, {
|
||||
...output,
|
||||
usage: GeminiGenerateContent.usage(state.usage),
|
||||
providerMetadata: GeminiGenerateContent.providerMetadata(state),
|
||||
detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string, State>(route, {
|
||||
unsupported: ["instructions", "speed"],
|
||||
body: { from: fromRequest },
|
||||
frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode),
|
||||
initial: () => ({ chunks: [] }),
|
||||
step,
|
||||
finish,
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
SpeechModel.fromRoute<GoogleSpeechOptions, string, State>(
|
||||
{
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
// Only `gemini-3.1-flash-tts-preview` and later stream; earlier TTS models reject `streamGenerateContent`.
|
||||
path: ({ request }) => GeminiGenerateContent.path(request.model.id, request.mode),
|
||||
},
|
||||
input,
|
||||
)
|
||||
|
||||
export const GoogleSpeech = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -0,0 +1,203 @@
|
||||
import { Effect, Schema, SchemaGetter } from "effect"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import {
|
||||
TranscriptionFinishEvent,
|
||||
TranscriptionModel,
|
||||
TranscriptionSegmentEvent,
|
||||
TranscriptionTextDeltaEvent,
|
||||
type TranscriptionRequestFor,
|
||||
type TranscriptionSegment,
|
||||
type TranscriptionWord,
|
||||
type TranscriptionEvent,
|
||||
} from "../transcription.js"
|
||||
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "google-transcription", name: "Google Transcription", provider: "google" })
|
||||
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Merged into `generationConfig`. The API rejects `customVocabulary` and `mode: "SMART"` alongside diarization or word
|
||||
* timestamps.
|
||||
*/
|
||||
export type GoogleTranscriptionOptions = {
|
||||
readonly audioTranscriptionConfig?: {
|
||||
readonly mode?: OpenString<"VERBATIM" | "SMART">
|
||||
readonly customVocabulary?: ReadonlyArray<string>
|
||||
readonly languageCodes?: ReadonlyArray<string>
|
||||
}
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = TranscriptionRequestFor<GoogleTranscriptionOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3. Streaming event schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const Seconds = Schema.String.check(Schema.isPattern(/^\d+(\.\d+)?s$/)).pipe(
|
||||
Schema.decodeTo(Schema.Number, {
|
||||
decode: SchemaGetter.transform((value) => Number.parseFloat(value)),
|
||||
encode: SchemaGetter.transform((value) => `${value}s`),
|
||||
}),
|
||||
)
|
||||
|
||||
const AudioTranscription = Schema.Struct({
|
||||
text: Schema.String,
|
||||
speakerLabel: Schema.optional(Schema.String),
|
||||
words: Schema.optional(
|
||||
Schema.Array(
|
||||
Schema.Struct({
|
||||
word: Schema.String,
|
||||
startOffset: Seconds,
|
||||
endOffset: Seconds,
|
||||
speakerLabel: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
),
|
||||
})
|
||||
|
||||
const decodeChunk = route.decodeFrame(
|
||||
GeminiGenerateContent.chunk(Schema.Struct({ audioTranscription: Schema.optional(AudioTranscription) })),
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. Parser state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface State extends GeminiGenerateContent.Metadata {
|
||||
readonly text: string
|
||||
readonly segments: Array<TranscriptionSegment>
|
||||
readonly words: Array<TranscriptionWord>
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const fromRequest = Effect.fn("GoogleTranscription.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
|
||||
// General Gemini models ignore `audioTranscriptionConfig` and answer the audio conversationally.
|
||||
if (!request.model.id.includes("transcribe"))
|
||||
return yield* route.unsupported(
|
||||
"transcription.model",
|
||||
`${request.model.id} is not a transcription model; use a transcribe model such as gemini-3.5-transcribe`,
|
||||
)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
contents: [{ role: "user", parts: [yield* GeminiGenerateContent.mediaPart(route.id, request.audio)] }],
|
||||
generationConfig: mergeJsonRecords(
|
||||
{
|
||||
audioTranscriptionConfig: {
|
||||
languageCodes: request.language === undefined ? undefined : [request.language],
|
||||
// Parts carry no offsets of their own, so segment times come from word offsets.
|
||||
wordTimestamp:
|
||||
request.diarize === true || request.timestamps === "word" || request.timestamps === "segment"
|
||||
? true
|
||||
: undefined,
|
||||
diarization: request.diarize === true ? true : undefined,
|
||||
},
|
||||
},
|
||||
request.providerOptions,
|
||||
),
|
||||
},
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Stream parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const turn = (part: Schema.Schema.Type<typeof AudioTranscription>) => {
|
||||
const words = (part.words ?? []).map((word) => ({
|
||||
text: word.word,
|
||||
startSeconds: word.startOffset,
|
||||
endSeconds: word.endOffset,
|
||||
speaker: word.speakerLabel ?? part.speakerLabel,
|
||||
}))
|
||||
const first = words[0]
|
||||
const last = words.at(-1)
|
||||
return {
|
||||
text: part.text,
|
||||
words,
|
||||
segment:
|
||||
first === undefined || last === undefined
|
||||
? undefined
|
||||
: {
|
||||
text: part.text,
|
||||
startSeconds: first.startSeconds,
|
||||
endSeconds: last.endSeconds,
|
||||
speaker: part.speakerLabel,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const step = Effect.fn("GoogleTranscription.step")(function* (state: State, frame: string) {
|
||||
const chunk = yield* decodeChunk(frame)
|
||||
const blocked = GeminiGenerateContent.blocked(route.name, chunk, frame)
|
||||
if (blocked !== undefined) return yield* blocked
|
||||
const turns = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) =>
|
||||
part.audioTranscription === undefined ? [] : [turn(part.audioTranscription)],
|
||||
)
|
||||
const segments = turns.flatMap((item) => (item.segment === undefined ? [] : [item.segment]))
|
||||
state.words.push(...turns.flatMap((item) => item.words))
|
||||
state.segments.push(...segments)
|
||||
// Each part is one whole speaker turn without surrounding whitespace, so turns join with a space.
|
||||
const text = turns
|
||||
.map((item) => item.text)
|
||||
.filter((item) => item.length > 0)
|
||||
.join(" ")
|
||||
const delta = text.length === 0 || state.text.length === 0 ? text : ` ${text}`
|
||||
const events: ReadonlyArray<TranscriptionEvent> = [
|
||||
...(delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta })]),
|
||||
...segments.map((segment) => TranscriptionSegmentEvent.make({ segment })),
|
||||
]
|
||||
return [{ ...GeminiGenerateContent.track(state, chunk), text: state.text + delta }, events] as const
|
||||
})
|
||||
|
||||
const finish = (state: State) => {
|
||||
if (state.finishReason === undefined) return Effect.fail(route.incomplete())
|
||||
return Effect.succeed([
|
||||
TranscriptionFinishEvent.make({
|
||||
text: state.text,
|
||||
segments: state.segments.length === 0 ? undefined : state.segments,
|
||||
words: state.words.length === 0 ? undefined : state.words,
|
||||
usage: GeminiGenerateContent.usage(state.usage),
|
||||
providerMetadata: GeminiGenerateContent.providerMetadata(state),
|
||||
}),
|
||||
])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, string, State>(route, {
|
||||
unsupported: ["prompt", "speakers"],
|
||||
body: { from: fromRequest },
|
||||
frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode),
|
||||
initial: () => ({ text: "", segments: [], words: [] }),
|
||||
step,
|
||||
finish,
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
TranscriptionModel.fromRoute<GoogleTranscriptionOptions, string, State>(
|
||||
{
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) => GeminiGenerateContent.path(request.model.id, request.mode),
|
||||
},
|
||||
input,
|
||||
)
|
||||
|
||||
export const GoogleTranscription = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -0,0 +1,220 @@
|
||||
import { Duration, Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import type { Status } from "../generation.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
|
||||
import { ProviderShared, optionalArray } from "./shared.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "google-video", name: "Google Veo", provider: "google" })
|
||||
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
||||
/** Veo keeps generated files for two days; the asset carries that deadline so callers materialize in time. */
|
||||
const FILE_RETENTION = Duration.days(2)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Provider-native `parameters`. Common fields (`aspectRatio`, `resolution`, `durationSeconds`, `seed`) live on the request. */
|
||||
export type GoogleVideoOptions = {
|
||||
readonly personGeneration?: OpenString<"allow_all" | "allow_adult" | "dont_allow">
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = VideoRequestFor<GoogleVideoOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Token and response schemas
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** The long-running operation name, e.g. `models/veo-3.1-generate-preview/operations/abc123`. */
|
||||
export const Token = Schema.Struct({ operation: Schema.String })
|
||||
export type Token = Schema.Schema.Type<typeof Token>
|
||||
|
||||
const StartResponse = Schema.Struct({ name: Schema.String })
|
||||
|
||||
const Operation = Schema.Struct({
|
||||
done: Schema.optional(Schema.Boolean),
|
||||
error: Schema.optional(Schema.Struct({ message: Schema.optional(Schema.String) })),
|
||||
response: Schema.optional(
|
||||
Schema.Struct({
|
||||
generateVideoResponse: Schema.optional(
|
||||
Schema.Struct({
|
||||
generatedSamples: optionalArray(
|
||||
Schema.Struct({
|
||||
video: Schema.optional(
|
||||
Schema.Struct({
|
||||
uri: Schema.optional(Schema.String),
|
||||
mimeType: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
),
|
||||
raiMediaFilteredCount: Schema.optional(Schema.Number),
|
||||
raiMediaFilteredReasons: optionalArray(Schema.String),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
),
|
||||
metadata: Schema.optional(Schema.Unknown),
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Veo takes inline media only; a prior Veo output is `Media.url` with transient auth, so materialize it first.
|
||||
const inlineMedia = (asset: Media.Asset) =>
|
||||
ProviderShared.requireInlineMedia(route.name, asset).pipe(
|
||||
Effect.map((inline) => ({ inlineData: { mimeType: inline.mime, data: inline.base64 } })),
|
||||
)
|
||||
|
||||
const fromRequest = Effect.fn("GoogleVideo.fromRequest")(function* (request: Request) {
|
||||
if (request.n !== undefined && request.n > 1)
|
||||
return yield* route.unsupported(
|
||||
"video.n",
|
||||
`${route.name} generates one video per request; call it once per video instead of n=${request.n}`,
|
||||
)
|
||||
if (request.audio === false)
|
||||
return yield* route.unsupported(
|
||||
"video.audio",
|
||||
`${route.name} always generates audio; audio: false cannot be honored`,
|
||||
)
|
||||
if (request.frames?.last !== undefined && request.frames.first === undefined)
|
||||
return yield* ProviderShared.invalidRequest(`${route.name} requires frames.first when frames.last is set`)
|
||||
const image = request.frames?.first === undefined ? undefined : yield* inlineMedia(request.frames.first)
|
||||
const lastFrame = request.frames?.last === undefined ? undefined : yield* inlineMedia(request.frames.last)
|
||||
const video = request.video === undefined ? undefined : yield* inlineMedia(request.video)
|
||||
const referenceImages = yield* Effect.forEach(request.references ?? [], (asset) =>
|
||||
inlineMedia(asset).pipe(Effect.map((image) => ({ image, referenceType: "asset" }))),
|
||||
)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
instances: [
|
||||
{
|
||||
prompt: request.prompt,
|
||||
image,
|
||||
lastFrame,
|
||||
referenceImages: referenceImages.length === 0 ? undefined : referenceImages,
|
||||
video,
|
||||
},
|
||||
],
|
||||
parameters: mergeJsonRecords(
|
||||
{
|
||||
aspectRatio: request.aspectRatio,
|
||||
resolution: request.resolution,
|
||||
durationSeconds: request.durationSeconds,
|
||||
negativePrompt: request.negativePrompt,
|
||||
seed: request.seed,
|
||||
},
|
||||
request.providerOptions,
|
||||
),
|
||||
},
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
|
||||
token: { operation: value.name },
|
||||
snapshot: { id: value.name, status: "running" },
|
||||
}))
|
||||
|
||||
// Operations carry no status string: not done is running, done with `error` failed, otherwise completed.
|
||||
const statusOf = (operation: typeof Operation.Type): Status => {
|
||||
if (operation.done !== true) return "running"
|
||||
return operation.error === undefined ? "completed" : "failed"
|
||||
}
|
||||
|
||||
const decodeOperation = route.decodeJson(Operation)
|
||||
|
||||
const decodeStatus = Effect.fn("GoogleVideo.decodeStatus")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
const output = yield* decodeOperation(response)
|
||||
return { id: context.token.operation, status: statusOf(output.value) }
|
||||
})
|
||||
|
||||
const decodeResult = Effect.fn("GoogleVideo.decodeResult")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
const output = yield* decodeOperation(response)
|
||||
const operation = output.value
|
||||
const status = statusOf(operation)
|
||||
if (status === "running") return yield* output.pending(context.token.operation)
|
||||
if (status === "failed")
|
||||
return yield* output.ended(
|
||||
"failed",
|
||||
`${route.name} operation failed${operation.error?.message === undefined ? "" : `: ${operation.error.message}`}`,
|
||||
)
|
||||
const generated = operation.response?.generateVideoResponse
|
||||
// Downloads require the same API key as the poll; the asset carries it transiently and follows the redirect.
|
||||
const videos = yield* Effect.forEach(
|
||||
(generated?.generatedSamples ?? []).flatMap((sample) =>
|
||||
sample.video?.uri === undefined ? [] : [{ uri: sample.video.uri, mimeType: sample.video.mimeType }],
|
||||
),
|
||||
(video) =>
|
||||
MediaProtocol.expiringUrl(video.uri, FILE_RETENTION, {
|
||||
mediaType: video.mimeType ?? "video/mp4",
|
||||
headers: context.auth,
|
||||
}),
|
||||
)
|
||||
const reasons = generated?.raiMediaFilteredReasons ?? []
|
||||
const notices = reasons.map((reason) => ({
|
||||
type: "filtered" as const,
|
||||
message: `${route.name} filtered media: ${reason}`,
|
||||
providerMetadata: { google: { raiMediaFilteredReason: reason } },
|
||||
}))
|
||||
if (videos.length === 0 && (reasons.length > 0 || (generated?.raiMediaFilteredCount ?? 0) > 0))
|
||||
return yield* output.contentPolicy(
|
||||
`${route.name} filtered every video${reasons.length === 0 ? "" : `: ${reasons.join("; ")}`}`,
|
||||
)
|
||||
if (videos.length === 0) return yield* output.invalid(`${route.name} operation completed without any video`)
|
||||
return new VideoResponse({
|
||||
videos,
|
||||
notices: notices.length === 0 ? undefined : notices,
|
||||
providerMetadata: {
|
||||
google: {
|
||||
operation: context.token.operation,
|
||||
raiMediaFilteredCount: generated?.raiMediaFilteredCount,
|
||||
metadata: operation.metadata,
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const operationPath = (token: Token) => `/${token.operation}`
|
||||
|
||||
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>(route, {
|
||||
token: Token,
|
||||
start: { body: { from: fromRequest }, decode: decodeStart },
|
||||
status: { path: operationPath, decode: decodeStatus },
|
||||
result: { path: operationPath, decode: decodeResult },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
VideoModel.fromRoute<GoogleVideoOptions, Token>(
|
||||
{
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) => `/models/${request.model.id}:predictLongRunning`,
|
||||
},
|
||||
input,
|
||||
)
|
||||
|
||||
export const GoogleVideo = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -4,20 +4,17 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { JsonObject, ProviderShared, optionalNull } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
const ADAPTER = "meta-images"
|
||||
const NAME = "Meta Images"
|
||||
const PROVIDER = ProviderID.make("meta")
|
||||
const route = MediaProtocol.identity({ id: "meta-images", name: "Meta Images", provider: "meta" })
|
||||
export const DEFAULT_BASE_URL = "https://api.meta.ai/v1"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type OpenString<Known extends string> = Known | (string & {})
|
||||
|
||||
/** Provider-native options. Common fields (`n`, `size`, `format`, `images`) live on the request. */
|
||||
export type ImageOptions = {
|
||||
readonly responseFormat?: OpenString<"b64_json" | "url">
|
||||
@@ -70,13 +67,9 @@ const Response = Schema.Struct({
|
||||
|
||||
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
|
||||
|
||||
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
|
||||
const inline = asset.inline()
|
||||
if (inline) return Effect.succeed({ image_url: inline.dataUrl })
|
||||
const url = ProviderShared.mediaUrl(asset)
|
||||
if (url) return Effect.succeed({ image_url: url })
|
||||
return Effect.fail(ProviderShared.invalidRequest(`${NAME} accepts image bytes and URLs`))
|
||||
}
|
||||
// Meta has no file handles: refs are rejected even when they name this provider.
|
||||
const reference = (asset: Media.Asset) =>
|
||||
ProviderShared.mediaReference(asset, undefined, route.name).pipe(Effect.map((item) => ({ image_url: item.value })))
|
||||
|
||||
const fromRequest = Effect.fn("MetaImages.fromRequest")(function* (request: Request) {
|
||||
const images = yield* Effect.forEach(request.images ?? [], reference)
|
||||
@@ -105,24 +98,23 @@ const fromRequest = Effect.fn("MetaImages.fromRequest")(function* (request: Requ
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeDocument = route.decodeJson(Response)
|
||||
|
||||
const decodeResponse = Effect.fn("MetaImages.decodeResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.DecodeContext<Request>,
|
||||
) {
|
||||
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, Response)(response)
|
||||
const output = yield* decodeDocument(response)
|
||||
const decoded = output.value
|
||||
const requested = context.body.type === "json" ? context.body.value.output_format : undefined
|
||||
const format = decoded.output_format ?? (typeof requested === "string" ? requested : "webp")
|
||||
const mediaType = `image/${format}`
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) => {
|
||||
if (item.b64_json)
|
||||
return MediaInput.decodedAsset(output.invalid, `${NAME} result ${index}`, item.b64_json, mediaType, {
|
||||
info: { format },
|
||||
})
|
||||
if (item.url) return Effect.succeed(Media.url(item.url, { mediaType, info: { format } }))
|
||||
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
|
||||
})
|
||||
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) =>
|
||||
MediaInput.imageOutput(output.invalid, `${route.name} result ${index}`, item, mediaType, {
|
||||
info: { format },
|
||||
}),
|
||||
)
|
||||
if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`)
|
||||
return new ImageResponse({
|
||||
images,
|
||||
usage:
|
||||
@@ -143,20 +135,17 @@ const decodeResponse = Effect.fn("MetaImages.decodeResponse")(function* (
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
|
||||
id: ADAPTER,
|
||||
name: NAME,
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
|
||||
unsupported: ["mask", "aspectRatio", "seed"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput & { readonly baseURL: string }) =>
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<ImageOptions>(
|
||||
{
|
||||
id: ADAPTER,
|
||||
provider: PROVIDER,
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) => `/images/${isEdit(request) ? "edits" : "generations"}`,
|
||||
},
|
||||
input,
|
||||
|
||||
@@ -5,8 +5,7 @@ import { LLMEvent, LLMRequest, Message, ToolResultPart } from "../schema/index.j
|
||||
import { OpenResponses } from "./open-responses.js"
|
||||
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
|
||||
import { ResponsesHostedTools } from "./utils/responses-hosted-tools.js"
|
||||
import { ToolSchemaProjection } from "./utils/tool-schema.js"
|
||||
import { MetaImage } from "./utils/meta-image.js"
|
||||
import { detectMediaType } from "../utils/media-type.js"
|
||||
|
||||
const ADAPTER = "meta-responses"
|
||||
const NAME = "Meta Responses"
|
||||
@@ -103,12 +102,7 @@ const fromRequest = Effect.fn("MetaResponses.fromRequest")(function* (request: L
|
||||
? undefined
|
||||
: yield* Effect.forEach(projected.tools, (tool) =>
|
||||
Effect.gen(function* () {
|
||||
if (tool.native === undefined)
|
||||
return yield* OpenResponses.lowerTool(
|
||||
NAME,
|
||||
tool,
|
||||
ToolSchemaProjection.modelCompatibility(tool.inputSchema, request.model.compatibility?.toolSchema),
|
||||
)
|
||||
if (tool.native === undefined) return yield* OpenResponses.lowerTool(NAME, tool)
|
||||
return yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(NativeTool))(tool.native.meta)
|
||||
}),
|
||||
),
|
||||
@@ -151,7 +145,11 @@ const HOSTED_TOOLS = {
|
||||
),
|
||||
),
|
||||
)
|
||||
const mime = MetaImage.mediaType(data, item.output_format)
|
||||
// Responses image items can omit output_format, including when PNG/JPEG was requested.
|
||||
const mime =
|
||||
item.output_format === undefined
|
||||
? (detectMediaType(data) ?? "application/octet-stream")
|
||||
: `image/${item.output_format}`
|
||||
return {
|
||||
type: "content" as const,
|
||||
value: [{ type: "file" as const, uri: `data:${mime};base64,${item.result}`, mime }],
|
||||
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
ProviderInternalError,
|
||||
Usage,
|
||||
type FinishReason,
|
||||
type JsonSchema,
|
||||
type LLMRequest,
|
||||
type MediaPart,
|
||||
type ProviderMetadata,
|
||||
@@ -24,7 +23,6 @@ import { classifyProviderFailure } from "../provider-error.js"
|
||||
import { effortUpdate } from "../effort-updates.js"
|
||||
import { OpenResponsesOptions } from "./utils/open-responses-options.js"
|
||||
import { Lifecycle } from "./utils/lifecycle.js"
|
||||
import { ToolSchemaProjection } from "./utils/tool-schema.js"
|
||||
import { ToolStream } from "./utils/tool-stream.js"
|
||||
|
||||
const ADAPTER = "open-responses"
|
||||
@@ -443,23 +441,24 @@ interface ReasoningStreamItem {
|
||||
// =============================================================================
|
||||
// Request Lowering
|
||||
// =============================================================================
|
||||
export const lowerTool = Effect.fn("OpenResponses.lowerTool")(function* (
|
||||
protocolName: string,
|
||||
tool: ToolDefinition,
|
||||
inputSchema: JsonSchema,
|
||||
) {
|
||||
export const lowerTool = Effect.fn("OpenResponses.lowerTool")(function* (protocolName: string, tool: ToolDefinition) {
|
||||
if (tool.native !== undefined)
|
||||
return yield* ProviderShared.invalidRequest(`${protocolName} does not support provider-native tool ${tool.name}`)
|
||||
return {
|
||||
type: "function" as const,
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: inputSchema,
|
||||
parameters: tool.inputSchema,
|
||||
// The common tool definition does not currently express Responses strict-schema policy.
|
||||
strict: false,
|
||||
}
|
||||
})
|
||||
|
||||
export const lowerTools = (tools: ReadonlyArray<ToolDefinition>, adapter: ProviderAdapter) =>
|
||||
Effect.forEach(tools, (tool) =>
|
||||
tool.native !== undefined && adapter.nativeTool ? adapter.nativeTool(tool.native) : lowerTool(adapter.name, tool),
|
||||
)
|
||||
|
||||
export const lowerToolChoice = (protocolName: string, toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
|
||||
ProviderShared.matchToolChoice(protocolName, toolChoice, {
|
||||
auto: () => "auto" as const,
|
||||
@@ -818,22 +817,10 @@ export const fromRequestWithAdapter = Effect.fn("OpenResponses.fromRequestWithAd
|
||||
adapter: ProviderAdapter,
|
||||
) {
|
||||
const projected = ProviderShared.flattenToolRequest(request)
|
||||
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
|
||||
return {
|
||||
...(yield* lowerConversation(projected.request, adapter)),
|
||||
...lowerGeneration(request),
|
||||
tools:
|
||||
projected.tools.length === 0
|
||||
? undefined
|
||||
: yield* Effect.forEach(projected.tools, (tool) =>
|
||||
tool.native !== undefined && adapter.nativeTool
|
||||
? adapter.nativeTool(tool.native)
|
||||
: lowerTool(
|
||||
adapter.name,
|
||||
tool,
|
||||
ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility),
|
||||
),
|
||||
),
|
||||
tools: projected.tools.length === 0 ? undefined : yield* lowerTools(projected.tools, adapter),
|
||||
tool_choice:
|
||||
allowedToolChoice(request) ??
|
||||
(request.toolChoice ? yield* lowerToolChoice(adapter.name, request.toolChoice) : undefined),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { Effect, Option, Schema } from "effect"
|
||||
import { Tool } from "@opencode/schema/tool"
|
||||
import { Route } from "../route/client.js"
|
||||
import { Auth } from "../route/auth.js"
|
||||
@@ -17,7 +17,6 @@ import {
|
||||
type FinishReason,
|
||||
type FinishReasonDetails,
|
||||
type CacheHint,
|
||||
type JsonSchema,
|
||||
type LLMRequest,
|
||||
type MediaPart,
|
||||
type ReasoningPart,
|
||||
@@ -29,7 +28,6 @@ import { classifyProviderFailure } from "../provider-error.js"
|
||||
import { isRecord, JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
|
||||
import { OpenAIOptions } from "./utils/openai-options.js"
|
||||
import { Lifecycle } from "./utils/lifecycle.js"
|
||||
import { ToolSchemaProjection } from "./utils/tool-schema.js"
|
||||
import { ToolStream } from "./utils/tool-stream.js"
|
||||
|
||||
const ADAPTER = "openai-chat"
|
||||
@@ -76,6 +74,44 @@ const OpenAIChatAssistantToolCall = Schema.Struct({
|
||||
})
|
||||
type OpenAIChatAssistantToolCall = Schema.Schema.Type<typeof OpenAIChatAssistantToolCall>
|
||||
|
||||
// `reasoning_details` carries two dialects. OpenRouter's `reasoning.*` entries
|
||||
// must be replayed unmodified (`index` included), so they keep every field they
|
||||
// arrived with. Kimi's OpenAI-compatible surface streams preserved thinking as
|
||||
// bare `summary` / `encrypted` entries keyed by a stream-only `index`; Kimi does
|
||||
// not document this publicly, so the handling follows Kimi Code (Kimi's own
|
||||
// client): merge summary deltas by `index`, replay without `index`, and always
|
||||
// send `reasoning_content` alongside. Anything else is dropped at the boundary.
|
||||
const OpenRouterDetailFields = {
|
||||
id: Schema.optional(Schema.NullOr(Schema.String)),
|
||||
format: Schema.optional(Schema.String),
|
||||
index: Schema.optional(Schema.Number),
|
||||
signature: Schema.optional(Schema.NullOr(Schema.String)),
|
||||
}
|
||||
const ReasoningDetail = Schema.Union([
|
||||
Schema.StructWithRest(
|
||||
Schema.Struct({ type: Schema.Literal("reasoning.text"), text: Schema.optional(Schema.String), ...OpenRouterDetailFields }),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
),
|
||||
Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("reasoning.summary"),
|
||||
summary: Schema.optional(Schema.String),
|
||||
...OpenRouterDetailFields,
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
),
|
||||
Schema.StructWithRest(
|
||||
Schema.Struct({ type: Schema.Literal("reasoning.encrypted"), data: Schema.String, ...OpenRouterDetailFields }),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
),
|
||||
Schema.Struct({ type: Schema.Literal("summary"), summary: Schema.String, index: Schema.optional(Schema.Number) }),
|
||||
Schema.Struct({ type: Schema.Literal("encrypted"), encrypted: Schema.String, index: Schema.optional(Schema.Number) }),
|
||||
])
|
||||
type ReasoningDetail = Schema.Schema.Type<typeof ReasoningDetail>
|
||||
const decodeReasoningDetail = Schema.decodeUnknownOption(ReasoningDetail)
|
||||
const knownReasoningDetails = (details: ReadonlyArray<unknown>) =>
|
||||
details.flatMap((detail) => Option.toArray(decodeReasoningDetail(detail)))
|
||||
|
||||
// Intentionally omit Gemini's provider-specific `extra_content.google.thought_signature`
|
||||
// extension until direct Google OpenAI-compatible routing is supported here:
|
||||
// https://github.com/vercel/ai/issues/11590
|
||||
@@ -92,6 +128,10 @@ const OpenAIChatUserContent = Schema.Union([
|
||||
type: Schema.Literal("image_url"),
|
||||
image_url: Schema.Struct({ url: Schema.String }),
|
||||
}),
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("file"),
|
||||
file: Schema.Struct({ filename: Schema.String, file_data: Schema.String }),
|
||||
}),
|
||||
])
|
||||
|
||||
const OpenAIChatMessage = Schema.Union([
|
||||
@@ -265,7 +305,9 @@ export interface ParserState {
|
||||
readonly finishReason?: FinishReasonDetails
|
||||
readonly lifecycle: Lifecycle.State
|
||||
readonly reasoningField?: string
|
||||
readonly reasoningDetails: Array<unknown>
|
||||
/** A scalar reasoning field (`reasoning_content`, ...) has carried text in this stream. */
|
||||
readonly reasoningTextObserved: boolean
|
||||
readonly reasoningDetails: Array<ReasoningDetail>
|
||||
readonly reasoningDetailsObserved: boolean
|
||||
readonly reasoningEmitted: boolean
|
||||
readonly latestToolIndex?: number
|
||||
@@ -286,17 +328,12 @@ interface LoweringOptions {
|
||||
readonly toolCallID?: (id: string) => string
|
||||
}
|
||||
|
||||
const lowerTool = (
|
||||
tool: ToolDefinition,
|
||||
inputSchema: JsonSchema,
|
||||
options: LoweringOptions,
|
||||
supportsStrictMode: boolean,
|
||||
): OpenAIChatTool => ({
|
||||
const lowerTool = (tool: ToolDefinition, options: LoweringOptions, supportsStrictMode: boolean): OpenAIChatTool => ({
|
||||
type: "function",
|
||||
function: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: inputSchema,
|
||||
parameters: tool.inputSchema,
|
||||
...(supportsStrictMode ? { strict: false } : {}),
|
||||
},
|
||||
cache_control: options.cacheControl?.(tool.cache),
|
||||
@@ -320,6 +357,15 @@ const lowerToolCall = (part: ToolCallPart, options: LoweringOptions): OpenAIChat
|
||||
})
|
||||
|
||||
const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart) {
|
||||
// Chat Completions accepts PDFs, and no other documents, as inline `file` parts; file URLs are not supported.
|
||||
if (part.media.mediaType.toLowerCase() === "application/pdf")
|
||||
return {
|
||||
type: "file" as const,
|
||||
file: {
|
||||
filename: part.filename ?? "document.pdf",
|
||||
file_data: (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl,
|
||||
},
|
||||
}
|
||||
if (part.media.kind !== "image")
|
||||
return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.media.mediaType}`)
|
||||
const url =
|
||||
@@ -341,10 +387,21 @@ const reasoningDetails = (parts: ReadonlyArray<ReasoningPart>, native: unknown,
|
||||
return Array.isArray(details) ? details : []
|
||||
})
|
||||
if (parts.some((part) => Array.isArray(part.providerMetadata?.[providerMetadataKey]?.reasoningDetails)))
|
||||
return observed
|
||||
if (isRecord(native) && Array.isArray(native.reasoning_details)) return native.reasoning_details
|
||||
return knownReasoningDetails(observed).map(lowerReasoningDetail)
|
||||
if (isRecord(native) && Array.isArray(native.reasoning_details))
|
||||
return knownReasoningDetails(native.reasoning_details).map(lowerReasoningDetail)
|
||||
}
|
||||
|
||||
// Kimi rejects its stream-only `index` on requests
|
||||
// ("the reasoning_details ... must not contain streaming index").
|
||||
const lowerReasoningDetail = (detail: ReasoningDetail) => {
|
||||
if (detail.type === "summary") return { type: detail.type, summary: detail.summary }
|
||||
if (detail.type === "encrypted") return { type: detail.type, encrypted: detail.encrypted }
|
||||
return detail
|
||||
}
|
||||
|
||||
const isKimiDetail = (detail: { readonly type: string }) => detail.type === "summary" || detail.type === "encrypted"
|
||||
|
||||
const lowerUserMessage = Effect.fn("OpenAIChat.lowerUserMessage")(function* (
|
||||
message: OpenAIChatRequestMessage,
|
||||
options: LoweringOptions,
|
||||
@@ -410,6 +467,9 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func
|
||||
if (observedField !== undefined) return observedField
|
||||
if (nativeReasoning !== undefined) return "reasoning_content"
|
||||
if (!fullyStructured || requireReasoning) return "reasoning_content"
|
||||
// Kimi always expects `reasoning_content` on replayed assistant messages,
|
||||
// even when thinking arrived only through structured details.
|
||||
if (details?.some(isKimiDetail)) return "reasoning_content"
|
||||
})()
|
||||
const reasoningText = (() => {
|
||||
if (configuredField !== undefined)
|
||||
@@ -435,7 +495,7 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (
|
||||
options: LoweringOptions,
|
||||
) {
|
||||
const messages: OpenAIChatMessage[] = []
|
||||
const images: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
|
||||
const attachments: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
|
||||
for (const part of message.content) {
|
||||
if (!ProviderShared.supportsContent(part, ["tool-result"]))
|
||||
return yield* ProviderShared.unsupportedContent("OpenAI Chat", "tool", ["tool-result"])
|
||||
@@ -457,9 +517,9 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (
|
||||
cache_control: options.cacheControl?.(part.cache),
|
||||
})
|
||||
const files = content.filter((item) => item.type === "file")
|
||||
images.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item)))))
|
||||
attachments.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item)))))
|
||||
}
|
||||
return { messages, images }
|
||||
return { messages, attachments }
|
||||
})
|
||||
|
||||
const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (
|
||||
@@ -520,21 +580,21 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request:
|
||||
if (requireAssistantAfterTool && messages.at(-1)?.role === "tool")
|
||||
messages.push({ role: "assistant", content: "Done." })
|
||||
}
|
||||
const pendingImages: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
|
||||
const flushImages = () => {
|
||||
if (pendingImages.length === 0) return
|
||||
const pendingAttachments: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
|
||||
const flushAttachments = () => {
|
||||
if (pendingAttachments.length === 0) return
|
||||
bridgeTools()
|
||||
messages.push({ role: "user", content: pendingImages.splice(0) })
|
||||
messages.push({ role: "user", content: pendingAttachments.splice(0) })
|
||||
}
|
||||
for (const message of request.messages) {
|
||||
if (message.role === "user") bridgeTools()
|
||||
if (message.role === "system") {
|
||||
const part = yield* ProviderShared.wrappedSystemUpdate("OpenAI Chat", message)
|
||||
if (pendingImages.length > 0) {
|
||||
if (pendingAttachments.length > 0) {
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: [
|
||||
...pendingImages.splice(0),
|
||||
...pendingAttachments.splice(0),
|
||||
{ type: "text", text: part.text, cache_control: options.cacheControl?.(part.cache) },
|
||||
],
|
||||
})
|
||||
@@ -578,13 +638,13 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request:
|
||||
if (message.role === "tool") {
|
||||
const lowered = yield* lowerToolMessages(message, lowering)
|
||||
messages.push(...lowered.messages)
|
||||
pendingImages.push(...lowered.images)
|
||||
pendingAttachments.push(...lowered.attachments)
|
||||
continue
|
||||
}
|
||||
flushImages()
|
||||
flushAttachments()
|
||||
messages.push(...(yield* lowerMessage(message, reasoningField, requireReasoning, lowering)))
|
||||
}
|
||||
flushImages()
|
||||
flushAttachments()
|
||||
return messages
|
||||
})
|
||||
|
||||
@@ -736,7 +796,6 @@ export const fromRequest = Effect.fn("OpenAIChat.fromRequest")(function* (
|
||||
`OpenAI Chat reasoning field conflicts with reserved field ${reasoningField}`,
|
||||
)
|
||||
const generation = request.generation
|
||||
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
|
||||
const flattened = ProviderShared.flattenToolRequest(request)
|
||||
const provider = String(request.model.provider)
|
||||
const baseURL = request.model.route.endpoint.baseURL
|
||||
@@ -759,14 +818,7 @@ export const fromRequest = Effect.fn("OpenAIChat.fromRequest")(function* (
|
||||
? hasHistory
|
||||
? []
|
||||
: undefined
|
||||
: flattened.tools.map((tool) =>
|
||||
lowerTool(
|
||||
tool,
|
||||
ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility),
|
||||
options,
|
||||
supportsStrictMode,
|
||||
),
|
||||
),
|
||||
: flattened.tools.map((tool) => lowerTool(tool, options, supportsStrictMode)),
|
||||
tool_choice: hasActiveTools && request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined,
|
||||
stream: true as const,
|
||||
...(supportsUsageInStreaming ? { stream_options: { include_usage: true } } : {}),
|
||||
@@ -881,44 +933,74 @@ const reasoningDelta = (
|
||||
return undefined
|
||||
}
|
||||
|
||||
const detailText = (details: ReadonlyArray<unknown>) => {
|
||||
const detailText = (details: ReadonlyArray<ReasoningDetail>, hideKimiSummary: boolean) => {
|
||||
const text = details.flatMap((detail) => {
|
||||
if (!isRecord(detail)) return []
|
||||
if (detail.type === "reasoning.text" && typeof detail.text === "string" && detail.text) return [detail.text]
|
||||
if (detail.type === "reasoning.summary" && typeof detail.summary === "string" && detail.summary)
|
||||
return [detail.summary]
|
||||
if (detail.type === "reasoning.text") return detail.text ? [detail.text] : []
|
||||
if (detail.type === "reasoning.summary") return detail.summary ? [detail.summary] : []
|
||||
// Kimi streams the full thinking through `reasoning_content` and a separate
|
||||
// summary through details; show the summary only when nothing else does.
|
||||
if (detail.type === "summary") return detail.summary && !hideKimiSummary ? [detail.summary] : []
|
||||
return []
|
||||
})
|
||||
if (text.length > 0) return text.join("")
|
||||
}
|
||||
|
||||
const appendReasoningDetails = (result: Array<unknown>, details: ReadonlyArray<unknown>) => {
|
||||
const appendReasoningDetails = (result: Array<ReasoningDetail>, details: ReadonlyArray<ReasoningDetail>) => {
|
||||
for (const detail of details) {
|
||||
const previous = result.at(-1)
|
||||
if (
|
||||
!isRecord(previous) ||
|
||||
previous.type !== "reasoning.text" ||
|
||||
!isRecord(detail) ||
|
||||
detail.type !== "reasoning.text" ||
|
||||
conflictingReasoningTextDetails(previous, detail)
|
||||
) {
|
||||
const merged = previous === undefined ? undefined : mergeReasoningDetails(previous, detail)
|
||||
if (merged === undefined) {
|
||||
result.push(detail)
|
||||
continue
|
||||
}
|
||||
result[result.length - 1] = {
|
||||
...previous,
|
||||
...Object.fromEntries(Object.entries(detail).filter((entry) => entry[1] !== undefined)),
|
||||
text: `${typeof previous.text === "string" ? previous.text : ""}${typeof detail.text === "string" ? detail.text : ""}`,
|
||||
signature: mergeDetailValue(previous.signature, detail.signature),
|
||||
format: mergeDetailValue(previous.format, detail.format),
|
||||
}
|
||||
result[result.length - 1] = merged
|
||||
}
|
||||
}
|
||||
|
||||
const mergeDetailValue = (previous: unknown, current: unknown) =>
|
||||
// Consecutive text or summary deltas of the same kind accumulate into one
|
||||
// entry; encrypted entries are opaque and never merge.
|
||||
const mergeReasoningDetails = (previous: ReasoningDetail, detail: ReasoningDetail): ReasoningDetail | undefined => {
|
||||
if (conflictingReasoningDetails(previous, detail)) return undefined
|
||||
if (previous.type === "reasoning.text" && detail.type === "reasoning.text")
|
||||
return {
|
||||
...previous,
|
||||
...detail,
|
||||
text: `${previous.text ?? ""}${detail.text ?? ""}`,
|
||||
...mergeDetailIdentity(previous, detail),
|
||||
}
|
||||
if (previous.type === "reasoning.summary" && detail.type === "reasoning.summary")
|
||||
return {
|
||||
...previous,
|
||||
...detail,
|
||||
summary: `${previous.summary ?? ""}${detail.summary ?? ""}`,
|
||||
...mergeDetailIdentity(previous, detail),
|
||||
}
|
||||
if (previous.type === "summary" && detail.type === "summary")
|
||||
return { ...previous, ...detail, summary: previous.summary + detail.summary }
|
||||
}
|
||||
|
||||
type DetailIdentity = {
|
||||
readonly id?: string | null
|
||||
readonly index?: number
|
||||
readonly format?: string
|
||||
readonly signature?: string | null
|
||||
}
|
||||
|
||||
// The first non-empty signature and format win; a later delta may carry the
|
||||
// signature for text that streamed earlier.
|
||||
const mergeDetailIdentity = (previous: DetailIdentity, current: DetailIdentity) => {
|
||||
const signature = mergeDetailValue(previous.signature, current.signature)
|
||||
const format = mergeDetailValue(previous.format, current.format)
|
||||
return {
|
||||
...(signature === undefined ? {} : { signature }),
|
||||
...(format === undefined ? {} : { format }),
|
||||
}
|
||||
}
|
||||
|
||||
const mergeDetailValue = <T>(previous: T | undefined, current: T | undefined) =>
|
||||
previous || current || (previous !== undefined ? previous : current)
|
||||
|
||||
const conflictingReasoningTextDetails = (previous: Record<string, unknown>, current: Record<string, unknown>) =>
|
||||
const conflictingReasoningDetails = (previous: DetailIdentity, current: DetailIdentity) =>
|
||||
conflictingDetailValue(previous.id, current.id) ||
|
||||
conflictingDetailValue(previous.index, current.index) ||
|
||||
conflictingDetailValue(previous.format, current.format) ||
|
||||
@@ -930,7 +1012,7 @@ const conflictingDetailValue = (previous: unknown, current: unknown) =>
|
||||
const reasoningMetadata = (
|
||||
providerMetadataKey: string,
|
||||
field: ParserState["reasoningField"],
|
||||
details?: ReadonlyArray<unknown>,
|
||||
details?: ReadonlyArray<ReasoningDetail>,
|
||||
) => ({
|
||||
[providerMetadataKey]: {
|
||||
...(field ? { reasoningField: field } : {}),
|
||||
@@ -993,11 +1075,16 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
|
||||
}
|
||||
|
||||
const reasoningField = state.reasoningField ?? reasoning?.field
|
||||
const detailDelta = Array.isArray(delta?.reasoning_details) ? delta.reasoning_details : undefined
|
||||
const reasoningTextObserved = state.reasoningTextObserved || reasoning !== undefined
|
||||
const detailDelta = Array.isArray(delta?.reasoning_details)
|
||||
? knownReasoningDetails(delta.reasoning_details)
|
||||
: undefined
|
||||
if (detailDelta !== undefined) appendReasoningDetails(state.reasoningDetails, detailDelta)
|
||||
const reasoningDetailsObserved = state.reasoningDetailsObserved || detailDelta !== undefined
|
||||
const deltaMetadata = reasoningMetadata(state.providerMetadataKey, reasoningField)
|
||||
const text = detailDelta?.length ? (detailText(detailDelta) ?? reasoning?.text) : reasoning?.text
|
||||
const text = detailDelta?.length
|
||||
? (detailText(detailDelta, reasoningTextObserved) ?? reasoning?.text)
|
||||
: reasoning?.text
|
||||
if (text !== undefined) lifecycle = Lifecycle.reasoningDelta(lifecycle, events, "reasoning-0", text, deltaMetadata)
|
||||
else if (
|
||||
reasoningDetailsObserved &&
|
||||
@@ -1093,6 +1180,7 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
|
||||
finishReason,
|
||||
lifecycle,
|
||||
reasoningField,
|
||||
reasoningTextObserved,
|
||||
reasoningDetails: state.reasoningDetails,
|
||||
reasoningDetailsObserved,
|
||||
reasoningEmitted,
|
||||
@@ -1173,6 +1261,7 @@ export const protocol = Protocol.make({
|
||||
toolCallEvents: [],
|
||||
lifecycle: Lifecycle.initial(),
|
||||
reasoningField: request.model.compatibility?.reasoningField,
|
||||
reasoningTextObserved: false,
|
||||
reasoningDetails: [],
|
||||
reasoningDetailsObserved: false,
|
||||
reasoningEmitted: false,
|
||||
|
||||
@@ -1,16 +1,21 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import {
|
||||
ImageFinishEvent,
|
||||
ImageModel,
|
||||
ImageOutputEvent,
|
||||
ImagePartialEvent,
|
||||
type ImageEvent,
|
||||
type ImageRequestFor,
|
||||
} from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
|
||||
import { mergeJsonRecords, type MediaUsage, type OpenString } from "../schema/index.js"
|
||||
import { ProviderShared } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
const ADAPTER = "openai-images"
|
||||
const NAME = "OpenAI Images"
|
||||
const PROVIDER = ProviderID.make("openai")
|
||||
const route = MediaProtocol.identity({ id: "openai-images", name: "OpenAI Images", provider: "openai" })
|
||||
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
||||
export const PATH = "/images/generations"
|
||||
export const EDIT_PATH = "/images/edits"
|
||||
@@ -19,14 +24,14 @@ export const EDIT_PATH = "/images/edits"
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type OpenAIImageString<Known extends string> = Known | (string & {})
|
||||
|
||||
/** Provider-native options. Common fields (`n`, `size`, `format`, `images`, `mask`) live on the request. */
|
||||
export type OpenAIImageOptions = {
|
||||
readonly quality?: OpenAIImageString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
|
||||
readonly background?: OpenAIImageString<"auto" | "opaque" | "transparent">
|
||||
readonly moderation?: OpenAIImageString<"auto" | "low">
|
||||
readonly quality?: OpenString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
|
||||
readonly background?: OpenString<"auto" | "opaque" | "transparent">
|
||||
readonly moderation?: OpenString<"auto" | "low">
|
||||
readonly outputCompression?: number
|
||||
/** Previews sent before the final image when streaming (default 2); ignored by `Image.generate`. */
|
||||
readonly partialImages?: number
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = ImageRequestFor<OpenAIImageOptions>
|
||||
@@ -35,26 +40,68 @@ export type Request = ImageRequestFor<OpenAIImageOptions>
|
||||
// 2. Response schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const OpenAIImageResponse = Schema.Struct({
|
||||
data: Schema.Array(
|
||||
Schema.Struct({
|
||||
b64_json: Schema.optional(Schema.String),
|
||||
url: Schema.optional(Schema.String),
|
||||
revised_prompt: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
output_format: Schema.optional(Schema.String),
|
||||
usage: Schema.optional(
|
||||
Schema.Struct({
|
||||
input_tokens: Schema.optional(Schema.Number),
|
||||
output_tokens: Schema.optional(Schema.Number),
|
||||
total_tokens: Schema.optional(Schema.Number),
|
||||
input_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
output_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
}),
|
||||
),
|
||||
const Usage = Schema.Struct({
|
||||
input_tokens: Schema.optional(Schema.Number),
|
||||
output_tokens: Schema.optional(Schema.Number),
|
||||
total_tokens: Schema.optional(Schema.Number),
|
||||
input_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
output_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
})
|
||||
|
||||
/** What the provider actually rendered; it can differ from the request when `auto` or a default applied. */
|
||||
const Settings = {
|
||||
output_format: Schema.optional(Schema.String),
|
||||
size: Schema.optional(Schema.String),
|
||||
quality: Schema.optional(Schema.String),
|
||||
background: Schema.optional(Schema.String),
|
||||
}
|
||||
|
||||
const OpenAIImageResponse = Schema.Struct({
|
||||
data: Schema.Array(Schema.Struct({ b64_json: Schema.String })),
|
||||
...Settings,
|
||||
usage: Schema.optional(Usage),
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3. Streaming event schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const StreamEvent = Schema.Union([
|
||||
Schema.Struct({
|
||||
type: Schema.Literals(["image_generation.partial_image", "image_edit.partial_image"]),
|
||||
b64_json: Schema.String,
|
||||
partial_image_index: Schema.Number,
|
||||
...Settings,
|
||||
output_format: Schema.String,
|
||||
}),
|
||||
Schema.Struct({
|
||||
type: Schema.Literals(["image_generation.completed", "image_edit.completed"]),
|
||||
b64_json: Schema.String,
|
||||
...Settings,
|
||||
output_format: Schema.String,
|
||||
usage: Schema.optional(Usage),
|
||||
}),
|
||||
])
|
||||
|
||||
const decodeEvent = route.decodeFrame(StreamEvent)
|
||||
const decodeDocument = Schema.decodeUnknownEffect(Schema.fromJsonString(OpenAIImageResponse))
|
||||
|
||||
/** `generate` reads the whole JSON response as one frame, with the requested format for responses that omit it. */
|
||||
type Frame = string | { readonly document: string; readonly requested: string | undefined }
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. Parser state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface State {
|
||||
readonly completed: number
|
||||
readonly format?: string
|
||||
readonly size?: string
|
||||
readonly quality?: string
|
||||
readonly background?: string
|
||||
readonly usage?: MediaUsage
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -64,39 +111,35 @@ const RESERVED_FORM_FIELDS = new Set(["model", "prompt", "image", "image[]", "im
|
||||
|
||||
const nativeOptions = (options: OpenAIImageOptions | undefined) => {
|
||||
if (!options) return undefined
|
||||
const { outputCompression, ...native } = options
|
||||
const { outputCompression, partialImages: _, ...native } = options
|
||||
return { output_compression: outputCompression, ...native }
|
||||
}
|
||||
|
||||
const streamOptions = (request: MediaProtocol.Addressed<Request>) => {
|
||||
if (request.mode !== "stream") return Effect.succeed(undefined)
|
||||
if (request.n !== undefined && request.n > 1)
|
||||
return Effect.fail(
|
||||
route.unsupported("media.n", `${route.name} streams one image; use Image.generate for n=${request.n}`),
|
||||
)
|
||||
return Effect.succeed({ stream: true, partial_images: request.providerOptions?.partialImages ?? 2 })
|
||||
}
|
||||
|
||||
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
|
||||
|
||||
const isInline = (asset: Media.Asset) => asset.inline() !== undefined
|
||||
const isInline = (asset: Media.Asset) => asset.source.type === "bytes" || asset.source.type === "base64"
|
||||
|
||||
const blob = (data: Uint8Array, mediaType: string) => {
|
||||
const buffer = new ArrayBuffer(data.byteLength)
|
||||
new Uint8Array(buffer).set(data)
|
||||
return new Blob([buffer], { type: mediaType })
|
||||
}
|
||||
|
||||
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
|
||||
const inline = asset.inline()
|
||||
if (inline) return Effect.succeed({ image_url: inline.dataUrl })
|
||||
const url = ProviderShared.mediaUrl(asset)
|
||||
if (url) return Effect.succeed({ image_url: url })
|
||||
const id = MediaInput.refID(asset, PROVIDER)
|
||||
if (id) return Effect.succeed({ file_id: id })
|
||||
return Effect.fail(
|
||||
ProviderShared.invalidRequest("OpenAI Images accepts image URLs, data URLs, bytes, and OpenAI file IDs"),
|
||||
const reference = (asset: Media.Asset) =>
|
||||
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(
|
||||
Effect.map((item) => (item.type === "ref" ? { file_id: item.value } : { image_url: item.value })),
|
||||
)
|
||||
}
|
||||
|
||||
const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Request) {
|
||||
const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
|
||||
const images = request.images ?? []
|
||||
const mask = request.mask
|
||||
if (mask !== undefined && images.length === 0)
|
||||
return yield* ProviderShared.invalidRequest("An OpenAI image mask requires at least one input image")
|
||||
const fields = mergeJsonRecords(
|
||||
{ n: request.n, size: request.size, output_format: request.format },
|
||||
{ n: request.n, size: request.size, output_format: request.format, ...(yield* streamOptions(request)) },
|
||||
nativeOptions(request.providerOptions),
|
||||
request.http?.body,
|
||||
)
|
||||
@@ -104,16 +147,17 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Re
|
||||
// Owned bytes go through multipart edits; remote URLs and file IDs use the JSON edits body instead.
|
||||
if (images.length > 0 && images.every(isInline) && (mask === undefined || isInline(mask))) {
|
||||
const form = new FormData()
|
||||
form.append("model", request.model.id)
|
||||
form.append("prompt", request.prompt)
|
||||
Object.entries(fields ?? {}).forEach(([key, value]) => {
|
||||
if (RESERVED_FORM_FIELDS.has(key)) return
|
||||
form.append(key, typeof value === "string" ? value : ProviderShared.encodeJson(value))
|
||||
})
|
||||
const uploads = yield* Effect.forEach(images, (image) => MediaInput.inlineBytes(ADAPTER, image))
|
||||
uploads.forEach((data, index) => form.append("image[]", blob(data, images[index].mediaType), `image-${index}`))
|
||||
MediaInput.appendFields(
|
||||
form,
|
||||
{ model: request.model.id, prompt: request.prompt },
|
||||
{ overlay: fields, reserved: RESERVED_FORM_FIELDS },
|
||||
)
|
||||
const uploads = yield* Effect.forEach(images, (image) => MediaInput.inlineBytes(route.id, image))
|
||||
uploads.forEach((data, index) =>
|
||||
form.append("image[]", MediaInput.blob(data, images[index].mediaType), `image-${index}`),
|
||||
)
|
||||
if (mask !== undefined)
|
||||
form.append("mask", blob(yield* MediaInput.inlineBytes(ADAPTER, mask), mask.mediaType), "mask")
|
||||
form.append("mask", MediaInput.blob(yield* MediaInput.inlineBytes(route.id, mask), mask.mediaType), "mask")
|
||||
return MediaProtocol.multipart(form)
|
||||
}
|
||||
|
||||
@@ -133,71 +177,121 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Re
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// 6. Stream parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const requestedFormat = (body: MediaProtocol.Body) => {
|
||||
if (body.type === "binary") return undefined
|
||||
const value = body.type === "json" ? body.value.output_format : body.value.get("output_format")
|
||||
return typeof value === "string" ? value : undefined
|
||||
}
|
||||
|
||||
const decodeResponse = Effect.fn("OpenAIImages.decodeResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.DecodeContext<Request>,
|
||||
) {
|
||||
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, OpenAIImageResponse)(response)
|
||||
const decoded = output.value
|
||||
const format = decoded.output_format ?? requestedFormat(context.body) ?? "png"
|
||||
const mediaType = `image/${format}`
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) => {
|
||||
const providerMetadata =
|
||||
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } }
|
||||
if (item.b64_json)
|
||||
return MediaInput.decodedAsset(output.invalid, `${NAME} result ${index}`, item.b64_json, mediaType, {
|
||||
info: { format },
|
||||
providerMetadata,
|
||||
})
|
||||
if (item.url) return Effect.succeed(Media.url(item.url, { mediaType, info: { format }, providerMetadata }))
|
||||
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
|
||||
})
|
||||
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
|
||||
return new ImageResponse({
|
||||
images,
|
||||
usage:
|
||||
decoded.usage === undefined
|
||||
? undefined
|
||||
: {
|
||||
type: "tokens",
|
||||
input: decoded.usage.input_tokens,
|
||||
output: decoded.usage.output_tokens,
|
||||
total: decoded.usage.total_tokens,
|
||||
details: { openai: decoded.usage },
|
||||
},
|
||||
providerMetadata: { openai: { outputFormat: format } },
|
||||
const usage = (value: Schema.Schema.Type<typeof Usage> | undefined): MediaUsage | undefined =>
|
||||
value === undefined
|
||||
? undefined
|
||||
: {
|
||||
type: "tokens",
|
||||
input: value.input_tokens,
|
||||
output: value.output_tokens,
|
||||
total: value.total_tokens,
|
||||
details: { openai: value },
|
||||
}
|
||||
|
||||
/** `size` echoes the rendered `WIDTHxHEIGHT`; `auto` or any other value leaves the dimensions unknown. */
|
||||
const info = (format: string, size: string | undefined): Media.Info => {
|
||||
const match = size?.match(/^(\d+)x(\d+)$/)
|
||||
return match ? { format, width: Number(match[1]), height: Number(match[2]) } : { format }
|
||||
}
|
||||
|
||||
const eventImage = (frame: string, label: string, data: string, format: string, size: string | undefined) =>
|
||||
MediaInput.decodedAsset((message, cause) => route.frameError(message, frame, cause), label, data, `image/${format}`, {
|
||||
info: info(format, size),
|
||||
})
|
||||
|
||||
const onEvent = Effect.fn("OpenAIImages.onEvent")(function* (state: State, frame: string) {
|
||||
const event = yield* decodeEvent(frame)
|
||||
const format = event.output_format
|
||||
if ("partial_image_index" in event) {
|
||||
const image = yield* eventImage(frame, `${route.name} partial image`, event.b64_json, format, event.size)
|
||||
return [state, [ImagePartialEvent.make({ index: event.partial_image_index, image })]] as const
|
||||
}
|
||||
const image = yield* eventImage(frame, `${route.name} result ${state.completed}`, event.b64_json, format, event.size)
|
||||
return [
|
||||
{
|
||||
completed: state.completed + 1,
|
||||
format,
|
||||
size: event.size,
|
||||
quality: event.quality,
|
||||
background: event.background,
|
||||
usage: usage(event.usage),
|
||||
},
|
||||
[ImageOutputEvent.make({ index: state.completed, image })],
|
||||
] as const
|
||||
})
|
||||
|
||||
const onDocument = Effect.fn("OpenAIImages.onDocument")(function* (frame: Exclude<Frame, string>) {
|
||||
const invalid = (message: string, cause?: unknown) => route.frameError(message, frame.document, cause)
|
||||
const decoded = yield* decodeDocument(frame.document).pipe(
|
||||
Effect.mapError((cause) => invalid(`${route.name} returned an invalid response`, cause)),
|
||||
)
|
||||
const format = decoded.output_format ?? frame.requested ?? "png"
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) =>
|
||||
MediaInput.decodedAsset(invalid, `${route.name} result ${index}`, item.b64_json, `image/${format}`, {
|
||||
info: info(format, decoded.size),
|
||||
}),
|
||||
)
|
||||
if (images.length === 0) return yield* invalid(`${route.name} returned no images`)
|
||||
const state: State = {
|
||||
completed: images.length,
|
||||
format,
|
||||
size: decoded.size,
|
||||
quality: decoded.quality,
|
||||
background: decoded.background,
|
||||
usage: usage(decoded.usage),
|
||||
}
|
||||
return [state, images.map((image, index) => ImageOutputEvent.make({ index, image }))] as const
|
||||
})
|
||||
|
||||
const step = (state: State, frame: Frame) => (typeof frame === "string" ? onEvent(state, frame) : onDocument(frame))
|
||||
|
||||
const finish = (state: State) => {
|
||||
if (state.completed === 0) return Effect.fail(route.incomplete())
|
||||
return Effect.succeed([
|
||||
ImageFinishEvent.make({
|
||||
usage: state.usage,
|
||||
providerMetadata: {
|
||||
openai: {
|
||||
outputFormat: state.format,
|
||||
size: state.size,
|
||||
quality: state.quality,
|
||||
background: state.background,
|
||||
},
|
||||
},
|
||||
}),
|
||||
])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
|
||||
id: ADAPTER,
|
||||
name: NAME,
|
||||
export const protocol = MediaProtocol.stream<Request, ImageEvent, Frame, State>(route, {
|
||||
unsupported: ["aspectRatio", "seed"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
frames: (bytes, context) =>
|
||||
context.request.mode === "stream"
|
||||
? Framing.sse.frame(bytes)
|
||||
: Framing.document
|
||||
.frame(bytes)
|
||||
.pipe(Stream.map((document) => ({ document, requested: requestedFormat(context.body) }))),
|
||||
initial: () => ({ completed: 0 }),
|
||||
step,
|
||||
finish,
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<OpenAIImageOptions>(
|
||||
{
|
||||
id: ADAPTER,
|
||||
provider: PROVIDER,
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
|
||||
},
|
||||
ImageModel.fromRoute<OpenAIImageOptions, Frame, State>(
|
||||
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH) },
|
||||
input,
|
||||
)
|
||||
|
||||
|
||||
@@ -5,14 +5,12 @@ import { Auth } from "../route/auth.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { HttpTransport } from "../route/transport/index.js"
|
||||
import { LLMRequest, mergeJsonRecords, type JsonSchema, type ToolDefinition, type ToolEntry } from "../schema/index.js"
|
||||
import { LLMRequest, type ToolDefinition, type ToolEntry } from "../schema/index.js"
|
||||
import { resolveEffortUpdates } from "../effort-updates.js"
|
||||
import { OpenResponses } from "./open-responses.js"
|
||||
import { OpenResponsesOptions } from "./utils/open-responses-options.js"
|
||||
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
|
||||
import { OpenAIImage } from "./utils/openai-image.js"
|
||||
import { ResponsesHostedTools } from "./utils/responses-hosted-tools.js"
|
||||
import { ToolSchemaProjection } from "./utils/tool-schema.js"
|
||||
import { OpenResponsesChannel } from "./open-responses-channel.js"
|
||||
import { ResponsesCompaction } from "./utils/responses-compaction.js"
|
||||
import { ResponsesCheckpoint } from "./utils/responses-checkpoint.js"
|
||||
@@ -41,7 +39,16 @@ const OpenAIResponsesImageGenerationTool = Schema.Struct({
|
||||
output_format: Schema.optional(Schema.Literals(["png", "jpeg", "webp"])),
|
||||
partial_images: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))),
|
||||
quality: Schema.optional(Schema.Literals(["auto", "low", "medium", "high"])),
|
||||
size: Schema.optional(OpenAIImage.Size),
|
||||
size: Schema.optional(
|
||||
Schema.String.check(
|
||||
Schema.makeFilter((value) => {
|
||||
if (value === "auto") return undefined
|
||||
const match = /^(\d+)x(\d+)$/.exec(value)
|
||||
if (!match) return "image size must be `auto` or `{width}x{height}`"
|
||||
return Number(match[1]) > 0 && Number(match[2]) > 0 ? undefined : "image dimensions must be positive integers"
|
||||
}),
|
||||
),
|
||||
),
|
||||
})
|
||||
|
||||
const OpenAIResponsesHostedToolItem = Schema.Union([
|
||||
@@ -128,11 +135,6 @@ export const CompactionTrigger = Schema.Struct({ type: Schema.Literal("compactio
|
||||
const CheckpointBody = Schema.Struct({
|
||||
...OpenAIResponsesBody.fields,
|
||||
input: Schema.Array(Schema.Union([OpenAIResponsesInputItem, CompactionTrigger])),
|
||||
store: Schema.Literal(false),
|
||||
prompt_cache_retention: optionalNull(Schema.String),
|
||||
prompt_cache_options: optionalNull(
|
||||
Schema.Struct({ mode: Schema.optional(Schema.String), ttl: Schema.optional(Schema.String) }),
|
||||
),
|
||||
})
|
||||
|
||||
const adapter = {
|
||||
@@ -141,12 +143,14 @@ const adapter = {
|
||||
restoreHostedToolItem: (item: unknown) => (Schema.is(OpenAIResponsesHostedToolItem)(item) ? item : undefined),
|
||||
} satisfies OpenResponses.ProviderAdapter
|
||||
|
||||
// Only GPT-6 Astra accepts `configuration_update`, and never alongside automatic `context_management` compaction.
|
||||
// GPT-6 Astra, Sol, and Luna accept `configuration_update` only in standard mode (not `reasoning.mode: "pro"` or
|
||||
// `-pro` slugs), and never alongside automatic `context_management` compaction.
|
||||
const supportsEffortUpdates = (request: LLMRequest) => {
|
||||
if (request.providerOptions?.contextManagement !== undefined) return false
|
||||
if (Schema.is(Schema.Struct({ mode: Schema.Literal("pro") }))(request.http?.body?.reasoning)) return false
|
||||
const override = request.model.compatibility?.supportsEffortUpdates
|
||||
if (override !== undefined) return override
|
||||
return /(?:^|\/)gpt-6-astra$/i.test(request.model.id)
|
||||
return /(?:^|\/)gpt-6-(?:astra|sol|luna)$/i.test(request.model.id)
|
||||
}
|
||||
|
||||
const nativeImageToolInput = (tool: ToolDefinition) => {
|
||||
@@ -159,23 +163,19 @@ const nativeImageTool = (tool: ToolDefinition) => {
|
||||
return Schema.is(OpenAIResponsesImageGenerationTool)(native) ? native : undefined
|
||||
}
|
||||
|
||||
const lowerTool = Effect.fn("OpenAIResponses.lowerTool")(function* (tool: ToolDefinition, inputSchema: JsonSchema) {
|
||||
const lowerTool = Effect.fn("OpenAIResponses.lowerTool")(function* (tool: ToolDefinition) {
|
||||
const native = nativeImageToolInput(tool)
|
||||
if (native !== undefined) {
|
||||
if (Schema.is(OpenAIResponsesImageGenerationTool)(native)) return native
|
||||
return yield* ProviderShared.invalidRequest("OpenAI Responses image generation tool options are invalid")
|
||||
}
|
||||
return yield* OpenResponses.lowerTool(NAME, tool, inputSchema)
|
||||
return yield* OpenResponses.lowerTool(NAME, tool)
|
||||
})
|
||||
|
||||
// Native namespaces hold only function tools, so deeper levels flatten into
|
||||
// the leaf names the same way non-native protocols flatten the whole tree.
|
||||
const lowerToolEntry = Effect.fn("OpenAIResponses.lowerToolEntry")(function* (
|
||||
tool: ToolEntry,
|
||||
compatibility: Parameters<typeof ToolSchemaProjection.modelCompatibility>[1],
|
||||
) {
|
||||
if (tool.type === "tool")
|
||||
return yield* lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, compatibility))
|
||||
const lowerToolEntry = Effect.fn("OpenAIResponses.lowerToolEntry")(function* (tool: ToolEntry) {
|
||||
if (tool.type === "tool") return yield* lowerTool(tool)
|
||||
// OpenAI requires a namespace description; fall back to a generic one so a
|
||||
// missing description never blocks the request.
|
||||
return {
|
||||
@@ -183,11 +183,13 @@ const lowerToolEntry = Effect.fn("OpenAIResponses.lowerToolEntry")(function* (
|
||||
name: tool.name,
|
||||
description: tool.description ?? `Tools in the ${tool.name} namespace.`,
|
||||
tools: yield* Effect.forEach(ProviderShared.flattenTools(tool.tools), (leaf) =>
|
||||
OpenResponses.lowerTool(NAME, leaf, ToolSchemaProjection.modelCompatibility(leaf.inputSchema, compatibility)),
|
||||
OpenResponses.lowerTool(NAME, leaf),
|
||||
),
|
||||
}
|
||||
})
|
||||
|
||||
const lowerTools = (request: LLMRequest) => Effect.forEach(request.tools, lowerToolEntry)
|
||||
|
||||
const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>, tools: ReadonlyArray<ToolEntry>) =>
|
||||
ProviderShared.matchToolChoice(NAME, toolChoice, {
|
||||
auto: () => "auto" as const,
|
||||
@@ -207,15 +209,11 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request:
|
||||
)(request.providerOptions?.contextManagement)
|
||||
const options = OpenResponsesOptions.resolve(request)
|
||||
const updates = resolveEffortUpdates(request, options.reasoningEffort)
|
||||
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
|
||||
return yield* decodeBody({
|
||||
...(yield* OpenResponses.lowerConversation(updates.request, adapter)),
|
||||
...OpenResponses.lowerGeneration(request, { ...options, reasoningEffort: updates.effort }),
|
||||
context_management: management?.map((edit) => ({ type: edit.type, compact_threshold: edit.compactThreshold })),
|
||||
tools:
|
||||
request.tools.length === 0
|
||||
? undefined
|
||||
: yield* Effect.forEach(request.tools, (tool) => lowerToolEntry(tool, toolSchemaCompatibility)),
|
||||
tools: request.tools.length === 0 ? undefined : yield* lowerTools(request),
|
||||
tool_choice:
|
||||
request.tools.length === 0
|
||||
? undefined
|
||||
@@ -227,7 +225,6 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request:
|
||||
const checkpointBody = {
|
||||
schema: CheckpointBody,
|
||||
from: Effect.fn("OpenAIResponses.checkpointBody")(function* (request: LLMRequest) {
|
||||
const native = yield* fromRequest(LLMRequest.update(request, { toolChoice: undefined }))
|
||||
const overlay = request.http?.body
|
||||
// Complete history is required for stateless replay and SSE recovery. Raw input overrides bypass that contract.
|
||||
if (
|
||||
@@ -238,18 +235,13 @@ const checkpointBody = {
|
||||
return yield* ProviderShared.invalidRequest(
|
||||
"Trigger compaction requires complete canonical history, not an input or continuation override",
|
||||
)
|
||||
return yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(CheckpointBody))({
|
||||
...mergeJsonRecords(native, overlay),
|
||||
input: [...native.input, { type: "compaction_trigger" }],
|
||||
stream: true,
|
||||
store: false,
|
||||
parallel_tool_calls: true,
|
||||
tool_choice: undefined,
|
||||
context_management: undefined,
|
||||
text: undefined,
|
||||
max_output_tokens: undefined,
|
||||
max_tool_calls: undefined,
|
||||
})
|
||||
if (overlay?.stream !== undefined && overlay.stream !== true)
|
||||
return yield* ProviderShared.invalidRequest("Trigger compaction requires a streamed response")
|
||||
const native = yield* fromRequest(request)
|
||||
return {
|
||||
...native,
|
||||
input: [...native.input, { type: "compaction_trigger" as const }],
|
||||
}
|
||||
}),
|
||||
}
|
||||
|
||||
@@ -331,7 +323,10 @@ export const transport = channelTransport({
|
||||
})
|
||||
|
||||
export const route = Route.make({
|
||||
compact: { endpoint: ResponsesCompaction.make(adapter), trigger: ResponsesCheckpoint.make(checkpointBody) },
|
||||
compact: {
|
||||
endpoint: ResponsesCompaction.make(adapter, lowerTools),
|
||||
trigger: ResponsesCheckpoint.make(checkpointBody),
|
||||
},
|
||||
id: ADAPTER,
|
||||
provider: "openai",
|
||||
providerMetadataKey: "openai",
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type MediaUsage } from "../schema/index.js"
|
||||
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
|
||||
import { SpeechStream } from "./utils/speech-stream.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "openai-speech", name: "OpenAI Speech", provider: "openai" })
|
||||
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
||||
export const PATH = "/audio/speech"
|
||||
/** `pcm` is raw 24 kHz, 16-bit signed little-endian mono samples without a header. */
|
||||
const PCM_SAMPLE_RATE = 24000
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** `voice`, `instructions`, `speed`, and `format` are common request fields; other native body fields pass through. */
|
||||
export type OpenAISpeechOptions = {
|
||||
/** Defaults to `"sse"` in `stream` mode on models that support it; the merged value selects the response framing. */
|
||||
readonly stream_format?: "sse" | "audio"
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = SpeechRequestFor<OpenAISpeechOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3. Streaming event schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const SpeechStreamEvent = Schema.Union([
|
||||
Schema.Struct({ type: Schema.Literal("speech.audio.delta"), audio: Schema.Uint8ArrayFromBase64 }),
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("speech.audio.done"),
|
||||
usage: Schema.optional(
|
||||
Schema.Struct({
|
||||
input_tokens: Schema.optional(Schema.Number),
|
||||
output_tokens: Schema.optional(Schema.Number),
|
||||
total_tokens: Schema.optional(Schema.Number),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
])
|
||||
|
||||
const decodeEvent = route.decodeFrame(SpeechStreamEvent)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. Parser state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface State extends SpeechStream.Audio {
|
||||
readonly done: boolean
|
||||
readonly usage?: MediaUsage
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// `sse` is not supported for `tts-1` or `tts-1-hd`; those models stream the raw audio body instead.
|
||||
const supportsSse = (model: string) => !/^tts-1(-hd)?(-|$)/.test(model)
|
||||
|
||||
const fromRequest = Effect.fn("OpenAISpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
|
||||
// Not in `unsupported`: that list would also reject `timestamps: false`, which asks for nothing.
|
||||
if (request.timestamps === true)
|
||||
return yield* route.unsupported("media.timestamps", `${route.name} does not return timestamps`)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
model: request.model.id,
|
||||
input: request.text,
|
||||
voice: request.voice,
|
||||
instructions: request.instructions,
|
||||
response_format: request.format,
|
||||
speed: request.speed,
|
||||
stream_format: request.mode === "stream" && supportsSse(request.model.id) ? "sse" : undefined,
|
||||
},
|
||||
request.providerOptions,
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Stream parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const isSse = (body: MediaProtocol.Body) => body.type === "json" && body.value.stream_format === "sse"
|
||||
|
||||
const onEvent = Effect.fn("OpenAISpeech.onEvent")(function* (state: State, frame: string) {
|
||||
const event = yield* decodeEvent(frame)
|
||||
if (event.type === "speech.audio.delta") return SpeechStream.delta(state, event.audio)
|
||||
const usage = event.usage
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
done: true,
|
||||
usage:
|
||||
usage === undefined
|
||||
? undefined
|
||||
: {
|
||||
type: "tokens" as const,
|
||||
input: usage.input_tokens,
|
||||
output: usage.output_tokens,
|
||||
total: usage.total_tokens,
|
||||
details: { openai: usage },
|
||||
},
|
||||
},
|
||||
[],
|
||||
] as const
|
||||
})
|
||||
|
||||
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
|
||||
if (isSse(context.body) && !state.done) return Effect.fail(route.incomplete())
|
||||
const format = context.request.format ?? "mp3"
|
||||
return SpeechStream.finish(route, state, {
|
||||
...(format === "pcm" ? SpeechStream.pcm("pcm_s16le", PCM_SAMPLE_RATE) : SpeechStream.container(format)),
|
||||
usage: state.usage,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>(route, {
|
||||
unsupported: ["language"],
|
||||
body: { from: fromRequest },
|
||||
frames: (bytes, context) => (isSse(context.body) ? Framing.sse.frame(bytes) : bytes),
|
||||
initial: () => ({ chunks: [], done: false }),
|
||||
step: SpeechStream.step(onEvent),
|
||||
finish,
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
SpeechModel.fromRoute<OpenAISpeechOptions, string | Uint8Array, State>(
|
||||
{ protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
|
||||
input,
|
||||
)
|
||||
|
||||
export const OpenAISpeech = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -0,0 +1,289 @@
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import { classifyProviderFailure } from "../provider-error.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { AIError, mergeJsonRecords, type MediaUsage } from "../schema/index.js"
|
||||
import {
|
||||
TranscriptionFinishEvent,
|
||||
TranscriptionModel,
|
||||
TranscriptionSegmentEvent,
|
||||
TranscriptionTextDeltaEvent,
|
||||
type TranscriptionEvent,
|
||||
type TranscriptionRequestFor,
|
||||
type TranscriptionSegment,
|
||||
} from "../transcription.js"
|
||||
import { mediaTypeExtension } from "../utils/media-type.js"
|
||||
import { ProviderShared } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "openai-transcription", name: "OpenAI Transcription", provider: "openai" })
|
||||
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
||||
export const PATH = "/audio/transcriptions"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type OpenAITranscriptionOptions = {
|
||||
readonly chunking_strategy?:
|
||||
| "auto"
|
||||
| {
|
||||
readonly type: "server_vad"
|
||||
readonly prefix_padding_ms?: number
|
||||
readonly silence_duration_ms?: number
|
||||
readonly threshold?: number
|
||||
}
|
||||
readonly include?: ReadonlyArray<"logprobs">
|
||||
readonly keywords?: ReadonlyArray<string>
|
||||
readonly known_speaker_names?: ReadonlyArray<string>
|
||||
readonly known_speaker_references?: ReadonlyArray<string>
|
||||
readonly temperature?: number
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = TranscriptionRequestFor<OpenAITranscriptionOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3. Streaming event schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const Segment = Schema.Struct({
|
||||
text: Schema.String,
|
||||
start: Schema.Number,
|
||||
end: Schema.Number,
|
||||
speaker: Schema.optional(Schema.String),
|
||||
})
|
||||
|
||||
const Usage = Schema.Union([
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("tokens"),
|
||||
input_tokens: Schema.optional(Schema.Number),
|
||||
output_tokens: Schema.optional(Schema.Number),
|
||||
total_tokens: Schema.optional(Schema.Number),
|
||||
input_token_details: Schema.optional(
|
||||
Schema.Struct({ audio_tokens: Schema.optional(Schema.Number), text_tokens: Schema.optional(Schema.Number) }),
|
||||
),
|
||||
}),
|
||||
Schema.Struct({ type: Schema.Literal("duration"), seconds: Schema.Number }),
|
||||
])
|
||||
|
||||
const transcriptFields = {
|
||||
text: Schema.String,
|
||||
language: Schema.optional(Schema.String),
|
||||
languages: Schema.optional(Schema.Array(Schema.Struct({ code: Schema.String }))),
|
||||
duration: Schema.optional(Schema.Number),
|
||||
segments: Schema.optional(Schema.Array(Segment)),
|
||||
words: Schema.optional(
|
||||
Schema.Array(Schema.Struct({ word: Schema.String, start: Schema.Number, end: Schema.Number })),
|
||||
),
|
||||
usage: Schema.optional(Usage),
|
||||
}
|
||||
|
||||
/** OpenAI may add stream event types; frames outside `EVENT_TYPES` are ignored. */
|
||||
const EventType = Schema.Struct({ type: Schema.String })
|
||||
const Event = Schema.Union([
|
||||
Schema.Struct({ type: Schema.Literal("transcript.text.delta"), delta: Schema.String }),
|
||||
Schema.Struct({ type: Schema.Literal("transcript.text.segment"), ...Segment.fields }),
|
||||
Schema.Struct({ type: Schema.Literal("transcript.text.done"), ...transcriptFields }),
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("error"),
|
||||
message: Schema.optional(Schema.String),
|
||||
error: Schema.optional(Schema.Struct({ message: Schema.optional(Schema.String) })),
|
||||
}),
|
||||
])
|
||||
const EVENT_TYPES = new Set(["transcript.text.delta", "transcript.text.segment", "transcript.text.done", "error"])
|
||||
const Transcript = Schema.Struct(transcriptFields)
|
||||
type Transcript = Schema.Schema.Type<typeof Transcript>
|
||||
|
||||
const decodeEventType = route.decodeFrame(EventType)
|
||||
const decodeEvent = route.decodeFrame(Event)
|
||||
const decodeTranscript = route.decodeFrame(Transcript)
|
||||
|
||||
type Frame = string | { readonly document: string }
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. Parser state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface State {
|
||||
readonly segments: Array<TranscriptionSegment>
|
||||
readonly transcript?: Transcript
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface Capabilities {
|
||||
readonly stream: boolean
|
||||
readonly timestamps: ReadonlyArray<"segment" | "word">
|
||||
readonly diarize: boolean
|
||||
readonly languageField: "language" | "languages"
|
||||
}
|
||||
|
||||
const TRANSCRIBE: Capabilities = { stream: true, timestamps: [], diarize: false, languageField: "language" }
|
||||
|
||||
const capabilities = (model: string): Capabilities => {
|
||||
if (model.startsWith("whisper")) return { ...TRANSCRIBE, stream: false, timestamps: ["segment", "word"] }
|
||||
if (model.includes("diarize")) return { ...TRANSCRIBE, timestamps: ["segment"], diarize: true }
|
||||
// `gpt-transcribe` replaces `language` with `languages[]` and rejects both together.
|
||||
if (model.startsWith("gpt-transcribe")) return { ...TRANSCRIBE, languageField: "languages" }
|
||||
return TRANSCRIBE
|
||||
}
|
||||
|
||||
/** whisper-1 ignores `stream`, so its `stream` mode sends a plain request and emits only `finish`. */
|
||||
const streamsEvents = (request: MediaProtocol.Addressed<Request>) =>
|
||||
request.mode === "stream" && capabilities(request.model.id).stream
|
||||
|
||||
const validate = (request: MediaProtocol.Addressed<Request>, model: Capabilities) => {
|
||||
const id = request.model.id
|
||||
if (request.diarize === true && !model.diarize)
|
||||
return Effect.fail(route.unsupported("media.diarize", `${id} does not diarize; use gpt-4o-transcribe-diarize`))
|
||||
if (request.prompt !== undefined && model.diarize)
|
||||
return Effect.fail(route.unsupported("media.prompt", `${id} does not accept a prompt`))
|
||||
if (
|
||||
request.timestamps === undefined ||
|
||||
request.timestamps === "none" ||
|
||||
model.timestamps.includes(request.timestamps)
|
||||
)
|
||||
return Effect.void
|
||||
return Effect.fail(route.unsupported("media.timestamps", `${id} does not return ${request.timestamps} timestamps`))
|
||||
}
|
||||
|
||||
const RESERVED_FORM_FIELDS = new Set([
|
||||
"file",
|
||||
"model",
|
||||
"prompt",
|
||||
"language",
|
||||
"response_format",
|
||||
"timestamp_granularities",
|
||||
"stream",
|
||||
])
|
||||
|
||||
const fromRequest = Effect.fn("OpenAITranscription.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
|
||||
const model = capabilities(request.model.id)
|
||||
yield* validate(request, model)
|
||||
// The API detects the audio format from the upload's filename extension.
|
||||
const extension = mediaTypeExtension(request.audio.mediaType)
|
||||
if (extension === undefined)
|
||||
return yield* ProviderShared.invalidRequest(
|
||||
`${route.name} cannot name a ${request.audio.mediaType} upload; send mp3, mp4, m4a, wav, webm, ogg, or flac audio`,
|
||||
)
|
||||
const audio = yield* MediaInput.inlineBytes(route.id, request.audio)
|
||||
const responseFormat = model.diarize
|
||||
? "diarized_json"
|
||||
: request.timestamps === undefined || request.timestamps === "none"
|
||||
? undefined
|
||||
: "verbose_json"
|
||||
const form = new FormData()
|
||||
form.append("file", MediaInput.blob(audio, request.audio.mediaType), `audio.${extension}`)
|
||||
MediaInput.appendFields(
|
||||
form,
|
||||
{
|
||||
model: request.model.id,
|
||||
language: model.languageField === "language" ? request.language : undefined,
|
||||
languages: model.languageField === "languages" && request.language !== undefined ? [request.language] : undefined,
|
||||
prompt: request.prompt,
|
||||
response_format: responseFormat,
|
||||
timestamp_granularities: responseFormat === "verbose_json" ? [request.timestamps] : undefined,
|
||||
// Diarizing audio longer than 30 seconds requires a chunking strategy.
|
||||
chunking_strategy: model.diarize ? "auto" : undefined,
|
||||
stream: streamsEvents(request) ? true : undefined,
|
||||
},
|
||||
{
|
||||
overlay: mergeJsonRecords(request.providerOptions, request.http?.body),
|
||||
reserved: RESERVED_FORM_FIELDS,
|
||||
repeatArrays: true,
|
||||
},
|
||||
)
|
||||
return MediaProtocol.multipart(form)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Stream parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const segment = (value: Schema.Schema.Type<typeof Segment>): TranscriptionSegment => ({
|
||||
text: value.text.trim(),
|
||||
startSeconds: value.start,
|
||||
endSeconds: value.end,
|
||||
speaker: value.speaker,
|
||||
})
|
||||
|
||||
const onEvent = Effect.fn("OpenAITranscription.onEvent")(function* (state: State, frame: string) {
|
||||
if (!EVENT_TYPES.has((yield* decodeEventType(frame)).type)) return [state, []] as const
|
||||
const event = yield* decodeEvent(frame)
|
||||
if (event.type === "error")
|
||||
return yield* new AIError({
|
||||
reason: classifyProviderFailure({
|
||||
message: `${route.name} stream failed: ${event.message ?? event.error?.message ?? "unknown error"}`,
|
||||
rawBody: frame,
|
||||
}),
|
||||
})
|
||||
if (event.type === "transcript.text.done") return [{ ...state, transcript: event }, []] as const
|
||||
if (event.type === "transcript.text.delta")
|
||||
return [state, event.delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta: event.delta })]] as const
|
||||
const next = segment(event)
|
||||
state.segments.push(next)
|
||||
return [state, [TranscriptionSegmentEvent.make({ segment: next })]] as const
|
||||
})
|
||||
|
||||
const step = (state: State, frame: Frame) =>
|
||||
typeof frame === "string"
|
||||
? onEvent(state, frame)
|
||||
: decodeTranscript(frame.document).pipe(Effect.map((transcript) => [{ ...state, transcript }, []] as const))
|
||||
|
||||
const usage = (value: Transcript["usage"]): MediaUsage | undefined => {
|
||||
if (value === undefined) return undefined
|
||||
if (value.type === "duration") return { type: "seconds", seconds: value.seconds }
|
||||
return {
|
||||
type: "tokens",
|
||||
input: value.input_tokens,
|
||||
output: value.output_tokens,
|
||||
total: value.total_tokens,
|
||||
details: { openai: value },
|
||||
}
|
||||
}
|
||||
|
||||
const finish = (state: State) => {
|
||||
const transcript = state.transcript
|
||||
if (transcript === undefined) return Effect.fail(route.incomplete())
|
||||
const segments = transcript.segments?.map(segment) ?? state.segments
|
||||
return Effect.succeed([
|
||||
TranscriptionFinishEvent.make({
|
||||
text: transcript.text,
|
||||
segments: segments.length === 0 ? undefined : segments,
|
||||
words: transcript.words?.map((word) => ({ text: word.word, startSeconds: word.start, endSeconds: word.end })),
|
||||
language: (transcript.language ?? transcript.languages?.[0]?.code)?.toLowerCase(),
|
||||
durationSeconds: transcript.duration,
|
||||
usage: usage(transcript.usage),
|
||||
}),
|
||||
])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, Frame, State>(route, {
|
||||
unsupported: ["speakers"],
|
||||
body: { from: fromRequest },
|
||||
frames: (bytes, context) =>
|
||||
streamsEvents(context.request)
|
||||
? Framing.sse.frame(bytes)
|
||||
: Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))),
|
||||
initial: () => ({ segments: [] }),
|
||||
step,
|
||||
finish,
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
TranscriptionModel.fromRoute<OpenAITranscriptionOptions, Frame, State>(
|
||||
{ protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
|
||||
input,
|
||||
)
|
||||
|
||||
export const OpenAITranscription = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -0,0 +1,180 @@
|
||||
import { Duration, Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import type { Status } from "../generation.js"
|
||||
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type AIError } from "../schema/index.js"
|
||||
import { ProviderShared, optionalNull } from "./shared.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "replicate-images", name: "Replicate", provider: "replicate" })
|
||||
export const DEFAULT_BASE_URL = "https://api.replicate.com"
|
||||
const OUTPUT_RETENTION = Duration.hours(1)
|
||||
const MAX_DATA_URL_BYTES = 256 * 1024
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Model-defined `input` fields under the model's own names; `Media.Asset` values lower to URLs or data URLs. */
|
||||
export type ReplicateImageOptions = Record<string, unknown>
|
||||
|
||||
export type Request = ImageRequestFor<ReplicateImageOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Token and response schemas
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const Token = Schema.Struct({ id: Schema.String, getURL: Schema.String, cancelURL: Schema.String })
|
||||
export type Token = Schema.Schema.Type<typeof Token>
|
||||
|
||||
const Prediction = Schema.Struct({
|
||||
id: Schema.String,
|
||||
status: Schema.String,
|
||||
output: optionalNull(Schema.Unknown),
|
||||
error: optionalNull(Schema.Unknown),
|
||||
data_removed: optionalNull(Schema.Boolean),
|
||||
completed_at: optionalNull(Schema.String),
|
||||
metrics: optionalNull(Schema.Struct({ predict_time: optionalNull(Schema.Number) })),
|
||||
urls: Schema.Struct({ get: Schema.String, cancel: Schema.String }),
|
||||
})
|
||||
|
||||
const Output = Schema.Union([Schema.String, Schema.Array(Schema.String)])
|
||||
const isOutput = Schema.is(Output)
|
||||
|
||||
const STATUS = {
|
||||
starting: "queued",
|
||||
processing: "running",
|
||||
succeeded: "completed",
|
||||
failed: "failed",
|
||||
canceled: "cancelled",
|
||||
// The prediction hit its `Cancel-After` deadline before it started running.
|
||||
aborted: "expired",
|
||||
} as const satisfies Record<string, Status>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Official models run by `owner/name`; anything else (`owner/name:version` or a bare version id) is a pinned version.
|
||||
const isOfficial = (model: string) => /^[^/:]+\/[^/:]+$/.test(model)
|
||||
|
||||
const inlineSize = (source: Media.Source) => {
|
||||
if (source.type === "bytes") return source.data.byteLength
|
||||
if (source.type === "base64") return source.data.length * 0.75
|
||||
return 0
|
||||
}
|
||||
|
||||
const fileInput = (asset: Media.Asset) => {
|
||||
if (inlineSize(asset.source) > MAX_DATA_URL_BYTES)
|
||||
return Effect.fail(
|
||||
ProviderShared.invalidRequest(
|
||||
`${route.name} data URL inputs are limited to 256 KB; pass a larger file by https URL`,
|
||||
),
|
||||
)
|
||||
return ProviderShared.mediaReference(asset, undefined, route.name).pipe(Effect.map((reference) => reference.value))
|
||||
}
|
||||
|
||||
const inputValue = (value: unknown): Effect.Effect<unknown, AIError> => {
|
||||
if (value instanceof Media.Asset) return fileInput(value)
|
||||
if (Array.isArray(value) && value.some((item) => item instanceof Media.Asset))
|
||||
return Effect.forEach(value, (item) => (item instanceof Media.Asset ? fileInput(item) : Effect.succeed(item)))
|
||||
return Effect.succeed(value)
|
||||
}
|
||||
|
||||
const fromRequest = Effect.fn("ReplicateImages.fromRequest")(function* (request: Request) {
|
||||
const native = yield* Effect.forEach(Object.entries(request.providerOptions ?? {}), ([key, value]) =>
|
||||
inputValue(value).pipe(Effect.map((lowered) => [key, lowered] as const)),
|
||||
)
|
||||
const input = mergeJsonRecords({ prompt: request.prompt }, Object.fromEntries(native))
|
||||
const model = request.model.id
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(isOfficial(model) ? { input } : { version: model, input }, request.http?.body) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodePrediction = route.decodeJson(Prediction)
|
||||
|
||||
const decodeStart = Effect.fn("ReplicateImages.decodeStart")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const output = yield* decodePrediction(response)
|
||||
const prediction = output.value
|
||||
return {
|
||||
token: { id: prediction.id, getURL: prediction.urls.get, cancelURL: prediction.urls.cancel },
|
||||
// `Prefer: wait` can return an already-finished prediction, so `await` skips straight to the result.
|
||||
snapshot: { id: prediction.id, status: yield* MediaProtocol.status(STATUS, prediction.status, output) },
|
||||
}
|
||||
})
|
||||
|
||||
const decodeStatus = Effect.fn("ReplicateImages.decodeStatus")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
const output = yield* decodePrediction(response)
|
||||
return { id: context.token.id, status: yield* MediaProtocol.status(STATUS, output.value.status, output) }
|
||||
})
|
||||
|
||||
const decodeResult = Effect.fn("ReplicateImages.decodeResult")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
const output = yield* decodePrediction(response)
|
||||
const prediction = output.value
|
||||
const status = yield* MediaProtocol.status(STATUS, prediction.status, output)
|
||||
if (status === "failed" || status === "cancelled" || status === "expired")
|
||||
return yield* output.ended(
|
||||
status,
|
||||
`${route.name} prediction ${context.token.id} ${prediction.status}${typeof prediction.error === "string" ? `: ${prediction.error}` : ""}`,
|
||||
)
|
||||
if (status !== "completed") return yield* output.pending(context.token.id)
|
||||
if (prediction.data_removed === true)
|
||||
return yield* output.ended("expired", `${route.name} removed the output of prediction ${context.token.id}`)
|
||||
if (!isOutput(prediction.output))
|
||||
return yield* output.invalid(`${route.name} prediction ${context.token.id} returned output that is not image URLs`)
|
||||
const urls = typeof prediction.output === "string" ? [prediction.output] : prediction.output
|
||||
if (urls.length === 0) return yield* output.invalid(`${route.name} prediction ${context.token.id} returned no images`)
|
||||
const predictTime = prediction.metrics?.predict_time ?? undefined
|
||||
const completedAt = prediction.completed_at ?? undefined
|
||||
const expiresAt =
|
||||
completedAt === undefined ? undefined : Date.parse(completedAt) + Duration.toMillis(OUTPUT_RETENTION)
|
||||
return new ImageResponse({
|
||||
images: urls.map((url) => Media.url(url, { expiresAt })),
|
||||
usage: predictTime === undefined ? undefined : { type: "compute", seconds: predictTime },
|
||||
providerMetadata: { replicate: { predictionId: prediction.id } },
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.queued<Request, ImageResponse, Token>(route, {
|
||||
token: Token,
|
||||
unsupported: ["images", "mask", "n", "size", "aspectRatio", "seed", "format"],
|
||||
start: { body: { from: fromRequest }, decode: decodeStart },
|
||||
status: { path: (token) => token.getURL, decode: decodeStatus },
|
||||
result: { path: (token) => token.getURL, decode: decodeResult },
|
||||
cancel: { method: "POST", path: (token) => token.cancelURL },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<ReplicateImageOptions, Token>(
|
||||
{
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) =>
|
||||
isOfficial(request.model.id) ? `/v1/models/${request.model.id}/predictions` : "/v1/predictions",
|
||||
},
|
||||
input,
|
||||
)
|
||||
|
||||
export const ReplicateImages = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -0,0 +1,196 @@
|
||||
import { Duration, Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import type { Status } from "../generation.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
|
||||
import { ProviderShared, optionalArray, optionalNull } from "./shared.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "runway-video", name: "Runway", provider: "runway" })
|
||||
export const DEFAULT_BASE_URL = "https://api.dev.runwayml.com/v1"
|
||||
/** Every Runway request must pin the API version. */
|
||||
export const API_VERSION = "2024-11-06"
|
||||
export const TEXT_TO_VIDEO_PATH = "/text_to_video"
|
||||
export const IMAGE_TO_VIDEO_PATH = "/image_to_video"
|
||||
export const VIDEO_TO_VIDEO_PATH = "/video_to_video"
|
||||
export const TASKS_PATH = "/tasks"
|
||||
/** Output URLs are valid for 24–48 hours; the asset carries the conservative bound. */
|
||||
const OUTPUT_RETENTION = Duration.hours(24)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Provider-native options. Common fields lower to Runway's names: `aspectRatio` → `ratio` (Runway expects pixel
|
||||
* ratios such as `1280:720` for most models), `durationSeconds` → `duration`, `audio`, `negativePrompt`,
|
||||
* `resolution`, `references`, and `frames` → `promptImage`.
|
||||
*/
|
||||
export type RunwayVideoOptions = {
|
||||
readonly contentModeration?: { readonly publicFigureThreshold?: OpenString<"auto" | "low"> }
|
||||
readonly outputFormat?: OpenString<"mp4" | "prores" | "png_sequence">
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = VideoRequestFor<RunwayVideoOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Token and response schemas
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const Token = Schema.Struct({ taskID: Schema.String })
|
||||
export type Token = Schema.Schema.Type<typeof Token>
|
||||
|
||||
const Cost = Schema.Struct({ credits: Schema.Number })
|
||||
|
||||
const StartResponse = Schema.Struct({ id: Schema.String })
|
||||
|
||||
const Task = Schema.Struct({
|
||||
status: Schema.String,
|
||||
progress: optionalNull(Schema.Number),
|
||||
output: optionalArray(Schema.String),
|
||||
failure: optionalNull(Schema.String),
|
||||
failureCode: optionalNull(Schema.String),
|
||||
cost: Schema.optional(Cost),
|
||||
estimatedCost: Schema.optional(Cost),
|
||||
})
|
||||
|
||||
const STATUS = {
|
||||
PENDING: "queued",
|
||||
THROTTLED: "queued",
|
||||
RUNNING: "running",
|
||||
SUCCEEDED: "completed",
|
||||
FAILED: "failed",
|
||||
CANCELLED: "cancelled",
|
||||
} as const satisfies Record<string, Status>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Runway accepts HTTPS URLs, `runway://` upload URIs, and data URIs, all as one string.
|
||||
const mediaUri = (asset: Media.Asset) =>
|
||||
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(Effect.map((reference) => reference.value))
|
||||
|
||||
const fromRequest = Effect.fn("RunwayVideo.fromRequest")(function* (request: Request) {
|
||||
const first = request.frames?.first === undefined ? undefined : yield* mediaUri(request.frames.first)
|
||||
const last = request.frames?.last === undefined ? undefined : yield* mediaUri(request.frames.last)
|
||||
const promptImage = [
|
||||
...(first === undefined ? [] : [{ uri: first, position: "first" }]),
|
||||
...(last === undefined ? [] : [{ uri: last, position: "last" }]),
|
||||
]
|
||||
const videoUri = request.video === undefined ? undefined : yield* mediaUri(request.video)
|
||||
const references = yield* Effect.forEach(request.references ?? [], (asset) =>
|
||||
mediaUri(asset).pipe(Effect.map((uri) => ({ uri }))),
|
||||
)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
model: request.model.id,
|
||||
promptText: request.prompt,
|
||||
promptImage: promptImage.length === 0 ? undefined : promptImage,
|
||||
videoUri,
|
||||
references: references.length === 0 ? undefined : references,
|
||||
ratio: request.aspectRatio,
|
||||
duration: request.durationSeconds,
|
||||
resolution: request.resolution,
|
||||
audio: request.audio,
|
||||
negativePrompt: request.negativePrompt,
|
||||
seed: request.seed,
|
||||
},
|
||||
request.providerOptions,
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
|
||||
token: { taskID: value.id },
|
||||
snapshot: { id: value.id, status: "queued" },
|
||||
}))
|
||||
|
||||
const decodeTask = route.decodeJson(Task)
|
||||
|
||||
const decodeStatus = Effect.fn("RunwayVideo.decodeStatus")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
const output = yield* decodeTask(response)
|
||||
const status = yield* MediaProtocol.status(STATUS, output.value.status, output)
|
||||
return { id: context.token.taskID, status, progress: output.value.progress ?? undefined }
|
||||
})
|
||||
|
||||
const decodeResult = Effect.fn("RunwayVideo.decodeResult")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
const output = yield* decodeTask(response)
|
||||
const task = output.value
|
||||
const status = yield* MediaProtocol.status(STATUS, task.status, output)
|
||||
if (status === "failed") {
|
||||
const code = task.failureCode ?? undefined
|
||||
const message = `${route.name} task failed${code === undefined ? "" : ` (${code})`}${task.failure ? `: ${task.failure}` : ""}`
|
||||
// Runway failure codes are dotted paths; every moderation outcome carries a SAFETY segment.
|
||||
if (code !== undefined && /(^|\.)SAFETY(\.|$)/.test(code)) return yield* output.contentPolicy(message)
|
||||
return yield* output.ended("failed", message)
|
||||
}
|
||||
if (status === "cancelled")
|
||||
return yield* output.ended("cancelled", `${route.name} task ${context.token.taskID} was cancelled`)
|
||||
if (status !== "completed") return yield* output.pending(context.token.taskID)
|
||||
const urls = task.output ?? []
|
||||
if (urls.length === 0) return yield* output.invalid(`${route.name} task succeeded without any output`)
|
||||
return new VideoResponse({
|
||||
videos: yield* Effect.forEach(urls, (url) =>
|
||||
MediaProtocol.expiringUrl(url, OUTPUT_RETENTION, { mediaType: "video/mp4" }),
|
||||
),
|
||||
usage: task.cost === undefined ? undefined : { type: "credits", credits: task.cost.credits },
|
||||
providerMetadata: {
|
||||
runway: {
|
||||
taskId: context.token.taskID,
|
||||
estimatedCredits: task.estimatedCost?.credits,
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const taskPath = (token: Token) => `${TASKS_PATH}/${token.taskID}`
|
||||
|
||||
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>(route, {
|
||||
token: Token,
|
||||
unsupported: ["n"],
|
||||
start: { body: { from: fromRequest }, decode: decodeStart },
|
||||
status: { path: taskPath, decode: decodeStatus },
|
||||
result: { path: taskPath, decode: decodeResult },
|
||||
cancel: { method: "DELETE", path: taskPath, activeOnly: true },
|
||||
})
|
||||
|
||||
const startPath = (request: Request) => {
|
||||
if (request.video !== undefined) return VIDEO_TO_VIDEO_PATH
|
||||
if (request.frames?.first !== undefined || request.frames?.last !== undefined) return IMAGE_TO_VIDEO_PATH
|
||||
return TEXT_TO_VIDEO_PATH
|
||||
}
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
VideoModel.fromRoute<RunwayVideoOptions, Token>(
|
||||
{
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
headers: { "X-Runway-Version": API_VERSION },
|
||||
path: ({ request }) => startPath(request),
|
||||
},
|
||||
input,
|
||||
)
|
||||
|
||||
export const RunwayVideo = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -1,6 +1,5 @@
|
||||
import { Tool } from "@opencode/schema/tool"
|
||||
import { Effect, Option, Schema, Stream } from "effect"
|
||||
import * as Sse from "effect/unstable/encoding/Sse"
|
||||
import { Effect, Option, Schema } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import { Media } from "../media.js"
|
||||
import {
|
||||
@@ -13,17 +12,16 @@ import {
|
||||
ToolDefinition,
|
||||
type ContentPart,
|
||||
type MediaPart,
|
||||
type OpenString,
|
||||
type ProviderID,
|
||||
type TextPart,
|
||||
type ToolEntry,
|
||||
type ToolResultPart,
|
||||
} from "../schema/index.js"
|
||||
import { Json, decodeJson, encodeJson } from "../utils/json.js"
|
||||
import { isRecord } from "../utils/record.js"
|
||||
export { isRecord }
|
||||
export { Json, decodeJson, encodeJson, isRecord }
|
||||
|
||||
export const Json = Schema.fromJsonString(Schema.Unknown)
|
||||
export const decodeJson = Schema.decodeUnknownSync(Json)
|
||||
export const encodeJson = Schema.encodeSync(Json)
|
||||
const isJson = Schema.is(Schema.Json)
|
||||
export const JsonObject = Schema.Record(Schema.String, Schema.Unknown)
|
||||
export const optionalArray = <const S extends Schema.Top>(schema: S) => Schema.optional(Schema.Array(schema))
|
||||
@@ -35,7 +33,7 @@ export const lenient = <const S extends Schema.Top>(schema: S) =>
|
||||
)
|
||||
/** Provider-defined string enum: known values for autocomplete, any string accepted at runtime. */
|
||||
export const knownString = <Known extends string>() =>
|
||||
Schema.declare<Known | (string & {})>((value): value is Known | (string & {}) => typeof value === "string", {
|
||||
Schema.declare<OpenString<Known>>((value): value is OpenString<Known> => typeof value === "string", {
|
||||
expected: "string",
|
||||
})
|
||||
|
||||
@@ -112,6 +110,14 @@ export const sumTokens = (...values: ReadonlyArray<number | undefined>): number
|
||||
return values.reduce((acc: number, value) => acc + (value ?? 0), 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Caps an explicit thinking budget at half the output limit. Thinking counts against the output limit, so a budget
|
||||
* near it leaves the answer, a tool call, or a summary without room. Smaller budgets, special values such as `-1` and
|
||||
* `0`, and requests without an output limit pass through unchanged.
|
||||
*/
|
||||
export const fitThinkingBudget = (budget: number, maxTokens: number | undefined, minimum = 1) =>
|
||||
maxTokens === undefined || budget <= maxTokens / 2 ? budget : Math.max(minimum, Math.floor(maxTokens / 2))
|
||||
|
||||
export const eventError = (route: string, message: string, body?: string, cause?: unknown) =>
|
||||
new AIError({
|
||||
reason: new InvalidProviderOutputError({ route, message, body, cause }),
|
||||
@@ -192,6 +198,29 @@ export const inlineRequired = (route: string, asset: Media.Asset) =>
|
||||
/** The remote URL of a `url` asset, for protocols that accept `http(s)` references natively. */
|
||||
export const mediaUrl = (asset: Media.Asset) => (asset.source.type === "url" ? asset.source.url : undefined)
|
||||
|
||||
export type MediaReference = { readonly type: "dataUrl" | "url" | "ref"; readonly value: string }
|
||||
|
||||
/**
|
||||
* The one string a provider can address an asset by: inline payloads as a data URL, `url` sources as their URL, and
|
||||
* this provider's own `ref` as its id. Other providers' refs are never forwarded and fail typed; omit `provider` for
|
||||
* APIs with no file handles at all.
|
||||
*/
|
||||
export const mediaReference = (
|
||||
asset: Media.Asset,
|
||||
provider: ProviderID | undefined,
|
||||
label: string,
|
||||
): Effect.Effect<MediaReference, AIError> => {
|
||||
const inline = asset.inline()
|
||||
if (inline) return Effect.succeed({ type: "dataUrl", value: inline.dataUrl })
|
||||
const url = mediaUrl(asset)
|
||||
if (url) return Effect.succeed({ type: "url", value: url })
|
||||
if (provider !== undefined && asset.source.type === "ref" && asset.source.provider === provider)
|
||||
return Effect.succeed({ type: "ref", value: asset.source.id })
|
||||
const accepted = provider === undefined ? "" : `, and ${provider} references`
|
||||
const got = asset.source.type === "ref" ? `; got ${asset.source.provider}:${asset.source.id}` : ""
|
||||
return Effect.fail(invalidRequest(`${label} accepts inline bytes, data URLs, http(s) URLs${accepted}${got}`))
|
||||
}
|
||||
|
||||
/**
|
||||
* Lift a tool-result file into a `MediaPart`. Tool files carry either a data URL, an `http(s)` URL, or raw base64 in
|
||||
* `uri`; the declared `mime` wins over any data-URL prefix so tool authors control the type the model sees.
|
||||
@@ -206,8 +235,6 @@ export const toolFileMedia = (item: Tool.FileContent): MediaPart => {
|
||||
return Message.media(asset, { filename: item.name })
|
||||
}
|
||||
|
||||
export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
|
||||
|
||||
export const toolResultText = (part: ToolResultPart) => {
|
||||
if (part.result.type === "text") return String(part.result.value)
|
||||
if (part.result.type === "error") {
|
||||
@@ -229,47 +256,6 @@ export const errorText = (error: unknown) => {
|
||||
return "Unknown stream error"
|
||||
}
|
||||
|
||||
/**
|
||||
* `framing` step for Server-Sent Events. Decodes UTF-8, runs the SSE channel
|
||||
* decoder, optionally filters named events, and drops empty events. `[DONE]`
|
||||
* is dropped by default or retained for protocols that use it as their stream
|
||||
* boundary. Retry control events are ignored without interrupting the stream.
|
||||
* Decoder failures become provider output errors so the public error channel
|
||||
* stays `AIError`.
|
||||
*/
|
||||
export const sseFraming = (
|
||||
bytes: Stream.Stream<Uint8Array, AIError>,
|
||||
events?: ReadonlySet<string>,
|
||||
includeDone = false,
|
||||
): Stream.Stream<string, AIError> =>
|
||||
bytes.pipe(
|
||||
Stream.decodeText(),
|
||||
Stream.mapAccumEffect(
|
||||
() => {
|
||||
const output: Sse.Event[] = []
|
||||
return {
|
||||
output,
|
||||
parser: Sse.makeParser((event) => {
|
||||
if (event._tag === "Event") output.push(event)
|
||||
}),
|
||||
}
|
||||
},
|
||||
(state, chunk) =>
|
||||
Effect.gen(function* () {
|
||||
const error = state.parser.feed(chunk)
|
||||
if (error) return yield* eventError("sse", error.message, chunk, error)
|
||||
return [state, state.output.splice(0)] as const
|
||||
}),
|
||||
),
|
||||
Stream.filter(
|
||||
(event) =>
|
||||
(events === undefined || events.has(event.event)) &&
|
||||
event.data.length > 0 &&
|
||||
(event.data !== "[DONE]" || includeDone || (events !== undefined && event.event !== "message")),
|
||||
),
|
||||
Stream.map((event) => event.data),
|
||||
)
|
||||
|
||||
/**
|
||||
* Canonical invalid-request constructor shared by protocol lowering.
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { ProviderShared } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "stability-images", name: "Stability AI", provider: "stability" })
|
||||
const upscaleRoute = MediaProtocol.identity({ id: "stability-upscale", name: "Stability AI", provider: "stability" })
|
||||
export const DEFAULT_BASE_URL = "https://api.stability.ai"
|
||||
const RESULTS_PATH = "/v2beta/results"
|
||||
const UPSCALE_MODEL = "creative"
|
||||
/** Base64 JSON instead of raw bytes, so the seed and finish reason arrive with the image. */
|
||||
const HEADERS = { accept: "application/json" }
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type StabilityStylePreset = OpenString<
|
||||
| "enhance"
|
||||
| "anime"
|
||||
| "photographic"
|
||||
| "digital-art"
|
||||
| "comic-book"
|
||||
| "fantasy-art"
|
||||
| "line-art"
|
||||
| "analog-film"
|
||||
| "neon-punk"
|
||||
| "isometric"
|
||||
| "low-poly"
|
||||
| "origami"
|
||||
| "modeling-compound"
|
||||
| "cinematic"
|
||||
| "3d-model"
|
||||
| "pixel-art"
|
||||
| "tile-texture"
|
||||
>
|
||||
|
||||
export type StabilityImageOptions = {
|
||||
readonly negative_prompt?: string
|
||||
readonly style_preset?: StabilityStylePreset
|
||||
readonly strength?: number
|
||||
readonly cfg_scale?: number
|
||||
} & Record<string, unknown>
|
||||
|
||||
/** Creative upscale options; the one `images` source is the image to upscale. */
|
||||
export type StabilityUpscaleOptions = {
|
||||
readonly negative_prompt?: string
|
||||
readonly style_preset?: StabilityStylePreset
|
||||
readonly creativity?: number
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = ImageRequestFor<StabilityImageOptions>
|
||||
export type UpscaleRequest = ImageRequestFor<StabilityUpscaleOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Token and response schemas
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const Token = Schema.Struct({ id: Schema.String })
|
||||
export type Token = Schema.Schema.Type<typeof Token>
|
||||
|
||||
// Generate names the base64 image `image`; `/v2beta/results/{id}` names it `result`.
|
||||
const ImageDocument = Schema.Struct({
|
||||
image: Schema.optional(Schema.String),
|
||||
result: Schema.optional(Schema.String),
|
||||
seed: Schema.optional(Schema.Number),
|
||||
finish_reason: Schema.optional(Schema.String),
|
||||
})
|
||||
|
||||
const Started = Schema.Struct({ id: Schema.String })
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** `sd3.5-*` ids share the `sd3` endpoint and select the model with a form field. */
|
||||
const endpoint = (model: string) => (model.startsWith("sd3") ? "sd3" : model)
|
||||
|
||||
const RESERVED_FORM_FIELDS = new Set(["image", "prompt", "mode", "model"])
|
||||
|
||||
const form = Effect.fn("StabilityImages.form")(function* (
|
||||
identity: MediaProtocol.Identity,
|
||||
fields: Record<string, unknown>,
|
||||
native: Record<string, unknown> | undefined,
|
||||
source: Request["images"],
|
||||
) {
|
||||
if ((source?.length ?? 0) > 1)
|
||||
return yield* identity.unsupported("media.images", `${identity.name} takes one source image`)
|
||||
const body = new FormData()
|
||||
MediaInput.appendFields(body, fields, { overlay: native, reserved: RESERVED_FORM_FIELDS })
|
||||
const image = source?.[0]
|
||||
if (image !== undefined)
|
||||
body.append("image", MediaInput.blob(yield* MediaInput.inlineBytes(identity.id, image), image.mediaType), "image")
|
||||
return MediaProtocol.multipart(body)
|
||||
})
|
||||
|
||||
const fromRequest = Effect.fn("StabilityImages.fromRequest")(function* (request: Request) {
|
||||
if (request.n !== undefined && request.n > 1)
|
||||
return yield* route.unsupported("media.n", `${route.name} generates one image per request; call it once per image`)
|
||||
const target = endpoint(request.model.id)
|
||||
const edit = (request.images?.length ?? 0) > 0
|
||||
if (edit && target === "core")
|
||||
return yield* route.unsupported("media.images", `${route.name} core is text-to-image only; use ultra or sd3.5-*`)
|
||||
return yield* form(
|
||||
route,
|
||||
{
|
||||
prompt: request.prompt,
|
||||
aspect_ratio: request.aspectRatio,
|
||||
seed: request.seed,
|
||||
output_format: request.format,
|
||||
model: target === "sd3" ? request.model.id : undefined,
|
||||
mode: target === "sd3" && edit ? "image-to-image" : undefined,
|
||||
},
|
||||
mergeJsonRecords(request.providerOptions, request.http?.body),
|
||||
request.images,
|
||||
)
|
||||
})
|
||||
|
||||
const fromUpscaleRequest = Effect.fn("StabilityImages.fromUpscaleRequest")(function* (request: UpscaleRequest) {
|
||||
if ((request.images?.length ?? 0) === 0)
|
||||
return yield* ProviderShared.invalidRequest(`${upscaleRoute.name} upscale requires the source image in images`)
|
||||
return yield* form(
|
||||
upscaleRoute,
|
||||
{ prompt: request.prompt, seed: request.seed, output_format: request.format },
|
||||
mergeJsonRecords(request.providerOptions, request.http?.body),
|
||||
request.images,
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeImageDocument = (identity: MediaProtocol.Identity) => {
|
||||
const decode = identity.decodeJson(ImageDocument)
|
||||
return Effect.fn("StabilityImages.decodeImage")(function* (response: HttpClientResponse.HttpClientResponse) {
|
||||
const output = yield* decode(response)
|
||||
const document = output.value
|
||||
const data = document.image ?? document.result
|
||||
if (data === undefined) return yield* output.invalid(`${identity.name} returned no image`)
|
||||
const image = yield* MediaInput.decodedAsset(output.invalid, `${identity.name} result`, data, undefined)
|
||||
return new ImageResponse({
|
||||
images: [image],
|
||||
notices:
|
||||
document.finish_reason === "CONTENT_FILTERED"
|
||||
? [{ type: "moderated", message: `${identity.name} blurred the image for violating its content policy` }]
|
||||
: undefined,
|
||||
providerMetadata: { stability: { seed: document.seed, finishReason: document.finish_reason } },
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
const decodeResponse = decodeImageDocument(route)
|
||||
const decodeUpscaleImage = decodeImageDocument(upscaleRoute)
|
||||
|
||||
const decodeStart = upscaleRoute.decodeStarted(Started, (value) => ({
|
||||
token: { id: value.id },
|
||||
snapshot: { id: value.id, status: "queued" },
|
||||
}))
|
||||
|
||||
// `/v2beta/results/{id}` answers 202 while in progress and 200 with the finished image document.
|
||||
const decodeStatus = (response: HttpClientResponse.HttpClientResponse, context: MediaProtocol.PollContext<Token>) =>
|
||||
Effect.succeed({
|
||||
id: context.token.id,
|
||||
status: response.status === 202 ? ("running" as const) : ("completed" as const),
|
||||
})
|
||||
|
||||
const decodeUpscaleResult = Effect.fn("StabilityImages.decodeUpscaleResult")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
if (response.status === 202) {
|
||||
const output = yield* upscaleRoute.text(response)
|
||||
return yield* output.pending(context.token.id)
|
||||
}
|
||||
return yield* decodeUpscaleImage(response)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
|
||||
unsupported: ["size", "mask"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
})
|
||||
|
||||
export const upscaleProtocol = MediaProtocol.queued<UpscaleRequest, ImageResponse, Token>(upscaleRoute, {
|
||||
token: Token,
|
||||
unsupported: ["n", "size", "aspectRatio", "mask"],
|
||||
start: { body: { from: fromUpscaleRequest }, decode: decodeStart },
|
||||
status: { path: (token) => `${RESULTS_PATH}/${token.id}`, decode: decodeStatus },
|
||||
result: { path: (token) => `${RESULTS_PATH}/${token.id}`, decode: decodeUpscaleResult },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<StabilityImageOptions>(
|
||||
{
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
headers: HEADERS,
|
||||
path: ({ request }) => `/v2beta/stable-image/generate/${endpoint(request.model.id)}`,
|
||||
},
|
||||
input,
|
||||
)
|
||||
|
||||
export const upscaleModel = (input: Omit<MediaRoute.ModelInput, "id">) =>
|
||||
ImageModel.fromRoute<StabilityUpscaleOptions, Token>(
|
||||
{
|
||||
protocol: upscaleProtocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
headers: HEADERS,
|
||||
// Only the creative upscaler is asynchronous; fast and conservative answer inline.
|
||||
path: `/v2beta/stable-image/upscale/${UPSCALE_MODEL}`,
|
||||
},
|
||||
{ ...input, id: UPSCALE_MODEL },
|
||||
)
|
||||
|
||||
export const StabilityImages = {
|
||||
protocol,
|
||||
upscaleProtocol,
|
||||
model,
|
||||
upscaleModel,
|
||||
} as const
|
||||
@@ -0,0 +1,91 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import type { Status } from "../../generation.js"
|
||||
import type { Media } from "../../media.js"
|
||||
import { MediaProtocol } from "../../route/media-protocol.js"
|
||||
import type { AIError } from "../../schema/index.js"
|
||||
import { ProviderShared, optionalNull } from "../shared.js"
|
||||
|
||||
export const DEFAULT_BASE_URL = "https://queue.fal.run"
|
||||
|
||||
/** fal hands back absolute follow-up URLs on submit; they are authoritative for status, result, and cancel. */
|
||||
export const Token = Schema.Struct({
|
||||
requestID: Schema.String,
|
||||
statusURL: Schema.String,
|
||||
responseURL: Schema.String,
|
||||
cancelURL: Schema.String,
|
||||
})
|
||||
export type Token = Schema.Schema.Type<typeof Token>
|
||||
|
||||
const StartResponse = Schema.Struct({
|
||||
request_id: Schema.String,
|
||||
status_url: Schema.String,
|
||||
response_url: Schema.String,
|
||||
cancel_url: Schema.String,
|
||||
queue_position: optionalNull(Schema.Number),
|
||||
})
|
||||
|
||||
const QueueStatus = Schema.Struct({
|
||||
status: Schema.String,
|
||||
queue_position: optionalNull(Schema.Number),
|
||||
error: optionalNull(Schema.Unknown),
|
||||
})
|
||||
|
||||
const STATUS = {
|
||||
IN_QUEUE: "queued",
|
||||
IN_PROGRESS: "running",
|
||||
COMPLETED: "completed",
|
||||
} as const satisfies Record<string, Status>
|
||||
|
||||
// fal accepts public URLs and data URIs; there is no provider file handle to forward.
|
||||
export const mediaUrl = (asset: Media.Asset, name: string) =>
|
||||
ProviderShared.mediaReference(asset, undefined, name).pipe(Effect.map((reference) => reference.value))
|
||||
|
||||
export const protocol = <Request, Response>(
|
||||
route: MediaProtocol.Identity,
|
||||
input: {
|
||||
readonly unsupported?: ReadonlyArray<keyof Request & string>
|
||||
readonly from: (request: Request) => Effect.Effect<MediaProtocol.Body, AIError>
|
||||
readonly decodeResult: (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) => Effect.Effect<Response, AIError>
|
||||
},
|
||||
) => {
|
||||
const decodeQueueStatus = route.decodeJson(QueueStatus)
|
||||
return MediaProtocol.queued<Request, Response, Token>(route, {
|
||||
token: Token,
|
||||
unsupported: input.unsupported,
|
||||
start: {
|
||||
body: { from: input.from },
|
||||
decode: route.decodeStarted(StartResponse, (value) => ({
|
||||
token: {
|
||||
requestID: value.request_id,
|
||||
statusURL: value.status_url,
|
||||
responseURL: value.response_url,
|
||||
cancelURL: value.cancel_url,
|
||||
},
|
||||
snapshot: { id: value.request_id, status: "queued", position: value.queue_position ?? undefined },
|
||||
})),
|
||||
},
|
||||
status: {
|
||||
path: (token) => token.statusURL,
|
||||
decode: Effect.fn("FalQueue.decodeStatus")(function* (response, context) {
|
||||
const output = yield* decodeQueueStatus(response)
|
||||
const decoded = output.value
|
||||
const status = yield* MediaProtocol.status(STATUS, decoded.status, output)
|
||||
// fal reports request failures as COMPLETED with an `error`; the response endpoint carries the details.
|
||||
const failed = status === "completed" && decoded.error !== undefined && decoded.error !== null
|
||||
return {
|
||||
id: context.token.requestID,
|
||||
status: failed ? "failed" : status,
|
||||
position: status === "queued" ? (decoded.queue_position ?? undefined) : undefined,
|
||||
}
|
||||
}),
|
||||
},
|
||||
result: { path: (token) => token.responseURL, decode: input.decodeResult },
|
||||
cancel: { method: "PUT", path: (token) => token.cancelURL },
|
||||
})
|
||||
}
|
||||
|
||||
export * as FalQueue from "./fal-queue.js"
|
||||
@@ -0,0 +1,113 @@
|
||||
import { Effect, Schema, type Stream } from "effect"
|
||||
import type { Media } from "../../media.js"
|
||||
import { Framing } from "../../route/framing.js"
|
||||
import type { MediaProtocol } from "../../route/media-protocol.js"
|
||||
import { AIError, ContentPolicyError, ProviderID, type MediaUsage, type ProviderMetadata } from "../../schema/index.js"
|
||||
import { ProviderShared } from "../shared.js"
|
||||
import { MediaInput } from "./media-input.js"
|
||||
|
||||
const PROVIDER = ProviderID.make("google")
|
||||
|
||||
const UsageMetadata = Schema.Struct({
|
||||
promptTokenCount: Schema.optional(Schema.Number),
|
||||
candidatesTokenCount: Schema.optional(Schema.Number),
|
||||
totalTokenCount: Schema.optional(Schema.Number),
|
||||
})
|
||||
type UsageMetadata = Schema.Schema.Type<typeof UsageMetadata>
|
||||
|
||||
export const chunk = <const Part extends Schema.Top>(part: Part) =>
|
||||
Schema.Struct({
|
||||
candidates: Schema.optional(
|
||||
Schema.Array(
|
||||
Schema.Struct({
|
||||
content: Schema.optional(Schema.Struct({ parts: Schema.optional(Schema.Array(part)) })),
|
||||
finishReason: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
),
|
||||
promptFeedback: Schema.optional(
|
||||
Schema.Struct({
|
||||
blockReason: Schema.optional(Schema.String),
|
||||
blockReasonMessage: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
usageMetadata: Schema.optional(UsageMetadata),
|
||||
modelVersion: Schema.optional(Schema.String),
|
||||
responseId: Schema.optional(Schema.String),
|
||||
})
|
||||
|
||||
interface Chunk {
|
||||
readonly candidates?: ReadonlyArray<{ readonly finishReason?: string }>
|
||||
readonly promptFeedback?: { readonly blockReason?: string; readonly blockReasonMessage?: string }
|
||||
readonly usageMetadata?: UsageMetadata
|
||||
readonly modelVersion?: string
|
||||
readonly responseId?: string
|
||||
}
|
||||
|
||||
export interface Metadata {
|
||||
readonly usage?: UsageMetadata
|
||||
readonly finishReason?: string
|
||||
readonly modelVersion?: string
|
||||
readonly responseId?: string
|
||||
}
|
||||
|
||||
export const track = <State extends Metadata>(state: State, chunk: Chunk): State => ({
|
||||
...state,
|
||||
usage: chunk.usageMetadata ?? state.usage,
|
||||
finishReason: chunk.candidates?.[0]?.finishReason ?? state.finishReason,
|
||||
modelVersion: chunk.modelVersion ?? state.modelVersion,
|
||||
responseId: chunk.responseId ?? state.responseId,
|
||||
})
|
||||
|
||||
export const blocked = (name: string, chunk: Chunk, frame: string) => {
|
||||
const feedback = chunk.promptFeedback
|
||||
if (feedback?.blockReason === undefined) return undefined
|
||||
return new AIError({
|
||||
reason: new ContentPolicyError({
|
||||
message: `${name} blocked the request (${feedback.blockReason})${
|
||||
feedback.blockReasonMessage === undefined ? "" : `: ${feedback.blockReasonMessage}`
|
||||
}`,
|
||||
body: frame,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
export const usage = (usage: UsageMetadata | undefined): MediaUsage | undefined =>
|
||||
usage === undefined
|
||||
? undefined
|
||||
: {
|
||||
type: "tokens",
|
||||
input: usage.promptTokenCount,
|
||||
output: usage.candidatesTokenCount,
|
||||
total: ProviderShared.totalTokens(usage.promptTokenCount, usage.candidatesTokenCount, usage.totalTokenCount),
|
||||
details: { google: usage },
|
||||
}
|
||||
|
||||
export const providerMetadata = (state: Metadata): ProviderMetadata => ({
|
||||
google: { finishReason: state.finishReason, modelVersion: state.modelVersion, responseId: state.responseId },
|
||||
})
|
||||
|
||||
export const path = (model: string, mode: MediaProtocol.Mode) =>
|
||||
mode === "stream" ? `/models/${model}:streamGenerateContent?alt=sse` : `/models/${model}:generateContent`
|
||||
|
||||
// `generateContent` answers with one document shaped exactly like a streamed chunk, so it is a single frame.
|
||||
export const frames = (bytes: Stream.Stream<Uint8Array, AIError>, mode: MediaProtocol.Mode) =>
|
||||
mode === "stream" ? Framing.sse.frame(bytes) : Framing.document.frame(bytes)
|
||||
|
||||
// Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs.
|
||||
export const mediaPart = (
|
||||
route: string,
|
||||
asset: Media.Asset,
|
||||
): Effect.Effect<
|
||||
| { readonly fileData: { readonly mimeType: string; readonly fileUri: string } }
|
||||
| { readonly inlineData: { readonly mimeType: string; readonly data: string } },
|
||||
AIError
|
||||
> => {
|
||||
const fileUri = MediaInput.refID(asset, PROVIDER)
|
||||
if (fileUri !== undefined) return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri } })
|
||||
return ProviderShared.requireInlineMedia(route, asset).pipe(
|
||||
Effect.map((media) => ({ inlineData: { mimeType: media.mime, data: media.base64 } })),
|
||||
)
|
||||
}
|
||||
|
||||
export * as GeminiGenerateContent from "./gemini-generate-content.js"
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { JsonSchema } from "../../schema/index.js"
|
||||
import { isRecord } from "../../utils/record.js"
|
||||
|
||||
// Gemini's `parametersJsonSchema` accepts standard JSON Schema, but rejects a few shapes that
|
||||
// published tool schemas commonly contain. Rewrite only those and send everything else unchanged.
|
||||
const SCHEMA_MAPS = new Set([
|
||||
"properties",
|
||||
"patternProperties",
|
||||
"$defs",
|
||||
"definitions",
|
||||
"dependentSchemas",
|
||||
"dependencies",
|
||||
])
|
||||
const VALUES = new Set(["const", "default", "enum", "examples", "dependentRequired"])
|
||||
|
||||
const mapValues = (record: Record<string, unknown>, map: (value: unknown, key: string) => unknown) =>
|
||||
Object.fromEntries(Object.entries(record).map(([key, value]) => [key, map(value, key)]))
|
||||
|
||||
const normalizeNode = (schema: unknown): unknown => {
|
||||
if (Array.isArray(schema)) return schema.map(normalizeNode)
|
||||
if (!isRecord(schema)) return schema
|
||||
const properties = isRecord(schema.properties) ? schema.properties : undefined
|
||||
return Object.fromEntries(
|
||||
Object.entries(schema).flatMap(([key, value]) => {
|
||||
if (VALUES.has(key)) return [[key, value]]
|
||||
if (SCHEMA_MAPS.has(key) && isRecord(value)) return [[key, mapValues(value, normalizeNode)]]
|
||||
// `required` may only name declared properties.
|
||||
if (key === "required" && properties && Array.isArray(value))
|
||||
return [[key, value.filter((name) => typeof name === "string" && Object.hasOwn(properties, name))]]
|
||||
// Draft-04 boolean exclusive bounds become the numeric form.
|
||||
if (key === "exclusiveMinimum" && typeof value === "boolean")
|
||||
return value && typeof schema.minimum === "number" ? [[key, schema.minimum]] : []
|
||||
if (key === "exclusiveMaximum" && typeof value === "boolean")
|
||||
return value && typeof schema.maximum === "number" ? [[key, schema.maximum]] : []
|
||||
if (key === "minimum" && schema.exclusiveMinimum === true) return []
|
||||
if (key === "maximum" && schema.exclusiveMaximum === true) return []
|
||||
// Draft-07 tuples (`items` array plus `additionalItems`) are `prefixItems` plus `items` in 2020-12.
|
||||
if (key === "items" && Array.isArray(value)) return [["prefixItems", value.map(normalizeNode)]]
|
||||
if (key === "additionalItems" && Array.isArray(schema.items)) return [["items", normalizeNode(value)]]
|
||||
return [[key, normalizeNode(value)]]
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// Gemini accepts a recursive `$ref` only when the loop passes through an optional property or
|
||||
// potentially empty array `items`. Replace other self-references with an unconstrained schema.
|
||||
const cutLoops = (schema: unknown, target: string, safe: boolean): unknown => {
|
||||
if (Array.isArray(schema)) return schema.map((item) => cutLoops(item, target, safe))
|
||||
if (!isRecord(schema)) return schema
|
||||
if (schema.$ref === target && !safe) return {}
|
||||
const required = Array.isArray(schema.required) ? schema.required : []
|
||||
return mapValues(schema, (value, key) => {
|
||||
if (VALUES.has(key)) return value
|
||||
if (key === "items") return cutLoops(value, target, safe || !(Number(schema.minItems) > 0))
|
||||
if (key === "properties" && isRecord(value))
|
||||
return mapValues(value, (child, name) => cutLoops(child, target, safe || !required.includes(name)))
|
||||
if (SCHEMA_MAPS.has(key) && isRecord(value)) return mapValues(value, (child) => cutLoops(child, target, safe))
|
||||
return cutLoops(value, target, safe)
|
||||
})
|
||||
}
|
||||
|
||||
export const normalize = (schema: JsonSchema): JsonSchema => {
|
||||
const normalized = normalizeNode(schema)
|
||||
if (!isRecord(normalized)) return {}
|
||||
const result = cutLoops(
|
||||
mapValues(normalized, (value, key) =>
|
||||
(key === "$defs" || key === "definitions") && isRecord(value)
|
||||
? mapValues(value, (def, name) => cutLoops(def, `#/${key}/${name}`, false))
|
||||
: value,
|
||||
),
|
||||
"#",
|
||||
false,
|
||||
)
|
||||
return isRecord(result) ? result : {}
|
||||
}
|
||||
|
||||
export * as GeminiJsonSchema from "./gemini-json-schema.js"
|
||||
@@ -1,119 +0,0 @@
|
||||
import { isRecord } from "../../utils/record.js"
|
||||
|
||||
// Gemini accepts a JSON Schema-like dialect for tool parameters, but rejects a
|
||||
// handful of common JSON Schema shapes. Keep this projection isolated so the
|
||||
// Gemini protocol file still reads like the other protocol modules.
|
||||
const SCHEMA_INTENT_KEYS = [
|
||||
"type",
|
||||
"properties",
|
||||
"items",
|
||||
"prefixItems",
|
||||
"enum",
|
||||
"const",
|
||||
"$ref",
|
||||
"additionalProperties",
|
||||
"patternProperties",
|
||||
"required",
|
||||
"not",
|
||||
"if",
|
||||
"then",
|
||||
"else",
|
||||
]
|
||||
|
||||
const hasCombiner = (schema: unknown) =>
|
||||
isRecord(schema) && (Array.isArray(schema.anyOf) || Array.isArray(schema.oneOf) || Array.isArray(schema.allOf))
|
||||
|
||||
const hasSchemaIntent = (schema: unknown) =>
|
||||
isRecord(schema) && (hasCombiner(schema) || SCHEMA_INTENT_KEYS.some((key) => key in schema))
|
||||
|
||||
const sanitizeNode = (schema: unknown): unknown => {
|
||||
if (!isRecord(schema)) return Array.isArray(schema) ? schema.map(sanitizeNode) : schema
|
||||
|
||||
const result: Record<string, unknown> = Object.fromEntries(
|
||||
Object.entries(schema).map(([key, value]) => [
|
||||
key,
|
||||
key === "enum" && Array.isArray(value) ? value.map(String) : sanitizeNode(value),
|
||||
]),
|
||||
)
|
||||
|
||||
if (Array.isArray(result.enum) && (result.type === "integer" || result.type === "number")) result.type = "string"
|
||||
|
||||
const properties = result.properties
|
||||
if (result.type === "object" && isRecord(properties) && Array.isArray(result.required)) {
|
||||
result.required = result.required.filter((field) => typeof field === "string" && field in properties)
|
||||
}
|
||||
|
||||
if (result.type === "array" && !hasCombiner(result)) {
|
||||
result.items = result.items ?? {}
|
||||
if (isRecord(result.items) && !hasSchemaIntent(result.items)) result.items = { ...result.items, type: "string" }
|
||||
}
|
||||
|
||||
if (typeof result.type === "string" && result.type !== "object" && !hasCombiner(result)) {
|
||||
delete result.properties
|
||||
delete result.required
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
const emptyObjectSchema = (schema: Record<string, unknown>) =>
|
||||
schema.type === "object" &&
|
||||
(!isRecord(schema.properties) || Object.keys(schema.properties).length === 0) &&
|
||||
!schema.additionalProperties
|
||||
|
||||
const projectNode = (schema: unknown, nested = false): Record<string, unknown> | undefined => {
|
||||
if (!isRecord(schema)) return undefined
|
||||
if (!nested && emptyObjectSchema(schema)) return undefined
|
||||
const types = Array.isArray(schema.type) ? schema.type.filter((type) => type !== "null") : undefined
|
||||
const anyOf = Array.isArray(schema.anyOf) ? schema.anyOf : undefined
|
||||
const hasNullAnyOf = anyOf?.some((item) => isRecord(item) && item.type === "null") ?? false
|
||||
const anyOfTypes = hasNullAnyOf ? anyOf?.filter((item) => !isRecord(item) || item.type !== "null") : anyOf
|
||||
const flattenedAnyOf = hasNullAnyOf && anyOfTypes?.length === 1 ? projectNode(anyOfTypes[0], true) : undefined
|
||||
const result = Object.fromEntries(
|
||||
[
|
||||
["description", schema.description],
|
||||
["required", schema.required],
|
||||
["format", schema.format],
|
||||
["type", types ? (types.length === 0 ? "null" : undefined) : schema.type],
|
||||
[
|
||||
"nullable",
|
||||
(Array.isArray(schema.type) && schema.type.includes("null") && types && types.length > 0) || hasNullAnyOf
|
||||
? true
|
||||
: undefined,
|
||||
],
|
||||
["enum", schema.const !== undefined ? [schema.const] : schema.enum],
|
||||
[
|
||||
"properties",
|
||||
isRecord(schema.properties)
|
||||
? Object.fromEntries(Object.entries(schema.properties).map(([key, value]) => [key, projectNode(value, true)]))
|
||||
: undefined,
|
||||
],
|
||||
[
|
||||
"items",
|
||||
Array.isArray(schema.items)
|
||||
? schema.items.map((item) => projectNode(item, true))
|
||||
: schema.items === undefined
|
||||
? undefined
|
||||
: projectNode(schema.items, true),
|
||||
],
|
||||
["allOf", Array.isArray(schema.allOf) ? schema.allOf.map((item) => projectNode(item, true)) : undefined],
|
||||
[
|
||||
"anyOf",
|
||||
anyOfTypes
|
||||
? hasNullAnyOf && anyOfTypes.length === 1
|
||||
? undefined
|
||||
: anyOfTypes.map((item) => projectNode(item, true))
|
||||
: types && types.length > 0
|
||||
? types.map((type) => ({ type }))
|
||||
: undefined,
|
||||
],
|
||||
["oneOf", Array.isArray(schema.oneOf) ? schema.oneOf.map((item) => projectNode(item, true)) : undefined],
|
||||
["minLength", schema.minLength],
|
||||
].filter((entry) => entry[1] !== undefined),
|
||||
)
|
||||
return flattenedAnyOf ? { ...result, ...flattenedAnyOf } : result
|
||||
}
|
||||
|
||||
export const convert = (schema: unknown) => projectNode(sanitizeNode(schema))
|
||||
|
||||
export * as GeminiToolSchema from "./gemini-tool-schema.js"
|
||||
@@ -1,6 +1,8 @@
|
||||
import { Effect, Encoding } from "effect"
|
||||
import { Media } from "../../media.js"
|
||||
import type { AIError, ProviderID } from "../../schema/index.js"
|
||||
import type { MediaProtocol } from "../../route/media-protocol.js"
|
||||
import { mergeJsonRecords, type AIError, type ProviderID } from "../../schema/index.js"
|
||||
import { encodeJson } from "../../utils/json.js"
|
||||
import { ProviderShared } from "../shared.js"
|
||||
|
||||
/** Owned bytes for multipart uploads; decodes `base64` sources and rejects remote sources. */
|
||||
@@ -13,6 +15,31 @@ export const inlineBytes = (route: string, asset: Media.Asset): Effect.Effect<Ui
|
||||
)
|
||||
}
|
||||
|
||||
/** Copied because `BlobPart` requires a plain `ArrayBuffer`. */
|
||||
export const blob = (data: Uint8Array, mediaType: string) => {
|
||||
const buffer = new ArrayBuffer(data.byteLength)
|
||||
new Uint8Array(buffer).set(data)
|
||||
return new Blob([buffer], { type: mediaType })
|
||||
}
|
||||
|
||||
const isScalar = (value: unknown): value is string | number | boolean =>
|
||||
typeof value === "string" || typeof value === "number" || typeof value === "boolean"
|
||||
|
||||
export const query = (route: string, values: Record<string, unknown>): Effect.Effect<MediaProtocol.Query, AIError> => {
|
||||
const entries = Object.entries(values).filter(([, value]) => value !== undefined)
|
||||
const invalid = entries.find(([, value]) => !isScalar(value) && !(Array.isArray(value) && value.every(isScalar)))
|
||||
if (invalid !== undefined)
|
||||
return Effect.fail(ProviderShared.invalidRequest(`${route} cannot send "${invalid[0]}" as a query parameter`))
|
||||
return Effect.succeed(
|
||||
Object.fromEntries(entries.map(([key, value]) => [key, Array.isArray(value) ? value.map(String) : String(value)])),
|
||||
)
|
||||
}
|
||||
|
||||
export const dimensions = (size: string) => {
|
||||
const [width, height] = size.split("x").map(Number)
|
||||
return { width, height }
|
||||
}
|
||||
|
||||
/** Provider file handle when the ref belongs to this provider; refs from other providers are never forwarded. */
|
||||
export const refID = (asset: Media.Asset, provider: ProviderID) =>
|
||||
asset.source.type === "ref" && asset.source.provider === provider ? asset.source.id : undefined
|
||||
@@ -30,4 +57,38 @@ export const decodedAsset = (
|
||||
Effect.map((bytes) => Media.bytes(bytes, mediaType, options)),
|
||||
)
|
||||
|
||||
/** One image of an OpenAI-shaped `data` array, which carries either `b64_json` or a `url`. */
|
||||
export const imageOutput = (
|
||||
invalid: (message: string, cause?: unknown) => AIError,
|
||||
label: string,
|
||||
item: { readonly b64_json?: string | null; readonly url?: string | null },
|
||||
mediaType: string | undefined,
|
||||
options?: Media.AssetOptions,
|
||||
) => {
|
||||
if (item.b64_json) return decodedAsset(invalid, label, item.b64_json, mediaType, options)
|
||||
if (item.url) return Effect.succeed(Media.url(item.url, { ...options, mediaType }))
|
||||
return Effect.fail(invalid(`${label} has neither image data nor a URL`))
|
||||
}
|
||||
|
||||
/**
|
||||
* Append multipart text fields: strings as-is, other values as JSON, or arrays as repeated `key[]` parts with
|
||||
* `repeatArrays`. `overlay` keys in `reserved` are dropped so `http.body` cannot replace route-owned fields.
|
||||
*/
|
||||
export const appendFields = (
|
||||
form: FormData,
|
||||
fields: Record<string, unknown>,
|
||||
options: {
|
||||
readonly overlay?: Record<string, unknown>
|
||||
readonly reserved: ReadonlySet<string>
|
||||
readonly repeatArrays?: true
|
||||
},
|
||||
) => {
|
||||
const overlay = Object.entries(options.overlay ?? {}).filter(([key]) => !options.reserved.has(key))
|
||||
Object.entries(mergeJsonRecords(fields, Object.fromEntries(overlay)) ?? {}).forEach(([key, value]) => {
|
||||
if (Array.isArray(value) && options.repeatArrays)
|
||||
return value.forEach((item) => form.append(`${key}[]`, String(item)))
|
||||
form.append(key, typeof value === "string" ? value : encodeJson(value))
|
||||
})
|
||||
}
|
||||
|
||||
export * as MediaInput from "./media-input.js"
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
// Responses image items can omit output_format, including when PNG/JPEG was requested.
|
||||
export const mediaType = (data: Uint8Array, format?: string) => {
|
||||
if (format !== undefined) return `image/${format}`
|
||||
if (data[0] === 137 && data[1] === 80 && data[2] === 78 && data[3] === 71) return "image/png"
|
||||
if (data[0] === 255 && data[1] === 216 && data[2] === 255) return "image/jpeg"
|
||||
if (new TextDecoder().decode(data.slice(0, 4)) === "RIFF" && new TextDecoder().decode(data.slice(8, 12)) === "WEBP")
|
||||
return "image/webp"
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
export * as MetaImage from "./meta-image.js"
|
||||
@@ -1,20 +0,0 @@
|
||||
import { Schema } from "effect"
|
||||
|
||||
const dimensions = (value: string) => {
|
||||
const match = /^(\d+)x(\d+)$/.exec(value)
|
||||
if (!match) return undefined
|
||||
return { width: Number(match[1]), height: Number(match[2]) }
|
||||
}
|
||||
|
||||
export const Size = Schema.String.check(
|
||||
Schema.makeFilter((value) => {
|
||||
if (value === "auto") return undefined
|
||||
const parsed = dimensions(value)
|
||||
if (!parsed) return "image size must be `auto` or `{width}x{height}`"
|
||||
return parsed.width > 0 && parsed.height > 0 ? undefined : "image dimensions must be positive integers"
|
||||
}),
|
||||
)
|
||||
|
||||
export const OpenAIImage = {
|
||||
Size,
|
||||
} as const
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import { Route, type RouteBody, type TriggerCompactOperation } from "../../route/client.js"
|
||||
import { Protocol } from "../../route/protocol.js"
|
||||
import { CompactionCheckpointResponse, HttpOptions, LLMEvent, LLMRequest } from "../../schema/index.js"
|
||||
import { CompactionCheckpointResponse, LLMEvent, LLMRequest } from "../../schema/index.js"
|
||||
import { OpenResponses } from "../open-responses.js"
|
||||
import { ProviderShared } from "../shared.js"
|
||||
|
||||
@@ -109,12 +109,8 @@ export const make = <Body>(body: RouteBody<Body>): TriggerCompactOperation =>
|
||||
transport: source.transport,
|
||||
})
|
||||
const native = yield* body.from(request)
|
||||
// The body builder already applied and validated overlays. Do not let transport reapply them.
|
||||
const preparedRequest = LLMRequest.update(request, {
|
||||
http: request.http === undefined ? undefined : new HttpOptions({ ...request.http, body: undefined }),
|
||||
})
|
||||
const prepared = yield* route.prepareTransport(native, preparedRequest, options)
|
||||
yield* route.streamPrepared(prepared, preparedRequest, { http: executor }, options).pipe(Stream.runDrain)
|
||||
const prepared = yield* route.prepareTransport(native, request, options)
|
||||
yield* route.streamPrepared(prepared, request, { http: executor }, options).pipe(Stream.runDrain)
|
||||
if (!result) return yield* ProviderShared.eventError(source.id, "Compaction response ended without a checkpoint")
|
||||
return result
|
||||
})
|
||||
|
||||
@@ -19,12 +19,18 @@ import { OpenResponses } from "../open-responses.js"
|
||||
import { JsonObject, optionalNull, ProviderShared } from "../shared.js"
|
||||
import { Media } from "../../media.js"
|
||||
|
||||
// /compact has a smaller wire contract than /responses; keep the request controls it accepts.
|
||||
const Body = Schema.Struct({
|
||||
model: Schema.String,
|
||||
input: Schema.Array(Schema.Unknown),
|
||||
instructions: optionalNull(Schema.String),
|
||||
previous_response_id: optionalNull(Schema.String),
|
||||
service_tier: optionalNull(Schema.String),
|
||||
reasoning: Schema.optional(JsonObject),
|
||||
text: Schema.optional(JsonObject),
|
||||
include: OpenResponses.coreFields.include,
|
||||
parallel_tool_calls: OpenResponses.coreFields.parallel_tool_calls,
|
||||
tools: Schema.optional(Schema.Array(JsonObject)),
|
||||
prompt_cache_key: optionalNull(Schema.String),
|
||||
prompt_cache_retention: optionalNull(Schema.String),
|
||||
prompt_cache_options: optionalNull(
|
||||
@@ -74,17 +80,27 @@ const Response = Schema.Struct({
|
||||
usage: Schema.optional(Schema.StructWithRest(OpenResponses.OpenResponsesUsage, [JsonObject])),
|
||||
})
|
||||
|
||||
export const make = (adapter: OpenResponses.ProviderAdapter): CompactOperation =>
|
||||
export const make = (
|
||||
adapter: OpenResponses.ProviderAdapter,
|
||||
lowerTools: (request: LLMRequest) => Effect.Effect<ReadonlyArray<Record<string, unknown>>, AIError>,
|
||||
): CompactOperation =>
|
||||
Effect.fn("ResponsesCompaction.execute")(function* (request, executor, options) {
|
||||
const route = request.model.route
|
||||
// The standalone compaction endpoint rejects histories containing configuration updates.
|
||||
const native = yield* OpenResponses.lowerConversation(stripEffortUpdates(request), adapter)
|
||||
const generation = OpenResponses.lowerGeneration(request)
|
||||
const tools = request.tools.length === 0 ? undefined : yield* lowerTools(request)
|
||||
const body = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(Body))(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
...native,
|
||||
service_tier: request.providerOptions?.serviceTier,
|
||||
prompt_cache_key: ProviderShared.promptCacheKey(request),
|
||||
service_tier: generation.service_tier,
|
||||
reasoning: generation.reasoning,
|
||||
text: generation.text,
|
||||
include: generation.include,
|
||||
parallel_tool_calls: generation.parallel_tool_calls,
|
||||
tools,
|
||||
prompt_cache_key: generation.prompt_cache_key,
|
||||
},
|
||||
request.http?.body,
|
||||
),
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import { Effect } from "effect"
|
||||
import { Media } from "../../media.js"
|
||||
import type { MediaProtocol } from "../../route/media-protocol.js"
|
||||
import type { AIError, MediaUsage, ProviderMetadata } from "../../schema/index.js"
|
||||
import {
|
||||
SpeechAudioDeltaEvent,
|
||||
SpeechFinishEvent,
|
||||
SpeechTimestampsEvent,
|
||||
type SpeechEvent,
|
||||
type SpeechVoice,
|
||||
} from "../../speech.js"
|
||||
import { concatBytes } from "../../utils/bytes.js"
|
||||
|
||||
export interface Audio {
|
||||
/** Appended in place: the route creates fresh state for each response through `initial`. */
|
||||
readonly chunks: Array<Uint8Array>
|
||||
}
|
||||
|
||||
export type StepResult<State> = readonly [State, ReadonlyArray<SpeechEvent>]
|
||||
|
||||
/** Empty chunks (keep-alive records) emit nothing. */
|
||||
export const delta = <State extends Audio>(state: State, chunk: Uint8Array): StepResult<State> => {
|
||||
if (chunk.length === 0) return [state, []]
|
||||
state.chunks.push(chunk)
|
||||
return [state, [SpeechAudioDeltaEvent.make({ chunk })]]
|
||||
}
|
||||
|
||||
export const step =
|
||||
<State extends Audio>(onRecord: (state: State, frame: string) => Effect.Effect<StepResult<State>, AIError>) =>
|
||||
(state: State, frame: string | Uint8Array) =>
|
||||
typeof frame === "string" ? onRecord(state, frame) : Effect.succeed(delta(state, frame))
|
||||
|
||||
export const timestamps = (
|
||||
texts: ReadonlyArray<string>,
|
||||
starts: ReadonlyArray<number>,
|
||||
ends: ReadonlyArray<number>,
|
||||
): ReadonlyArray<SpeechEvent> =>
|
||||
texts.length === 0
|
||||
? []
|
||||
: [
|
||||
SpeechTimestampsEvent.make({
|
||||
items: texts.map((text, index) => ({ text, startSeconds: starts[index] ?? 0, endSeconds: ends[index] ?? 0 })),
|
||||
}),
|
||||
]
|
||||
|
||||
export const voiceID = (voice: SpeechVoice | undefined) => (typeof voice === "object" ? voice.id : voice)
|
||||
|
||||
const CONTAINER_MEDIA_TYPES: Readonly<Record<string, string>> = {
|
||||
mp3: "audio/mpeg",
|
||||
wav: "audio/wav",
|
||||
opus: "audio/ogg",
|
||||
aac: "audio/aac",
|
||||
flac: "audio/flac",
|
||||
}
|
||||
|
||||
export const container = (format: string, sampleRate?: number) => ({
|
||||
mediaType: CONTAINER_MEDIA_TYPES[format],
|
||||
info: { format, sampleRate },
|
||||
})
|
||||
|
||||
const PCM_MEDIA_TYPES = {
|
||||
pcm_s16le: "audio/pcm",
|
||||
pcm_f32le: "audio/pcm",
|
||||
pcm_mulaw: "audio/mulaw",
|
||||
pcm_alaw: "audio/alaw",
|
||||
} as const
|
||||
|
||||
export type PcmEncoding = keyof typeof PCM_MEDIA_TYPES
|
||||
|
||||
export const pcm = (encoding: PcmEncoding, sampleRate: number | undefined, mediaType?: string) => ({
|
||||
mediaType: mediaType ?? PCM_MEDIA_TYPES[encoding],
|
||||
info: { format: "pcm", encoding, sampleRate, channels: 1 },
|
||||
})
|
||||
|
||||
export const sampleRate = (mediaType: string | undefined) => {
|
||||
const rate = /rate=(\d+)/i.exec(mediaType ?? "")?.[1]
|
||||
return rate === undefined ? undefined : Number(rate)
|
||||
}
|
||||
|
||||
/** A declared `mediaType` wins over sniffing: headerless PCM can start with bytes that look like an MPEG frame sync. */
|
||||
export const finish = (
|
||||
route: MediaProtocol.Identity,
|
||||
state: Audio,
|
||||
output: {
|
||||
readonly mediaType: string | undefined
|
||||
readonly info?: Media.Info
|
||||
readonly usage?: MediaUsage
|
||||
readonly providerMetadata?: ProviderMetadata
|
||||
readonly detail?: string
|
||||
},
|
||||
): Effect.Effect<ReadonlyArray<SpeechEvent>, AIError> => {
|
||||
if (state.chunks.length === 0)
|
||||
return Effect.fail(
|
||||
route.frameError(`The provider returned no audio${output.detail === undefined ? "" : ` (${output.detail})`}`),
|
||||
)
|
||||
return Effect.succeed([
|
||||
SpeechFinishEvent.make({
|
||||
audio: Media.bytes(concatBytes(state.chunks), output.mediaType, { info: output.info }),
|
||||
usage: output.usage,
|
||||
providerMetadata: output.providerMetadata,
|
||||
}),
|
||||
])
|
||||
}
|
||||
|
||||
export const headerUsage = (type: "characters" | "credits", value: string | undefined): MediaUsage | undefined => {
|
||||
const amount = Number(value)
|
||||
if (!Number.isFinite(amount)) return undefined
|
||||
return type === "credits" ? { type, credits: amount } : { type, characters: amount }
|
||||
}
|
||||
|
||||
export * as SpeechStream from "./speech-stream.js"
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { JsonSchema, LanguageModelToolSchemaCompatibility } from "../../schema/index.js"
|
||||
import { ToolDefinition, type JsonSchema, type LanguageModel, type LLMRequest } from "../../schema/index.js"
|
||||
import { isRecord } from "../../utils/record.js"
|
||||
import { GeminiToolSchema } from "./gemini-tool-schema.js"
|
||||
import { GeminiJsonSchema } from "./gemini-json-schema.js"
|
||||
|
||||
const tupleItemsSchema = (items: ReadonlyArray<unknown>) => {
|
||||
const projected = items.map(moonshotNode)
|
||||
@@ -9,12 +9,24 @@ const tupleItemsSchema = (items: ReadonlyArray<unknown>) => {
|
||||
return { anyOf: projected }
|
||||
}
|
||||
|
||||
// Moonshot rejects an `enum` without a sibling `type`. Its `type` may be one type, or one type plus "null".
|
||||
const enumType = (values: ReadonlyArray<unknown>) => {
|
||||
const types = [
|
||||
...new Set(values.map((value) => (value === null ? "null" : Array.isArray(value) ? "array" : typeof value))),
|
||||
]
|
||||
if (types.length === 1) return types[0]
|
||||
if (types.length === 2 && types.includes("null")) return [...types.filter((type) => type !== "null"), "null"]
|
||||
return undefined
|
||||
}
|
||||
|
||||
const moonshotNode = (schema: unknown): unknown => {
|
||||
if (Array.isArray(schema)) return schema.map(moonshotNode)
|
||||
if (!isRecord(schema)) return schema
|
||||
if (typeof schema.$ref === "string") return { $ref: schema.$ref }
|
||||
return Object.fromEntries(
|
||||
Object.entries(schema).flatMap(([key, value]) => {
|
||||
const type = schema.type === undefined && Array.isArray(schema.enum) ? enumType(schema.enum) : undefined
|
||||
return Object.fromEntries([
|
||||
...(type === undefined ? [] : [["type", type]]),
|
||||
...Object.entries(schema).flatMap(([key, value]) => {
|
||||
if (key === "items" && Array.isArray(value)) return [[key, tupleItemsSchema(value)]]
|
||||
if (key === "prefixItems") {
|
||||
if ("items" in schema) return []
|
||||
@@ -23,7 +35,7 @@ const moonshotNode = (schema: unknown): unknown => {
|
||||
if (key === "unevaluatedItems") return []
|
||||
return [[key, moonshotNode(value)]]
|
||||
}),
|
||||
)
|
||||
])
|
||||
}
|
||||
|
||||
const moonshot = (schema: JsonSchema): JsonSchema => {
|
||||
@@ -34,25 +46,59 @@ const moonshot = (schema: JsonSchema): JsonSchema => {
|
||||
const openAI = (schema: JsonSchema): JsonSchema => schema
|
||||
const responses = openAI
|
||||
|
||||
const gemini = (schema: JsonSchema): JsonSchema => GeminiToolSchema.convert(schema) ?? {}
|
||||
const gemini = GeminiJsonSchema.normalize
|
||||
|
||||
const modelCompatibility = (
|
||||
schema: JsonSchema,
|
||||
compatibility: LanguageModelToolSchemaCompatibility | undefined,
|
||||
): JsonSchema => {
|
||||
if (compatibility === undefined) return schema
|
||||
switch (compatibility) {
|
||||
const MODEL_NAMES = [
|
||||
[/gemini/i, "gemini"],
|
||||
[/kimi/i, "moonshot"],
|
||||
] as const
|
||||
|
||||
// Tool arguments are always a JSON object, and most providers reject a tool schema whose root does not
|
||||
// declare `type: "object"`, such as `{}` or a bare `properties` map. Effect encodes an empty struct as
|
||||
// `anyOf` object or array; every object matches its bare object branch, so that `anyOf` is dropped.
|
||||
const objectRoot = (schema: JsonSchema): JsonSchema => {
|
||||
if (schema.type !== undefined) return schema
|
||||
if (
|
||||
Array.isArray(schema.anyOf) &&
|
||||
schema.anyOf.some((branch) => isRecord(branch) && branch.type === "object" && Object.keys(branch).length === 1)
|
||||
)
|
||||
return { type: "object", ...Object.fromEntries(Object.entries(schema).filter(([key]) => key !== "anyOf")) }
|
||||
return { type: "object", ...schema }
|
||||
}
|
||||
|
||||
// Every tool schema gets an object root. Then an explicit `sanitizer` wins, and `none` opts out.
|
||||
// Otherwise the protocol's own default applies (the Gemini API always uses Gemini's rules), then the
|
||||
// model name selects the family's rules so models reached through gateways and OpenAI-compatible
|
||||
// endpoints get the same handling.
|
||||
const modelCompatibility = (schema: JsonSchema, model: LanguageModel): JsonSchema => {
|
||||
const root = objectRoot(schema)
|
||||
switch (
|
||||
model.compatibility?.sanitizer ??
|
||||
model.route.sanitizer ??
|
||||
MODEL_NAMES.find(([name]) => name.test(model.id))?.[1]
|
||||
) {
|
||||
case "gemini":
|
||||
return gemini(schema)
|
||||
return gemini(root)
|
||||
case "moonshot":
|
||||
return moonshot(schema)
|
||||
return moonshot(root)
|
||||
case "none":
|
||||
case undefined:
|
||||
return root
|
||||
}
|
||||
}
|
||||
|
||||
// Applied once to every request before any protocol builds its body, including tools in namespaces.
|
||||
const tools = (entries: LLMRequest["tools"], model: LanguageModel): LLMRequest["tools"] =>
|
||||
entries.map((tool) =>
|
||||
tool.type === "tool"
|
||||
? new ToolDefinition({ ...tool, inputSchema: modelCompatibility(tool.inputSchema, model) })
|
||||
: { ...tool, tools: tools(tool.tools, model) },
|
||||
)
|
||||
|
||||
export const ToolSchemaProjection = {
|
||||
gemini,
|
||||
modelCompatibility,
|
||||
moonshot,
|
||||
openAI,
|
||||
responses,
|
||||
tools,
|
||||
} as const
|
||||
|
||||
@@ -4,13 +4,11 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
import { ProviderShared, optionalNull } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
const ADAPTER = "xai-images"
|
||||
const NAME = "xAI Images"
|
||||
const PROVIDER = ProviderID.make("xai")
|
||||
const route = MediaProtocol.identity({ id: "xai-images", name: "xAI Images", provider: "xai" })
|
||||
export const DEFAULT_BASE_URL = "https://api.x.ai/v1"
|
||||
export const PATH = "/images/generations"
|
||||
export const EDIT_PATH = "/images/edits"
|
||||
@@ -19,13 +17,11 @@ export const EDIT_PATH = "/images/edits"
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type XAIImageString<Known extends string> = Known | (string & {})
|
||||
|
||||
/** Provider-native options. Common fields (`n`, `aspectRatio`, `images`) live on the request. */
|
||||
export type XAIImageOptions = {
|
||||
readonly resolution?: XAIImageString<"1k" | "2k">
|
||||
readonly responseFormat?: XAIImageString<"url" | "b64_json">
|
||||
readonly response_format?: XAIImageString<"url" | "b64_json">
|
||||
readonly resolution?: OpenString<"1k" | "2k">
|
||||
readonly responseFormat?: OpenString<"url" | "b64_json">
|
||||
readonly response_format?: OpenString<"url" | "b64_json">
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = ImageRequestFor<XAIImageOptions>
|
||||
@@ -58,15 +54,12 @@ const nativeOptions = (options: XAIImageOptions | undefined) => {
|
||||
|
||||
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
|
||||
|
||||
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
|
||||
const inline = asset.inline()
|
||||
if (inline) return Effect.succeed({ url: inline.dataUrl, type: "image_url" as const })
|
||||
const url = ProviderShared.mediaUrl(asset)
|
||||
if (url) return Effect.succeed({ url, type: "image_url" as const })
|
||||
const id = MediaInput.refID(asset, PROVIDER)
|
||||
if (id) return Effect.succeed({ file_id: id })
|
||||
return Effect.fail(ProviderShared.invalidRequest(`${NAME} accepts image URLs, data URLs, bytes, and xAI file IDs`))
|
||||
}
|
||||
const reference = (asset: Media.Asset) =>
|
||||
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(
|
||||
Effect.map((item) =>
|
||||
item.type === "ref" ? { file_id: item.value } : { url: item.value, type: "image_url" as const },
|
||||
),
|
||||
)
|
||||
|
||||
const fromRequest = Effect.fn("XAIImages.fromRequest")(function* (request: Request) {
|
||||
const references = yield* Effect.forEach(request.images ?? [], reference)
|
||||
@@ -91,33 +84,25 @@ const fromRequest = Effect.fn("XAIImages.fromRequest")(function* (request: Reque
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeDocument = route.decodeJson(XAIImageResponse)
|
||||
|
||||
const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, XAIImageResponse)(response)
|
||||
const output = yield* decodeDocument(response)
|
||||
const decoded = output.value
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) => {
|
||||
const providerMetadata =
|
||||
item.revised_prompt === undefined || item.revised_prompt === null
|
||||
? undefined
|
||||
: { xai: { revisedPrompt: item.revised_prompt } }
|
||||
if (item.b64_json)
|
||||
return MediaInput.decodedAsset(
|
||||
output.invalid,
|
||||
`${NAME} result ${index}`,
|
||||
item.b64_json,
|
||||
item.mime_type ?? undefined,
|
||||
{
|
||||
providerMetadata,
|
||||
},
|
||||
)
|
||||
if (item.url)
|
||||
return Effect.succeed(Media.url(item.url, { mediaType: item.mime_type ?? undefined, providerMetadata }))
|
||||
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
|
||||
})
|
||||
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) =>
|
||||
MediaInput.imageOutput(output.invalid, `${route.name} result ${index}`, item, item.mime_type ?? undefined, {
|
||||
providerMetadata:
|
||||
item.revised_prompt === undefined || item.revised_prompt === null
|
||||
? undefined
|
||||
: { xai: { revisedPrompt: item.revised_prompt } },
|
||||
}),
|
||||
)
|
||||
if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`)
|
||||
const usage = ProviderShared.isRecord(decoded.usage) ? decoded.usage : undefined
|
||||
// xAI reports image counts rather than tokens, seconds, or credits; the raw record stays in provider metadata.
|
||||
// xAI reports a USD cost (`cost_in_usd_ticks`) rather than tokens, seconds, or credits; the raw record stays in
|
||||
// provider metadata.
|
||||
return new ImageResponse({
|
||||
images,
|
||||
providerMetadata: usage === undefined ? undefined : { xai: { usage } },
|
||||
@@ -128,9 +113,7 @@ const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* (
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
|
||||
id: ADAPTER,
|
||||
name: NAME,
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
|
||||
unsupported: ["mask", "size", "seed", "format"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
@@ -138,13 +121,7 @@ export const protocol = MediaProtocol.inline<Request, ImageResponse>({
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<XAIImageOptions>(
|
||||
{
|
||||
id: ADAPTER,
|
||||
provider: PROVIDER,
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
|
||||
},
|
||||
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH) },
|
||||
input,
|
||||
)
|
||||
|
||||
|
||||
@@ -50,7 +50,8 @@ const fromRequest = Effect.fn("XAIResponses.fromRequest")(function* (request: LL
|
||||
operation: "in-band-compaction",
|
||||
provider: request.model.provider,
|
||||
route: request.model.route.id,
|
||||
message: "xAI requires explicit compaction through LLMClient.compact; automatic context management is not supported",
|
||||
message:
|
||||
"xAI requires explicit compaction through LLMClient.compact; automatic context management is not supported",
|
||||
})
|
||||
return yield* decodeBody(yield* OpenResponses.fromRequestWithAdapter(request, adapter))
|
||||
})
|
||||
@@ -93,6 +94,8 @@ export const protocol = Protocol.make({
|
||||
},
|
||||
})
|
||||
|
||||
export const compact = ResponsesCompaction.make(adapter)
|
||||
export const compact = ResponsesCompaction.make(adapter, (request) =>
|
||||
OpenResponses.lowerTools(ProviderShared.flattenTools(request.tools), adapter),
|
||||
)
|
||||
|
||||
export * as XAIResponses from "./xai-responses.js"
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import type { Status } from "../generation.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords } from "../schema/index.js"
|
||||
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
|
||||
import { ProviderShared, optionalNull } from "./shared.js"
|
||||
|
||||
const route = MediaProtocol.identity({ id: "xai-video", name: "xAI Video", provider: "xai" })
|
||||
export const DEFAULT_BASE_URL = "https://api.x.ai/v1"
|
||||
export const PATH = "/videos/generations"
|
||||
export const EDIT_PATH = "/videos/edits"
|
||||
export const EXTEND_PATH = "/videos/extensions"
|
||||
export const STATUS_PATH = "/videos"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Provider-native options. Common fields (`frames`, `references`, `video`, `durationSeconds`, `aspectRatio`,
|
||||
* `resolution`, `audio`) live on the request. `mode` selects the endpoint a `video` input is sent to.
|
||||
*/
|
||||
export type XAIVideoOptions = {
|
||||
readonly mode?: "edit" | "extend"
|
||||
readonly reference_audios?: ReadonlyArray<{ readonly voice_id: string }>
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = VideoRequestFor<XAIVideoOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Token and response schemas
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const Token = Schema.Struct({ requestID: Schema.String })
|
||||
export type Token = Schema.Schema.Type<typeof Token>
|
||||
|
||||
const StartResponse = Schema.Struct({ request_id: Schema.String })
|
||||
|
||||
const VideoStatus = Schema.Struct({
|
||||
status: Schema.String,
|
||||
progress: optionalNull(Schema.Number),
|
||||
video: optionalNull(
|
||||
Schema.Struct({
|
||||
url: optionalNull(Schema.String),
|
||||
duration: optionalNull(Schema.Number),
|
||||
respect_moderation: optionalNull(Schema.Boolean),
|
||||
}),
|
||||
),
|
||||
error: optionalNull(
|
||||
Schema.Struct({
|
||||
code: optionalNull(Schema.String),
|
||||
message: optionalNull(Schema.String),
|
||||
}),
|
||||
),
|
||||
model: optionalNull(Schema.String),
|
||||
})
|
||||
|
||||
const STATUS = {
|
||||
pending: "running",
|
||||
done: "completed",
|
||||
failed: "failed",
|
||||
expired: "expired",
|
||||
} as const satisfies Record<string, Status>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const mediaInput = (asset: Media.Asset) =>
|
||||
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(
|
||||
Effect.map((reference) => (reference.type === "ref" ? { file_id: reference.value } : { url: reference.value })),
|
||||
)
|
||||
|
||||
const nativeOptions = (options: XAIVideoOptions | undefined) => {
|
||||
if (!options) return undefined
|
||||
const { mode: _mode, ...native } = options
|
||||
return native
|
||||
}
|
||||
|
||||
const fromRequest = Effect.fn("XAIVideo.fromRequest")(function* (request: Request) {
|
||||
const image = request.frames?.first === undefined ? undefined : yield* mediaInput(request.frames.first)
|
||||
const lastFrame = request.frames?.last === undefined ? undefined : yield* mediaInput(request.frames.last)
|
||||
const video = request.video === undefined ? undefined : yield* mediaInput(request.video)
|
||||
const references = yield* Effect.forEach(request.references ?? [], mediaInput)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
model: request.model.id,
|
||||
prompt: request.prompt,
|
||||
image,
|
||||
last_frame: lastFrame,
|
||||
reference_images: references.length === 0 ? undefined : references,
|
||||
video,
|
||||
duration: request.durationSeconds,
|
||||
aspect_ratio: request.aspectRatio,
|
||||
resolution: request.resolution,
|
||||
generate_audio: request.audio,
|
||||
},
|
||||
nativeOptions(request.providerOptions),
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
|
||||
token: { requestID: value.request_id },
|
||||
snapshot: { id: value.request_id, status: "running" },
|
||||
}))
|
||||
|
||||
// `progress` is undocumented but observed live as a 0..100 percentage (recorded cassette: 1 → 10 → 37 → 100).
|
||||
const fraction = (progress: number | null | undefined) =>
|
||||
progress !== undefined && progress !== null && progress >= 0 && progress <= 100 ? progress / 100 : undefined
|
||||
|
||||
const decodeVideoStatus = route.decodeJson(VideoStatus)
|
||||
|
||||
const decodeStatus = Effect.fn("XAIVideo.decodeStatus")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
const output = yield* decodeVideoStatus(response)
|
||||
const status = yield* MediaProtocol.status(STATUS, output.value.status, output)
|
||||
return { id: context.token.requestID, status, progress: fraction(output.value.progress) }
|
||||
})
|
||||
|
||||
const decodeResult = Effect.fn("XAIVideo.decodeResult")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) {
|
||||
const output = yield* decodeVideoStatus(response)
|
||||
const decoded = output.value
|
||||
const status = yield* MediaProtocol.status(STATUS, decoded.status, output)
|
||||
if (status === "running") return yield* output.pending(context.token.requestID)
|
||||
if (status === "failed") {
|
||||
const code = decoded.error?.code ?? undefined
|
||||
const message = decoded.error?.message ?? undefined
|
||||
return yield* output.ended(
|
||||
"failed",
|
||||
`${route.name} generation failed${code === undefined ? "" : ` (${code})`}${message === undefined ? "" : `: ${message}`}`,
|
||||
)
|
||||
}
|
||||
if (status !== "completed")
|
||||
return yield* output.ended("expired", `${route.name} request ${context.token.requestID} expired`)
|
||||
// `respect_moderation: false` marks a filtered result; a URL may still be present, so report it as a notice.
|
||||
const notices =
|
||||
decoded.video?.respect_moderation === false
|
||||
? [{ type: "moderated" as const, message: `${route.name} flagged the generated video for moderation` }]
|
||||
: undefined
|
||||
const url = decoded.video?.url ?? undefined
|
||||
if (url === undefined && notices !== undefined)
|
||||
return yield* output.contentPolicy(`${route.name} withheld the video for moderation`)
|
||||
if (url === undefined) return yield* output.invalid(`${route.name} completed without a video URL`)
|
||||
const duration = decoded.video?.duration ?? undefined
|
||||
return new VideoResponse({
|
||||
videos: [
|
||||
Media.url(url, {
|
||||
mediaType: "video/mp4",
|
||||
info: duration === undefined ? undefined : { durationSeconds: duration },
|
||||
}),
|
||||
],
|
||||
notices,
|
||||
providerMetadata: { xai: { requestId: context.token.requestID, model: decoded.model ?? undefined } },
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const statusPath = (token: Token) => `${STATUS_PATH}/${token.requestID}`
|
||||
|
||||
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>(route, {
|
||||
token: Token,
|
||||
unsupported: ["n", "seed", "negativePrompt"],
|
||||
start: { body: { from: fromRequest }, decode: decodeStart },
|
||||
status: { path: statusPath, decode: decodeStatus },
|
||||
result: { path: statusPath, decode: decodeResult },
|
||||
})
|
||||
|
||||
// A source video goes to `/videos/edits` unless `providerOptions.mode` asks for an extension.
|
||||
const startPath = (request: Request) => {
|
||||
if (request.video === undefined) return PATH
|
||||
return request.providerOptions?.mode === "extend" ? EXTEND_PATH : EDIT_PATH
|
||||
}
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
VideoModel.fromRoute<XAIVideoOptions, Token>(
|
||||
{
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) => startPath(request),
|
||||
},
|
||||
input,
|
||||
)
|
||||
|
||||
export const XAIVideo = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
@@ -1,26 +1,22 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { Duration, Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
|
||||
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
|
||||
|
||||
const ADAPTER = "zai-images"
|
||||
const NAME = "Z.ai Images"
|
||||
const PROVIDER = ProviderID.make("zai")
|
||||
const route = MediaProtocol.identity({ id: "zai-images", name: "Z.ai Images", provider: "zai" })
|
||||
export const DEFAULT_BASE_URL = "https://api.z.ai/api/paas/v4"
|
||||
export const PATH = "/images/generations"
|
||||
const OUTPUT_RETENTION = Duration.days(30)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type ZAIImageString<Known extends string> = Known | (string & {})
|
||||
|
||||
/** Provider-native options. The common `size` field lives on the request. */
|
||||
export type ZAIImageOptions = {
|
||||
readonly quality?: ZAIImageString<"hd" | "standard">
|
||||
readonly quality?: OpenString<"hd" | "standard">
|
||||
readonly userID?: string
|
||||
} & Record<string, unknown>
|
||||
|
||||
@@ -69,23 +65,25 @@ const fromRequest = Effect.fn("ZAIImages.fromRequest")(function* (request: Reque
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeDocument = route.decodeJson(ZAIImageResponse)
|
||||
|
||||
const decodeResponse = Effect.fn("ZAIImages.decodeResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, ZAIImageResponse)(response)
|
||||
const output = yield* decodeDocument(response)
|
||||
const decoded = output.value
|
||||
if (decoded.data.length === 0) return yield* output.invalid(`${NAME} returned no images`)
|
||||
if (decoded.data.length === 0) return yield* output.invalid(`${route.name} returned no images`)
|
||||
const filters = decoded.content_filter ?? []
|
||||
return new ImageResponse({
|
||||
// Z.ai returns only URLs and no content type; the media type resolves when the asset is materialized.
|
||||
images: decoded.data.map((item) => Media.url(item.url)),
|
||||
images: yield* Effect.forEach(decoded.data, (item) => MediaProtocol.expiringUrl(item.url, OUTPUT_RETENTION)),
|
||||
// Z.ai reports applied content filters alongside a successful result; surface them instead of dropping them.
|
||||
notices:
|
||||
filters.length === 0
|
||||
? undefined
|
||||
: filters.map((filter) => ({
|
||||
type: "moderated" as const,
|
||||
message: `${NAME} applied a content filter${filter.role === undefined ? "" : ` for ${filter.role}`}${
|
||||
message: `${route.name} applied a content filter${filter.role === undefined ? "" : ` for ${filter.role}`}${
|
||||
filter.level === undefined ? "" : ` at level ${filter.level}`
|
||||
}`,
|
||||
providerMetadata: { zai: filter },
|
||||
@@ -105,19 +103,14 @@ const decodeResponse = Effect.fn("ZAIImages.decodeResponse")(function* (
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
|
||||
id: ADAPTER,
|
||||
name: NAME,
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
|
||||
unsupported: ["images", "mask", "n", "aspectRatio", "seed", "format"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<ZAIImageOptions>(
|
||||
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
|
||||
input,
|
||||
)
|
||||
ImageModel.fromRoute<ZAIImageOptions>({ protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, input)
|
||||
|
||||
export const ZAIImages = {
|
||||
protocol,
|
||||
|
||||
@@ -80,12 +80,19 @@ const SERVER_CODES = new Set([
|
||||
"slow_down",
|
||||
"serviceunavailableexception",
|
||||
])
|
||||
const INVALID_REQUEST_CODES = new Set(["invalid_prompt", "invalid_request_error", "validationexception"])
|
||||
// `invalid_request` is the Vercel AI Gateway's code for an upstream request rejection.
|
||||
const INVALID_REQUEST_CODES = new Set([
|
||||
"invalid_prompt",
|
||||
"invalid_request",
|
||||
"invalid_request_error",
|
||||
"validationexception",
|
||||
])
|
||||
// Azure OpenAI reports `content_filter` with `innererror.code` ResponsibleAIPolicyViolation.
|
||||
// OpenRouter tags provider failures with a typed `error_type`; its Responses skin also
|
||||
// emits `image_content_policy_violation` as the native code.
|
||||
const CONTENT_POLICY_CODES = new Set([
|
||||
"content_filter",
|
||||
"content_moderation",
|
||||
"responsibleaipolicyviolation",
|
||||
"content_policy_violation",
|
||||
"image_content_policy_violation",
|
||||
@@ -204,6 +211,8 @@ function providerCodes(value: unknown) {
|
||||
const exception = isRecord(decoded.exception) ? decoded.exception : undefined
|
||||
return [
|
||||
decoded.code,
|
||||
// Stability's `{ id, name, errors }` bodies carry the code in `name`.
|
||||
Array.isArray(decoded.errors) ? decoded.name : undefined,
|
||||
decoded.error_type,
|
||||
error?.code,
|
||||
error?.type,
|
||||
|
||||
@@ -31,6 +31,7 @@ export interface Settings extends ProviderPackage.Settings {
|
||||
readonly profile?: string
|
||||
readonly region?: string
|
||||
readonly topP?: number
|
||||
readonly thinking?: BedrockConverse.OptionsInput["thinking"]
|
||||
}
|
||||
export const routes = [BedrockConverse.route]
|
||||
|
||||
@@ -71,6 +72,7 @@ export const model: ProviderPackage.Definition<Settings>["model"] = (modelID, se
|
||||
generation: settings.topP === undefined ? undefined : { topP: settings.topP },
|
||||
headers: settings.headers === undefined ? undefined : { ...settings.headers },
|
||||
http: settings.body === undefined ? undefined : { body: { ...settings.body } },
|
||||
providerOptions: settings.thinking === undefined ? undefined : { thinking: settings.thinking },
|
||||
profile: settings.profile,
|
||||
region: settings.region,
|
||||
}).model(modelID)
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { Auth } from "../route/auth.js"
|
||||
import type { ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { AssemblyAITranscription } from "../protocols/assemblyai-transcription.js"
|
||||
|
||||
export type { AssemblyAITranscriptionOptions } from "../protocols/assemblyai-transcription.js"
|
||||
|
||||
export const id = ProviderID.make("assemblyai")
|
||||
|
||||
export type Config = ProviderAuthOption<"optional"> & {
|
||||
/** `https://api.eu.assemblyai.com` for the EU region. */
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
// The key is the whole `authorization` value, without a scheme.
|
||||
const auth = (options: ProviderAuthOption<"optional">) => {
|
||||
if ("auth" in options && options.auth) return options.auth
|
||||
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
|
||||
.orElse(Auth.config("ASSEMBLYAI_API_KEY"))
|
||||
.pipe(Auth.header("authorization"))
|
||||
}
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
const transcription = (modelID: string | ModelID) => AssemblyAITranscription.model({ ...media, id: modelID })
|
||||
return {
|
||||
id,
|
||||
transcription,
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
export const transcription = provider.transcription
|
||||
@@ -1,11 +1,11 @@
|
||||
import { Auth } from "../route/auth.js"
|
||||
import { type AtLeastOne, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import type { Route, RouteDefaultsInput, CompactionOperations } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import * as OpenAIChat from "../protocols/openai-chat.js"
|
||||
import * as OpenAIResponses from "../protocols/openai-responses.js"
|
||||
import { ProviderShared } from "../protocols/shared.js"
|
||||
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
|
||||
export const id = ProviderID.make("azure")
|
||||
@@ -108,7 +108,7 @@ const configuredRoute = <Body, Prepared, Compact extends CompactionOperations |
|
||||
})
|
||||
|
||||
function endpoint(input: Config, modelID: string | ModelID) {
|
||||
const baseURL = ProviderShared.trimBaseUrl(input.baseURL ?? resourceBaseURL(input.resourceName!))
|
||||
const baseURL = Endpoint.trimBaseUrl(input.baseURL ?? resourceBaseURL(input.resourceName!))
|
||||
const query = { "api-version": input.apiVersion ?? "v1", ...input.queryParams }
|
||||
|
||||
if (input.useDeploymentBasedUrls) return { baseURL: `${baseURL}/deployments/${modelID}`, query }
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import { Auth } from "../route/auth.js"
|
||||
import type { ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { BlackForestLabsImages } from "../protocols/bfl-images.js"
|
||||
|
||||
export type { BlackForestLabsImageOptions } from "../protocols/bfl-images.js"
|
||||
|
||||
export const id = ProviderID.make("black-forest-labs")
|
||||
|
||||
export type Config = ProviderAuthOption<"optional"> & {
|
||||
/** `https://api.eu.bfl.ai` or `https://api.us.bfl.ai` pin inference to one region. */
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
const auth = (options: ProviderAuthOption<"optional">) => {
|
||||
if ("auth" in options && options.auth) return options.auth
|
||||
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
|
||||
.orElse(Auth.config("BFL_API_KEY"))
|
||||
.pipe(Auth.header("x-key"))
|
||||
}
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
const image = (modelID: string | ModelID) => BlackForestLabsImages.model({ ...media, id: modelID })
|
||||
return {
|
||||
id,
|
||||
image,
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
export const image = provider.image
|
||||
@@ -0,0 +1,29 @@
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { CartesiaSpeech } from "../protocols/cartesia-speech.js"
|
||||
|
||||
export type { CartesiaEncoding, CartesiaSpeechOptions } from "../protocols/cartesia-speech.js"
|
||||
|
||||
export const id = ProviderID.make("cartesia")
|
||||
|
||||
export type Config = ProviderAuthOption<"optional"> & {
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "CARTESIA_API_KEY")
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
const speech = (modelID: string | ModelID) => CartesiaSpeech.model({ ...media, id: modelID })
|
||||
return {
|
||||
id,
|
||||
speech,
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
export const speech = provider.speech
|
||||
@@ -0,0 +1,38 @@
|
||||
import { Auth } from "../route/auth.js"
|
||||
import type { ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { DeepgramSpeech } from "../protocols/deepgram-speech.js"
|
||||
import { DeepgramTranscription } from "../protocols/deepgram-transcription.js"
|
||||
|
||||
export type { DeepgramEncoding, DeepgramSpeechOptions } from "../protocols/deepgram-speech.js"
|
||||
export type { DeepgramTranscriptionOptions } from "../protocols/deepgram-transcription.js"
|
||||
|
||||
export const id = ProviderID.make("deepgram")
|
||||
|
||||
export type Config = ProviderAuthOption<"optional"> & {
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
const auth = (options: ProviderAuthOption<"optional">) => {
|
||||
if ("auth" in options && options.auth) return options.auth
|
||||
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
|
||||
.orElse(Auth.config("DEEPGRAM_API_KEY"))
|
||||
.pipe(Auth.scheme("Token"))
|
||||
}
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
return {
|
||||
id,
|
||||
speech: (modelID: string | ModelID) => DeepgramSpeech.model({ ...media, id: modelID }),
|
||||
transcription: (modelID: string | ModelID) => DeepgramTranscription.model({ ...media, id: modelID }),
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
export const speech = provider.speech
|
||||
export const transcription = provider.transcription
|
||||
@@ -0,0 +1,35 @@
|
||||
import { Auth } from "../route/auth.js"
|
||||
import type { ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { ElevenLabsSpeech } from "../protocols/elevenlabs-speech.js"
|
||||
|
||||
export type { ElevenLabsOutputFormat, ElevenLabsSpeechOptions } from "../protocols/elevenlabs-speech.js"
|
||||
|
||||
export const id = ProviderID.make("elevenlabs")
|
||||
|
||||
export type Config = ProviderAuthOption<"optional"> & {
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
const auth = (options: ProviderAuthOption<"optional">) => {
|
||||
if ("auth" in options && options.auth) return options.auth
|
||||
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
|
||||
.orElse(Auth.config("ELEVENLABS_API_KEY"))
|
||||
.pipe(Auth.header("xi-api-key"))
|
||||
}
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
const speech = (modelID: string | ModelID) => ElevenLabsSpeech.model({ ...media, id: modelID })
|
||||
return {
|
||||
id,
|
||||
speech,
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
export const speech = provider.speech
|
||||
@@ -0,0 +1,39 @@
|
||||
import { Auth } from "../route/auth.js"
|
||||
import type { ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { FalImages } from "../protocols/fal-images.js"
|
||||
import { FalVideo } from "../protocols/fal-video.js"
|
||||
|
||||
export type { FalImageOptions } from "../protocols/fal-images.js"
|
||||
export type { FalVideoOptions } from "../protocols/fal-video.js"
|
||||
|
||||
export const id = ProviderID.make("fal")
|
||||
|
||||
export type Config = ProviderAuthOption<"optional"> & {
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
// fal authenticates with `Authorization: Key <FAL_KEY>` rather than a bearer token.
|
||||
const auth = (options: ProviderAuthOption<"optional">) => {
|
||||
if ("auth" in options && options.auth) return options.auth
|
||||
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
|
||||
.orElse(Auth.config("FAL_KEY"))
|
||||
.pipe(Auth.scheme("Key"))
|
||||
}
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
return {
|
||||
id,
|
||||
image: (modelID: string | ModelID) => FalImages.model({ ...media, id: modelID }),
|
||||
video: (modelID: string | ModelID) => FalVideo.model({ ...media, id: modelID }),
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
export const image = provider.image
|
||||
export const video = provider.video
|
||||
@@ -1,12 +1,19 @@
|
||||
import type { RouteDefaultsInput } from "../route/client.js"
|
||||
import { Auth } from "../route/auth.js"
|
||||
import type { ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { HttpOptions, ProviderID, mergeHttpOptions, type ModelID } from "../schema/index.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { Gemini } from "../protocols/gemini.js"
|
||||
import { GoogleImages } from "../protocols/google-images.js"
|
||||
import { GoogleSpeech } from "../protocols/google-speech.js"
|
||||
import { GoogleTranscription } from "../protocols/google-transcription.js"
|
||||
import { GoogleVideo } from "../protocols/google-video.js"
|
||||
|
||||
export type { GoogleImageOptions } from "../protocols/google-images.js"
|
||||
export type { GoogleSpeechOptions } from "../protocols/google-speech.js"
|
||||
export type { GoogleTranscriptionOptions } from "../protocols/google-transcription.js"
|
||||
export type { GoogleVideoOptions } from "../protocols/google-video.js"
|
||||
export type GeminiOptionsInput = Gemini.OptionsInput
|
||||
export type GeminiProviderOptionsInput = Gemini.ProviderOptionsInput
|
||||
|
||||
@@ -40,18 +47,14 @@ const configuredRoute = (input: Config) => {
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const route = configuredRoute(input)
|
||||
const image = (modelID: string | ModelID) =>
|
||||
GoogleImages.model({
|
||||
id: modelID,
|
||||
auth: auth(input),
|
||||
baseURL: input.baseURL,
|
||||
headers: input.headers,
|
||||
http: mergeHttpOptions(input.http === undefined ? undefined : HttpOptions.make(input.http)),
|
||||
})
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => route.model<Gemini.ProviderOptionsInput>({ id: modelID }),
|
||||
image,
|
||||
image: (modelID: string | ModelID) => GoogleImages.model({ ...media, id: modelID }),
|
||||
video: (modelID: string | ModelID) => GoogleVideo.model({ ...media, id: modelID }),
|
||||
speech: (modelID: string | ModelID) => GoogleSpeech.model({ ...media, id: modelID }),
|
||||
transcription: (modelID: string | ModelID) => GoogleTranscription.model({ ...media, id: modelID }),
|
||||
configure,
|
||||
}
|
||||
}
|
||||
@@ -70,3 +73,6 @@ export const model: ProviderPackage.Definition<Settings, Gemini.ProviderOptionsI
|
||||
}).model(modelID)
|
||||
|
||||
export const image = provider.image
|
||||
export const video = provider.video
|
||||
export const speech = provider.speech
|
||||
export const transcription = provider.transcription
|
||||
|
||||
@@ -3,13 +3,19 @@ export * as Anthropic from "./anthropic.js"
|
||||
export * as AnthropicCompatible from "./anthropic-compatible.js"
|
||||
export * as AmazonBedrock from "./amazon-bedrock.js"
|
||||
export * as AmazonBedrockMantle from "./amazon-bedrock-mantle.js"
|
||||
export * as AssemblyAI from "./assemblyai.js"
|
||||
export * as Azure from "./azure.js"
|
||||
export * as Baseten from "./baseten.js"
|
||||
export * as BlackForestLabs from "./black-forest-labs.js"
|
||||
export * as Cartesia from "./cartesia.js"
|
||||
export * as Cerebras from "./cerebras.js"
|
||||
export * as CloudflareAIGateway from "./cloudflare-ai-gateway.js"
|
||||
export * as CloudflareWorkersAI from "./cloudflare-workers-ai.js"
|
||||
export * as DeepInfra from "./deepinfra.js"
|
||||
export * as Deepgram from "./deepgram.js"
|
||||
export * as DeepSeek from "./deepseek.js"
|
||||
export * as ElevenLabs from "./elevenlabs.js"
|
||||
export * as Fal from "./fal.js"
|
||||
export * as Fireworks from "./fireworks.js"
|
||||
export * as Google from "./google.js"
|
||||
export * as GoogleVertex from "./google-vertex.js"
|
||||
@@ -26,6 +32,9 @@ export * as OpenAICompatible from "./openai-compatible.js"
|
||||
export * as OpenAICompatibleResponses from "./openai-compatible-responses.js"
|
||||
export * as OpenCodeZen from "./opencode-zen.js"
|
||||
export * as OpenRouter from "./openrouter.js"
|
||||
export * as Replicate from "./replicate.js"
|
||||
export * as Runway from "./runway.js"
|
||||
export * as Stability from "./stability.js"
|
||||
export * as TogetherAI from "./togetherai.js"
|
||||
export * as TypeSafeAI from "./typesafe-ai.js"
|
||||
export * as VercelAIGateway from "./vercel-ai-gateway.js"
|
||||
|
||||
@@ -7,7 +7,8 @@ import { MetaImages } from "../protocols/meta-images.js"
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { HttpOptions, ProviderID, ToolDefinition, type ModelID } from "../schema/index.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, ToolDefinition, type ModelID, type OpenString } from "../schema/index.js"
|
||||
import type { OpenResponsesProviderOptionsInput } from "./open-responses-options.js"
|
||||
|
||||
export const id = ProviderID.make("meta")
|
||||
@@ -48,8 +49,8 @@ export const webSearch = (options: WebSearchOptions = {}) =>
|
||||
|
||||
export interface ImageGenerationOptions {
|
||||
readonly size?: string
|
||||
readonly outputFormat?: "webp" | "png" | "jpeg" | (string & {})
|
||||
readonly reasoningStrength?: "low" | "high" | (string & {})
|
||||
readonly outputFormat?: OpenString<"webp" | "png" | "jpeg">
|
||||
readonly reasoningStrength?: OpenString<"low" | "high">
|
||||
readonly enableImageSearch?: boolean
|
||||
readonly enableWebSearch?: boolean
|
||||
readonly enableShell?: boolean
|
||||
@@ -139,14 +140,8 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
id: modelID,
|
||||
compatibility: { requireSignature: false },
|
||||
})
|
||||
const image = (modelID: string | ModelID) =>
|
||||
MetaImages.model({
|
||||
id: modelID,
|
||||
baseURL: endpoint ?? baseURL,
|
||||
auth: options.auth,
|
||||
headers: input.headers,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
})
|
||||
const media = MediaRoute.deployment(input, options.auth)
|
||||
const image = (modelID: string | ModelID) => MetaImages.model({ ...media, id: modelID })
|
||||
return { id, model: responses, responses, chat, messages, image, configure }
|
||||
}
|
||||
|
||||
|
||||
@@ -113,19 +113,19 @@ export const configure = (input: Config = {}) => {
|
||||
supportsStore: false,
|
||||
supportsStrictMode: false,
|
||||
supportsPromptCacheKey: true,
|
||||
toolSchema: "moonshot",
|
||||
sanitizer: "moonshot",
|
||||
reasoningField: "reasoning_content",
|
||||
},
|
||||
})
|
||||
const messages = (modelID: string | ModelID) =>
|
||||
messagesRoute.with(defaults).model<MessagesOptionsInput>({
|
||||
id: modelID,
|
||||
compatibility: { requireSignature: false, toolSchema: "moonshot" },
|
||||
compatibility: { requireSignature: false, sanitizer: "moonshot" },
|
||||
})
|
||||
const responses = (modelID: string | ModelID) =>
|
||||
responsesRoute
|
||||
.with(defaults)
|
||||
.model<ResponsesOptionsInput>({ id: modelID, compatibility: { toolSchema: "moonshot" } })
|
||||
.model<ResponsesOptionsInput>({ id: modelID, compatibility: { sanitizer: "moonshot" } })
|
||||
return { id, model: chat, chat, messages, responses, configure }
|
||||
}
|
||||
|
||||
|
||||
@@ -32,10 +32,7 @@ const openAIProviderOptions = (options: OpenAIOptionsInput | undefined): Provide
|
||||
return result
|
||||
}
|
||||
|
||||
export const gpt5DefaultOptions = (
|
||||
modelID: string,
|
||||
options: { readonly textVerbosity?: boolean } = {},
|
||||
): ProviderOptions | undefined => {
|
||||
export const gpt5DefaultOptions = (modelID: string): ProviderOptions | undefined => {
|
||||
const id = modelID.toLowerCase()
|
||||
if (!id.includes("gpt-5") || id.includes("gpt-5-chat") || id.includes("gpt-5-pro")) return undefined
|
||||
return openAIProviderOptions({
|
||||
@@ -47,27 +44,19 @@ export const gpt5DefaultOptions = (
|
||||
// this, callers using the default model facade get reasoning summaries
|
||||
// they cannot replay statelessly.
|
||||
include: ["reasoning.encrypted_content"],
|
||||
textVerbosity:
|
||||
options.textVerbosity === true && id.includes("gpt-5.") && !id.includes("codex") && !id.includes("-chat")
|
||||
? "low"
|
||||
: undefined,
|
||||
})
|
||||
}
|
||||
|
||||
export const openAIDefaultOptions = (
|
||||
modelID: string,
|
||||
options: { readonly textVerbosity?: boolean } = {},
|
||||
): ProviderOptions | undefined =>
|
||||
mergeProviderOptions(openAIProviderOptions({ store: false }), gpt5DefaultOptions(modelID, options))
|
||||
export const openAIDefaultOptions = (modelID: string): ProviderOptions | undefined =>
|
||||
mergeProviderOptions(openAIProviderOptions({ store: false }), gpt5DefaultOptions(modelID))
|
||||
|
||||
export const withOpenAIOptions = <Options extends { readonly providerOptions?: OpenAIProviderOptionsInput }>(
|
||||
modelID: string,
|
||||
options: Options,
|
||||
defaults: { readonly textVerbosity?: boolean } = {},
|
||||
): Omit<Options, "providerOptions"> & { readonly providerOptions?: ProviderOptions } => {
|
||||
return {
|
||||
...options,
|
||||
providerOptions: mergeProviderOptions(openAIDefaultOptions(modelID, defaults), options.providerOptions),
|
||||
providerOptions: mergeProviderOptions(openAIDefaultOptions(modelID), options.providerOptions),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,26 @@
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import type { Route, RouteDefaultsInput, CompactionOperations } from "../route/client.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { HttpOptions, ProviderID, ToolDefinition, mergeHttpOptions, type ModelID } from "../schema/index.js"
|
||||
import {
|
||||
HttpOptions,
|
||||
ProviderID,
|
||||
ToolDefinition,
|
||||
mergeHttpOptions,
|
||||
type ModelID,
|
||||
type OpenString,
|
||||
} from "../schema/index.js"
|
||||
import * as OpenAIChat from "../protocols/openai-chat.js"
|
||||
import * as OpenAIResponses from "../protocols/openai-responses.js"
|
||||
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { OpenAIImages, type OpenAIImageString } from "../protocols/openai-images.js"
|
||||
import { OpenAIImages } from "../protocols/openai-images.js"
|
||||
import { OpenAISpeech } from "../protocols/openai-speech.js"
|
||||
import { OpenAITranscription } from "../protocols/openai-transcription.js"
|
||||
|
||||
export type { OpenAIOptionsInput, OpenAIResponseIncludable } from "./openai-options.js"
|
||||
export type { OpenAIImageOptions } from "../protocols/openai-images.js"
|
||||
export type { OpenAISpeechOptions } from "../protocols/openai-speech.js"
|
||||
export type { OpenAITranscriptionOptions } from "../protocols/openai-transcription.js"
|
||||
|
||||
export const id = ProviderID.make("openai")
|
||||
|
||||
@@ -25,14 +37,14 @@ export type Config = RouteDefaultsInput &
|
||||
}
|
||||
|
||||
export interface ImageGenerationOptions {
|
||||
readonly action?: OpenAIImageString<"auto" | "generate" | "edit">
|
||||
readonly background?: OpenAIImageString<"auto" | "opaque" | "transparent">
|
||||
readonly inputFidelity?: OpenAIImageString<"low" | "high">
|
||||
readonly action?: OpenString<"auto" | "generate" | "edit">
|
||||
readonly background?: OpenString<"auto" | "opaque" | "transparent">
|
||||
readonly inputFidelity?: OpenString<"low" | "high">
|
||||
readonly outputCompression?: number
|
||||
readonly outputFormat?: OpenAIImageString<"png" | "jpeg" | "webp">
|
||||
readonly outputFormat?: OpenString<"png" | "jpeg" | "webp">
|
||||
readonly partialImages?: number
|
||||
readonly quality?: OpenAIImageString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
|
||||
readonly size?: OpenAIImageString<
|
||||
readonly quality?: OpenString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
|
||||
readonly size?: OpenString<
|
||||
"auto" | "256x256" | "512x512" | "1024x1024" | "1536x1024" | "1024x1536" | "1792x1024" | "1024x1792"
|
||||
>
|
||||
}
|
||||
@@ -88,24 +100,24 @@ export const configure = (input: Config = {}) => {
|
||||
const modelDefaults = defaults(input)
|
||||
const responses = (id: string | ModelID) =>
|
||||
responsesRoute
|
||||
.with(withOpenAIOptions(id, modelDefaults, { textVerbosity: true }))
|
||||
.with(withOpenAIOptions(id, modelDefaults))
|
||||
.model<OpenAIProviderOptionsInput>({ id })
|
||||
const chat = (id: string | ModelID) =>
|
||||
chatRoute.with(withOpenAIOptions(id, modelDefaults)).model<OpenAIProviderOptionsInput>({
|
||||
id,
|
||||
compatibility: { supportsPromptCacheKey: true },
|
||||
})
|
||||
const image = (modelID: string | ModelID) =>
|
||||
OpenAIImages.model({
|
||||
id: modelID,
|
||||
auth: auth(input),
|
||||
baseURL: input.baseURL,
|
||||
headers: input.headers,
|
||||
http: mergeHttpOptions(
|
||||
input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
input.queryParams === undefined ? undefined : new HttpOptions({ query: input.queryParams }),
|
||||
),
|
||||
})
|
||||
const deployment = MediaRoute.deployment(input, auth(input))
|
||||
const media = {
|
||||
...deployment,
|
||||
http: mergeHttpOptions(
|
||||
deployment.http,
|
||||
input.queryParams === undefined ? undefined : new HttpOptions({ query: input.queryParams }),
|
||||
),
|
||||
}
|
||||
const image = (modelID: string | ModelID) => OpenAIImages.model({ ...media, id: modelID })
|
||||
const speech = (modelID: string | ModelID) => OpenAISpeech.model({ ...media, id: modelID })
|
||||
const transcription = (modelID: string | ModelID) => OpenAITranscription.model({ ...media, id: modelID })
|
||||
|
||||
return {
|
||||
id,
|
||||
@@ -113,6 +125,8 @@ export const configure = (input: Config = {}) => {
|
||||
responses,
|
||||
chat,
|
||||
image,
|
||||
speech,
|
||||
transcription,
|
||||
configure,
|
||||
}
|
||||
}
|
||||
@@ -159,3 +173,5 @@ export const chatModel: ProviderPackage.Definition<Settings, OpenAIProviderOptio
|
||||
export const responses = provider.responses
|
||||
export const chat = provider.chat
|
||||
export const image = provider.image
|
||||
export const speech = provider.speech
|
||||
export const transcription = provider.transcription
|
||||
|
||||
@@ -20,7 +20,7 @@ export const configure = (input: Options = {}) => {
|
||||
auth: AuthOptions.bearer(input, "OPENCODE_API_KEY"),
|
||||
baseURL: input.baseURL ?? baseURL,
|
||||
headers: input.headers,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
http: HttpOptions.make(input.http),
|
||||
})
|
||||
return { id, experimental: { evaluation }, configure }
|
||||
}
|
||||
|
||||
@@ -3,29 +3,27 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { HttpOptions, ProviderID, type CacheHint, type ModelID } from "../schema/index.js"
|
||||
import { HttpOptions, ProviderID, type CacheHint, type ModelID, type OpenString } from "../schema/index.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { SystemOne } from "../experimental/system-one.js"
|
||||
import { OpenAIChat } from "../protocols/openai-chat.js"
|
||||
import { newBreakpoints, ttlBucket } from "../protocols/utils/cache.js"
|
||||
import { isRecord } from "../protocols/shared.js"
|
||||
import { isRecord, ProviderShared } from "../protocols/shared.js"
|
||||
|
||||
export const id = ProviderID.make("openrouter")
|
||||
const baseURL = "https://openrouter.ai/api/v1"
|
||||
const ADAPTER = "openrouter"
|
||||
|
||||
type OpenRouterString<Known extends string> = Known | (string & {})
|
||||
|
||||
export interface OpenRouterProviderRouting {
|
||||
readonly [key: string]: unknown
|
||||
readonly order?: ReadonlyArray<string>
|
||||
readonly allow_fallbacks?: boolean
|
||||
readonly require_parameters?: boolean
|
||||
readonly data_collection?: OpenRouterString<"allow" | "deny">
|
||||
readonly data_collection?: OpenString<"allow" | "deny">
|
||||
readonly only?: ReadonlyArray<string>
|
||||
readonly ignore?: ReadonlyArray<string>
|
||||
readonly quantizations?: ReadonlyArray<string>
|
||||
readonly sort?: OpenRouterString<"price" | "throughput" | "latency">
|
||||
readonly sort?: OpenString<"price" | "throughput" | "latency">
|
||||
readonly max_price?: Readonly<{
|
||||
prompt?: number | string
|
||||
completion?: number | string
|
||||
@@ -41,7 +39,7 @@ export type OpenRouterPlugin =
|
||||
id: "web"
|
||||
max_results?: number
|
||||
search_prompt?: string
|
||||
engine?: OpenRouterString<"native" | "exa">
|
||||
engine?: OpenString<"native" | "exa">
|
||||
}>
|
||||
| Readonly<{ id: "file-parser"; max_files?: number; pdf?: { engine?: string } }>
|
||||
| Readonly<{ id: "moderation" }>
|
||||
@@ -58,7 +56,7 @@ export interface OpenRouterOptions {
|
||||
readonly reasoning?: Readonly<{
|
||||
enabled?: boolean
|
||||
exclude?: boolean
|
||||
effort?: OpenRouterString<"none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max">
|
||||
effort?: OpenString<"none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max">
|
||||
max_tokens?: number
|
||||
}>
|
||||
readonly usage?: boolean | Readonly<{ include: boolean }>
|
||||
@@ -66,7 +64,7 @@ export interface OpenRouterOptions {
|
||||
readonly web_search_options?: Readonly<{
|
||||
max_results?: number
|
||||
search_prompt?: string
|
||||
engine?: OpenRouterString<"native" | "exa">
|
||||
engine?: OpenString<"native" | "exa">
|
||||
}>
|
||||
}
|
||||
|
||||
@@ -125,7 +123,7 @@ export const protocol = Protocol.make({
|
||||
return {
|
||||
...body,
|
||||
messages,
|
||||
...bodyOptions(request.providerOptions),
|
||||
...bodyOptions(request.providerOptions, request.generation?.maxTokens),
|
||||
} as OpenRouterBody
|
||||
}),
|
||||
),
|
||||
@@ -145,7 +143,14 @@ const cacheControl = () => {
|
||||
}
|
||||
}
|
||||
|
||||
const bodyOptions = (input: unknown) => {
|
||||
// OpenRouter forwards `reasoning.max_tokens` as the upstream thinking budget. Upstreams such as Anthropic and Alibaba
|
||||
// reject one that is not below the output limit; 1,024 is Anthropic's minimum budget.
|
||||
const fitReasoning = (reasoning: Record<string, unknown>, maxTokens: number | undefined) =>
|
||||
typeof reasoning.max_tokens === "number"
|
||||
? { ...reasoning, max_tokens: ProviderShared.fitThinkingBudget(reasoning.max_tokens, maxTokens, 1_024) }
|
||||
: reasoning
|
||||
|
||||
const bodyOptions = (input: unknown, maxTokens: number | undefined) => {
|
||||
const openrouter = isRecord(input) ? input : {}
|
||||
const { usage, models, provider, plugins, web_search_options, debug, user, reasoning, promptCacheKey, ...options } =
|
||||
openrouter
|
||||
@@ -164,7 +169,7 @@ const bodyOptions = (input: unknown) => {
|
||||
...(isRecord(web_search_options) ? { web_search_options } : {}),
|
||||
...(isRecord(debug) ? { debug } : {}),
|
||||
...(typeof user === "string" ? { user } : {}),
|
||||
...(isRecord(reasoning) ? { reasoning } : {}),
|
||||
...(isRecord(reasoning) ? { reasoning: fitReasoning(reasoning, maxTokens) } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,7 +203,7 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
auth: AuthOptions.bearer(input, "OPENROUTER_API_KEY"),
|
||||
baseURL: input.baseURL ?? baseURL,
|
||||
headers: input.headers,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
http: HttpOptions.make(input.http),
|
||||
})
|
||||
return {
|
||||
id,
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { ReplicateImages } from "../protocols/replicate-images.js"
|
||||
|
||||
export type { ReplicateImageOptions } from "../protocols/replicate-images.js"
|
||||
|
||||
export const id = ProviderID.make("replicate")
|
||||
|
||||
export type Config = ProviderAuthOption<"optional"> & {
|
||||
readonly baseURL?: string
|
||||
/** `{ Prefer: "wait=60" }` holds the submission open until the prediction finishes (up to 60 seconds). */
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "REPLICATE_API_TOKEN")
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
const image = (modelID: string | ModelID) => ReplicateImages.model({ ...media, id: modelID })
|
||||
return {
|
||||
id,
|
||||
image,
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
export const image = provider.image
|
||||
@@ -0,0 +1,29 @@
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { RunwayVideo } from "../protocols/runway-video.js"
|
||||
|
||||
export type { RunwayVideoOptions } from "../protocols/runway-video.js"
|
||||
|
||||
export const id = ProviderID.make("runway")
|
||||
|
||||
export type Config = ProviderAuthOption<"optional"> & {
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "RUNWAYML_API_SECRET")
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
const video = (modelID: string | ModelID) => RunwayVideo.model({ ...media, id: modelID })
|
||||
return {
|
||||
id,
|
||||
video,
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
export const video = provider.video
|
||||
@@ -0,0 +1,30 @@
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { StabilityImages } from "../protocols/stability-images.js"
|
||||
|
||||
export type { StabilityImageOptions, StabilityUpscaleOptions } from "../protocols/stability-images.js"
|
||||
|
||||
export const id = ProviderID.make("stability")
|
||||
|
||||
export type Config = ProviderAuthOption<"optional"> & {
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "STABILITY_API_KEY")
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
return {
|
||||
id,
|
||||
image: (modelID: string | ModelID) => StabilityImages.model({ ...media, id: modelID }),
|
||||
upscale: () => StabilityImages.upscaleModel(media),
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
export const image = provider.image
|
||||
export const upscale = provider.upscale
|
||||
@@ -20,7 +20,7 @@ export const configure = (input: Options = {}) => {
|
||||
auth: AuthOptions.bearer(input, "TYPESAFE_API_KEY"),
|
||||
baseURL: input.baseURL ?? baseURL,
|
||||
headers: input.headers,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
http: HttpOptions.make(input.http),
|
||||
})
|
||||
return { id, experimental: { evaluation }, configure }
|
||||
}
|
||||
|
||||
@@ -67,7 +67,7 @@ export const configure = (input: Options = {}) => {
|
||||
EvaluationModel.make<EvaluationOptions>({
|
||||
id: modelID,
|
||||
provider: id,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
http: HttpOptions.make(input.http),
|
||||
route: {
|
||||
id: "vercel-evaluation",
|
||||
evaluate: (req, send) =>
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { OpenAIChat } from "../protocols/openai-chat.js"
|
||||
import { OpenResponsesChannel } from "../protocols/open-responses-channel.js"
|
||||
import { XAIResponses } from "../protocols/xai-responses.js"
|
||||
import { XAIImages } from "../protocols/xai-images.js"
|
||||
import { XAIVideo } from "../protocols/xai-video.js"
|
||||
import type { OpenAIOptionsInput } from "./openai-options.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
|
||||
@@ -27,6 +29,7 @@ export type Settings = ProviderPackage.Settings &
|
||||
}
|
||||
|
||||
export type { XAIImageOptions } from "../protocols/xai-images.js"
|
||||
export type { XAIVideoOptions } from "../protocols/xai-video.js"
|
||||
|
||||
const RESPONSES_WEBSOCKET_ROTATE_AFTER_MS = 24 * 60 * 1000
|
||||
|
||||
@@ -87,20 +90,14 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
const chatRoute = configuredChatRoute(input)
|
||||
const responses = (modelID: string | ModelID) => responsesRoute.model<XAIProviderOptionsInput>({ id: modelID })
|
||||
const chat = (modelID: string | ModelID) => chatRoute.model<XAIProviderOptionsInput>({ id: modelID })
|
||||
const image = (modelID: string | ModelID) =>
|
||||
XAIImages.model({
|
||||
id: modelID,
|
||||
auth: auth(input),
|
||||
baseURL: input.baseURL ?? baseURL,
|
||||
headers: input.headers,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
})
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
return {
|
||||
id,
|
||||
model: responses,
|
||||
responses,
|
||||
chat,
|
||||
image,
|
||||
image: (modelID: string | ModelID) => XAIImages.model({ ...media, id: modelID }),
|
||||
video: (modelID: string | ModelID) => XAIVideo.model({ ...media, id: modelID }),
|
||||
configure,
|
||||
}
|
||||
}
|
||||
@@ -121,3 +118,4 @@ export const model: ProviderPackage.Definition<
|
||||
export const responses = provider.responses
|
||||
export const chat = provider.chat
|
||||
export const image = provider.image
|
||||
export const video = provider.video
|
||||
|
||||
@@ -5,7 +5,8 @@ import { OpenAIChat } from "../protocols/openai-chat.js"
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
|
||||
export const id = ProviderID.make("zai")
|
||||
|
||||
@@ -48,14 +49,8 @@ export const configure = (input: Config = {}) => {
|
||||
auth: auth(input),
|
||||
})
|
||||
.model<ChatOptionsInput>({ id: modelID, compatibility: ZAIChat.compatibility })
|
||||
const image = (modelID: string | ModelID) =>
|
||||
ZAIImages.model({
|
||||
id: modelID,
|
||||
auth: auth(input),
|
||||
baseURL: input.baseURL,
|
||||
headers: input.headers,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
})
|
||||
const media = MediaRoute.deployment(input, auth(input))
|
||||
const image = (modelID: string | ModelID) => ZAIImages.model({ ...media, id: modelID })
|
||||
|
||||
return {
|
||||
id,
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
import { Config, Effect, Redacted } from "effect"
|
||||
import { Config, Effect, Option, Redacted } from "effect"
|
||||
import { Headers } from "effect/unstable/http"
|
||||
import { AuthenticationError, InvalidRequestError, AIError, type HttpOptions } from "../schema/index.js"
|
||||
import { AuthenticationError, AIError, type HttpOptions } from "../schema/index.js"
|
||||
|
||||
export class MissingCredentialError extends Error {
|
||||
readonly _tag = "MissingCredentialError"
|
||||
|
||||
constructor(readonly source: string) {
|
||||
super(`Missing auth credential: ${source}`)
|
||||
constructor(
|
||||
readonly source: string,
|
||||
message = `Missing auth credential: ${source}`,
|
||||
) {
|
||||
super(message)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +19,7 @@ type Secret = string | Redacted.Redacted | Config.Config<string | Redacted.Redac
|
||||
|
||||
export interface AuthInput {
|
||||
readonly request: { readonly http?: HttpOptions }
|
||||
readonly method: "POST" | "GET"
|
||||
readonly method: "POST" | "GET" | "PUT" | "DELETE"
|
||||
readonly url: string
|
||||
readonly body: string
|
||||
readonly headers: Headers.Headers
|
||||
@@ -89,7 +92,14 @@ export const optional = (secret: Secret | undefined, source = "optional value")
|
||||
? credential(Effect.fail(new MissingCredentialError(source)))
|
||||
: credentialFromSecret(secret, source)
|
||||
|
||||
export const config = (name: string) => credentialFromSecret(Config.redacted(name), name)
|
||||
export const config = (name: string) =>
|
||||
credential(
|
||||
Effect.gen(function* () {
|
||||
const secret = yield* Config.option(Config.redacted(name))
|
||||
if (Option.isSome(secret) && Redacted.value(secret.value) !== "") return secret.value
|
||||
return yield* Effect.fail(new MissingCredentialError(name, `${name} is not set`))
|
||||
}),
|
||||
)
|
||||
|
||||
export const effect = (load: Effect.Effect<Redacted.Redacted, CredentialError>) => credential(load)
|
||||
|
||||
@@ -134,16 +144,21 @@ export function bearerHeader(name: string, source?: Secret | Credential) {
|
||||
return render(source)
|
||||
}
|
||||
|
||||
/** `Authorization: <scheme> <secret>` for providers whose scheme is not `Bearer`, such as fal's `Key`. */
|
||||
export function scheme(name: string): (source: Secret | Credential) => Definition
|
||||
export function scheme(name: string, source: Secret | Credential): Definition
|
||||
export function scheme(name: string, source?: Secret | Credential) {
|
||||
const render = (input: Secret | Credential) =>
|
||||
fromCredential(credentialInput(input), (secret) => ({ authorization: `${name} ${secret}` }))
|
||||
if (source === undefined) return render
|
||||
return render(source)
|
||||
}
|
||||
|
||||
const toAIError = (error: AuthError): AIError => {
|
||||
if (error instanceof MissingCredentialError || error instanceof Config.ConfigError) {
|
||||
return new AIError({
|
||||
reason:
|
||||
error instanceof MissingCredentialError
|
||||
? new AuthenticationError({ message: error.message, cause: error })
|
||||
: new InvalidRequestError({ message: `Failed to resolve auth config: ${error.message}`, cause: error }),
|
||||
})
|
||||
}
|
||||
return error
|
||||
if (error instanceof AIError) return error
|
||||
const message =
|
||||
error instanceof MissingCredentialError ? error.message : `Failed to resolve auth config: ${error.message}`
|
||||
return new AIError({ reason: new AuthenticationError({ message, cause: error }) })
|
||||
}
|
||||
|
||||
export const toEffect =
|
||||
|
||||
@@ -11,7 +11,8 @@ import { applyEffortUpdates } from "../effort-updates.js"
|
||||
import { normalizeToolHistory } from "../tool-history.js"
|
||||
import { sanitizeSurrogates } from "../utils/sanitize.js"
|
||||
import * as ProviderShared from "../protocols/shared.js"
|
||||
import type { ProtocolID, ProviderOptions } from "../schema/index.js"
|
||||
import { ToolSchemaProjection } from "../protocols/utils/tool-schema.js"
|
||||
import type { LanguageModelSanitizerCompatibility, ProtocolID, ProviderOptions } from "../schema/index.js"
|
||||
import {
|
||||
AIError,
|
||||
CompactionResponse,
|
||||
@@ -57,6 +58,7 @@ export interface Route<
|
||||
readonly defaults: RouteDefaults
|
||||
readonly body: RouteBody<Body>
|
||||
readonly supportsEffortUpdates?: (request: LLMRequest) => boolean
|
||||
readonly sanitizer?: LanguageModelSanitizerCompatibility
|
||||
readonly with: {
|
||||
<Next extends CompactionOperations | undefined>(
|
||||
patch: RoutePatch<Body, Prepared> & { readonly compact: Next },
|
||||
@@ -152,7 +154,7 @@ const mergeRouteDefaults = (base: RouteDefaults | undefined, patch: RouteDefault
|
||||
providerOptions: mergeProviderOptions(base?.providerOptions, patch.providerOptions),
|
||||
http: mergeHttpOptions(
|
||||
base?.http,
|
||||
httpOptions(patch.http),
|
||||
HttpOptions.make(patch.http),
|
||||
headers === undefined ? undefined : new HttpOptions({ headers }),
|
||||
),
|
||||
}
|
||||
@@ -172,11 +174,6 @@ const mergeHeaders = (...items: ReadonlyArray<Record<string, string> | undefined
|
||||
export const generationOptions = (input: GenerationOptions.Input | undefined) =>
|
||||
input === undefined ? undefined : GenerationOptions.make(input)
|
||||
|
||||
export const httpOptions = (input: HttpOptionsInput | undefined) => {
|
||||
if (input === undefined) return input
|
||||
return HttpOptions.make(input)
|
||||
}
|
||||
|
||||
export interface Interface {
|
||||
readonly compact: CompactMethod
|
||||
readonly stream: StreamMethod
|
||||
@@ -261,7 +258,9 @@ const unsupportedCompaction = (request: LLMRequest, mechanism: string | undefine
|
||||
})
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/LLMClient") {}
|
||||
export class LLMClientService extends Context.Service<LLMClientService, Interface>()("@opencode/LLMClient") {}
|
||||
export const Service = LLMClientService
|
||||
export type Service = LLMClientService
|
||||
|
||||
const resolveRequestOptions = (request: LLMRequest) => {
|
||||
const messages = normalizeToolHistory(request.messages)
|
||||
@@ -391,6 +390,7 @@ function makeFromTransport<Body, Prepared, Frame, Event, State>(
|
||||
defaults: routeInput.defaults ?? {},
|
||||
body: protocol.body,
|
||||
supportsEffortUpdates: protocol.supportsEffortUpdates,
|
||||
sanitizer: protocol.sanitizer,
|
||||
with: (patch: RoutePatch<Body, Prepared>) => {
|
||||
const { compact, id, provider, providerMetadataKey, auth, transport, endpoint, ...defaults } = patch
|
||||
return build({
|
||||
@@ -562,7 +562,9 @@ const prepareRequest = (request: LLMRequest) => {
|
||||
tool.type === "tool" ? tool : { ...tool, tools: dedupe(tool.tools) },
|
||||
)
|
||||
const resolved = applyCachePolicy(
|
||||
applyEffortUpdates(LLMRequest.update(sanitized, { tools: dedupe(sanitized.tools) })),
|
||||
applyEffortUpdates(
|
||||
LLMRequest.update(sanitized, { tools: ToolSchemaProjection.tools(dedupe(sanitized.tools), sanitized.model) }),
|
||||
),
|
||||
)
|
||||
const headers = resolved.model.route.headers?.({ request: resolved })
|
||||
return headers === undefined
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { LLMRequest } from "../schema/index.js"
|
||||
import * as ProviderShared from "../protocols/shared.js"
|
||||
|
||||
export interface EndpointInput<Body, Request = LLMRequest> {
|
||||
readonly request: Request
|
||||
@@ -47,6 +46,8 @@ export const merge = <Body, Request = LLMRequest>(
|
||||
query: patch.query === undefined ? base.query : { ...base.query, ...patch.query },
|
||||
})
|
||||
|
||||
export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
|
||||
|
||||
const renderPart = <Body, Request>(part: EndpointPart<Body, Request>, input: EndpointInput<Body, Request>) =>
|
||||
typeof part === "function" ? part(input) : part
|
||||
|
||||
@@ -54,7 +55,7 @@ export const render = <Body, Request = LLMRequest>(
|
||||
endpoint: Definition<Body, Request>,
|
||||
input: EndpointInput<Body, Request>,
|
||||
) => {
|
||||
const url = new URL(`${ProviderShared.trimBaseUrl(endpoint.baseURL ?? "")}${renderPart(endpoint.path, input)}`)
|
||||
const url = new URL(`${trimBaseUrl(endpoint.baseURL ?? "")}${renderPart(endpoint.path, input)}`)
|
||||
for (const [key, value] of Object.entries(endpoint.query ?? {})) url.searchParams.set(key, value)
|
||||
return url
|
||||
}
|
||||
|
||||
@@ -19,4 +19,8 @@ export type HttpMiddleware = (
|
||||
handler: HttpHandler,
|
||||
) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error>
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/AI/RequestExecutor") {}
|
||||
export class RequestExecutorService extends Context.Service<RequestExecutorService, Interface>()(
|
||||
"@opencode/AI/RequestExecutor",
|
||||
) {}
|
||||
export const Service = RequestExecutorService
|
||||
export type Service = RequestExecutorService
|
||||
|
||||
@@ -255,4 +255,20 @@ export const layer: Layer.Layer<Service, never, HttpClient.HttpClient> = Layer.e
|
||||
|
||||
export const fetchLayer = layer.pipe(Layer.provide(FetchHttpClient.layer))
|
||||
|
||||
/** Run `fn` on every request: it sees the raw response before status classification, inside middleware already on `executor`, and outside per-call middleware. */
|
||||
export const middleware = (fn: HttpMiddleware, executor: Layer.Layer<Service> = fetchLayer): Layer.Layer<Service> =>
|
||||
Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const inner = yield* Service
|
||||
return Service.of({
|
||||
execute: (request, next) =>
|
||||
inner.execute(
|
||||
request,
|
||||
next === undefined ? fn : (input, handler) => fn(input, (forwarded) => next(forwarded, handler)),
|
||||
),
|
||||
})
|
||||
}),
|
||||
).pipe(Layer.provide(executor))
|
||||
|
||||
export * as RequestExecutor from "./executor.js"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { Stream } from "effect"
|
||||
import * as ProviderShared from "../protocols/shared.js"
|
||||
import type { AIError } from "../schema/index.js"
|
||||
import { Effect, Stream } from "effect"
|
||||
import { makeParser, type Event } from "effect/unstable/encoding/Sse"
|
||||
import { AIError, InvalidProviderOutputError } from "../schema/index.js"
|
||||
|
||||
/**
|
||||
* Decode a streaming HTTP response body into provider-protocol frames.
|
||||
@@ -12,6 +12,8 @@ import type { AIError } from "../schema/index.js"
|
||||
* `[DONE]`; protocols that use it as a terminal select `sseWithDone`.
|
||||
* - AWS event stream — length-prefixed binary frames with CRC checksums.
|
||||
* Each emitted frame is one parsed binary event record.
|
||||
* - Media streams — newline-delimited JSON (`lines`) or the whole body as one
|
||||
* frame (`document`); chunked binary bodies need no framing.
|
||||
*
|
||||
* The frame type is opaque to this layer; the protocol's event schema decodes
|
||||
* each frame before its state machine handles it.
|
||||
@@ -23,19 +25,90 @@ export interface Definition<Frame> {
|
||||
readonly body?: (frame: Frame) => string | undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* `framing` step for Server-Sent Events. Decodes UTF-8, runs the SSE channel
|
||||
* decoder, optionally filters named events, and drops empty events and known
|
||||
* keepalives that proxies send as data. `[DONE]` is dropped by default or
|
||||
* retained for protocols that use it as their stream boundary. Retry control events are ignored without
|
||||
* interrupting the stream. Decoder failures become provider output errors so
|
||||
* the public error channel stays `AIError`.
|
||||
*/
|
||||
export const sseFraming = (
|
||||
bytes: Stream.Stream<Uint8Array, AIError>,
|
||||
events?: ReadonlySet<string>,
|
||||
includeDone = false,
|
||||
): Stream.Stream<string, AIError> =>
|
||||
bytes.pipe(
|
||||
Stream.decodeText(),
|
||||
Stream.mapAccumEffect(
|
||||
() => {
|
||||
const output: Event[] = []
|
||||
return {
|
||||
output,
|
||||
parser: makeParser((event) => {
|
||||
if (event._tag === "Event") output.push(event)
|
||||
}),
|
||||
}
|
||||
},
|
||||
(state, chunk) =>
|
||||
Effect.gen(function* () {
|
||||
const error = state.parser.feed(chunk)
|
||||
if (error)
|
||||
return yield* new AIError({
|
||||
reason: new InvalidProviderOutputError({
|
||||
route: "sse",
|
||||
message: error.message,
|
||||
body: chunk,
|
||||
cause: error,
|
||||
}),
|
||||
})
|
||||
return [state, state.output.splice(0)] as const
|
||||
}),
|
||||
),
|
||||
Stream.filter(
|
||||
(event) =>
|
||||
(events === undefined || events.has(event.event)) &&
|
||||
event.data.length > 0 &&
|
||||
// Some OpenAI-compatible proxies serialize an empty flush as a bare
|
||||
// `data: null`, between events or after `[DONE]`. No protocol has a
|
||||
// null event, so it carries nothing and must not abort the stream.
|
||||
event.data !== "null" &&
|
||||
// Vertex AI partner models (e.g. `xai/grok-4.6`) send their SSE
|
||||
// keepalive comment as `data: : keepalive` while reasoning.
|
||||
event.data !== ": keepalive" &&
|
||||
(event.data !== "[DONE]" || includeDone || (events !== undefined && event.event !== "message")),
|
||||
),
|
||||
Stream.map((event) => event.data),
|
||||
)
|
||||
|
||||
/** Server-Sent Events framing. Used by every JSON-streaming HTTP provider. */
|
||||
export const sse: Definition<string> = { id: "sse", frame: ProviderShared.sseFraming }
|
||||
export const sse: Definition<string> = { id: "sse", frame: sseFraming }
|
||||
|
||||
/** Server-Sent Events framing that retains the conventional `[DONE]` sentinel. */
|
||||
export const sseWithDone: Definition<string> = {
|
||||
id: "sse",
|
||||
frame: (bytes) => ProviderShared.sseFraming(bytes, undefined, true),
|
||||
frame: (bytes) => sseFraming(bytes, undefined, true),
|
||||
}
|
||||
|
||||
/** SSE framing restricted to protocol-recognized event names. */
|
||||
export const sseEvents = (events: ReadonlySet<string>): Definition<string> => ({
|
||||
id: "sse",
|
||||
frame: (bytes) => ProviderShared.sseFraming(bytes, events),
|
||||
frame: (bytes) => sseFraming(bytes, events),
|
||||
})
|
||||
|
||||
export const lines: Definition<string> = {
|
||||
id: "lines",
|
||||
frame: (bytes) =>
|
||||
bytes.pipe(
|
||||
Stream.decodeText(),
|
||||
Stream.splitLines,
|
||||
Stream.filter((line) => line.trim().length > 0),
|
||||
),
|
||||
}
|
||||
|
||||
export const document: Definition<string> = {
|
||||
id: "document",
|
||||
frame: (bytes) => Stream.fromEffect(Stream.mkString(bytes.pipe(Stream.decodeText()))),
|
||||
}
|
||||
|
||||
export * as Framing from "./framing.js"
|
||||
|
||||
@@ -7,7 +7,7 @@ export type {
|
||||
RouteDefaultsInput,
|
||||
AnyRoute,
|
||||
Interface as LLMClientShape,
|
||||
Service as LLMClientService,
|
||||
LLMClientService,
|
||||
StreamOptions,
|
||||
CompactMethod,
|
||||
CompactionOperations,
|
||||
|
||||
@@ -1,17 +1,54 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { Clock, Duration, Effect, Schema, type Stream } from "effect"
|
||||
import { HttpClientResponse } from "effect/unstable/http"
|
||||
import { AIError, HttpContext, InvalidProviderOutputError } from "../schema/index.js"
|
||||
import type { Snapshot, Status } from "../generation.js"
|
||||
import { Media } from "../media.js"
|
||||
import type { AuthInput } from "./auth.js"
|
||||
import {
|
||||
AIError,
|
||||
ContentPolicyError,
|
||||
HttpContext,
|
||||
InvalidProviderOutputError,
|
||||
InvalidRequestError,
|
||||
ProviderID,
|
||||
ProviderInternalError,
|
||||
UnsupportedOperationError,
|
||||
} from "../schema/index.js"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bodies
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type Body =
|
||||
| { readonly type: "json"; readonly value: Record<string, unknown> }
|
||||
| { readonly type: "multipart"; readonly value: FormData }
|
||||
/** Array values become repeated parameters (`keyterm=a&keyterm=b`). */
|
||||
export type Query = Readonly<Record<string, string | ReadonlyArray<string>>>
|
||||
|
||||
export const json = (value: Record<string, unknown>): Body => ({ type: "json", value })
|
||||
/** `query` is appended to the endpoint URL before the route and caller `http.query` overlays. */
|
||||
export type Body =
|
||||
| { readonly type: "json"; readonly value: Record<string, unknown>; readonly query?: Query }
|
||||
| { readonly type: "multipart"; readonly value: FormData }
|
||||
| {
|
||||
readonly type: "binary"
|
||||
readonly value: Uint8Array
|
||||
readonly contentType: string
|
||||
readonly query?: Query
|
||||
}
|
||||
|
||||
export const json = (value: Record<string, unknown>, query?: Query): Body => ({
|
||||
type: "json",
|
||||
value,
|
||||
query,
|
||||
})
|
||||
export const multipart = (value: FormData): Body => ({ type: "multipart", value })
|
||||
export const binary = (value: Uint8Array, contentType: string, query?: Query): Body => ({
|
||||
type: "binary",
|
||||
value,
|
||||
contentType,
|
||||
query,
|
||||
})
|
||||
|
||||
export type Send = (path: string, body: Body) => Effect.Effect<HttpClientResponse.HttpClientResponse, AIError>
|
||||
|
||||
/** Runs after unsupported-field rejection and before `body.from`, for providers that need an upload first. */
|
||||
export type Prepare<Request> = (request: Request, send: Send) => Effect.Effect<Request, AIError>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Protocol kinds
|
||||
@@ -26,7 +63,7 @@ export interface DecodeContext<Request> {
|
||||
export interface Inline<Request, Response> {
|
||||
readonly kind: "inline"
|
||||
readonly id: string
|
||||
readonly name: string
|
||||
readonly provider: ProviderID
|
||||
/** Common request fields this protocol cannot lower; the route rejects them before `body.from` runs. */
|
||||
readonly unsupported?: ReadonlyArray<keyof Request & string>
|
||||
readonly body: { readonly from: (request: Request) => Effect.Effect<Body, AIError> }
|
||||
@@ -39,11 +76,113 @@ export interface Inline<Request, Response> {
|
||||
}
|
||||
|
||||
export const inline = <Request, Response>(
|
||||
input: Omit<Inline<Request, Response>, "kind">,
|
||||
): Inline<Request, Response> => ({
|
||||
kind: "inline",
|
||||
...input,
|
||||
})
|
||||
route: Identity,
|
||||
input: Omit<Inline<Request, Response>, "kind" | "id" | "provider">,
|
||||
): Inline<Request, Response> => ({ kind: "inline", id: route.id, provider: route.provider, ...input })
|
||||
|
||||
/** What `start` learned from the submission response: the route-owned handle plus the first observation. */
|
||||
export interface Started<Token> {
|
||||
readonly token: Token
|
||||
readonly snapshot: Snapshot
|
||||
}
|
||||
|
||||
/**
|
||||
* A follow-up call's inputs: the decoded token and the auth headers the route sent, so a protocol can attach them
|
||||
* to output URLs that require the same credentials to download (Veo). `materialize` downloads an output through the
|
||||
* route's executor, for URLs that expire too soon to hand back (BFL).
|
||||
*/
|
||||
export interface PollContext<Token> {
|
||||
readonly token: Token
|
||||
readonly auth: Record<string, string>
|
||||
readonly materialize: (asset: Media.Asset) => Effect.Effect<Media.Asset, AIError>
|
||||
}
|
||||
|
||||
/**
|
||||
* Submit, then poll. `start` posts the body to the route endpoint; `status`, `result`, and `cancel` are follow-up
|
||||
* calls addressed by the token. Paths are relative to the route base URL unless the provider hands back absolute
|
||||
* URLs (fal `status_url`), in which case they are used verbatim. `result` is always its own GET: providers that
|
||||
* return the output inside the status body (Veo, xAI, Runway) point `result.path` at the status path and decode the
|
||||
* same document, so `Generation.await` and `Video.resume(...).await()` behave identically everywhere.
|
||||
*/
|
||||
export interface Queued<Request, Response, Token> {
|
||||
readonly kind: "queued"
|
||||
readonly id: string
|
||||
readonly provider: ProviderID
|
||||
/** Common request fields this protocol cannot lower; the route rejects them before `start.body.from` runs. */
|
||||
readonly unsupported?: ReadonlyArray<keyof Request & string>
|
||||
/** Serializable handle. `Generation.token` carries the encoded form so it can be persisted and resumed elsewhere. */
|
||||
readonly token: Schema.Codec<Token, unknown>
|
||||
readonly start: {
|
||||
readonly prepare?: Prepare<Request>
|
||||
readonly body: { readonly from: (request: Request) => Effect.Effect<Body, AIError> }
|
||||
readonly decode: (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: DecodeContext<Request>,
|
||||
) => Effect.Effect<Started<Token>, AIError>
|
||||
}
|
||||
readonly status: {
|
||||
readonly path: (token: Token) => string
|
||||
readonly decode: (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: PollContext<Token>,
|
||||
) => Effect.Effect<Snapshot, AIError>
|
||||
}
|
||||
readonly result: {
|
||||
readonly path: (token: Token) => string
|
||||
readonly decode: (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: PollContext<Token>,
|
||||
) => Effect.Effect<Response, AIError>
|
||||
}
|
||||
readonly cancel?: {
|
||||
readonly method: AuthInput["method"]
|
||||
readonly path: (token: Token) => string
|
||||
/**
|
||||
* Fetch a fresh status first and skip the call for terminal generations, for providers whose cancel endpoint
|
||||
* destroys finished work (Runway's `DELETE /v1/tasks/{id}` deletes completed tasks and their outputs).
|
||||
*/
|
||||
readonly activeOnly?: boolean
|
||||
}
|
||||
}
|
||||
|
||||
export const queued = <Request, Response, Token>(
|
||||
route: Identity,
|
||||
input: Omit<Queued<Request, Response, Token>, "kind" | "id" | "provider">,
|
||||
): Queued<Request, Response, Token> => ({ kind: "queued", id: route.id, provider: route.provider, ...input })
|
||||
|
||||
export type Mode = "generate" | "stream"
|
||||
|
||||
export type Addressed<Request> = Request & { readonly mode: Mode }
|
||||
|
||||
export interface ResponseContext<Request> extends DecodeContext<Addressed<Request>> {
|
||||
readonly http: HttpContext
|
||||
}
|
||||
|
||||
/**
|
||||
* One request whose body is parsed incrementally, like LLM protocols: `frames` → `step`* → `finish`. `generate` and
|
||||
* `stream` share this state machine; `request.mode` lets a protocol pick a different body, path, or framing.
|
||||
*/
|
||||
export interface Streamed<Request, Event, Frame, State> {
|
||||
readonly kind: "stream"
|
||||
readonly id: string
|
||||
readonly provider: ProviderID
|
||||
/** Common request fields this protocol cannot lower; the route rejects them before `body.from` runs. */
|
||||
readonly unsupported?: ReadonlyArray<keyof Request & string>
|
||||
readonly body: { readonly from: (request: Addressed<Request>) => Effect.Effect<Body, AIError> }
|
||||
readonly frames: (
|
||||
bytes: Stream.Stream<Uint8Array, AIError>,
|
||||
context: DecodeContext<Addressed<Request>>,
|
||||
) => Stream.Stream<Frame, AIError>
|
||||
readonly initial: () => State
|
||||
readonly step: (state: State, frame: Frame) => Effect.Effect<readonly [State, ReadonlyArray<Event>], AIError>
|
||||
/** Emit exactly one terminal event, or fail when the provider stopped before completing. */
|
||||
readonly finish: (state: State, context: ResponseContext<Request>) => Effect.Effect<ReadonlyArray<Event>, AIError>
|
||||
}
|
||||
|
||||
export const stream = <Request, Event, Frame, State>(
|
||||
route: Identity,
|
||||
input: Omit<Streamed<Request, Event, Frame, State>, "kind" | "id" | "provider">,
|
||||
): Streamed<Request, Event, Frame, State> => ({ kind: "stream", id: route.id, provider: route.provider, ...input })
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response helpers
|
||||
@@ -52,44 +191,122 @@ export const inline = <Request, Response>(
|
||||
const context = (response: HttpClientResponse.HttpClientResponse) =>
|
||||
new HttpContext({ url: response.request.url, status: response.status, headers: response.headers })
|
||||
|
||||
/** Read a text body while retaining the original payload and HTTP context on every downstream error. */
|
||||
export const text = Effect.fn("MediaProtocol.text")(function* (
|
||||
route: string,
|
||||
name: string,
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const http = context(response)
|
||||
const body = yield* response.text.pipe(
|
||||
Effect.mapError(
|
||||
(cause) =>
|
||||
/** One protocol's route id, display name, and provider, with the decoders and errors that carry them. */
|
||||
export const identity = (input: { readonly id: string; readonly name: string; readonly provider: string }) => {
|
||||
const provider = ProviderID.make(input.provider)
|
||||
const frameError = (message: string, body?: string, cause?: unknown) =>
|
||||
new AIError({ reason: new InvalidProviderOutputError({ route: input.id, message, body, cause }) })
|
||||
|
||||
/**
|
||||
* Read a text body while retaining the original payload and HTTP context on every downstream error. `invalid` is a
|
||||
* malformed provider document; `ended` is a generation that reached a terminal status without output (`failed` is
|
||||
* provider-side, `cancelled`/`expired` mean the result will never exist); `pending` is a `result()` read before the
|
||||
* generation finished, which is caller misuse; `contentPolicy` is a moderated result.
|
||||
*/
|
||||
const text = Effect.fn("MediaProtocol.text")(function* (response: HttpClientResponse.HttpClientResponse) {
|
||||
const http = context(response)
|
||||
const body = yield* response.text.pipe(
|
||||
Effect.mapError(
|
||||
(cause) =>
|
||||
new AIError({
|
||||
reason: new InvalidProviderOutputError({
|
||||
route: input.id,
|
||||
message: `Failed to read the ${input.name} response`,
|
||||
http,
|
||||
cause,
|
||||
}),
|
||||
}),
|
||||
),
|
||||
)
|
||||
return {
|
||||
body,
|
||||
http,
|
||||
invalid: (message: string, cause?: unknown) =>
|
||||
new AIError({ reason: new InvalidProviderOutputError({ route: input.id, message, body, http, cause }) }),
|
||||
ended: (status: Exclude<Status, "queued" | "running" | "completed">, message: string) =>
|
||||
new AIError({
|
||||
reason: new InvalidProviderOutputError({
|
||||
route,
|
||||
message: `Failed to read the ${name} response`,
|
||||
reason:
|
||||
status === "failed"
|
||||
? new ProviderInternalError({ message, body, http })
|
||||
: new InvalidRequestError({ message, body, http }),
|
||||
}),
|
||||
pending: (id: string) =>
|
||||
new AIError({
|
||||
reason: new InvalidRequestError({
|
||||
message: `${input.name} generation ${id} has not finished; await it before reading the result`,
|
||||
body,
|
||||
http,
|
||||
cause,
|
||||
}),
|
||||
}),
|
||||
),
|
||||
)
|
||||
return {
|
||||
body,
|
||||
http,
|
||||
invalid: (message: string, cause?: unknown) =>
|
||||
new AIError({ reason: new InvalidProviderOutputError({ route, message, body, http, cause }) }),
|
||||
}
|
||||
})
|
||||
|
||||
/** Read and Schema-decode a JSON body. Decode failures keep the raw body as `reason.body`. */
|
||||
export const decodeJson = <A>(route: string, name: string, schema: Schema.Codec<A, unknown>) => {
|
||||
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
|
||||
return Effect.fn("MediaProtocol.decodeJson")(function* (response: HttpClientResponse.HttpClientResponse) {
|
||||
const output = yield* text(route, name, response)
|
||||
const value = yield* decode(output.body).pipe(
|
||||
Effect.mapError((cause) => output.invalid(`${name} returned an invalid response`, cause)),
|
||||
)
|
||||
return { ...output, value }
|
||||
contentPolicy: (message: string) => new AIError({ reason: new ContentPolicyError({ message, body, http }) }),
|
||||
}
|
||||
})
|
||||
|
||||
/** Read and Schema-decode a JSON body. Decode failures keep the raw body as `reason.body`. */
|
||||
const decodeJson = <A>(schema: Schema.Codec<A, unknown>) => {
|
||||
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
|
||||
return Effect.fn("MediaProtocol.decodeJson")(function* (response: HttpClientResponse.HttpClientResponse) {
|
||||
const output = yield* text(response)
|
||||
const value = yield* decode(output.body).pipe(
|
||||
Effect.mapError((cause) => output.invalid(`${input.name} returned an invalid response`, cause)),
|
||||
)
|
||||
return { ...output, value }
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
id: input.id,
|
||||
name: input.name,
|
||||
provider,
|
||||
text,
|
||||
decodeJson,
|
||||
/** Decode a submission response into the token and first snapshot. */
|
||||
decodeStarted: <A, Token>(schema: Schema.Codec<A, unknown>, started: (value: A) => Started<Token>) => {
|
||||
const decode = decodeJson(schema)
|
||||
return (response: HttpClientResponse.HttpClientResponse) =>
|
||||
decode(response).pipe(Effect.map((output) => started(output.value)))
|
||||
},
|
||||
/** Schema-decode one JSON stream frame. Decode failures keep the frame as `reason.body`. */
|
||||
decodeFrame: <A>(schema: Schema.Codec<A, unknown>) => {
|
||||
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
|
||||
return (frame: string) =>
|
||||
decode(frame).pipe(
|
||||
Effect.mapError((cause) => frameError(`${input.name} sent an invalid stream event`, frame, cause)),
|
||||
)
|
||||
},
|
||||
/** A stream-time failure; the frame stays on `reason.body`. */
|
||||
frameError,
|
||||
incomplete: () =>
|
||||
new AIError({
|
||||
reason: new InvalidProviderOutputError({
|
||||
route: input.id,
|
||||
message: "The provider response ended unexpectedly.",
|
||||
classification: "incomplete-stream",
|
||||
}),
|
||||
}),
|
||||
unsupported: (operation: string, message: string) =>
|
||||
new AIError({ reason: new UnsupportedOperationError({ operation, provider, route: input.id, message }) }),
|
||||
}
|
||||
}
|
||||
|
||||
export type Identity = ReturnType<typeof identity>
|
||||
|
||||
export type Output = Effect.Success<ReturnType<Identity["text"]>>
|
||||
|
||||
/** Map a provider status string through the protocol's table; unknown values are an invalid provider document. */
|
||||
export const status = <Table extends Record<string, Status>>(
|
||||
table: Table,
|
||||
raw: string,
|
||||
output: Output,
|
||||
): Effect.Effect<Status, AIError> => {
|
||||
if (!Object.hasOwn(table, raw)) return Effect.fail(output.invalid(`Unknown generation status "${raw}"`))
|
||||
return Effect.succeed(table[raw])
|
||||
}
|
||||
|
||||
/** A `url` asset whose provider-declared retention window starts now. */
|
||||
export const expiringUrl = (url: string, retention: Duration.Duration, options?: Parameters<typeof Media.url>[1]) =>
|
||||
Clock.currentTimeMillis.pipe(
|
||||
Effect.map((now) => Media.url(url, { ...options, expiresAt: now + Duration.toMillis(retention) })),
|
||||
)
|
||||
|
||||
export * as MediaProtocol from "./media-protocol.js"
|
||||
|
||||
+351
-51
@@ -1,11 +1,22 @@
|
||||
import { Effect } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import { Auth } from "./auth.js"
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import { Headers, HttpClientRequest, type HttpClientResponse } from "effect/unstable/http"
|
||||
import { Auth, type AuthInput } from "./auth.js"
|
||||
import { Endpoint } from "./endpoint.js"
|
||||
import type { Interface } from "./executor-service.js"
|
||||
import { RequestExecutorService, type Interface } from "./executor-service.js"
|
||||
import { RequestExecutor } from "./executor.js"
|
||||
import { MediaProtocol } from "./media-protocol.js"
|
||||
import { ProviderShared } from "../protocols/shared.js"
|
||||
import { AIError, HttpOptions, ProviderID, mergeHttpOptions } from "../schema/index.js"
|
||||
import { Generation, isTerminal } from "../generation.js"
|
||||
import type { Media } from "../media.js"
|
||||
import {
|
||||
AIError,
|
||||
AIErrorReason,
|
||||
HttpOptions,
|
||||
InvalidRequestError,
|
||||
ProviderID,
|
||||
UnsupportedOperationError,
|
||||
mergeHttpOptions,
|
||||
} from "../schema/index.js"
|
||||
import { encodeJson } from "../utils/json.js"
|
||||
import { sanitizeSurrogates } from "../utils/sanitize.js"
|
||||
|
||||
export type Execute = Interface["execute"]
|
||||
@@ -25,73 +36,360 @@ export interface ModelInput {
|
||||
readonly http?: HttpOptions
|
||||
}
|
||||
|
||||
/** A provider facade's `configure(...)` input as the `ModelInput` every media selector shares, minus the model id. */
|
||||
export const deployment = (
|
||||
input: { readonly baseURL?: string; readonly headers?: Record<string, string>; readonly http?: HttpOptions.Input },
|
||||
auth: Auth.Definition,
|
||||
): Omit<ModelInput, "id"> => ({
|
||||
auth,
|
||||
baseURL: input.baseURL,
|
||||
headers: input.headers,
|
||||
http: HttpOptions.make(input.http),
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Route
|
||||
// Routes
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface Route<Request extends MediaRequest, Response> {
|
||||
/** One request, one response. */
|
||||
export interface InlineRoute<Request extends MediaRequest, Response> {
|
||||
readonly kind: "inline"
|
||||
readonly id: string
|
||||
readonly provider: ProviderID
|
||||
readonly protocol: string
|
||||
readonly generate: (request: Request, execute: Execute) => Effect.Effect<Response, AIError>
|
||||
}
|
||||
|
||||
export interface MakeInput<Request extends MediaRequest, Response> {
|
||||
/** Submit, then poll through the returned `Generation`. */
|
||||
export interface QueuedRoute<Request extends MediaRequest, Response> {
|
||||
readonly kind: "queued"
|
||||
readonly id: string
|
||||
readonly provider: string | ProviderID
|
||||
readonly protocol: MediaProtocol.Inline<Request, Response>
|
||||
readonly provider: ProviderID
|
||||
readonly protocol: string
|
||||
readonly start: (request: Request, execute: Execute) => Effect.Effect<Generation<Response>, AIError>
|
||||
/** Rebuild a handle from a persisted `Generation.token`; fails typed when the token is not this route's. */
|
||||
readonly resume: (
|
||||
model: MediaRequest["model"],
|
||||
token: unknown,
|
||||
execute: Execute,
|
||||
) => Effect.Effect<Generation<Response>, AIError>
|
||||
}
|
||||
|
||||
/** One request whose response parses into events; `generate` runs the same stream and collects it. */
|
||||
export interface StreamRoute<Request extends MediaRequest, Event, Response> {
|
||||
readonly kind: "stream"
|
||||
readonly id: string
|
||||
readonly provider: ProviderID
|
||||
readonly protocol: string
|
||||
readonly stream: (request: Request, execute: Execute) => Stream.Stream<Event, AIError>
|
||||
readonly generate: (request: Request, execute: Execute) => Effect.Effect<Response, AIError>
|
||||
}
|
||||
|
||||
export type AnyRoute<Request extends MediaRequest, Event, Response> =
|
||||
| InlineRoute<Request, Response>
|
||||
| StreamRoute<Request, Event, Response>
|
||||
| QueuedRoute<Request, Response>
|
||||
|
||||
export interface Composition<Request extends MediaRequest> {
|
||||
readonly endpoint: Endpoint.Definition<MediaProtocol.Body, Request>
|
||||
readonly auth: Auth.Definition
|
||||
/** Deployment headers applied before transport authentication. */
|
||||
readonly headers?: Record<string, string>
|
||||
}
|
||||
|
||||
export interface InlineInput<Request extends MediaRequest, Response> extends Composition<Request> {
|
||||
readonly protocol: MediaProtocol.Inline<Request, Response>
|
||||
}
|
||||
|
||||
export interface QueuedInput<Request extends MediaRequest, Response, Token> extends Composition<Request> {
|
||||
readonly protocol: MediaProtocol.Queued<Request, Response, Token>
|
||||
}
|
||||
|
||||
export interface StreamInput<Request extends MediaRequest, Event, Response, Frame, State>
|
||||
extends Composition<MediaProtocol.Addressed<Request>> {
|
||||
readonly protocol: MediaProtocol.Streamed<Request, Event, Frame, State>
|
||||
readonly collect: (events: ReadonlyArray<Event>) => Effect.Effect<Response, AIError>
|
||||
}
|
||||
|
||||
/**
|
||||
* Compose an inline media protocol with an endpoint and auth into a runnable route. The route owns the transport
|
||||
* plumbing every media protocol would otherwise duplicate: option merging, surrogate sanitizing, unsupported-field rejection, URL and query
|
||||
* rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol for decoding.
|
||||
* plumbing every media protocol would otherwise duplicate: option merging, surrogate sanitizing, unsupported-field
|
||||
* rejection, URL and query rendering, auth headers, JSON, multipart, or binary encoding, and handing responses back
|
||||
* to the protocol.
|
||||
*/
|
||||
export const make = <Request extends MediaRequest, Response>(
|
||||
input: MakeInput<Request, Response>,
|
||||
): Route<Request, Response> => {
|
||||
const provider = ProviderID.make(input.provider)
|
||||
const routeHttp = input.headers === undefined ? undefined : new HttpOptions({ headers: input.headers })
|
||||
const authorize = Auth.toEffect(input.auth)
|
||||
export const inline = <Request extends MediaRequest, Response>(
|
||||
input: InlineInput<Request, Response>,
|
||||
): InlineRoute<Request, Response> => {
|
||||
const transport = makeTransport(input)
|
||||
return {
|
||||
id: input.id,
|
||||
provider,
|
||||
kind: "inline",
|
||||
id: input.protocol.id,
|
||||
provider: input.protocol.provider,
|
||||
protocol: input.protocol.id,
|
||||
generate: Effect.fn(`MediaRoute.generate`)(function* (request: Request, execute: Execute) {
|
||||
yield* rejectUnsupported(input.id, provider, request, input.protocol.unsupported)
|
||||
const http = mergeHttpOptions(routeHttp, request.model.http, request.http)
|
||||
// Sanitize after merging so model-level overlays are covered; the model value is restored, not sanitized.
|
||||
const resolved: Request = { ...sanitizeSurrogates({ ...request, http }), model: request.model }
|
||||
const body = yield* input.protocol.body.from(resolved)
|
||||
const url = Endpoint.render(input.endpoint, { request: resolved, body })
|
||||
for (const [key, value] of Object.entries(http?.query ?? {})) url.searchParams.set(key, value)
|
||||
const encoded = body.type === "json" ? ProviderShared.encodeJson(body.value) : "[multipart/form-data]"
|
||||
const baseHeaders = Headers.fromInput(http?.headers)
|
||||
const headers = yield* authorize({
|
||||
request: resolved,
|
||||
method: "POST",
|
||||
url: url.toString(),
|
||||
body: encoded,
|
||||
// The HTTP client sets the multipart boundary; a caller-supplied content-type would corrupt it.
|
||||
headers: body.type === "multipart" ? Headers.remove(baseHeaders, "content-type") : baseHeaders,
|
||||
})
|
||||
const transport = HttpClientRequest.post(url.toString()).pipe(
|
||||
HttpClientRequest.setHeaders(headers),
|
||||
body.type === "json"
|
||||
? HttpClientRequest.bodyText(encoded, "application/json")
|
||||
: HttpClientRequest.bodyFormData(body.value),
|
||||
const submitted = yield* transport.submit(
|
||||
request,
|
||||
{ unsupported: input.protocol.unsupported, from: input.protocol.body.from },
|
||||
execute,
|
||||
)
|
||||
const response = yield* execute(transport)
|
||||
return yield* input.protocol.response.decode(response, { request: resolved, body })
|
||||
return yield* input.protocol.response.decode(submitted.response, submitted.context)
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/** Common fields are never silently dropped: a present field the protocol declared unsupported fails typed. */
|
||||
/**
|
||||
* Compose a queued media protocol the same way, adding `start`/`resume` handles whose polls reuse the route's auth,
|
||||
* deployment headers, and (for `start`) the request's `http` overlay. The token is decoded once at the boundary and
|
||||
* closed over by the resulting `Generation.Route`.
|
||||
*/
|
||||
export const queued = <Request extends MediaRequest, Response, Token>(
|
||||
input: QueuedInput<Request, Response, Token>,
|
||||
): QueuedRoute<Request, Response> => {
|
||||
const transport = makeTransport(input)
|
||||
const protocol = input.protocol
|
||||
const decodeToken = Schema.decodeUnknownEffect(protocol.token)
|
||||
// A protocol producing a token its own codec rejects is a programmer defect, not a provider error.
|
||||
const encodeToken = Schema.encodeSync(protocol.token)
|
||||
|
||||
const generationRoute = (token: Token, http: HttpOptions | undefined, execute: Execute) => {
|
||||
const materialize = (asset: Media.Asset) =>
|
||||
asset.materialize().pipe(Effect.provideService(RequestExecutorService, { execute }))
|
||||
const poll = <A>(operation: {
|
||||
readonly path: (token: Token) => string
|
||||
readonly decode: (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.PollContext<Token>,
|
||||
) => Effect.Effect<A, AIError>
|
||||
}) =>
|
||||
transport
|
||||
.call("GET", operation.path(token), http, execute)
|
||||
.pipe(Effect.flatMap((sent) => operation.decode(sent.response, { token, auth: sent.auth, materialize })))
|
||||
const status = poll(protocol.status)
|
||||
const cancel = protocol.cancel
|
||||
const send =
|
||||
cancel === undefined
|
||||
? undefined
|
||||
: transport.call(cancel.method, cancel.path(token), http, execute).pipe(Effect.asVoid)
|
||||
return {
|
||||
status,
|
||||
result: poll(protocol.result),
|
||||
cancel:
|
||||
send !== undefined && cancel?.activeOnly
|
||||
? status.pipe(Effect.flatMap((snapshot) => (isTerminal(snapshot.status) ? Effect.void : send)))
|
||||
: send,
|
||||
}
|
||||
}
|
||||
|
||||
const start = Effect.fn("MediaRoute.start")(function* (request: Request, execute: Execute) {
|
||||
const submitted = yield* transport.submit(
|
||||
request,
|
||||
{ unsupported: protocol.unsupported, prepare: protocol.start.prepare, from: protocol.start.body.from },
|
||||
execute,
|
||||
)
|
||||
const started = yield* protocol.start.decode(submitted.response, submitted.context)
|
||||
const route = generationRoute(started.token, submitted.context.request.http, execute)
|
||||
return new Generation(route, encodeToken(started.token), started.snapshot)
|
||||
})
|
||||
|
||||
const resume = Effect.fn("MediaRoute.resume")(function* (
|
||||
model: MediaRequest["model"],
|
||||
raw: unknown,
|
||||
execute: Execute,
|
||||
) {
|
||||
const token = yield* decodeToken(raw).pipe(
|
||||
Effect.mapError(
|
||||
(cause) =>
|
||||
new AIError({
|
||||
reason: new InvalidRequestError({
|
||||
message: `${protocol.id} cannot resume a generation from this token`,
|
||||
cause,
|
||||
}),
|
||||
}),
|
||||
),
|
||||
)
|
||||
const route = generationRoute(token, transport.http(model), execute)
|
||||
return new Generation(route, encodeToken(token), yield* route.status)
|
||||
})
|
||||
|
||||
return { kind: "queued", id: protocol.id, provider: protocol.provider, protocol: protocol.id, start, resume }
|
||||
}
|
||||
|
||||
/** Compose a streaming media protocol; `generate` runs the same stream in `generate` mode and folds it with `collect`. */
|
||||
export const stream = <Request extends MediaRequest, Event, Response, Frame, State>(
|
||||
input: StreamInput<Request, Event, Response, Frame, State>,
|
||||
): StreamRoute<Request, Event, Response> => {
|
||||
const transport = makeTransport(input)
|
||||
const protocol = input.protocol
|
||||
const events = (request: Request, execute: Execute, mode: MediaProtocol.Mode) =>
|
||||
Stream.unwrap(
|
||||
Effect.gen(function* () {
|
||||
const submitted = yield* transport.submit(
|
||||
{ ...request, mode },
|
||||
{ unsupported: protocol.unsupported, from: protocol.body.from },
|
||||
execute,
|
||||
)
|
||||
const http = RequestExecutor.responseHttp(submitted.response)
|
||||
return Stream.suspend(() => {
|
||||
// Parser state is local to one response, exactly like `Route.make`'s LLM stream loop.
|
||||
let state = protocol.initial()
|
||||
return protocol.frames(RequestExecutor.responseStream(submitted.response), submitted.context).pipe(
|
||||
Stream.mapEffect((frame) =>
|
||||
protocol.step(state, frame).pipe(
|
||||
Effect.map(([next, output]) => {
|
||||
state = next
|
||||
return output
|
||||
}),
|
||||
),
|
||||
),
|
||||
Stream.flattenIterable,
|
||||
Stream.concat(
|
||||
Stream.suspend(() => Stream.fromIterableEffect(protocol.finish(state, { ...submitted.context, http }))),
|
||||
),
|
||||
Stream.mapError((error) =>
|
||||
error.reason.http !== undefined
|
||||
? error
|
||||
: new AIError({
|
||||
reason: AIErrorReason.make({
|
||||
...error.reason,
|
||||
message: error.reason.message,
|
||||
cause: error.reason.cause,
|
||||
http,
|
||||
}),
|
||||
}),
|
||||
),
|
||||
)
|
||||
})
|
||||
}),
|
||||
)
|
||||
return {
|
||||
kind: "stream",
|
||||
id: protocol.id,
|
||||
provider: protocol.provider,
|
||||
protocol: protocol.id,
|
||||
stream: (request, execute) => events(request, execute, "stream"),
|
||||
generate: (request, execute) =>
|
||||
events(request, execute, "generate").pipe(Stream.runCollect, Effect.flatMap(input.collect)),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Transport plumbing shared by every kind
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const makeTransport = <Request extends MediaRequest>(
|
||||
input: Composition<Request> & { readonly protocol: { readonly id: string; readonly provider: ProviderID } },
|
||||
) => {
|
||||
const routeHttp = input.headers === undefined ? undefined : new HttpOptions({ headers: input.headers })
|
||||
const authorize = Auth.toEffect(input.auth)
|
||||
const baseURL = (path: string) => new URL(`${Endpoint.trimBaseUrl(input.endpoint.baseURL ?? "")}${path}`)
|
||||
/** `auth` is only what `Auth` added or changed, never untouched deployment headers. */
|
||||
const send = Effect.fn("MediaRoute.send")(function* (
|
||||
call: {
|
||||
readonly method: AuthInput["method"]
|
||||
readonly url: URL
|
||||
readonly headers: Headers.Headers
|
||||
readonly request: AuthInput["request"]
|
||||
readonly body?: MediaProtocol.Body
|
||||
},
|
||||
execute: Execute,
|
||||
) {
|
||||
const encoded = encode(call.body, call.headers)
|
||||
const url = call.url.toString()
|
||||
const headers = yield* authorize({
|
||||
request: call.request,
|
||||
method: call.method,
|
||||
url,
|
||||
body: encoded.text,
|
||||
headers: encoded.headers,
|
||||
})
|
||||
const response = yield* execute(
|
||||
encoded.apply(HttpClientRequest.make(call.method)(url).pipe(HttpClientRequest.setHeaders(headers))),
|
||||
)
|
||||
return {
|
||||
response,
|
||||
auth: Object.fromEntries(Object.entries(headers).filter(([key, value]) => encoded.headers[key] !== value)),
|
||||
}
|
||||
})
|
||||
return {
|
||||
/** Route and model overlays; `start` additionally merges the request's own `http`. */
|
||||
http: (model: MediaRequest["model"]) => mergeHttpOptions(routeHttp, model.http),
|
||||
/** POST the protocol body to the route endpoint. */
|
||||
submit: Effect.fn("MediaRoute.submit")(function* (
|
||||
request: Request,
|
||||
protocol: {
|
||||
readonly unsupported?: ReadonlyArray<keyof Request & string>
|
||||
readonly prepare?: MediaProtocol.Prepare<Request>
|
||||
readonly from: (request: Request) => Effect.Effect<MediaProtocol.Body, AIError>
|
||||
},
|
||||
execute: Execute,
|
||||
) {
|
||||
yield* rejectUnsupported(input.protocol.id, input.protocol.provider, request, protocol.unsupported)
|
||||
const http = mergeHttpOptions(routeHttp, request.model.http, request.http)
|
||||
const headers = Headers.fromInput(http?.headers)
|
||||
const prepared =
|
||||
protocol.prepare === undefined
|
||||
? request
|
||||
: yield* protocol.prepare(request, (path, body) =>
|
||||
send({ method: "POST", url: baseURL(path), headers, request, body }, execute).pipe(
|
||||
Effect.map((sent) => sent.response),
|
||||
),
|
||||
)
|
||||
// Sanitize after merging so model-level overlays are covered; the model value is restored, not sanitized.
|
||||
const resolved: Request = { ...sanitizeSurrogates({ ...prepared, http }), model: request.model }
|
||||
const body = yield* protocol.from(resolved)
|
||||
const url = withQuery(
|
||||
withQuery(
|
||||
Endpoint.render(input.endpoint, { request: resolved, body }),
|
||||
body.type === "multipart" ? undefined : body.query,
|
||||
),
|
||||
http?.query,
|
||||
)
|
||||
const sent = yield* send({ method: "POST", url, headers, request: resolved, body }, execute)
|
||||
return { response: sent.response, context: { request: resolved, body } }
|
||||
}),
|
||||
/** Bodiless follow-up call (status, result, cancel) with the same auth and headers as `submit`. */
|
||||
call: (method: AuthInput["method"], path: string, http: HttpOptions | undefined, execute: Execute) => {
|
||||
// Provider-issued absolute URLs (fal `status_url`) are used as-is; everything else resolves against the base.
|
||||
const url = withQuery(/^https?:\/\//.test(path) ? new URL(path) : baseURL(path), http?.query)
|
||||
for (const [key, value] of Object.entries(input.endpoint.query ?? {})) url.searchParams.set(key, value)
|
||||
return send({ method, url, headers: Headers.fromInput(http?.headers), request: { http } }, execute)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const withQuery = (url: URL, query: MediaProtocol.Query | undefined) => {
|
||||
for (const [key, value] of Object.entries(query ?? {})) {
|
||||
url.searchParams.delete(key)
|
||||
for (const item of typeof value === "string" ? [value] : value) url.searchParams.append(key, item)
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
const encode = (body: MediaProtocol.Body | undefined, headers: Headers.Headers) => {
|
||||
if (body === undefined) return { text: "", headers, apply: (request: HttpClientRequest.HttpClientRequest) => request }
|
||||
if (body.type === "json") {
|
||||
const text = encodeJson(body.value)
|
||||
return { text, headers, apply: HttpClientRequest.bodyText(text, "application/json") }
|
||||
}
|
||||
if (body.type === "binary")
|
||||
return {
|
||||
text: `[${body.contentType}]`,
|
||||
headers,
|
||||
apply: HttpClientRequest.bodyUint8Array(body.value, body.contentType),
|
||||
}
|
||||
return {
|
||||
text: "[multipart/form-data]",
|
||||
// The HTTP client sets the multipart boundary; a caller-supplied content-type would corrupt it.
|
||||
headers: Headers.remove(headers, "content-type"),
|
||||
apply: HttpClientRequest.bodyFormData(body.value),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Common fields are never silently dropped: a present field the protocol declared unsupported fails typed. `false`
|
||||
* counts as present because some booleans mean something when false (video `audio`); protocols reject opt-in
|
||||
* booleans such as speech `timestamps` with `=== true` in `body.from` instead of listing them.
|
||||
*/
|
||||
const rejectUnsupported = <Request extends object>(
|
||||
route: string,
|
||||
provider: ProviderID,
|
||||
@@ -104,11 +402,13 @@ const rejectUnsupported = <Request extends object>(
|
||||
})
|
||||
if (present.length === 0) return Effect.void
|
||||
return Effect.fail(
|
||||
ProviderShared.unsupportedOperation({
|
||||
operation: `media.${present[0]}`,
|
||||
provider,
|
||||
route,
|
||||
message: `${provider}/${route} does not support ${present.join(", ")}`,
|
||||
new AIError({
|
||||
reason: new UnsupportedOperationError({
|
||||
operation: `media.${present[0]}`,
|
||||
provider,
|
||||
route,
|
||||
message: `${provider}/${route} does not support ${present.join(", ")}`,
|
||||
}),
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { Schema, type Effect } from "effect"
|
||||
import type { AIError, LLMEvent, LLMRequest, ProtocolID } from "../schema/index.js"
|
||||
import type { AIError, LanguageModelSanitizerCompatibility, LLMEvent, LLMRequest, ProtocolID } from "../schema/index.js"
|
||||
|
||||
/**
|
||||
* The semantic API contract of one model server family.
|
||||
@@ -43,6 +43,8 @@ export interface Protocol<Body, Frame, Event, State> {
|
||||
readonly stream: ProtocolStream<Frame, Event, State>
|
||||
/** Whether `body.from` lowers `Message.effort(...)` markers; wrappers around another `body.from` must forward it. */
|
||||
readonly supportsEffortUpdates?: (request: LLMRequest) => boolean
|
||||
/** Tool schema sanitizer for every model on this protocol unless the model's compatibility sets one; wrappers around another `body.from` must forward it. */
|
||||
readonly sanitizer?: LanguageModelSanitizerCompatibility
|
||||
}
|
||||
|
||||
export interface ProtocolBody<Body> {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user