Compare commits

..
Author SHA1 Message Date
Aiden Cline 00aee1daed fix(core): serialize MCP OAuth refreshes across processes 2026-09-23 17:35:36 -05:00
551 changed files with 5684 additions and 44144 deletions
+37 -37
View File
@@ -32,7 +32,7 @@
},
"packages/ai": {
"name": "@opencode/ai",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@aws-sdk/credential-providers": "3.1057.0",
"@opencode/schema": "workspace:*",
@@ -54,7 +54,7 @@
},
"packages/app": {
"name": "@opencode/app",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@corvu/drawer": "catalog:",
"@dnd-kit/abstract": "0.5.0",
@@ -112,7 +112,7 @@
},
"packages/cli": {
"name": "@opencode/cli",
"version": "2.0.16",
"version": "2.0.15",
"bin": {
"opencode": "./bin/opencode.cjs",
"opencode2": "./bin/opencode2.cjs",
@@ -177,7 +177,7 @@
},
"packages/client": {
"name": "@opencode/client",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@opencode/protocol": "workspace:*",
"@opencode/schema": "workspace:*",
@@ -203,7 +203,7 @@
},
"packages/codemode": {
"name": "@opencode/codemode",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"acorn": "8.15.0",
"effect": "catalog:",
@@ -216,7 +216,7 @@
},
"packages/console/app": {
"name": "@opencode/console-app",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@cloudflare/vite-plugin": "1.15.2",
"@ibm/plex": "6.4.1",
@@ -252,7 +252,7 @@
},
"packages/console/core": {
"name": "@opencode/console-core",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@aws-sdk/client-sts": "3.782.0",
"@jsx-email/render": "1.1.1",
@@ -279,7 +279,7 @@
},
"packages/console/function": {
"name": "@opencode/console-function",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@openauthjs/openauth": "0.0.0-20250322224806",
"@opencode/console-core": "workspace:*",
@@ -296,7 +296,7 @@
},
"packages/console/mail": {
"name": "@opencode/console-mail",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@jsx-email/all": "2.2.3",
"@jsx-email/cli": "1.4.3",
@@ -320,7 +320,7 @@
},
"packages/console/support": {
"name": "@opencode/console-support",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@cloudflare/vite-plugin": "1.15.2",
"@opencode/console-core": "workspace:*",
@@ -340,7 +340,7 @@
},
"packages/core": {
"name": "@opencode/core",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@ai-sdk/cohere": "3.0.27",
"@ai-sdk/gateway": "3.0.104",
@@ -368,7 +368,7 @@
"drizzle-orm": "catalog:",
"effect": "catalog:",
"fuzzysort": "3.1.0",
"gitlab-ai-provider": "6.18.0",
"gitlab-ai-provider": "6.16.0",
"google-auth-library": "10.5.0",
"gray-matter": "4.0.3",
"htmlparser2": "8.0.2",
@@ -408,7 +408,7 @@
},
"packages/desktop": {
"name": "@opencode/desktop",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@zip.js/zip.js": "2.7.62",
"electron-context-menu": "5.0.0",
@@ -457,7 +457,7 @@
},
"packages/enterprise": {
"name": "@opencode/enterprise",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@hono/standard-validator": "catalog:",
"@opencode-ai/sdk": "1.18.21",
@@ -494,7 +494,7 @@
},
"packages/function": {
"name": "@opencode/function",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@octokit/auth-app": "8.0.1",
"@octokit/rest": "catalog:",
@@ -510,7 +510,7 @@
},
"packages/http-recorder": {
"name": "@opencode/http-recorder",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@effect/platform-node-shared": "4.0.0-rc.112",
},
@@ -529,7 +529,7 @@
},
"packages/httpapi-codegen": {
"name": "@opencode/httpapi-codegen",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"effect": "catalog:",
"prettier": "3.6.2",
@@ -542,7 +542,7 @@
},
"packages/latex": {
"name": "@opencode/latex",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@opencode/plugin": "workspace:*",
"@opentui/core": "catalog:",
@@ -556,7 +556,7 @@
},
"packages/merman": {
"name": "@opencode/merman",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@opencode/plugin": "workspace:*",
"@opentui/core": "catalog:",
@@ -571,7 +571,7 @@
},
"packages/plugin": {
"name": "@opencode/plugin",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@ai-sdk/provider": "3.0.8",
"@opencode/ai": "workspace:*",
@@ -610,7 +610,7 @@
},
"packages/plugin-browser": {
"name": "@opencode/plugin-browser",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@opencode/plugin": "workspace:*",
"@opencode/schema": "workspace:*",
@@ -640,7 +640,7 @@
},
"packages/protocol": {
"name": "@opencode/protocol",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@opencode/schema": "workspace:*",
"effect": "catalog:",
@@ -655,7 +655,7 @@
},
"packages/schema": {
"name": "@opencode/schema",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@standard-schema/spec": "catalog:",
"effect": "catalog:",
@@ -679,7 +679,7 @@
},
"packages/sdk": {
"name": "@opencode/sdk",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@opencode/client": "workspace:*",
"@opencode/core": "workspace:*",
@@ -700,7 +700,7 @@
},
"packages/server": {
"name": "@opencode/server",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@effect/platform-node": "catalog:",
"@effect/platform-node-shared": "catalog:",
@@ -722,7 +722,7 @@
},
"packages/session-ui": {
"name": "@opencode/session-ui",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@kobalte/core": "catalog:",
"@opencode/client": "workspace:*",
@@ -757,7 +757,7 @@
},
"packages/simulation": {
"name": "@opencode/simulation",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@opencode/ai": "workspace:*",
"@opencode/core": "workspace:*",
@@ -777,7 +777,7 @@
},
"packages/stats/app": {
"name": "@opencode/stats-app",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@ibm/plex": "6.4.1",
"@kobalte/core": "catalog:",
@@ -811,7 +811,7 @@
},
"packages/stats/core": {
"name": "@opencode/stats-core",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@aws-sdk/client-athena": "3.933.0",
"@planetscale/database": "1.19.0",
@@ -830,7 +830,7 @@
},
"packages/stats/server": {
"name": "@opencode/stats-server",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@aws-sdk/client-firehose": "3.933.0",
"@effect/platform-node": "catalog:",
@@ -876,7 +876,7 @@
},
"packages/theme": {
"name": "@opencode/theme",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@opentui/core": "catalog:",
"effect": "catalog:",
@@ -890,7 +890,7 @@
},
"packages/tui": {
"name": "@opencode/tui",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@opencode/client": "workspace:*",
"@opencode/core": "workspace:*",
@@ -925,7 +925,7 @@
},
"packages/ui": {
"name": "@opencode/ui",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@kobalte/core": "catalog:",
"@pierre/diffs": "catalog:",
@@ -960,7 +960,7 @@
},
"packages/util": {
"name": "@opencode/util",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@effect/opentelemetry": "catalog:",
"@effect/platform-node": "catalog:",
@@ -997,7 +997,7 @@
},
"packages/web": {
"name": "@opencode/web",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"@astrojs/cloudflare": "12.6.3",
"@astrojs/markdown-remark": "6.3.1",
@@ -1038,7 +1038,7 @@
},
"services/update": {
"name": "@opencode/update",
"version": "2.0.16",
"version": "2.0.15",
"dependencies": {
"jose": "6.0.11",
"semver": "catalog:",
@@ -4114,7 +4114,7 @@
"github-slugger": ["github-slugger@2.0.0", "", {}, "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw=="],
"gitlab-ai-provider": ["gitlab-ai-provider@6.18.0", "", { "dependencies": { "@anthropic-ai/sdk": "^0.71.0", "@anycable/core": "^0.9.2", "graphql-request": "^6.1.0", "isomorphic-ws": "^5.0.0", "openai": "^6.16.0", "socket.io-client": "^4.8.1", "vscode-jsonrpc": "^8.2.1", "zod": "^3.25.76" }, "peerDependencies": { "@ai-sdk/provider": ">=3.0.0", "@ai-sdk/provider-utils": ">=4.0.0" } }, "sha512-dXTXkNt1SFCL7jGlqazHL6iUJEug21qq0tx2s+Tui7jWNUqIAHEzY9WY1+PnZlhxvB56jPPBcFAT0attpjnopg=="],
"gitlab-ai-provider": ["gitlab-ai-provider@6.16.0", "", { "dependencies": { "@anthropic-ai/sdk": "^0.71.0", "@anycable/core": "^0.9.2", "graphql-request": "^6.1.0", "isomorphic-ws": "^5.0.0", "openai": "^6.16.0", "socket.io-client": "^4.8.1", "vscode-jsonrpc": "^8.2.1", "zod": "^3.25.76" }, "peerDependencies": { "@ai-sdk/provider": ">=3.0.0", "@ai-sdk/provider-utils": ">=4.0.0" } }, "sha512-HMC3sKgWYaYSsgm86Cnq2e6laHlYkhiFQ6rFD5qsVghv9//6h4Ofr7j5R2KjQx9Hl8EBercPsmzjoEjEN7dX5Q=="],
"glob": ["glob@13.0.5", "", { "dependencies": { "minimatch": "^10.2.1", "minipass": "^7.1.2", "path-scurry": "^2.0.0" } }, "sha512-BzXxZg24Ibra1pbQ/zE7Kys4Ua1ks7Bn6pKLkVPZ9FZe4JQS6/Q7ef3LG1H+k7lUf5l4T3PLSyYyYJVYUvfgTw=="],
+4 -4
View File
@@ -1,8 +1,8 @@
{
"nodeModules": {
"x86_64-linux": "sha256-+Clo0VPDdruHSoBNvV/wKAM8iR6HJPtB00oa8yl9ujU=",
"aarch64-linux": "sha256-4wU5v36GTXjwyt5ls4FH+5G43Ujd+dKVSJR21w3lhbA=",
"aarch64-darwin": "sha256-pThjoD6baddQ6biy7k1ByXwGwLAeWe/+w0tcYmt1uWs=",
"x86_64-darwin": "sha256-bCBl63CqBiqilb+YdaOLBYYZx/yf47c1aqgDOkgdegg="
"x86_64-linux": "sha256-LQ1GAz1qF4R5P4j/kkgUygsQvxm7KStVlMf24nmyq44=",
"aarch64-linux": "sha256-PsNR3VaClA1O1vSE0z/Hb3XRT1pb6PvNRXuK+giIx6s=",
"aarch64-darwin": "sha256-7VCS+GT7tDYAMgqNCnLh1zvHAZkLTnJJ4vtfM88ruT8=",
"x86_64-darwin": "sha256-HyxXjiFVd8vcKvSb/BuHAY5Jj3MGgdXEvuhDuglp4ss="
}
}
+1 -1
View File
@@ -2,7 +2,7 @@
"$schema": "https://json.schemastore.org/package.json",
"name": "opencode",
"description": "AI-powered development tool",
"version": "2.0.16",
"version": "2.0.15",
"private": true,
"type": "module",
"packageManager": "bun@1.4.2",
+4 -4
View File
@@ -55,7 +55,7 @@ const request = LLM.request({
prompt: "Say hello.",
})
const response = yield * LLMClient.generate(request) // inside Effect.gen
const response = yield * LLMClient.generate(request)
```
`LLM.request(...)` builds an `LLMRequest`. `LLMClient.generate(...)` reads the executable route carried by `request.model.route`, builds the provider-native body, asks the route's transport for a real `HttpClientRequest.HttpClientRequest`, sends it through `RequestExecutor.Service`, parses the provider stream into common `LLMEvent`s, and finally returns an `LLMResponse`.
@@ -96,11 +96,11 @@ When a provider supports multiple physical transports, selection remains executi
### Media Routes
Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute.inline(...)` / `queued(...)` / `stream(...)` (`src/route/media.ts`) compose a `MediaProtocol` kind with `Endpoint` and `Auth` and own the transport plumbing: `http` option merging, URL/query rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol. `MediaProtocol.inline` (`src/route/media-protocol.ts`) is `body.from(request)` plus `response.decode(response, context)`; each protocol declares `const route = MediaProtocol.identity({ id, name, provider })` once and decodes through `route.decodeJson` / `route.text` / `route.decodeStarted` so decode failures retain the raw body and HTTP context, raising `route.unsupported(operation, message)` for requests it cannot lower, and passes `route` as the first argument to `MediaProtocol.inline` / `queued` / `stream`. `Generation` (`src/generation.ts`) is the provider-neutral handle for a queued generation over a `GenerationRoute` (`status`, `result`, `cancel`). Image protocol files follow the same section order as LLM protocols and declare unsupported common fields once through the protocol's `unsupported` list.
Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute.make(...)` (`src/route/media.ts`) composes a `MediaProtocol` kind with `Endpoint` and `Auth` and owns the transport plumbing: `http` option merging, URL/query rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol. `MediaProtocol.inline` (`src/route/media-protocol.ts`) is `body.from(request)` plus `response.decode(response, context)`; use `MediaProtocol.decodeJson` / `text` / `bytes` so decode failures retain the raw body and HTTP context. `Generation` (`src/generation.ts`) is the provider-neutral handle for a queued generation over a `GenerationRoute` (`status`, `result`, `cancel`, `pollHint`). Image protocol files follow the same section order as LLM protocols and declare unsupported common fields once through `MediaInput.rejectUnsupported`.
`MediaProtocol.queued` is the submit-then-poll kind every video route uses: `start` (body + decode into `{ token, snapshot }`), `status`, `result`, and optional `cancel`, each addressed by a route-owned `token` whose `Schema.Codec` makes it serializable. `MediaRoute.inline` and `MediaRoute.queued` compose the two kinds with `Endpoint` and `Auth`; the queued route decodes the token once at the boundary (`start` output or `resume` input) and closes over it in a token-free `GenerationRoute` (`status`/`result`/`cancel` are plain Effects), so `Generation` never sees the token's shape and only carries the encoded JSON for persistence. Polls reuse the route's auth and deployment headers plus the request's `http` overlay after `start`, and resolve relative paths against the route base URL (provider-issued absolute URLs such as fal's `status_url` pass through). `result` is always its own GET even when the provider returns output inside the status document, so `Generation.await` behaves the same after `start` and after `resume`. `PollContext.auth` carries only what `Auth` added or changed so protocols can hand download credentials to output assets as transient `Media.Asset.headers` (Veo) — never part of `source` or JSON. Status strings map through a per-protocol `STATUS` table via `MediaProtocol.status`; terminal generations without output fail through `output.ended` / `output.contentPolicy` with the provider document on `reason.body`. `GenerationAwaitOptions` (`AwaitOptions` in `src/generation.ts`, `{ poll?: Poll }`) is the one options type for `await`, `events`, `Video.generate`, and `Video.stream`.
`MediaProtocol.queued` is the submit-then-poll kind every video route uses: `start` (body + decode into `{ token, snapshot }`), `status`, `result`, and optional `cancel`, each addressed by a route-owned `token` whose `Schema.Codec` makes it serializable. `MediaRoute.inline` and `MediaRoute.queued` compose the two kinds with `Endpoint` and `Auth`; the queued route decodes the token once at the boundary (`start` output or `resume` input) and closes over it in a token-free `GenerationRoute` (`status`/`result`/`cancel` are plain Effects), so `Generation` never sees the token's shape and only carries the encoded JSON for persistence. Polls reuse the route's auth and deployment headers plus the request's `http` overlay after `start`, and resolve relative paths against the route base URL (provider-issued absolute URLs such as fal's `status_url` pass through). `result` is always its own GET even when the provider returns output inside the status document, so `Generation.await` behaves the same after `start` and after `resume`. `PollContext.auth` carries only what `Auth` added so protocols can hand download credentials to output assets as transient `Media.Asset.headers` (Veo) — never part of `source` or JSON. Status strings map through a per-protocol `STATUS` table via `MediaProtocol.status`; terminal generations without output fail through `output.ended` / `output.contentPolicy` with the provider document on `reason.body`. `Generation.AwaitOptions` (`{ poll?: Poll }`) is the one options type for `await`, `events`, `Video.generate`, and `Video.stream`.
`MediaProtocol.stream` is the incremental kind every speech route uses, with the same discipline as LLM protocols. `MediaRoute.stream` submits the caller's request as `MediaProtocol.Addressed<Request>` (`{ ...request, mode }`, `mode: "generate" | "stream"`), so one provider stays one protocol: `body.from`, the endpoint path, and `frames` read `request.mode` to pick the body, path, and framing. `frames(bytes, context)` returns frames — `Framing.sse`, `Framing.lines`, `Framing.document` (a single-document response shaped like a streamed record), or the raw `bytes` for chunked audio. `initial()` is fresh per-response parser state; `step` folds each frame into it and emits modality events; `finish(state, context)` runs once after the last frame with the request, body, and observed `http` (header-only usage lives there) and emits exactly one terminal event or fails with `route.incomplete()`. Keep parser state to real accumulators and derive anything the request or body determines in `finish`. `generate` runs the same stream and folds it with the modality's `collect`. Request-derived URL parameters go on the body's `query` (array values repeat the parameter), applied before route and caller `http.query`. Decode frames with `route.decodeFrame` and raise stream-time failures with `route.frameError` (the frame stays on `reason.body`); protocols never thread HTTP context, because the route fills `reason.http` on stream errors that lack it. Speech protocols share `protocols/utils/speech-stream.ts` for deltas, timestamps, voice ids, PCM and container descriptions, and the terminal asset.
`MediaProtocol.stream` is the incremental kind every speech route uses, with the same discipline as LLM protocols. `MediaRoute.stream` submits the caller's request as `MediaProtocol.Addressed<Request>` (`{ ...request, mode }`, `mode: "generate" | "stream"`), so one provider stays one protocol: `body.from`, the endpoint path, and `frames` read `request.mode` to pick the body, path, and framing. `frames(bytes, context)` returns frames — `Framing.sse`, `Framing.lines`, `Framing.document` (a single-document response shaped like a streamed record), or the raw `bytes` for chunked audio. `initial()` is fresh per-response parser state; `step` folds each frame into it and emits modality events; `finish(state, context)` runs once after the last frame with the request, body, and observed `http` (header-only usage lives there) and emits exactly one terminal event or fails with `MediaProtocol.incomplete`. Keep parser state to real accumulators and derive anything the request or body determines in `finish`. `generate` runs the same stream and folds it with the modality's `collect`. Request-derived URL parameters go on the body's `query` (array values repeat the parameter), applied before route and caller `http.query`. Decode frames with `MediaProtocol.decodeFrame` and raise stream-time failures with `MediaProtocol.frameError` (the frame stays on `reason.body`); protocols never thread HTTP context, because the route fills `reason.http` on stream errors that lack it. Speech protocols share `protocols/utils/speech-stream.ts` for deltas, timestamps, voice ids, PCM and container descriptions, and the terminal asset.
Transcription uses all three kinds (OpenAI and Gemini stream, Deepgram is inline, AssemblyAI is queued): every route carries its `kind` and `TranscriptionClient` dispatches on it. `ImageRoute` is the same union; both clients dispatch through `MediaRoute.dispatch` and models compose through `composeAnyRoute`, and fal queue protocols come from `protocols/utils/fal-queue.ts`. Bodies are `json`, `multipart`, or `binary` (a raw upload), and a queued protocol that must upload media before submitting implements `start.prepare` (`MediaProtocol.Prepare`; AssemblyAI `/v2/upload`).
+114 -171
View File
@@ -1,10 +1,11 @@
# @opencode/ai
Schema-first APIs for text, images, video, speech, and transcription, built with Effect.
Schema-first language model and image-generation APIs built with Effect.
```ts
import { Effect } from "effect"
import { AIClient, LLM } from "@opencode/ai"
import { Effect, Layer } from "effect"
import { LLM, LLMClient } from "@opencode/ai"
import { RequestExecutor } from "@opencode/ai/route"
import { OpenAI } from "@opencode/ai/providers"
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
@@ -17,23 +18,23 @@ const request = LLM.request({
})
const program = Effect.gen(function* () {
const response = yield* LLM.generate(request)
const response = yield* LLMClient.generate(request)
console.log(response.text)
})
// Every modality client plus the HTTP request executor; `AIClient.layerWith(executor)` swaps the executor.
await Effect.runPromise(program.pipe(Effect.provide(AIClient.layer)))
const llmLayer = LLMClient.layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
await Effect.runPromise(program.pipe(Effect.provide(llmLayer)))
```
Run `LLM.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses,
Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
The same configured facade names image, video, speech, and transcription models. `Image.generate` resolves the
provider's image route from the model and returns `Media.Asset`s with lazily decoded bytes:
The same configured facade names image models. `Image.request` resolves the provider's image route from the ref and
returns `Media.Asset`s with lazily decoded bytes:
```ts
import { NodeFileSystem } from "@effect/platform-node"
import { Image, Media } from "@opencode/ai"
import { Image, ImageClient, Media } from "@opencode/ai"
const image = Effect.gen(function* () {
const response = yield* Image.generate({
@@ -45,28 +46,13 @@ const image = Effect.gen(function* () {
yield* Media.write(response.image, "./garden.png")
})
// `Media.file` / `Media.write` use the Effect `FileSystem` service; provide your platform's layer.
await Effect.runPromise(image.pipe(Effect.provide(AIClient.layer), Effect.provide(NodeFileSystem.layer)))
// `asset.bytes()` / `Media.write` also need the executor, so merge it into the environment instead of hiding it.
const imageLayer = ImageClient.layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer))
await Effect.runPromise(image.pipe(Effect.provide(imageLayer), Effect.provide(NodeFileSystem.layer)))
```
Advanced: each client also has its own `layer`, which requires `RequestExecutor.Service`. Compose client layers with
`Layer.provideMerge`, not `Layer.provide`: `asset.bytes()`, `Media.write`, and Gemini's `media` output parts need the
executor too, and hiding it fails type-checking with `RequestExecutorService` left in the requirements.
To share a policy such as logging across every client, wrap the executor once with `RequestExecutor.middleware`:
```ts
import { RequestExecutor } from "@opencode/ai/route"
const logged = RequestExecutor.middleware((request, next) =>
Effect.log(`${request.method} ${request.url}`).pipe(Effect.andThen(next(request))),
)
const everything = AIClient.layerWith(logged) // or AI.make({ layer: logged })
```
Prefer promises? `@opencode/ai/promise` exposes the same LLM and media APIs over one managed runtime, plus asset
helpers; `ai.file` and `ai.write` load `node:fs/promises` on first use, so no Effect `FileSystem` is needed:
Prefer promises? `@opencode/ai/promise` exposes the same LLM and image APIs over one managed runtime:
```ts
import { AI } from "@opencode/ai/promise"
@@ -74,7 +60,6 @@ import { AI } from "@opencode/ai/promise"
const ai = AI.make()
const text = await ai.llm.generate({ model: openai.responses("gpt-4o-mini"), prompt: "Say hello." })
const generated = await ai.image.generate({ model: openai.image("gpt-image-2"), prompt: "A lighthouse" })
await ai.write(generated.image, "./lighthouse.png") // also ai.file(path), ai.bytes(asset), ai.base64(asset), ai.materialize(asset)
for await (const event of ai.llm.stream({ model: openai.responses("gpt-4o-mini"), prompt: "Stream hello." })) {
// LLMEvent
}
@@ -337,9 +322,10 @@ and `moonshot/responses`; each exports `model(modelID, settings)`.
MiniMax defaults to its Messages API and reads `MINIMAX_API_KEY` when `apiKey` is omitted:
```ts
import { Effect } from "effect"
import { AIClient, LLM } from "@opencode/ai"
import { Effect, Layer } from "effect"
import { LLM, LLMClient } from "@opencode/ai"
import { MiniMax } from "@opencode/ai/providers"
import { RequestExecutor } from "@opencode/ai/route"
const minimax = MiniMax.configure({ apiKey: process.env.MINIMAX_API_KEY })
const request = LLM.request({
@@ -349,7 +335,8 @@ const request = LLM.request({
generation: { maxTokens: 1536 },
})
const response = await Effect.runPromise(LLM.generate(request).pipe(Effect.provide(AIClient.layer)))
const layer = LLMClient.layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
const response = await Effect.runPromise(LLMClient.generate(request).pipe(Effect.provide(layer)))
console.log(response.text)
```
@@ -418,14 +405,14 @@ Use `Image.generate` for one-off generation or editing:
import { Image, Media } from "@opencode/ai"
const generation = Image.generate({
model: meta.image("muse-image-1.0"),
model: meta("muse-image-1.0"),
prompt: "A flat black square on a white background.",
n: 1,
providerOptions: { reasoningStrength: "low" },
})
const edit = Image.generate({
model: meta.image("muse-image-1.0"),
model: meta("muse-image-1.0"),
prompt: "Make the square purple.",
images: [Media.bytes(imageBytes, "image/webp")],
format: "png",
@@ -472,25 +459,6 @@ const program = Effect.gen(function* () {
})
```
Common fields are portable in shape, not in support. Unsupported fields fail with a typed `AIError` before any network
call rather than being dropped, so check this table before swapping only the `model`:
| Provider | `n` | `size` | `aspectRatio` | `seed` | `format` | `images` | `mask` |
| --------------------- | --- | --------- | ------------- | ------ | -------- | ------------------------- | ------------------- |
| OpenAI | ✓¹ | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ |
| Google (Gemini) | 1 | ✗ | ✓ | ✓ | ✗ | ✓ (no public URLs) | ✗ |
| xAI | ✓ | ✗ | ✓ | ✗ | ✗ | ✓ | ✗ |
| Z.ai | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ |
| Meta | ✓ | ✓ (hint) | ✗ | ✗ | ✓ | ✓ | ✗ |
| Black Forest Labs | 1 | per model | per model | ✓ | ✓ | per model (1–8) | `flux-pro-1.0-fill` |
| fal | ✓ | per model | per model | ✓ | ✓ | 1 (several on `/edit`) | ✓ |
| Replicate | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ (use `providerOptions`) | ✗ |
| Stability `image` | 1 | ✗ | ✓ | ✓ | ✓ | 1 (not on `core`) | ✗ |
| Stability `upscale()` | ✗ | ✗ | ✗ | ✓ | ✓ | exactly 1 (required) | ✗ |
✓ lowers natively; ✗ fails whenever the field is set (including `n: 1`); `1` means `n > 1` fails. ¹ `Image.stream` on OpenAI generates one image. fal
rejects `size` and `aspectRatio` together; which one a fal or BFL model takes depends on the model.
`Media.Asset` is the one asset type shared by image requests, image responses, LLM messages, and tool results.
`asset.source` is the serializable `Media.Source` (`bytes`, `base64`, `url`, or `ref`); `asset.bytes()`,
`asset.base64()`, and `asset.dataUrl()` decode or download lazily and cache; `asset.materialize()` pulls a `url`
@@ -500,8 +468,9 @@ asset into owned bytes before the provider URL expires. Construct assets with `M
Pass ordered image inputs to the same method for editing, composition, or image-conditioned generation:
```ts
const composed = Effect.gen(function* () {
const response = yield* Image.generate({
const response =
yield *
Image.generate({
model,
prompt: "Combine these product photos into one studio scene",
images: [
@@ -512,25 +481,23 @@ const composed = Effect.gen(function* () {
providerOptions,
http,
})
return response.images
})
```
`Media.ref(provider, id)` represents provider file handles such as OpenAI file IDs or Gemini Files URIs; routes
only forward refs that belong to their own provider (OpenAI, xAI, and Gemini images accept them). No shipped route
returns a ref yet, and `asset.bytes()` / `materialize()` on a ref fail by design. Raw strings are not accepted as
image inputs, avoiding ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a
non-empty array selects the provider's edit behavior (see the table above for routes that limit the count). OpenAI
only forward refs that belong to their own provider. Raw strings are not accepted as image inputs, avoiding
ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a
non-empty array selects the provider's edit behavior without enforcing provider image-count limits locally. OpenAI
uses multipart for byte/data-URL edits and its JSON reference body for URL or file-ID edits. The common `mask`
field selects inpainting; routes that cannot honor it fail with `UnsupportedOperation`:
```ts
const inpainted = Image.generate({
model: openai.image("gpt-image-2"),
prompt,
images: [Media.bytes(sourceBytes, "image/png")],
mask: Media.bytes(maskBytes, "image/png"),
})
yield *
Image.generate({
model: openai.image("gpt-image-2"),
prompt,
images: [Media.bytes(sourceBytes, "image/png")],
mask: Media.bytes(maskBytes, "image/png"),
})
```
On multipart requests, `http.body` can override option fields but not structural `model`, `prompt`, `image[]`,
@@ -541,31 +508,31 @@ not accept image inputs. These cases fail with a typed `AIError` before network
Provider-native image options belong to each request. Raw `http.body` fields have final precedence over them:
```ts
const medium = Image.generate({
model: openai.image("gpt-image-2"),
prompt,
providerOptions: { quality: "medium" },
http,
})
yield *
Image.generate({
model: openai.image("gpt-image-2"),
prompt,
providerOptions: { quality: "medium" },
http,
})
```
xAI image models use the same request API with xAI-native controls:
```ts
import { XAI } from "@opencode/ai/providers"
const xai = Image.generate({
model: XAI.configure({ apiKey }).image("any-model-id"),
prompt,
n: 2,
aspectRatio: "16:9",
providerOptions: {
resolution: "1k",
responseFormat: "b64_json",
future_option: true,
},
http,
})
yield *
Image.generate({
model: XAI.configure({ apiKey })("any-model-id"),
prompt,
n: 2,
aspectRatio: "16:9",
providerOptions: {
resolution: "1k",
responseFormat: "b64_json",
future_option: true,
},
http,
})
```
Google's current Gemini image models use the same direct API:
@@ -575,7 +542,7 @@ import { Google } from "@opencode/ai/providers"
const googleProgram = Effect.gen(function* () {
const response = yield* Image.generate({
model: Google.configure({ apiKey }).image("any-model-id"),
model: Google.configure({ apiKey })("any-model-id"),
prompt: "A robot tending a rooftop garden",
aspectRatio: "16:9",
seed: 42,
@@ -600,18 +567,17 @@ their mapped aliases, and `http.body` is the final deep overlay. The selected mo
Z.ai image models infer open Z.ai-native options from the selected model:
```ts
import { ZAI } from "@opencode/ai/providers"
const zai = Image.generate({
model: ZAI.configure({ apiKey }).image("any-model-id"),
prompt,
providerOptions: {
quality: "hd",
userID: "user-123",
future_option: true,
},
http,
})
yield *
Image.generate({
model: ZAI.configure({ apiKey })("any-model-id"),
prompt,
providerOptions: {
quality: "hd",
userID: "user-123",
future_option: true,
},
http,
})
```
Z.ai does not include trustworthy MIME metadata for output URLs, so generated images use
@@ -625,14 +591,12 @@ and emits `image-partial` events before each final `image`; `Image.generate` kee
`dall-e-*` models do not stream and fail typed:
```ts
import { Stream } from "effect"
import { ImageEvent } from "@opencode/ai"
const previews = Image.stream({
model: openai.image("gpt-image-2"),
prompt: "A lighthouse at dusk",
providerOptions: { partialImages: 2 },
}).pipe(Stream.runForEach((event) => (ImageEvent.is.imagePartial(event) ? showPreview(event.image) : Effect.void)))
yield *
Image.stream({
model: openai.image("gpt-image-2"),
prompt: "A lighthouse at dusk",
providerOptions: { partialImages: 2 },
}).pipe(Stream.runForEach((event) => (ImageEvent.is.imagePartial(event) ? showPreview(event.image) : Effect.void)))
```
The provider may send fewer previews than requested when the final image is ready first.
@@ -648,20 +612,13 @@ import { BlackForestLabs, Stability } from "@opencode/ai/providers"
const bfl = BlackForestLabs.configure({ apiKey: process.env.BFL_API_KEY })
const submit = Effect.gen(function* () {
const generation = yield* Image.start({ model: bfl.image("flux-2-pro"), prompt, size: "1024x768" })
persist({ provider: "black-forest-labs", modelID: "flux-2-pro", token: generation.token })
})
const generation = yield * Image.start({ model: bfl.image("flux-2-pro"), prompt, size: "1024x768" })
persist(generation.token)
const finish = Effect.gen(function* () {
const saved = load()
const resumed = yield* Image.resume(bfl.image(saved.modelID), saved.token)
return yield* resumed.await({ poll: { interval: "2 seconds" } })
})
const resumed = yield * Image.resume(bfl.image("flux-2-pro"), loadToken())
const response = yield * resumed.await({ poll: { interval: "2 seconds" } })
```
The token carries no route identity, so persist the provider and model ID alongside it: `resume` needs the model.
- **Black Forest Labs** — results are downloaded before returning, because `result.sample` expires in 10 minutes.
- **Replicate** — inputs are model-defined, so only `prompt` lowers: sizing, count, seed, format, and files go in
`providerOptions` under the model's names, with files as `Media.Asset` (data URLs up to 256 KB, larger by URL).
@@ -671,13 +628,12 @@ The token carries no route identity, so persist the provider and model ID alongs
```ts
const stability = Stability.configure({ apiKey: process.env.STABILITY_API_KEY })
const upscaled = Effect.gen(function* () {
const small = yield* Media.file("./small.png")
return yield* Image.generate(
{ model: stability.upscale(), prompt: "A lighthouse", images: [small] },
const upscaled =
yield *
Image.generate(
{ model: stability.upscale(), prompt: "A lighthouse", images: [yield * Media.file("./small.png")] },
{ poll: { interval: "5 seconds" } },
)
})
```
Imagen is not available: Google shut it down on the Gemini API, and Vertex discontinued the Imagen 4 models on
@@ -710,8 +666,8 @@ Common fields (`frames`, `references`, `video`, `durationSeconds`, `aspectRatio`
under `providerOptions`, inferred from the selected model.
```ts
import { Video } from "@opencode/ai"
import { Google, Runway } from "@opencode/ai/providers"
import { Video, VideoClient } from "@opencode/ai"
import { Google } from "@opencode/ai/providers"
const google = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY })
@@ -740,7 +696,6 @@ const controlled = Effect.gen(function* () {
generation.id // provider operation / task / request id
generation.status // "queued" | "running" | "completed" | "failed" | "cancelled" | "expired"
generation.token // route-owned JSON: `{ operation }`, `{ requestID }`, `{ taskID }`, or fal's follow-up URLs
// The token carries no route identity: persist the provider and model ID alongside it, since `resume` needs the model.
const saved = JSON.stringify(generation.token)
const resumed = yield* Video.resume(google.video("veo-3.1-generate-preview"), JSON.parse(saved))
@@ -751,8 +706,8 @@ const controlled = Effect.gen(function* () {
const events = Video.stream({ model: Runway.configure({ apiKey }).video("gen4.5"), prompt }, { poll })
```
Status polls, result fetches, cancels, and asset downloads all run through the same request executor with the route's
auth. `Generation.await` and `Generation.events` fail with a
`VideoClient.layer` needs `RequestExecutor.Service`, and status polls, result fetches, cancels, and asset downloads
all run through the same executor with the route's auth. `Generation.await` and `Generation.events` fail with a
`Timeout` reason when `poll.timeout` (default 10 minutes) elapses. Failed,
cancelled, and expired generations fail typed with the provider's terminal document on `reason.body`; moderation
outcomes (Veo `raiMediaFilteredReasons`, xAI `respect_moderation`, Runway `SAFETY.*` codes) surface as `notices` when
@@ -771,18 +726,15 @@ Provider notes:
- **Runway** expects pixel ratios in `aspectRatio` for most models (`"1280:720"`), pins `X-Runway-Version`, reports
`usage: { type: "credits" }`, and its output URLs expire after 24–48 hours.
The promise client exposes the same surface: `ai.video.start(...)` resolves to a handle with `await`, `events`,
`result`, `refresh`, `cancel`, and `token`; `ai.video.generate`, `ai.video.resume(model, token)`, and
`ai.video.stream` mirror the Effect API. The handle's `status` and `progress` are a snapshot from when it was
created; `refresh()` resolves to a new handle.
The promise client exposes the same surface: `ai.video.start(...)` resolves to a handle with `await`, `refresh`,
`cancel`, and `token`; `ai.video.generate`, `ai.video.resume(model, token)`, and `ai.video.stream` mirror the Effect
API.
```ts
import { ai } from "@opencode/ai/promise"
const generation = await ai.video.start({ model, prompt })
for await (const event of generation.events({ poll: { interval: 10_000 } })) console.log(event.type)
const video = await generation.result({ signal })
await ai.write(video.video, "./kite.mp4")
const video = await generation.await({ poll: { interval: 10_000 }, signal })
```
## Speech generation
@@ -834,6 +786,7 @@ ElevenLabs and Cartesia. `{ id }` selects an OpenAI custom voice (`{ id: "voice_
plain string elsewhere. There is no cross-provider voice catalog. `format` is the container-level word (`mp3`, `wav`,
`pcm`, `opus`, `aac`, `flac`); sample rates and bitrates live under `providerOptions`, and a value the route cannot
produce fails as `UnsupportedOperation`. Streams buffer every chunk so `finish` can carry the whole clip.
`SpeechClient.layer` needs `RequestExecutor.Service`.
Provider notes:
@@ -861,7 +814,7 @@ The promise client mirrors the Effect API; `ai.speech.stream` is an `AsyncIterab
import { ai } from "@opencode/ai/promise"
const response = await ai.speech.generate({ model, text: "Hello from OpenCode.", voice: "coral" })
await ai.write(response.audio, "hello.mp3")
await Bun.write("hello.mp3", await ai.run(response.audio.bytes()))
for await (const event of ai.speech.stream({ model, text: "Hello from OpenCode.", voice: "coral" })) {
if (event.type === "audio-delta") player.write(event.chunk)
@@ -878,7 +831,6 @@ facades. Common fields (`language`, `prompt`, `timestamps: "none" | "segment" |
natively or fail with a typed `AIError` before any network call; a route may return more than asked.
```ts
import { Console, Effect, Stream } from "effect"
import { Media, Transcription, TranscriptionEvent } from "@opencode/ai"
import { AssemblyAI, Deepgram, OpenAI } from "@opencode/ai/providers"
@@ -905,7 +857,7 @@ const program = Effect.gen(function* () {
Stream.runDrain,
)
// Queued: persist the token with the provider and model ID (the token alone cannot pick the model), resume, and await.
// Queued: persist the token, resume from another process, and await.
const model = AssemblyAI.configure({ apiKey }).transcription("universal-3-5-pro")
const generation = yield* Transcription.start({ model, audio })
const resumed = yield* Transcription.resume(model, JSON.parse(JSON.stringify(generation.token)))
@@ -914,7 +866,7 @@ const program = Effect.gen(function* () {
```
Inline routes emit only `finish` from `stream` (no faked deltas); queued routes emit `generation-queued` /
`generation-progress` before it.
`generation-progress` before it. `TranscriptionClient.layer` needs `RequestExecutor.Service`.
Provider notes:
@@ -926,7 +878,6 @@ Provider notes:
The promise client mirrors the Effect API:
```ts
const audio = await ai.file("./call.mp3")
const text = (await ai.transcription.generate({ model, audio })).text
for await (const event of ai.transcription.stream({ model, audio })) if (event.type === "text-delta") write(event.delta)
const generation = await ai.transcription.start({ model: assemblyai, audio })
@@ -946,8 +897,7 @@ const transcript = await generation.await({ poll: { interval: 3_000 } })
- **`Generation`** — provider-neutral handle for an in-flight media generation (`await`, `refresh`, `cancel`, `events`) used by queued media routes.
- **`Speech.request` / `Speech.generate` / `Speech.stream`** — text-to-speech through a provider-neutral request; `SpeechClient` is its Effect service and layer.
- **`Transcription.request` / `generate` / `stream` / `start` / `resume`** — speech-to-text over inline, streaming, and queued routes; `TranscriptionClient` is its Effect service and layer.
- **`AIClient.layer` / `AIClient.layerWith(executor)`** — every modality client plus the request executor in one layer.
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm`, `image`, `video`, `speech`, and `transcription` as Promise / `AsyncIterable` APIs, plus `file`, `write`, `bytes`, `base64`, and `materialize` for assets.
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm`, `image`, `video`, `speech`, and `transcription` as Promise / `AsyncIterable` APIs.
## Testing
@@ -1010,13 +960,11 @@ This is different from prompt caching, server-side history storage, or truncatio
Prefer this operation, where supported, when the application owns compaction policy and durable context updates.
```ts
const compacted = Effect.gen(function* () {
const result = yield* LLMClient.compact(request)
const next = LLMRequest.update(request, {
messages: result.replacement,
})
return yield* LLMClient.generate(next)
const result = yield * LLMClient.compact(request)
const next = LLMRequest.update(request, {
messages: result.replacement,
})
const response = yield * LLMClient.generate(next)
```
`replacement` replaces the complete input window. Do not append it to the original transcript or extract only the encrypted item: the provider may retain additional messages in its output. Retained user and assistant messages remain ordinary messages with typed text, media, or reasoning parts, in their original order. Provider-specific message IDs, status, and phase use `providerMetadata`, not a raw output array hidden in an assistant message. Unsupported returned item types fail explicitly.
@@ -1032,16 +980,16 @@ The input must still fit the model's context window. Explicit compaction is not
OpenAI Responses also exposes a separate, explicitly selected mechanism:
```ts
const checkpoint = Effect.gen(function* () {
const result = yield* LLMClient.compact(request, {
const result =
yield *
LLMClient.compact(request, {
mechanism: "trigger",
webSocket, // Optional: without it, the request uses HTTP/SSE.
})
result.checkpoint // Successful encrypted CompactionPart.
result.responseID
result.usage
})
result.checkpoint // Successful encrypted CompactionPart.
result.responseID
result.usage
```
This appends a native `compaction_trigger` control item to the full input and sends a normal Responses request, with tools and instructions retained, `stream: true`, `store: false`, and parallel tool calls enabled. It removes normal-answer text/output-format controls, forced tool choices, output-token/tool-call limits, and automatic `context_management`. Body overlays cannot replace `input` or supply `previous_response_id`/`conversation`; the complete canonical history is required for safe stateless replay. Request metadata, auth, headers, query parameters, service tier, and supported prompt-cache settings are preserved.
@@ -1055,11 +1003,9 @@ The supplied WebSocket executor can reuse a compatible append baseline for the c
Trigger support is separate from endpoint support. Only the OpenAI Responses route advertises it; Azure, xAI, Chat, and compatible Responses routes do not inherit it. Untyped calls still fail before sending: missing route capabilities return `UnsupportedOperation`, while unknown mechanism names and invalid inputs return `InvalidRequest`. Dynamic callers must narrow for the selected mechanism:
```ts
const narrowed = Effect.gen(function* () {
if (LLMClient.canCompact(request, { mechanism: "trigger" })) {
const result = yield* LLMClient.compact(request, { mechanism: "trigger" })
}
})
if (LLMClient.canCompact(request, { mechanism: "trigger" })) {
const result = yield * LLMClient.compact(request, { mechanism: "trigger" })
}
```
This capability describes protocol implementation, **not universal availability on OpenAI API deployments**. The host application owns subscription/deployment eligibility, OAuth, endpoint selection, and deployment-specific headers. Local protocol/socket tests do not establish live provider support.
@@ -1068,10 +1014,9 @@ This capability describes protocol implementation, **not universal availability
`providerOptions.contextManagement` lets the provider decide when to compact during an ordinary `generate` or `stream` call. This is an advanced option for callers that own persistence and recovery: persist the complete assistant message, including its checkpoint, before continuing. Enabling the option does not provide durable checkpoint storage, interruption recovery, or model-switch policy. Keep the prior context until a successful checkpoint has been persisted.
Enable OpenAI compaction with typed provider options:
Inside an `Effect.gen`, enable OpenAI compaction with typed provider options:
```ts
import { Effect } from "effect"
import { LLM, LLMClient, LLMRequest, Message } from "@opencode/ai"
import { OpenAI } from "@opencode/ai/providers"
@@ -1082,11 +1027,9 @@ const request = LLM.request({
contextManagement: [{ type: "compaction", compactThreshold: 200_000 }],
},
})
const continued = Effect.gen(function* () {
const response = yield* LLMClient.generate(request)
return LLMRequest.update(request, {
messages: [...request.messages, response.message, Message.user("Continue")],
})
const response = yield * LLMClient.generate(request)
const next = LLMRequest.update(request, {
messages: [...request.messages, response.message, Message.user("Continue")],
})
```
@@ -1314,7 +1257,7 @@ Compose a route with `Route.make({ protocol, endpoint, auth, framing, ... })`. T
## Effect
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `AIClient.layer` (or `AIClient.layerWith(executor)`) for every modality, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `LLMClient.layer` for LLM dispatch and `ImageClient.layer` for image dispatch, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
## See also
+69 -85
View File
@@ -1,6 +1,6 @@
# Media generation in `@opencode/ai` — public API direction
Status: phases 1–4 implemented (through Image queued routes and partial images); phase 5 proposal.
Status: phases 1–3 implemented (Speech and Transcription); phases 4–5 proposal.
## Goal
@@ -54,7 +54,7 @@ Speech.request({ model: openai.speech("gpt-4o-mini-tts"), text })
Transcription.request({ model: openai.transcription("gpt-4o-transcribe"), audio })
```
The request namespace and the selector share one word (`Image.request` + `.image(...)`). That redundancy is accepted: a callable facade returning a lazily resolved ref would be a second way to construct the same model, and the type machinery to infer `providerOptions` through it is not worth one word. Where a provider has two routes for one modality, the selectors stay explicit (`openai.chat`, `stability.image` inline vs `stability.upscale()` queued), and one default per modality per provider is part of the facade definition (OpenAI image → Images API, Google image → Gemini-native; Imagen is shut down, so there is no `google.imagen`). The facade selector (`openai.image(id)`) is the public path for media models. Modality-specific package entrypoints (`model(modelID, settings)` beside today's LLM paths such as `@opencode/ai/providers/openai/responses`) are deferred until Core has a modality-aware model resolver; Core's resolver accepts only `LanguageModel` today.
The request namespace and the selector share one word (`Image.request` + `.image(...)`). That redundancy is accepted: a callable facade returning a lazily resolved ref would be a second way to construct the same model, and the type machinery to infer `providerOptions` through it is not worth one word. Where a provider has two routes for one modality, the selectors stay explicit (`openai.chat`, `stability.image` inline vs `stability.upscale()` queued), and one default per modality per provider is part of the facade definition (OpenAI image → Images API, Google image → Gemini-native; Imagen is shut down, so there is no `google.imagen`). Provider package entrypoints keep `model(modelID, settings)` per modality-specific path, e.g. `@opencode/ai/providers/openai/responses`.
### `Media` — the asset type
@@ -86,18 +86,10 @@ class Media.Asset {
Media.bytes(data, mediaType?) Media.base64(data, mediaType?)
Media.url(url, options?) Media.ref(provider, id)
Media.file(path) // Effect<Asset, AIError, FileSystem>: reads + sniffs
Media.write(asset, path) // Effect<void, AIError, FileSystem | RequestExecutor.Service>
Media.file(path) // Bun/Node: reads + sniffs; Effect FileSystem variant for layers
Media.write(asset, path) // convenience, uses FileSystem
```
`Media.file` and `Media.write` stay Effect-only: bring your platform's `FileSystem` layer. The Promise client owns the
runtime path: `ai.file(path)` and `ai.write(asset, path)` read and write through `node:fs/promises` (loaded on first
use) with the same media-type sniffing and `InvalidRequest` failures, and `ai.bytes`, `ai.base64`, and
`ai.materialize` run the asset methods in its runtime.
A `ref` source is accepted as input only by routes whose provider issues file handles. No shipped route produces one
yet, so `bytes()` and `materialize()` on a ref fail by design until a producer exists.
Raw-PCM outputs (Gemini TTS, Cartesia raw, Deepgram WS) carry `info.encoding/sampleRate/channels` because there is no container header.
### Modality namespaces
@@ -112,31 +104,27 @@ import { OpenAI, Google, ElevenLabs, Fal } from "@opencode/ai/providers"
#### Image
```ts
Effect.gen(function* () {
const request = Image.request({
model: openai.image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
images: [yield* Media.file("./ref.png")], // references / edit sources
mask: yield* Media.file("./mask.png"),
n: 2,
size: "1536x1024", // OpenAI sizes by pixels; Gemini/xAI take aspectRatio instead
format: "webp",
providerOptions: { quality: "high", background: "transparent" }, // typed per model
})
const response = yield* Image.generate(request) // ImageResponse
response.image // Media.Asset (first)
response.images // Media.Asset[]
response.usage // Usage union (see below)
response.notices // moderation / partial-result notices
Image.stream(request) // Stream<ImageEvent>
// ImageEvent: generation-queued | generation-progress | image-partial { index, image } | image { index, image } | finish { usage }
const request = Image.request({
model: openai.image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
images: [Media.file("./ref.png")], // references / edit sources
mask: Media.file("./mask.png"),
n: 2,
size: "1536x1024", // or aspectRatio: "3:2"
seed: 7,
format: "webp",
providerOptions: { quality: "high", background: "transparent" }, // typed per model
})
```
`size` and `aspectRatio` are not interchangeable; each route rejects fields it cannot lower — see the README's Image
portability matrix.
const response = yield* Image.generate(request) // ImageResponse
response.image // Media.Asset (first)
response.images // Media.Asset[]
response.usage // Usage union (see below)
response.notices // moderation / partial-result notices
yield* Image.stream(request) // Stream<ImageEvent>
// ImageEvent: generation-queued | generation-progress | image-partial { index, image } | image { index, image } | finish { usage }
```
Editing is not a separate function; `images`/`mask` on the request select the edit path in the route (OpenAI `/images/edits`, Gemini multimodal parts, xAI `/images/edits`). Routes that cannot honor `mask` fail with `Unsupported`.
@@ -147,43 +135,40 @@ Editing is not a separate function; `images`/`mask` on the request select the ed
Shipped in phase 2 (`src/video.ts`, `src/video-client.ts`, protocols `google-video`, `xai-video`, `fal-video`, `runway-video`).
```ts
Effect.gen(function* () {
const request = Video.request({
model: google.video("veo-3.1-generate-preview"),
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
frames: { first: yield* Media.file("./start.png"), last: yield* Media.file("./end.png") },
references: [yield* Media.file("./style.png")],
video: Media.bytes(previous, "video/mp4"), // edit / extend source
durationSeconds: 8,
aspectRatio: "16:9",
resolution: "1080p",
audio: true,
n: 1,
seed: 7,
negativePrompt: "text, watermark", // common, not provider-native
providerOptions: { personGeneration: "allow_adult" },
})
// Simple: wait for it.
const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } })
response.video // Media.Asset: url with expiresAt (+ transient `headers` for Veo downloads)
response.usage // credits on Runway; the other three report none
response.notices // Veo raiMediaFilteredReasons → filtered, xAI respect_moderation → moderated
yield* response.video.materialize() // pull bytes before the URL expires
// Explicit generation control.
const generation = yield* Video.start(request) // Generation<VideoResponse>
generation.id; generation.status; generation.progress; generation.position; generation.token
yield* generation.await({ poll }) // VideoResponse
yield* generation.cancel() // fal PUT cancel_url, Runway DELETE /tasks/{id}; no-op for Veo and xAI
// Resume from another process. The token is validated against the route's codec and refreshed once. It carries no
// route identity, so persist the provider and model ID alongside it: `resume` needs the model.
const resumed = yield* Video.resume(model, JSON.parse(saved))
// Progress as a stream.
Video.stream(request, { poll }) // Stream<VideoEvent>: generation-queued { id, position } | generation-progress { id, progress } | video { index, video } | finish { usage, notices }
const request = Video.request({
model: google.video("veo-3.1-generate-preview"),
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
frames: { first: Media.file("./start.png"), last: Media.file("./end.png") },
references: [Media.file("./style.png")],
video: Media.bytes(previous, "video/mp4"), // edit / extend source
durationSeconds: 8,
aspectRatio: "16:9",
resolution: "1080p",
audio: true,
n: 1,
seed: 7,
negativePrompt: "text, watermark", // common, not provider-native
providerOptions: { personGeneration: "allow_adult" },
})
// Simple: wait for it.
const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } })
response.video // Media.Asset: url with expiresAt (+ transient `headers` for Veo downloads)
response.usage // credits on Runway; the other three report none
response.notices // Veo raiMediaFilteredReasons → filtered, xAI respect_moderation → moderated
yield* response.video.materialize() // pull bytes before the URL expires
// Explicit generation control.
const generation = yield* Video.start(request) // Generation<VideoResponse>
generation.id; generation.status; generation.progress; generation.position; generation.token
yield* generation.await({ poll }) // VideoResponse
yield* generation.cancel() // fal PUT cancel_url, Runway DELETE /tasks/{id}; no-op for Veo and xAI
// Resume from another process. The token is validated against the route's codec and refreshed once.
const resumed = yield* Video.resume(model, JSON.parse(saved))
// Progress as a stream.
yield* Video.stream(request, { poll }) // Stream<VideoEvent>: generation-queued { id, position } | generation-progress { id, progress } | video { index, video } | finish { usage, notices }
```
Tokens are route-owned JSON: Veo `{ operation }`, xAI `{ requestID }`, Runway `{ taskID }`, fal
@@ -337,20 +322,21 @@ with the realtime work in phase 5. ElevenLabs Scribe is not implemented yet.
```ts
class Generation<Response> {
readonly id: string
readonly route: GenerationRoute<Response> // token-free: { status, result, cancel?: Effect } closed over the decoded token
readonly route: GenerationRoute<Response> // token-free: { status, result, cancel?: Effect; pollHint? } closed over the decoded token
readonly token: unknown // route-owned serializable JSON
readonly status: "queued" | "running" | "completed" | "failed" | "cancelled" | "expired"
readonly progress?: number // 0..1, normalized
readonly position?: number
readonly expiresAt?: number
refresh(): Effect<Generation<Response>, AIError>
result(): Effect<Response, AIError>
await(options?: GenerationAwaitOptions): Effect<Response, AIError>
await(options?: AwaitOptions): Effect<Response, AIError>
cancel(): Effect<void, AIError>
events(options?: GenerationAwaitOptions): Stream<GenerationEvent, AIError> // fails with Timeout past poll.timeout, checked per observation
events(options?: AwaitOptions): Stream<GenerationEvent, AIError> // fails with Timeout past poll.timeout, checked per observation
}
GenerationAwaitOptions = { poll?: Poll }
Poll = { interval?: Duration; timeout?: Duration }
AwaitOptions = { poll?: Poll }
Poll = { interval?: Duration; timeout?: Duration; schedule?: Schedule } // route may override from provider hints (`openai-poll-after-ms`)
```
`Generation` is not video-specific. Image routes on BFL, fal, and Replicate are queued; `Image.start` exists for them. A route declares itself `inline` or `queued`; `generate` on a queued route is `start` then `await`.
@@ -379,17 +365,15 @@ const ai = AI.make() // ManagedRuntime over Reque
// AI.make({ layer }) to inject a custom executor / recorder / middleware
const image = await ai.image.generate({ model, prompt })
await ai.bytes(image.image) // also ai.base64, ai.materialize, ai.write(asset, path)
const reference = await ai.file("./ref.png")
await image.image.bytes()
for await (const event of ai.speech.stream({ model, text, voice })) { … }
const generation = await ai.video.start({ model, prompt }) // snapshot handle; refresh() returns a new one
for await (const event of generation.events({ poll: { interval: 10_000 } })) { … }
const generation = await ai.video.start({ model, prompt })
const video = await generation.await({ poll: { interval: 10_000 }, signal })
const resumed = await ai.video.resume(model, JSON.parse(saved)) // persist provider + model ID with the token
const resumed = ai.video.resume(model, JSON.parse(saved))
const text = await ai.llm.generate({ model, prompt })
const text = await ai.llm.generate({ model, prompt }) // closes today's gap: LLM has no promise API either
for await (const event of ai.llm.stream(request)) { … }
await ai.dispose()
@@ -417,16 +401,16 @@ Existing facades gain per-modality selectors; the modality routes each facade pr
| `Runway` | | | ✓ | | | |
| `Luma`, `Kling`, `MiniMax` | | per provider | | | | |
New facades follow the existing one-file-per-provider rule. The facade selector is the public path for media models; modality-specific package entrypoints (for example `@opencode/ai/providers/openai/images`) are deferred until Core has a modality-aware model resolver.
New facades follow the existing one-file-per-provider rule. Package entrypoints are modality-specific, such as `@opencode/ai/providers/openai/images`, and return the concrete model.
`ImageModel<Options>` gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, and `TranscriptionModel` follow the same generic. They share an internal `MediaModel` base class (ids, route, `http` overlays) that is not part of the public exports; `Generation` and the promise client work with the concrete modality models.
`ImageModel<Options>` already gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, `TranscriptionModel` follow the same generic. A shared `MediaModel` union is what `Generation` and the promise client key on.
### Routes and protocols
Media does not fit the LLM four-axis route (SSE frames → event state machine) except for streaming TTS/STT. Reuse `Endpoint`, `Auth`, `Framing`, `RequestExecutor`, and add media protocol kinds:
- `MediaProtocol.inline` — `body.from(request)` (JSON, multipart, or query), `response.decode(response)` (JSON, or binary body → `Media.Asset`).
- `MediaProtocol.queued` — `start` (body + decode to `{ token, snapshot }`), `status`, `result`, optional `cancel`, and a `token` codec. `result` is always a separate GET (against the status document for Veo/xAI/Runway, fal's `response_url` otherwise) so `await` after `start` and after `resume` share one path. `PollContext.auth` hands the auth headers the route sent to the protocol for output URLs that need them (Veo downloads); they become transient `Media.Asset.headers`, never part of `source`. There is no separate `download` step: `Media.Asset.bytes()` downloads through the executor with those headers. `MediaRoute.inline(...)` / `MediaRoute.queued(...)` compose each kind with endpoint and auth; the queued route decodes the token once and hands `Generation` a token-free `{ status, result, cancel? }`.
- `MediaProtocol.queued` — `start` (body + decode to `{ token, snapshot }`), `status`, `result`, optional `cancel`, `pollHint`, and a `token` codec. `result` is always a separate GET (against the status document for Veo/xAI/Runway, fal's `response_url` otherwise) so `await` after `start` and after `resume` share one path. `PollContext.auth` hands the auth headers the route sent to the protocol for output URLs that need them (Veo downloads); they become transient `Media.Asset.headers`, never part of `source`. There is no separate `download` step: `Media.Asset.bytes()` downloads through the executor with those headers. `MediaRoute.inline(...)` / `MediaRoute.queued(...)` compose each kind with endpoint and auth; the queued route decodes the token once and hands `Generation` a token-free `{ status, result, cancel? }`.
- `MediaProtocol.stream` — `body.from(request)` over the request plus its `mode`, `frames` (a function that picks the framing for the call: `Framing.sse`, `lines`, `document`, or the raw bytes), fresh per-response `initial()` state, `step` emitting modality events, and `finish(state, context)` — with the observed response for header-only usage — emitting exactly one terminal event or failing as an incomplete stream. The route fills `reason.http` on stream errors. `MediaRoute.stream(...)` exposes `stream` and `generate` (the same stream folded by the modality's `collect`).
`MediaRoute.inline` / `MediaRoute.queued` / `MediaRoute.stream` compose one protocol kind with endpoint/auth and tag the route with its `kind`; `ImageModel`/`VideoModel`/`SpeechModel`/`TranscriptionModel` share the `MediaModel` base (`src/media-model.ts`).
+20 -7
View File
@@ -1,7 +1,18 @@
import { Config, Effect, Formatter, Schema, Stream } from "effect"
import { Config, Effect, Formatter, Layer, Schema, Stream } from "effect"
import { NodeFileSystem } from "@effect/platform-node"
import { AIClient, Image, LLM, LLMRequest, Media, Message, ProviderID, Tool, ToolRuntime } from "@opencode/ai"
import { Route, Auth, Endpoint, Framing, Protocol } from "@opencode/ai/route"
import {
Image,
ImageClient,
LLM,
LLMClient,
LLMRequest,
Media,
Message,
ProviderID,
Tool,
ToolRuntime,
} from "@opencode/ai"
import { Route, Auth, Endpoint, Framing, Protocol, RequestExecutor } from "@opencode/ai/route"
import { OpenAI } from "@opencode/ai/providers"
/**
@@ -232,10 +243,12 @@ const generateImage = Effect.gen(function* () {
yield* Media.write(response.image, "tutorial-image.jpg").pipe(Effect.provide(NodeFileSystem.layer))
})
// Provide every modality client and the HTTP request executor once with
// `AIClient.layer` (`AIClient.layerWith(executor)` swaps the executor). Keep one
// path enabled at a time so the tutorial can demonstrate generate, stream, or
// Provide the LLM runtime and the HTTP request executor once. Keep one path
// enabled at a time so the tutorial can demonstrate generate, stream, or
// tool-loop behavior without spending tokens on every example.
const requestExecutorLayer = RequestExecutor.fetchLayer
const llmClientLayer = LLMClient.layer.pipe(Layer.provide(requestExecutorLayer))
const imageClientLayer = ImageClient.layer.pipe(Layer.provide(requestExecutorLayer))
const program = Effect.gen(function* () {
// yield* generateOnce
@@ -244,6 +257,6 @@ const program = Effect.gen(function* () {
// yield* generateDynamicObject.pipe(Effect.andThen((response) => Effect.sync(() => console.log(response.object))))
// yield* generateImage
yield* streamWithTools
}).pipe(Effect.provide(AIClient.layer))
}).pipe(Effect.provide(Layer.mergeAll(requestExecutorLayer, llmClientLayer, imageClientLayer)))
Effect.runPromise(program)
+1 -1
View File
@@ -1,6 +1,6 @@
{
"$schema": "https://json.schemastore.org/package.json",
"version": "2.0.16",
"version": "2.0.15",
"name": "@opencode/ai",
"type": "module",
"license": "MIT",
-24
View File
@@ -1,24 +0,0 @@
import { Layer } from "effect"
import { ImageClient } from "./image-client.js"
import { LLMClient } from "./route/client.js"
import { RequestExecutor } from "./route/executor.js"
import { SpeechClient } from "./speech-client.js"
import { TranscriptionClient } from "./transcription-client.js"
import { VideoClient } from "./video-client.js"
/** Every modality client over `executor`, which stays in the output so `asset.bytes()` and `Media.write` resolve. */
export const layerWith = <E, R>(executor: Layer.Layer<RequestExecutor.Service, E, R>) =>
Layer.mergeAll(
LLMClient.layer,
ImageClient.layer,
VideoClient.layer,
SpeechClient.layer,
TranscriptionClient.layer,
).pipe(Layer.provideMerge(executor))
/** Every modality client plus the executor over `RequestExecutor.fetchLayer`: the one layer most programs need. */
export const layer = layerWith(RequestExecutor.fetchLayer)
export type Services = Layer.Success<typeof layer>
export * as AIClient from "./ai-client.js"
@@ -86,7 +86,7 @@ export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer
})
}),
)
export const fetchLayer = layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer))
export const fetchLayer = layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
export const EvaluationClient = {
Service,
+1 -1
View File
@@ -214,7 +214,7 @@ export function request(input: EvaluationRequest | EvaluationRequestInput) {
return new EvaluationRequest({
...input,
model: input.model as unknown as EvaluationModel,
http: HttpOptions.make(input.http),
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}
+17 -2
View File
@@ -11,6 +11,7 @@ export interface Snapshot {
/** Normalized 0..1 when the provider reports progress. */
readonly progress?: number
readonly position?: number
readonly expiresAt?: number
}
/**
@@ -21,11 +22,15 @@ export interface Route<Response> {
readonly status: Effect.Effect<Snapshot, AIError>
readonly result: Effect.Effect<Response, AIError>
readonly cancel?: Effect.Effect<void, AIError>
/** Provider polling hint (e.g. `openai-poll-after-ms`) that overrides the default interval for the next poll. */
readonly pollHint?: (snapshot: Snapshot) => Duration.Duration | undefined
}
export interface Poll {
readonly interval?: Duration.Input
readonly timeout?: Duration.Input
/** Full override of the polling schedule; `interval` and `pollHint` are ignored when supplied. */
readonly schedule?: Schedule.Schedule<unknown, Snapshot>
}
export interface AwaitOptions {
@@ -58,6 +63,7 @@ export class Generation<Response> {
readonly status: Status
readonly progress?: number
readonly position?: number
readonly expiresAt?: number
constructor(
readonly route: Route<Response>,
@@ -69,6 +75,7 @@ export class Generation<Response> {
this.status = snapshot.status
this.progress = snapshot.progress
this.position = snapshot.position
this.expiresAt = snapshot.expiresAt
}
get snapshot(): Snapshot {
@@ -77,6 +84,7 @@ export class Generation<Response> {
status: this.status,
progress: this.progress,
position: this.position,
expiresAt: this.expiresAt,
}
}
@@ -160,8 +168,15 @@ export class Generation<Response> {
)
}
private schedule(poll: Poll | undefined) {
return Schedule.spaced(poll?.interval ?? DEFAULT_POLL_INTERVAL)
private schedule(poll: Poll | undefined): Schedule.Schedule<unknown, Generation<Response>> {
if (poll?.schedule) return poll.schedule.pipe(Schedule.setInputType<Generation<Response>>())
const interval = poll?.interval ?? DEFAULT_POLL_INTERVAL
const pollHint = this.route.pollHint
const spaced = Schedule.spaced(interval).pipe(Schedule.setInputType<Generation<Response>>())
if (!pollHint) return spaced
return spaced.pipe(
Schedule.modifyDelay((metadata) => Effect.succeed(pollHint(metadata.input.snapshot) ?? interval)),
)
}
}
+1 -3
View File
@@ -30,9 +30,7 @@ export interface Interface {
) => Effect.Effect<Generation<ImageResponse>, AIError>
}
export class ImageClientService extends Context.Service<ImageClientService, Interface>()("@opencode/ImageClient") {}
export const Service = ImageClientService
export type Service = ImageClientService
export class Service extends Context.Service<Service, Interface>()("@opencode/ImageClient") {}
export const generate = <Options extends ImageOptions>(
request: ImageRequestFor<Options>,
+4 -4
View File
@@ -4,7 +4,7 @@ import { Media } from "./media.js"
import { MediaModel, composeAnyRoute, tryRequest } from "./media-model.js"
import { MediaRoute } from "./route/media.js"
import type { MediaProtocol } from "./route/media-protocol.js"
import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js"
import { AIError, HttpOptions, MediaUsage, ProviderMetadata } from "./schema/index.js"
import { ImageClient, Service } from "./image-client.js"
// ---------------------------------------------------------------------------
@@ -45,7 +45,7 @@ export class ImageModel<Options extends ImageOptions = ImageOptions> extends Med
) {
return new ImageModel<Options>({
id: input.id,
provider: route.protocol.provider,
provider: route.provider,
http: input.http,
route: composeAnyRoute(route, input, collectResponse),
})
@@ -97,7 +97,7 @@ export const ImageSize = Schema.declare<ImageSize>(
export type ImageAspectRatio = Media.AspectRatio
export const ImageAspectRatio = Media.AspectRatio
export type ImageFormat = OpenString<"png" | "jpeg" | "webp">
export type ImageFormat = "png" | "jpeg" | "webp" | (string & {})
export class ImageRequest extends Schema.Class<ImageRequest>("Image.Request")({
model: ImageModelSchema,
@@ -225,7 +225,7 @@ export function request(input: ImageRequest | ImageRequestInput) {
if (input instanceof ImageRequest) return input
return new ImageRequest({
...input,
http: HttpOptions.make(input.http),
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}
+1 -2
View File
@@ -1,4 +1,3 @@
export { AIClient } from "./ai-client.js"
export { LLMClient } from "./route/client.js"
export { ImageClient } from "./image-client.js"
export { Auth } from "./route/auth.js"
@@ -9,7 +8,7 @@ export type {
RouteLanguageModelInput,
RouteRoutedLanguageModelInput,
Interface as LLMClientShape,
LLMClientService,
Service as LLMClientService,
} from "./route/client.js"
export * from "./schema/index.js"
export {
+1 -1
View File
@@ -61,7 +61,7 @@ export const request = <const SelectedLanguageModel extends LanguageModel>(
toolChoice: requestToolChoice ? ToolChoice.make(requestToolChoice) : undefined,
generation: requestGeneration === undefined ? undefined : GenerationOptions.make(requestGeneration),
providerOptions: requestProviderOptions,
http: HttpOptions.make(requestHttp),
http: requestHttp === undefined ? undefined : HttpOptions.make(requestHttp),
})
}
+4
View File
@@ -34,6 +34,8 @@ export namespace MediaModel {
/** A protocol plus its canonical start path; `ModelInput.baseURL` overrides `baseURL` per deployment. */
export interface RouteInput<Request extends MediaRoute.MediaRequest, Protocol> {
readonly id: string
readonly provider: string | ProviderID
readonly protocol: Protocol
readonly path: Endpoint.EndpointPart<MediaProtocol.Body, Request>
readonly baseURL?: string
@@ -54,6 +56,8 @@ export const composeRoute = <Request extends MediaRoute.MediaRequest, Protocol,
input: MediaRoute.ModelInput,
): Route =>
compose({
id: route.id,
provider: route.provider,
protocol: route.protocol,
endpoint: Endpoint.path(route.path, { baseURL: input.baseURL ?? route.baseURL }),
auth: input.auth,
+2 -2
View File
@@ -6,7 +6,7 @@ import { ProviderID } from "./schema/ids.js"
import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "./schema/errors.js"
import { ProviderMetadata } from "./schema/options.js"
import { Service } from "./route/executor-service.js"
import { detectMediaType, fileMediaType } from "./utils/media-type.js"
import { detectMediaType, extensionMediaType } from "./utils/media-type.js"
export { detectMediaType } from "./utils/media-type.js"
@@ -309,7 +309,7 @@ export const file = (path: string, options?: AssetOptions): Effect.Effect<Asset,
const data = yield* fs
.readFile(path)
.pipe(Effect.mapError((cause) => invalid(`Failed to read media file ${path}`, cause)))
return bytes(data, fileMediaType(data, path), options)
return bytes(data, detectMediaType(data) ?? extensionMediaType(path), options)
})
/** Materialize an asset and write its bytes through `FileSystem`. */
+30 -64
View File
@@ -1,14 +1,14 @@
import { Effect, Layer, ManagedRuntime, Stream } from "effect"
import { AIClient } from "./ai-client.js"
import type { AwaitOptions, Event, Generation, Snapshot } from "./generation.js"
import type { AwaitOptions, Generation, Snapshot } from "./generation.js"
import { Image, ImageModel, ImageRequest, type ImageOptions, type ImageRequestInput } from "./image.js"
import { ImageClient } from "./image-client.js"
import { LLM } from "./index.js"
import { Media } from "./media.js"
import { tryRequest } from "./media-model.js"
import { LLMClient } from "./route/client.js"
import { RequestExecutor } from "./route/executor.js"
import { AIError, InvalidRequestError, LanguageModel, LLMRequest } from "./schema/index.js"
import { LanguageModel, LLMRequest } from "./schema/index.js"
import type { RequestInput } from "./llm.js"
import { Speech, SpeechModel, SpeechRequest, type SpeechRequestInput } from "./speech.js"
import { SpeechClient } from "./speech-client.js"
import {
Transcription,
TranscriptionModel,
@@ -16,16 +16,17 @@ import {
type TranscriptionOptions,
type TranscriptionRequestInput,
} from "./transcription.js"
import { fileMediaType } from "./utils/media-type.js"
import { TranscriptionClient } from "./transcription-client.js"
import { Video, VideoModel, VideoRequest, type VideoOptions, type VideoRequestInput } from "./video.js"
import { VideoClient } from "./video-client.js"
/**
* Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM, image, video, speech,
* and transcription clients over a request executor; every method runs the corresponding Effect API and rethrows
* `AIError` unchanged. `file` and `write` load `node:fs/promises` on first use, so importing this module does not.
* `AIError` unchanged.
*/
export interface Options {
/** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or `RequestExecutor.middleware(fn)` here. */
/** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or middleware here. */
readonly layer?: Layer.Layer<RequestExecutor.Service>
}
@@ -33,20 +34,19 @@ export interface RunOptions {
readonly signal?: AbortSignal
}
export type Services = AIClient.Services
export type Services =
| Layer.Success<typeof LLMClient.layer>
| Layer.Success<typeof ImageClient.layer>
| Layer.Success<typeof VideoClient.layer>
| Layer.Success<typeof SpeechClient.layer>
| Layer.Success<typeof TranscriptionClient.layer>
| RequestExecutor.Service
/**
* Promise view of a `Generation`. Its fields are a snapshot taken when the handle was created; `refresh()` resolves to a
* new handle rather than updating this one.
*/
/** Promise view of a `Generation`: its snapshot plus `await`, `refresh`, and `cancel` returning promises. */
export type GenerationHandle<Response> = Snapshot & {
/** Serializable JSON; pass it back to `resume` from another process. */
readonly token: unknown
readonly await: (options?: AwaitOptions & RunOptions) => Promise<Response>
/** Status observations until the first terminal one, polling like `await`; abort ends iteration without throwing. */
readonly events: (options?: AwaitOptions & RunOptions) => AsyncIterable<Event>
/** The result without polling; fails when the generation has not completed. */
readonly result: (options?: RunOptions) => Promise<Response>
readonly refresh: (options?: RunOptions) => Promise<GenerationHandle<Response>>
readonly cancel: (options?: RunOptions) => Promise<void>
}
@@ -65,9 +65,17 @@ const abortEffect = (signal: AbortSignal | undefined) =>
})
export const make = (options: Options = {}) => {
const runtime = ManagedRuntime.make(AIClient.layerWith(options.layer ?? RequestExecutor.fetchLayer))
const runtime = ManagedRuntime.make(
Layer.mergeAll(
LLMClient.layer,
ImageClient.layer,
VideoClient.layer,
SpeechClient.layer,
TranscriptionClient.layer,
).pipe(Layer.provideMerge(options.layer ?? RequestExecutor.fetchLayer)),
)
/** Run any package Effect (for example `LLMClient.compact(...)`) inside this runtime. */
/** Run any package Effect (for example `asset.bytes()`) inside this runtime. */
const run = <A, E>(effect: Effect.Effect<A, E, Services>, options?: RunOptions) =>
runtime.runPromise(effect, { signal: options?.signal })
@@ -87,15 +95,12 @@ export const make = (options: Options = {}) => {
...generation.snapshot,
token: generation.token,
await: (options) => run(generation.await({ poll: options?.poll }), options),
events: (options) => iterate(generation.events({ poll: options?.poll }), options),
result: (options) => run(generation.result(), options),
refresh: (options) => run(generation.refresh(), options).then(handle),
cancel: (options) => run(generation.cancel(), options),
})
// The typed `generate`/`stream` overloads take a concrete input or a request, not the union; normalize once here.
const llmRequest = (input: RequestInput | LLMRequest) =>
input instanceof LLMRequest ? Effect.succeed(input) : tryRequest(() => LLM.request(input))
const llmRequest = (input: RequestInput | LLMRequest) => (input instanceof LLMRequest ? input : LLM.request(input))
const imageRequest = (input: ImageRequestInput | ImageRequest) =>
input instanceof ImageRequest ? input : Image.request(input)
const videoRequest = (input: VideoRequestInput | VideoRequest) =>
@@ -107,48 +112,12 @@ export const make = (options: Options = {}) => {
return {
run,
/** Decoded asset bytes, downloading `url` sources through the executor. */
bytes: (asset: Media.Asset, options?: RunOptions) => run(asset.bytes(), options),
base64: (asset: Media.Asset, options?: RunOptions) => run(asset.base64(), options),
/** Pull a `url` asset into owned bytes before the provider URL expires. */
materialize: (asset: Media.Asset, options?: RunOptions) => run(asset.materialize(), options),
/** Read a file into an asset like `Media.file`: sniffed media type, then the extension's. */
file: async (path: string, options?: Media.AssetOptions & RunOptions) => {
const { readFile } = await import("node:fs/promises")
return run(
Effect.tryPromise({
try: (signal) => readFile(path, { signal }),
catch: (cause) => fileError(`Failed to read media file ${path}`, cause),
}).pipe(
Effect.map((buffer) => {
const data = new Uint8Array(buffer)
return Media.bytes(data, fileMediaType(data, path), options)
}),
),
options,
)
},
/** Write an asset's bytes like `Media.write`, downloading `url` sources through the executor. */
write: async (asset: Media.Asset, path: string, options?: RunOptions) => {
const { writeFile } = await import("node:fs/promises")
return run(
asset.bytes().pipe(
Effect.flatMap((data) =>
Effect.tryPromise({
try: (signal) => writeFile(path, data, { signal }),
catch: (cause) => fileError(`Failed to write media file ${path}`, cause),
}),
),
),
options,
)
},
llm: {
request: LLM.request,
generate: <const Model extends LanguageModel>(input: RequestInput<Model> | LLMRequest, options?: RunOptions) =>
run(Effect.flatMap(llmRequest(input), LLM.generate), options),
run(LLM.generate(llmRequest(input)), options),
stream: <const Model extends LanguageModel>(input: RequestInput<Model> | LLMRequest, options?: RunOptions) =>
iterate(Stream.unwrap(Effect.map(llmRequest(input), LLM.stream)), options),
iterate(LLM.stream(llmRequest(input)), options),
},
image: {
request: Image.request,
@@ -217,9 +186,6 @@ export const make = (options: Options = {}) => {
export type Client = ReturnType<typeof make>
const fileError = (message: string, cause: unknown) =>
new AIError({ reason: new InvalidRequestError({ message, cause }) })
/** Default client over `RequestExecutor.fetchLayer` for scripts; the runtime builds its layer on first use. */
export const ai = make()
@@ -1034,6 +1034,7 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
const format = outputConfig?.format ?? undefined
const updates = resolveEffortUpdates(request, options.effort ?? outputConfig?.effort ?? undefined)
const generation = request.generation
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
// Allocate the 4-breakpoint budget in invalidation order: tools → system →
// messages. Tools live highest in the cache hierarchy, so when callers
// over-mark we keep their tool hints and shed the message-tail ones first.
@@ -1043,7 +1044,11 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
flattened.tools.length === 0
? undefined
: flattened.tools.map((tool) =>
lowerTool(breakpoints, tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, request.model)),
lowerTool(
breakpoints,
tool,
ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility),
),
)
// Anthropic rejects tool_choice when tools are absent; "none" is only meaningful with tools present.
const toolChoice = tools === undefined || !request.toolChoice ? undefined : yield* lowerToolChoice(request.toolChoice)
@@ -4,12 +4,14 @@ import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "assemblyai-transcription", name: "AssemblyAI", provider: "assemblyai" })
const ADAPTER = "assemblyai-transcription"
const NAME = "AssemblyAI"
const PROVIDER = ProviderID.make("assemblyai")
export const DEFAULT_BASE_URL = "https://api.assemblyai.com"
export const PATH = "/v2/transcript"
export const UPLOAD_PATH = "/v2/upload"
@@ -31,7 +33,7 @@ export type AssemblyAITranscriptionOptions = {
readonly fallback_language?: string
readonly code_switching?: boolean
}
readonly speech_models?: ReadonlyArray<OpenString<"universal-3-5-pro" | "universal-2">>
readonly speech_models?: ReadonlyArray<"universal-3-5-pro" | "universal-2" | (string & {})>
} & Record<string, unknown>
export type Request = TranscriptionRequestFor<AssemblyAITranscriptionOptions>
@@ -88,12 +90,12 @@ const STATUS = {
// 5. Request body construction
// ---------------------------------------------------------------------------
const decodeUpload = route.decodeJson(Upload)
const decodeUpload = MediaProtocol.decodeJson(ADAPTER, NAME, Upload)
/** `/v2/transcript` only takes a URL, so inline audio is uploaded to `/v2/upload` first. */
const prepare = Effect.fn("AssemblyAITranscription.prepare")(function* (request: Request, send: MediaProtocol.Send) {
if (request.audio.source.type !== "bytes" && request.audio.source.type !== "base64") return request
const audio = yield* MediaInput.inlineBytes(route.id, request.audio)
const audio = yield* MediaInput.inlineBytes(ADAPTER, request.audio)
const uploaded = yield* send(UPLOAD_PATH, MediaProtocol.binary(audio, "application/octet-stream")).pipe(
Effect.flatMap(decodeUpload),
)
@@ -101,7 +103,7 @@ const prepare = Effect.fn("AssemblyAITranscription.prepare")(function* (request:
})
const fromRequest = Effect.fn("AssemblyAITranscription.fromRequest")(function* (request: Request) {
const audio = yield* ProviderShared.mediaReference(request.audio, route.provider, route.name)
const audio = yield* ProviderShared.mediaReference(request.audio, PROVIDER, NAME)
return MediaProtocol.json(
mergeJsonRecords(
{
@@ -124,7 +126,7 @@ const fromRequest = Effect.fn("AssemblyAITranscription.fromRequest")(function* (
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeTranscript = route.decodeJson(Transcript)
const decodeTranscript = MediaProtocol.decodeJson(ADAPTER, NAME, Transcript)
const decodeStart = Effect.fn("AssemblyAITranscription.decodeStart")(function* (
response: HttpClientResponse.HttpClientResponse,
@@ -154,9 +156,9 @@ const decodeResult = Effect.fn("AssemblyAITranscription.decodeResult")(function*
const status = yield* MediaProtocol.status(STATUS, transcript.status, output)
const error = transcript.error ?? undefined
if (status === "failed")
return yield* output.ended("failed", `${route.name} transcription failed${error === undefined ? "" : `: ${error}`}`)
return yield* output.ended("failed", `${NAME} transcription failed${error === undefined ? "" : `: ${error}`}`)
if (status !== "completed")
return yield* output.invalid(`${route.name} transcript ${context.token.transcriptID} has not finished`)
return yield* output.invalid(`${NAME} transcript ${context.token.transcriptID} has not finished`)
const duration = transcript.audio_duration ?? undefined
return new TranscriptionResponse({
text: transcript.text ?? "",
@@ -188,7 +190,9 @@ const decodeResult = Effect.fn("AssemblyAITranscription.decodeResult")(function*
const transcriptPath = (token: Token) => `${PATH}/${token.transcriptID}`
export const protocol = MediaProtocol.queued<Request, TranscriptionResponse, Token>(route, {
export const protocol = MediaProtocol.queued<Request, TranscriptionResponse, Token>({
id: ADAPTER,
name: NAME,
token: Token,
start: { prepare, body: { from: fromRequest }, decode: decodeStart },
status: { path: transcriptPath, decode: decodeStatus },
@@ -197,7 +201,7 @@ export const protocol = MediaProtocol.queued<Request, TranscriptionResponse, Tok
export const model = (input: MediaRoute.ModelInput) =>
TranscriptionModel.fromRoute<AssemblyAITranscriptionOptions, Token>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
+6 -19
View File
@@ -11,7 +11,7 @@ import {
type FinishReasonDetails,
type JsonSchema,
type LLMRequest,
type LanguageModel,
type LanguageModelToolSchemaCompatibility,
type ProviderMetadata,
type ReasoningPart,
type ToolCallPart,
@@ -230,13 +230,13 @@ const lowerToolSpec = (tool: ToolDefinition, inputSchema: JsonSchema): BedrockTo
})
const lowerTools = (
model: LanguageModel,
compatibility: LanguageModelToolSchemaCompatibility | undefined,
breakpoints: BedrockCache.Breakpoints,
tools: ReadonlyArray<ToolDefinition>,
): BedrockTool[] => {
const result: BedrockTool[] = []
for (const tool of tools) {
result.push(lowerToolSpec(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, model)))
result.push(lowerToolSpec(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, compatibility)))
const cachePoint = BedrockCache.block(breakpoints, tool.cache)
if (cachePoint) result.push(cachePoint)
}
@@ -430,30 +430,17 @@ const lowerSystem = (breakpoints: BedrockCache.Breakpoints, system: ReadonlyArra
return content.length === 0 ? undefined : content
}
// Nova 2 rejects `maxTokens` at high reasoning effort, where its output can exceed the field's maximum. Other models
// that take `reasoningConfig`, such as Grok on Bedrock, accept it.
const isNova2 = (model: LanguageModel) => /\bamazon\.nova-2-/.test(model.id)
const isHighReasoningEffort = Schema.is(
Schema.Struct({
additionalModelRequestFields: Schema.Struct({
reasoningConfig: Schema.Struct({ maxReasoningEffort: Schema.Literal("high") }),
}),
}),
)
const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request: LLMRequest) {
const toolChoice = request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined
const flattened = ProviderShared.flattenToolRequest(request)
const generation = request.generation
const maxTokens =
isNova2(request.model) && isHighReasoningEffort(request.http?.body) ? undefined : generation?.maxTokens
// Bedrock-Claude shares Anthropic's 4-breakpoint cap. Spend the budget in
// tools → system → messages order to favour the highest-impact prefixes.
const breakpoints = BedrockCache.breakpoints(request.model.id)
const toolConfig = (() => {
if (flattened.tools.length === 0) return undefined
return {
tools: lowerTools(request.model, breakpoints, flattened.tools),
tools: lowerTools(request.model.compatibility?.toolSchema, breakpoints, flattened.tools),
// Converse has no native "none". Keep definitions stable for prompt
// caching and omit only the unsupported choice.
toolChoice,
@@ -468,14 +455,14 @@ const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request:
}
const inferenceConfig = (() => {
if (
maxTokens === undefined &&
generation?.maxTokens === undefined &&
generation?.temperature === undefined &&
generation?.topP === undefined &&
(generation?.stop === undefined || generation.stop.length === 0)
)
return undefined
return {
maxTokens,
maxTokens: generation?.maxTokens,
temperature: generation?.temperature,
topP: generation?.topP,
stopSequences: generation?.stop,
+33 -16
View File
@@ -5,11 +5,13 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "bfl-images", name: "Black Forest Labs", provider: "black-forest-labs" })
const ADAPTER = "bfl-images"
const NAME = "Black Forest Labs"
const PROVIDER = ProviderID.make("black-forest-labs")
export const DEFAULT_BASE_URL = "https://api.bfl.ai"
// ---------------------------------------------------------------------------
@@ -92,26 +94,33 @@ const capabilities = (model: string): Capabilities => {
return { sizing: "dimensions", imageField: "input_image", maxImages: 8, mask: false }
}
const unsupported = (model: string, field: string, message: string) =>
ProviderShared.unsupportedOperation({
operation: `media.${field}`,
provider: PROVIDER,
route: ADAPTER,
message: `${model} ${message}`,
})
const validate = (request: Request, model: Capabilities) => {
const id = request.model.id
const images = request.images?.length ?? 0
if (request.n !== undefined && request.n > 1)
return Effect.fail(route.unsupported("media.n", `${id} generates one image per request; call it once per image`))
return Effect.fail(unsupported(id, "n", "generates one image per request; call it once per image"))
if (request.size !== undefined && model.sizing !== "dimensions")
return Effect.fail(route.unsupported("media.size", `${id} does not take size (width and height)`))
return Effect.fail(unsupported(id, "size", "does not take size (width and height)"))
if (request.aspectRatio !== undefined && model.sizing !== "aspectRatio")
return Effect.fail(route.unsupported("media.aspectRatio", `${id} does not take aspectRatio`))
if (images > model.maxImages)
return Effect.fail(route.unsupported("media.images", `${id} takes at most ${model.maxImages} images`))
return Effect.fail(unsupported(id, "aspectRatio", "does not take aspectRatio"))
if (images > model.maxImages) return Effect.fail(unsupported(id, "images", `takes at most ${model.maxImages} images`))
if (request.mask !== undefined && !model.mask)
return Effect.fail(route.unsupported("media.mask", `${id} does not inpaint; use flux-pro-1.0-fill`))
return Effect.fail(unsupported(id, "mask", "does not inpaint; use flux-pro-1.0-fill"))
return Effect.void
}
const imageInput = (asset: Media.Asset) => {
const value = asset.inline()?.base64 ?? ProviderShared.mediaUrl(asset)
if (value === undefined)
return Effect.fail(ProviderShared.invalidRequest(`${route.name} accepts inline images or https URLs`))
return Effect.fail(ProviderShared.invalidRequest(`${NAME} accepts inline images or https URLs`))
return Effect.succeed(value)
}
@@ -144,12 +153,12 @@ const fromRequest = Effect.fn("BlackForestLabsImages.fromRequest")(function* (re
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
const decodeStart = MediaProtocol.decodeStarted(ADAPTER, NAME, StartResponse, (value) => ({
token: { id: value.id, pollingURL: value.polling_url },
snapshot: { id: value.id, status: "queued" },
}))
const decodeDocument = route.decodeJson(Result)
const decodeDocument = MediaProtocol.decodeJson(ADAPTER, NAME, Result)
const decodeStatus = Effect.fn("BlackForestLabsImages.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
@@ -166,11 +175,11 @@ const decodeResult = Effect.fn("BlackForestLabsImages.decodeResult")(function* (
const output = yield* decodeDocument(response)
const document = output.value
const status = yield* MediaProtocol.status(STATUS, document.status, output)
if (isModerated(document.status)) return yield* output.contentPolicy(`${route.name} moderated the generation`)
if (isModerated(document.status)) return yield* output.contentPolicy(`${NAME} moderated the generation`)
if (status === "failed" || status === "expired")
return yield* output.ended(status, `${route.name} generation ${context.token.id} ended with ${document.status}`)
return yield* output.ended(status, `${NAME} generation ${context.token.id} ended with ${document.status}`)
if (status !== "completed" || document.result === undefined || document.result === null)
return yield* output.invalid(`${route.name} generation ${context.token.id} has no result`)
return yield* output.invalid(`${NAME} generation ${context.token.id} has no result`)
const { sample, seed, prompt, ...rest } = document.result
return new ImageResponse({
// `sample` is a signed URL that expires 10 minutes after the result is ready, so it is downloaded now.
@@ -187,7 +196,9 @@ const decodeResult = Effect.fn("BlackForestLabsImages.decodeResult")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.queued<Request, ImageResponse, Token>(route, {
export const protocol = MediaProtocol.queued<Request, ImageResponse, Token>({
id: ADAPTER,
name: NAME,
token: Token,
start: { body: { from: fromRequest }, decode: decodeStart },
status: { path: (token) => token.pollingURL, decode: decodeStatus },
@@ -196,7 +207,13 @@ export const protocol = MediaProtocol.queued<Request, ImageResponse, Token>(rout
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<BlackForestLabsImageOptions, Token>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => `/v1/${request.model.id}` },
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => `/v1/${request.model.id}`,
},
input,
)
+25 -15
View File
@@ -3,12 +3,14 @@ import { classifyProviderFailure } from "../provider-error.js"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { AIError, mergeJsonRecords, type OpenString } from "../schema/index.js"
import { AIError, ProviderID, mergeJsonRecords } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { SpeechStream } from "./utils/speech-stream.js"
const route = MediaProtocol.identity({ id: "cartesia-speech", name: "Cartesia", provider: "cartesia" })
const ADAPTER = "cartesia-speech"
const NAME = "Cartesia"
const PROVIDER = ProviderID.make("cartesia")
export const DEFAULT_BASE_URL = "https://api.cartesia.ai"
export const API_VERSION = "2026-08-14"
export const BYTES_PATH = "/tts/bytes"
@@ -20,6 +22,8 @@ const DEFAULT_BIT_RATE = 128000
// 1. Public model input
// ---------------------------------------------------------------------------
export type CartesiaSpeechString<Known extends string> = Known | (string & {})
export type CartesiaEncoding = SpeechStream.PcmEncoding
export type CartesiaSpeechOptions = {
@@ -28,7 +32,7 @@ export type CartesiaSpeechOptions = {
readonly encoding?: CartesiaEncoding
readonly generation_config?: {
readonly volume?: number
readonly emotion?: OpenString<"neutral" | "calm" | "angry" | "content" | "sad" | "scared">
readonly emotion?: CartesiaSpeechString<"neutral" | "calm" | "angry" | "content" | "sad" | "scared">
}
readonly pronunciation_dict_id?: string
} & Record<string, unknown>
@@ -56,7 +60,7 @@ const SseEvent = Schema.Struct({
error_code: optionalNull(Schema.String),
})
const decodeEvent = route.decodeFrame(SseEvent)
const decodeEvent = MediaProtocol.decodeFrame(ADAPTER, NAME, SseEvent)
// ---------------------------------------------------------------------------
// 4. Parser state
@@ -80,14 +84,16 @@ const outputFormat = Effect.fn("CartesiaSpeech.outputFormat")(function* (request
const format = request.format ?? (sse ? "pcm" : "mp3")
const container = CONTAINERS[format]
if (container === undefined)
return yield* route.unsupported(
"media.format",
`${route.name} supports the pcm, wav, and mp3 formats, not "${format}"`,
return yield* SpeechStream.unsupportedFormat(
PROVIDER,
ADAPTER,
`${NAME} supports the pcm, wav, and mp3 formats, not "${format}"`,
)
if (sse && container !== "raw")
return yield* route.unsupported(
"media.format",
`${route.name} streams and timestamps only raw PCM; request format "pcm" instead of "${format}"`,
return yield* SpeechStream.unsupportedFormat(
PROVIDER,
ADAPTER,
`${NAME} streams and timestamps only raw PCM; request format "pcm" instead of "${format}"`,
)
const sampleRate = request.providerOptions?.sampleRate ?? DEFAULT_SAMPLE_RATE
if (container === "mp3")
@@ -98,7 +104,7 @@ const outputFormat = Effect.fn("CartesiaSpeech.outputFormat")(function* (request
const fromRequest = Effect.fn("CartesiaSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
const voice = SpeechStream.voiceID(request.voice)
if (voice === undefined)
return yield* ProviderShared.invalidRequest(`${route.name} requires a voice id; pass it as \`voice\``)
return yield* ProviderShared.invalidRequest(`${NAME} requires a voice id; pass it as \`voice\``)
const { sampleRate: _sampleRate, bitRate: _bitRate, encoding: _encoding, ...native } = request.providerOptions ?? {}
return MediaProtocol.json(
mergeJsonRecords(
@@ -132,7 +138,7 @@ const onEvent = Effect.fn("CartesiaSpeech.onEvent")(function* (state: State, fra
if (event.type === "error")
return yield* new AIError({
reason: classifyProviderFailure({
message: `${route.name} stream failed${event.title === undefined ? "" : ` (${event.title})`}: ${event.message ?? "unknown error"}`,
message: `${NAME} stream failed${event.title === undefined ? "" : ` (${event.title})`}: ${event.message ?? "unknown error"}`,
status: event.status_code,
rawBody: frame,
}),
@@ -144,10 +150,10 @@ const finish = Effect.fn("CartesiaSpeech.finish")(function* (
state: State,
context: MediaProtocol.ResponseContext<Request>,
) {
if (usesSse(context.request) && !state.done) return yield* route.incomplete()
if (usesSse(context.request) && !state.done) return yield* MediaProtocol.incomplete(ADAPTER)
const format = yield* outputFormat(context.request)
return yield* SpeechStream.finish(
route,
ADAPTER,
state,
format.container === "raw"
? SpeechStream.pcm(format.encoding, format.sample_rate)
@@ -159,7 +165,9 @@ const finish = Effect.fn("CartesiaSpeech.finish")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>(route, {
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>({
id: ADAPTER,
name: NAME,
unsupported: ["instructions"],
body: { from: fromRequest },
frames: (bytes, context) => (usesSse(context.request) ? Framing.sse.frame(bytes) : bytes),
@@ -171,6 +179,8 @@ export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<CartesiaSpeechOptions, string | Uint8Array, State>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
headers: { "Cartesia-Version": API_VERSION },
+18 -11
View File
@@ -1,12 +1,14 @@
import { Effect } from "effect"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { MediaInput } from "./utils/media-input.js"
import { SpeechStream } from "./utils/speech-stream.js"
const route = MediaProtocol.identity({ id: "deepgram-speech", name: "Deepgram", provider: "deepgram" })
const ADAPTER = "deepgram-speech"
const NAME = "Deepgram"
const PROVIDER = ProviderID.make("deepgram")
export const DEFAULT_BASE_URL = "https://api.deepgram.com"
export const PATH = "/v1/speak"
@@ -14,11 +16,13 @@ export const PATH = "/v1/speak"
// 1. Public model input
// ---------------------------------------------------------------------------
export type DeepgramEncoding = OpenString<"linear16" | "mulaw" | "alaw" | "mp3" | "opus" | "flac" | "aac">
export type DeepgramSpeechString<Known extends string> = Known | (string & {})
export type DeepgramEncoding = DeepgramSpeechString<"linear16" | "mulaw" | "alaw" | "mp3" | "opus" | "flac" | "aac">
export type DeepgramSpeechOptions = {
readonly encoding?: DeepgramEncoding
readonly container?: OpenString<"wav" | "ogg" | "none">
readonly container?: DeepgramSpeechString<"wav" | "ogg" | "none">
readonly sampleRate?: number
readonly bitRate?: number
readonly mip_opt_out?: boolean
@@ -56,7 +60,7 @@ const audioFormat = (request: Request) => {
const queryParameters = (request: Request) => {
const { encoding: _encoding, container: _container, sampleRate, bitRate, ...native } = request.providerOptions ?? {}
return MediaInput.query(route.id, {
return MediaInput.query(ADAPTER, {
...native,
model: request.model.id,
...audioFormat(request),
@@ -72,9 +76,10 @@ const fromRequest = Effect.fn("DeepgramSpeech.fromRequest")(function* (request:
FORMATS[request.format] === undefined &&
request.providerOptions?.encoding === undefined
)
return yield* route.unsupported(
"media.format",
`${route.name} has no encoding for format "${request.format}"; pass providerOptions.encoding`,
return yield* SpeechStream.unsupportedFormat(
PROVIDER,
ADAPTER,
`${NAME} has no encoding for format "${request.format}"; pass providerOptions.encoding`,
)
return MediaProtocol.json(
mergeJsonRecords({ text: request.text }, request.http?.body) ?? {},
@@ -99,7 +104,7 @@ const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) =
const encoding = HEADERLESS_ENCODINGS[format.encoding ?? ""]
const requestID = headers["dg-request-id"]
const modelName = headers["dg-model-name"]
return SpeechStream.finish(route, state, {
return SpeechStream.finish(ADAPTER, state, {
...(format.container === "none" && encoding !== undefined
? SpeechStream.pcm(encoding, SpeechStream.sampleRate(mediaType), mediaType)
: // Deepgram's default encoding is MP3; WAV is a container around any encoding.
@@ -116,7 +121,9 @@ const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) =
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, Uint8Array, State>(route, {
export const protocol = MediaProtocol.stream<Request, SpeechEvent, Uint8Array, State>({
id: ADAPTER,
name: NAME,
unsupported: ["voice", "language", "instructions", "timestamps"],
body: { from: fromRequest },
frames: (bytes) => bytes,
@@ -127,7 +134,7 @@ export const protocol = MediaProtocol.stream<Request, SpeechEvent, Uint8Array, S
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<DeepgramSpeechOptions, Uint8Array, State>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
@@ -2,12 +2,14 @@ import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "deepgram-transcription", name: "Deepgram", provider: "deepgram" })
const ADAPTER = "deepgram-transcription"
const NAME = "Deepgram"
const PROVIDER = ProviderID.make("deepgram")
export const DEFAULT_BASE_URL = "https://api.deepgram.com"
export const PATH = "/v1/listen"
@@ -22,7 +24,7 @@ export type DeepgramTranscriptionOptions = {
readonly utterances?: boolean
readonly detect_language?: boolean | ReadonlyArray<string>
readonly keyterm?: ReadonlyArray<string>
readonly diarize_model?: OpenString<"latest" | "v1" | "v2">
readonly diarize_model?: "latest" | "v1" | "v2" | (string & {})
readonly filler_words?: boolean
readonly numerals?: boolean
readonly mip_opt_out?: boolean
@@ -77,7 +79,7 @@ const ListenResponse = Schema.Struct({
const query = (request: Request) =>
MediaInput.query(
route.id,
ADAPTER,
mergeJsonRecords(
{
model: request.model.id,
@@ -98,10 +100,8 @@ const fromRequest = Effect.fn("DeepgramTranscription.fromRequest")(function* (re
if (url !== undefined)
return MediaProtocol.json(mergeJsonRecords({ url }, request.http?.body) ?? {}, yield* query(request))
if (request.http?.body !== undefined)
return yield* ProviderShared.invalidRequest(
`${route.name} sends inline audio as the raw body, so http.body cannot apply`,
)
const audio = yield* MediaInput.inlineBytes(route.id, request.audio)
return yield* ProviderShared.invalidRequest(`${NAME} sends inline audio as the raw body, so http.body cannot apply`)
const audio = yield* MediaInput.inlineBytes(ADAPTER, request.audio)
return MediaProtocol.binary(audio, request.audio.mediaType, yield* query(request))
})
@@ -109,7 +109,7 @@ const fromRequest = Effect.fn("DeepgramTranscription.fromRequest")(function* (re
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeListen = route.decodeJson(ListenResponse)
const decodeListen = MediaProtocol.decodeJson(ADAPTER, NAME, ListenResponse)
const speaker = (value: number | undefined) => (value === undefined ? undefined : String(value))
@@ -131,7 +131,7 @@ const decodeResponse = Effect.fn("DeepgramTranscription.decodeResponse")(functio
const output = yield* decodeListen(response)
const channel = output.value.results.channels[0]
const alternative = channel?.alternatives?.[0]
if (alternative === undefined) return yield* output.invalid(`${route.name} returned no transcript`)
if (alternative === undefined) return yield* output.invalid(`${NAME} returned no transcript`)
const duration = output.value.metadata?.duration
const requestID = output.value.metadata?.request_id
return new TranscriptionResponse({
@@ -171,14 +171,19 @@ const decodeResponse = Effect.fn("DeepgramTranscription.decodeResponse")(functio
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, TranscriptionResponse>(route, {
export const protocol = MediaProtocol.inline<Request, TranscriptionResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["prompt", "speakers"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput) =>
TranscriptionModel.fromRoute<DeepgramTranscriptionOptions>({ protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, input)
TranscriptionModel.fromRoute<DeepgramTranscriptionOptions>(
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
export const DeepgramTranscription = {
protocol,
+23 -15
View File
@@ -2,12 +2,14 @@ import { Effect, Schema } from "effect"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { SpeechStream } from "./utils/speech-stream.js"
const route = MediaProtocol.identity({ id: "elevenlabs-speech", name: "ElevenLabs", provider: "elevenlabs" })
const ADAPTER = "elevenlabs-speech"
const NAME = "ElevenLabs"
const PROVIDER = ProviderID.make("elevenlabs")
export const DEFAULT_BASE_URL = "https://api.elevenlabs.io"
export const PATH = "/v1/text-to-speech"
@@ -15,7 +17,9 @@ export const PATH = "/v1/text-to-speech"
// 1. Public model input
// ---------------------------------------------------------------------------
export type ElevenLabsOutputFormat = OpenString<
export type ElevenLabsSpeechString<Known extends string> = Known | (string & {})
export type ElevenLabsOutputFormat = ElevenLabsSpeechString<
| "mp3_22050_32"
| "mp3_24000_48"
| "mp3_44100_32"
@@ -55,7 +59,7 @@ export type ElevenLabsSpeechOptions = {
readonly use_speaker_boost?: boolean
}
readonly seed?: number
readonly apply_text_normalization?: OpenString<"auto" | "on" | "off">
readonly apply_text_normalization?: ElevenLabsSpeechString<"auto" | "on" | "off">
} & Record<string, unknown>
export type Request = SpeechRequestFor<ElevenLabsSpeechOptions>
@@ -75,7 +79,7 @@ const TimestampedAudio = Schema.Struct({
alignment: optionalNull(Alignment),
})
const decodeRecord = route.decodeFrame(TimestampedAudio)
const decodeRecord = MediaProtocol.decodeFrame(ADAPTER, NAME, TimestampedAudio)
// ---------------------------------------------------------------------------
// 4. Parser state
@@ -98,21 +102,23 @@ const OUTPUT_FORMATS: Readonly<Record<string, string>> = {
const outputFormat = Effect.fn("ElevenLabsSpeech.outputFormat")(function* (request: MediaProtocol.Addressed<Request>) {
const format = request.providerOptions?.outputFormat ?? OUTPUT_FORMATS[request.format ?? "mp3"]
if (format === undefined)
return yield* route.unsupported(
"media.format",
`${route.name} has no default output format for "${request.format}"; pass providerOptions.outputFormat`,
return yield* SpeechStream.unsupportedFormat(
PROVIDER,
ADAPTER,
`${NAME} has no default output format for "${request.format}"; pass providerOptions.outputFormat`,
)
if (request.mode === "stream" && format.startsWith("wav_"))
return yield* route.unsupported(
"media.format",
`${route.name} streams mp3, pcm, opus, ulaw, and alaw but not "${format}"; use generate for WAV`,
return yield* SpeechStream.unsupportedFormat(
PROVIDER,
ADAPTER,
`${NAME} streams mp3, pcm, opus, ulaw, and alaw but not "${format}"; use generate for WAV`,
)
return format
})
const fromRequest = Effect.fn("ElevenLabsSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
if (request.voice === undefined)
return yield* ProviderShared.invalidRequest(`${route.name} requires a voice id; pass it as \`voice\``)
return yield* ProviderShared.invalidRequest(`${NAME} requires a voice id; pass it as \`voice\``)
const { outputFormat: _outputFormat, ...native } = request.providerOptions ?? {}
return MediaProtocol.json(
mergeJsonRecords(
@@ -174,7 +180,7 @@ const finish = Effect.fn("ElevenLabsSpeech.finish")(function* (
context: MediaProtocol.ResponseContext<Request>,
) {
const requestID = context.http.headers["request-id"]
return yield* SpeechStream.finish(route, state, {
return yield* SpeechStream.finish(ADAPTER, state, {
...describeOutput(yield* outputFormat(context.request)),
// `character-cost` is billed credits, not a character count (3 for 20 characters on `eleven_flash_v2_5`).
usage: SpeechStream.headerUsage("credits", context.http.headers["character-cost"]),
@@ -186,7 +192,9 @@ const finish = Effect.fn("ElevenLabsSpeech.finish")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>(route, {
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>({
id: ADAPTER,
name: NAME,
unsupported: ["instructions"],
body: { from: fromRequest },
frames: (bytes, context) => {
@@ -200,7 +208,7 @@ export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<ElevenLabsSpeechOptions, string | Uint8Array, State>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => path(request) },
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => path(request) },
input,
)
+39 -21
View File
@@ -4,21 +4,28 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { FalQueue } from "./utils/fal-queue.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "fal-images", name: "fal Images", provider: "fal" })
const ADAPTER = "fal-images"
const NAME = "fal Images"
const PROVIDER = ProviderID.make("fal")
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type FalImageOptions = {
readonly image_size?: OpenString<
"square_hd" | "square" | "portrait_4_3" | "portrait_16_9" | "landscape_4_3" | "landscape_16_9"
>
readonly image_size?:
| "square_hd"
| "square"
| "portrait_4_3"
| "portrait_16_9"
| "landscape_4_3"
| "landscape_16_9"
| (string & {})
readonly enable_safety_checker?: boolean
} & Record<string, unknown>
@@ -54,19 +61,25 @@ const sizing = (model: string) => {
return undefined
}
const unsupported = (model: string, field: string, message: string) =>
ProviderShared.unsupportedOperation({
operation: `media.${field}`,
provider: PROVIDER,
route: ADAPTER,
message: `${model} ${message}`,
})
const validate = (request: Request) => {
const id = request.model.id
const field = sizing(id)
if (request.size !== undefined && request.aspectRatio !== undefined)
return Effect.fail(ProviderShared.invalidRequest(`${route.name} accepts either size or aspectRatio, not both`))
return Effect.fail(ProviderShared.invalidRequest(`${NAME} accepts either size or aspectRatio, not both`))
if (request.size !== undefined && field === "aspect_ratio")
return Effect.fail(route.unsupported("media.size", `${id} sizes by aspectRatio`))
return Effect.fail(unsupported(id, "size", "sizes by aspectRatio"))
if (request.aspectRatio !== undefined && field === "image_size")
return Effect.fail(route.unsupported("media.aspectRatio", `${id} sizes by size (image_size)`))
return Effect.fail(unsupported(id, "aspectRatio", "sizes by size (image_size)"))
if ((request.images?.length ?? 0) > 1 && !isEdit(id))
return Effect.fail(
route.unsupported("media.images", `${id} takes one image_url; use an /edit endpoint for several images`),
)
return Effect.fail(unsupported(id, "images", "takes one image_url; use an /edit endpoint for several images"))
return Effect.void
}
@@ -75,7 +88,7 @@ const isEdit = (model: string) => model.endsWith("/edit")
const fromRequest = Effect.fn("FalImages.fromRequest")(function* (request: Request) {
yield* validate(request)
const images = yield* Effect.forEach(request.images ?? [], (image) => FalQueue.mediaUrl(image, route.name))
const images = yield* Effect.forEach(request.images ?? [], (image) => FalQueue.mediaUrl(image, NAME))
const edit = isEdit(request.model.id)
return MediaProtocol.json(
mergeJsonRecords(
@@ -88,7 +101,7 @@ const fromRequest = Effect.fn("FalImages.fromRequest")(function* (request: Reque
output_format: request.format,
image_urls: edit && images.length > 0 ? images : undefined,
image_url: edit ? undefined : images[0],
mask_url: request.mask === undefined ? undefined : yield* FalQueue.mediaUrl(request.mask, route.name),
mask_url: request.mask === undefined ? undefined : yield* FalQueue.mediaUrl(request.mask, NAME),
},
request.providerOptions,
request.http?.body,
@@ -100,7 +113,7 @@ const fromRequest = Effect.fn("FalImages.fromRequest")(function* (request: Reque
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeQueueResult = route.decodeJson(QueueResult)
const decodeQueueResult = MediaProtocol.decodeJson(ADAPTER, NAME, QueueResult)
const decodeResult = Effect.fn("FalImages.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
@@ -108,7 +121,7 @@ const decodeResult = Effect.fn("FalImages.decodeResult")(function* (
) {
const output = yield* decodeQueueResult(response)
const { images, seed, has_nsfw_concepts, ...rest } = output.value
if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`)
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
// With the safety checker on, flagged images come back blacked out rather than omitted.
const flagged = (has_nsfw_concepts ?? []).flatMap((value, index) => (value ? [index] : []))
return new ImageResponse({
@@ -121,10 +134,7 @@ const decodeResult = Effect.fn("FalImages.decodeResult")(function* (
notices:
flagged.length === 0
? undefined
: flagged.map((index) => ({
type: "moderated" as const,
message: `${route.name} flagged image ${index} as NSFW`,
})),
: flagged.map((index) => ({ type: "moderated" as const, message: `${NAME} flagged image ${index} as NSFW` })),
providerMetadata: { fal: { requestId: context.token.requestID, seed: seed ?? undefined, ...rest } },
})
})
@@ -133,14 +143,22 @@ const decodeResult = Effect.fn("FalImages.decodeResult")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = FalQueue.protocol<Request, ImageResponse>(route, {
export const protocol = FalQueue.protocol<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
from: fromRequest,
decodeResult,
})
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<FalImageOptions, FalQueue.Token>(
{ protocol, baseURL: FalQueue.DEFAULT_BASE_URL, path: ({ request }) => `/${request.model.id}` },
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: FalQueue.DEFAULT_BASE_URL,
path: ({ request }) => `/${request.model.id}`,
},
input,
)
+27 -13
View File
@@ -3,24 +3,28 @@ import type { HttpClientResponse } from "effect/unstable/http"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
import { optionalNull } from "./shared.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { FalQueue } from "./utils/fal-queue.js"
const route = MediaProtocol.identity({ id: "fal-video", name: "fal Video", provider: "fal" })
const ADAPTER = "fal-video"
const NAME = "fal Video"
const PROVIDER = ProviderID.make("fal")
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type FalVideoString<Known extends string> = Known | (string & {})
/**
* Provider-native input. fal video endpoints are model-specific: `duration` is a string enum whose values differ per
* model (`"8s"` for Veo, `"5"` for Kling), and last-frame fields are named per model (`end_image_url`,
* `last_frame_url`, `tail_image_url`), so those pass through here instead of lowering from common fields.
*/
export type FalVideoOptions = {
readonly duration?: OpenString<"4s" | "6s" | "8s" | "5" | "10">
readonly duration?: FalVideoString<"4s" | "6s" | "8s" | "5" | "10">
} & Record<string, unknown>
export type Request = VideoRequestFor<FalVideoOptions>
@@ -48,13 +52,15 @@ const QueueResult = Schema.StructWithRest(
const fromRequest = Effect.fn("FalVideo.fromRequest")(function* (request: Request) {
if (request.frames?.last !== undefined)
return yield* route.unsupported(
"video.frames.last",
`${route.name} names the last frame per model; pass it through providerOptions (e.g. end_image_url) instead of frames.last`,
)
return yield* ProviderShared.unsupportedOperation({
operation: "video.frames.last",
provider: PROVIDER,
route: ADAPTER,
message: `${NAME} names the last frame per model; pass it through providerOptions (e.g. end_image_url) instead of frames.last`,
})
const imageUrl =
request.frames?.first === undefined ? undefined : yield* FalQueue.mediaUrl(request.frames.first, route.name)
const videoUrl = request.video === undefined ? undefined : yield* FalQueue.mediaUrl(request.video, route.name)
request.frames?.first === undefined ? undefined : yield* FalQueue.mediaUrl(request.frames.first, NAME)
const videoUrl = request.video === undefined ? undefined : yield* FalQueue.mediaUrl(request.video, NAME)
return MediaProtocol.json(
mergeJsonRecords(
{
@@ -77,7 +83,7 @@ const fromRequest = Effect.fn("FalVideo.fromRequest")(function* (request: Reques
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeQueueResult = route.decodeJson(QueueResult)
const decodeQueueResult = MediaProtocol.decodeJson(ADAPTER, NAME, QueueResult)
const decodeResult = Effect.fn("FalVideo.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
@@ -103,7 +109,9 @@ const decodeResult = Effect.fn("FalVideo.decodeResult")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = FalQueue.protocol<Request, VideoResponse>(route, {
export const protocol = FalQueue.protocol<Request, VideoResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["n", "durationSeconds", "references"],
from: fromRequest,
decodeResult,
@@ -111,7 +119,13 @@ export const protocol = FalQueue.protocol<Request, VideoResponse>(route, {
export const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<FalVideoOptions, FalQueue.Token>(
{ protocol, baseURL: FalQueue.DEFAULT_BASE_URL, path: ({ request }) => `/${request.model.id}` },
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: FalQueue.DEFAULT_BASE_URL,
path: ({ request }) => `/${request.model.id}`,
},
input,
)
+32 -7
View File
@@ -10,8 +10,8 @@ import {
LLMEvent,
Usage,
type FinishReason,
type JsonSchema,
type LLMRequest,
type LanguageModel,
type MediaPart,
type ProviderMetadata,
type ProviderOptions,
@@ -23,6 +23,7 @@ import { classifyProviderFailure } from "../provider-error.js"
import { Media } from "../media.js"
import { JsonObject, knownString, lenient, optionalArray, optionalNull, ProviderShared } from "./shared.js"
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
import { GeminiToolSchema } from "./utils/gemini-tool-schema.js"
import { Lifecycle } from "./utils/lifecycle.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
@@ -132,7 +133,7 @@ const GeminiSystemInstruction = Schema.Struct({
const GeminiFunctionDeclaration = Schema.Struct({
name: Schema.String,
description: Schema.String,
parametersJsonSchema: JsonObject,
parameters: Schema.optional(JsonObject),
})
const GeminiTool = Schema.Struct({
@@ -265,15 +266,36 @@ interface ParserState {
readonly seenCallIds?: ReadonlySet<string>
}
// =============================================================================
// Tool Schema Conversion
// =============================================================================
// Tool-schema conversion has two distinct concerns:
//
// 1. Sanitize — fix common authoring mistakes Gemini rejects: integer/number
// enums (must be strings), `required` entries that don't match a property,
// untyped arrays (`items` must be present), and `properties`/`required`
// keys on non-object scalars. Mirrors OpenCode's historical Gemini rules.
//
// 2. Project — lossy mapping from JSON Schema to Gemini's schema dialect:
// drop empty root parameter schemas while preserving nested empty objects,
// expand type arrays into `anyOf`, derive `nullable: true` from null members,
// coerce `const` to `[const]` enum, recurse properties/items, and propagate
// only an allowlisted set of keys (description, required, format, type,
// nullable, enum, properties, items, allOf, anyOf, oneOf, minLength).
// Anything outside the allowlist (e.g. `additionalProperties`, `$ref`) is
// silently dropped.
//
// Sanitize runs first, then project. The implementation lives in
// `utils/gemini-tool-schema` so this protocol keeps the same shape as the other
// provider protocols.
// =============================================================================
// Request Lowering
// =============================================================================
// Tool schemas go in `parametersJsonSchema`, which accepts standard JSON Schema. Gemini's schema
// rules are this API's default, including for tuned endpoints whose IDs do not name Gemini.
const lowerTool = (tool: ToolDefinition, model: LanguageModel) => ({
const lowerTool = (tool: ToolDefinition, inputSchema: JsonSchema) => ({
name: tool.name,
description: tool.description,
parametersJsonSchema: ToolSchemaProjection.modelCompatibility(tool.inputSchema, model, "gemini"),
parameters: GeminiToolSchema.convert(inputSchema),
})
const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
@@ -443,6 +465,7 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
const hasTools = flattened.tools.length > 0
const generation = request.generation
const options = yield* decodeOptions(request.providerOptions ?? {})
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
const generationConfig = {
maxOutputTokens: generation?.maxTokens,
temperature: generation?.temperature,
@@ -468,7 +491,9 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
tools: hasTools
? [
{
functionDeclarations: flattened.tools.map((tool) => lowerTool(tool, request.model)),
functionDeclarations: flattened.tools.map((tool) =>
lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility)),
),
},
]
: undefined,
+25 -19
View File
@@ -3,22 +3,26 @@ import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { ProviderShared } from "./shared.js"
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "google-images", name: "Google Images", provider: "google" })
const ADAPTER = "google-images"
const NAME = "Google Images"
const PROVIDER = ProviderID.make("google")
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type GoogleImageString<Known extends string> = Known | (string & {})
/** Provider-native options. Common fields (`aspectRatio`, `seed`, `images`) live on the request. */
export type GoogleImageOptions = {
readonly imageSize?: OpenString<"1K" | "2K" | "4K">
readonly thinkingLevel?: OpenString<"MINIMAL" | "LOW" | "MEDIUM" | "HIGH">
readonly imageSize?: GoogleImageString<"1K" | "2K" | "4K">
readonly thinkingLevel?: GoogleImageString<"MINIMAL" | "LOW" | "MEDIUM" | "HIGH">
readonly includeThoughts?: boolean
} & Record<string, unknown>
@@ -100,13 +104,13 @@ const generationConfig = (request: Request) => {
const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request: Request) {
if (request.n !== undefined && request.n > 1)
return yield* route.unsupported(
"image.n",
`${route.name} generates one image per request; call it once per image instead of n=${request.n}`,
)
const parts = yield* Effect.forEach(request.images ?? [], (image) =>
GeminiGenerateContent.mediaPart(route.name, image),
)
return yield* ProviderShared.unsupportedOperation({
operation: "image.n",
provider: PROVIDER,
route: ADAPTER,
message: `${NAME} generates one image per request; call it once per image instead of n=${request.n}`,
})
const parts = yield* Effect.forEach(request.images ?? [], (image) => GeminiGenerateContent.mediaPart(NAME, image))
return MediaProtocol.json(
mergeJsonRecords(
{
@@ -122,12 +126,10 @@ const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request: Re
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeDocument = route.decodeJson(GoogleImageResponse)
const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* decodeDocument(response)
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, GoogleImageResponse)(response)
const decoded = output.value
const candidates = decoded.candidates ?? []
const candidateMetadata = candidates.map((candidate, candidateIndex) => ({
@@ -167,7 +169,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
const images = yield* Effect.forEach(encoded, (item) =>
MediaInput.decodedAsset(
output.invalid,
`${route.name} candidate ${item.candidateIndex} part ${item.partIndex}`,
`${NAME} candidate ${item.candidateIndex} part ${item.partIndex}`,
item.inlineData.data,
item.inlineData.mimeType,
{
@@ -190,7 +192,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
candidate.finishReason === undefined ? [] : [candidate.finishReason],
)
return yield* output.invalid(
`${route.name} returned no final images${
`${NAME} returned no final images${
finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})`
}; inspect body for prompt feedback and candidate details`,
)
@@ -202,7 +204,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
: [
{
type: "filtered" as const,
message: `${route.name} reported prompt feedback`,
message: `${NAME} reported prompt feedback`,
providerMetadata: { google: { promptFeedback: decoded.promptFeedback } },
},
]),
@@ -212,7 +214,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
: [
{
type: "filtered" as const,
message: `${route.name} candidate ${candidate.index ?? index} finished with ${candidate.finishReason}${
message: `${NAME} candidate ${candidate.index ?? index} finished with ${candidate.finishReason}${
candidate.finishMessage === undefined ? "" : `: ${candidate.finishMessage}`
}`,
providerMetadata: {
@@ -262,7 +264,9 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["mask", "size", "format"],
body: { from: fromRequest },
response: { decode: decodeResponse },
@@ -271,6 +275,8 @@ export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<GoogleImageOptions>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => `/models/${request.model.id}:generateContent`,
+16 -9
View File
@@ -1,12 +1,14 @@
import { Effect, Schema } from "effect"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
import { SpeechStream } from "./utils/speech-stream.js"
const route = MediaProtocol.identity({ id: "google-speech", name: "Google Speech", provider: "google" })
const ADAPTER = "google-speech"
const NAME = "Google Speech"
const PROVIDER = ProviderID.make("google")
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
const DEFAULT_SAMPLE_RATE = 24000
@@ -41,7 +43,7 @@ const GenerateContentChunk = GeminiGenerateContent.chunk(
}),
)
const decodeChunk = route.decodeFrame(GenerateContentChunk)
const decodeChunk = MediaProtocol.decodeFrame(ADAPTER, NAME, GenerateContentChunk)
// ---------------------------------------------------------------------------
// 4. Parser state
@@ -57,9 +59,10 @@ interface State extends SpeechStream.Audio, GeminiGenerateContent.Metadata {
const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
if (request.format !== undefined && request.format !== "pcm")
return yield* route.unsupported(
"media.format",
`${route.name} only returns raw PCM; request format "pcm" or omit it, then wrap the samples yourself`,
return yield* SpeechStream.unsupportedFormat(
PROVIDER,
ADAPTER,
`${NAME} only returns raw PCM; request format "pcm" or omit it, then wrap the samples yourself`,
)
const voiceName = SpeechStream.voiceID(request.voice)
return MediaProtocol.json(
@@ -88,7 +91,7 @@ const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request: Me
const step = Effect.fn("GoogleSpeech.step")(function* (state: State, frame: string) {
const chunk = yield* decodeChunk(frame)
const blocked = GeminiGenerateContent.blocked(route.name, chunk, frame)
const blocked = GeminiGenerateContent.blocked(NAME, chunk, frame)
if (blocked !== undefined) return yield* blocked
const audio = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) =>
part.inlineData === undefined ? [] : [part.inlineData],
@@ -99,7 +102,7 @@ const step = Effect.fn("GoogleSpeech.step")(function* (state: State, frame: stri
const finish = (state: State) => {
const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE
return SpeechStream.finish(route, state, {
return SpeechStream.finish(ADAPTER, state, {
...SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`),
usage: GeminiGenerateContent.usage(state.usage),
providerMetadata: GeminiGenerateContent.providerMetadata(state),
@@ -111,7 +114,9 @@ const finish = (state: State) => {
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string, State>(route, {
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string, State>({
id: ADAPTER,
name: NAME,
unsupported: ["instructions", "speed", "timestamps"],
body: { from: fromRequest },
frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode),
@@ -123,6 +128,8 @@ export const protocol = MediaProtocol.stream<Request, SpeechEvent, string, State
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<GoogleSpeechOptions, string, State>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
// Only `gemini-3.1-flash-tts-preview` and later stream; earlier TTS models reject `streamGenerateContent`.
@@ -1,7 +1,7 @@
import { Effect, Schema, SchemaGetter } from "effect"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import {
TranscriptionFinishEvent,
TranscriptionModel,
@@ -12,9 +12,12 @@ import {
type TranscriptionWord,
type TranscriptionEvent,
} from "../transcription.js"
import { ProviderShared } from "./shared.js"
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
const route = MediaProtocol.identity({ id: "google-transcription", name: "Google Transcription", provider: "google" })
const ADAPTER = "google-transcription"
const NAME = "Google Transcription"
const PROVIDER = ProviderID.make("google")
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
// ---------------------------------------------------------------------------
@@ -27,7 +30,7 @@ export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1bet
*/
export type GoogleTranscriptionOptions = {
readonly audioTranscriptionConfig?: {
readonly mode?: OpenString<"VERBATIM" | "SMART">
readonly mode?: "VERBATIM" | "SMART" | (string & {})
readonly customVocabulary?: ReadonlyArray<string>
readonly languageCodes?: ReadonlyArray<string>
}
@@ -61,7 +64,9 @@ const AudioTranscription = Schema.Struct({
),
})
const decodeChunk = route.decodeFrame(
const decodeChunk = MediaProtocol.decodeFrame(
ADAPTER,
NAME,
GeminiGenerateContent.chunk(Schema.Struct({ audioTranscription: Schema.optional(AudioTranscription) })),
)
@@ -82,14 +87,16 @@ interface State extends GeminiGenerateContent.Metadata {
const fromRequest = Effect.fn("GoogleTranscription.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
// General Gemini models ignore `audioTranscriptionConfig` and answer the audio conversationally.
if (!request.model.id.includes("transcribe"))
return yield* route.unsupported(
"transcription.model",
`${request.model.id} is not a transcription model; use a transcribe model such as gemini-3.5-transcribe`,
)
return yield* ProviderShared.unsupportedOperation({
operation: "transcription.model",
provider: PROVIDER,
route: ADAPTER,
message: `${request.model.id} is not a transcription model; use a transcribe model such as gemini-3.5-transcribe`,
})
return MediaProtocol.json(
mergeJsonRecords(
{
contents: [{ role: "user", parts: [yield* GeminiGenerateContent.mediaPart(route.id, request.audio)] }],
contents: [{ role: "user", parts: [yield* GeminiGenerateContent.mediaPart(ADAPTER, request.audio)] }],
generationConfig: mergeJsonRecords(
{
audioTranscriptionConfig: {
@@ -140,7 +147,7 @@ const turn = (part: Schema.Schema.Type<typeof AudioTranscription>) => {
const step = Effect.fn("GoogleTranscription.step")(function* (state: State, frame: string) {
const chunk = yield* decodeChunk(frame)
const blocked = GeminiGenerateContent.blocked(route.name, chunk, frame)
const blocked = GeminiGenerateContent.blocked(NAME, chunk, frame)
if (blocked !== undefined) return yield* blocked
const turns = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) =>
part.audioTranscription === undefined ? [] : [turn(part.audioTranscription)],
@@ -162,7 +169,7 @@ const step = Effect.fn("GoogleTranscription.step")(function* (state: State, fram
})
const finish = (state: State) => {
if (state.finishReason === undefined) return Effect.fail(route.incomplete())
if (state.finishReason === undefined) return Effect.fail(MediaProtocol.incomplete(ADAPTER))
return Effect.succeed([
TranscriptionFinishEvent.make({
text: state.text,
@@ -178,7 +185,9 @@ const finish = (state: State) => {
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, string, State>(route, {
export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, string, State>({
id: ADAPTER,
name: NAME,
unsupported: ["prompt", "speakers"],
body: { from: fromRequest },
frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode),
@@ -190,6 +199,8 @@ export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, string
export const model = (input: MediaRoute.ModelInput) =>
TranscriptionModel.fromRoute<GoogleTranscriptionOptions, string, State>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => GeminiGenerateContent.path(request.model.id, request.mode),
+33 -21
View File
@@ -4,11 +4,13 @@ import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
import { ProviderShared, optionalArray } from "./shared.js"
const route = MediaProtocol.identity({ id: "google-video", name: "Google Veo", provider: "google" })
const ADAPTER = "google-video"
const NAME = "Google Veo"
const PROVIDER = ProviderID.make("google")
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
/** Veo keeps generated files for two days; the asset carries that deadline so callers materialize in time. */
const FILE_RETENTION = Duration.days(2)
@@ -17,9 +19,11 @@ const FILE_RETENTION = Duration.days(2)
// 1. Public model input
// ---------------------------------------------------------------------------
export type GoogleVideoString<Known extends string> = Known | (string & {})
/** Provider-native `parameters`. Common fields (`aspectRatio`, `resolution`, `durationSeconds`, `seed`) live on the request. */
export type GoogleVideoOptions = {
readonly personGeneration?: OpenString<"allow_all" | "allow_adult" | "dont_allow">
readonly personGeneration?: GoogleVideoString<"allow_all" | "allow_adult" | "dont_allow">
} & Record<string, unknown>
export type Request = VideoRequestFor<GoogleVideoOptions>
@@ -66,23 +70,27 @@ const Operation = Schema.Struct({
// Veo takes inline media only; a prior Veo output is `Media.url` with transient auth, so materialize it first.
const inlineMedia = (asset: Media.Asset) =>
ProviderShared.requireInlineMedia(route.name, asset).pipe(
ProviderShared.requireInlineMedia(NAME, asset).pipe(
Effect.map((inline) => ({ inlineData: { mimeType: inline.mime, data: inline.base64 } })),
)
const fromRequest = Effect.fn("GoogleVideo.fromRequest")(function* (request: Request) {
if (request.n !== undefined && request.n > 1)
return yield* route.unsupported(
"video.n",
`${route.name} generates one video per request; call it once per video instead of n=${request.n}`,
)
return yield* ProviderShared.unsupportedOperation({
operation: "video.n",
provider: PROVIDER,
route: ADAPTER,
message: `${NAME} generates one video per request; call it once per video instead of n=${request.n}`,
})
if (request.audio === false)
return yield* route.unsupported(
"video.audio",
`${route.name} always generates audio; audio: false cannot be honored`,
)
return yield* ProviderShared.unsupportedOperation({
operation: "video.audio",
provider: PROVIDER,
route: ADAPTER,
message: `${NAME} always generates audio; audio: false cannot be honored`,
})
if (request.frames?.last !== undefined && request.frames.first === undefined)
return yield* ProviderShared.invalidRequest(`${route.name} requires frames.first when frames.last is set`)
return yield* ProviderShared.invalidRequest(`${NAME} requires frames.first when frames.last is set`)
const image = request.frames?.first === undefined ? undefined : yield* inlineMedia(request.frames.first)
const lastFrame = request.frames?.last === undefined ? undefined : yield* inlineMedia(request.frames.last)
const video = request.video === undefined ? undefined : yield* inlineMedia(request.video)
@@ -121,7 +129,7 @@ const fromRequest = Effect.fn("GoogleVideo.fromRequest")(function* (request: Req
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
const decodeStart = MediaProtocol.decodeStarted(ADAPTER, NAME, StartResponse, (value) => ({
token: { operation: value.name },
snapshot: { id: value.name, status: "running" },
}))
@@ -132,7 +140,7 @@ const statusOf = (operation: typeof Operation.Type): Status => {
return operation.error === undefined ? "completed" : "failed"
}
const decodeOperation = route.decodeJson(Operation)
const decodeOperation = MediaProtocol.decodeJson(ADAPTER, NAME, Operation)
const decodeStatus = Effect.fn("GoogleVideo.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
@@ -150,11 +158,11 @@ const decodeResult = Effect.fn("GoogleVideo.decodeResult")(function* (
const operation = output.value
const status = statusOf(operation)
if (status === "running")
return yield* output.invalid(`${route.name} operation ${context.token.operation} has not finished`)
return yield* output.invalid(`${NAME} operation ${context.token.operation} has not finished`)
if (status === "failed")
return yield* output.ended(
"failed",
`${route.name} operation failed${operation.error?.message === undefined ? "" : `: ${operation.error.message}`}`,
`${NAME} operation failed${operation.error?.message === undefined ? "" : `: ${operation.error.message}`}`,
)
const generated = operation.response?.generateVideoResponse
// Downloads require the same API key as the poll; the asset carries it transiently and follows the redirect.
@@ -171,14 +179,14 @@ const decodeResult = Effect.fn("GoogleVideo.decodeResult")(function* (
const reasons = generated?.raiMediaFilteredReasons ?? []
const notices = reasons.map((reason) => ({
type: "filtered" as const,
message: `${route.name} filtered media: ${reason}`,
message: `${NAME} filtered media: ${reason}`,
providerMetadata: { google: { raiMediaFilteredReason: reason } },
}))
if (videos.length === 0 && (reasons.length > 0 || (generated?.raiMediaFilteredCount ?? 0) > 0))
return yield* output.contentPolicy(
`${route.name} filtered every video${reasons.length === 0 ? "" : `: ${reasons.join("; ")}`}`,
`${NAME} filtered every video${reasons.length === 0 ? "" : `: ${reasons.join("; ")}`}`,
)
if (videos.length === 0) return yield* output.invalid(`${route.name} operation completed without any video`)
if (videos.length === 0) return yield* output.invalid(`${NAME} operation completed without any video`)
return new VideoResponse({
videos,
notices: notices.length === 0 ? undefined : notices,
@@ -198,7 +206,9 @@ const decodeResult = Effect.fn("GoogleVideo.decodeResult")(function* (
const operationPath = (token: Token) => `/${token.operation}`
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>(route, {
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>({
id: ADAPTER,
name: NAME,
token: Token,
start: { body: { from: fromRequest }, decode: decodeStart },
status: { path: operationPath, decode: decodeStatus },
@@ -208,6 +218,8 @@ export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>(rout
export const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<GoogleVideoOptions, Token>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => `/models/${request.model.id}:predictLongRunning`,
+23 -16
View File
@@ -4,17 +4,20 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { JsonObject, ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "meta-images", name: "Meta Images", provider: "meta" })
export const DEFAULT_BASE_URL = "https://api.meta.ai/v1"
const ADAPTER = "meta-images"
const NAME = "Meta Images"
const PROVIDER = ProviderID.make("meta")
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
type OpenString<Known extends string> = Known | (string & {})
/** Provider-native options. Common fields (`n`, `size`, `format`, `images`) live on the request. */
export type ImageOptions = {
readonly responseFormat?: OpenString<"b64_json" | "url">
@@ -69,7 +72,7 @@ const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
// Meta has no file handles: refs are rejected even when they name this provider.
const reference = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, undefined, route.name).pipe(Effect.map((item) => ({ image_url: item.value })))
ProviderShared.mediaReference(asset, undefined, NAME).pipe(Effect.map((item) => ({ image_url: item.value })))
const fromRequest = Effect.fn("MetaImages.fromRequest")(function* (request: Request) {
const images = yield* Effect.forEach(request.images ?? [], reference)
@@ -98,23 +101,24 @@ const fromRequest = Effect.fn("MetaImages.fromRequest")(function* (request: Requ
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeDocument = route.decodeJson(Response)
const decodeResponse = Effect.fn("MetaImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.DecodeContext<Request>,
) {
const output = yield* decodeDocument(response)
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, Response)(response)
const decoded = output.value
const requested = context.body.type === "json" ? context.body.value.output_format : undefined
const format = decoded.output_format ?? (typeof requested === "string" ? requested : "webp")
const mediaType = `image/${format}`
const images = yield* Effect.forEach(decoded.data, (item, index) =>
MediaInput.imageOutput(output.invalid, `${route.name} result ${index}`, item, mediaType, {
info: { format },
}),
)
if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`)
const images = yield* Effect.forEach(decoded.data, (item, index) => {
if (item.b64_json)
return MediaInput.decodedAsset(output.invalid, `${NAME} result ${index}`, item.b64_json, mediaType, {
info: { format },
})
if (item.url) return Effect.succeed(Media.url(item.url, { mediaType, info: { format } }))
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
})
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
return new ImageResponse({
images,
usage:
@@ -135,17 +139,20 @@ const decodeResponse = Effect.fn("MetaImages.decodeResponse")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["mask", "aspectRatio", "seed"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput) =>
export const model = (input: MediaRoute.ModelInput & { readonly baseURL: string }) =>
ImageModel.fromRoute<ImageOptions>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => `/images/${isEdit(request) ? "edits" : "generations"}`,
},
input,
+3 -7
View File
@@ -6,7 +6,7 @@ import { OpenResponses } from "./open-responses.js"
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
import { ResponsesHostedTools } from "./utils/responses-hosted-tools.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
import { detectMediaType } from "../utils/media-type.js"
import { MetaImage } from "./utils/meta-image.js"
const ADAPTER = "meta-responses"
const NAME = "Meta Responses"
@@ -107,7 +107,7 @@ const fromRequest = Effect.fn("MetaResponses.fromRequest")(function* (request: L
return yield* OpenResponses.lowerTool(
NAME,
tool,
ToolSchemaProjection.modelCompatibility(tool.inputSchema, request.model),
ToolSchemaProjection.modelCompatibility(tool.inputSchema, request.model.compatibility?.toolSchema),
)
return yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(NativeTool))(tool.native.meta)
}),
@@ -151,11 +151,7 @@ const HOSTED_TOOLS = {
),
),
)
// Responses image items can omit output_format, including when PNG/JPEG was requested.
const mime =
item.output_format === undefined
? (detectMediaType(data) ?? "application/octet-stream")
: `image/${item.output_format}`
const mime = MetaImage.mediaType(data, item.output_format)
return {
type: "content" as const,
value: [{ type: "file" as const, uri: `data:${mime};base64,${item.result}`, mime }],
+3 -10
View File
@@ -13,7 +13,6 @@ import {
UnknownProviderError,
Usage,
type FinishReasonDetails,
type JsonSchema,
type LLMRequest,
type MediaPart,
type ToolCallPart,
@@ -23,7 +22,6 @@ import { classifyProviderFailure } from "../provider-error.js"
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
import { Lifecycle } from "./utils/lifecycle.js"
import { MistralToolID } from "./utils/mistral-tool-id.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
import { ToolStream } from "./utils/tool-stream.js"
const ADAPTER = "mistral-chat"
@@ -368,9 +366,9 @@ const lowerMessages = Effect.fn("MistralChat.lowerMessages")(function* (request:
return messages
})
const lowerTool = (tool: ToolDefinition, inputSchema: JsonSchema): MistralTool => ({
const lowerTool = (tool: ToolDefinition): MistralTool => ({
type: "function",
function: { name: tool.name, description: tool.description, parameters: inputSchema, strict: false },
function: { name: tool.name, description: tool.description, parameters: tool.inputSchema, strict: false },
})
export const fromRequest = Effect.fn("MistralChat.fromRequest")(function* (request: LLMRequest) {
@@ -396,12 +394,7 @@ export const fromRequest = Effect.fn("MistralChat.fromRequest")(function* (reque
return {
model: request.model.id,
messages: yield* lowerMessages(flattened.request),
tools:
flattened.tools.length > 0
? flattened.tools.map((tool) =>
lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, request.model)),
)
: undefined,
tools: flattened.tools.length > 0 ? flattened.tools.map(lowerTool) : undefined,
tool_choice: toolChoice,
stream: true as const,
max_tokens: request.generation?.maxTokens,
+6 -1
View File
@@ -818,6 +818,7 @@ export const fromRequestWithAdapter = Effect.fn("OpenResponses.fromRequestWithAd
adapter: ProviderAdapter,
) {
const projected = ProviderShared.flattenToolRequest(request)
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
return {
...(yield* lowerConversation(projected.request, adapter)),
...lowerGeneration(request),
@@ -827,7 +828,11 @@ export const fromRequestWithAdapter = Effect.fn("OpenResponses.fromRequestWithAd
: yield* Effect.forEach(projected.tools, (tool) =>
tool.native !== undefined && adapter.nativeTool
? adapter.nativeTool(tool.native)
: lowerTool(adapter.name, tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, request.model)),
: lowerTool(
adapter.name,
tool,
ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility),
),
),
tool_choice:
allowedToolChoice(request) ??
+2 -1
View File
@@ -803,6 +803,7 @@ export const fromRequest = Effect.fn("OpenAIChat.fromRequest")(function* (
`OpenAI Chat reasoning field conflicts with reserved field ${reasoningField}`,
)
const generation = request.generation
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
const flattened = ProviderShared.flattenToolRequest(request)
const provider = String(request.model.provider)
const baseURL = request.model.route.endpoint.baseURL
@@ -828,7 +829,7 @@ export const fromRequest = Effect.fn("OpenAIChat.fromRequest")(function* (
: flattened.tools.map((tool) =>
lowerTool(
tool,
ToolSchemaProjection.modelCompatibility(tool.inputSchema, request.model),
ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility),
options,
supportsStrictMode,
),
+66 -34
View File
@@ -11,11 +11,13 @@ import { Media } from "../media.js"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type MediaUsage, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords, type MediaUsage } from "../schema/index.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "openai-images", name: "OpenAI Images", provider: "openai" })
const ADAPTER = "openai-images"
const NAME = "OpenAI Images"
const PROVIDER = ProviderID.make("openai")
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/images/generations"
export const EDIT_PATH = "/images/edits"
@@ -24,11 +26,13 @@ export const EDIT_PATH = "/images/edits"
// 1. Public model input
// ---------------------------------------------------------------------------
export type OpenAIImageString<Known extends string> = Known | (string & {})
/** Provider-native options. Common fields (`n`, `size`, `format`, `images`, `mask`) live on the request. */
export type OpenAIImageOptions = {
readonly quality?: OpenString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
readonly background?: OpenString<"auto" | "opaque" | "transparent">
readonly moderation?: OpenString<"auto" | "low">
readonly quality?: OpenAIImageString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
readonly background?: OpenAIImageString<"auto" | "opaque" | "transparent">
readonly moderation?: OpenAIImageString<"auto" | "low">
readonly outputCompression?: number
/** Previews sent before the final image when streaming (default 2); ignored by `Image.generate`. */
readonly partialImages?: number
@@ -79,7 +83,7 @@ const StreamEvent = Schema.Union([
}),
])
const decodeEvent = route.decodeFrame(StreamEvent)
const decodeEvent = MediaProtocol.decodeFrame(ADAPTER, NAME, StreamEvent)
const decodeDocument = Schema.decodeUnknownEffect(Schema.fromJsonString(OpenAIImageResponse))
/** `generate` reads the whole JSON response as one frame, with the requested format for responses that omit it. */
@@ -112,11 +116,21 @@ const streamOptions = (request: MediaProtocol.Addressed<Request>) => {
if (request.mode !== "stream") return Effect.succeed(undefined)
if (request.model.id.startsWith("dall-e"))
return Effect.fail(
route.unsupported("media.stream", `${request.model.id} does not stream; use Image.generate or a GPT image model`),
ProviderShared.unsupportedOperation({
operation: "media.stream",
provider: PROVIDER,
route: ADAPTER,
message: `${request.model.id} does not stream; use Image.generate or a GPT image model`,
}),
)
if (request.n !== undefined && request.n > 1)
return Effect.fail(
route.unsupported("media.n", `${route.name} streams one image; use Image.generate for n=${request.n}`),
ProviderShared.unsupportedOperation({
operation: "media.n",
provider: PROVIDER,
route: ADAPTER,
message: `${NAME} streams one image; use Image.generate for n=${request.n}`,
}),
)
return Effect.succeed({ stream: true, partial_images: request.providerOptions?.partialImages ?? 2 })
}
@@ -126,7 +140,7 @@ const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
const isInline = (asset: Media.Asset) => asset.source.type === "bytes" || asset.source.type === "base64"
const reference = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(
ProviderShared.mediaReference(asset, PROVIDER, NAME).pipe(
Effect.map((item) => (item.type === "ref" ? { file_id: item.value } : { image_url: item.value })),
)
@@ -144,17 +158,18 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Me
// Owned bytes go through multipart edits; remote URLs and file IDs use the JSON edits body instead.
if (images.length > 0 && images.every(isInline) && (mask === undefined || isInline(mask))) {
const form = new FormData()
MediaInput.appendFields(
form,
{ model: request.model.id, prompt: request.prompt },
{ overlay: fields, reserved: RESERVED_FORM_FIELDS },
)
const uploads = yield* Effect.forEach(images, (image) => MediaInput.inlineBytes(route.id, image))
form.append("model", request.model.id)
form.append("prompt", request.prompt)
Object.entries(fields ?? {}).forEach(([key, value]) => {
if (RESERVED_FORM_FIELDS.has(key)) return
form.append(key, typeof value === "string" ? value : ProviderShared.encodeJson(value))
})
const uploads = yield* Effect.forEach(images, (image) => MediaInput.inlineBytes(ADAPTER, image))
uploads.forEach((data, index) =>
form.append("image[]", MediaInput.blob(data, images[index].mediaType), `image-${index}`),
)
if (mask !== undefined)
form.append("mask", MediaInput.blob(yield* MediaInput.inlineBytes(route.id, mask), mask.mediaType), "mask")
form.append("mask", MediaInput.blob(yield* MediaInput.inlineBytes(ADAPTER, mask), mask.mediaType), "mask")
return MediaProtocol.multipart(form)
}
@@ -195,18 +210,22 @@ const usage = (value: Schema.Schema.Type<typeof Usage> | undefined): MediaUsage
}
const eventImage = (frame: string, label: string, data: string, format: string) =>
MediaInput.decodedAsset((message, cause) => route.frameError(message, frame, cause), label, data, `image/${format}`, {
info: { format },
})
MediaInput.decodedAsset(
(message, cause) => MediaProtocol.frameError(ADAPTER, message, frame, cause),
label,
data,
`image/${format}`,
{ info: { format } },
)
const onEvent = Effect.fn("OpenAIImages.onEvent")(function* (state: State, frame: string) {
const event = yield* decodeEvent(frame)
const format = event.output_format
if ("partial_image_index" in event) {
const image = yield* eventImage(frame, `${route.name} partial image`, event.b64_json, format)
const image = yield* eventImage(frame, `${NAME} partial image`, event.b64_json, format)
return [state, [ImagePartialEvent.make({ index: event.partial_image_index, image })]] as const
}
const image = yield* eventImage(frame, `${route.name} result ${state.completed}`, event.b64_json, format)
const image = yield* eventImage(frame, `${NAME} result ${state.completed}`, event.b64_json, format)
return [
{ ...state, completed: state.completed + 1, format, usage: usage(event.usage) },
[ImageOutputEvent.make({ index: state.completed, image })],
@@ -214,20 +233,25 @@ const onEvent = Effect.fn("OpenAIImages.onEvent")(function* (state: State, frame
})
const onDocument = Effect.fn("OpenAIImages.onDocument")(function* (frame: Exclude<Frame, string>) {
const invalid = (message: string, cause?: unknown) => route.frameError(message, frame.document, cause)
const invalid = (message: string, cause?: unknown) =>
MediaProtocol.frameError(ADAPTER, message, frame.document, cause)
const decoded = yield* decodeDocument(frame.document).pipe(
Effect.mapError((cause) => invalid(`${route.name} returned an invalid response`, cause)),
Effect.mapError((cause) => invalid(`${NAME} returned an invalid response`, cause)),
)
const format = decoded.output_format ?? frame.requested ?? "png"
const mediaType = `image/${format}`
const images = yield* Effect.forEach(decoded.data, (item, index) =>
MediaInput.imageOutput(invalid, `${route.name} result ${index}`, item, mediaType, {
info: { format },
providerMetadata:
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } },
}),
)
if (images.length === 0) return yield* invalid(`${route.name} returned no images`)
const images = yield* Effect.forEach(decoded.data, (item, index) => {
const providerMetadata =
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } }
if (item.b64_json)
return MediaInput.decodedAsset(invalid, `${NAME} result ${index}`, item.b64_json, mediaType, {
info: { format },
providerMetadata,
})
if (item.url) return Effect.succeed(Media.url(item.url, { mediaType, info: { format }, providerMetadata }))
return Effect.fail(invalid(`${NAME} result ${index} has neither image data nor a URL`))
})
if (images.length === 0) return yield* invalid(`${NAME} returned no images`)
const state: State = { completed: images.length, format, usage: usage(decoded.usage) }
return [state, images.map((image, index) => ImageOutputEvent.make({ index, image }))] as const
})
@@ -235,7 +259,7 @@ const onDocument = Effect.fn("OpenAIImages.onDocument")(function* (frame: Exclud
const step = (state: State, frame: Frame) => (typeof frame === "string" ? onEvent(state, frame) : onDocument(frame))
const finish = (state: State) => {
if (state.completed === 0) return Effect.fail(route.incomplete())
if (state.completed === 0) return Effect.fail(MediaProtocol.incomplete(ADAPTER))
return Effect.succeed([
ImageFinishEvent.make({ usage: state.usage, providerMetadata: { openai: { outputFormat: state.format } } }),
])
@@ -245,7 +269,9 @@ const finish = (state: State) => {
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, ImageEvent, Frame, State>(route, {
export const protocol = MediaProtocol.stream<Request, ImageEvent, Frame, State>({
id: ADAPTER,
name: NAME,
unsupported: ["aspectRatio", "seed"],
body: { from: fromRequest },
frames: (bytes, context) =>
@@ -261,7 +287,13 @@ export const protocol = MediaProtocol.stream<Request, ImageEvent, Frame, State>(
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<OpenAIImageOptions, Frame, State>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH) },
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
},
input,
)
+11 -22
View File
@@ -5,18 +5,12 @@ import { Auth } from "../route/auth.js"
import { Endpoint } from "../route/endpoint.js"
import { Protocol } from "../route/protocol.js"
import { HttpTransport } from "../route/transport/index.js"
import {
LLMRequest,
mergeJsonRecords,
type JsonSchema,
type LanguageModel,
type ToolDefinition,
type ToolEntry,
} from "../schema/index.js"
import { LLMRequest, mergeJsonRecords, type JsonSchema, type ToolDefinition, type ToolEntry } from "../schema/index.js"
import { resolveEffortUpdates } from "../effort-updates.js"
import { OpenResponses } from "./open-responses.js"
import { OpenResponsesOptions } from "./utils/open-responses-options.js"
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
import { OpenAIImage } from "./utils/openai-image.js"
import { ResponsesHostedTools } from "./utils/responses-hosted-tools.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
import { OpenResponsesChannel } from "./open-responses-channel.js"
@@ -47,16 +41,7 @@ const OpenAIResponsesImageGenerationTool = Schema.Struct({
output_format: Schema.optional(Schema.Literals(["png", "jpeg", "webp"])),
partial_images: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))),
quality: Schema.optional(Schema.Literals(["auto", "low", "medium", "high"])),
size: Schema.optional(
Schema.String.check(
Schema.makeFilter((value) => {
if (value === "auto") return undefined
const match = /^(\d+)x(\d+)$/.exec(value)
if (!match) return "image size must be `auto` or `{width}x{height}`"
return Number(match[1]) > 0 && Number(match[2]) > 0 ? undefined : "image dimensions must be positive integers"
}),
),
),
size: Schema.optional(OpenAIImage.Size),
})
const OpenAIResponsesHostedToolItem = Schema.Union([
@@ -185,9 +170,12 @@ const lowerTool = Effect.fn("OpenAIResponses.lowerTool")(function* (tool: ToolDe
// Native namespaces hold only function tools, so deeper levels flatten into
// the leaf names the same way non-native protocols flatten the whole tree.
const lowerToolEntry = Effect.fn("OpenAIResponses.lowerToolEntry")(function* (tool: ToolEntry, model: LanguageModel) {
const lowerToolEntry = Effect.fn("OpenAIResponses.lowerToolEntry")(function* (
tool: ToolEntry,
compatibility: Parameters<typeof ToolSchemaProjection.modelCompatibility>[1],
) {
if (tool.type === "tool")
return yield* lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, model))
return yield* lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, compatibility))
// OpenAI requires a namespace description; fall back to a generic one so a
// missing description never blocks the request.
return {
@@ -195,7 +183,7 @@ const lowerToolEntry = Effect.fn("OpenAIResponses.lowerToolEntry")(function* (to
name: tool.name,
description: tool.description ?? `Tools in the ${tool.name} namespace.`,
tools: yield* Effect.forEach(ProviderShared.flattenTools(tool.tools), (leaf) =>
OpenResponses.lowerTool(NAME, leaf, ToolSchemaProjection.modelCompatibility(leaf.inputSchema, model)),
OpenResponses.lowerTool(NAME, leaf, ToolSchemaProjection.modelCompatibility(leaf.inputSchema, compatibility)),
),
}
})
@@ -219,6 +207,7 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request:
)(request.providerOptions?.contextManagement)
const options = OpenResponsesOptions.resolve(request)
const updates = resolveEffortUpdates(request, options.reasoningEffort)
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
return yield* decodeBody({
...(yield* OpenResponses.lowerConversation(updates.request, adapter)),
...OpenResponses.lowerGeneration(request, { ...options, reasoningEffort: updates.effort }),
@@ -226,7 +215,7 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request:
tools:
request.tools.length === 0
? undefined
: yield* Effect.forEach(request.tools, (tool) => lowerToolEntry(tool, request.model)),
: yield* Effect.forEach(request.tools, (tool) => lowerToolEntry(tool, toolSchemaCompatibility)),
tool_choice:
request.tools.length === 0
? undefined
+12 -12
View File
@@ -2,11 +2,13 @@ import { Effect, Schema } from "effect"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type MediaUsage } from "../schema/index.js"
import { ProviderID, mergeJsonRecords, type MediaUsage } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { SpeechStream } from "./utils/speech-stream.js"
const route = MediaProtocol.identity({ id: "openai-speech", name: "OpenAI Speech", provider: "openai" })
const ADAPTER = "openai-speech"
const NAME = "OpenAI Speech"
const PROVIDER = ProviderID.make("openai")
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/audio/speech"
/** `pcm` is raw 24 kHz, 16-bit signed little-endian mono samples without a header. */
@@ -16,11 +18,7 @@ const PCM_SAMPLE_RATE = 24000
// 1. Public model input
// ---------------------------------------------------------------------------
/** `voice`, `instructions`, `speed`, and `format` are common request fields; other native body fields pass through. */
export type OpenAISpeechOptions = {
/** Defaults to `"sse"` in `stream` mode on models that support it; the merged value selects the response framing. */
readonly stream_format?: "sse" | "audio"
} & Record<string, unknown>
export type OpenAISpeechOptions = Record<string, unknown>
export type Request = SpeechRequestFor<OpenAISpeechOptions>
@@ -42,7 +40,7 @@ const SpeechStreamEvent = Schema.Union([
}),
])
const decodeEvent = route.decodeFrame(SpeechStreamEvent)
const decodeEvent = MediaProtocol.decodeFrame(ADAPTER, NAME, SpeechStreamEvent)
// ---------------------------------------------------------------------------
// 4. Parser state
@@ -108,9 +106,9 @@ const onEvent = Effect.fn("OpenAISpeech.onEvent")(function* (state: State, frame
})
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
if (isSse(context.body) && !state.done) return Effect.fail(route.incomplete())
if (isSse(context.body) && !state.done) return Effect.fail(MediaProtocol.incomplete(ADAPTER))
const format = context.request.format ?? "mp3"
return SpeechStream.finish(route, state, {
return SpeechStream.finish(ADAPTER, state, {
...(format === "pcm" ? SpeechStream.pcm("pcm_s16le", PCM_SAMPLE_RATE) : SpeechStream.container(format)),
usage: state.usage,
})
@@ -120,7 +118,9 @@ const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) =
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>(route, {
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>({
id: ADAPTER,
name: NAME,
unsupported: ["language", "timestamps"],
body: { from: fromRequest },
frames: (bytes, context) => (isSse(context.body) ? Framing.sse.frame(bytes) : bytes),
@@ -131,7 +131,7 @@ export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<OpenAISpeechOptions, string | Uint8Array, State>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
@@ -2,7 +2,7 @@ import { Effect, Schema, Stream } from "effect"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type MediaUsage } from "../schema/index.js"
import { ProviderID, mergeJsonRecords, type MediaUsage } from "../schema/index.js"
import {
TranscriptionFinishEvent,
TranscriptionModel,
@@ -16,7 +16,9 @@ import { mediaTypeExtension } from "../utils/media-type.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "openai-transcription", name: "OpenAI Transcription", provider: "openai" })
const ADAPTER = "openai-transcription"
const NAME = "OpenAI Transcription"
const PROVIDER = ProviderID.make("openai")
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/audio/transcriptions"
@@ -83,8 +85,8 @@ const Event = Schema.Union([
const Transcript = Schema.Struct(transcriptFields)
type Transcript = Schema.Schema.Type<typeof Transcript>
const decodeEvent = route.decodeFrame(Event)
const decodeTranscript = route.decodeFrame(Transcript)
const decodeEvent = MediaProtocol.decodeFrame(ADAPTER, NAME, Event)
const decodeTranscript = MediaProtocol.decodeFrame(ADAPTER, NAME, Transcript)
type Frame = string | { readonly document: string }
@@ -118,21 +120,24 @@ const capabilities = (model: string): Capabilities => {
return TRANSCRIBE
}
const unsupported = (operation: string, message: string) =>
Effect.fail(ProviderShared.unsupportedOperation({ operation, provider: PROVIDER, route: ADAPTER, message }))
const validate = (request: MediaProtocol.Addressed<Request>, model: Capabilities) => {
const id = request.model.id
if (request.mode === "stream" && !model.stream)
return Effect.fail(route.unsupported("media.stream", `${id} does not stream; use Transcription.generate`))
return unsupported("media.stream", `${id} does not stream; use Transcription.generate`)
if (request.diarize === true && !model.diarize)
return Effect.fail(route.unsupported("media.diarize", `${id} does not diarize; use gpt-4o-transcribe-diarize`))
return unsupported("media.diarize", `${id} does not diarize; use gpt-4o-transcribe-diarize`)
if (request.prompt !== undefined && model.diarize)
return Effect.fail(route.unsupported("media.prompt", `${id} does not accept a prompt`))
return unsupported("media.prompt", `${id} does not accept a prompt`)
if (
request.timestamps === undefined ||
request.timestamps === "none" ||
model.timestamps.includes(request.timestamps)
)
return Effect.void
return Effect.fail(route.unsupported("media.timestamps", `${id} does not return ${request.timestamps} timestamps`))
return unsupported("media.timestamps", `${id} does not return ${request.timestamps} timestamps`)
}
const RESERVED_FORM_FIELDS = new Set([
@@ -145,6 +150,11 @@ const RESERVED_FORM_FIELDS = new Set([
"stream",
])
const appendField = (form: FormData, key: string, value: unknown) => {
if (Array.isArray(value)) return value.forEach((item) => form.append(`${key}[]`, String(item)))
form.append(key, typeof value === "object" && value !== null ? ProviderShared.encodeJson(value) : String(value))
}
const fromRequest = Effect.fn("OpenAITranscription.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
const model = capabilities(request.model.id)
yield* validate(request, model)
@@ -152,18 +162,18 @@ const fromRequest = Effect.fn("OpenAITranscription.fromRequest")(function* (requ
const extension = mediaTypeExtension(request.audio.mediaType)
if (extension === undefined)
return yield* ProviderShared.invalidRequest(
`${route.name} cannot name a ${request.audio.mediaType} upload; send mp3, mp4, m4a, wav, webm, ogg, or flac audio`,
`${NAME} cannot name a ${request.audio.mediaType} upload; send mp3, mp4, m4a, wav, webm, ogg, or flac audio`,
)
const audio = yield* MediaInput.inlineBytes(route.id, request.audio)
const audio = yield* MediaInput.inlineBytes(ADAPTER, request.audio)
const responseFormat = model.diarize
? "diarized_json"
: request.timestamps === undefined || request.timestamps === "none"
? undefined
: "verbose_json"
const form = new FormData()
form.append("file", MediaInput.blob(audio, request.audio.mediaType), `audio.${extension}`)
MediaInput.appendFields(
form,
const native = Object.entries(mergeJsonRecords(request.providerOptions, request.http?.body) ?? {}).filter(
([key]) => !RESERVED_FORM_FIELDS.has(key),
)
const fields = mergeJsonRecords(
{
model: request.model.id,
language: model.languageField === "language" ? request.language : undefined,
@@ -175,12 +185,11 @@ const fromRequest = Effect.fn("OpenAITranscription.fromRequest")(function* (requ
chunking_strategy: model.diarize ? "auto" : undefined,
stream: request.mode === "stream" ? true : undefined,
},
{
overlay: mergeJsonRecords(request.providerOptions, request.http?.body),
reserved: RESERVED_FORM_FIELDS,
repeatArrays: true,
},
Object.fromEntries(native),
)
const form = new FormData()
form.append("file", MediaInput.blob(audio, request.audio.mediaType), `audio.${extension}`)
Object.entries(fields ?? {}).forEach(([key, value]) => appendField(form, key, value))
return MediaProtocol.multipart(form)
})
@@ -224,7 +233,7 @@ const usage = (value: Transcript["usage"]): MediaUsage | undefined => {
const finish = (state: State) => {
const transcript = state.transcript
if (transcript === undefined) return Effect.fail(route.incomplete())
if (transcript === undefined) return Effect.fail(MediaProtocol.incomplete(ADAPTER))
const segments = transcript.segments?.map(segment) ?? state.segments
return Effect.succeed([
TranscriptionFinishEvent.make({
@@ -242,7 +251,9 @@ const finish = (state: State) => {
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, Frame, State>(route, {
export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, Frame, State>({
id: ADAPTER,
name: NAME,
unsupported: ["speakers"],
body: { from: fromRequest },
frames: (bytes, context) =>
@@ -256,7 +267,7 @@ export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, Frame,
export const model = (input: MediaRoute.ModelInput) =>
TranscriptionModel.fromRoute<OpenAITranscriptionOptions, Frame, State>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
+17 -14
View File
@@ -5,10 +5,12 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type AIError } from "../schema/index.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { ProviderShared, optionalNull } from "./shared.js"
const route = MediaProtocol.identity({ id: "replicate-images", name: "Replicate", provider: "replicate" })
const ADAPTER = "replicate-images"
const NAME = "Replicate"
const PROVIDER = ProviderID.make("replicate")
export const DEFAULT_BASE_URL = "https://api.replicate.com"
const OUTPUT_RETENTION = Duration.hours(1)
const MAX_DATA_URL_BYTES = 256 * 1024
@@ -69,11 +71,9 @@ const inlineSize = (source: Media.Source) => {
const fileInput = (asset: Media.Asset) => {
if (inlineSize(asset.source) > MAX_DATA_URL_BYTES)
return Effect.fail(
ProviderShared.invalidRequest(
`${route.name} data URL inputs are limited to 256 KB; pass a larger file by https URL`,
),
ProviderShared.invalidRequest(`${NAME} data URL inputs are limited to 256 KB; pass a larger file by https URL`),
)
return ProviderShared.mediaReference(asset, undefined, route.name).pipe(Effect.map((reference) => reference.value))
return ProviderShared.mediaReference(asset, undefined, NAME).pipe(Effect.map((reference) => reference.value))
}
const inputValue = (value: unknown): Effect.Effect<unknown, AIError> => {
@@ -98,7 +98,7 @@ const fromRequest = Effect.fn("ReplicateImages.fromRequest")(function* (request:
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodePrediction = route.decodeJson(Prediction)
const decodePrediction = MediaProtocol.decodeJson(ADAPTER, NAME, Prediction)
const decodeStart = Effect.fn("ReplicateImages.decodeStart")(function* (
response: HttpClientResponse.HttpClientResponse,
@@ -130,16 +130,15 @@ const decodeResult = Effect.fn("ReplicateImages.decodeResult")(function* (
if (status === "failed" || status === "cancelled" || status === "expired")
return yield* output.ended(
status,
`${route.name} prediction ${context.token.id} ${prediction.status}${typeof prediction.error === "string" ? `: ${prediction.error}` : ""}`,
`${NAME} prediction ${context.token.id} ${prediction.status}${typeof prediction.error === "string" ? `: ${prediction.error}` : ""}`,
)
if (status !== "completed")
return yield* output.invalid(`${route.name} prediction ${context.token.id} has not finished`)
if (status !== "completed") return yield* output.invalid(`${NAME} prediction ${context.token.id} has not finished`)
if (prediction.data_removed === true)
return yield* output.ended("expired", `${route.name} removed the output of prediction ${context.token.id}`)
return yield* output.ended("expired", `${NAME} removed the output of prediction ${context.token.id}`)
if (!isOutput(prediction.output))
return yield* output.invalid(`${route.name} prediction ${context.token.id} returned output that is not image URLs`)
return yield* output.invalid(`${NAME} prediction ${context.token.id} returned output that is not image URLs`)
const urls = typeof prediction.output === "string" ? [prediction.output] : prediction.output
if (urls.length === 0) return yield* output.invalid(`${route.name} prediction ${context.token.id} returned no images`)
if (urls.length === 0) return yield* output.invalid(`${NAME} prediction ${context.token.id} returned no images`)
const predictTime = prediction.metrics?.predict_time ?? undefined
const completedAt = prediction.completed_at ?? undefined
const expiresAt =
@@ -155,7 +154,9 @@ const decodeResult = Effect.fn("ReplicateImages.decodeResult")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.queued<Request, ImageResponse, Token>(route, {
export const protocol = MediaProtocol.queued<Request, ImageResponse, Token>({
id: ADAPTER,
name: NAME,
token: Token,
unsupported: ["images", "mask", "n", "size", "aspectRatio", "seed", "format"],
start: { body: { from: fromRequest }, decode: decodeStart },
@@ -167,6 +168,8 @@ export const protocol = MediaProtocol.queued<Request, ImageResponse, Token>(rout
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<ReplicateImageOptions, Token>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) =>
+20 -13
View File
@@ -4,11 +4,13 @@ import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
import { ProviderShared, optionalArray, optionalNull } from "./shared.js"
const route = MediaProtocol.identity({ id: "runway-video", name: "Runway", provider: "runway" })
const ADAPTER = "runway-video"
const NAME = "Runway"
const PROVIDER = ProviderID.make("runway")
export const DEFAULT_BASE_URL = "https://api.dev.runwayml.com/v1"
/** Every Runway request must pin the API version. */
export const API_VERSION = "2024-11-06"
@@ -23,14 +25,16 @@ const OUTPUT_RETENTION = Duration.hours(24)
// 1. Public model input
// ---------------------------------------------------------------------------
export type RunwayVideoString<Known extends string> = Known | (string & {})
/**
* Provider-native options. Common fields lower to Runway's names: `aspectRatio` → `ratio` (Runway expects pixel
* ratios such as `1280:720` for most models), `durationSeconds` → `duration`, `audio`, `negativePrompt`,
* `resolution`, `references`, and `frames` → `promptImage`.
*/
export type RunwayVideoOptions = {
readonly contentModeration?: { readonly publicFigureThreshold?: OpenString<"auto" | "low"> }
readonly outputFormat?: OpenString<"mp4" | "prores" | "png_sequence">
readonly contentModeration?: { readonly publicFigureThreshold?: RunwayVideoString<"auto" | "low"> }
readonly outputFormat?: RunwayVideoString<"mp4" | "prores" | "png_sequence">
} & Record<string, unknown>
export type Request = VideoRequestFor<RunwayVideoOptions>
@@ -71,7 +75,7 @@ const STATUS = {
// Runway accepts HTTPS URLs, `runway://` upload URIs, and data URIs, all as one string.
const mediaUri = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(Effect.map((reference) => reference.value))
ProviderShared.mediaReference(asset, PROVIDER, NAME).pipe(Effect.map((reference) => reference.value))
const fromRequest = Effect.fn("RunwayVideo.fromRequest")(function* (request: Request) {
const first = request.frames?.first === undefined ? undefined : yield* mediaUri(request.frames.first)
@@ -109,12 +113,12 @@ const fromRequest = Effect.fn("RunwayVideo.fromRequest")(function* (request: Req
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
const decodeStart = MediaProtocol.decodeStarted(ADAPTER, NAME, StartResponse, (value) => ({
token: { taskID: value.id },
snapshot: { id: value.id, status: "queued" },
}))
const decodeTask = route.decodeJson(Task)
const decodeTask = MediaProtocol.decodeJson(ADAPTER, NAME, Task)
const decodeStatus = Effect.fn("RunwayVideo.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
@@ -134,17 +138,16 @@ const decodeResult = Effect.fn("RunwayVideo.decodeResult")(function* (
const status = yield* MediaProtocol.status(STATUS, task.status, output)
if (status === "failed") {
const code = task.failureCode ?? undefined
const message = `${route.name} task failed${code === undefined ? "" : ` (${code})`}${task.failure ? `: ${task.failure}` : ""}`
const message = `${NAME} task failed${code === undefined ? "" : ` (${code})`}${task.failure ? `: ${task.failure}` : ""}`
// Runway failure codes are dotted paths; every moderation outcome carries a SAFETY segment.
if (code !== undefined && /(^|\.)SAFETY(\.|$)/.test(code)) return yield* output.contentPolicy(message)
return yield* output.ended("failed", message)
}
if (status === "cancelled")
return yield* output.ended("cancelled", `${route.name} task ${context.token.taskID} was cancelled`)
if (status !== "completed")
return yield* output.invalid(`${route.name} task ${context.token.taskID} has not finished`)
return yield* output.ended("cancelled", `${NAME} task ${context.token.taskID} was cancelled`)
if (status !== "completed") return yield* output.invalid(`${NAME} task ${context.token.taskID} has not finished`)
const urls = task.output ?? []
if (urls.length === 0) return yield* output.invalid(`${route.name} task succeeded without any output`)
if (urls.length === 0) return yield* output.invalid(`${NAME} task succeeded without any output`)
return new VideoResponse({
videos: yield* Effect.forEach(urls, (url) =>
MediaProtocol.expiringUrl(url, OUTPUT_RETENTION, { mediaType: "video/mp4" }),
@@ -165,7 +168,9 @@ const decodeResult = Effect.fn("RunwayVideo.decodeResult")(function* (
const taskPath = (token: Token) => `${TASKS_PATH}/${token.taskID}`
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>(route, {
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>({
id: ADAPTER,
name: NAME,
token: Token,
unsupported: ["n"],
start: { body: { from: fromRequest }, decode: decodeStart },
@@ -183,6 +188,8 @@ const startPath = (request: Request) => {
export const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<RunwayVideoOptions, Token>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
headers: { "X-Runway-Version": API_VERSION },
+58 -7
View File
@@ -1,5 +1,6 @@
import { Tool } from "@opencode/schema/tool"
import { Effect, Option, Schema } from "effect"
import { Effect, Option, Schema, Stream } from "effect"
import * as Sse from "effect/unstable/encoding/Sse"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { Media } from "../media.js"
import {
@@ -12,16 +13,17 @@ import {
ToolDefinition,
type ContentPart,
type MediaPart,
type OpenString,
type ProviderID,
type TextPart,
type ToolEntry,
type ToolResultPart,
} from "../schema/index.js"
import { Json, decodeJson, encodeJson } from "../utils/json.js"
import { isRecord } from "../utils/record.js"
export { Json, decodeJson, encodeJson, isRecord }
export { isRecord }
export const Json = Schema.fromJsonString(Schema.Unknown)
export const decodeJson = Schema.decodeUnknownSync(Json)
export const encodeJson = Schema.encodeSync(Json)
const isJson = Schema.is(Schema.Json)
export const JsonObject = Schema.Record(Schema.String, Schema.Unknown)
export const optionalArray = <const S extends Schema.Top>(schema: S) => Schema.optional(Schema.Array(schema))
@@ -33,7 +35,7 @@ export const lenient = <const S extends Schema.Top>(schema: S) =>
)
/** Provider-defined string enum: known values for autocomplete, any string accepted at runtime. */
export const knownString = <Known extends string>() =>
Schema.declare<OpenString<Known>>((value): value is OpenString<Known> => typeof value === "string", {
Schema.declare<Known | (string & {})>((value): value is Known | (string & {}) => typeof value === "string", {
expected: "string",
})
@@ -209,8 +211,7 @@ export const mediaReference = (
if (provider !== undefined && asset.source.type === "ref" && asset.source.provider === provider)
return Effect.succeed({ type: "ref", value: asset.source.id })
const accepted = provider === undefined ? "" : `, and ${provider} references`
const got = asset.source.type === "ref" ? `; got ${asset.source.provider}:${asset.source.id}` : ""
return Effect.fail(invalidRequest(`${label} accepts inline bytes, data URLs, http(s) URLs${accepted}${got}`))
return Effect.fail(invalidRequest(`${label} accepts inline bytes, data URLs, http(s) URLs${accepted}`))
}
/**
@@ -227,6 +228,8 @@ export const toolFileMedia = (item: Tool.FileContent): MediaPart => {
return Message.media(asset, { filename: item.name })
}
export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
export const toolResultText = (part: ToolResultPart) => {
if (part.result.type === "text") return String(part.result.value)
if (part.result.type === "error") {
@@ -248,6 +251,54 @@ export const errorText = (error: unknown) => {
return "Unknown stream error"
}
/**
* `framing` step for Server-Sent Events. Decodes UTF-8, runs the SSE channel
* decoder, optionally filters named events, and drops empty events and known
* keepalives that proxies send as data. `[DONE]` is dropped by default or
* retained for protocols that use it as their stream boundary. Retry control events are ignored without
* interrupting the stream. Decoder failures become provider output errors so
* the public error channel stays `AIError`.
*/
export const sseFraming = (
bytes: Stream.Stream<Uint8Array, AIError>,
events?: ReadonlySet<string>,
includeDone = false,
): Stream.Stream<string, AIError> =>
bytes.pipe(
Stream.decodeText(),
Stream.mapAccumEffect(
() => {
const output: Sse.Event[] = []
return {
output,
parser: Sse.makeParser((event) => {
if (event._tag === "Event") output.push(event)
}),
}
},
(state, chunk) =>
Effect.gen(function* () {
const error = state.parser.feed(chunk)
if (error) return yield* eventError("sse", error.message, chunk, error)
return [state, state.output.splice(0)] as const
}),
),
Stream.filter(
(event) =>
(events === undefined || events.has(event.event)) &&
event.data.length > 0 &&
// Some OpenAI-compatible proxies serialize an empty flush as a bare
// `data: null`, between events or after `[DONE]`. No protocol has a
// null event, so it carries nothing and must not abort the stream.
event.data !== "null" &&
// Vertex AI partner models (e.g. `xai/grok-4.6`) send their SSE
// keepalive comment as `data: : keepalive` while reasoning.
event.data !== ": keepalive" &&
(event.data !== "[DONE]" || includeDone || (events !== undefined && event.event !== "message")),
),
Stream.map((event) => event.data),
)
/**
* Canonical invalid-request constructor shared by protocol lowering.
*/
+42 -27
View File
@@ -3,12 +3,14 @@ import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "stability-images", name: "Stability AI", provider: "stability" })
const upscaleRoute = MediaProtocol.identity({ id: "stability-upscale", name: "Stability AI", provider: "stability" })
const ADAPTER = "stability-images"
const UPSCALE_ADAPTER = "stability-upscale"
const NAME = "Stability AI"
const PROVIDER = ProviderID.make("stability")
export const DEFAULT_BASE_URL = "https://api.stability.ai"
const RESULTS_PATH = "/v2beta/results"
const UPSCALE_MODEL = "creative"
@@ -19,7 +21,7 @@ const HEADERS = { accept: "application/json" }
// 1. Public model input
// ---------------------------------------------------------------------------
export type StabilityStylePreset = OpenString<
export type StabilityStylePreset =
| "enhance"
| "anime"
| "photographic"
@@ -37,7 +39,7 @@ export type StabilityStylePreset = OpenString<
| "3d-model"
| "pixel-art"
| "tile-texture"
>
| (string & {})
export type StabilityImageOptions = {
readonly negative_prompt?: string
@@ -82,31 +84,36 @@ const endpoint = (model: string) => (model.startsWith("sd3") ? "sd3" : model)
const RESERVED_FORM_FIELDS = new Set(["image", "prompt", "mode", "model"])
const unsupported = (route: string, operation: string, message: string) =>
ProviderShared.unsupportedOperation({ operation, provider: PROVIDER, route, message })
const form = Effect.fn("StabilityImages.form")(function* (
identity: MediaProtocol.Identity,
route: string,
fields: Record<string, unknown>,
native: Record<string, unknown> | undefined,
source: Request["images"],
) {
if ((source?.length ?? 0) > 1)
return yield* identity.unsupported("media.images", `${identity.name} takes one source image`)
if ((source?.length ?? 0) > 1) return yield* unsupported(route, "media.images", `${NAME} takes one source image`)
const body = new FormData()
MediaInput.appendFields(body, fields, { overlay: native, reserved: RESERVED_FORM_FIELDS })
const overlay = Object.entries(native ?? {}).filter(([key]) => !RESERVED_FORM_FIELDS.has(key))
Object.entries(mergeJsonRecords(fields, Object.fromEntries(overlay)) ?? {}).forEach(([key, value]) =>
body.append(key, typeof value === "string" ? value : ProviderShared.encodeJson(value)),
)
const image = source?.[0]
if (image !== undefined)
body.append("image", MediaInput.blob(yield* MediaInput.inlineBytes(identity.id, image), image.mediaType), "image")
body.append("image", MediaInput.blob(yield* MediaInput.inlineBytes(route, image), image.mediaType), "image")
return MediaProtocol.multipart(body)
})
const fromRequest = Effect.fn("StabilityImages.fromRequest")(function* (request: Request) {
if (request.n !== undefined && request.n > 1)
return yield* route.unsupported("media.n", `${route.name} generates one image per request; call it once per image`)
return yield* unsupported(ADAPTER, "media.n", `${NAME} generates one image per request; call it once per image`)
const target = endpoint(request.model.id)
const edit = (request.images?.length ?? 0) > 0
if (edit && target === "core")
return yield* route.unsupported("media.images", `${route.name} core is text-to-image only; use ultra or sd3.5-*`)
return yield* unsupported(ADAPTER, "media.images", `${NAME} core is text-to-image only; use ultra or sd3.5-*`)
return yield* form(
route,
ADAPTER,
{
prompt: request.prompt,
aspect_ratio: request.aspectRatio,
@@ -122,9 +129,9 @@ const fromRequest = Effect.fn("StabilityImages.fromRequest")(function* (request:
const fromUpscaleRequest = Effect.fn("StabilityImages.fromUpscaleRequest")(function* (request: UpscaleRequest) {
if ((request.images?.length ?? 0) === 0)
return yield* ProviderShared.invalidRequest(`${upscaleRoute.name} upscale requires the source image in images`)
return yield* ProviderShared.invalidRequest(`${NAME} upscale requires the source image in images`)
return yield* form(
upscaleRoute,
UPSCALE_ADAPTER,
{ prompt: request.prompt, seed: request.seed, output_format: request.format },
mergeJsonRecords(request.providerOptions, request.http?.body),
request.images,
@@ -135,29 +142,29 @@ const fromUpscaleRequest = Effect.fn("StabilityImages.fromUpscaleRequest")(funct
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeImageDocument = (identity: MediaProtocol.Identity) => {
const decode = identity.decodeJson(ImageDocument)
const decodeImageDocument = (route: string) => {
const decode = MediaProtocol.decodeJson(route, NAME, ImageDocument)
return Effect.fn("StabilityImages.decodeImage")(function* (response: HttpClientResponse.HttpClientResponse) {
const output = yield* decode(response)
const document = output.value
const data = document.image ?? document.result
if (data === undefined) return yield* output.invalid(`${identity.name} returned no image`)
const image = yield* MediaInput.decodedAsset(output.invalid, `${identity.name} result`, data, undefined)
if (data === undefined) return yield* output.invalid(`${NAME} returned no image`)
const image = yield* MediaInput.decodedAsset(output.invalid, `${NAME} result`, data, undefined)
return new ImageResponse({
images: [image],
notices:
document.finish_reason === "CONTENT_FILTERED"
? [{ type: "moderated", message: `${identity.name} blurred the image for violating its content policy` }]
? [{ type: "moderated", message: `${NAME} blurred the image for violating its content policy` }]
: undefined,
providerMetadata: { stability: { seed: document.seed, finishReason: document.finish_reason } },
})
})
}
const decodeResponse = decodeImageDocument(route)
const decodeUpscaleImage = decodeImageDocument(upscaleRoute)
const decodeResponse = decodeImageDocument(ADAPTER)
const decodeUpscaleImage = decodeImageDocument(UPSCALE_ADAPTER)
const decodeStart = upscaleRoute.decodeStarted(Started, (value) => ({
const decodeStart = MediaProtocol.decodeStarted(UPSCALE_ADAPTER, NAME, Started, (value) => ({
token: { id: value.id },
snapshot: { id: value.id, status: "queued" },
}))
@@ -174,8 +181,8 @@ const decodeUpscaleResult = Effect.fn("StabilityImages.decodeUpscaleResult")(fun
context: MediaProtocol.PollContext<Token>,
) {
if (response.status === 202) {
const output = yield* upscaleRoute.text(response)
return yield* output.invalid(`${upscaleRoute.name} upscale ${context.token.id} has not finished`)
const output = yield* MediaProtocol.text(UPSCALE_ADAPTER, NAME, response)
return yield* output.invalid(`${NAME} upscale ${context.token.id} has not finished`)
}
return yield* decodeUpscaleImage(response)
})
@@ -184,13 +191,17 @@ const decodeUpscaleResult = Effect.fn("StabilityImages.decodeUpscaleResult")(fun
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["size", "mask"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const upscaleProtocol = MediaProtocol.queued<UpscaleRequest, ImageResponse, Token>(upscaleRoute, {
export const upscaleProtocol = MediaProtocol.queued<UpscaleRequest, ImageResponse, Token>({
id: UPSCALE_ADAPTER,
name: NAME,
token: Token,
unsupported: ["n", "size", "aspectRatio", "mask"],
start: { body: { from: fromUpscaleRequest }, decode: decodeStart },
@@ -201,6 +212,8 @@ export const upscaleProtocol = MediaProtocol.queued<UpscaleRequest, ImageRespons
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<StabilityImageOptions>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
headers: HEADERS,
@@ -212,6 +225,8 @@ export const model = (input: MediaRoute.ModelInput) =>
export const upscaleModel = (input: Omit<MediaRoute.ModelInput, "id">) =>
ImageModel.fromRoute<StabilityUpscaleOptions, Token>(
{
id: UPSCALE_ADAPTER,
provider: PROVIDER,
protocol: upscaleProtocol,
baseURL: DEFAULT_BASE_URL,
headers: HEADERS,
+15 -14
View File
@@ -41,24 +41,25 @@ const STATUS = {
export const mediaUrl = (asset: Media.Asset, name: string) =>
ProviderShared.mediaReference(asset, undefined, name).pipe(Effect.map((reference) => reference.value))
export const protocol = <Request, Response>(
route: MediaProtocol.Identity,
input: {
readonly unsupported?: ReadonlyArray<keyof Request & string>
readonly from: (request: Request) => Effect.Effect<MediaProtocol.Body, AIError>
readonly decodeResult: (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) => Effect.Effect<Response, AIError>
},
) => {
const decodeQueueStatus = route.decodeJson(QueueStatus)
return MediaProtocol.queued<Request, Response, Token>(route, {
export const protocol = <Request, Response>(input: {
readonly id: string
readonly name: string
readonly unsupported?: ReadonlyArray<keyof Request & string>
readonly from: (request: Request) => Effect.Effect<MediaProtocol.Body, AIError>
readonly decodeResult: (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) => Effect.Effect<Response, AIError>
}) => {
const decodeQueueStatus = MediaProtocol.decodeJson(input.id, input.name, QueueStatus)
return MediaProtocol.queued<Request, Response, Token>({
id: input.id,
name: input.name,
token: Token,
unsupported: input.unsupported,
start: {
body: { from: input.from },
decode: route.decodeStarted(StartResponse, (value) => ({
decode: MediaProtocol.decodeStarted(input.id, input.name, StartResponse, (value) => ({
token: {
requestID: value.request_id,
statusURL: value.status_url,
@@ -1,77 +0,0 @@
import type { JsonSchema } from "../../schema/index.js"
import { isRecord } from "../../utils/record.js"
// Gemini's `parametersJsonSchema` accepts standard JSON Schema, but rejects a few shapes that
// published tool schemas commonly contain. Rewrite only those and send everything else unchanged.
const SCHEMA_MAPS = new Set([
"properties",
"patternProperties",
"$defs",
"definitions",
"dependentSchemas",
"dependencies",
])
const VALUES = new Set(["const", "default", "enum", "examples", "dependentRequired"])
const mapValues = (record: Record<string, unknown>, map: (value: unknown, key: string) => unknown) =>
Object.fromEntries(Object.entries(record).map(([key, value]) => [key, map(value, key)]))
const normalizeNode = (schema: unknown): unknown => {
if (Array.isArray(schema)) return schema.map(normalizeNode)
if (!isRecord(schema)) return schema
const properties = isRecord(schema.properties) ? schema.properties : undefined
return Object.fromEntries(
Object.entries(schema).flatMap(([key, value]) => {
if (VALUES.has(key)) return [[key, value]]
if (SCHEMA_MAPS.has(key) && isRecord(value)) return [[key, mapValues(value, normalizeNode)]]
// `required` may only name declared properties.
if (key === "required" && properties && Array.isArray(value))
return [[key, value.filter((name) => typeof name === "string" && Object.hasOwn(properties, name))]]
// Draft-04 boolean exclusive bounds become the numeric form.
if (key === "exclusiveMinimum" && typeof value === "boolean")
return value && typeof schema.minimum === "number" ? [[key, schema.minimum]] : []
if (key === "exclusiveMaximum" && typeof value === "boolean")
return value && typeof schema.maximum === "number" ? [[key, schema.maximum]] : []
if (key === "minimum" && schema.exclusiveMinimum === true) return []
if (key === "maximum" && schema.exclusiveMaximum === true) return []
// Draft-07 tuples (`items` array plus `additionalItems`) are `prefixItems` plus `items` in 2020-12.
if (key === "items" && Array.isArray(value)) return [["prefixItems", value.map(normalizeNode)]]
if (key === "additionalItems" && Array.isArray(schema.items)) return [["items", normalizeNode(value)]]
return [[key, normalizeNode(value)]]
}),
)
}
// Gemini accepts a recursive `$ref` only when the loop passes through an optional property or
// potentially empty array `items`. Replace other self-references with an unconstrained schema.
const cutLoops = (schema: unknown, target: string, safe: boolean): unknown => {
if (Array.isArray(schema)) return schema.map((item) => cutLoops(item, target, safe))
if (!isRecord(schema)) return schema
if (schema.$ref === target && !safe) return {}
const required = Array.isArray(schema.required) ? schema.required : []
return mapValues(schema, (value, key) => {
if (VALUES.has(key)) return value
if (key === "items") return cutLoops(value, target, safe || !(Number(schema.minItems) > 0))
if (key === "properties" && isRecord(value))
return mapValues(value, (child, name) => cutLoops(child, target, safe || !required.includes(name)))
if (SCHEMA_MAPS.has(key) && isRecord(value)) return mapValues(value, (child) => cutLoops(child, target, safe))
return cutLoops(value, target, safe)
})
}
export const normalize = (schema: JsonSchema): JsonSchema => {
const normalized = normalizeNode(schema)
if (!isRecord(normalized)) return {}
const result = cutLoops(
mapValues(normalized, (value, key) =>
(key === "$defs" || key === "definitions") && isRecord(value)
? mapValues(value, (def, name) => cutLoops(def, `#/${key}/${name}`, false))
: value,
),
"#",
false,
)
return isRecord(result) ? result : {}
}
export * as GeminiJsonSchema from "./gemini-json-schema.js"
@@ -0,0 +1,119 @@
import { isRecord } from "../../utils/record.js"
// Gemini accepts a JSON Schema-like dialect for tool parameters, but rejects a
// handful of common JSON Schema shapes. Keep this projection isolated so the
// Gemini protocol file still reads like the other protocol modules.
const SCHEMA_INTENT_KEYS = [
"type",
"properties",
"items",
"prefixItems",
"enum",
"const",
"$ref",
"additionalProperties",
"patternProperties",
"required",
"not",
"if",
"then",
"else",
]
const hasCombiner = (schema: unknown) =>
isRecord(schema) && (Array.isArray(schema.anyOf) || Array.isArray(schema.oneOf) || Array.isArray(schema.allOf))
const hasSchemaIntent = (schema: unknown) =>
isRecord(schema) && (hasCombiner(schema) || SCHEMA_INTENT_KEYS.some((key) => key in schema))
const sanitizeNode = (schema: unknown): unknown => {
if (!isRecord(schema)) return Array.isArray(schema) ? schema.map(sanitizeNode) : schema
const result: Record<string, unknown> = Object.fromEntries(
Object.entries(schema).map(([key, value]) => [
key,
key === "enum" && Array.isArray(value) ? value.map(String) : sanitizeNode(value),
]),
)
if (Array.isArray(result.enum) && (result.type === "integer" || result.type === "number")) result.type = "string"
const properties = result.properties
if (result.type === "object" && isRecord(properties) && Array.isArray(result.required)) {
result.required = result.required.filter((field) => typeof field === "string" && field in properties)
}
if (result.type === "array" && !hasCombiner(result)) {
result.items = result.items ?? {}
if (isRecord(result.items) && !hasSchemaIntent(result.items)) result.items = { ...result.items, type: "string" }
}
if (typeof result.type === "string" && result.type !== "object" && !hasCombiner(result)) {
delete result.properties
delete result.required
}
return result
}
const emptyObjectSchema = (schema: Record<string, unknown>) =>
schema.type === "object" &&
(!isRecord(schema.properties) || Object.keys(schema.properties).length === 0) &&
!schema.additionalProperties
const projectNode = (schema: unknown, nested = false): Record<string, unknown> | undefined => {
if (!isRecord(schema)) return undefined
if (!nested && emptyObjectSchema(schema)) return undefined
const types = Array.isArray(schema.type) ? schema.type.filter((type) => type !== "null") : undefined
const anyOf = Array.isArray(schema.anyOf) ? schema.anyOf : undefined
const hasNullAnyOf = anyOf?.some((item) => isRecord(item) && item.type === "null") ?? false
const anyOfTypes = hasNullAnyOf ? anyOf?.filter((item) => !isRecord(item) || item.type !== "null") : anyOf
const flattenedAnyOf = hasNullAnyOf && anyOfTypes?.length === 1 ? projectNode(anyOfTypes[0], true) : undefined
const result = Object.fromEntries(
[
["description", schema.description],
["required", schema.required],
["format", schema.format],
["type", types ? (types.length === 0 ? "null" : undefined) : schema.type],
[
"nullable",
(Array.isArray(schema.type) && schema.type.includes("null") && types && types.length > 0) || hasNullAnyOf
? true
: undefined,
],
["enum", schema.const !== undefined ? [schema.const] : schema.enum],
[
"properties",
isRecord(schema.properties)
? Object.fromEntries(Object.entries(schema.properties).map(([key, value]) => [key, projectNode(value, true)]))
: undefined,
],
[
"items",
Array.isArray(schema.items)
? schema.items.map((item) => projectNode(item, true))
: schema.items === undefined
? undefined
: projectNode(schema.items, true),
],
["allOf", Array.isArray(schema.allOf) ? schema.allOf.map((item) => projectNode(item, true)) : undefined],
[
"anyOf",
anyOfTypes
? hasNullAnyOf && anyOfTypes.length === 1
? undefined
: anyOfTypes.map((item) => projectNode(item, true))
: types && types.length > 0
? types.map((type) => ({ type }))
: undefined,
],
["oneOf", Array.isArray(schema.oneOf) ? schema.oneOf.map((item) => projectNode(item, true)) : undefined],
["minLength", schema.minLength],
].filter((entry) => entry[1] !== undefined),
)
return flattenedAnyOf ? { ...result, ...flattenedAnyOf } : result
}
export const convert = (schema: unknown) => projectNode(sanitizeNode(schema))
export * as GeminiToolSchema from "./gemini-tool-schema.js"
+1 -36
View File
@@ -1,8 +1,7 @@
import { Effect, Encoding } from "effect"
import { Media } from "../../media.js"
import type { MediaProtocol } from "../../route/media-protocol.js"
import { mergeJsonRecords, type AIError, type ProviderID } from "../../schema/index.js"
import { encodeJson } from "../../utils/json.js"
import type { AIError, ProviderID } from "../../schema/index.js"
import { ProviderShared } from "../shared.js"
/** Owned bytes for multipart uploads; decodes `base64` sources and rejects remote sources. */
@@ -57,38 +56,4 @@ export const decodedAsset = (
Effect.map((bytes) => Media.bytes(bytes, mediaType, options)),
)
/** One image of an OpenAI-shaped `data` array, which carries either `b64_json` or a `url`. */
export const imageOutput = (
invalid: (message: string, cause?: unknown) => AIError,
label: string,
item: { readonly b64_json?: string | null; readonly url?: string | null },
mediaType: string | undefined,
options?: Media.AssetOptions,
) => {
if (item.b64_json) return decodedAsset(invalid, label, item.b64_json, mediaType, options)
if (item.url) return Effect.succeed(Media.url(item.url, { ...options, mediaType }))
return Effect.fail(invalid(`${label} has neither image data nor a URL`))
}
/**
* Append multipart text fields: strings as-is, other values as JSON, or arrays as repeated `key[]` parts with
* `repeatArrays`. `overlay` keys in `reserved` are dropped so `http.body` cannot replace route-owned fields.
*/
export const appendFields = (
form: FormData,
fields: Record<string, unknown>,
options: {
readonly overlay?: Record<string, unknown>
readonly reserved: ReadonlySet<string>
readonly repeatArrays?: true
},
) => {
const overlay = Object.entries(options.overlay ?? {}).filter(([key]) => !options.reserved.has(key))
Object.entries(mergeJsonRecords(fields, Object.fromEntries(overlay)) ?? {}).forEach(([key, value]) => {
if (Array.isArray(value) && options.repeatArrays)
return value.forEach((item) => form.append(`${key}[]`, String(item)))
form.append(key, typeof value === "string" ? value : encodeJson(value))
})
}
export * as MediaInput from "./media-input.js"
@@ -0,0 +1,11 @@
// Responses image items can omit output_format, including when PNG/JPEG was requested.
export const mediaType = (data: Uint8Array, format?: string) => {
if (format !== undefined) return `image/${format}`
if (data[0] === 137 && data[1] === 80 && data[2] === 78 && data[3] === 71) return "image/png"
if (data[0] === 255 && data[1] === 216 && data[2] === 255) return "image/jpeg"
if (new TextDecoder().decode(data.slice(0, 4)) === "RIFF" && new TextDecoder().decode(data.slice(8, 12)) === "WEBP")
return "image/webp"
return "application/octet-stream"
}
export * as MetaImage from "./meta-image.js"
@@ -0,0 +1,20 @@
import { Schema } from "effect"
const dimensions = (value: string) => {
const match = /^(\d+)x(\d+)$/.exec(value)
if (!match) return undefined
return { width: Number(match[1]), height: Number(match[2]) }
}
export const Size = Schema.String.check(
Schema.makeFilter((value) => {
if (value === "auto") return undefined
const parsed = dimensions(value)
if (!parsed) return "image size must be `auto` or `{width}x{height}`"
return parsed.width > 0 && parsed.height > 0 ? undefined : "image dimensions must be positive integers"
}),
)
export const OpenAIImage = {
Size,
} as const
@@ -1,7 +1,7 @@
import { Effect } from "effect"
import { Media } from "../../media.js"
import type { MediaProtocol } from "../../route/media-protocol.js"
import type { AIError, MediaUsage, ProviderMetadata } from "../../schema/index.js"
import { MediaProtocol } from "../../route/media-protocol.js"
import type { AIError, MediaUsage, ProviderID, ProviderMetadata } from "../../schema/index.js"
import {
SpeechAudioDeltaEvent,
SpeechFinishEvent,
@@ -10,6 +10,7 @@ import {
type SpeechVoice,
} from "../../speech.js"
import { concatBytes } from "../../utils/bytes.js"
import { ProviderShared } from "../shared.js"
export interface Audio {
/** Appended in place: the route creates fresh state for each response through `initial`. */
@@ -77,9 +78,12 @@ export const sampleRate = (mediaType: string | undefined) => {
return rate === undefined ? undefined : Number(rate)
}
export const unsupportedFormat = (provider: ProviderID, route: string, message: string) =>
ProviderShared.unsupportedOperation({ operation: "media.format", provider, route, message })
/** A declared `mediaType` wins over sniffing: headerless PCM can start with bytes that look like an MPEG frame sync. */
export const finish = (
route: MediaProtocol.Identity,
route: string,
state: Audio,
output: {
readonly mediaType: string | undefined
@@ -91,7 +95,10 @@ export const finish = (
): Effect.Effect<ReadonlyArray<SpeechEvent>, AIError> => {
if (state.chunks.length === 0)
return Effect.fail(
route.frameError(`The provider returned no audio${output.detail === undefined ? "" : ` (${output.detail})`}`),
MediaProtocol.frameError(
route,
`The provider returned no audio${output.detail === undefined ? "" : ` (${output.detail})`}`,
),
)
return Effect.succeed([
SpeechFinishEvent.make({
+8 -34
View File
@@ -1,6 +1,6 @@
import type { JsonSchema, LanguageModel, LanguageModelSanitizerCompatibility } from "../../schema/index.js"
import type { JsonSchema, LanguageModelToolSchemaCompatibility } from "../../schema/index.js"
import { isRecord } from "../../utils/record.js"
import { GeminiJsonSchema } from "./gemini-json-schema.js"
import { GeminiToolSchema } from "./gemini-tool-schema.js"
const tupleItemsSchema = (items: ReadonlyArray<unknown>) => {
const projected = items.map(moonshotNode)
@@ -46,44 +46,18 @@ const moonshot = (schema: JsonSchema): JsonSchema => {
const openAI = (schema: JsonSchema): JsonSchema => schema
const responses = openAI
const gemini = GeminiJsonSchema.normalize
const gemini = (schema: JsonSchema): JsonSchema => GeminiToolSchema.convert(schema) ?? {}
const MODEL_NAMES = [
[/gemini/i, "gemini"],
[/kimi/i, "moonshot"],
] as const
// Tool arguments are always a JSON object, and most providers reject a tool schema whose root does not
// declare `type: "object"`, such as `{}` or a bare `properties` map. Effect encodes an empty struct as
// `anyOf` object or array; every object matches its bare object branch, so that `anyOf` is dropped.
const objectRoot = (schema: JsonSchema): JsonSchema => {
if (schema.type !== undefined) return schema
if (
Array.isArray(schema.anyOf) &&
schema.anyOf.some((branch) => isRecord(branch) && branch.type === "object" && Object.keys(branch).length === 1)
)
return { type: "object", ...Object.fromEntries(Object.entries(schema).filter(([key]) => key !== "anyOf")) }
return { type: "object", ...schema }
}
// Every tool schema gets an object root. Then an explicit `sanitizer` wins, and `none` opts out.
// Otherwise the protocol's own default applies (the Gemini API always uses Gemini's rules), then the
// model name selects the family's rules so models reached through gateways and OpenAI-compatible
// endpoints get the same handling.
const modelCompatibility = (
schema: JsonSchema,
model: LanguageModel,
protocolDefault?: LanguageModelSanitizerCompatibility,
compatibility: LanguageModelToolSchemaCompatibility | undefined,
): JsonSchema => {
const root = objectRoot(schema)
switch (model.compatibility?.sanitizer ?? protocolDefault ?? MODEL_NAMES.find(([name]) => name.test(model.id))?.[1]) {
if (compatibility === undefined) return schema
switch (compatibility) {
case "gemini":
return gemini(root)
return gemini(schema)
case "moonshot":
return moonshot(root)
case "none":
case undefined:
return root
return moonshot(schema)
}
}
+41 -20
View File
@@ -4,11 +4,13 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "xai-images", name: "xAI Images", provider: "xai" })
const ADAPTER = "xai-images"
const NAME = "xAI Images"
const PROVIDER = ProviderID.make("xai")
export const DEFAULT_BASE_URL = "https://api.x.ai/v1"
export const PATH = "/images/generations"
export const EDIT_PATH = "/images/edits"
@@ -17,11 +19,13 @@ export const EDIT_PATH = "/images/edits"
// 1. Public model input
// ---------------------------------------------------------------------------
export type XAIImageString<Known extends string> = Known | (string & {})
/** Provider-native options. Common fields (`n`, `aspectRatio`, `images`) live on the request. */
export type XAIImageOptions = {
readonly resolution?: OpenString<"1k" | "2k">
readonly responseFormat?: OpenString<"url" | "b64_json">
readonly response_format?: OpenString<"url" | "b64_json">
readonly resolution?: XAIImageString<"1k" | "2k">
readonly responseFormat?: XAIImageString<"url" | "b64_json">
readonly response_format?: XAIImageString<"url" | "b64_json">
} & Record<string, unknown>
export type Request = ImageRequestFor<XAIImageOptions>
@@ -55,7 +59,7 @@ const nativeOptions = (options: XAIImageOptions | undefined) => {
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
const reference = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(
ProviderShared.mediaReference(asset, PROVIDER, NAME).pipe(
Effect.map((item) =>
item.type === "ref" ? { file_id: item.value } : { url: item.value, type: "image_url" as const },
),
@@ -84,22 +88,31 @@ const fromRequest = Effect.fn("XAIImages.fromRequest")(function* (request: Reque
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeDocument = route.decodeJson(XAIImageResponse)
const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* decodeDocument(response)
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, XAIImageResponse)(response)
const decoded = output.value
const images = yield* Effect.forEach(decoded.data, (item, index) =>
MediaInput.imageOutput(output.invalid, `${route.name} result ${index}`, item, item.mime_type ?? undefined, {
providerMetadata:
item.revised_prompt === undefined || item.revised_prompt === null
? undefined
: { xai: { revisedPrompt: item.revised_prompt } },
}),
)
if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`)
const images = yield* Effect.forEach(decoded.data, (item, index) => {
const providerMetadata =
item.revised_prompt === undefined || item.revised_prompt === null
? undefined
: { xai: { revisedPrompt: item.revised_prompt } }
if (item.b64_json)
return MediaInput.decodedAsset(
output.invalid,
`${NAME} result ${index}`,
item.b64_json,
item.mime_type ?? undefined,
{
providerMetadata,
},
)
if (item.url)
return Effect.succeed(Media.url(item.url, { mediaType: item.mime_type ?? undefined, providerMetadata }))
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
})
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
const usage = ProviderShared.isRecord(decoded.usage) ? decoded.usage : undefined
// xAI reports image counts rather than tokens, seconds, or credits; the raw record stays in provider metadata.
return new ImageResponse({
@@ -112,7 +125,9 @@ const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["mask", "size", "seed", "format"],
body: { from: fromRequest },
response: { decode: decodeResponse },
@@ -120,7 +135,13 @@ export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<XAIImageOptions>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH) },
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
},
input,
)
+18 -13
View File
@@ -4,11 +4,13 @@ import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
import { ProviderShared, optionalNull } from "./shared.js"
const route = MediaProtocol.identity({ id: "xai-video", name: "xAI Video", provider: "xai" })
const ADAPTER = "xai-video"
const NAME = "xAI Video"
const PROVIDER = ProviderID.make("xai")
export const DEFAULT_BASE_URL = "https://api.x.ai/v1"
export const PATH = "/videos/generations"
export const EDIT_PATH = "/videos/edits"
@@ -70,7 +72,7 @@ const STATUS = {
// ---------------------------------------------------------------------------
const mediaInput = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(
ProviderShared.mediaReference(asset, PROVIDER, NAME).pipe(
Effect.map((reference) => (reference.type === "ref" ? { file_id: reference.value } : { url: reference.value })),
)
@@ -109,7 +111,7 @@ const fromRequest = Effect.fn("XAIVideo.fromRequest")(function* (request: Reques
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
const decodeStart = MediaProtocol.decodeStarted(ADAPTER, NAME, StartResponse, (value) => ({
token: { requestID: value.request_id },
snapshot: { id: value.request_id, status: "running" },
}))
@@ -118,7 +120,7 @@ const decodeStart = route.decodeStarted(StartResponse, (value) => ({
const fraction = (progress: number | null | undefined) =>
progress !== undefined && progress !== null && progress >= 0 && progress <= 100 ? progress / 100 : undefined
const decodeVideoStatus = route.decodeJson(VideoStatus)
const decodeVideoStatus = MediaProtocol.decodeJson(ADAPTER, NAME, VideoStatus)
const decodeStatus = Effect.fn("XAIVideo.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
@@ -136,27 +138,26 @@ const decodeResult = Effect.fn("XAIVideo.decodeResult")(function* (
const output = yield* decodeVideoStatus(response)
const decoded = output.value
const status = yield* MediaProtocol.status(STATUS, decoded.status, output)
if (status === "running")
return yield* output.invalid(`${route.name} request ${context.token.requestID} has not finished`)
if (status === "running") return yield* output.invalid(`${NAME} request ${context.token.requestID} has not finished`)
if (status === "failed") {
const code = decoded.error?.code ?? undefined
const message = decoded.error?.message ?? undefined
return yield* output.ended(
"failed",
`${route.name} generation failed${code === undefined ? "" : ` (${code})`}${message === undefined ? "" : `: ${message}`}`,
`${NAME} generation failed${code === undefined ? "" : ` (${code})`}${message === undefined ? "" : `: ${message}`}`,
)
}
if (status !== "completed")
return yield* output.ended("expired", `${route.name} request ${context.token.requestID} expired`)
return yield* output.ended("expired", `${NAME} request ${context.token.requestID} expired`)
// `respect_moderation: false` marks a filtered result; a URL may still be present, so report it as a notice.
const notices =
decoded.video?.respect_moderation === false
? [{ type: "moderated" as const, message: `${route.name} flagged the generated video for moderation` }]
? [{ type: "moderated" as const, message: `${NAME} flagged the generated video for moderation` }]
: undefined
const url = decoded.video?.url ?? undefined
if (url === undefined && notices !== undefined)
return yield* output.contentPolicy(`${route.name} withheld the video for moderation`)
if (url === undefined) return yield* output.invalid(`${route.name} completed without a video URL`)
return yield* output.contentPolicy(`${NAME} withheld the video for moderation`)
if (url === undefined) return yield* output.invalid(`${NAME} completed without a video URL`)
const duration = decoded.video?.duration ?? undefined
return new VideoResponse({
videos: [
@@ -176,7 +177,9 @@ const decodeResult = Effect.fn("XAIVideo.decodeResult")(function* (
const statusPath = (token: Token) => `${STATUS_PATH}/${token.requestID}`
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>(route, {
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>({
id: ADAPTER,
name: NAME,
token: Token,
unsupported: ["n", "seed", "negativePrompt"],
start: { body: { from: fromRequest }, decode: decodeStart },
@@ -193,6 +196,8 @@ const startPath = (request: Request) => {
export const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<XAIVideoOptions, Token>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => startPath(request),
+17 -10
View File
@@ -4,9 +4,11 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
const route = MediaProtocol.identity({ id: "zai-images", name: "Z.ai Images", provider: "zai" })
const ADAPTER = "zai-images"
const NAME = "Z.ai Images"
const PROVIDER = ProviderID.make("zai")
export const DEFAULT_BASE_URL = "https://api.z.ai/api/paas/v4"
export const PATH = "/images/generations"
@@ -14,9 +16,11 @@ export const PATH = "/images/generations"
// 1. Public model input
// ---------------------------------------------------------------------------
export type ZAIImageString<Known extends string> = Known | (string & {})
/** Provider-native options. The common `size` field lives on the request. */
export type ZAIImageOptions = {
readonly quality?: OpenString<"hd" | "standard">
readonly quality?: ZAIImageString<"hd" | "standard">
readonly userID?: string
} & Record<string, unknown>
@@ -65,14 +69,12 @@ const fromRequest = Effect.fn("ZAIImages.fromRequest")(function* (request: Reque
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeDocument = route.decodeJson(ZAIImageResponse)
const decodeResponse = Effect.fn("ZAIImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* decodeDocument(response)
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, ZAIImageResponse)(response)
const decoded = output.value
if (decoded.data.length === 0) return yield* output.invalid(`${route.name} returned no images`)
if (decoded.data.length === 0) return yield* output.invalid(`${NAME} returned no images`)
const filters = decoded.content_filter ?? []
return new ImageResponse({
// Z.ai returns only URLs and no content type; the media type resolves when the asset is materialized.
@@ -83,7 +85,7 @@ const decodeResponse = Effect.fn("ZAIImages.decodeResponse")(function* (
? undefined
: filters.map((filter) => ({
type: "moderated" as const,
message: `${route.name} applied a content filter${filter.role === undefined ? "" : ` for ${filter.role}`}${
message: `${NAME} applied a content filter${filter.role === undefined ? "" : ` for ${filter.role}`}${
filter.level === undefined ? "" : ` at level ${filter.level}`
}`,
providerMetadata: { zai: filter },
@@ -103,14 +105,19 @@ const decodeResponse = Effect.fn("ZAIImages.decodeResponse")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["images", "mask", "n", "aspectRatio", "seed", "format"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<ZAIImageOptions>({ protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, input)
ImageModel.fromRoute<ZAIImageOptions>(
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
export const ZAIImages = {
protocol,
+11 -5
View File
@@ -1,12 +1,12 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { AssemblyAITranscription } from "../protocols/assemblyai-transcription.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { AssemblyAITranscription, DEFAULT_BASE_URL } from "../protocols/assemblyai-transcription.js"
export type { AssemblyAITranscriptionOptions } from "../protocols/assemblyai-transcription.js"
export const id = ProviderID.make("assemblyai")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
/** `https://api.eu.assemblyai.com` for the EU region. */
@@ -24,8 +24,14 @@ const auth = (options: ProviderAuthOption<"optional">) => {
}
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const transcription = (modelID: string | ModelID) => AssemblyAITranscription.model({ ...media, id: modelID })
const transcription = (modelID: string | ModelID) =>
AssemblyAITranscription.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
transcription,
+2 -2
View File
@@ -1,11 +1,11 @@
import { Auth } from "../route/auth.js"
import { type AtLeastOne, type ProviderAuthOption } from "../route/auth-options.js"
import type { Route, RouteDefaultsInput, CompactionOperations } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import type { ProviderPackage } from "../provider-package.js"
import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js"
import * as OpenAIChat from "../protocols/openai-chat.js"
import * as OpenAIResponses from "../protocols/openai-responses.js"
import { ProviderShared } from "../protocols/shared.js"
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options.js"
export const id = ProviderID.make("azure")
@@ -108,7 +108,7 @@ const configuredRoute = <Body, Prepared, Compact extends CompactionOperations |
})
function endpoint(input: Config, modelID: string | ModelID) {
const baseURL = Endpoint.trimBaseUrl(input.baseURL ?? resourceBaseURL(input.resourceName!))
const baseURL = ProviderShared.trimBaseUrl(input.baseURL ?? resourceBaseURL(input.resourceName!))
const query = { "api-version": input.apiVersion ?? "v1", ...input.queryParams }
if (input.useDeploymentBasedUrls) return { baseURL: `${baseURL}/deployments/${modelID}`, query }
+11 -5
View File
@@ -1,12 +1,12 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { BlackForestLabsImages } from "../protocols/bfl-images.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { BlackForestLabsImages, DEFAULT_BASE_URL } from "../protocols/bfl-images.js"
export type { BlackForestLabsImageOptions } from "../protocols/bfl-images.js"
export const id = ProviderID.make("black-forest-labs")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
/** `https://api.eu.bfl.ai` or `https://api.us.bfl.ai` pin inference to one region. */
@@ -23,8 +23,14 @@ const auth = (options: ProviderAuthOption<"optional">) => {
}
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const image = (modelID: string | ModelID) => BlackForestLabsImages.model({ ...media, id: modelID })
const image = (modelID: string | ModelID) =>
BlackForestLabsImages.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
image,
+11 -5
View File
@@ -1,11 +1,11 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { CartesiaSpeech } from "../protocols/cartesia-speech.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { CartesiaSpeech, DEFAULT_BASE_URL } from "../protocols/cartesia-speech.js"
export type { CartesiaEncoding, CartesiaSpeechOptions } from "../protocols/cartesia-speech.js"
export const id = ProviderID.make("cartesia")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
@@ -16,8 +16,14 @@ export type Config = ProviderAuthOption<"optional"> & {
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "CARTESIA_API_KEY")
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const speech = (modelID: string | ModelID) => CartesiaSpeech.model({ ...media, id: modelID })
const speech = (modelID: string | ModelID) =>
CartesiaSpeech.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
speech,
+12 -6
View File
@@ -1,14 +1,14 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { DeepgramSpeech } from "../protocols/deepgram-speech.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { DEFAULT_BASE_URL, DeepgramSpeech } from "../protocols/deepgram-speech.js"
import { DeepgramTranscription } from "../protocols/deepgram-transcription.js"
export type { DeepgramEncoding, DeepgramSpeechOptions } from "../protocols/deepgram-speech.js"
export type { DeepgramTranscriptionOptions } from "../protocols/deepgram-transcription.js"
export const id = ProviderID.make("deepgram")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
@@ -24,11 +24,17 @@ const auth = (options: ProviderAuthOption<"optional">) => {
}
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const media = (modelID: string | ModelID) => ({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
speech: (modelID: string | ModelID) => DeepgramSpeech.model({ ...media, id: modelID }),
transcription: (modelID: string | ModelID) => DeepgramTranscription.model({ ...media, id: modelID }),
speech: (modelID: string | ModelID) => DeepgramSpeech.model(media(modelID)),
transcription: (modelID: string | ModelID) => DeepgramTranscription.model(media(modelID)),
configure,
}
}
+11 -5
View File
@@ -1,12 +1,12 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { ElevenLabsSpeech } from "../protocols/elevenlabs-speech.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { DEFAULT_BASE_URL, ElevenLabsSpeech } from "../protocols/elevenlabs-speech.js"
export type { ElevenLabsOutputFormat, ElevenLabsSpeechOptions } from "../protocols/elevenlabs-speech.js"
export const id = ProviderID.make("elevenlabs")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
@@ -22,8 +22,14 @@ const auth = (options: ProviderAuthOption<"optional">) => {
}
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const speech = (modelID: string | ModelID) => ElevenLabsSpeech.model({ ...media, id: modelID })
const speech = (modelID: string | ModelID) =>
ElevenLabsSpeech.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
speech,
+12 -5
View File
@@ -1,14 +1,15 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { FalImages } from "../protocols/fal-images.js"
import { FalVideo } from "../protocols/fal-video.js"
import { FalQueue } from "../protocols/utils/fal-queue.js"
export type { FalImageOptions } from "../protocols/fal-images.js"
export type { FalVideoOptions } from "../protocols/fal-video.js"
export const id = ProviderID.make("fal")
const baseURL = FalQueue.DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
@@ -25,11 +26,17 @@ const auth = (options: ProviderAuthOption<"optional">) => {
}
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const media = (modelID: string | ModelID) => ({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
image: (modelID: string | ModelID) => FalImages.model({ ...media, id: modelID }),
video: (modelID: string | ModelID) => FalVideo.model({ ...media, id: modelID }),
image: (modelID: string | ModelID) => FalImages.model(media(modelID)),
video: (modelID: string | ModelID) => FalVideo.model(media(modelID)),
configure,
}
}
+12 -7
View File
@@ -1,9 +1,8 @@
import type { RouteDefaultsInput } from "../route/client.js"
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import type { ProviderPackage } from "../provider-package.js"
import { ProviderID, type ModelID } from "../schema/index.js"
import { HttpOptions, ProviderID, mergeHttpOptions, type ModelID } from "../schema/index.js"
import { Gemini } from "../protocols/gemini.js"
import { GoogleImages } from "../protocols/google-images.js"
import { GoogleSpeech } from "../protocols/google-speech.js"
@@ -47,14 +46,20 @@ const configuredRoute = (input: Config) => {
export const configure = (input: Config = {}) => {
const route = configuredRoute(input)
const media = MediaRoute.deployment(input, auth(input))
const media = (modelID: string | ModelID) => ({
id: modelID,
auth: auth(input),
baseURL: input.baseURL,
headers: input.headers,
http: mergeHttpOptions(input.http === undefined ? undefined : HttpOptions.make(input.http)),
})
return {
id,
model: (modelID: string | ModelID) => route.model<Gemini.ProviderOptionsInput>({ id: modelID }),
image: (modelID: string | ModelID) => GoogleImages.model({ ...media, id: modelID }),
video: (modelID: string | ModelID) => GoogleVideo.model({ ...media, id: modelID }),
speech: (modelID: string | ModelID) => GoogleSpeech.model({ ...media, id: modelID }),
transcription: (modelID: string | ModelID) => GoogleTranscription.model({ ...media, id: modelID }),
image: (modelID: string | ModelID) => GoogleImages.model(media(modelID)),
video: (modelID: string | ModelID) => GoogleVideo.model(media(modelID)),
speech: (modelID: string | ModelID) => GoogleSpeech.model(media(modelID)),
transcription: (modelID: string | ModelID) => GoogleTranscription.model(media(modelID)),
configure,
}
}
+11 -6
View File
@@ -7,8 +7,7 @@ import { MetaImages } from "../protocols/meta-images.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, ToolDefinition, type ModelID, type OpenString } from "../schema/index.js"
import { HttpOptions, ProviderID, ToolDefinition, type ModelID } from "../schema/index.js"
import type { OpenResponsesProviderOptionsInput } from "./open-responses-options.js"
export const id = ProviderID.make("meta")
@@ -49,8 +48,8 @@ export const webSearch = (options: WebSearchOptions = {}) =>
export interface ImageGenerationOptions {
readonly size?: string
readonly outputFormat?: OpenString<"webp" | "png" | "jpeg">
readonly reasoningStrength?: OpenString<"low" | "high">
readonly outputFormat?: "webp" | "png" | "jpeg" | (string & {})
readonly reasoningStrength?: "low" | "high" | (string & {})
readonly enableImageSearch?: boolean
readonly enableWebSearch?: boolean
readonly enableShell?: boolean
@@ -140,8 +139,14 @@ export const configure = (input: LanguageModelOptions = {}) => {
id: modelID,
compatibility: { requireSignature: false },
})
const media = MediaRoute.deployment(input, options.auth)
const image = (modelID: string | ModelID) => MetaImages.model({ ...media, id: modelID })
const image = (modelID: string | ModelID) =>
MetaImages.model({
id: modelID,
baseURL: endpoint ?? baseURL,
auth: options.auth,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return { id, model: responses, responses, chat, messages, image, configure }
}
+3 -3
View File
@@ -113,19 +113,19 @@ export const configure = (input: Config = {}) => {
supportsStore: false,
supportsStrictMode: false,
supportsPromptCacheKey: true,
sanitizer: "moonshot",
toolSchema: "moonshot",
reasoningField: "reasoning_content",
},
})
const messages = (modelID: string | ModelID) =>
messagesRoute.with(defaults).model<MessagesOptionsInput>({
id: modelID,
compatibility: { requireSignature: false, sanitizer: "moonshot" },
compatibility: { requireSignature: false, toolSchema: "moonshot" },
})
const responses = (modelID: string | ModelID) =>
responsesRoute
.with(defaults)
.model<ResponsesOptionsInput>({ id: modelID, compatibility: { sanitizer: "moonshot" } })
.model<ResponsesOptionsInput>({ id: modelID, compatibility: { toolSchema: "moonshot" } })
return { id, model: chat, chat, messages, responses, configure }
}
+18 -24
View File
@@ -1,19 +1,11 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import type { Route, RouteDefaultsInput, CompactionOperations } from "../route/client.js"
import { MediaRoute } from "../route/media.js"
import type { ProviderPackage } from "../provider-package.js"
import {
HttpOptions,
ProviderID,
ToolDefinition,
mergeHttpOptions,
type ModelID,
type OpenString,
} from "../schema/index.js"
import { HttpOptions, ProviderID, ToolDefinition, mergeHttpOptions, type ModelID } from "../schema/index.js"
import * as OpenAIChat from "../protocols/openai-chat.js"
import * as OpenAIResponses from "../protocols/openai-responses.js"
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options.js"
import { OpenAIImages } from "../protocols/openai-images.js"
import { OpenAIImages, type OpenAIImageString } from "../protocols/openai-images.js"
import { OpenAISpeech } from "../protocols/openai-speech.js"
import { OpenAITranscription } from "../protocols/openai-transcription.js"
@@ -37,14 +29,14 @@ export type Config = RouteDefaultsInput &
}
export interface ImageGenerationOptions {
readonly action?: OpenString<"auto" | "generate" | "edit">
readonly background?: OpenString<"auto" | "opaque" | "transparent">
readonly inputFidelity?: OpenString<"low" | "high">
readonly action?: OpenAIImageString<"auto" | "generate" | "edit">
readonly background?: OpenAIImageString<"auto" | "opaque" | "transparent">
readonly inputFidelity?: OpenAIImageString<"low" | "high">
readonly outputCompression?: number
readonly outputFormat?: OpenString<"png" | "jpeg" | "webp">
readonly outputFormat?: OpenAIImageString<"png" | "jpeg" | "webp">
readonly partialImages?: number
readonly quality?: OpenString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
readonly size?: OpenString<
readonly quality?: OpenAIImageString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
readonly size?: OpenAIImageString<
"auto" | "256x256" | "512x512" | "1024x1024" | "1536x1024" | "1024x1536" | "1792x1024" | "1024x1792"
>
}
@@ -107,17 +99,19 @@ export const configure = (input: Config = {}) => {
id,
compatibility: { supportsPromptCacheKey: true },
})
const deployment = MediaRoute.deployment(input, auth(input))
const media = {
...deployment,
const media = (modelID: string | ModelID) => ({
id: modelID,
auth: auth(input),
baseURL: input.baseURL,
headers: input.headers,
http: mergeHttpOptions(
deployment.http,
input.http === undefined ? undefined : HttpOptions.make(input.http),
input.queryParams === undefined ? undefined : new HttpOptions({ query: input.queryParams }),
),
}
const image = (modelID: string | ModelID) => OpenAIImages.model({ ...media, id: modelID })
const speech = (modelID: string | ModelID) => OpenAISpeech.model({ ...media, id: modelID })
const transcription = (modelID: string | ModelID) => OpenAITranscription.model({ ...media, id: modelID })
})
const image = (modelID: string | ModelID) => OpenAIImages.model(media(modelID))
const speech = (modelID: string | ModelID) => OpenAISpeech.model(media(modelID))
const transcription = (modelID: string | ModelID) => OpenAITranscription.model(media(modelID))
return {
id,
+1 -1
View File
@@ -20,7 +20,7 @@ export const configure = (input: Options = {}) => {
auth: AuthOptions.bearer(input, "OPENCODE_API_KEY"),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: HttpOptions.make(input.http),
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return { id, experimental: { evaluation }, configure }
}
+9 -7
View File
@@ -3,7 +3,7 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { Protocol } from "../route/protocol.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { HttpOptions, ProviderID, type CacheHint, type ModelID, type OpenString } from "../schema/index.js"
import { HttpOptions, ProviderID, type CacheHint, type ModelID } from "../schema/index.js"
import type { ProviderPackage } from "../provider-package.js"
import { SystemOne } from "../experimental/system-one.js"
import { OpenAIChat } from "../protocols/openai-chat.js"
@@ -14,16 +14,18 @@ export const id = ProviderID.make("openrouter")
const baseURL = "https://openrouter.ai/api/v1"
const ADAPTER = "openrouter"
type OpenRouterString<Known extends string> = Known | (string & {})
export interface OpenRouterProviderRouting {
readonly [key: string]: unknown
readonly order?: ReadonlyArray<string>
readonly allow_fallbacks?: boolean
readonly require_parameters?: boolean
readonly data_collection?: OpenString<"allow" | "deny">
readonly data_collection?: OpenRouterString<"allow" | "deny">
readonly only?: ReadonlyArray<string>
readonly ignore?: ReadonlyArray<string>
readonly quantizations?: ReadonlyArray<string>
readonly sort?: OpenString<"price" | "throughput" | "latency">
readonly sort?: OpenRouterString<"price" | "throughput" | "latency">
readonly max_price?: Readonly<{
prompt?: number | string
completion?: number | string
@@ -39,7 +41,7 @@ export type OpenRouterPlugin =
id: "web"
max_results?: number
search_prompt?: string
engine?: OpenString<"native" | "exa">
engine?: OpenRouterString<"native" | "exa">
}>
| Readonly<{ id: "file-parser"; max_files?: number; pdf?: { engine?: string } }>
| Readonly<{ id: "moderation" }>
@@ -56,7 +58,7 @@ export interface OpenRouterOptions {
readonly reasoning?: Readonly<{
enabled?: boolean
exclude?: boolean
effort?: OpenString<"none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max">
effort?: OpenRouterString<"none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max">
max_tokens?: number
}>
readonly usage?: boolean | Readonly<{ include: boolean }>
@@ -64,7 +66,7 @@ export interface OpenRouterOptions {
readonly web_search_options?: Readonly<{
max_results?: number
search_prompt?: string
engine?: OpenString<"native" | "exa">
engine?: OpenRouterString<"native" | "exa">
}>
}
@@ -196,7 +198,7 @@ export const configure = (input: LanguageModelOptions = {}) => {
auth: AuthOptions.bearer(input, "OPENROUTER_API_KEY"),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: HttpOptions.make(input.http),
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
+11 -5
View File
@@ -1,11 +1,11 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { ReplicateImages } from "../protocols/replicate-images.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { DEFAULT_BASE_URL, ReplicateImages } from "../protocols/replicate-images.js"
export type { ReplicateImageOptions } from "../protocols/replicate-images.js"
export const id = ProviderID.make("replicate")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
@@ -17,8 +17,14 @@ export type Config = ProviderAuthOption<"optional"> & {
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "REPLICATE_API_TOKEN")
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const image = (modelID: string | ModelID) => ReplicateImages.model({ ...media, id: modelID })
const image = (modelID: string | ModelID) =>
ReplicateImages.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
image,
+11 -5
View File
@@ -1,11 +1,11 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { RunwayVideo } from "../protocols/runway-video.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { DEFAULT_BASE_URL, RunwayVideo } from "../protocols/runway-video.js"
export type { RunwayVideoOptions } from "../protocols/runway-video.js"
export const id = ProviderID.make("runway")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
@@ -16,8 +16,14 @@ export type Config = ProviderAuthOption<"optional"> & {
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "RUNWAYML_API_SECRET")
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const video = (modelID: string | ModelID) => RunwayVideo.model({ ...media, id: modelID })
const video = (modelID: string | ModelID) =>
RunwayVideo.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
video,
+11 -6
View File
@@ -1,11 +1,11 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { StabilityImages } from "../protocols/stability-images.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { DEFAULT_BASE_URL, StabilityImages } from "../protocols/stability-images.js"
export type { StabilityImageOptions, StabilityUpscaleOptions } from "../protocols/stability-images.js"
export const id = ProviderID.make("stability")
const baseURL = DEFAULT_BASE_URL
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
@@ -16,11 +16,16 @@ export type Config = ProviderAuthOption<"optional"> & {
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "STABILITY_API_KEY")
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const deployment = {
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
}
return {
id,
image: (modelID: string | ModelID) => StabilityImages.model({ ...media, id: modelID }),
upscale: () => StabilityImages.upscaleModel(media),
image: (modelID: string | ModelID) => StabilityImages.model({ ...deployment, id: modelID }),
upscale: () => StabilityImages.upscaleModel(deployment),
configure,
}
}
+1 -1
View File
@@ -20,7 +20,7 @@ export const configure = (input: Options = {}) => {
auth: AuthOptions.bearer(input, "TYPESAFE_API_KEY"),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: HttpOptions.make(input.http),
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return { id, experimental: { evaluation }, configure }
}
@@ -67,7 +67,7 @@ export const configure = (input: Options = {}) => {
EvaluationModel.make<EvaluationOptions>({
id: modelID,
provider: id,
http: HttpOptions.make(input.http),
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
route: {
id: "vercel-evaluation",
evaluate: (req, send) =>
+10 -5
View File
@@ -1,8 +1,7 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, type ModelID } from "../schema/index.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { OpenAIChat } from "../protocols/openai-chat.js"
import { OpenResponsesChannel } from "../protocols/open-responses-channel.js"
import { XAIResponses } from "../protocols/xai-responses.js"
@@ -90,14 +89,20 @@ export const configure = (input: LanguageModelOptions = {}) => {
const chatRoute = configuredChatRoute(input)
const responses = (modelID: string | ModelID) => responsesRoute.model<XAIProviderOptionsInput>({ id: modelID })
const chat = (modelID: string | ModelID) => chatRoute.model<XAIProviderOptionsInput>({ id: modelID })
const media = MediaRoute.deployment(input, auth(input))
const media = (modelID: string | ModelID) => ({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
model: responses,
responses,
chat,
image: (modelID: string | ModelID) => XAIImages.model({ ...media, id: modelID }),
video: (modelID: string | ModelID) => XAIVideo.model({ ...media, id: modelID }),
image: (modelID: string | ModelID) => XAIImages.model(media(modelID)),
video: (modelID: string | ModelID) => XAIVideo.model(media(modelID)),
configure,
}
}
+9 -4
View File
@@ -5,8 +5,7 @@ import { OpenAIChat } from "../protocols/openai-chat.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, type ModelID } from "../schema/index.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
export const id = ProviderID.make("zai")
@@ -49,8 +48,14 @@ export const configure = (input: Config = {}) => {
auth: auth(input),
})
.model<ChatOptionsInput>({ id: modelID, compatibility: ZAIChat.compatibility })
const media = MediaRoute.deployment(input, auth(input))
const image = (modelID: string | ModelID) => ZAIImages.model({ ...media, id: modelID })
const image = (modelID: string | ModelID) =>
ZAIImages.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
+14 -19
View File
@@ -1,15 +1,12 @@
import { Config, Effect, Option, Redacted } from "effect"
import { Config, Effect, Redacted } from "effect"
import { Headers } from "effect/unstable/http"
import { AuthenticationError, AIError, type HttpOptions } from "../schema/index.js"
import { AuthenticationError, InvalidRequestError, AIError, type HttpOptions } from "../schema/index.js"
export class MissingCredentialError extends Error {
readonly _tag = "MissingCredentialError"
constructor(
readonly source: string,
message = `Missing auth credential: ${source}`,
) {
super(message)
constructor(readonly source: string) {
super(`Missing auth credential: ${source}`)
}
}
@@ -92,14 +89,7 @@ export const optional = (secret: Secret | undefined, source = "optional value")
? credential(Effect.fail(new MissingCredentialError(source)))
: credentialFromSecret(secret, source)
export const config = (name: string) =>
credential(
Effect.gen(function* () {
const secret = yield* Config.option(Config.redacted(name))
if (Option.isSome(secret) && Redacted.value(secret.value) !== "") return secret.value
return yield* Effect.fail(new MissingCredentialError(name, `${name} is not set`))
}),
)
export const config = (name: string) => credentialFromSecret(Config.redacted(name), name)
export const effect = (load: Effect.Effect<Redacted.Redacted, CredentialError>) => credential(load)
@@ -155,10 +145,15 @@ export function scheme(name: string, source?: Secret | Credential) {
}
const toAIError = (error: AuthError): AIError => {
if (error instanceof AIError) return error
const message =
error instanceof MissingCredentialError ? error.message : `Failed to resolve auth config: ${error.message}`
return new AIError({ reason: new AuthenticationError({ message, cause: error }) })
if (error instanceof MissingCredentialError || error instanceof Config.ConfigError) {
return new AIError({
reason:
error instanceof MissingCredentialError
? new AuthenticationError({ message: error.message, cause: error })
: new InvalidRequestError({ message: `Failed to resolve auth config: ${error.message}`, cause: error }),
})
}
return error
}
export const toEffect =
+7 -4
View File
@@ -152,7 +152,7 @@ const mergeRouteDefaults = (base: RouteDefaults | undefined, patch: RouteDefault
providerOptions: mergeProviderOptions(base?.providerOptions, patch.providerOptions),
http: mergeHttpOptions(
base?.http,
HttpOptions.make(patch.http),
httpOptions(patch.http),
headers === undefined ? undefined : new HttpOptions({ headers }),
),
}
@@ -172,6 +172,11 @@ const mergeHeaders = (...items: ReadonlyArray<Record<string, string> | undefined
export const generationOptions = (input: GenerationOptions.Input | undefined) =>
input === undefined ? undefined : GenerationOptions.make(input)
export const httpOptions = (input: HttpOptionsInput | undefined) => {
if (input === undefined) return input
return HttpOptions.make(input)
}
export interface Interface {
readonly compact: CompactMethod
readonly stream: StreamMethod
@@ -256,9 +261,7 @@ const unsupportedCompaction = (request: LLMRequest, mechanism: string | undefine
})
}
export class LLMClientService extends Context.Service<LLMClientService, Interface>()("@opencode/LLMClient") {}
export const Service = LLMClientService
export type Service = LLMClientService
export class Service extends Context.Service<Service, Interface>()("@opencode/LLMClient") {}
const resolveRequestOptions = (request: LLMRequest) => {
const messages = normalizeToolHistory(request.messages)
+2 -3
View File
@@ -1,4 +1,5 @@
import type { LLMRequest } from "../schema/index.js"
import * as ProviderShared from "../protocols/shared.js"
export interface EndpointInput<Body, Request = LLMRequest> {
readonly request: Request
@@ -46,8 +47,6 @@ export const merge = <Body, Request = LLMRequest>(
query: patch.query === undefined ? base.query : { ...base.query, ...patch.query },
})
export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
const renderPart = <Body, Request>(part: EndpointPart<Body, Request>, input: EndpointInput<Body, Request>) =>
typeof part === "function" ? part(input) : part
@@ -55,7 +54,7 @@ export const render = <Body, Request = LLMRequest>(
endpoint: Definition<Body, Request>,
input: EndpointInput<Body, Request>,
) => {
const url = new URL(`${trimBaseUrl(endpoint.baseURL ?? "")}${renderPart(endpoint.path, input)}`)
const url = new URL(`${ProviderShared.trimBaseUrl(endpoint.baseURL ?? "")}${renderPart(endpoint.path, input)}`)
for (const [key, value] of Object.entries(endpoint.query ?? {})) url.searchParams.set(key, value)
return url
}
+1 -5
View File
@@ -19,8 +19,4 @@ export type HttpMiddleware = (
handler: HttpHandler,
) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error>
export class RequestExecutorService extends Context.Service<RequestExecutorService, Interface>()(
"@opencode/AI/RequestExecutor",
) {}
export const Service = RequestExecutorService
export type Service = RequestExecutorService
export class Service extends Context.Service<Service, Interface>()("@opencode/AI/RequestExecutor") {}
-16
View File
@@ -255,20 +255,4 @@ export const layer: Layer.Layer<Service, never, HttpClient.HttpClient> = Layer.e
export const fetchLayer = layer.pipe(Layer.provide(FetchHttpClient.layer))
/** Run `fn` on every request: it sees the raw response before status classification, inside middleware already on `executor`, and outside per-call middleware. */
export const middleware = (fn: HttpMiddleware, executor: Layer.Layer<Service> = fetchLayer): Layer.Layer<Service> =>
Layer.effect(
Service,
Effect.gen(function* () {
const inner = yield* Service
return Service.of({
execute: (request, next) =>
inner.execute(
request,
next === undefined ? fn : (input, handler) => fn(input, (forwarded) => next(forwarded, handler)),
),
})
}),
).pipe(Layer.provide(executor))
export * as RequestExecutor from "./executor.js"
+6 -62
View File
@@ -1,6 +1,6 @@
import { Effect, Stream } from "effect"
import { makeParser, type Event } from "effect/unstable/encoding/Sse"
import { AIError, InvalidProviderOutputError } from "../schema/index.js"
import { Stream } from "effect"
import * as ProviderShared from "../protocols/shared.js"
import type { AIError } from "../schema/index.js"
/**
* Decode a streaming HTTP response body into provider-protocol frames.
@@ -25,75 +25,19 @@ export interface Definition<Frame> {
readonly body?: (frame: Frame) => string | undefined
}
/**
* `framing` step for Server-Sent Events. Decodes UTF-8, runs the SSE channel
* decoder, optionally filters named events, and drops empty events and known
* keepalives that proxies send as data. `[DONE]` is dropped by default or
* retained for protocols that use it as their stream boundary. Retry control events are ignored without
* interrupting the stream. Decoder failures become provider output errors so
* the public error channel stays `AIError`.
*/
export const sseFraming = (
bytes: Stream.Stream<Uint8Array, AIError>,
events?: ReadonlySet<string>,
includeDone = false,
): Stream.Stream<string, AIError> =>
bytes.pipe(
Stream.decodeText(),
Stream.mapAccumEffect(
() => {
const output: Event[] = []
return {
output,
parser: makeParser((event) => {
if (event._tag === "Event") output.push(event)
}),
}
},
(state, chunk) =>
Effect.gen(function* () {
const error = state.parser.feed(chunk)
if (error)
return yield* new AIError({
reason: new InvalidProviderOutputError({
route: "sse",
message: error.message,
body: chunk,
cause: error,
}),
})
return [state, state.output.splice(0)] as const
}),
),
Stream.filter(
(event) =>
(events === undefined || events.has(event.event)) &&
event.data.length > 0 &&
// Some OpenAI-compatible proxies serialize an empty flush as a bare
// `data: null`, between events or after `[DONE]`. No protocol has a
// null event, so it carries nothing and must not abort the stream.
event.data !== "null" &&
// Vertex AI partner models (e.g. `xai/grok-4.6`) send their SSE
// keepalive comment as `data: : keepalive` while reasoning.
event.data !== ": keepalive" &&
(event.data !== "[DONE]" || includeDone || (events !== undefined && event.event !== "message")),
),
Stream.map((event) => event.data),
)
/** Server-Sent Events framing. Used by every JSON-streaming HTTP provider. */
export const sse: Definition<string> = { id: "sse", frame: sseFraming }
export const sse: Definition<string> = { id: "sse", frame: ProviderShared.sseFraming }
/** Server-Sent Events framing that retains the conventional `[DONE]` sentinel. */
export const sseWithDone: Definition<string> = {
id: "sse",
frame: (bytes) => sseFraming(bytes, undefined, true),
frame: (bytes) => ProviderShared.sseFraming(bytes, undefined, true),
}
/** SSE framing restricted to protocol-recognized event names. */
export const sseEvents = (events: ReadonlySet<string>): Definition<string> => ({
id: "sse",
frame: (bytes) => sseFraming(bytes, events),
frame: (bytes) => ProviderShared.sseFraming(bytes, events),
})
export const lines: Definition<string> = {
+1 -1
View File
@@ -7,7 +7,7 @@ export type {
RouteDefaultsInput,
AnyRoute,
Interface as LLMClientShape,
LLMClientService,
Service as LLMClientService,
StreamOptions,
CompactMethod,
CompactionOperations,
+96 -98
View File
@@ -9,9 +9,7 @@ import {
HttpContext,
InvalidProviderOutputError,
InvalidRequestError,
ProviderID,
ProviderInternalError,
UnsupportedOperationError,
} from "../schema/index.js"
// ---------------------------------------------------------------------------
@@ -63,7 +61,7 @@ export interface DecodeContext<Request> {
export interface Inline<Request, Response> {
readonly kind: "inline"
readonly id: string
readonly provider: ProviderID
readonly name: string
/** Common request fields this protocol cannot lower; the route rejects them before `body.from` runs. */
readonly unsupported?: ReadonlyArray<keyof Request & string>
readonly body: { readonly from: (request: Request) => Effect.Effect<Body, AIError> }
@@ -76,9 +74,11 @@ export interface Inline<Request, Response> {
}
export const inline = <Request, Response>(
route: Identity,
input: Omit<Inline<Request, Response>, "kind" | "id" | "provider">,
): Inline<Request, Response> => ({ kind: "inline", id: route.id, provider: route.provider, ...input })
input: Omit<Inline<Request, Response>, "kind">,
): Inline<Request, Response> => ({
kind: "inline",
...input,
})
/** What `start` learned from the submission response: the route-owned handle plus the first observation. */
export interface Started<Token> {
@@ -107,7 +107,7 @@ export interface PollContext<Token> {
export interface Queued<Request, Response, Token> {
readonly kind: "queued"
readonly id: string
readonly provider: ProviderID
readonly name: string
/** Common request fields this protocol cannot lower; the route rejects them before `start.body.from` runs. */
readonly unsupported?: ReadonlyArray<keyof Request & string>
/** Serializable handle. `Generation.token` carries the encoded form so it can be persisted and resumed elsewhere. */
@@ -141,9 +141,11 @@ export interface Queued<Request, Response, Token> {
}
export const queued = <Request, Response, Token>(
route: Identity,
input: Omit<Queued<Request, Response, Token>, "kind" | "id" | "provider">,
): Queued<Request, Response, Token> => ({ kind: "queued", id: route.id, provider: route.provider, ...input })
input: Omit<Queued<Request, Response, Token>, "kind">,
): Queued<Request, Response, Token> => ({
kind: "queued",
...input,
})
export type Mode = "generate" | "stream"
@@ -160,7 +162,7 @@ export interface ResponseContext<Request> extends DecodeContext<Addressed<Reques
export interface Streamed<Request, Event, Frame, State> {
readonly kind: "stream"
readonly id: string
readonly provider: ProviderID
readonly name: string
/** Common request fields this protocol cannot lower; the route rejects them before `body.from` runs. */
readonly unsupported?: ReadonlyArray<keyof Request & string>
readonly body: { readonly from: (request: Addressed<Request>) => Effect.Effect<Body, AIError> }
@@ -175,9 +177,11 @@ export interface Streamed<Request, Event, Frame, State> {
}
export const stream = <Request, Event, Frame, State>(
route: Identity,
input: Omit<Streamed<Request, Event, Frame, State>, "kind" | "id" | "provider">,
): Streamed<Request, Event, Frame, State> => ({ kind: "stream", id: route.id, provider: route.provider, ...input })
input: Omit<Streamed<Request, Event, Frame, State>, "kind">,
): Streamed<Request, Event, Frame, State> => ({
kind: "stream",
...input,
})
// ---------------------------------------------------------------------------
// Response helpers
@@ -186,98 +190,71 @@ export const stream = <Request, Event, Frame, State>(
const context = (response: HttpClientResponse.HttpClientResponse) =>
new HttpContext({ url: response.request.url, status: response.status, headers: response.headers })
/** One protocol's route id, display name, and provider, with the decoders and errors that carry them. */
export const identity = (input: { readonly id: string; readonly name: string; readonly provider: string }) => {
const provider = ProviderID.make(input.provider)
const frameError = (message: string, body?: string, cause?: unknown) =>
new AIError({ reason: new InvalidProviderOutputError({ route: input.id, message, body, cause }) })
/**
* Read a text body while retaining the original payload and HTTP context on every downstream error. `invalid` is a
* malformed provider document; `ended` is a generation that reached a terminal status without output (`failed` is
* provider-side, `cancelled`/`expired` mean the result will never exist); `contentPolicy` is a moderated result.
*/
const text = Effect.fn("MediaProtocol.text")(function* (response: HttpClientResponse.HttpClientResponse) {
const http = context(response)
const body = yield* response.text.pipe(
Effect.mapError(
(cause) =>
new AIError({
reason: new InvalidProviderOutputError({
route: input.id,
message: `Failed to read the ${input.name} response`,
http,
cause,
}),
}),
),
)
return {
body,
http,
invalid: (message: string, cause?: unknown) =>
new AIError({ reason: new InvalidProviderOutputError({ route: input.id, message, body, http, cause }) }),
ended: (status: Exclude<Status, "queued" | "running" | "completed">, message: string) =>
/**
* Read a text body while retaining the original payload and HTTP context on every downstream error. `invalid` is a
* malformed provider document; `ended` is a generation that reached a terminal status without output (`failed` is
* provider-side, `cancelled`/`expired` mean the result will never exist); `contentPolicy` is a moderated result.
*/
export const text = Effect.fn("MediaProtocol.text")(function* (
route: string,
name: string,
response: HttpClientResponse.HttpClientResponse,
) {
const http = context(response)
const body = yield* response.text.pipe(
Effect.mapError(
(cause) =>
new AIError({
reason:
status === "failed"
? new ProviderInternalError({ message, body, http })
: new InvalidRequestError({ message, body, http }),
reason: new InvalidProviderOutputError({
route,
message: `Failed to read the ${name} response`,
http,
cause,
}),
}),
contentPolicy: (message: string) => new AIError({ reason: new ContentPolicyError({ message, body, http }) }),
}
})
/** Read and Schema-decode a JSON body. Decode failures keep the raw body as `reason.body`. */
const decodeJson = <A>(schema: Schema.Codec<A, unknown>) => {
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
return Effect.fn("MediaProtocol.decodeJson")(function* (response: HttpClientResponse.HttpClientResponse) {
const output = yield* text(response)
const value = yield* decode(output.body).pipe(
Effect.mapError((cause) => output.invalid(`${input.name} returned an invalid response`, cause)),
)
return { ...output, value }
})
}
),
)
return {
id: input.id,
name: input.name,
provider,
text,
decodeJson,
/** Decode a submission response into the token and first snapshot. */
decodeStarted: <A, Token>(schema: Schema.Codec<A, unknown>, started: (value: A) => Started<Token>) => {
const decode = decodeJson(schema)
return (response: HttpClientResponse.HttpClientResponse) =>
decode(response).pipe(Effect.map((output) => started(output.value)))
},
/** Schema-decode one JSON stream frame. Decode failures keep the frame as `reason.body`. */
decodeFrame: <A>(schema: Schema.Codec<A, unknown>) => {
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
return (frame: string) =>
decode(frame).pipe(
Effect.mapError((cause) => frameError(`${input.name} sent an invalid stream event`, frame, cause)),
)
},
/** A stream-time failure; the frame stays on `reason.body`. */
frameError,
incomplete: () =>
body,
http,
invalid: (message: string, cause?: unknown) =>
new AIError({ reason: new InvalidProviderOutputError({ route, message, body, http, cause }) }),
ended: (status: Exclude<Status, "queued" | "running" | "completed">, message: string) =>
new AIError({
reason: new InvalidProviderOutputError({
route: input.id,
message: "The provider response ended unexpectedly.",
classification: "incomplete-stream",
}),
reason:
status === "failed"
? new ProviderInternalError({ message, body, http })
: new InvalidRequestError({ message, body, http }),
}),
unsupported: (operation: string, message: string) =>
new AIError({ reason: new UnsupportedOperationError({ operation, provider, route: input.id, message }) }),
contentPolicy: (message: string) => new AIError({ reason: new ContentPolicyError({ message, body, http }) }),
}
})
export type Output = Effect.Success<ReturnType<typeof text>>
/** Read and Schema-decode a JSON body. Decode failures keep the raw body as `reason.body`. */
export const decodeJson = <A>(route: string, name: string, schema: Schema.Codec<A, unknown>) => {
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
return Effect.fn("MediaProtocol.decodeJson")(function* (response: HttpClientResponse.HttpClientResponse) {
const output = yield* text(route, name, response)
const value = yield* decode(output.body).pipe(
Effect.mapError((cause) => output.invalid(`${name} returned an invalid response`, cause)),
)
return { ...output, value }
})
}
export type Identity = ReturnType<typeof identity>
export type Output = Effect.Success<ReturnType<Identity["text"]>>
/** Decode a submission response into the token and first snapshot. */
export const decodeStarted = <A, Token>(
route: string,
name: string,
schema: Schema.Codec<A, unknown>,
started: (value: A) => Started<Token>,
) => {
const decode = decodeJson(route, name, schema)
return (response: HttpClientResponse.HttpClientResponse) =>
decode(response).pipe(Effect.map((output) => started(output.value)))
}
/** Map a provider status string through the protocol's table; unknown values are an invalid provider document. */
export const status = <Table extends Record<string, Status>>(
@@ -290,6 +267,27 @@ export const status = <Table extends Record<string, Status>>(
return Effect.succeed(normalized)
}
export const frameError = (route: string, message: string, body?: string, cause?: unknown) =>
new AIError({ reason: new InvalidProviderOutputError({ route, message, body, cause }) })
export const incomplete = (route: string) =>
new AIError({
reason: new InvalidProviderOutputError({
route,
message: "The provider response ended unexpectedly.",
classification: "incomplete-stream",
}),
})
/** Schema-decode one JSON stream frame. Decode failures keep the frame as `reason.body`. */
export const decodeFrame = <A>(route: string, name: string, schema: Schema.Codec<A, unknown>) => {
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
return (frame: string) =>
decode(frame).pipe(
Effect.mapError((cause) => frameError(route, `${name} sent an invalid stream event`, frame, cause)),
)
}
/** A `url` asset whose provider-declared retention window starts now. */
export const expiringUrl = (url: string, retention: Duration.Duration, options?: Parameters<typeof Media.url>[1]) =>
Clock.currentTimeMillis.pipe(
+37 -47
View File
@@ -2,21 +2,26 @@ import { Effect, Schema, Stream } from "effect"
import { Headers, HttpClientRequest, type HttpClientResponse } from "effect/unstable/http"
import { Auth, type AuthInput } from "./auth.js"
import { Endpoint } from "./endpoint.js"
import { RequestExecutorService, type Interface } from "./executor-service.js"
import { Service as RequestExecutorService, type Interface } from "./executor-service.js"
import { RequestExecutor } from "./executor.js"
import { MediaProtocol } from "./media-protocol.js"
import { Generation, resultEvents, type AwaitOptions, type Observation } from "../generation.js"
import {
Generation,
resultEvents,
type AwaitOptions,
type Observation,
type Route as GenerationRoute,
} from "../generation.js"
import type { Media } from "../media.js"
import { ProviderShared } from "../protocols/shared.js"
import {
AIError,
AIErrorReason,
HttpOptions,
InvalidRequestError,
ProviderID,
UnsupportedOperationError,
mergeHttpOptions,
} from "../schema/index.js"
import { encodeJson } from "../utils/json.js"
import { sanitizeSurrogates } from "../utils/sanitize.js"
export type Execute = Interface["execute"]
@@ -36,17 +41,6 @@ export interface ModelInput {
readonly http?: HttpOptions
}
/** A provider facade's `configure(...)` input as the `ModelInput` every media selector shares, minus the model id. */
export const deployment = (
input: { readonly baseURL?: string; readonly headers?: Record<string, string>; readonly http?: HttpOptions.Input },
auth: Auth.Definition,
): Omit<ModelInput, "id"> => ({
auth,
baseURL: input.baseURL,
headers: input.headers,
http: HttpOptions.make(input.http),
})
// ---------------------------------------------------------------------------
// Routes
// ---------------------------------------------------------------------------
@@ -91,6 +85,8 @@ export type AnyRoute<Request extends MediaRequest, Event, Response> =
| QueuedRoute<Request, Response>
export interface Composition<Request extends MediaRequest> {
readonly id: string
readonly provider: string | ProviderID
readonly endpoint: Endpoint.Definition<MediaProtocol.Body, Request>
readonly auth: Auth.Definition
/** Deployment headers applied before transport authentication. */
@@ -123,8 +119,8 @@ export const inline = <Request extends MediaRequest, Response>(
const transport = makeTransport(input)
return {
kind: "inline",
id: input.protocol.id,
provider: input.protocol.provider,
id: input.id,
provider: transport.provider,
protocol: input.protocol.id,
generate: Effect.fn(`MediaRoute.generate`)(function* (request: Request, execute: Execute) {
const submitted = yield* transport.submit(
@@ -165,7 +161,7 @@ export const queued = <Request extends MediaRequest, Response, Token>(
.call("GET", operation.path(token), http, execute)
.pipe(Effect.flatMap((sent) => operation.decode(sent.response, { token, auth: sent.auth, materialize })))
const cancel = protocol.cancel
return {
const route: GenerationRoute<Response> = {
status: poll(protocol.status),
result: poll(protocol.result),
cancel:
@@ -173,6 +169,7 @@ export const queued = <Request extends MediaRequest, Response, Token>(
? undefined
: transport.call(cancel.method, cancel.path(token), http, execute).pipe(Effect.asVoid),
}
return route
}
const start = Effect.fn("MediaRoute.start")(function* (request: Request, execute: Execute) {
@@ -196,7 +193,7 @@ export const queued = <Request extends MediaRequest, Response, Token>(
(cause) =>
new AIError({
reason: new InvalidRequestError({
message: `${protocol.id} cannot resume a generation from this token`,
message: `${input.id} cannot resume a generation from this token`,
cause,
}),
}),
@@ -206,7 +203,7 @@ export const queued = <Request extends MediaRequest, Response, Token>(
return new Generation(route, encodeToken(token), yield* route.status)
})
return { kind: "queued", id: protocol.id, provider: protocol.provider, protocol: protocol.id, start, resume }
return { kind: "queued", id: input.id, provider: transport.provider, protocol: protocol.id, start, resume }
}
/** Compose a streaming media protocol; `generate` runs the same stream in `generate` mode and folds it with `collect`. */
@@ -258,8 +255,8 @@ export const stream = <Request extends MediaRequest, Event, Response, Frame, Sta
)
return {
kind: "stream",
id: protocol.id,
provider: protocol.provider,
id: input.id,
provider: transport.provider,
protocol: protocol.id,
stream: (request, execute) => events(request, execute, "stream"),
generate: (request, execute) =>
@@ -273,13 +270,11 @@ export const dispatch = <Event, Response>(input: {
readonly responseEvents: (response: Response) => ReadonlyArray<Event>
}) => {
const notQueued = (route: { readonly provider: ProviderID; readonly id: string }, operation: string) =>
new AIError({
reason: new UnsupportedOperationError({
operation: `${input.modality}.${operation}`,
provider: route.provider,
route: route.id,
message: `${route.provider}/${route.id} is not a queued route; use generate or stream`,
}),
ProviderShared.unsupportedOperation({
operation: `${input.modality}.${operation}`,
provider: route.provider,
route: route.id,
message: `${route.provider}/${route.id} is not a queued route; use generate or stream`,
})
const start = <Request extends MediaRequest>(route: AnyRoute<Request, Event, Response>, request: Request) => {
if (route.kind !== "queued") return Effect.fail(notQueued(route, "start"))
@@ -324,13 +319,12 @@ export const dispatch = <Event, Response>(input: {
// Transport plumbing shared by every kind
// ---------------------------------------------------------------------------
const makeTransport = <Request extends MediaRequest>(
input: Composition<Request> & { readonly protocol: { readonly id: string; readonly provider: ProviderID } },
) => {
const makeTransport = <Request extends MediaRequest>(input: Composition<Request>) => {
const provider = ProviderID.make(input.provider)
const routeHttp = input.headers === undefined ? undefined : new HttpOptions({ headers: input.headers })
const authorize = Auth.toEffect(input.auth)
const baseURL = (path: string) => new URL(`${Endpoint.trimBaseUrl(input.endpoint.baseURL ?? "")}${path}`)
/** `auth` is only what `Auth` added or changed, never untouched deployment headers. */
const baseURL = (path: string) => new URL(`${ProviderShared.trimBaseUrl(input.endpoint.baseURL ?? "")}${path}`)
/** `auth` is only what `Auth` added, never deployment headers. */
const send = Effect.fn("MediaRoute.send")(function* (
call: {
readonly method: AuthInput["method"]
@@ -353,12 +347,10 @@ const makeTransport = <Request extends MediaRequest>(
const response = yield* execute(
encoded.apply(HttpClientRequest.make(call.method)(url).pipe(HttpClientRequest.setHeaders(headers))),
)
return {
response,
auth: Object.fromEntries(Object.entries(headers).filter(([key, value]) => encoded.headers[key] !== value)),
}
return { response, auth: Object.fromEntries(Object.entries(headers).filter(([key]) => !(key in call.headers))) }
})
return {
provider,
/** Route and model overlays; `start` additionally merges the request's own `http`. */
http: (model: MediaRequest["model"]) => mergeHttpOptions(routeHttp, model.http),
/** POST the protocol body to the route endpoint. */
@@ -371,7 +363,7 @@ const makeTransport = <Request extends MediaRequest>(
},
execute: Execute,
) {
yield* rejectUnsupported(input.protocol.id, input.protocol.provider, request, protocol.unsupported)
yield* rejectUnsupported(input.id, provider, request, protocol.unsupported)
const http = mergeHttpOptions(routeHttp, request.model.http, request.http)
const headers = Headers.fromInput(http?.headers)
const prepared =
@@ -416,7 +408,7 @@ const withQuery = (url: URL, query: MediaProtocol.Query | undefined) => {
const encode = (body: MediaProtocol.Body | undefined, headers: Headers.Headers) => {
if (body === undefined) return { text: "", headers, apply: (request: HttpClientRequest.HttpClientRequest) => request }
if (body.type === "json") {
const text = encodeJson(body.value)
const text = ProviderShared.encodeJson(body.value)
return { text, headers, apply: HttpClientRequest.bodyText(text, "application/json") }
}
if (body.type === "binary")
@@ -446,13 +438,11 @@ const rejectUnsupported = <Request extends object>(
})
if (present.length === 0) return Effect.void
return Effect.fail(
new AIError({
reason: new UnsupportedOperationError({
operation: `media.${present[0]}`,
provider,
route,
message: `${provider}/${route} does not support ${present.join(", ")}`,
}),
ProviderShared.unsupportedOperation({
operation: `media.${present[0]}`,
provider,
route,
message: `${provider}/${route} does not support ${present.join(", ")}`,
}),
)
}
+7 -16
View File
@@ -57,13 +57,8 @@ export class HttpOptions extends Schema.Class<HttpOptions>("AI.HttpOptions")({
export namespace HttpOptions {
export type Input = HttpOptions | ConstructorParameters<typeof HttpOptions>[0]
/** Normalize HTTP option input into the canonical `HttpOptions` class; `undefined` stays `undefined`. */
export function make(input: Input): HttpOptions
export function make(input: Input | undefined): HttpOptions | undefined
export function make(input: Input | undefined) {
if (input === undefined || input instanceof HttpOptions) return input
return new HttpOptions(input)
}
/** Normalize HTTP option input into the canonical `HttpOptions` class. */
export const make = (input: Input) => (input instanceof HttpOptions ? input : new HttpOptions(input))
}
export const mergeHttpOptions = (...items: ReadonlyArray<HttpOptions | undefined>): HttpOptions | undefined => {
@@ -145,24 +140,20 @@ export namespace LanguageModelDefaults {
return new LanguageModelDefaults({
generation: input.generation === undefined ? undefined : GenerationOptions.make(input.generation),
providerOptions: input.providerOptions,
http: HttpOptions.make(input.http),
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}
}
/** Provider-defined string enum: known values for autocomplete, any string accepted. */
export type OpenString<Known extends string> = Known | (string & {})
export const ReasoningEfforts = ["none", "minimal", "low", "medium", "high", "xhigh", "max"] as const
export type ReasoningEffort = OpenString<(typeof ReasoningEfforts)[number]>
export type ReasoningEffort = (typeof ReasoningEfforts)[number] | (string & {})
export const ReasoningEffort = Schema.declare<ReasoningEffort>(
(value): value is ReasoningEffort => typeof value === "string",
{ title: "ReasoningEffort" },
)
/** Tool schema sanitizer for a model family. `none` opts out of the protocol and model-name defaults. */
export const LanguageModelSanitizerCompatibility = Schema.Literals(["gemini", "moonshot", "none"])
export type LanguageModelSanitizerCompatibility = Schema.Schema.Type<typeof LanguageModelSanitizerCompatibility>
export const LanguageModelToolSchemaCompatibility = Schema.Literals(["gemini", "moonshot"])
export type LanguageModelToolSchemaCompatibility = Schema.Schema.Type<typeof LanguageModelToolSchemaCompatibility>
export const LanguageModelMaxTokensFieldCompatibility = Schema.Literals(["max_completion_tokens", "max_tokens"])
export type LanguageModelMaxTokensFieldCompatibility = Schema.Schema.Type<
@@ -172,7 +163,7 @@ export type LanguageModelMaxTokensFieldCompatibility = Schema.Schema.Type<
export class LanguageModelCompatibility extends Schema.Class<LanguageModelCompatibility>(
"LLM.LanguageModelCompatibility",
)({
sanitizer: Schema.optional(LanguageModelSanitizerCompatibility),
toolSchema: Schema.optional(LanguageModelToolSchemaCompatibility),
reasoningField: Schema.optional(Schema.String),
/** Require every assistant message to include its reasoning field, even when empty. */
requireReasoning: Schema.optional(Schema.Boolean),
+1 -3
View File
@@ -12,9 +12,7 @@ export interface Interface {
) => Stream.Stream<SpeechEvent, AIError>
}
export class SpeechClientService extends Context.Service<SpeechClientService, Interface>()("@opencode/SpeechClient") {}
export const Service = SpeechClientService
export type Service = SpeechClientService
export class Service extends Context.Service<Service, Interface>()("@opencode/SpeechClient") {}
export const generate = <Options extends SpeechOptions>(
request: SpeechRequestFor<Options>,
+4 -4
View File
@@ -3,7 +3,7 @@ import { Media } from "./media.js"
import { MediaModel, composeRoute, tryRequest } from "./media-model.js"
import { MediaRoute } from "./route/media.js"
import type { MediaProtocol } from "./route/media-protocol.js"
import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js"
import { AIError, HttpOptions, MediaUsage, ProviderMetadata } from "./schema/index.js"
import { SpeechClient, Service } from "./speech-client.js"
// ---------------------------------------------------------------------------
@@ -35,7 +35,7 @@ export class SpeechModel<Options extends SpeechOptions = SpeechOptions> extends
) {
return new SpeechModel<Options>({
id: input.id,
provider: route.protocol.provider,
provider: route.provider,
http: input.http,
route: composeRoute(
(composition) => MediaRoute.stream({ ...composition, collect: collectResponse }),
@@ -71,7 +71,7 @@ export const SpeechVoice = Schema.Union([Schema.String, Schema.Struct({ id: Sche
})
export type SpeechVoice = Schema.Schema.Type<typeof SpeechVoice>
export type SpeechFormat = OpenString<"mp3" | "wav" | "pcm" | "opus" | "aac" | "flac">
export type SpeechFormat = "mp3" | "wav" | "pcm" | "opus" | "aac" | "flac" | (string & {})
/** Granularity is provider-native: characters on ElevenLabs, words on Cartesia. */
export const SpeechTimestamp = Schema.Struct({
@@ -188,7 +188,7 @@ export function request(input: SpeechRequest | SpeechRequestInput) {
if (input instanceof SpeechRequest) return input
return new SpeechRequest({
...input,
http: HttpOptions.make(input.http),
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}
+1 -5
View File
@@ -30,11 +30,7 @@ export interface Interface {
) => Effect.Effect<Generation<TranscriptionResponse>, AIError>
}
export class TranscriptionClientService extends Context.Service<TranscriptionClientService, Interface>()(
"@opencode/TranscriptionClient",
) {}
export const Service = TranscriptionClientService
export type Service = TranscriptionClientService
export class Service extends Context.Service<Service, Interface>()("@opencode/TranscriptionClient") {}
export const generate = <Options extends TranscriptionOptions>(
request: TranscriptionRequestFor<Options>,
+2 -2
View File
@@ -50,7 +50,7 @@ export class TranscriptionModel<Options extends TranscriptionOptions = Transcrip
) {
return new TranscriptionModel<Options>({
id: input.id,
provider: route.protocol.provider,
provider: route.provider,
http: input.http,
route: composeAnyRoute(route, input, collectResponse),
})
@@ -236,7 +236,7 @@ export function request(input: TranscriptionRequest | TranscriptionRequestInput)
if (input instanceof TranscriptionRequest) return input
return new TranscriptionRequest({
...input,
http: HttpOptions.make(input.http),
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}
-5
View File
@@ -1,5 +0,0 @@
import { Schema } from "effect"
export const Json = Schema.fromJsonString(Schema.Unknown)
export const decodeJson = Schema.decodeUnknownSync(Json)
export const encodeJson = Schema.encodeSync(Json)
+1 -4
View File
@@ -48,12 +48,9 @@ const EXTENSIONS: Readonly<Record<string, string>> = {
csv: "text/csv",
}
const extensionMediaType = (path: string): string | undefined =>
export const extensionMediaType = (path: string): string | undefined =>
EXTENSIONS[path.slice(path.lastIndexOf(".") + 1).toLowerCase()]
/** Media type of a file's contents: sniffed magic bytes, then the path's extension. */
export const fileMediaType = (bytes: Uint8Array, path: string) => detectMediaType(bytes) ?? extensionMediaType(path)
const EXTENSION_ALIASES: Readonly<Record<string, string>> = {
"audio/mp3": "mp3",
"audio/m4a": "m4a",
+1 -3
View File
@@ -29,9 +29,7 @@ export interface Interface {
) => Stream.Stream<VideoEvent, AIError>
}
export class VideoClientService extends Context.Service<VideoClientService, Interface>()("@opencode/VideoClient") {}
export const Service = VideoClientService
export type Service = VideoClientService
export class Service extends Context.Service<Service, Interface>()("@opencode/VideoClient") {}
export const start = <Options extends VideoOptions>(
request: VideoRequestFor<Options>,
+4 -4
View File
@@ -4,7 +4,7 @@ import { Media } from "./media.js"
import { MediaModel, composeRoute, tryRequest } from "./media-model.js"
import { MediaRoute } from "./route/media.js"
import type { MediaProtocol } from "./route/media-protocol.js"
import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js"
import { AIError, HttpOptions, MediaUsage, ProviderMetadata } from "./schema/index.js"
import { VideoClient, Service } from "./video-client.js"
// ---------------------------------------------------------------------------
@@ -32,7 +32,7 @@ export class VideoModel<Options extends VideoOptions = VideoOptions> extends Med
) {
return new VideoModel<Options>({
id: input.id,
provider: route.protocol.provider,
provider: route.provider,
http: input.http,
route: composeRoute(MediaRoute.queued, route, input),
})
@@ -57,7 +57,7 @@ export const VideoModelSchema = Schema.declare((value): value is VideoModel => v
export type VideoAspectRatio = Media.AspectRatio
export const VideoAspectRatio = Media.AspectRatio
export type VideoResolution = OpenString<"480p" | "720p" | "1080p" | "4k">
export type VideoResolution = "480p" | "720p" | "1080p" | "4k" | (string & {})
/** Pinned frames. Routes that accept only a first frame fail typed when `last` is present. */
export const VideoFrames = Schema.Struct({
@@ -171,7 +171,7 @@ export function request(input: VideoRequest | VideoRequestInput) {
if (input instanceof VideoRequest) return input
return new VideoRequest({
...input,
http: HttpOptions.make(input.http),
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}
-9
View File
@@ -92,15 +92,6 @@ describe("Auth", () => {
}),
)
it.effect("reports a missing config credential as Authentication naming the variable", () =>
Effect.gen(function* () {
const error = yield* Auth.toEffect(Auth.config("OPENAI_API_KEY").bearer())(input).pipe(withEnv({}), Effect.flip)
expect(error.reason._tag).toBe("Authentication")
expect(error.message).toContain("OPENAI_API_KEY is not set")
}),
)
it.effect("can intentionally leave auth untouched", () =>
Effect.gen(function* () {
const headers = yield* Auth.none.apply(input)
+4 -6
View File
@@ -144,7 +144,7 @@ describe("request option precedence", () => {
type: "function",
name: "crm",
description: "Top-level CRM tool",
parameters: { type: "object" },
parameters: {},
strict: false,
},
{
@@ -152,17 +152,15 @@ describe("request option precedence", () => {
name: "crm",
description: "CRM tools",
tools: [
{ type: "function", name: "lookup", description: "new", parameters: { type: "object" }, strict: false },
{ type: "function", name: "search", description: "search", parameters: { type: "object" }, strict: false },
{ type: "function", name: "lookup", description: "new", parameters: {}, strict: false },
{ type: "function", name: "search", description: "search", parameters: {}, strict: false },
],
},
{
type: "namespace",
name: "support",
description: "Support tools",
tools: [
{ type: "function", name: "lookup", description: "support", parameters: { type: "object" }, strict: false },
],
tools: [{ type: "function", name: "lookup", description: "support", parameters: {}, strict: false }],
},
])
}),
+2 -22
View File
@@ -1,11 +1,11 @@
import { describe, expect } from "bun:test"
import { Deferred, Effect, Fiber, Layer, Ref, Stream } from "effect"
import { Deferred, Effect, Fiber, Ref, Stream } from "effect"
import { Headers, HttpClientError, HttpClientRequest } from "effect/unstable/http"
import { LLM, AIError, HttpContext, InvalidProviderOutputError, TransportError } from "../src/index.js"
import { LLMClient, RequestExecutor, WebSocketTransport, type WebSocketChannelExecutor } from "../src/route.js"
import { route } from "../src/protocols/openai-chat.js"
import { configure } from "../src/providers/openai.js"
import { dynamicResponse, fixedResponse, handlerLayer, systemError } from "./lib/http.js"
import { dynamicResponse, fixedResponse, systemError } from "./lib/http.js"
import { deltaChunk } from "./lib/openai-chunks.js"
import { sseEvents, sseRaw } from "./lib/sse.js"
import { it } from "./lib/effect.js"
@@ -195,26 +195,6 @@ describe("RequestExecutor", () => {
),
)
it.effect("runs shared middleware outside per-call middleware", () => {
const calls: Array<string> = []
const record = (name: string) => Effect.sync(() => calls.push(name))
const base = RequestExecutor.layer.pipe(
Layer.provide(handlerLayer((input) => record("handler").pipe(Effect.as(input.respond("ok"))))),
)
return Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
yield* executor.execute(request, (input, next) => record("per-call").pipe(Effect.andThen(next(input))))
expect(calls).toEqual(["outer", "per-call", "handler"])
calls.length = 0
yield* executor.execute(request)
expect(calls).toEqual(["outer", "handler"])
}).pipe(
Effect.provide(
RequestExecutor.middleware((input, next) => record("outer").pipe(Effect.andThen(next(input))), base),
),
)
})
it.effect("classifies context overflow responses", () =>
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
+1 -27
View File
@@ -1,11 +1,8 @@
import { describe, expect, test } from "bun:test"
import { Effect, Layer } from "effect"
import {
AIClient,
AIError,
Generation,
Image,
ImageClient,
LanguageModel,
LLM,
LLMClient,
@@ -14,11 +11,10 @@ import {
Speech,
SpeechClient,
SpeechEvent,
TranscriptionClient,
Video,
VideoClient,
} from "@opencode/ai"
import { Route, Protocol, RequestExecutor, WebSocketTransport } from "@opencode/ai/route"
import { Route, Protocol, WebSocketTransport } from "@opencode/ai/route"
import { Provider as ProviderSubpath } from "@opencode/ai/provider"
import {
AssemblyAI,
@@ -80,28 +76,6 @@ describe("public exports", () => {
expect(EvaluationClient.fetchLayer).toBeDefined()
})
test("AIClient.layerWith shares one executor across every client", async () => {
let built = 0
const counting = Layer.effect(
RequestExecutor.Service,
Effect.sync(() => {
built++
return RequestExecutor.Service.of({ execute: () => Effect.die("unexpected request") })
}),
)
await Effect.runPromise(
Effect.gen(function* () {
yield* LLMClient.Service
yield* ImageClient.Service
yield* VideoClient.Service
yield* SpeechClient.Service
yield* TranscriptionClient.Service
yield* RequestExecutor.Service
}).pipe(Effect.provide(AIClient.layerWith(counting))),
)
expect(built).toBe(1)
})
test("route barrel exposes route-authoring APIs", () => {
expect(Route.make).toBeFunction()
expect(Protocol.make).toBeFunction()

Some files were not shown because too many files have changed in this diff Show More