Compare commits

..
Author SHA1 Message Date
usrnk1 d45d7d2ee6 Merge branch 'v2' into adaptive-command-menu 2026-09-21 16:09:06 +02:00
usrnk1 3c8301e155 feat(desktop): adapt composer suggestion menu 2026-09-16 15:47:36 +02:00
470 changed files with 4260 additions and 11712 deletions
+2 -3
View File
@@ -112,12 +112,11 @@ jobs:
- name: Run unit tests
timeout-minutes: 20
run: |
# The runners have four vCPUs, and each Bun test process performs its own concurrent work.
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
GITHUB_ACTIONS=false bun turbo test --concurrency=3
GITHUB_ACTIONS=false bun turbo test
exit 0
fi
GITHUB_ACTIONS=false bun turbo test --affected --concurrency=3
GITHUB_ACTIONS=false bun turbo test --affected
env:
OPENCODE_EXPERIMENTAL_DISABLE_FILEWATCHER: ${{ runner.os == 'Windows' && 'true' || 'false' }}
TURBO_SCM_BASE: ${{ github.event_name == 'pull_request' && format('{0}^1', github.sha) || github.event.before }}
+35 -36
View File
@@ -32,7 +32,7 @@
},
"packages/ai": {
"name": "@opencode/ai",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@aws-sdk/credential-providers": "3.1057.0",
"@opencode/schema": "workspace:*",
@@ -54,7 +54,7 @@
},
"packages/app": {
"name": "@opencode/app",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@corvu/drawer": "catalog:",
"@dnd-kit/abstract": "0.5.0",
@@ -92,7 +92,6 @@
"solid-js": "catalog:",
"solid-presence": "0.2.0",
"tailwindcss": "4.3.3",
"uqr": "0.1.3",
},
"devDependencies": {
"@happy-dom/global-registrator": "20.0.11",
@@ -112,7 +111,7 @@
},
"packages/cli": {
"name": "@opencode/cli",
"version": "2.0.14",
"version": "2.0.12",
"bin": {
"opencode2": "./bin/opencode2.cjs",
},
@@ -176,7 +175,7 @@
},
"packages/client": {
"name": "@opencode/client",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@opencode/protocol": "workspace:*",
"@opencode/schema": "workspace:*",
@@ -202,7 +201,7 @@
},
"packages/codemode": {
"name": "@opencode/codemode",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"acorn": "8.15.0",
"effect": "catalog:",
@@ -215,7 +214,7 @@
},
"packages/console/app": {
"name": "@opencode/console-app",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@cloudflare/vite-plugin": "1.15.2",
"@ibm/plex": "6.4.1",
@@ -251,7 +250,7 @@
},
"packages/console/core": {
"name": "@opencode/console-core",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@aws-sdk/client-sts": "3.782.0",
"@jsx-email/render": "1.1.1",
@@ -278,7 +277,7 @@
},
"packages/console/function": {
"name": "@opencode/console-function",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@openauthjs/openauth": "0.0.0-20250322224806",
"@opencode/console-core": "workspace:*",
@@ -295,7 +294,7 @@
},
"packages/console/mail": {
"name": "@opencode/console-mail",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@jsx-email/all": "2.2.3",
"@jsx-email/cli": "1.4.3",
@@ -319,7 +318,7 @@
},
"packages/console/support": {
"name": "@opencode/console-support",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@cloudflare/vite-plugin": "1.15.2",
"@opencode/console-core": "workspace:*",
@@ -339,7 +338,7 @@
},
"packages/core": {
"name": "@opencode/core",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@ai-sdk/cohere": "3.0.27",
"@ai-sdk/gateway": "3.0.104",
@@ -407,7 +406,7 @@
},
"packages/desktop": {
"name": "@opencode/desktop",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@zip.js/zip.js": "2.7.62",
"electron-context-menu": "5.0.0",
@@ -456,7 +455,7 @@
},
"packages/enterprise": {
"name": "@opencode/enterprise",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@hono/standard-validator": "catalog:",
"@opencode-ai/sdk": "1.18.21",
@@ -493,7 +492,7 @@
},
"packages/function": {
"name": "@opencode/function",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@octokit/auth-app": "8.0.1",
"@octokit/rest": "catalog:",
@@ -509,7 +508,7 @@
},
"packages/http-recorder": {
"name": "@opencode/http-recorder",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@effect/platform-node-shared": "4.0.0-rc.112",
},
@@ -528,7 +527,7 @@
},
"packages/httpapi-codegen": {
"name": "@opencode/httpapi-codegen",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"effect": "catalog:",
"prettier": "3.6.2",
@@ -541,7 +540,7 @@
},
"packages/latex": {
"name": "@opencode/latex",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@opencode/plugin": "workspace:*",
"@opentui/core": "catalog:",
@@ -555,7 +554,7 @@
},
"packages/merman": {
"name": "@opencode/merman",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@opencode/plugin": "workspace:*",
"@opentui/core": "catalog:",
@@ -570,7 +569,7 @@
},
"packages/plugin": {
"name": "@opencode/plugin",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@ai-sdk/provider": "3.0.8",
"@opencode/ai": "workspace:*",
@@ -609,7 +608,7 @@
},
"packages/plugin-browser": {
"name": "@opencode/plugin-browser",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@opencode/plugin": "workspace:*",
"@opencode/schema": "workspace:*",
@@ -639,7 +638,7 @@
},
"packages/protocol": {
"name": "@opencode/protocol",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@opencode/schema": "workspace:*",
"effect": "catalog:",
@@ -654,7 +653,7 @@
},
"packages/schema": {
"name": "@opencode/schema",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@standard-schema/spec": "catalog:",
"effect": "catalog:",
@@ -678,7 +677,7 @@
},
"packages/sdk": {
"name": "@opencode/sdk",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@opencode/client": "workspace:*",
"@opencode/core": "workspace:*",
@@ -699,7 +698,7 @@
},
"packages/server": {
"name": "@opencode/server",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@effect/platform-node": "catalog:",
"@effect/platform-node-shared": "catalog:",
@@ -721,7 +720,7 @@
},
"packages/session-ui": {
"name": "@opencode/session-ui",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@kobalte/core": "catalog:",
"@opencode/client": "workspace:*",
@@ -756,7 +755,7 @@
},
"packages/simulation": {
"name": "@opencode/simulation",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@opencode/ai": "workspace:*",
"@opencode/core": "workspace:*",
@@ -776,7 +775,7 @@
},
"packages/stats/app": {
"name": "@opencode/stats-app",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@ibm/plex": "6.4.1",
"@kobalte/core": "catalog:",
@@ -810,7 +809,7 @@
},
"packages/stats/core": {
"name": "@opencode/stats-core",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@aws-sdk/client-athena": "3.933.0",
"@planetscale/database": "1.19.0",
@@ -829,7 +828,7 @@
},
"packages/stats/server": {
"name": "@opencode/stats-server",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@aws-sdk/client-firehose": "3.933.0",
"@effect/platform-node": "catalog:",
@@ -875,7 +874,7 @@
},
"packages/theme": {
"name": "@opencode/theme",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@opentui/core": "catalog:",
"effect": "catalog:",
@@ -889,7 +888,7 @@
},
"packages/tui": {
"name": "@opencode/tui",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@opencode/client": "workspace:*",
"@opencode/core": "workspace:*",
@@ -924,7 +923,7 @@
},
"packages/ui": {
"name": "@opencode/ui",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@kobalte/core": "catalog:",
"@pierre/diffs": "catalog:",
@@ -959,7 +958,7 @@
},
"packages/util": {
"name": "@opencode/util",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@effect/opentelemetry": "catalog:",
"@effect/platform-node": "catalog:",
@@ -992,7 +991,7 @@
},
"packages/web": {
"name": "@opencode/web",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"@astrojs/cloudflare": "12.6.3",
"@astrojs/markdown-remark": "6.3.1",
@@ -1033,7 +1032,7 @@
},
"services/update": {
"name": "@opencode/update",
"version": "2.0.14",
"version": "2.0.12",
"dependencies": {
"jose": "6.0.11",
"semver": "catalog:",
+4 -4
View File
@@ -1,8 +1,8 @@
{
"nodeModules": {
"x86_64-linux": "sha256-WBGj4wU5PQs+0zbDbScFhJs0GL8qWnswIsJb/Kee5Qc=",
"aarch64-linux": "sha256-Z8hkyNMdqCpuv24q50QhO2RECINvENjBhLDY5FKcnFc=",
"aarch64-darwin": "sha256-HZHb9HcXMOc7bKR7eADQSyfWxX37GAcoRi/EMSpRiP4=",
"x86_64-darwin": "sha256-CcWCOAcW4t9ADs48fMhumSbDcQWdk91LpkG6fe/gWlk="
"x86_64-linux": "sha256-/jah4P2a0aGJNJ0aMdlFEbGHAXx33lqxHbc7UmpLFDg=",
"aarch64-linux": "sha256-L3SoZ24qNXicsE2FK6LATQjOmjPxY679RjugrUyO/1Y=",
"aarch64-darwin": "sha256-pI9NT8KWUPi+JCk6DYMqIAYmBqTbi13uL4VdNV3WS6Y=",
"x86_64-darwin": "sha256-rMAGhTTz46KA5Ya7E5J0af7Bn1QzTDhTaNfNJm8qfsw="
}
}
+1 -1
View File
@@ -2,7 +2,7 @@
"$schema": "https://json.schemastore.org/package.json",
"name": "opencode",
"description": "AI-powered development tool",
"version": "2.0.14",
"version": "2.0.12",
"private": true,
"type": "module",
"packageManager": "bun@1.4.2",
+2 -15
View File
@@ -10,15 +10,7 @@
## Conventions
Per-type constructors live on the type, not as top-level re-exports. Use `Message.system(...)`, `Message.user(...)`, `Message.assistant(...)`, `Message.tool(...)`, `Message.media(...)`, `LanguageModel.make(...)`, `ToolDefinition.make(...)`, `ToolCallPart.make(...)`, `ToolResultPart.make(...)`, `ToolChoice.make(...)`, `ToolChoice.named(...)`, `SystemPart.make(...)`, and `GenerationOptions.make(...)` directly. The top-level `LLM` namespace is reserved for request-shaped call APIs: `LLM.request`, `LLM.generate`, `LLM.stream`, and `LLM.generateObject`. Use `LLMRequest.update(...)` when deriving canonical request data; do not add a duplicate `LLM.updateRequest(...)` path. Two ways to construct the same thing is one too many.
Modality namespaces mirror `LLM` exactly: `Image.request`, `Image.generate`, `Image.stream` (later `Video`, `Speech`, `Transcription`). Common request fields (`images`, `mask`, `n`, `size`, `aspectRatio`, `seed`, `format`) lower natively or fail with a typed `AIError`; provider-native controls always live under `providerOptions`, never under a modality-specific `options` key.
Media payloads are always `Media.Asset` (`src/media.ts`). Construct them with `Media.bytes`, `Media.base64`, `Media.url`, `Media.ref`, `Media.fromDataUrl`, or `Media.file`; never introduce a parallel `data: string | Uint8Array` shape. `MediaPart.media`, `ImageRequest.images`/`mask`, `ImageResponse.images`, and the `media` `LLMEvent` all share it. Protocols branch on `asset.source.type` and `asset.kind` and use `ProviderShared.inlineMedia` / `requireInlineMedia` / `mediaUrl` / `MediaInput.refID` rather than re-deriving base64 or URL handling.
`schema/messages.ts → media.ts → route/executor-service.ts` is an accepted runtime dependency from the schema layer on the executor service tag: `Media.Asset.bytes()` must be able to download `url` sources, and the tag lives in that leaf module precisely so the schema barrel never imports the executor implementation (which imports the schema barrel back). Do not move the tag into `route/executor.ts` or import `route/executor.ts` from `src/schema/*` or `src/media.ts`.
Nothing in `src/*` except `src/promise.ts` may know about Promises. `@opencode/ai/promise` (`AI.make({ layer? })`, default `ai`) is the single Promise/`AsyncIterable` surface for LLM and media; it runs the Effect APIs in one `ManagedRuntime` and rethrows `AIError` unchanged.
Per-type constructors live on the type, not as top-level re-exports. Use `Message.system(...)`, `Message.user(...)`, `Message.assistant(...)`, `Message.tool(...)`, `LanguageModel.make(...)`, `ToolDefinition.make(...)`, `ToolCallPart.make(...)`, `ToolResultPart.make(...)`, `ToolChoice.make(...)`, `ToolChoice.named(...)`, `SystemPart.make(...)`, and `GenerationOptions.make(...)` directly. The top-level `LLM` namespace is reserved for request-shaped call APIs: `LLM.request`, `LLM.generate`, `LLM.stream`, and `LLM.generateObject`. Use `LLMRequest.update(...)` when deriving canonical request data; do not add a duplicate `LLM.updateRequest(...)` path. Two ways to construct the same thing is one too many.
- Prefer forward compatibility for provider-defined options that OpenCode only passes through. For pass-through string enums, expose known values for autocomplete while accepting future values with `Known | (string & {})`, and accept any string at runtime. Closed literals are appropriate when OpenCode branches on a value, transforms its associated structure, or otherwise cannot correctly handle an unknown variant. New options whose shape or behavior requires implementation remain unsupported until they are handled; do not blindly forward unknown structures.
- Order reasoning-effort values from lowest to highest: `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`. Provider-specific subsets follow the same relative order in types, schemas, option lists, and tests.
@@ -94,10 +86,6 @@ The four-axis decomposition is the reason DeepSeek, TogetherAI, Cerebras, Basete
When a provider supports multiple physical transports, selection remains execution policy below its semantic route. `OpenResponsesChannel.transport(...)` owns the provider-neutral Responses WebSocket concept: it prepares one final request, executes HTTP by default, strips WebSocket-disallowed fields, and passes a generic channel exchange to a per-call `WebSocketChannelExecutor` when supplied. Provider-specific Responses routes opt in with handshake and connection-age policy. `Route.streamPrepared` owns decoding and acknowledges channel completion only after successful full consumption.
### Media Routes
Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute.make(...)` (`src/route/media.ts`) composes a `MediaProtocol` kind with `Endpoint` and `Auth` and owns the transport plumbing: `http` option merging, URL/query rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol. `MediaProtocol.inline` (`src/route/media-protocol.ts`) is `body.from(request)` plus `response.decode(response, context)`; use `MediaProtocol.decodeJson` / `text` / `bytes` so decode failures retain the raw body and HTTP context. `Generation` (`src/generation.ts`) is the provider-neutral handle for a queued generation over a `GenerationRoute` (`status`, `result`, `cancel`, `pollHint`); the first video route implements it. Image protocol files follow the same section order as LLM protocols and declare unsupported common fields once through `MediaInput.rejectUnsupported`.
### URL Construction
`Endpoint` owns `{ baseURL, path, query }`. Each protocol route includes a canonical endpoint when the provider has one (e.g. `https://api.openai.com/v1`); provider helpers override endpoint fields by configuring the route before selecting a model. Generic OpenAI-compatible routes have no canonical URL and require configuration before execution.
@@ -106,12 +94,11 @@ For providers where the URL is derived from typed inputs (Azure resource name, B
### Provider Facades
Provider-facing APIs are configured facades over route values. Endpoint/auth/resource/API-version setup happens before model selection, and model selectors accept only a model or deployment id. Media models use per-modality selectors on the same facade (`openai.image(id)`, later `.video` / `.speech` / `.transcription`) that mirror `openai.responses(id)`; the one-word overlap with the request namespace is accepted over a second construction path:
Provider-facing APIs are configured facades over route values. Endpoint/auth/resource/API-version setup happens before model selection, and model selectors accept only a model or deployment id:
```ts
const openai = OpenAI.configure({ apiKey, baseURL })
const model = openai.responses("gpt-4o-mini")
const image = openai.image("gpt-image-2")
const azure = Azure.configure({ resourceName, apiKey, apiVersion: "v1" })
const deployment = azure.responses("my-deployment")
+54 -154
View File
@@ -8,10 +8,10 @@ import { LLM, LLMClient } from "@opencode/ai"
import { RequestExecutor } from "@opencode/ai/route"
import { OpenAI } from "@opencode/ai/providers"
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
const model = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY }).responses("gpt-4o-mini")
const request = LLM.request({
model: openai.responses("gpt-4o-mini"), // `.chat(...)` selects the Chat Completions API instead
model,
system: "You are concise.",
prompt: "Say hello in one short sentence.",
generation: { maxTokens: 40 },
@@ -29,94 +29,6 @@ await Effect.runPromise(program.pipe(Effect.provide(llmLayer)))
Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
The same configured facade names image models. `Image.request` resolves the provider's image route from the ref and
returns `Media.Asset`s with lazily decoded bytes:
```ts
import { NodeFileSystem } from "@effect/platform-node"
import { Image, ImageClient, Media } from "@opencode/ai"
const image = Effect.gen(function* () {
const response = yield* Image.generate({
model: openai.image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
size: "1024x1024",
providerOptions: { quality: "high" }, // typed per image model
})
yield* Media.write(response.image, "./garden.png")
})
// `asset.bytes()` / `Media.write` also need the executor, so merge it into the environment instead of hiding it.
const imageLayer = ImageClient.layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer))
await Effect.runPromise(image.pipe(Effect.provide(imageLayer), Effect.provide(NodeFileSystem.layer)))
```
Prefer promises? `@opencode/ai/promise` exposes the same LLM and image APIs over one managed runtime:
```ts
import { AI } from "@opencode/ai/promise"
const ai = AI.make()
const text = await ai.llm.generate({ model: openai.responses("gpt-4o-mini"), prompt: "Say hello." })
const generated = await ai.image.generate({ model: openai.image("gpt-image-2"), prompt: "A lighthouse" })
for await (const event of ai.llm.stream({ model: openai.responses("gpt-4o-mini"), prompt: "Stream hello." })) {
// LLMEvent
}
await ai.dispose()
```
## Experimental evaluation
Evaluation models compare shared state with typed choice, score, and boolean questions. The API is
isolated under an experimental entrypoint and provider namespace while the contract evolves:
```ts
import { Effect } from "effect"
import { Evaluation, EvaluationClient } from "@opencode/ai/experimental"
import { TypeSafeAI } from "@opencode/ai/providers"
const model = TypeSafeAI.configure().experimental.evaluation("jev-latest")
const program = Evaluation.run({
model,
state: "I was charged twice. Please refund the duplicate payment.",
questions: {
department: {
type: "choice",
instructions: "Which team should handle this?",
criteria: { billing: "Payments and refunds", technical: "Bugs and outages" },
},
urgency: {
type: "score",
instructions: "How urgent is this?",
criteria: ["Can wait", "Needs prompt attention", "Blocking revenue"],
},
refund: { type: "boolean", instructions: "Is the customer asking for a refund?" },
},
})
const response = await Effect.runPromise(program.pipe(Effect.provide(EvaluationClient.fetchLayer)))
console.log(response.answers.department.choice)
console.log(response.answers.refund.probability)
```
`TypeSafeAI` reads `TYPESAFE_API_KEY`. `OpenCodeZen` exposes the same selector and reads
`OPENCODE_API_KEY`. OpenRouter and Vercel AI Gateway use the same provider shape:
```ts
import { OpenRouter, VercelAIGateway } from "@opencode/ai/providers"
OpenRouter.configure().experimental.evaluation("typesafe/jev-1.13")
VercelAIGateway.configure().experimental.evaluation("typesafe-ai/jev")
```
OpenRouter reads `OPENROUTER_API_KEY`. Vercel reads `AI_GATEWAY_API_KEY`, then `VERCEL_OIDC_TOKEN`.
The common API uses `boolean`; System One routes lower it to native `noul`.
Choice and score confidence plus score legends remain available in provider metadata, and the
provider's rounded probabilities are returned unchanged.
## Alibaba Cloud Model Studio
`Alibaba` provides standard Model Studio inference. Configure a region explicitly, then select
@@ -402,25 +314,23 @@ citations or separate result blocks. Retain `response.message` for either API's
Use `Image.generate` for one-off generation or editing:
```ts
import { Image, Media } from "@opencode/ai"
import { Image, ImageInput } from "@opencode/ai"
const generation = Image.generate({
model: meta("muse-image-1.0"),
model: meta.image("muse-image-1.0"),
prompt: "A flat black square on a white background.",
n: 1,
providerOptions: { reasoningStrength: "low" },
options: { n: 1, reasoningStrength: "low" },
})
const edit = Image.generate({
model: meta("muse-image-1.0"),
model: meta.image("muse-image-1.0"),
prompt: "Make the square purple.",
images: [Media.bytes(imageBytes, "image/webp")],
format: "png",
providerOptions: { reasoningStrength: "low" },
images: [ImageInput.bytes(imageBytes, "image/webp")],
options: { outputFormat: "png", reasoningStrength: "low" },
})
```
The default image format is WEBP; `format` also accepts PNG/JPEG and `responseFormat: "url"`
The default image format is WEBP; `outputFormat` also accepts PNG/JPEG and `responseFormat: "url"`
returns a signed URL. `size` is an aspect-ratio hint. For conversational images, select
`meta.responses("muse-image-1.0")` with `tools: [Meta.imageGeneration({ reasoningStrength: "low" })]`.
Generated images are provider-executed tool results with file content. Retain `response.message` to
@@ -431,40 +341,29 @@ Meta Responses is explicitly HTTP/SSE-only and does not use WebSockets, even whe
## Image generation
Use `Image.generate` with an image model for direct asset generation. `Image.request` mirrors `LLM.request`: the
model comes from the facade's `.image(...)` selector (mirroring `.responses(...)`), common fields
(`images`, `mask`, `n`, `size`, `aspectRatio`, `seed`, `format`) lower natively or fail typed, and
`providerOptions` is inferred from the selected model:
Use `Image.generate` with an image model for direct asset generation:
```ts
import { Image, Media } from "@opencode/ai"
import { Image, ImageInput } from "@opencode/ai"
import { OpenAI } from "@opencode/ai/providers"
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
const program = Effect.gen(function* () {
const response = yield* Image.generate({
model: openai.image("gpt-image-2"),
model: OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY }).image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
n: 2,
size: "1024x1024",
format: "webp",
providerOptions: {
options: {
n: 2,
size: "1024x1024",
quality: "high", // inferred from the OpenAI image model
outputFormat: "webp",
future_option: true, // unknown native options pass through unchanged
},
})
return response.images // Media.Asset[] with owned bytes or a provider URL
return response.images // GeneratedImage[] with owned bytes or a provider URL
})
```
`Media.Asset` is the one asset type shared by image requests, image responses, LLM messages, and tool results.
`asset.source` is the serializable `Media.Source` (`bytes`, `base64`, `url`, or `ref`); `asset.bytes()`,
`asset.base64()`, and `asset.dataUrl()` decode or download lazily and cache; `asset.materialize()` pulls a `url`
asset into owned bytes before the provider URL expires. Construct assets with `Media.bytes`, `Media.base64`,
`Media.url`, `Media.ref(provider, id)`, `Media.fromDataUrl`, or `Media.file(path)`.
Pass ordered image inputs to the same method for editing, composition, or image-conditioned generation:
```ts
@@ -474,45 +373,49 @@ const response =
model,
prompt: "Combine these product photos into one studio scene",
images: [
Media.bytes(firstBytes, "image/png"),
Media.url("https://example.com/second.webp"),
Media.ref("openai", "file_123"),
ImageInput.bytes(firstBytes, "image/png"),
ImageInput.url("https://example.com/second.webp"),
ImageInput.file("file_123"),
],
providerOptions,
options,
http,
})
```
`Media.ref(provider, id)` represents provider file handles such as OpenAI file IDs or Gemini Files URIs; routes
only forward refs that belong to their own provider. Raw strings are not accepted as image inputs, avoiding
ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a
non-empty array selects the provider's edit behavior without enforcing provider image-count limits locally. OpenAI
uses multipart for byte/data-URL edits and its JSON reference body for URL or file-ID edits. The common `mask`
field selects inpainting; routes that cannot honor it fail with `UnsupportedOperation`:
`ImageInput.fileUri(uri, mediaType)` represents provider file URIs such as Gemini Files. Raw strings are not
accepted as image inputs, avoiding ambiguity between base64, URLs, and provider IDs. Empty or omitted `images`
uses text-to-image generation; a non-empty array selects the provider's edit behavior without enforcing provider
image-count limits locally. `images` is the only common image-editing field. OpenAI uses multipart for byte/data-URL
edits and its JSON reference body for URL or file-ID edits. Its provider-specific `options.mask` accepts an
`ImageInput` for inpainting:
```ts
yield *
Image.generate({
model: openai.image("gpt-image-2"),
model: OpenAI.configure({ apiKey }).image("gpt-image-2"),
prompt,
images: [Media.bytes(sourceBytes, "image/png")],
mask: Media.bytes(maskBytes, "image/png"),
images: [ImageInput.bytes(sourceBytes, "image/png")],
options: { mask: ImageInput.bytes(maskBytes, "image/png") },
})
```
On multipart requests, `http.body` can override option fields but not structural `model`, `prompt`, `image[]`,
or `mask` fields, and the transport owns the multipart `Content-Type` boundary. For JSON requests, `http.body`
remains the final raw-native overlay. Gemini does not fetch public HTTP URLs, and hosted Z.ai image generation does
not accept image inputs. These cases fail with a typed `AIError` before network I/O.
The OpenAI adapter extracts this helper value into the edit request's native `mask` field rather than passing the
tagged `ImageInput` object through as an ordinary option. On multipart requests, `http.body` can override option
fields but not structural `model`, `prompt`, `image[]`, or `mask` fields, and the transport owns the multipart
`Content-Type` boundary. For JSON requests, `http.body` remains the final raw-native overlay. Gemini does not fetch
public HTTP URLs, and hosted Z.ai image generation does not accept image inputs. These cases fail with
`InvalidRequest` before network I/O.
Provider-native image options belong to each request. Raw `http.body` fields have final precedence over them:
```ts
const model = OpenAI.configure({ apiKey }).image("gpt-image-2")
yield *
Image.generate({
model: openai.image("gpt-image-2"),
model,
prompt,
providerOptions: { quality: "medium" },
options: { quality: "medium" },
http,
})
```
@@ -522,11 +425,11 @@ xAI image models use the same request API with xAI-native controls:
```ts
yield *
Image.generate({
model: XAI.configure({ apiKey })("any-model-id"),
model: XAI.configure({ apiKey }).image("any-model-id"),
prompt,
n: 2,
aspectRatio: "16:9",
providerOptions: {
options: {
n: 2,
aspectRatio: "16:9",
resolution: "1k",
responseFormat: "b64_json",
future_option: true,
@@ -542,12 +445,12 @@ import { Google } from "@opencode/ai/providers"
const googleProgram = Effect.gen(function* () {
const response = yield* Image.generate({
model: Google.configure({ apiKey })("any-model-id"),
model: Google.configure({ apiKey }).image("any-model-id"),
prompt: "A robot tending a rooftop garden",
aspectRatio: "16:9",
seed: 42,
providerOptions: {
options: {
aspectRatio: "16:9",
imageSize: "2K",
seed: 42,
thinkingLevel: "HIGH",
includeThoughts: true,
futureOption: true,
@@ -569,9 +472,9 @@ Z.ai image models infer open Z.ai-native options from the selected model:
```ts
yield *
Image.generate({
model: ZAI.configure({ apiKey })("any-model-id"),
model: ZAI.configure({ apiKey }).image("any-model-id"),
prompt,
providerOptions: {
options: {
quality: "hd",
userID: "user-123",
future_option: true,
@@ -581,8 +484,8 @@ yield *
```
Z.ai does not include trustworthy MIME metadata for output URLs, so generated images use
`application/octet-stream` until materialized. Output URLs expire after 30 days; call `asset.materialize()` and
persist the bytes promptly if they must remain available.
`application/octet-stream`. Output URLs expire after 30 days; download and persist them promptly if they must
remain available.
Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool:
@@ -600,7 +503,7 @@ const program = Effect.gen(function* () {
})
```
The hosted result is represented as a provider-executed tool call and tool result, and the generated image is also emitted as a first-class `media` `LLMEvent` (`response.message` then carries a `media` part). Gemini image-capable models emit the same `media` event for inline image output. Retaining `response.message` preserves the generated image for continuation on both routes.
The hosted result is represented as a provider-executed tool call and tool result. Its image is a `file` content item with a data URI, so retaining `response.message` preserves the generated image for continuation.
## Public API
@@ -609,11 +512,8 @@ The hosted result is represented as a provider-executed tool call and tool resul
- **`Message.user(...)` / `Message.assistant(...)` / `Message.tool(...)`** — message constructors from the canonical schema model.
- **`LanguageModel.make(...)` / `ToolCallPart.make(...)` / `ToolResultPart.make(...)` / `ToolDefinition.make(...)`** — model and tool-related constructors from the canonical schema model.
- **`LLMEvent.is.*`** — typed guards (`is.textDelta`, `is.toolCall`, `is.finish`, …) for filtering streams.
- **`Image.request` / `Image.generate` / `Image.stream`** — generate images through a provider-neutral image request and response model.
- **`Image.generate({...})`** — generate images through a provider-neutral image request and response model.
- **`ImageClient`** — Effect service and layer for image execution, parallel to `LLMClient`.
- **`Media`** — the shared asset type (`Media.Asset`, `Media.Source`) and constructors used by messages, tool results, and media requests.
- **`Generation`** — provider-neutral handle for an in-flight media generation (`await`, `refresh`, `cancel`, `events`) used by queued media routes.
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm` and `image` as Promise / `AsyncIterable` APIs.
## Testing
-328
View File
@@ -1,328 +0,0 @@
# Media generation in `@opencode/ai` — public API direction
Status: proposal. Branch `media-support`.
## Goal
`@opencode/ai` becomes the one package you reach for to generate anything: text, images, video, speech, transcripts, and later music and realtime. The LLM surface already exists and is shaped by three constraints: Effect-first, used by OpenCode Core, usable externally. Media has a different priority order: **external DX first**, Effect and Promise as peers, Core as one consumer among many.
The design below is derived from a survey of the raw provider APIs (OpenAI, Gemini/Veo/Imagen, xAI, Stability, BFL, fal, Replicate, Runway, Luma, Kling, MiniMax, ElevenLabs, Deepgram, Cartesia, AssemblyAI, Lyria) and of existing multi-provider SDKs.
## What the survey forces
1. **Three execution shapes, everywhere.** Inline sync (OpenAI images, all TTS, Gemini), async job with polling or webhook (every video provider, BFL, fal, Replicate, AssemblyAI), and bidirectional streams (ElevenLabs/Cartesia/Deepgram WS, realtime). Video has no sync provider at all.
2. **Output is never just bytes.** base64, signed URLs with TTLs from 10 minutes (BFL) to 2 days (Veo), URLs that need auth plus redirect (Veo), separate download endpoints (Sora `/content?variant=`), raw bodies (Stability, TTS). Multi-output is the norm.
3. **Inputs have roles.** First/last frame, mask, style/subject reference, source video for edit/extend, reference audio, prior generation id, provider-side file handles (`file_id`, `gs://`, `runway://`, `mm_file://`).
4. **Partial streaming is modality-specific.** Images: a few whole partial frames. Audio: ordered chunks plus timestamp events. Jobs: status/progress/logs. Video: none.
5. **Usage is a union**: tokens, seconds, characters (often only in headers), credits, compute time.
6. **Moderation can be partial success** (Veo strips audio but returns video). Deprecations are constant (Sora API shuts down 2026-09-24, Imagen on Gemini API 2026-08-17).
## Where existing SDKs are weak and we should not be
- No streaming TTS.
- Video handles are experimental start/status pairs; the polling loop lives inside the generate call.
- Unsupported inputs become silent warnings arrays, so a request can succeed while dropping your mask.
- `n` is fanned out into hidden parallel calls, which obscures cost and idempotency.
- Each modality has its own bespoke result type; the file abstraction is a lazy base64/bytes pair with no URL, expiry, or provider ref.
- Effect's own `unstable/ai` has no media generation. Nothing in the Effect ecosystem owns this.
## Design principles
- **Same shape as LLM.** `X.request(...)` → Schema class; `X.generate(request)` / `X.stream(request)`; `XClient.Service` + `layer`; typed `AIError`. If you know `LLM`, you know `Video`.
- **Execution shape is route policy, not API shape.** `Image.generate` returns an image whether the provider is inline or queued. Job control is available uniformly when you want it.
- **Errors, not warnings.** Unsupported common fields fail at the protocol boundary with a typed `AIError`, as the LLM routes do today. Provider-side partial results (filtered audio, moderated sample) surface as `notices` on the response, never as silent drops.
- **One asset type in, one asset type out**, shared with LLM messages and tool results.
- **Typed per-model options**, no hidden fan-out, no implicit retries that spend money.
- **Promise API is one mechanism for the whole package**, not a media-only wrapper.
- **One construction path per model.** Media models come from per-modality selectors on the configured facade (`openai.image("gpt-image-2")`), the same shape as `openai.responses("gpt-5")`.
## Public API
### Model selection
A model value is built as `OpenAI.configure({ apiKey }).responses("gpt-5")` or `.image("gpt-image-2")`: `configure` fixes credentials, endpoint, and defaults; the selector fixes which of the provider's APIs to hit and binds the typed `providerOptions` generic. Media follows the same shape with one selector per modality — `openai.image(id)` today, `.video(id)` / `.speech(id)` / `.transcription(id)` as those modalities land — mirroring `openai.responses(id)`. `Image.request` accepts `ImageModel` only, exactly as `LLM.request` accepts `LanguageModel`.
```ts
import { OpenAI, Google } from "@opencode/ai/providers"
const openai = OpenAI.configure({ apiKey }) // OpenAI(...) alone uses env auth (OPENAI_API_KEY)
LLM.request({ model: openai.responses("gpt-5"), prompt })
Image.request({ model: openai.image("gpt-image-2"), prompt })
Video.request({ model: google.video("veo-3.1-generate-preview"), prompt })
Speech.request({ model: openai.speech("gpt-4o-mini-tts"), text })
Transcription.request({ model: openai.transcription("gpt-4o-transcribe"), audio })
```
The request namespace and the selector share one word (`Image.request` + `.image(...)`). That redundancy is accepted: a callable facade returning a lazily resolved ref would be a second way to construct the same model, and the type machinery to infer `providerOptions` through it is not worth one word. Where a provider has two APIs for one modality, the selectors stay explicit (`openai.chat`, a future `google.imagen`), and one default per modality per provider is part of the facade definition (OpenAI image → Images API, Google image → Gemini-native since Imagen on the Gemini API shuts down 2026-08-17). Provider package entrypoints keep `model(modelID, settings)` per modality-specific path, e.g. `@opencode/ai/providers/openai/responses`.
### `Media` — the asset type
Replaces `MediaPart.data: string | Uint8Array`, `ImageInput`, `GeneratedImage`, and aligns `Tool.FileContent`.
```ts
import { Media } from "@opencode/ai"
Media.Source =
| { type: "bytes"; data: Uint8Array; mediaType: string }
| { type: "base64"; data: string; mediaType: string }
| { type: "url"; url: string; mediaType?: string; expiresAt?: number; headers?: Record<string, string> }
| { type: "ref"; provider: ProviderID; id: string; mediaType?: string } // file_id, gs://, runway://, prior generation
class Media.Asset {
readonly source: Media.Source
readonly mediaType: string // always resolved (sniffed when the provider omits it)
readonly kind: "image" | "video" | "audio" | "document" | "other"
readonly info?: { width?; height?; durationSeconds?; sampleRate?; channels?; encoding?; format? }
readonly expiresAt?: number
readonly providerMetadata?: ProviderMetadata
bytes(): Effect<Uint8Array, AIError, RequestExecutor.Service> // downloads/decodes lazily, cached
base64(): Effect<string, AIError, RequestExecutor.Service>
dataUrl(): Effect<string, AIError, RequestExecutor.Service>
materialize(): Effect<Media.Asset, AIError, RequestExecutor.Service> // url/ref → bytes, before the URL dies
}
Media.bytes(data, mediaType?) Media.base64(data, mediaType?)
Media.url(url, options?) Media.ref(provider, id)
Media.file(path) // Bun/Node: reads + sniffs; Effect FileSystem variant for layers
Media.write(asset, path) // convenience, uses FileSystem
```
Raw-PCM outputs (Gemini TTS, Cartesia raw, Deepgram WS) carry `info.encoding/sampleRate/channels` because there is no container header.
### Modality namespaces
Each namespace mirrors `LLM` exactly.
```ts
import { Image, Video, Speech, Transcription } from "@opencode/ai"
import { OpenAI, Google, ElevenLabs, Fal } from "@opencode/ai/providers"
```
#### Image
```ts
const request = Image.request({
model: openai.image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
images: [Media.file("./ref.png")], // references / edit sources
mask: Media.file("./mask.png"),
n: 2,
size: "1536x1024", // or aspectRatio: "3:2"
seed: 7,
format: "webp",
providerOptions: { quality: "high", background: "transparent" }, // typed per model
})
const response = yield* Image.generate(request) // ImageResponse
response.image // Media.Asset (first)
response.images // Media.Asset[]
response.usage // Usage union (see below)
response.notices // moderation / partial-result notices
yield* Image.stream(request) // Stream<ImageEvent>
// ImageEvent: generation-queued | generation-progress | image-partial { index, image } | image { index, image } | finish { usage }
```
Editing is not a separate function; `images`/`mask` on the request select the edit path in the route (OpenAI `/images/edits`, Gemini multimodal parts, xAI `/images/edits`). Routes that cannot honor `mask` fail with `Unsupported`.
#### Video
```ts
const request = Video.request({
model: google.video("veo-3.1-generate-preview"),
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
frames: { first: Media.file("./start.png"), last: Media.file("./end.png") },
references: [Media.url("https://…/style.png")],
video: Media.ref("openai", "video_123"), // edit / extend / remix source
durationSeconds: 8,
aspectRatio: "16:9",
resolution: "1080p",
audio: true,
n: 1,
providerOptions: { personGeneration: "dont_allow", negativePrompt: "text, watermark" },
})
// Simple: wait for it.
const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } })
response.video // Media.Asset (url with expiresAt, or bytes when the route downloads)
yield* response.video.materialize() // pull bytes before the URL expires
// Explicit generation control.
const generation = yield* Video.start(request) // Generation<VideoResponse>
generation.id; generation.status; generation.progress; generation.token // token is serializable JSON
yield* generation.await({ poll }) // VideoResponse
yield* generation.cancel()
// Resume from another process.
const resumed = yield* Video.resume(model, token) // Generation<VideoResponse>
// Progress as a stream.
yield* Video.stream(request) // Stream<VideoEvent>: generation-queued { position } | generation-progress { progress, logs } | video { index, video } | finish
```
Webhooks: `Video.complete(model, token, webhook)` finishes a generation from a webhook payload without polling. Token shape is route-owned and opaque (Veo operation name, fal `response_url`, Runway task id).
#### Speech (TTS)
```ts
const request = Speech.request({
model: elevenlabs.speech("eleven_v3"),
text: "Hello from OpenCode.",
voice: "JBFqnCBsd6RMkjVDRZzb", // name, uuid, or { id } — provider-normalized
format: "mp3", // mp3 | wav | pcm | opus | aac | flac | (string & {})
speed: 1.0,
language: "en",
instructions: "Warm, unhurried.",
providerOptions: { stability: 0.5 },
})
const response = yield* Speech.generate(request) // SpeechResponse: audio: Media.Asset, timestamps?, usage
yield* Speech.stream(request) // Stream<SpeechEvent>: audio-delta { chunk } | timestamps { words } | finish
```
Streaming TTS is first-class on day one: OpenAI `stream_format: sse`, ElevenLabs `/stream`, Cartesia SSE, Deepgram chunked. Input-streaming TTS (WS, text arrives incrementally) is a later `Speech.session(...)` scoped resource, not part of `generate`.
#### Transcription (STT)
```ts
const request = Transcription.request({
model: openai.transcription("gpt-4o-transcribe"),
audio: Media.file("./call.wav"),
language: "en",
prompt: "Names: Shoubhit, OpenCode.",
timestamps: "word", // none | segment | word
diarize: true,
providerOptions: { chunkingStrategy: "auto" },
})
const response = yield* Transcription.generate(request)
response.text; response.segments; response.words; response.language; response.durationSeconds
yield* Transcription.stream(request) // Stream<TranscriptionEvent>: text-delta | segment | finish
```
Realtime STT over WebSocket is the same future `session` shape as input-streaming TTS.
### `Generation` — shared async execution
```ts
class Generation<Response> {
readonly id: string
readonly model: MediaModel
readonly token: unknown // route-owned serializable JSON
readonly status: "queued" | "running" | "completed" | "failed" | "cancelled" | "expired"
readonly progress?: number // 0..1, normalized
readonly position?: number
readonly expiresAt?: number
refresh(): Effect<Generation<Response>, AIError>
await(options?: { poll?: Poll }): Effect<Response, AIError>
cancel(): Effect<void, AIError>
events(options?): Stream<GenerationEvent, AIError>
}
Poll = { interval?: Duration; timeout?: Duration; schedule?: Schedule } // route may override from provider hints (`openai-poll-after-ms`)
```
`Generation` is not video-specific. Image routes on BFL, fal, and Replicate are queued; `Image.start` exists for them. A route declares itself `inline` or `queued`; `generate` on a queued route is `start` then `await`.
### Usage
```ts
Usage =
| { type: "tokens"; input; output; total; details? }
| { type: "seconds"; seconds }
| { type: "characters"; characters }
| { type: "credits"; credits }
| { type: "compute"; seconds }
```
Header-only usage (ElevenLabs `character-cost`, Deepgram `dg-char-count`) is lifted into `usage` by the route.
### Promise API — `@opencode/ai/promise`
Mirrors the `packages/plugin/src/effect` and `packages/plugin/src/promise` split that already exists in this repo. One mechanism for LLM and media.
```ts
import { AI } from "@opencode/ai/promise"
const ai = AI.make() // ManagedRuntime over RequestExecutor.fetchLayer + all clients
// AI.make({ layer }) to inject a custom executor / recorder / middleware
const image = await ai.image.generate({ model, prompt })
await image.image.bytes()
for await (const event of ai.speech.stream({ model, text, voice })) { }
const generation = await ai.video.start({ model, prompt })
const video = await generation.await({ poll: { interval: 10_000 }, signal })
const resumed = ai.video.resume(model, JSON.parse(saved))
const text = await ai.llm.generate({ model, prompt }) // closes today's gap: LLM has no promise API either
for await (const event of ai.llm.stream(request)) { }
await ai.dispose()
```
Streams become `AsyncIterable` via `Stream.toAsyncIterable`. `AIError` is thrown as-is. `AbortSignal` maps to interruption. Nothing in `src/*` except this entrypoint knows about promises.
### Providers
Existing facades gain per-modality selectors; the modality routes each facade provides:
| Facade | llm | image | video | speech | transcription | other |
|---|---|---|---|---|---|---|
| `OpenAI` | responses (default), chat | Images API | Sora (deprecated 2026-09-24) | ✓ | ✓ | |
| `Google` | Gemini | Gemini-native (default), `imagen` | Veo | Gemini TTS | Gemini transcribe | |
| `XAI` | ✓ | ✓ | ✓ | | | |
| `ElevenLabs` | | | | ✓ | Scribe | soundEffect, music |
| `Fal` | | ✓ | ✓ | | | |
| `Replicate`, `Runway`, `Luma`, `Kling`, `MiniMax`, `Deepgram`, `Cartesia`, `AssemblyAI`, `BlackForestLabs`, `Stability` | | per provider | | | | |
New facades follow the existing one-file-per-provider rule. Package entrypoints are modality-specific, such as `@opencode/ai/providers/openai/images`, and return the concrete model.
`ImageModel<Options>` already gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, `TranscriptionModel` follow the same generic. A shared `MediaModel` union is what `Generation` and the promise client key on.
### Routes and protocols
Media does not fit the LLM four-axis route (SSE frames → event state machine) except for streaming TTS/STT. Reuse `Endpoint`, `Auth`, `Framing`, `RequestExecutor`, and add media protocol kinds:
- `MediaProtocol.inline``body.from(request)` (JSON, multipart, or query), `response.decode(response)` (JSON, or binary body → `Media.Asset`).
- `MediaProtocol.queued``start`, `status`, `result`, `cancel`, optional `download`, `pollHint`, `token` schema.
- `MediaProtocol.stream` — framing + `step` state machine emitting modality events, same discipline as LLM protocols.
`Route.make` for media composes one protocol kind with endpoint/auth. The existing `ImageRoute { generate(request, execute) }` is the ad-hoc version of `inline` and gets folded in.
### LLM integration
- `MediaPart` becomes `{ type: "media"; media: Media.Asset; … }` so protocols branch on `kind` and can pass `url`/`ref` sources through natively (OpenAI `image_url`, Gemini `fileData`).
- New `LLMEvent`s: `media { media: Media.Asset }` so Gemini inline image output is first-class instead of dropped. OpenAI Responses `image_generation_call` keeps its single carrier — the provider-executed `tool-result` with `file` content — because Core consumes hosted tool-result content today and has no `media` event handling yet; it switches to the `media` carrier when Core adopts the event, so the image is never emitted twice.
- `Message.assistant([...])` accepts media parts; Gemini multi-turn image editing replays them.
- `Tool.FileContent` aligns with `Media.Source`.
## Decisions
All settled:
1. **Per-modality selectors** (`openai.image(id)`, `.video`, `.speech`, `.transcription`) name media models, mirroring `openai.responses(id)`. The one-word overlap with the request namespace is accepted over a callable-facade `ModelRef` as a second construction path.
2. **`providerOptions` everywhere** (rename current `Image.options`) for consistency with LLM.
3. **No hidden `n` fan-out.** `n` lowers natively; routes that cannot do `n > 1` fail typed. Callers use `Effect.all` / `Promise.all` explicitly.
4. **Errors over warnings** for unsupported common fields; `notices` for provider-side partial results only.
5. **`Media.Asset` is a class** (lazy bytes, cached) with `Media.Source` as the serializable Schema for wire/persistence. `Asset.from(source)` / `asset.source` round-trip losslessly. Same pattern as `LanguageModel` today.
6. **Promise entrypoint**: `@opencode/ai/promise` exporting `AI.make(options?: { layer? })` plus a module-level default `ai` for scripts, covering LLM too.
7. **Modality set for v1**: `Image`, `Video`, `Speech`, `Transcription`. `Music`/`SoundEffect` and `session` (bidirectional WS, realtime) are designed-for but deferred.
8. **Sora is skipped** (API shuts down 2026-09-24). Video launches with Veo, xAI, fal, Runway.
## Build order
Foundation + Image ship together as the reference implementation, serially. Video, Speech, and Transcription then proceed in parallel on separate branches. Image jobs and partial streaming come last, after Video has hardened `Generation`.
## Phasing
1. **Foundation** — per-modality selectors, `Media`, `Generation`, `Poll`, `Usage` union, `MediaProtocol` kinds, `@opencode/ai/promise` with `llm` + `image`. Port the five existing image protocols onto it. Unify `MediaPart` and add the `media` LLM event (fixes Gemini image output being dropped).
2. **Video** — Veo, xAI, fal, Runway first. Then Luma, Kling, MiniMax, Replicate.
3. **Speech + Transcription** — OpenAI, ElevenLabs, Gemini TTS, Deepgram, Cartesia, AssemblyAI. Streaming TTS from the start.
4. **Image queued routes and partials** — BFL, fal, Replicate, Stability; OpenAI `partial_images` streaming.
5. **Later** — ElevenLabs music/SFX, Lyria, `Speech.session` / `Transcription.session`, realtime.
Core adoption (session attachments beyond png/jpeg/gif/webp/pdf, image-generation tool, TUI rendering) comes after phase 1 and is a Core concern.
+6 -42
View File
@@ -1,17 +1,5 @@
import { Config, Effect, Formatter, Layer, Schema, Stream } from "effect"
import { NodeFileSystem } from "@effect/platform-node"
import {
Image,
ImageClient,
LLM,
LLMClient,
LLMRequest,
Media,
Message,
ProviderID,
Tool,
ToolRuntime,
} from "@opencode/ai"
import { LLM, LLMClient, LLMRequest, Message, ProviderID, Tool, ToolRuntime } from "@opencode/ai"
import { Route, Auth, Endpoint, Framing, Protocol, RequestExecutor } from "@opencode/ai/route"
import { OpenAI } from "@opencode/ai/providers"
@@ -28,18 +16,15 @@ import { OpenAI } from "@opencode/ai/providers"
const apiKey = Config.redacted("OPENAI_API_KEY")
// 1. Configure a provider. The configured facade records provider identity,
// deployment options, authentication, and defaults. Per-modality selectors pick
// the API: `.responses(...)` / `.chat(...)` for LLM calls and `.image(...)` for
// image generation.
const openai = OpenAI.configure({
// 1. Pick a model. The provider helper records provider identity, protocol
// choice, capabilities, deployment options, authentication, and defaults.
const model = OpenAI.configure({
apiKey,
generation: { maxTokens: 160 },
providerOptions: {
store: false,
},
})
const model = openai.responses("gpt-4o-mini")
}).model("gpt-4o-mini")
// 2. Build a provider-neutral request. This is useful when reusing one request
// across generate and stream examples.
@@ -224,39 +209,18 @@ const FakeEcho = {
}),
}
// 8. Image generation uses the same facade and the same request/generate shape.
// `response.image` is a `Media.Asset`: bytes decode lazily and are cached, and
// `Media.write` persists them through the Effect `FileSystem`.
const generateImage = Effect.gen(function* () {
const response = yield* Image.generate({
model: openai.image("gpt-image-1-mini"),
prompt: "A flat black circle centered on a plain white background.",
size: "1024x1024",
format: "jpeg",
providerOptions: { quality: "low" },
})
console.log("\n== image ==")
console.log("media type:", response.image.mediaType)
console.log("bytes:", (yield* response.image.bytes()).byteLength)
console.log("usage", Formatter.formatJson(response.usage, { space: 2 }))
yield* Media.write(response.image, "tutorial-image.jpg").pipe(Effect.provide(NodeFileSystem.layer))
})
// Provide the LLM runtime and the HTTP request executor once. Keep one path
// enabled at a time so the tutorial can demonstrate generate, stream, or
// tool-loop behavior without spending tokens on every example.
const requestExecutorLayer = RequestExecutor.fetchLayer
const llmClientLayer = LLMClient.layer.pipe(Layer.provide(requestExecutorLayer))
const imageClientLayer = ImageClient.layer.pipe(Layer.provide(requestExecutorLayer))
const program = Effect.gen(function* () {
// yield* generateOnce
// yield* streamText
// yield* generateStructuredObject
// yield* generateDynamicObject.pipe(Effect.andThen((response) => Effect.sync(() => console.log(response.object))))
// yield* generateImage
yield* streamWithTools
}).pipe(Effect.provide(Layer.mergeAll(requestExecutorLayer, llmClientLayer, imageClientLayer)))
}).pipe(Effect.provide(Layer.mergeAll(requestExecutorLayer, llmClientLayer)))
Effect.runPromise(program)
+1 -2
View File
@@ -1,6 +1,6 @@
{
"$schema": "https://json.schemastore.org/package.json",
"version": "2.0.14",
"version": "2.0.12",
"name": "@opencode/ai",
"type": "module",
"license": "MIT",
@@ -15,7 +15,6 @@
],
"exports": {
".": "./src/index.ts",
"./promise": "./src/promise.ts",
"./testing": "./src/testing.ts",
"./*": "./src/*.ts"
},
-29
View File
@@ -1,29 +0,0 @@
export { EvaluationClient } from "./experimental/evaluation-client.js"
export {
BooleanAnswer,
BooleanQuestion,
ChoiceAnswer,
ChoiceQuestion,
Evaluation,
EvaluationAnswer,
EvaluationInput,
EvaluationModel,
EvaluationModelSchema,
EvaluationQuestion,
EvaluationRequest,
EvaluationResponse,
EvaluationRounding,
ScoreAnswer,
ScoreQuestion,
} from "./experimental/evaluation.js"
export type {
AnswerFor,
AnswersFor,
EvaluationModelOptions,
EvaluationOptions,
EvaluationQuestions,
EvaluationRequestFor,
EvaluationRequestInput,
EvaluationResponseFor,
EvaluationRoute,
} from "./experimental/evaluation.js"
@@ -1,96 +0,0 @@
import { Context, Effect, Layer } from "effect"
import { RequestExecutor } from "../route/executor.js"
import { AIError, InvalidProviderOutputError, mergeHttpOptions } from "../schema/index.js"
import { sanitizeSurrogates } from "../utils/sanitize.js"
import {
type EvaluationOptions,
type EvaluationQuestions,
type EvaluationRequestFor,
type EvaluationResponseFor,
} from "./evaluation.js"
export type Execute = RequestExecutor.Interface["execute"]
export interface Interface {
readonly evaluate: <Options extends EvaluationOptions, const Questions extends EvaluationQuestions>(
request: EvaluationRequestFor<Options, Questions>,
) => Effect.Effect<EvaluationResponseFor<Questions>, AIError>
}
export class Service extends Context.Service<Service, Interface>()("@opencode/AI/Experimental/EvaluationClient") {}
export const evaluate = <Options extends EvaluationOptions, const Questions extends EvaluationQuestions>(
request: EvaluationRequestFor<Options, Questions>,
): Effect.Effect<EvaluationResponseFor<Questions>, AIError, Service> =>
Effect.flatMap(Service, (client) => client.evaluate(request))
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
Service,
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
return Service.of({
evaluate: (request) =>
request.model.route
.evaluate(
{
...sanitizeSurrogates({
...request,
model: undefined,
http: mergeHttpOptions(request.model.http, request.http),
}),
model: request.model,
},
executor.execute,
)
.pipe(
Effect.flatMap((response) => {
const questions = Object.entries(request.questions)
if (
questions.length === Object.keys(response.answers).length &&
questions.every(([id, question]) => {
const answer = response.answers[id]
if (question.type === "boolean") return answer?.type === "boolean"
if (question.type === "choice") {
if (answer?.type !== "choice" || !Object.hasOwn(question.criteria, answer.choice)) return false
if (answer.probabilities === undefined) return true
const keys = Object.keys(question.criteria)
const probabilities = answer.probabilities
return (
Object.keys(probabilities).length === keys.length &&
keys.every((key) => Object.hasOwn(probabilities, key))
)
}
if (answer?.type !== "score" || answer.score < 0 || answer.score > question.criteria.length - 1)
return false
if (answer.probabilities === undefined) return true
const keys = question.criteria.map((_, index) => String(index))
const probabilities = answer.probabilities
return (
Object.keys(probabilities).length === keys.length &&
keys.every((key) => Object.hasOwn(probabilities, key))
)
})
)
return Effect.succeed(response as EvaluationResponseFor<typeof request.questions>)
return Effect.fail(
new AIError({
reason: new InvalidProviderOutputError({
route: request.model.route.id,
message: "Evaluation answers do not match the requested questions",
cause: response.answers,
}),
}),
)
}),
),
})
}),
)
export const fetchLayer = layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
export const EvaluationClient = {
Service,
layer,
fetchLayer,
evaluate,
} as const
-245
View File
@@ -1,245 +0,0 @@
import { Effect, Schema } from "effect"
import {
AIError,
HttpOptions,
InvalidRequestError,
ModelID,
ProviderID,
ProviderMetadata,
Usage,
} from "../schema/index.js"
import { EvaluationClient, Service, type Execute } from "./evaluation-client.js"
export const EvaluationInput = Schema.Union([Schema.String, Schema.JsonObject, Schema.Array(Schema.Json)])
export type EvaluationInput = Schema.Schema.Type<typeof EvaluationInput>
const EvaluationCriterion = Schema.NullOr(EvaluationInput)
const ChoiceCriteria = Schema.Record(Schema.String, EvaluationCriterion).pipe(
Schema.refine((x): x is typeof x => Object.keys(x).length > 0, {
message: "Choice criteria must be a nonempty option map",
}),
)
export const ChoiceQuestion = Schema.Struct({
type: Schema.Literal("choice"),
instructions: EvaluationInput,
criteria: ChoiceCriteria,
})
export type ChoiceQuestion = Schema.Schema.Type<typeof ChoiceQuestion>
export const ScoreQuestion = Schema.Struct({
type: Schema.Literal("score"),
instructions: EvaluationInput,
criteria: Schema.Array(EvaluationCriterion).check(Schema.isMinLength(2)),
})
export type ScoreQuestion = Schema.Schema.Type<typeof ScoreQuestion>
export const BooleanQuestion = Schema.Struct({
type: Schema.Literal("boolean"),
instructions: EvaluationInput,
criteria: Schema.optional(
Schema.Struct({
true: Schema.optional(EvaluationCriterion),
false: Schema.optional(EvaluationCriterion),
}),
),
})
export type BooleanQuestion = Schema.Schema.Type<typeof BooleanQuestion>
export const EvaluationQuestion = Schema.Union([ChoiceQuestion, ScoreQuestion, BooleanQuestion]).pipe(
Schema.toTaggedUnion("type"),
)
export type EvaluationQuestion = Schema.Schema.Type<typeof EvaluationQuestion>
export type EvaluationQuestions = Readonly<Record<string, EvaluationQuestion>>
const EvaluationQuestions = Schema.Record(Schema.String, EvaluationQuestion).pipe(
Schema.refine((x): x is typeof x => Object.keys(x).length > 0, {
message: "Evaluation questions must be a nonempty map",
}),
)
const Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 }))
export const ChoiceAnswer = Schema.Struct({
type: Schema.Literal("choice"),
choice: Schema.String,
probabilities: Schema.optional(Schema.Record(Schema.String, Probability)),
})
export type ChoiceAnswer = Schema.Schema.Type<typeof ChoiceAnswer>
export const ScoreAnswer = Schema.Struct({
type: Schema.Literal("score"),
score: Schema.Number,
probabilities: Schema.optional(Schema.Record(Schema.String, Probability)),
})
export type ScoreAnswer = Schema.Schema.Type<typeof ScoreAnswer>
export const BooleanAnswer = Schema.Struct({
type: Schema.Literal("boolean"),
probability: Probability,
})
export type BooleanAnswer = Schema.Schema.Type<typeof BooleanAnswer>
export const EvaluationAnswer = Schema.Union([ChoiceAnswer, ScoreAnswer, BooleanAnswer]).pipe(
Schema.toTaggedUnion("type"),
)
export type EvaluationAnswer = Schema.Schema.Type<typeof EvaluationAnswer>
export type AnswerFor<Question extends EvaluationQuestion> = Question extends {
readonly type: "choice"
readonly criteria: infer Criteria
}
? {
readonly type: "choice"
readonly choice: Extract<keyof Criteria, string>
readonly probabilities?: Readonly<Record<Extract<keyof Criteria, string>, number>>
}
: Question extends { readonly type: "score" }
? ScoreAnswer
: BooleanAnswer
export type AnswersFor<Questions extends EvaluationQuestions> = {
readonly [ID in keyof Questions]: AnswerFor<Questions[ID]>
}
export type EvaluationOptions = Record<string, unknown>
export interface EvaluationRoute<Options extends EvaluationOptions = EvaluationOptions> {
readonly id: string
readonly evaluate: (
request: EvaluationRequestFor<Options>,
execute: Execute,
) => Effect.Effect<EvaluationResponse, AIError>
}
export class EvaluationModel<Options extends EvaluationOptions = EvaluationOptions> {
declare protected readonly _Options: (options: Options) => Options
readonly id: ModelID
readonly provider: ProviderID
readonly route: EvaluationRoute<Options>
readonly http?: HttpOptions
constructor(input: EvaluationModel.Input<Options>) {
this.id = input.id
this.provider = input.provider
this.route = input.route
this.http = input.http
}
static make<Options extends EvaluationOptions = EvaluationOptions>(input: EvaluationModel.MakeInput<Options>) {
return new EvaluationModel<Options>({
id: ModelID.make(input.id),
provider: ProviderID.make(input.provider),
route: input.route,
http: input.http,
})
}
}
export namespace EvaluationModel {
export interface Input<Options extends EvaluationOptions = EvaluationOptions> {
readonly id: ModelID
readonly provider: ProviderID
readonly route: EvaluationRoute<Options>
readonly http?: HttpOptions
}
export interface MakeInput<Options extends EvaluationOptions = EvaluationOptions>
extends Omit<Input<Options>, "id" | "provider"> {
readonly id: string | ModelID
readonly provider: string | ProviderID
}
}
export const EvaluationModelSchema = Schema.declare(
(value): value is EvaluationModel => value instanceof EvaluationModel,
{
expected: "Evaluation.Model",
},
)
export class EvaluationRequest extends Schema.Class<EvaluationRequest>("Evaluation.Request")({
model: EvaluationModelSchema,
state: EvaluationInput,
questions: EvaluationQuestions,
options: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
http: Schema.optional(HttpOptions),
}) {
declare protected readonly _EvaluationRequest: void
}
export type EvaluationModelOptions<Model> = Model extends EvaluationModel<infer Options> ? Options : never
export type EvaluationRequestFor<
Options extends EvaluationOptions = EvaluationOptions,
Questions extends EvaluationQuestions = EvaluationQuestions,
> = Omit<EvaluationRequest, "model" | "questions" | "options"> & {
readonly model: EvaluationModel<Options>
readonly questions: Questions
readonly options?: Options
}
export type EvaluationRequestInput<
Model extends object = EvaluationModel,
Questions extends EvaluationQuestions = EvaluationQuestions,
> = Omit<ConstructorParameters<typeof EvaluationRequest>[0], "model" | "questions" | "options" | "http"> & {
readonly model: Model
readonly questions: Questions
readonly options?: NoInfer<EvaluationModelOptions<Model>>
readonly http?: HttpOptions.Input
} & (Model extends EvaluationModel<EvaluationModelOptions<Model>> ? unknown : never)
export class EvaluationRounding extends Schema.Class<EvaluationRounding>("Evaluation.Rounding")({
probabilityDecimals: Schema.optional(Schema.Int),
scoreDecimals: Schema.optional(Schema.Int),
}) {}
export class EvaluationResponse extends Schema.Class<EvaluationResponse>("Evaluation.Response")({
model: ModelID,
answers: Schema.Record(Schema.String, EvaluationAnswer),
usage: Schema.optional(Usage),
rounding: Schema.optional(EvaluationRounding),
providerMetadata: Schema.optional(ProviderMetadata),
}) {}
export type EvaluationResponseFor<Questions extends EvaluationQuestions> = Omit<EvaluationResponse, "answers"> & {
readonly answers: AnswersFor<Questions>
}
export function request<const Model extends object, const Questions extends EvaluationQuestions>(
input: EvaluationRequestInput<Model, Questions>,
): EvaluationRequestFor<EvaluationModelOptions<Model>, Questions>
export function request(input: EvaluationRequest): EvaluationRequest
export function request(input: EvaluationRequest | EvaluationRequestInput) {
if (input instanceof EvaluationRequest) return input
return new EvaluationRequest({
...input,
model: input.model as unknown as EvaluationModel,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}
export function run<const Model extends object, const Questions extends EvaluationQuestions>(
input: EvaluationRequestInput<Model, Questions>,
): Effect.Effect<EvaluationResponseFor<Questions>, AIError, Service>
export function run(input: EvaluationRequest): Effect.Effect<EvaluationResponse, AIError, Service>
export function run(input: EvaluationRequest | EvaluationRequestInput) {
return Effect.try({
try: () => (input instanceof EvaluationRequest ? input : request(input)),
catch: (cause) =>
new AIError({
reason: new InvalidRequestError({
message: cause instanceof Error ? cause.message : String(cause),
cause,
}),
}),
}).pipe(
Effect.flatMap((request) =>
EvaluationClient.evaluate(request as EvaluationRequestFor<EvaluationOptions, EvaluationQuestions>),
),
)
}
export const Evaluation = {
request,
run,
} as const
-194
View File
@@ -1,194 +0,0 @@
import { Effect, Schema } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import {
ChoiceQuestion,
EvaluationInput,
EvaluationModel,
EvaluationResponse,
EvaluationRounding,
ScoreQuestion,
type EvaluationAnswer,
type EvaluationOptions,
} from "./evaluation.js"
import { Auth, type Definition as AuthDefinition } from "../route/auth.js"
import {
AIError,
HttpContext,
HttpOptions,
InvalidProviderOutputError,
InvalidRequestError,
ModelID,
Usage,
mergeJsonRecords,
} from "../schema/index.js"
const Noul = Schema.Struct({
type: Schema.Literal("noul"),
instructions: EvaluationInput,
criteria: Schema.optional(
Schema.Struct({
true: Schema.optional(Schema.NullOr(EvaluationInput)),
false: Schema.optional(Schema.NullOr(EvaluationInput)),
}),
),
})
const Question = Schema.Union([
ChoiceQuestion.pipe(
Schema.refine((x): x is typeof x => Object.keys(x.criteria).length <= 255, {
message: "System One Choice questions support at most 255 options",
}),
),
ScoreQuestion.pipe(
Schema.refine((x): x is typeof x => x.criteria.length <= 10, {
message: "System One Score questions support at most 10 levels",
}),
),
Noul,
])
const Request = Schema.StructWithRest(
Schema.Struct({
model: Schema.String,
state: EvaluationInput,
questions: Schema.Record(Schema.String, Question),
}),
[Schema.Record(Schema.String, Schema.Any)],
)
const Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 }))
const NoulAnswer = Schema.Struct({ type: Schema.Literal("noul"), noul: Probability })
const Choice = Schema.Struct({
type: Schema.Literal("choice"),
choice: Schema.String,
probabilities: Schema.Record(Schema.String, Probability),
confidence: Schema.optional(Probability),
})
const Score = Schema.Struct({
type: Schema.Literal("score"),
score: Schema.Number,
probabilities: Schema.Record(Schema.String, Probability),
legend: Schema.optional(Schema.Record(Schema.String, Schema.Json)),
confidence: Schema.optional(Probability),
})
const Answer = Schema.Union([NoulAnswer, Choice, Score]).pipe(Schema.toTaggedUnion("type"))
const NativeUsage = Schema.StructWithRest(
Schema.Struct({
input_tokens: Schema.optional(Schema.Number),
output_tokens: Schema.optional(Schema.Number),
}),
[Schema.Record(Schema.String, Schema.Unknown)],
)
const Response = Schema.Struct({
model: Schema.String,
answers: Schema.Record(Schema.String, Answer),
usage: Schema.optional(NativeUsage),
id: Schema.optional(Schema.String),
provider: Schema.optional(Schema.String),
provider_metadata: Schema.optional(Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown))),
})
export interface ModelInput {
readonly id: string | ModelID
readonly provider: string
readonly providerMetadataKey: string
readonly auth: AuthDefinition
readonly baseURL: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions
}
export const model = <Options extends EvaluationOptions = EvaluationOptions>(cfg: ModelInput) =>
EvaluationModel.make<Options>({
id: cfg.id,
provider: cfg.provider,
http: cfg.http,
route: {
id: "system-one",
evaluate: (req, send) =>
Effect.gen(function* () {
const url = new URL(`${cfg.baseURL.replace(/\/$/, "")}/systemone`)
Object.entries(req.http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value))
const body = yield* Schema.encodeUnknownEffect(Schema.fromJsonString(Request))({
...mergeJsonRecords(req.options, req.http?.body),
model: req.model.id,
state: req.state,
questions: Object.fromEntries(
Object.entries(req.questions).map(([id, x]) => [id, x.type === "boolean" ? { ...x, type: "noul" } : x]),
),
}).pipe(
Effect.mapError(
(cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) }),
),
)
const headers = yield* Auth.toEffect(cfg.auth)({
request: req,
method: "POST",
url: url.toString(),
body,
headers: Headers.fromInput({ ...cfg.headers, ...req.http?.headers }),
})
const res = yield* send(
HttpClientRequest.post(url).pipe(
HttpClientRequest.setHeaders(headers),
HttpClientRequest.bodyText(body, "application/json"),
),
)
const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers })
const fail = (message: string, cause: unknown, body?: string) =>
new AIError({ reason: new InvalidProviderOutputError({ route: "system-one", message, body, http, cause }) })
const text = yield* res.text.pipe(
Effect.mapError((cause) => fail("Failed to read the System One response", cause)),
)
const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Response))(text).pipe(
Effect.mapError((cause) => fail("System One returned an invalid response", cause, text)),
)
const confidence: Record<string, number> = {}
const legend: Record<string, Record<string, Schema.Json>> = {}
const answers = Object.fromEntries(
Object.entries(data.answers).map(([id, answer]): [string, EvaluationAnswer] => {
if (answer.type === "noul") return [id, { type: "boolean", probability: answer.noul }]
if (answer.type === "choice") {
if (answer.confidence !== undefined) confidence[id] = answer.confidence
return [
id,
{
type: "choice",
choice: answer.choice,
probabilities: answer.probabilities,
},
]
}
if (answer.confidence !== undefined) confidence[id] = answer.confidence
if (answer.legend !== undefined) legend[id] = answer.legend
return [id, { type: "score", score: answer.score, probabilities: answer.probabilities }]
}),
)
const meta = {
...(data.id === undefined ? {} : { responseId: data.id }),
...(data.provider === undefined ? {} : { provider: data.provider }),
...data.provider_metadata?.[cfg.providerMetadataKey],
...(Object.keys(confidence).length === 0 ? {} : { confidence }),
...(Object.keys(legend).length === 0 ? {} : { legend }),
}
return new EvaluationResponse({
model: ModelID.make(data.model),
answers,
usage: data.usage
? new Usage({
inputTokens: data.usage.input_tokens,
outputTokens: data.usage.output_tokens,
totalTokens:
data.usage.input_tokens === undefined && data.usage.output_tokens === undefined
? undefined
: (data.usage.input_tokens ?? 0) + (data.usage.output_tokens ?? 0),
providerMetadata: { [cfg.providerMetadataKey]: data.usage },
})
: undefined,
rounding: new EvaluationRounding({ probabilityDecimals: 2, scoreDecimals: 2 }),
providerMetadata: Object.keys(meta).length === 0 ? undefined : { [cfg.providerMetadataKey]: meta },
})
}),
},
})
export const SystemOne = { model } as const
-137
View File
@@ -1,137 +0,0 @@
import { Duration, Effect, Schedule, Schema, Stream } from "effect"
import { AIError, TimeoutError } from "./schema/errors.js"
export const Status = Schema.Literals(["queued", "running", "completed", "failed", "cancelled", "expired"])
export type Status = Schema.Schema.Type<typeof Status>
/** Provider-neutral view of one generation observation. */
export interface Snapshot {
readonly id: string
readonly status: Status
/** Normalized 0..1 when the provider reports progress. */
readonly progress?: number
readonly position?: number
readonly expiresAt?: number
}
/**
* Route-owned generation operations. `token` is the route's serializable handle (operation name, task id, response URL)
* so a generation can be resumed from another process; its shape is opaque to `Generation`.
*/
export interface Route<Response> {
readonly status: (token: unknown) => Effect.Effect<Snapshot, AIError>
readonly result: (token: unknown) => Effect.Effect<Response, AIError>
readonly cancel?: (token: unknown) => Effect.Effect<void, AIError>
/** Provider polling hint (e.g. `openai-poll-after-ms`) that overrides the default interval for the next poll. */
readonly pollHint?: (snapshot: Snapshot) => Duration.Duration | undefined
}
export interface Poll {
readonly interval?: Duration.Input
readonly timeout?: Duration.Input
/** Full override of the polling schedule; `interval` and `pollHint` are ignored when supplied. */
readonly schedule?: Schedule.Schedule<unknown, Snapshot>
}
export const DEFAULT_POLL_INTERVAL = Duration.seconds(5)
export const DEFAULT_POLL_TIMEOUT = Duration.minutes(10)
export type Event =
| { readonly type: "generation-queued"; readonly id: string; readonly position?: number }
| { readonly type: "generation-progress"; readonly id: string; readonly progress?: number }
| { readonly type: "generation-finished"; readonly id: string; readonly status: Status }
const TERMINAL: ReadonlySet<Status> = new Set(["completed", "failed", "cancelled", "expired"])
export class Generation<Response> {
readonly id: string
readonly status: Status
readonly progress?: number
readonly position?: number
readonly expiresAt?: number
constructor(
readonly route: Route<Response>,
readonly token: unknown,
snapshot: Snapshot,
) {
this.id = snapshot.id
this.status = snapshot.status
this.progress = snapshot.progress
this.position = snapshot.position
this.expiresAt = snapshot.expiresAt
}
get snapshot(): Snapshot {
return {
id: this.id,
status: this.status,
progress: this.progress,
position: this.position,
expiresAt: this.expiresAt,
}
}
get terminal() {
return TERMINAL.has(this.status)
}
refresh(): Effect.Effect<Generation<Response>, AIError> {
return this.route.status(this.token).pipe(Effect.map((snapshot) => new Generation(this.route, this.token, snapshot)))
}
/** Poll until the generation reaches a terminal status, then fetch the result. Fails with a `Timeout` reason on deadline. */
await(options?: { readonly poll?: Poll }): Effect.Effect<Response, AIError> {
const timeout = Duration.fromInputUnsafe(options?.poll?.timeout ?? DEFAULT_POLL_TIMEOUT)
const settled = this.terminal ? Effect.succeed(this) : this.poll(options?.poll)
return settled.pipe(
// Non-completed terminal states also go through `result` so the route can surface its provider failure body.
Effect.flatMap((generation) => generation.route.result(generation.token)),
Effect.timeoutOrElse({
duration: timeout,
orElse: () =>
new AIError({
reason: new TimeoutError({
message: `Generation ${this.id} did not finish within ${Duration.format(timeout)}`,
timeoutMs: Duration.toMillis(timeout),
}),
}),
}),
)
}
cancel(): Effect.Effect<void, AIError> {
return this.route.cancel?.(this.token) ?? Effect.void
}
/** Status observations as a stream, ending after the first terminal observation. */
events(options?: { readonly poll?: Poll }): Stream.Stream<Event, AIError> {
const observations = this.terminal
? Stream.make(this)
: Stream.fromEffectSchedule(this.refresh(), this.schedule(options?.poll)).pipe(
Stream.takeUntil((generation) => generation.terminal),
)
return observations.pipe(
Stream.map((generation): Event => {
if (generation.terminal) return { type: "generation-finished", id: generation.id, status: generation.status }
if (generation.status === "queued") return { type: "generation-queued", id: generation.id, position: generation.position }
return { type: "generation-progress", id: generation.id, progress: generation.progress }
}),
)
}
private poll(poll: Poll | undefined) {
return this.refresh().pipe(Effect.repeat({ schedule: this.schedule(poll), until: (generation) => generation.terminal }))
}
private schedule(poll: Poll | undefined): Schedule.Schedule<unknown, Generation<Response>> {
if (poll?.schedule) return poll.schedule.pipe(Schedule.setInputType<Generation<Response>>())
const interval = poll?.interval ?? DEFAULT_POLL_INTERVAL
const pollHint = this.route.pollHint
const spaced = Schedule.spaced(interval).pipe(Schedule.setInputType<Generation<Response>>())
if (!pollHint) return spaced
return spaced.pipe(
Schedule.modifyDelay((metadata) => Effect.succeed(pollHint(metadata.input.snapshot) ?? interval)),
)
}
}
+18 -29
View File
@@ -1,21 +1,15 @@
import { Context, Effect, Layer, Stream } from "effect"
import { Context, Effect, Layer } from "effect"
import { RequestExecutor } from "./route/executor.js"
import type { AIError } from "./schema/index.js"
import {
responseEvents,
type ImageEvent,
type ImageOptions,
type ImageRequestFor,
type ImageResponse,
} from "./image.js"
import { mergeHttpOptions, type AIError } from "./schema/index.js"
import { sanitizeSurrogates } from "./utils/sanitize.js"
import type { ImageOptions, ImageRequest, ImageRequestFor, ImageResponse } from "./image.js"
export type Execute = RequestExecutor.Interface["execute"]
export interface Interface {
readonly generate: <Options extends ImageOptions>(
request: ImageRequestFor<Options>,
) => Effect.Effect<ImageResponse, AIError>
readonly stream: <Options extends ImageOptions>(
request: ImageRequestFor<Options>,
) => Stream.Stream<ImageEvent, AIError>
}
export class Service extends Context.Service<Service, Interface>()("@opencode/ImageClient") {}
@@ -28,27 +22,23 @@ export const generate = <Options extends ImageOptions>(
return yield* client.generate(request)
})
export const stream = <Options extends ImageOptions>(
request: ImageRequestFor<Options>,
): Stream.Stream<ImageEvent, AIError, Service> =>
Stream.unwrap(
Effect.gen(function* () {
const client = yield* Service
return client.stream(request)
}),
)
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
Service,
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
const generate = <Options extends ImageOptions>(request: ImageRequestFor<Options>) =>
request.model.route.generate(request, executor.execute)
return Service.of({
generate,
// Inline routes have no partial frames yet; the stream is the completed response expanded into events.
stream: (request) =>
Stream.unwrap(generate(request).pipe(Effect.map((response) => Stream.fromIterable(responseEvents(response))))),
generate: (request) =>
request.model.route.generate(
{
...sanitizeSurrogates({
...request,
model: undefined,
http: mergeHttpOptions(request.model.http, request.http),
}),
model: request.model,
},
executor.execute,
),
})
}),
)
@@ -57,5 +47,4 @@ export const ImageClient = {
Service,
layer,
generate,
stream,
} as const
+64 -143
View File
@@ -1,30 +1,22 @@
import { Effect, Schema, Stream } from "effect"
import { Media } from "./media.js"
import { Endpoint } from "./route/endpoint.js"
import { MediaRoute } from "./route/media.js"
import type { MediaProtocol } from "./route/media-protocol.js"
import { Effect, Schema } from "effect"
import {
AIError,
HttpOptions,
InvalidRequestError,
MediaUsage,
AIError,
ModelID,
ProviderID,
ProviderMetadata,
Usage,
} from "./schema/index.js"
import { ImageClient, Service } from "./image-client.js"
import { ImageClient, Service, type Execute as ImageExecute } from "./image-client.js"
// ---------------------------------------------------------------------------
// Model
// ---------------------------------------------------------------------------
export interface ImageRoute<Options extends ImageOptions = ImageOptions> {
readonly id: string
readonly generate: (request: ImageRequestFor<Options>, execute: ImageExecute) => Effect.Effect<ImageResponse, AIError>
}
export type ImageOptions = Record<string, unknown>
export type ImageRoute<Options extends ImageOptions = ImageOptions> = MediaRoute.Route<
ImageRequestFor<Options>,
ImageResponse
>
export class ImageModel<Options extends ImageOptions = ImageOptions> {
declare protected readonly _Options: (options: Options) => Options
readonly id: ModelID
@@ -47,26 +39,6 @@ export class ImageModel<Options extends ImageOptions = ImageOptions> {
http: input.http,
})
}
/** Compose an inline image protocol with its canonical path into a model for one deployment. */
static fromRoute<Options extends ImageOptions = ImageOptions>(
route: ImageModel.RouteInput<Options>,
input: MediaRoute.ModelInput,
) {
return ImageModel.make<Options>({
id: input.id,
provider: route.provider,
http: input.http,
route: MediaRoute.make({
id: route.id,
provider: route.provider,
protocol: route.protocol,
endpoint: Endpoint.path(route.path, { baseURL: input.baseURL ?? route.baseURL }),
auth: input.auth,
headers: input.headers,
}),
})
}
}
export namespace ImageModel {
@@ -82,85 +54,81 @@ export namespace ImageModel {
readonly id: string | ModelID
readonly provider: string | ProviderID
}
export interface RouteInput<Options extends ImageOptions = ImageOptions> {
readonly id: string
readonly provider: string | ProviderID
readonly protocol: MediaProtocol.Inline<ImageRequestFor<Options>, ImageResponse>
readonly path: Endpoint.EndpointPart<MediaProtocol.Body, ImageRequestFor<Options>>
/** Canonical base URL; `ModelInput.baseURL` overrides it per deployment. */
readonly baseURL?: string
}
}
export const ImageModelSchema = Schema.declare((value): value is ImageModel => value instanceof ImageModel, {
expected: "Image.Model",
})
// ---------------------------------------------------------------------------
// Request
// ---------------------------------------------------------------------------
const ImageBytesInput = Schema.Struct({
type: Schema.Literal("bytes"),
data: Schema.Uint8Array,
mediaType: Schema.String,
})
const ImageUrlInput = Schema.Struct({
type: Schema.Literal("url"),
url: Schema.String,
})
const ImageFileIDInput = Schema.Struct({
type: Schema.Literal("file-id"),
id: Schema.String,
})
const ImageFileURIInput = Schema.Struct({
type: Schema.Literal("file-uri"),
uri: Schema.String,
mediaType: Schema.String,
})
export type ImageSize = `${number}x${number}`
export const ImageSize = Schema.declare<ImageSize>(
(value): value is ImageSize => typeof value === "string" && /^\d+x\d+$/.test(value),
{ title: "ImageSize" },
)
export const ImageInputSchema = Schema.Union([
ImageBytesInput,
ImageUrlInput,
ImageFileIDInput,
ImageFileURIInput,
]).pipe(Schema.toTaggedUnion("type"))
export type ImageInput = Schema.Schema.Type<typeof ImageInputSchema>
export type ImageAspectRatio = `${number}:${number}`
export const ImageAspectRatio = Schema.declare<ImageAspectRatio>(
(value): value is ImageAspectRatio => typeof value === "string" && /^\d+(?:\.\d+)?:\d+(?:\.\d+)?$/.test(value),
{ title: "ImageAspectRatio" },
)
export type ImageFormat = "png" | "jpeg" | "webp" | (string & {})
export const ImageInput = {
bytes: (data: Uint8Array, mediaType: string): ImageInput => ({ type: "bytes", data, mediaType }),
url: (url: string): ImageInput => ({ type: "url", url }),
file: (id: string): ImageInput => ({ type: "file-id", id }),
fileUri: (uri: string, mediaType: string): ImageInput => ({ type: "file-uri", uri, mediaType }),
} as const
export class ImageRequest extends Schema.Class<ImageRequest>("Image.Request")({
model: ImageModelSchema,
prompt: Schema.String,
/** Edit sources or style/subject references, in order. */
images: Schema.optional(Schema.Array(Media.AssetSchema)),
/** Inpainting mask; routes that cannot honor it fail with `UnsupportedOperation`. */
mask: Schema.optional(Media.AssetSchema),
n: Schema.optional(Schema.Int),
size: Schema.optional(ImageSize),
aspectRatio: Schema.optional(ImageAspectRatio),
seed: Schema.optional(Schema.Number),
format: Schema.optional(Schema.String),
providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
images: Schema.optional(Schema.Array(ImageInputSchema)),
options: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
http: Schema.optional(HttpOptions),
}) {
declare protected readonly _ImageRequest: void
}
export type ImageRequestFor<Options extends ImageOptions = ImageOptions> = Omit<
ImageRequest,
"model" | "providerOptions"
> & {
export type ImageRequestFor<Options extends ImageOptions = ImageOptions> = Omit<ImageRequest, "model" | "options"> & {
readonly model: ImageModel<Options>
readonly providerOptions?: Options
readonly options?: Options
}
export type ImageModelOptions<Model> = Model extends ImageModel<infer Options> ? Options : never
export type ImageRequestInput<Model extends ImageModel = ImageModel> = Omit<
export type ImageRequestInput<Model extends object = ImageModel> = Omit<
ConstructorParameters<typeof ImageRequest>[0],
"model" | "providerOptions" | "http"
"model" | "options" | "http"
> & {
readonly model: Model
readonly format?: ImageFormat
readonly providerOptions?: NoInfer<ImageModelOptions<Model>>
readonly options?: NoInfer<ImageModelOptions<Model>>
readonly http?: HttpOptions.Input
}
} & (Model extends ImageModel<ImageModelOptions<Model>> ? unknown : never)
// ---------------------------------------------------------------------------
// Response and events
// ---------------------------------------------------------------------------
export class GeneratedImage extends Schema.Class<GeneratedImage>("Image.Generated")({
mediaType: Schema.String,
data: Schema.Union([Schema.String, Schema.Uint8Array]),
providerMetadata: Schema.optional(ProviderMetadata),
}) {}
export class ImageResponse extends Schema.Class<ImageResponse>("Image.Response")({
images: Schema.Array(Media.AssetSchema),
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
images: Schema.Array(GeneratedImage),
usage: Schema.optional(Usage),
providerMetadata: Schema.optional(ProviderMetadata),
}) {
get image() {
@@ -168,43 +136,7 @@ export class ImageResponse extends Schema.Class<ImageResponse>("Image.Response")
}
}
export const ImageOutputEvent = Schema.Struct({
type: Schema.tag("image"),
index: Schema.Number,
image: Media.AssetSchema,
}).annotate({ identifier: "Image.Event.Image" })
export const ImageFinishEvent = Schema.Struct({
type: Schema.tag("finish"),
usage: Schema.optional(MediaUsage),
notices: Schema.optional(Schema.Array(Media.Notice)),
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "Image.Event.Finish" })
const imageEventTagged = Schema.Union([ImageOutputEvent, ImageFinishEvent]).pipe(Schema.toTaggedUnion("type"))
export const ImageEvent = Object.assign(imageEventTagged, {
is: {
image: imageEventTagged.guards.image,
finish: imageEventTagged.guards.finish,
},
})
export type ImageEvent = Schema.Schema.Type<typeof imageEventTagged>
/** Inline routes produce every image at once; expand the response into the streaming event shape. */
export const responseEvents = (response: ImageResponse): ReadonlyArray<ImageEvent> => [
...response.images.map((image, index) => ImageOutputEvent.make({ index, image })),
ImageFinishEvent.make({
usage: response.usage,
notices: response.notices,
providerMetadata: response.providerMetadata,
}),
]
// ---------------------------------------------------------------------------
// Request-shaped call API
// ---------------------------------------------------------------------------
export function request<const Model extends ImageModel>(
export function request<const Model extends object>(
input: ImageRequestInput<Model>,
): ImageRequestFor<ImageModelOptions<Model>>
export function request(input: ImageRequest): ImageRequest
@@ -212,13 +144,18 @@ export function request(input: ImageRequest | ImageRequestInput) {
if (input instanceof ImageRequest) return input
return new ImageRequest({
...input,
model: input.model as unknown as ImageModel,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}
const requestEffect = (input: ImageRequest | ImageRequestInput) =>
Effect.try({
try: () => request(input),
export function generate<const Model extends object>(
input: ImageRequestInput<Model>,
): Effect.Effect<ImageResponse, AIError, Service>
export function generate(input: ImageRequest): Effect.Effect<ImageResponse, AIError, Service>
export function generate(input: ImageRequest | ImageRequestInput) {
return Effect.try({
try: () => (input instanceof ImageRequest ? input : request(input)),
catch: (error) =>
new AIError({
reason: new InvalidRequestError({
@@ -226,26 +163,10 @@ const requestEffect = (input: ImageRequest | ImageRequestInput) =>
cause: error,
}),
}),
})
export function generate<const Model extends ImageModel>(
input: ImageRequestInput<Model>,
): Effect.Effect<ImageResponse, AIError, Service>
export function generate(input: ImageRequest): Effect.Effect<ImageResponse, AIError, Service>
export function generate(input: ImageRequest | ImageRequestInput) {
return requestEffect(input).pipe(Effect.flatMap((request) => ImageClient.generate(request)))
}
export function stream<const Model extends ImageModel>(
input: ImageRequestInput<Model>,
): Stream.Stream<ImageEvent, AIError, Service>
export function stream(input: ImageRequest): Stream.Stream<ImageEvent, AIError, Service>
export function stream(input: ImageRequest | ImageRequestInput) {
return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => ImageClient.stream(request))))
}).pipe(Effect.flatMap((request) => ImageClient.generate(request as unknown as ImageRequestFor<ImageOptions>)))
}
export const Image = {
request,
generate,
stream,
} as const
+2 -20
View File
@@ -11,27 +11,9 @@ export type {
Service as LLMClientService,
} from "./route/client.js"
export * from "./schema/index.js"
export {
ImageAspectRatio,
ImageEvent,
ImageModel,
ImageModelSchema,
ImageRequest,
ImageResponse,
ImageSize,
} from "./image.js"
export type {
ImageFormat,
ImageModelOptions,
ImageOptions,
ImageRequestFor,
ImageRequestInput,
ImageRoute,
} from "./image.js"
export { GeneratedImage, ImageInput, ImageInputSchema, ImageModel, ImageRequest, ImageResponse } from "./image.js"
export type { ImageModelOptions, ImageOptions, ImageRequestFor, ImageRequestInput, ImageRoute } from "./image.js"
export { Image } from "./image.js"
export { Media } from "./media.js"
export { Generation } from "./generation.js"
export type { Event as GenerationEvent, Poll, Route as GenerationRoute, Snapshot as GenerationSnapshot, Status as GenerationStatus } from "./generation.js"
export { Tool, ToolFailure, toDefinitions } from "./tool.js"
export { ToolRuntime } from "./tool-runtime.js"
export type { DispatchResult as ToolDispatchResult, ToolSettlement } from "./tool-runtime.js"
-307
View File
@@ -1,307 +0,0 @@
export * as Media from "./media.js"
import { Effect, Encoding, FileSystem, Schema, SchemaGetter } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { ProviderID } from "./schema/ids.js"
import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "./schema/errors.js"
import { ProviderMetadata } from "./schema/options.js"
import { Service } from "./route/executor-service.js"
import { detectMediaType, extensionMediaType } from "./utils/media-type.js"
export { detectMediaType } from "./utils/media-type.js"
const OCTET_STREAM = "application/octet-stream"
// ---------------------------------------------------------------------------
// Source — the serializable wire/persistence form of a media asset
// ---------------------------------------------------------------------------
const BytesSource = Schema.Struct({
type: Schema.Literal("bytes"),
data: Schema.Uint8Array,
mediaType: Schema.String,
})
const Base64Source = Schema.Struct({
type: Schema.Literal("base64"),
data: Schema.String,
mediaType: Schema.String,
})
const UrlSource = Schema.Struct({
type: Schema.Literal("url"),
url: Schema.String,
mediaType: Schema.optional(Schema.String),
/** Epoch milliseconds after which the provider no longer serves the URL. */
expiresAt: Schema.optional(Schema.Number),
/** Headers required to fetch the URL, such as provider auth for Veo downloads. */
headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
})
/** A provider-side handle: OpenAI `file_id`, Gemini file URI, `gs://`, `runway://`, or a prior generation id. */
const RefSource = Schema.Struct({
type: Schema.Literal("ref"),
provider: ProviderID,
id: Schema.String,
mediaType: Schema.optional(Schema.String),
})
export const Source = Schema.Union([BytesSource, Base64Source, UrlSource, RefSource])
.pipe(Schema.toTaggedUnion("type"))
.annotate({ identifier: "Media.Source" })
export type Source = Schema.Schema.Type<typeof Source>
// ---------------------------------------------------------------------------
// Kind, Info, Notice
// ---------------------------------------------------------------------------
export const Kind = Schema.Literals(["image", "video", "audio", "document", "other"])
export type Kind = Schema.Schema.Type<typeof Kind>
export const kindOf = (mediaType: string): Kind => {
const lower = mediaType.toLowerCase()
if (lower.startsWith("image/")) return "image"
if (lower.startsWith("video/")) return "video"
if (lower.startsWith("audio/")) return "audio"
if (lower === "application/pdf" || lower.startsWith("text/")) return "document"
return "other"
}
/** Container-independent facts about the payload; raw PCM audio relies on these because it has no header. */
export const Info = Schema.Struct({
width: Schema.optional(Schema.Number),
height: Schema.optional(Schema.Number),
durationSeconds: Schema.optional(Schema.Number),
sampleRate: Schema.optional(Schema.Number),
channels: Schema.optional(Schema.Number),
encoding: Schema.optional(Schema.String),
format: Schema.optional(Schema.String),
}).annotate({ identifier: "Media.Info" })
export type Info = Schema.Schema.Type<typeof Info>
/** A provider-side partial result such as stripped audio or a moderated sample; never a silent drop. */
export const Notice = Schema.Struct({
type: Schema.Literals(["moderated", "filtered", "other"]),
message: Schema.String,
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "Media.Notice" })
export type Notice = Schema.Schema.Type<typeof Notice>
// ---------------------------------------------------------------------------
// Asset
// ---------------------------------------------------------------------------
const invalid = (message: string, cause?: unknown) =>
new AIError({ reason: new InvalidRequestError({ message, cause }) })
/** Synchronous view of an inline payload; `undefined` for `url` and `ref` sources, which carry no local bytes. */
export interface Inline {
readonly mime: string
readonly base64: string
readonly dataUrl: string
}
export class Asset {
readonly source: Source
/** Derived from the source: declared type, sniffed magic bytes, then `application/octet-stream`. */
readonly mediaType: string
readonly kind: Kind
readonly info?: Info
/** Epoch milliseconds after which a `url` source stops resolving. */
readonly expiresAt?: number
readonly providerMetadata?: ProviderMetadata
// Derived payload forms are cached on the instance because every protocol lowering re-reads the same payload. The
// cache is check-then-set (concurrent first reads of a `url` source may both download) and is never observable
// through `source`, so round-tripping through `Media.from(asset.source)` stays lossless.
#bytes: Uint8Array | undefined
#base64: string | undefined
constructor(input: Asset.Input) {
this.source = input.source
this.mediaType =
input.source.mediaType ??
(input.source.type === "bytes" ? detectMediaType(input.source.data) : undefined) ??
OCTET_STREAM
this.kind = kindOf(this.mediaType)
this.info = input.info
this.expiresAt = input.source.type === "url" ? input.source.expiresAt : undefined
this.providerMetadata = input.providerMetadata
}
/** Inline payload without effects, for protocols that embed base64 or data URLs directly. */
inline(): Inline | undefined {
const source = this.source
if (source.type !== "bytes" && source.type !== "base64") return undefined
const base64 = source.type === "base64" ? source.data : (this.#base64 ??= Encoding.encodeBase64(source.data))
const mime = this.mediaType.toLowerCase()
return { mime, base64, dataUrl: `data:${mime};base64,${base64}` }
}
/** Decoded payload; downloads `url` sources through the request executor and caches the result. */
bytes(): Effect.Effect<Uint8Array, AIError, Service> {
return Effect.suspend(() => {
const source = this.source
if (source.type === "bytes") return Effect.succeed(source.data)
if (this.#bytes !== undefined) return Effect.succeed(this.#bytes)
if (source.type === "ref")
return Effect.fail(invalid(`Cannot materialize provider ref ${source.provider}:${source.id}`))
const decoded =
source.type === "base64"
? Effect.fromResult(Encoding.decodeBase64(source.data)).pipe(
Effect.mapError((cause) => invalid(`Media asset contains invalid base64 data`, cause)),
)
: download(source)
return decoded.pipe(Effect.tap((data) => Effect.sync(() => (this.#bytes = data))))
})
}
base64(): Effect.Effect<string, AIError, Service> {
return Effect.suspend(() => {
const source = this.source
if (source.type === "base64") return Effect.succeed(source.data)
if (this.#base64 !== undefined) return Effect.succeed(this.#base64)
return this.bytes().pipe(Effect.map((data) => (this.#base64 = Encoding.encodeBase64(data))))
})
}
dataUrl(): Effect.Effect<string, AIError, Service> {
return this.base64().pipe(Effect.map((data) => `data:${this.mediaType};base64,${data}`))
}
/**
* The `AssetEncoded` JSON form with `bytes` sources as base64, matching `Schema.toCodecJson(AssetSchema)`, so a
* plain `JSON.stringify` of messages or events stays lossless and decodes back through the JSON codec.
*/
toJSON() {
const source = this.source
return {
source: source.type === "bytes" ? { ...source, data: Encoding.encodeBase64(source.data) } : source,
info: this.info,
providerMetadata: this.providerMetadata,
}
}
/** Pull `url` sources into owned bytes before the URL expires. Inline sources return themselves. */
materialize(): Effect.Effect<Asset, AIError, Service> {
if (this.source.type === "bytes" || this.source.type === "base64") return Effect.succeed(this)
return this.bytes().pipe(
Effect.map((data) =>
bytes(data, this.source.mediaType, { info: this.info, providerMetadata: this.providerMetadata }),
),
)
}
}
export namespace Asset {
export interface Input {
readonly source: Source
readonly info?: Info
readonly providerMetadata?: ProviderMetadata
}
}
/** JSON form of an asset: the serializable `Source` plus caller-supplied metadata. `bytes` sources encode as base64. */
export const AssetEncoded = Schema.Struct({
source: Source,
info: Schema.optional(Info),
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "Media.AssetEncoded" })
const encodeAsset = (asset: Asset): typeof AssetEncoded.Type => ({
source: asset.source,
info: asset.info,
providerMetadata: asset.providerMetadata,
})
const AssetInstance = Schema.declare((value): value is Asset => value instanceof Asset, {
expected: "Media.Asset",
})
/** `Asset` in the type domain and `AssetEncoded` on the wire, so messages and events holding assets serialize. */
export const AssetSchema = AssetEncoded.pipe(
Schema.decodeTo(AssetInstance, {
decode: SchemaGetter.transform((encoded) => new Asset(encoded)),
encode: SchemaGetter.transform(encodeAsset),
}),
)
const download = Effect.fn("Media.download")(function* (source: Extract<Source, { readonly type: "url" }>) {
const executor = yield* Service
const response = yield* executor.execute(
HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(source.headers ?? {})),
)
const buffer = yield* response.arrayBuffer.pipe(
Effect.mapError(
(cause) =>
new AIError({
reason: new InvalidProviderOutputError({
message: `Failed to read media from ${source.url}`,
http: new HttpContext({ url: response.request.url, status: response.status, headers: response.headers }),
cause,
}),
}),
),
)
return new Uint8Array(buffer)
})
// ---------------------------------------------------------------------------
// Constructors
// ---------------------------------------------------------------------------
export type AssetOptions = Omit<Asset.Input, "source">
export const from = (source: Source, options?: AssetOptions) => new Asset({ ...options, source })
export const bytes = (data: Uint8Array, mediaType?: string, options?: AssetOptions) =>
from({ type: "bytes", data, mediaType: mediaType ?? detectMediaType(data) ?? OCTET_STREAM }, options)
export const base64 = (data: string, mediaType: string, options?: AssetOptions) =>
from({ type: "base64", data, mediaType }, options)
export const url = (
value: string,
options?: AssetOptions & Omit<Extract<Source, { readonly type: "url" }>, "type" | "url">,
) => {
const { mediaType, expiresAt, headers, ...rest } = options ?? {}
return from({ type: "url", url: value, mediaType, expiresAt, headers }, rest)
}
export const ref = (provider: string | ProviderID, id: string, mediaType?: string, options?: AssetOptions) =>
from({ type: "ref", provider: ProviderID.make(provider), id, mediaType }, options)
const DATA_URL = /^data:([^;,]+)(?:;[^,]*)*;base64,(.*)$/s
/** Parse a `data:<mime>;base64,<data>` URL, or `undefined` when the value is not a base64 data URL. */
export const parseDataUrl = (value: string, options?: AssetOptions) => {
const match = DATA_URL.exec(value)
return match === null ? undefined : base64(match[2], match[1], options)
}
/** Parse a `data:<mime>;base64,<data>` URL. Malformed input throws a typed `AIError` because constructors are sync. */
export const fromDataUrl = (dataUrl: string, options?: AssetOptions) => {
const asset = parseDataUrl(dataUrl, options)
if (asset === undefined) throw invalid("Media data URLs must contain a MIME type and base64 data")
return asset
}
/** Read a file through `FileSystem` and sniff its media type from magic bytes, then the extension. */
export const file = (path: string, options?: AssetOptions): Effect.Effect<Asset, AIError, FileSystem.FileSystem> =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem
const data = yield* fs
.readFile(path)
.pipe(Effect.mapError((cause) => invalid(`Failed to read media file ${path}`, cause)))
return bytes(data, detectMediaType(data) ?? extensionMediaType(path), options)
})
/** Materialize an asset and write its bytes through `FileSystem`. */
export const write = (asset: Asset, path: string): Effect.Effect<void, AIError, FileSystem.FileSystem | Service> =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem
const data = yield* asset.bytes()
yield* fs
.writeFile(path, data)
.pipe(Effect.mapError((cause) => invalid(`Failed to write media file ${path}`, cause)))
})
-100
View File
@@ -1,100 +0,0 @@
import { Effect, Layer, ManagedRuntime, Stream } from "effect"
import { Image, ImageModel, ImageRequest, type ImageRequestInput } from "./image.js"
import { ImageClient } from "./image-client.js"
import { LLM } from "./index.js"
import { LLMClient } from "./route/client.js"
import { RequestExecutor } from "./route/executor.js"
import { LanguageModel, LLMRequest } from "./schema/index.js"
import type { RequestInput } from "./llm.js"
/**
* Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM and image clients
* over a request executor; every method runs the corresponding Effect API and rethrows `AIError` unchanged.
*/
export interface Options {
/** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or middleware here. */
readonly layer?: Layer.Layer<RequestExecutor.Service>
}
export interface RunOptions {
readonly signal?: AbortSignal
}
export type Services =
| Layer.Success<typeof LLMClient.layer>
| Layer.Success<typeof ImageClient.layer>
| RequestExecutor.Service
const abortEffect = (signal: AbortSignal | undefined) =>
signal === undefined
? Effect.never
: Effect.callback<void>((resume) => {
if (signal.aborted) {
resume(Effect.void)
return
}
const onAbort = () => resume(Effect.void)
signal.addEventListener("abort", onAbort, { once: true })
return Effect.sync(() => signal.removeEventListener("abort", onAbort))
})
export const make = (options: Options = {}) => {
const runtime = ManagedRuntime.make(
Layer.mergeAll(LLMClient.layer, ImageClient.layer).pipe(
Layer.provideMerge(options.layer ?? RequestExecutor.fetchLayer),
),
)
/** Run any package Effect (for example `asset.bytes()`) inside this runtime. */
const run = <A, E>(effect: Effect.Effect<A, E, Services>, options?: RunOptions) =>
runtime.runPromise(effect, { signal: options?.signal })
const iterate = <A, E>(stream: Stream.Stream<A, E, Services>, options?: RunOptions): AsyncIterable<A> =>
Stream.toAsyncIterable(
Stream.unwrap(
runtime.contextEffect.pipe(
Effect.map(
(context): Stream.Stream<A, E> =>
stream.pipe(Stream.interruptWhen(abortEffect(options?.signal)), Stream.provideContext(context)),
),
),
),
)
// The typed `generate`/`stream` overloads take a concrete input or a request, not the union; normalize once here.
const llmRequest = (input: RequestInput | LLMRequest) => (input instanceof LLMRequest ? input : LLM.request(input))
const imageRequest = (input: ImageRequestInput | ImageRequest) =>
input instanceof ImageRequest ? input : Image.request(input)
return {
run,
llm: {
request: LLM.request,
generate: <const Model extends LanguageModel>(
input: RequestInput<Model> | LLMRequest,
options?: RunOptions,
) => run(LLM.generate(llmRequest(input)), options),
stream: <const Model extends LanguageModel>(input: RequestInput<Model> | LLMRequest, options?: RunOptions) =>
iterate(LLM.stream(llmRequest(input)), options),
},
image: {
request: Image.request,
generate: <const Model extends ImageModel>(
input: ImageRequestInput<Model> | ImageRequest,
options?: RunOptions,
) => run(Image.generate(imageRequest(input)), options),
stream: <const Model extends ImageModel>(
input: ImageRequestInput<Model> | ImageRequest,
options?: RunOptions,
) => iterate(Image.stream(imageRequest(input)), options),
},
dispose: () => runtime.dispose(),
}
}
export type Client = ReturnType<typeof make>
/** Default client over `RequestExecutor.fetchLayer` for scripts; the runtime builds its layer on first use. */
export const ai = make()
export * as AI from "./promise.js"
@@ -658,7 +658,7 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (
part: MediaPart,
breakpoints?: Cache.Breakpoints,
) {
const mime = part.media.mediaType.toLowerCase()
const mime = part.mediaType.toLowerCase()
const cacheControlValue = breakpoints ? cacheControl(breakpoints, part.cache) : undefined
const fileId = fileIdFromMetadata(part.metadata)
@@ -687,9 +687,9 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (
} satisfies AnthropicDocumentBlock
}
const rawString = ProviderShared.mediaUrl(part.media)?.trim()
const rawString = typeof part.data === "string" ? part.data.trim() : undefined
// SDK URL sources: URLImageSource:3817 / URLPDFSource:3823 {type:"url", url}
if (rawString && isHttpUrl(rawString)) {
if (rawString && isHttpUrl(rawString) && !rawString.startsWith("data:")) {
if (mime.startsWith("image/"))
return {
type: "image" as const,
@@ -714,11 +714,20 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (
} satisfies AnthropicDocumentBlock
}
const media = yield* ProviderShared.requireInlineMedia("Anthropic Messages", part.media)
// SDK PlainTextSource:2716 {type:"text", media_type:"text/plain", data}
if (mime === "text/plain") {
const textData = Buffer.from(media.base64, "base64").toString("utf8")
const textData =
typeof part.data !== "string"
? Buffer.from(part.data).toString("utf8")
: part.data.startsWith("data:")
? (() => {
const comma = part.data.indexOf(",")
const payload = comma >= 0 ? part.data.slice(comma + 1) : part.data
return part.data.includes(";base64")
? Buffer.from(payload, "base64").toString("utf8")
: decodeURIComponent(payload)
})()
: part.data
return {
type: "document" as const,
source: { type: "text" as const, media_type: "text/plain" as const, data: textData },
@@ -733,6 +742,7 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (
} satisfies AnthropicDocumentBlock
}
const media = ProviderShared.normalizeMedia(part)
if (media.mime === "application/pdf")
return {
type: "document" as const,
@@ -751,7 +761,7 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (
: { citations: citationsFromMetadata(part.metadata)! }),
} satisfies AnthropicDocumentBlock
if (!media.mime.startsWith("image/"))
return yield* invalid(`Anthropic Messages does not support media type ${part.media.mediaType}`)
return yield* invalid(`Anthropic Messages does not support media type ${part.mediaType}`)
return {
type: "image" as const,
source: {
@@ -770,7 +780,7 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (
// content instead of JSON-stringifying base64 into a prompt string.
const lowerToolResultContentItem = Effect.fnUntraced(function* (item: Tool.Content) {
if (item.type === "text") return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock
return yield* lowerMedia(ProviderShared.toolFileMedia(item))
return yield* lowerMedia({ type: "media", mediaType: item.mime, data: item.uri, filename: item.name })
})
const lowerToolResultContent = Effect.fnUntraced(function* (part: ToolResultPart) {
@@ -303,7 +303,15 @@ const lowerToolResultContent = Effect.fn("BedrockConverse.lowerToolResultContent
content.push({ text: item.text })
continue
}
const media = yield* BedrockMedia.lower(ProviderShared.toolFileMedia(item), documentNames)
const media = yield* BedrockMedia.lower(
{
type: "media",
mediaType: item.mime,
data: item.uri,
filename: item.name,
},
documentNames,
)
content.push(...media)
}
return content
+6 -45
View File
@@ -20,7 +20,6 @@ import {
type ToolDefinition,
} from "../schema/index.js"
import { classifyProviderFailure } from "../provider-error.js"
import { Media } from "../media.js"
import { JsonObject, knownString, lenient, optionalArray, optionalNull, ProviderShared } from "./shared.js"
import { GeminiToolSchema } from "./utils/gemini-tool-schema.js"
import { Lifecycle } from "./utils/lifecycle.js"
@@ -75,18 +74,9 @@ const GeminiInlineDataPart = Schema.Struct({
mimeType: Schema.String,
data: Schema.String,
}),
thoughtSignature: optionalNull(Schema.String),
})
type GeminiInlineDataPart = Schema.Schema.Type<typeof GeminiInlineDataPart>
/** Gemini Files API reference; the only remote input Gemini accepts. */
const GeminiFileDataPart = Schema.Struct({
fileData: Schema.Struct({
mimeType: Schema.String,
fileUri: Schema.String,
}),
})
const GeminiFunctionCallPart = Schema.Struct({
functionCall: Schema.Struct({
id: optionalNull(Schema.String),
@@ -108,7 +98,6 @@ const GeminiFunctionResponsePart = Schema.Struct({
const GeminiContentPart = Schema.Union([
GeminiTextPart,
GeminiInlineDataPart,
GeminiFileDataPart,
GeminiFunctionCallPart,
GeminiFunctionResponsePart,
])
@@ -305,13 +294,9 @@ const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
tool: (name) => ({ functionCallingConfig: { mode: "ANY" as const, allowedFunctionNames: [name] } }),
})
// Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs.
const lowerContentPart = Effect.fn("Gemini.lowerContentPart")(function* (part: TextPart | MediaPart) {
const lowerUserPart = Effect.fn("Gemini.lowerUserPart")(function* (part: TextPart | MediaPart) {
if (part.type === "text") return { text: part.text }
const source = part.media.source
if (source.type === "ref" && source.provider === "google")
return { fileData: { mimeType: part.media.mediaType, fileUri: source.id } }
const media = yield* ProviderShared.requireInlineMedia("Gemini", part.media)
const media = ProviderShared.normalizeMedia(part)
return { inlineData: { mimeType: media.mime, data: media.base64 } }
})
@@ -359,7 +344,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
for (const part of message.content) {
if (!ProviderShared.supportsContent(part, ["text", "media"]))
return yield* ProviderShared.unsupportedContent("Gemini", "user", ["text", "media"])
parts.push(yield* lowerContentPart(part))
parts.push(yield* lowerUserPart(part))
}
contents.push({ role: "user", parts })
continue
@@ -370,23 +355,12 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
// Parallel Gemini 3 calls may carry one signature on the first call; unsigned sibling calls are valid.
let hasSignedToolCall = false
for (const part of message.content) {
if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call", "media"]))
return yield* ProviderShared.unsupportedContent("Gemini", "assistant", [
"text",
"reasoning",
"tool-call",
"media",
])
if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call"]))
return yield* ProviderShared.unsupportedContent("Gemini", "assistant", ["text", "reasoning", "tool-call"])
if (part.type === "text") {
parts.push({ text: part.text, thoughtSignature: thoughtSignature(part.providerMetadata, metadataKey) })
continue
}
// Generated images replay as model-role inline data so multi-turn image editing keeps the prior output.
if (part.type === "media") {
const lowered = yield* lowerContentPart(part)
parts.push({ ...lowered, thoughtSignature: thoughtSignature(part.providerMetadata, metadataKey) })
continue
}
if (part.type === "reasoning") {
parts.push({
text: part.text,
@@ -436,7 +410,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
const media: GeminiInlineDataPart[] = []
for (const item of content) {
if (item.type === "text") continue
const value = yield* ProviderShared.requireInlineMedia("Gemini", ProviderShared.toolFileMedia(item).media)
const value = ProviderShared.normalizeToolFile(item)
media.push({ inlineData: { mimeType: value.mime, data: value.base64 } })
}
if (legacyToolMedia && media.length > 0) (pendingMedia ??= []).push(...media)
@@ -688,19 +662,6 @@ const step = (state: ParserState, event: GeminiEvent) => {
// each block kind must retain the signature attached to its own parts.
if (signature !== undefined && "thought" in part && part.thought) reasoningSignature = signature
else if (signature !== undefined && "text" in part) textSignature = signature
// Image-capable Gemini models return generated images as inline data parts; surface them as first-class output.
if ("inlineData" in part) {
lifecycle = Lifecycle.stepStart(lifecycle, events)
events.push(
LLMEvent.media({
media: Media.base64(part.inlineData.data, part.inlineData.mimeType),
providerMetadata: signature
? providerMetadata(state.providerMetadataKey, { thoughtSignature: signature })
: undefined,
}),
)
continue
}
if ("text" in part && part.text.length > 0) {
if (part.thought) {
if (textId !== undefined) {
+208 -226
View File
@@ -1,36 +1,40 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { Effect, Encoding, Schema } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import {
GeneratedImage,
ImageModel,
ImageResponse,
type ImageInput,
type ImageRequestFor,
type ImageRoute,
} from "../image.js"
import { Auth, type Definition as AuthDefinition } from "../route/auth.js"
import { AIError, Usage, mergeHttpOptions, mergeJsonRecords, type HttpOptions } from "../schema/index.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
import { ImageInputs } from "./utils/image-input.js"
const ADAPTER = "google-images"
const NAME = "Google Images"
const PROVIDER = ProviderID.make("google")
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type GoogleImageString<Known extends string> = Known | (string & {})
/** Provider-native options. Common fields (`aspectRatio`, `seed`, `images`) live on the request. */
export type GoogleImageOptions = {
readonly aspectRatio?: GoogleImageString<
"1:1" | "2:3" | "3:2" | "3:4" | "4:3" | "4:5" | "5:4" | "9:16" | "16:9" | "21:9"
>
readonly imageSize?: GoogleImageString<"1K" | "2K" | "4K">
readonly seed?: number
readonly thinkingLevel?: GoogleImageString<"MINIMAL" | "LOW" | "MEDIUM" | "HIGH">
readonly includeThoughts?: boolean
} & Record<string, unknown>
export type Request = ImageRequestFor<GoogleImageOptions>
// ---------------------------------------------------------------------------
// 2. Response schema
// ---------------------------------------------------------------------------
export type GoogleImageBody = Record<string, unknown> & {
readonly contents: ReadonlyArray<{
readonly role: "user"
readonly parts: ReadonlyArray<Record<string, unknown>>
}>
readonly generationConfig: Record<string, unknown>
}
const GoogleUsage = Schema.StructWithRest(
Schema.Struct({
@@ -81,20 +85,30 @@ const GoogleImageResponse = Schema.Struct({
promptFeedback: Schema.optional(Schema.Unknown),
})
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
export interface ModelInput {
readonly id: string
readonly auth: AuthDefinition
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions
}
const generationConfig = (request: Request) => {
const { imageSize, thinkingLevel, includeThoughts, ...native } = request.providerOptions ?? {}
const imageConfig = { aspectRatio: request.aspectRatio, imageSize }
const thinkingConfig = { thinkingLevel, includeThoughts }
const nativeOptions = (options: GoogleImageOptions | undefined) => {
const { aspectRatio, imageSize, seed, thinkingLevel, includeThoughts, ...native } = options ?? {}
const image = {
aspectRatio,
imageSize,
}
const thinkingConfig = {
thinkingLevel,
includeThoughts,
}
return (
mergeJsonRecords(
{
responseModalities: ["IMAGE"],
imageConfig: Object.values(imageConfig).some((value) => value !== undefined) ? imageConfig : undefined,
seed: request.seed,
imageConfig: Object.values(image).some((value) => value !== undefined) ? image : undefined,
seed,
thinkingConfig: Object.values(thinkingConfig).some((value) => value !== undefined) ? thinkingConfig : undefined,
},
native,
@@ -102,208 +116,176 @@ const generationConfig = (request: Request) => {
)
}
// Gemini does not fetch public URLs; inline payloads or Gemini Files references are the only accepted inputs.
const imagePart = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ inlineData: { mimeType: inline.mime, data: inline.base64 } })
const id = MediaInput.refID(asset, PROVIDER)
if (id) return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri: id } })
if (asset.source.type === "ref")
return Effect.fail(
ProviderShared.invalidRequest(
"Google generateContent requires Gemini file references rather than other providers' file IDs",
),
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
if (!query) return url
const next = new URL(url)
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
return next.toString()
}
export const model = (input: ModelInput) => {
const route: ImageRoute<GoogleImageOptions> = {
id: ADAPTER,
generate: Effect.fn("GoogleImages.generate")(function* (request: ImageRequestFor<GoogleImageOptions>, execute) {
const imageParts = yield* Effect.forEach(request.images ?? [], googleImagePart)
const http = mergeHttpOptions(request.model.http, request.http)
const requestBody = mergeJsonRecords(
{
contents: [{ role: "user", parts: [{ text: request.prompt }, ...imageParts] }],
generationConfig: nativeOptions(request.options),
},
http?.body,
) as GoogleImageBody
const text = ProviderShared.encodeJson(requestBody)
const url = applyQuery(
`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}/models/${request.model.id}:generateContent`,
http?.query,
)
const headers = yield* Auth.toEffect(input.auth)({
request,
method: "POST",
url,
body: text,
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
})
const response = yield* execute(
HttpClientRequest.post(url).pipe(
HttpClientRequest.setHeaders(headers),
HttpClientRequest.bodyText(text, "application/json"),
),
)
const output = yield* ProviderShared.imageResponse(ADAPTER, "Google Images", response)
const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(GoogleImageResponse))(output.body).pipe(
Effect.mapError((cause) => output.invalid("Google Images returned an invalid response", cause)),
)
const candidates = decoded.candidates ?? []
const candidateMetadata = candidates.map((candidate, candidateIndex) => ({
index: candidate.index ?? candidateIndex,
finishReason: candidate.finishReason,
finishMessage: candidate.finishMessage,
safetyRatings: candidate.safetyRatings,
citationMetadata: candidate.citationMetadata,
groundingMetadata: candidate.groundingMetadata,
parts: (candidate.content?.parts ?? []).map((part) =>
part.inlineData === undefined
? {
type: "text",
text: part.text,
thought: part.thought,
thoughtSignature: part.thoughtSignature,
}
: {
type: "inlineData",
mediaType: part.inlineData.mimeType,
thought: part.thought,
thoughtSignature: part.thoughtSignature,
},
),
}))
const encoded = candidates.flatMap((candidate, candidateIndex) =>
(candidate.content?.parts ?? []).flatMap((part, partIndex) =>
part.inlineData === undefined || part.thought === true
? []
: [{ candidate, candidateIndex, partIndex, inlineData: part.inlineData }],
),
)
const images = yield* Effect.forEach(encoded, (item) =>
Effect.fromResult(Encoding.decodeBase64(item.inlineData.data)).pipe(
Effect.mapError((cause) =>
output.invalid(
`Google Images candidate ${item.candidateIndex} part ${item.partIndex} contains invalid base64 data`,
cause,
),
),
Effect.map(
(data) =>
new GeneratedImage({
mediaType: item.inlineData.mimeType,
data,
providerMetadata: {
google: {
candidateIndex: item.candidate.index ?? item.candidateIndex,
partIndex: item.partIndex,
finishReason: item.candidate.finishReason,
safetyRatings: item.candidate.safetyRatings,
citationMetadata: item.candidate.citationMetadata,
groundingMetadata: item.candidate.groundingMetadata,
thoughtSignature: item.candidate.content?.parts[item.partIndex]?.thoughtSignature,
},
},
}),
),
),
)
if (images.length === 0) {
const finishReasons = candidates.flatMap((candidate) =>
candidate.finishReason === undefined ? [] : [candidate.finishReason],
)
return yield* output.invalid(
`Google Images returned no final images${
finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})`
}; inspect body for prompt feedback and candidate details`,
)
}
const usage = decoded.usageMetadata
const outputTokens =
usage?.candidatesTokenCount === undefined
? undefined
: usage.candidatesTokenCount + (usage.thoughtsTokenCount ?? 0)
return new ImageResponse({
images,
usage:
usage === undefined
? undefined
: new Usage({
inputTokens: usage.promptTokenCount,
outputTokens,
nonCachedInputTokens: ProviderShared.subtractTokens(
usage.promptTokenCount,
usage.cachedContentTokenCount,
),
cacheReadInputTokens: usage.cachedContentTokenCount,
reasoningTokens: usage.thoughtsTokenCount,
totalTokens: ProviderShared.totalTokens(usage.promptTokenCount, outputTokens, usage.totalTokenCount),
providerMetadata: { google: usage },
}),
providerMetadata: {
google: {
modelVersion: decoded.modelVersion,
responseId: decoded.responseId,
promptFeedback: decoded.promptFeedback,
candidates: candidateMetadata,
},
},
})
}),
}
return ImageModel.make<GoogleImageOptions>({ id: input.id, provider: "google", route, http: input.http })
}
const googleImagePart = (image: ImageInput): Effect.Effect<Record<string, unknown>, AIError> => {
if (image.type === "bytes")
return Effect.succeed({ inlineData: { mimeType: image.mediaType, data: Encoding.encodeBase64(image.data) } })
if (image.type === "file-uri") return Effect.succeed({ fileData: { mimeType: image.mediaType, fileUri: image.uri } })
if (image.type === "url")
return ImageInputs.decodeDataUrl(image.url).pipe(
Effect.flatMap((decoded) => {
if (decoded === undefined)
return Effect.fail(
ImageInputs.invalid(
"Google generateContent does not fetch public image URLs; use bytes, a data URL, or a Gemini file URI",
),
)
return Effect.succeed({
inlineData: { mimeType: decoded.mediaType, data: Encoding.encodeBase64(decoded.data) },
})
}),
)
return Effect.fail(
ProviderShared.invalidRequest(
"Google generateContent does not fetch public image URLs; use bytes, a data URL, or a Gemini file reference",
),
ImageInputs.invalid("Google generateContent requires Gemini file URIs rather than provider file IDs"),
)
}
const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request: Request) {
if (request.n !== undefined && request.n > 1)
return yield* ProviderShared.unsupportedOperation({
operation: "image.n",
provider: PROVIDER,
route: ADAPTER,
message: `${NAME} generates one image per request; call it once per image instead of n=${request.n}`,
})
const parts = yield* Effect.forEach(request.images ?? [], imagePart)
return MediaProtocol.json(
mergeJsonRecords(
{
contents: [{ role: "user", parts: [{ text: request.prompt }, ...parts] }],
generationConfig: generationConfig(request),
},
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, GoogleImageResponse)(response)
const decoded = output.value
const candidates = decoded.candidates ?? []
const candidateMetadata = candidates.map((candidate, candidateIndex) => ({
index: candidate.index ?? candidateIndex,
finishReason: candidate.finishReason,
finishMessage: candidate.finishMessage,
safetyRatings: candidate.safetyRatings,
citationMetadata: candidate.citationMetadata,
groundingMetadata: candidate.groundingMetadata,
parts: (candidate.content?.parts ?? []).map((part) =>
part.inlineData === undefined
? { type: "text", text: part.text, thought: part.thought, thoughtSignature: part.thoughtSignature }
: {
type: "inlineData",
mediaType: part.inlineData.mimeType,
thought: part.thought,
thoughtSignature: part.thoughtSignature,
},
),
}))
// Thought parts are drafts; only non-thought inline data is a final image.
const encoded = candidates.flatMap((candidate, candidateIndex) =>
(candidate.content?.parts ?? []).flatMap((part, partIndex) =>
part.inlineData === undefined || part.thought === true
? []
: [
{
candidate,
candidateIndex,
partIndex,
inlineData: part.inlineData,
thoughtSignature: part.thoughtSignature,
},
],
),
)
const images = yield* Effect.forEach(encoded, (item) =>
MediaInput.decodedAsset(
output.invalid,
`${NAME} candidate ${item.candidateIndex} part ${item.partIndex}`,
item.inlineData.data,
item.inlineData.mimeType,
{
providerMetadata: {
google: {
candidateIndex: item.candidate.index ?? item.candidateIndex,
partIndex: item.partIndex,
finishReason: item.candidate.finishReason,
safetyRatings: item.candidate.safetyRatings,
citationMetadata: item.candidate.citationMetadata,
groundingMetadata: item.candidate.groundingMetadata,
thoughtSignature: item.thoughtSignature,
},
},
},
),
)
if (images.length === 0) {
const finishReasons = candidates.flatMap((candidate) =>
candidate.finishReason === undefined ? [] : [candidate.finishReason],
)
return yield* output.invalid(
`${NAME} returned no final images${
finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})`
}; inspect body for prompt feedback and candidate details`,
)
}
// Candidates that stopped for a safety or policy reason are partial results, not a silent drop.
const notices = [
...(decoded.promptFeedback === undefined
? []
: [
{
type: "filtered" as const,
message: `${NAME} reported prompt feedback`,
providerMetadata: { google: { promptFeedback: decoded.promptFeedback } },
},
]),
...candidates.flatMap((candidate, index) =>
candidate.finishReason === undefined || candidate.finishReason === "STOP"
? []
: [
{
type: "filtered" as const,
message: `${NAME} candidate ${candidate.index ?? index} finished with ${candidate.finishReason}${
candidate.finishMessage === undefined ? "" : `: ${candidate.finishMessage}`
}`,
providerMetadata: {
google: {
candidateIndex: candidate.index ?? index,
finishReason: candidate.finishReason,
finishMessage: candidate.finishMessage,
safetyRatings: candidate.safetyRatings,
},
},
},
],
),
]
const usage = decoded.usageMetadata
const outputTokens =
usage?.candidatesTokenCount === undefined ? undefined : usage.candidatesTokenCount + (usage.thoughtsTokenCount ?? 0)
return new ImageResponse({
images,
notices: notices.length === 0 ? undefined : notices,
usage:
usage === undefined
? undefined
: {
type: "tokens",
input: usage.promptTokenCount,
output: outputTokens,
total: ProviderShared.totalTokens(usage.promptTokenCount, outputTokens, usage.totalTokenCount),
details: {
reasoningTokens: usage.thoughtsTokenCount,
cacheReadInputTokens: usage.cachedContentTokenCount,
google: usage,
},
},
providerMetadata: {
google: {
modelVersion: decoded.modelVersion,
responseId: decoded.responseId,
promptFeedback: decoded.promptFeedback,
candidates: candidateMetadata,
},
},
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["mask", "size", "format"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<GoogleImageOptions>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => `/models/${request.model.id}:generateContent`,
},
input,
)
export const GoogleImages = {
protocol,
model,
} as const
+89 -121
View File
@@ -1,25 +1,17 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { Effect, Encoding, Schema } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { GeneratedImage, ImageModel, ImageResponse, type ImageRequestFor, type ImageRoute } from "../image.js"
import { Auth } from "../route/auth.js"
import { Usage, mergeHttpOptions, mergeJsonRecords, type HttpOptions } from "../schema/index.js"
import { JsonObject, ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const ADAPTER = "meta-images"
const NAME = "Meta Images"
const PROVIDER = ProviderID.make("meta")
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
import { ImageInputs } from "./utils/image-input.js"
type OpenString<Known extends string> = Known | (string & {})
/** Provider-native options. Common fields (`n`, `size`, `format`, `images`) live on the request. */
export type ImageOptions = {
readonly n?: number
/** Aspect ratio hint, not an exact output resolution. */
readonly size?: string
readonly outputFormat?: OpenString<"webp" | "png" | "jpeg">
readonly responseFormat?: OpenString<"b64_json" | "url">
readonly reasoningStrength?: OpenString<"low" | "high">
readonly toolEnablement?: {
@@ -30,19 +22,12 @@ export type ImageOptions = {
readonly [key: string]: unknown
}
export type Request = ImageRequestFor<ImageOptions>
// ---------------------------------------------------------------------------
// 2. Request body and response schemas
// ---------------------------------------------------------------------------
const Body = Schema.StructWithRest(
Schema.Struct({
model: Schema.String,
prompt: Schema.String,
images: Schema.optional(Schema.Array(JsonObject)),
n: Schema.optional(Schema.Number),
/** Aspect ratio hint, not an exact output resolution. */
size: Schema.optional(Schema.String),
output_format: Schema.optional(Schema.String),
response_format: Schema.optional(Schema.String),
@@ -64,102 +49,85 @@ const Response = Schema.Struct({
),
})
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ image_url: inline.dataUrl })
const url = ProviderShared.mediaUrl(asset)
if (url) return Effect.succeed({ image_url: url })
return Effect.fail(ProviderShared.invalidRequest(`${NAME} accepts image bytes and URLs`))
export const model = (input: {
readonly id: string
readonly auth: Auth.Definition
readonly baseURL: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions
}) => {
const route: ImageRoute<ImageOptions> = {
id: "meta-images",
generate: Effect.fn("MetaImages.generate")(function* (request: ImageRequestFor<ImageOptions>, execute) {
const http = mergeHttpOptions(request.model.http, request.http)
const images = yield* Effect.forEach(request.images ?? [], (image) => {
if (image.type === "bytes") return Effect.succeed({ image_url: ImageInputs.dataUrl(image) })
if (image.type === "url") return Effect.succeed({ image_url: image.url })
return ImageInputs.invalid("Meta Images accepts image bytes and URLs")
})
const { outputFormat, responseFormat, reasoningStrength, toolEnablement, ...native } = request.options ?? {}
const payload = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(Body))(
mergeJsonRecords(
{
model: request.model.id,
prompt: request.prompt,
images: images.length === 0 ? undefined : images,
output_format: outputFormat,
response_format: responseFormat,
reasoning_strength: reasoningStrength,
tool_enablement: toolEnablement,
},
native,
http?.body,
),
)
const body = ProviderShared.encodeJson(payload)
const url = new URL(`${input.baseURL.replace(/\/$/, "")}/images/${images.length === 0 ? "generations" : "edits"}`)
Object.entries(http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value))
const headers = yield* Auth.toEffect(input.auth)({
request,
method: "POST",
url: url.toString(),
body,
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
})
const response = yield* execute(
HttpClientRequest.post(url.toString()).pipe(
HttpClientRequest.setHeaders(headers),
HttpClientRequest.bodyText(body, "application/json"),
),
)
const output = yield* ProviderShared.imageResponse("meta-images", "Meta Images", response)
const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Response))(output.body).pipe(
Effect.mapError((cause) => output.invalid("Meta Images returned an invalid response", cause)),
)
const format = decoded.output_format ?? payload.output_format ?? "webp"
const generated = yield* Effect.forEach(decoded.data, (item, index) => {
if (item.b64_json)
return Effect.fromResult(Encoding.decodeBase64(item.b64_json)).pipe(
Effect.mapError((cause) => output.invalid(`Meta Images result ${index} contains invalid base64`, cause)),
Effect.map((data) => new GeneratedImage({ mediaType: `image/${format}`, data })),
)
if (item.url) return Effect.succeed(new GeneratedImage({ mediaType: `image/${format}`, data: item.url }))
return output.invalid(`Meta Images result ${index} has neither image data nor a URL`)
})
if (generated.length === 0) return yield* output.invalid("Meta Images returned no images")
return new ImageResponse({
images: generated,
usage:
decoded.usage === undefined
? undefined
: new Usage({
inputTokens: decoded.usage.input_tokens,
outputTokens: decoded.usage.output_tokens,
totalTokens: decoded.usage.total_tokens,
providerMetadata: { meta: decoded.usage },
}),
providerMetadata: { meta: { outputFormat: format } },
})
}),
}
return ImageModel.make<ImageOptions>({ id: input.id, provider: "meta", route, http: input.http })
}
const fromRequest = Effect.fn("MetaImages.fromRequest")(function* (request: Request) {
const images = yield* Effect.forEach(request.images ?? [], reference)
const { responseFormat, reasoningStrength, toolEnablement, ...native } = request.providerOptions ?? {}
const payload = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(Body))(
mergeJsonRecords(
{
model: request.model.id,
prompt: request.prompt,
images: images.length === 0 ? undefined : images,
n: request.n,
size: request.size,
output_format: request.format,
response_format: responseFormat,
reasoning_strength: reasoningStrength,
tool_enablement: toolEnablement,
},
native,
request.http?.body,
),
)
return MediaProtocol.json(payload)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeResponse = Effect.fn("MetaImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.DecodeContext<Request>,
) {
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, Response)(response)
const decoded = output.value
const requested = context.body.type === "json" ? context.body.value.output_format : undefined
const format = decoded.output_format ?? (typeof requested === "string" ? requested : "webp")
const mediaType = `image/${format}`
const images = yield* Effect.forEach(decoded.data, (item, index) => {
if (item.b64_json)
return MediaInput.decodedAsset(output.invalid, `${NAME} result ${index}`, item.b64_json, mediaType, {
info: { format },
})
if (item.url) return Effect.succeed(Media.url(item.url, { mediaType, info: { format } }))
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
})
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
return new ImageResponse({
images,
usage:
decoded.usage === undefined
? undefined
: {
type: "tokens",
input: decoded.usage.input_tokens,
output: decoded.usage.output_tokens,
total: decoded.usage.total_tokens,
details: { meta: decoded.usage },
},
providerMetadata: { meta: { outputFormat: format } },
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["mask", "aspectRatio", "seed"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput & { readonly baseURL: string }) =>
ImageModel.fromRoute<ImageOptions>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
path: ({ request }) => `/images/${isEdit(request) ? "edits" : "generations"}`,
},
input,
)
export * as MetaImages from "./meta-images.js"
+6 -8
View File
@@ -224,13 +224,11 @@ type MistralEvent = Schema.Schema.Type<typeof MistralEvent>
const MistralStreamEvent = Schema.Union([Schema.Literal(DONE), Protocol.jsonEvent(MistralEvent)])
const lowerMedia = Effect.fn("MistralChat.lowerMedia")(function* (part: MediaPart) {
const mime = part.media.mediaType.toLowerCase()
const url =
ProviderShared.mediaUrl(part.media) ??
(yield* ProviderShared.requireInlineMedia("Mistral Chat", part.media)).dataUrl
if (mime.startsWith("image/")) return { type: "image_url" as const, image_url: url }
if (mime === "application/pdf") return { type: "document_url" as const, document_url: url }
return yield* ProviderShared.invalidRequest(`Mistral Chat does not support media type ${part.media.mediaType}`)
const media = ProviderShared.normalizeMedia(part)
const url = typeof part.data === "string" && /^(?:https?:|data:)/.test(part.data) ? part.data : media.dataUrl
if (media.mime.startsWith("image/")) return { type: "image_url" as const, image_url: url }
if (media.mime === "application/pdf") return { type: "document_url" as const, document_url: url }
return yield* ProviderShared.invalidRequest(`Mistral Chat does not support media type ${part.mediaType}`)
})
const lowerUser = Effect.fn("MistralChat.lowerUser")(function* (message: LLMRequest["messages"][number]) {
@@ -318,7 +316,7 @@ const lowerToolResults = Effect.fn("MistralChat.lowerToolResults")(function* (
content.push({ type: "text", text: item.text })
continue
}
content.push(yield* lowerMedia(ProviderShared.toolFileMedia(item)))
content.push(yield* lowerMedia({ type: "media", mediaType: item.mime, data: item.uri, filename: item.name }))
}
output.push({
role: "tool",
+23 -25
View File
@@ -18,7 +18,6 @@ import {
type ToolDefinition,
type ToolResultPart,
} from "../schema/index.js"
import type { Media } from "../media.js"
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
import { classifyProviderFailure } from "../provider-error.js"
import { effortUpdate } from "../effort-updates.js"
@@ -404,7 +403,7 @@ export interface ProviderAdapter {
) => Effect.Effect<{ readonly type: string }, AIError>
readonly lowerMedia?: (input: {
readonly part: MediaPart
readonly media: Media.Inline | undefined
readonly media: ProviderShared.NormalizedMedia
readonly request: LLMRequest
}) => MediaInput | undefined
readonly restoreHostedToolItem?: (item: unknown) => HostedToolReplayItem | undefined
@@ -511,28 +510,29 @@ const lowerMedia = Effect.fn("OpenResponses.lowerMedia")(function* (
adapter: ProviderAdapter,
target: "message" | "tool-result",
) {
const media = part.media.inline()
const media = ProviderShared.normalizeMedia(part)
const providerMedia = adapter.lowerMedia?.({ part, media, request })
if (providerMedia) return providerMedia
const detail = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(OpenResponsesInputImage.fields.detail))(
part.providerMetadata?.[metadataKey(request.model)]?.detail,
)
const mime = part.media.mediaType.toLowerCase()
const url = ProviderShared.mediaUrl(part.media)
const location = url ?? (yield* ProviderShared.requireInlineMedia(adapter.name, part.media)).dataUrl
if (part.media.kind !== "image") {
if (target === "tool-result" && part.media.kind === "video")
return { type: "input_video" as const, video_url: location }
const url =
typeof part.data === "string" && (part.data.startsWith("https://") || part.data.startsWith("http://"))
? part.data
: undefined
if (!media.mime.startsWith("image/")) {
if (target === "tool-result" && media.mime.startsWith("video/"))
return { type: "input_video" as const, video_url: url ?? media.dataUrl }
return {
type: "input_file" as const,
filename: part.filename ?? (mime === "application/pdf" ? "document.pdf" : "file"),
filename: part.filename ?? (media.mime === "application/pdf" ? "document.pdf" : "file"),
detail,
...(url ? { file_url: url } : { file_data: location }),
...(url ? { file_url: url } : { file_data: media.dataUrl }),
}
}
return {
type: "input_image" as const,
image_url: location,
image_url: url ?? media.dataUrl,
detail,
}
})
@@ -562,7 +562,12 @@ const lowerToolResultContentItem = Effect.fnUntraced(function* (
adapter: ProviderAdapter,
) {
if (item.type === "text") return { type: "input_text" as const, text: item.text }
return yield* lowerMedia(ProviderShared.toolFileMedia(item), request, adapter, "tool-result")
return yield* lowerMedia(
{ type: "media", mediaType: item.mime, data: item.uri, filename: item.name },
request,
adapter,
"tool-result",
)
})
const lowerHostedToolResultContentItem = Effect.fnUntraced(function* (
@@ -571,7 +576,11 @@ const lowerHostedToolResultContentItem = Effect.fnUntraced(function* (
adapter: ProviderAdapter,
) {
if (item.type === "text") return { type: "input_text" as const, text: item.text }
return yield* lowerMessageMedia(ProviderShared.toolFileMedia(item), request, adapter)
return yield* lowerMessageMedia(
{ type: "media", mediaType: item.mime, data: item.uri, filename: item.name },
request,
adapter,
)
})
const lowerToolResultOutput = Effect.fnUntraced(function* (
@@ -717,22 +726,11 @@ const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (
})
continue
}
if (part.type === "media") {
flushText()
// Responses has no assistant-authored image item; replay generated media (e.g. from Gemini) as user input.
input.push({
type: "message",
role: "user",
content: [yield* lowerMessageMedia(part, request, adapter)],
})
continue
}
return yield* ProviderShared.unsupportedContent(adapter.name, "assistant", [
"text",
"reasoning",
"tool-call",
"tool-result",
"media",
])
}
flushText()
+12 -6
View File
@@ -320,10 +320,13 @@ const lowerToolCall = (part: ToolCallPart, options: LoweringOptions): OpenAIChat
})
const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart) {
if (part.media.kind !== "image")
return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.media.mediaType}`)
const media = ProviderShared.normalizeMedia(part)
if (!media.mime.startsWith("image/"))
return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.mediaType}`)
const url =
ProviderShared.mediaUrl(part.media) ?? (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl
typeof part.data === "string" && (part.data.startsWith("https://") || part.data.startsWith("http://"))
? part.data
: media.dataUrl
return { type: "image_url" as const, image_url: { url } }
})
@@ -457,7 +460,11 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (
cache_control: options.cacheControl?.(part.cache),
})
const files = content.filter((item) => item.type === "file")
images.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item)))))
images.push(
...(yield* Effect.forEach(files, (item) =>
lowerMedia({ type: "media", mediaType: item.mime, data: item.uri, filename: item.name }),
)),
)
}
return { messages, images }
})
@@ -711,8 +718,7 @@ const lowerOptions = (request: LLMRequest, supportsStore: boolean) => {
// Default off: strict providers 400 on unknown body fields, so only send
// the key where compatibility explicitly allows it. Header-based affinity
// (x-session-affinity, x-grok-conv-id, ...) is unaffected.
const cacheKey =
(request.model.compatibility?.supportsPromptCacheKey ?? false) ? ProviderShared.promptCacheKey(request) : undefined
const cacheKey = (request.model.compatibility?.supportsPromptCacheKey ?? false) ? ProviderShared.promptCacheKey(request) : undefined
return {
...(supportsStore && options.store !== undefined ? { store: options.store } : {}),
// For providers that support `store`, ensure stateless `store:false` is sent
+196 -149
View File
@@ -1,39 +1,43 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { Effect, Encoding, Schema } from "effect"
import { Headers, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
import {
ImageModel,
GeneratedImage,
ImageResponse,
type ImageInput,
type ImageRequestFor,
type ImageRoute,
} from "../image.js"
import { Auth, type Definition as AuthDefinition } from "../route/auth.js"
import { Usage, mergeHttpOptions, mergeJsonRecords, type HttpOptions } from "../schema/index.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
import { ImageInputs } from "./utils/image-input.js"
import { OpenAIImage } from "./utils/openai-image.js"
const ADAPTER = "openai-images"
const NAME = "OpenAI Images"
const PROVIDER = ProviderID.make("openai")
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/images/generations"
export const EDIT_PATH = "/images/edits"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type OpenAIImageString<Known extends string> = Known | (string & {})
/** Provider-native options. Common fields (`n`, `size`, `format`, `images`, `mask`) live on the request. */
export type OpenAIImageOptions = {
readonly mask?: ImageInput
readonly n?: number
readonly size?: OpenAIImageString<
"auto" | "256x256" | "512x512" | "1024x1024" | "1536x1024" | "1024x1536" | "1792x1024" | "1024x1792"
>
readonly quality?: OpenAIImageString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
readonly background?: OpenAIImageString<"auto" | "opaque" | "transparent">
readonly moderation?: OpenAIImageString<"auto" | "low">
readonly outputFormat?: OpenAIImageString<"png" | "jpeg" | "webp">
readonly outputCompression?: number
} & Record<string, unknown>
export type Request = ImageRequestFor<OpenAIImageOptions>
// ---------------------------------------------------------------------------
// 2. Response schema
// ---------------------------------------------------------------------------
export type OpenAIImageBody = Record<string, unknown> & {
readonly model: string
readonly prompt: string
}
const OpenAIImageResponse = Schema.Struct({
data: Schema.Array(
@@ -55,153 +59,196 @@ const OpenAIImageResponse = Schema.Struct({
),
})
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
/** Multipart field names the route owns; `http.body` overlays cannot smuggle replacements for them. */
const RESERVED_FORM_FIELDS = new Set(["model", "prompt", "image", "image[]", "images", "mask"])
export interface ModelInput {
readonly id: string
readonly auth: AuthDefinition
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions
}
const nativeOptions = (options: OpenAIImageOptions | undefined) => {
if (!options) return undefined
const { outputCompression, ...native } = options
return { output_compression: outputCompression, ...native }
}
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
const isInline = (asset: Media.Asset) => asset.inline() !== undefined
const blob = (data: Uint8Array, mediaType: string) => {
const buffer = new ArrayBuffer(data.byteLength)
new Uint8Array(buffer).set(data)
return new Blob([buffer], { type: mediaType })
}
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ image_url: inline.dataUrl })
const url = ProviderShared.mediaUrl(asset)
if (url) return Effect.succeed({ image_url: url })
const id = MediaInput.refID(asset, PROVIDER)
if (id) return Effect.succeed({ file_id: id })
return Effect.fail(
ProviderShared.invalidRequest("OpenAI Images accepts image URLs, data URLs, bytes, and OpenAI file IDs"),
)
}
const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Request) {
const images = request.images ?? []
const mask = request.mask
if (mask !== undefined && images.length === 0)
return yield* ProviderShared.invalidRequest("An OpenAI image mask requires at least one input image")
const fields = mergeJsonRecords(
{ n: request.n, size: request.size, output_format: request.format },
nativeOptions(request.providerOptions),
request.http?.body,
)
// Owned bytes go through multipart edits; remote URLs and file IDs use the JSON edits body instead.
if (images.length > 0 && images.every(isInline) && (mask === undefined || isInline(mask))) {
const form = new FormData()
form.append("model", request.model.id)
form.append("prompt", request.prompt)
Object.entries(fields ?? {}).forEach(([key, value]) => {
if (RESERVED_FORM_FIELDS.has(key)) return
form.append(key, typeof value === "string" ? value : ProviderShared.encodeJson(value))
})
const uploads = yield* Effect.forEach(images, (image) => MediaInput.inlineBytes(ADAPTER, image))
uploads.forEach((data, index) => form.append("image[]", blob(data, images[index].mediaType), `image-${index}`))
if (mask !== undefined)
form.append("mask", blob(yield* MediaInput.inlineBytes(ADAPTER, mask), mask.mediaType), "mask")
return MediaProtocol.multipart(form)
const { mask: _, outputFormat, outputCompression, ...native } = options
return {
output_format: outputFormat,
output_compression: outputCompression,
...native,
}
const references = yield* Effect.forEach(images, reference)
const maskReference = mask === undefined ? undefined : yield* reference(mask)
return MediaProtocol.json(
mergeJsonRecords(
{
model: request.model.id,
prompt: request.prompt,
images: references.length === 0 ? undefined : references,
mask: maskReference,
},
fields,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const requestedFormat = (body: MediaProtocol.Body) => {
const value = body.type === "json" ? body.value.output_format : body.value.get("output_format")
return typeof value === "string" ? value : undefined
}
const decodeResponse = Effect.fn("OpenAIImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.DecodeContext<Request>,
) {
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, OpenAIImageResponse)(response)
const decoded = output.value
const format = decoded.output_format ?? requestedFormat(context.body) ?? "png"
const mediaType = `image/${format}`
const images = yield* Effect.forEach(decoded.data, (item, index) => {
const providerMetadata =
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } }
if (item.b64_json)
return MediaInput.decodedAsset(output.invalid, `${NAME} result ${index}`, item.b64_json, mediaType, {
info: { format },
providerMetadata,
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
if (!query) return url
const next = new URL(url)
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
return next.toString()
}
export const model = (input: ModelInput) => {
const route: ImageRoute<OpenAIImageOptions> = {
id: ADAPTER,
generate: Effect.fn("OpenAIImages.generate")(function* (request: ImageRequestFor<OpenAIImageOptions>, execute) {
const mask = request.options?.mask
if (mask !== undefined && (request.images?.length ?? 0) === 0)
return yield* ImageInputs.invalid("An OpenAI image mask requires at least one input image")
const http = mergeHttpOptions(request.model.http, request.http)
const sourceImages = request.images ?? []
const multipartImages = yield* Effect.forEach(sourceImages, (image) => {
if (image.type === "bytes") return Effect.succeed({ data: image.data, mediaType: image.mediaType })
if (image.type === "url") return ImageInputs.decodeDataUrl(image.url)
return Effect.undefined
})
if (item.url) return Effect.succeed(Media.url(item.url, { mediaType, info: { format }, providerMetadata }))
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
const multipartMask =
mask === undefined
? undefined
: mask.type === "bytes"
? { data: mask.data, mediaType: mask.mediaType }
: mask.type === "url"
? yield* ImageInputs.decodeDataUrl(mask.url)
: undefined
const useMultipart =
sourceImages.length > 0 &&
multipartImages.every((image) => image !== undefined) &&
(mask === undefined || multipartMask !== undefined)
const path = sourceImages.length === 0 ? PATH : EDIT_PATH
const url = applyQuery(`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}${path}`, http?.query)
if (useMultipart) {
const form = new FormData()
form.append("model", request.model.id)
form.append("prompt", request.prompt)
Object.entries(mergeJsonRecords(nativeOptions(request.options), http?.body) ?? {}).forEach(([key, value]) => {
if (["model", "prompt", "image", "image[]", "images", "mask"].includes(key)) return
form.append(key, typeof value === "string" ? value : ProviderShared.encodeJson(value))
})
multipartImages.forEach((image, index) => {
if (image === undefined) return
form.append("image[]", imageBlob(image.data, image.mediaType), `image-${index}`)
})
if (multipartMask !== undefined)
form.append("mask", imageBlob(multipartMask.data, multipartMask.mediaType), "mask")
const headers = yield* Auth.toEffect(input.auth)({
request,
method: "POST",
url,
body: "[multipart/form-data]",
headers: Headers.remove(Headers.fromInput({ ...input.headers, ...http?.headers }), "content-type"),
})
const response = yield* execute(
HttpClientRequest.post(url).pipe(HttpClientRequest.setHeaders(headers), HttpClientRequest.bodyFormData(form)),
)
return yield* parseResponse(response, request.options, http?.body)
}
const references = sourceImages.map((image) => {
if (image.type === "bytes") return { image_url: ImageInputs.dataUrl(image) }
if (image.type === "url") return { image_url: image.url }
if (image.type === "file-id") return { file_id: image.id }
return undefined
})
if (references.some((image) => image === undefined))
return yield* ImageInputs.invalid("OpenAI Images accepts image URLs, data URLs, bytes, and file IDs")
const maskReference =
mask === undefined
? undefined
: mask.type === "bytes"
? { image_url: ImageInputs.dataUrl(mask) }
: mask.type === "url"
? { image_url: mask.url }
: mask.type === "file-id"
? { file_id: mask.id }
: undefined
if (mask !== undefined && maskReference === undefined)
return yield* ImageInputs.invalid("OpenAI Images accepts masks as URLs, data URLs, bytes, or file IDs")
const requestBody = mergeJsonRecords(
{
model: request.model.id,
prompt: request.prompt,
images: references.length === 0 ? undefined : references,
mask: maskReference,
},
nativeOptions(request.options),
http?.body,
) as OpenAIImageBody
const text = ProviderShared.encodeJson(requestBody)
const headers = yield* Auth.toEffect(input.auth)({
request,
method: "POST",
url,
body: text,
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
})
const response = yield* execute(
HttpClientRequest.post(url).pipe(
HttpClientRequest.setHeaders(headers),
HttpClientRequest.bodyText(text, "application/json"),
),
)
return yield* parseResponse(response, request.options, http?.body)
}),
}
return ImageModel.make<OpenAIImageOptions>({ id: input.id, provider: "openai", route, http: input.http })
}
const parseResponse = Effect.fn("OpenAIImages.parseResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
options: OpenAIImageOptions | undefined,
overlay: Record<string, unknown> | undefined,
) {
const output = yield* ProviderShared.imageResponse(ADAPTER, "OpenAI Images", response)
const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(OpenAIImageResponse))(output.body).pipe(
Effect.mapError((cause) => output.invalid("OpenAI Images returned an invalid response", cause)),
)
const requestBody = mergeJsonRecords(nativeOptions(options), overlay)
const format =
decoded.output_format ?? (typeof requestBody?.output_format === "string" ? requestBody.output_format : "png")
const images = yield* Effect.forEach(decoded.data, (item, index) => {
if (item.b64_json)
return Effect.fromResult(Encoding.decodeBase64(item.b64_json)).pipe(
Effect.mapError((cause) => output.invalid(`OpenAI Images result ${index} contains invalid base64 data`, cause)),
Effect.map(
(data) =>
new GeneratedImage({
mediaType: `image/${format}`,
data,
providerMetadata:
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } },
}),
),
)
if (item.url)
return Effect.succeed(
new GeneratedImage({
mediaType: `image/${format}`,
data: item.url,
providerMetadata:
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } },
}),
)
return Effect.fail(output.invalid(`OpenAI Images result ${index} has neither image data nor a URL`))
})
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
if (images.length === 0) return yield* output.invalid("OpenAI Images returned no images")
return new ImageResponse({
images,
usage:
decoded.usage === undefined
? undefined
: {
type: "tokens",
input: decoded.usage.input_tokens,
output: decoded.usage.output_tokens,
total: decoded.usage.total_tokens,
details: { openai: decoded.usage },
},
: new Usage({
inputTokens: decoded.usage.input_tokens,
outputTokens: decoded.usage.output_tokens,
totalTokens: decoded.usage.total_tokens,
providerMetadata: { openai: decoded.usage },
}),
providerMetadata: { openai: { outputFormat: format } },
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["aspectRatio", "seed"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<OpenAIImageOptions>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
},
input,
)
const imageBlob = (data: Uint8Array, mediaType: string) => {
const buffer = new ArrayBuffer(data.byteLength)
new Uint8Array(buffer).set(data)
return new Blob([buffer], { type: mediaType })
}
export const OpenAIImages = {
protocol,
model,
} as const
+46 -26
View File
@@ -1,13 +1,14 @@
import { Buffer } from "node:buffer"
import { Tool } from "@opencode/schema/tool"
import { Effect, Option, Schema, Stream } from "effect"
import * as Sse from "effect/unstable/encoding/Sse"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { Media } from "../media.js"
import { Headers, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
import {
InvalidProviderOutputError,
InvalidRequestError,
UnsupportedOperationError,
AIError,
HttpContext,
LLMRequest,
Message,
ToolDefinition,
@@ -178,34 +179,25 @@ export const wrappedSystemUpdate = Effect.fn("ProviderShared.wrappedSystemUpdate
export const parseToolInput = (route: string, name: string, raw: string) =>
parseJson(route, raw || "{}", `Invalid JSON input for ${route} tool call ${name}`)
/** Inline view or a typed `InvalidRequest` for routes that cannot fetch URLs or dereference provider refs. */
export const requireInlineMedia = (route: string, asset: Media.Asset): Effect.Effect<Media.Inline, AIError> => {
const inline = asset.inline()
return inline ? Effect.succeed(inline) : Effect.fail(inlineRequired(route, asset))
export interface NormalizedMedia {
readonly mime: string
readonly base64: string
readonly dataUrl: string
}
export const inlineRequired = (route: string, asset: Media.Asset) =>
invalidRequest(
`${route} requires inline media (bytes or base64); ${asset.source.type} sources must be materialized first`,
)
/** The remote URL of a `url` asset, for protocols that accept `http(s)` references natively. */
export const mediaUrl = (asset: Media.Asset) => (asset.source.type === "url" ? asset.source.url : undefined)
/**
* Lift a tool-result file into a `MediaPart`. Tool files carry either a data URL, an `http(s)` URL, or raw base64 in
* `uri`; the declared `mime` wins over any data-URL prefix so tool authors control the type the model sees.
*/
export const toolFileMedia = (item: Tool.FileContent): MediaPart => {
const parsed = Media.parseDataUrl(item.uri)
const asset = parsed
? Media.from({ ...parsed.source, mediaType: item.mime })
: /^https?:\/\//.test(item.uri)
? Media.url(item.uri, { mediaType: item.mime })
: Media.base64(item.uri, item.mime)
return Message.media(asset, { filename: item.name })
export const normalizeMedia = (part: MediaPart): NormalizedMedia => {
const mime = part.mediaType.toLowerCase()
if (typeof part.data !== "string") {
const base64 = Buffer.from(part.data).toString("base64")
return { mime, base64, dataUrl: `data:${mime};base64,${base64}` }
}
if (!part.data.startsWith("data:")) return { mime, base64: part.data, dataUrl: `data:${mime};base64,${part.data}` }
return { mime, base64: part.data.slice(part.data.indexOf(",") + 1), dataUrl: part.data }
}
export const normalizeToolFile = (part: Tool.FileContent) =>
normalizeMedia({ type: "media", mediaType: part.mime, data: part.uri, filename: part.name })
export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
export const toolResultText = (part: ToolResultPart) => {
@@ -333,6 +325,34 @@ export const flattenToolRequest = (request: LLMRequest) => {
}
}
export const imageResponse = Effect.fn("ProviderShared.imageResponse")(function* (
route: string,
name: string,
response: HttpClientResponse.HttpClientResponse,
) {
const http = new HttpContext({ url: response.request.url, status: response.status, headers: response.headers })
const body = yield* response.text.pipe(
Effect.mapError(
(cause) =>
new AIError({
reason: new InvalidProviderOutputError({
route,
message: `Failed to read the ${name} response`,
http,
cause,
}),
}),
),
)
return {
body,
invalid: (message: string, cause?: unknown) =>
new AIError({
reason: new InvalidProviderOutputError({ route, message, body, http, cause }),
}),
}
})
export const matchToolChoice = <Auto, None, Required, Tool>(
route: string,
toolChoice: NonNullable<LLMRequest["toolChoice"]>,
@@ -77,7 +77,7 @@ function documentName(filename: string | undefined, names: Set<string>) {
}
const mediaBase64 = Effect.fn("BedrockMedia.mediaBase64")(function* (part: MediaPart) {
const media = yield* ProviderShared.requireInlineMedia("Bedrock Converse", part.media)
const media = ProviderShared.normalizeMedia(part)
const bytes = yield* Effect.fromResult(Encoding.decodeBase64(media.base64)).pipe(
Effect.mapError((cause) =>
ProviderShared.invalidRequest("Bedrock Converse media data must be valid base64", cause),
@@ -92,15 +92,13 @@ const mediaBase64 = Effect.fn("BedrockMedia.mediaBase64")(function* (part: Media
// get an image-specific error so the caller knows it's a format-support issue,
// not a kind-detection issue.
export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart, documentNames: Set<string>) {
const mime = part.media.mediaType.toLowerCase()
const mime = part.mediaType.toLowerCase()
const imageFormat = IMAGE_FORMATS[mime as keyof typeof IMAGE_FORMATS]
if (imageFormat) {
return [{ image: { format: imageFormat, source: { bytes: yield* mediaBase64(part) } } } satisfies ImageBlock]
}
if (mime.startsWith("image/"))
return yield* ProviderShared.invalidRequest(
`Bedrock Converse does not support image media type ${part.media.mediaType}`,
)
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`)
const documentFormat = DOCUMENT_FORMATS[mime as keyof typeof DOCUMENT_FORMATS]
if (documentFormat) {
const name = documentName(part.filename, documentNames)
@@ -114,7 +112,7 @@ export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart,
]
: [block]
}
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.media.mediaType}`)
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
})
export * as BedrockMedia from "./bedrock-media.js"
@@ -0,0 +1,31 @@
import { Effect, Encoding } from "effect"
import type { ImageInput } from "../../image.js"
import { InvalidRequestError, AIError } from "../../schema/index.js"
const invalid = (message: string, cause?: unknown) =>
new AIError({
reason: new InvalidRequestError({ message, cause }),
})
export const dataUrl = (input: Extract<ImageInput, { readonly type: "bytes" }>) =>
`data:${input.mediaType};base64,${Encoding.encodeBase64(input.data)}`
export const decodeDataUrl = (
url: string,
): Effect.Effect<{ readonly mediaType: string; readonly data: Uint8Array } | undefined, AIError> => {
if (!url.startsWith("data:")) return Effect.undefined
const match = /^data:([^;,]+);base64,(.*)$/s.exec(url)
if (!match) return Effect.fail(invalid("Image data URLs must contain a MIME type and base64 data"))
return Effect.fromResult(Encoding.decodeBase64(match[2])).pipe(
Effect.mapError((cause) => invalid("Image data URL contains invalid base64 data", cause)),
Effect.map((data) => ({ mediaType: match[1], data })),
)
}
export const invalidImageInput = invalid
export const ImageInputs = {
dataUrl,
decodeDataUrl,
invalid: invalidImageInput,
} as const
@@ -1,33 +0,0 @@
import { Effect, Encoding } from "effect"
import { Media } from "../../media.js"
import type { AIError, ProviderID } from "../../schema/index.js"
import { ProviderShared } from "../shared.js"
/** Owned bytes for multipart uploads; decodes `base64` sources and rejects remote sources. */
export const inlineBytes = (route: string, asset: Media.Asset): Effect.Effect<Uint8Array, AIError> => {
if (asset.source.type === "bytes") return Effect.succeed(asset.source.data)
const inline = asset.inline()
if (!inline) return Effect.fail(ProviderShared.inlineRequired(route, asset))
return Effect.fromResult(Encoding.decodeBase64(inline.base64)).pipe(
Effect.mapError((cause) => ProviderShared.invalidRequest(`${route} media contains invalid base64 data`, cause)),
)
}
/** Provider file handle when the ref belongs to this provider; refs from other providers are never forwarded. */
export const refID = (asset: Media.Asset, provider: ProviderID) =>
asset.source.type === "ref" && asset.source.provider === provider ? asset.source.id : undefined
/** Decode a provider's base64 output once into an owned `bytes` asset, sniffing the type when it is not declared. */
export const decodedAsset = (
invalid: (message: string, cause?: unknown) => AIError,
label: string,
data: string,
mediaType: string | undefined,
options?: Media.AssetOptions,
) =>
Effect.fromResult(Encoding.decodeBase64(data)).pipe(
Effect.mapError((cause) => invalid(`${label} contains invalid base64 data`, cause)),
Effect.map((bytes) => Media.bytes(bytes, mediaType, options)),
)
export * as MediaInput from "./media-input.js"
@@ -17,7 +17,6 @@ import { RequestExecutor } from "../../route/executor.js"
import { HttpTransport } from "../../route/transport/index.js"
import { OpenResponses } from "../open-responses.js"
import { JsonObject, optionalNull, ProviderShared } from "../shared.js"
import { Media } from "../../media.js"
const Body = Schema.Struct({
model: Schema.String,
@@ -158,22 +157,20 @@ function toMessage(item: (typeof Response.Type.output)[number], model: LLMReques
if (part.type === "input_image")
return {
type: "media",
media: replayMedia(part.image_url, "image/*"),
data: part.image_url,
mediaType: /^data:([^;,]+)/.exec(part.image_url)?.[1] ?? "image/*",
providerMetadata: part.detail === undefined ? undefined : { [key]: { detail: part.detail } },
}
const data = part.file_url === undefined ? part.file_data : part.file_url
return {
type: "media",
media: replayMedia(part.file_url === undefined ? part.file_data : part.file_url, "application/octet-stream"),
data,
filename: part.filename,
mediaType: /^data:([^;,]+)/.exec(data)?.[1] ?? "application/octet-stream",
providerMetadata: part.detail === undefined ? undefined : { [key]: { detail: part.detail } },
}
}),
})
}
/** Replayed compaction items carry either a data URL or a remote URL; the data URL's own type wins when present. */
const replayMedia = (value: string, fallbackType: string) =>
Media.parseDataUrl(value) ??
(/^https?:\/\//.test(value) ? Media.url(value, { mediaType: fallbackType }) : Media.base64(value, fallbackType))
export * as ResponsesCompaction from "./responses-compaction.js"
+146 -112
View File
@@ -1,38 +1,61 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { Effect, Encoding, Schema } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { GeneratedImage, ImageModel, ImageResponse, type ImageRequestFor, type ImageRoute } from "../image.js"
import { Auth, type Definition as AuthDefinition } from "../route/auth.js"
import { Usage, mergeHttpOptions, mergeJsonRecords, type HttpOptions } from "../schema/index.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
import { ImageInputs } from "./utils/image-input.js"
const ADAPTER = "xai-images"
const NAME = "xAI Images"
const PROVIDER = ProviderID.make("xai")
export const DEFAULT_BASE_URL = "https://api.x.ai/v1"
export const PATH = "/images/generations"
export const EDIT_PATH = "/images/edits"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type XAIImageString<Known extends string> = Known | (string & {})
/** Provider-native options. Common fields (`n`, `aspectRatio`, `images`) live on the request. */
export type XAIImageOptions = {
readonly n?: number
readonly aspectRatio?: XAIImageString<
| "1:1"
| "3:4"
| "4:3"
| "9:16"
| "16:9"
| "2:3"
| "3:2"
| "9:19.5"
| "19.5:9"
| "9:20"
| "20:9"
| "1:2"
| "2:1"
| "auto"
>
readonly aspect_ratio?: XAIImageString<
| "1:1"
| "3:4"
| "4:3"
| "9:16"
| "16:9"
| "2:3"
| "3:2"
| "9:19.5"
| "19.5:9"
| "9:20"
| "20:9"
| "1:2"
| "2:1"
| "auto"
>
readonly resolution?: XAIImageString<"1k" | "2k">
readonly responseFormat?: XAIImageString<"url" | "b64_json">
readonly response_format?: XAIImageString<"url" | "b64_json">
} & Record<string, unknown>
export type Request = ImageRequestFor<XAIImageOptions>
// ---------------------------------------------------------------------------
// 2. Response schema
// ---------------------------------------------------------------------------
type XAIImageBody = Record<string, unknown> & {
readonly model: string
readonly prompt: string
}
const XAIImageResponse = Schema.Struct({
data: Schema.Array(
@@ -46,109 +69,120 @@ const XAIImageResponse = Schema.Struct({
usage: Schema.optional(Schema.Unknown),
})
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
export interface ModelInput {
readonly id: string
readonly auth: AuthDefinition
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions
}
const nativeOptions = (options: XAIImageOptions | undefined) => {
if (!options) return undefined
const { responseFormat, ...native } = options
return { response_format: responseFormat, ...native }
const { aspectRatio, responseFormat, ...native } = options
return {
aspect_ratio: aspectRatio,
response_format: responseFormat,
...native,
}
}
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ url: inline.dataUrl, type: "image_url" as const })
const url = ProviderShared.mediaUrl(asset)
if (url) return Effect.succeed({ url, type: "image_url" as const })
const id = MediaInput.refID(asset, PROVIDER)
if (id) return Effect.succeed({ file_id: id })
return Effect.fail(ProviderShared.invalidRequest(`${NAME} accepts image URLs, data URLs, bytes, and xAI file IDs`))
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
if (!query) return url
const next = new URL(url)
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
return next.toString()
}
const fromRequest = Effect.fn("XAIImages.fromRequest")(function* (request: Request) {
const references = yield* Effect.forEach(request.images ?? [], reference)
return MediaProtocol.json(
mergeJsonRecords(
{
model: request.model.id,
prompt: request.prompt,
// xAI takes one edit source as `image` and several as `images`.
image: references.length === 1 ? references[0] : undefined,
images: references.length > 1 ? references : undefined,
n: request.n,
aspect_ratio: request.aspectRatio,
},
nativeOptions(request.providerOptions),
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, XAIImageResponse)(response)
const decoded = output.value
const images = yield* Effect.forEach(decoded.data, (item, index) => {
const providerMetadata =
item.revised_prompt === undefined || item.revised_prompt === null
? undefined
: { xai: { revisedPrompt: item.revised_prompt } }
if (item.b64_json)
return MediaInput.decodedAsset(
output.invalid,
`${NAME} result ${index}`,
item.b64_json,
item.mime_type ?? undefined,
export const model = (input: ModelInput) => {
const route: ImageRoute<XAIImageOptions> = {
id: ADAPTER,
generate: Effect.fn("XAIImages.generate")(function* (request: ImageRequestFor<XAIImageOptions>, execute) {
const http = mergeHttpOptions(request.model.http, request.http)
const imageReferences = (request.images ?? []).map((image) => {
if (image.type === "bytes") return { url: ImageInputs.dataUrl(image), type: "image_url" as const }
if (image.type === "url") return { url: image.url, type: "image_url" as const }
if (image.type === "file-id") return { file_id: image.id }
return undefined
})
if (imageReferences.some((image) => image === undefined))
return yield* ImageInputs.invalid("xAI Images accepts image URLs, data URLs, bytes, and file IDs")
const requestBody = mergeJsonRecords(
{
providerMetadata,
model: request.model.id,
prompt: request.prompt,
image: imageReferences.length === 1 ? imageReferences[0] : undefined,
images: imageReferences.length > 1 ? imageReferences : undefined,
},
nativeOptions(request.options),
http?.body,
) as XAIImageBody
const text = ProviderShared.encodeJson(requestBody)
const url = applyQuery(
`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}${imageReferences.length === 0 ? PATH : EDIT_PATH}`,
http?.query,
)
if (item.url)
return Effect.succeed(Media.url(item.url, { mediaType: item.mime_type ?? undefined, providerMetadata }))
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
})
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
const usage = ProviderShared.isRecord(decoded.usage) ? decoded.usage : undefined
// xAI reports image counts rather than tokens, seconds, or credits; the raw record stays in provider metadata.
return new ImageResponse({
images,
providerMetadata: usage === undefined ? undefined : { xai: { usage } },
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["mask", "size", "seed", "format"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<XAIImageOptions>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
},
input,
)
const headers = yield* Auth.toEffect(input.auth)({
request,
method: "POST",
url,
body: text,
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
})
const response = yield* execute(
HttpClientRequest.post(url).pipe(
HttpClientRequest.setHeaders(headers),
HttpClientRequest.bodyText(text, "application/json"),
),
)
const output = yield* ProviderShared.imageResponse(ADAPTER, "xAI Images", response)
const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(XAIImageResponse))(output.body).pipe(
Effect.mapError((cause) => output.invalid("xAI Images returned an invalid response", cause)),
)
const images = yield* Effect.forEach(decoded.data, (item, index) => {
const mediaType = item.mime_type ?? "application/octet-stream"
if (item.b64_json)
return Effect.fromResult(Encoding.decodeBase64(item.b64_json)).pipe(
Effect.mapError((cause) =>
output.invalid(`xAI Images result ${index} contains invalid base64 data`, cause),
),
Effect.map(
(data) =>
new GeneratedImage({
mediaType,
data,
providerMetadata:
item.revised_prompt === undefined || item.revised_prompt === null
? undefined
: { xai: { revisedPrompt: item.revised_prompt } },
}),
),
)
if (item.url)
return Effect.succeed(
new GeneratedImage({
mediaType,
data: item.url,
providerMetadata:
item.revised_prompt === undefined || item.revised_prompt === null
? undefined
: { xai: { revisedPrompt: item.revised_prompt } },
}),
)
return Effect.fail(output.invalid(`xAI Images result ${index} has neither image data nor a URL`))
})
if (images.length === 0) return yield* output.invalid("xAI Images returned no images")
const usage = ProviderShared.isRecord(decoded.usage) ? decoded.usage : undefined
return new ImageResponse({
images,
usage: usage === undefined ? undefined : new Usage({ providerMetadata: { xai: usage } }),
providerMetadata: usage === undefined ? undefined : { xai: { usage } },
})
}),
}
return ImageModel.make<XAIImageOptions>({ id: input.id, provider: "xai", route, http: input.http })
}
export const XAIImages = {
protocol,
model,
} as const
+83 -85
View File
@@ -1,34 +1,29 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { GeneratedImage, ImageModel, ImageResponse, type ImageRequestFor, type ImageRoute } from "../image.js"
import { Auth, type Definition as AuthDefinition } from "../route/auth.js"
import { mergeHttpOptions, mergeJsonRecords, type HttpOptions } from "../schema/index.js"
import { ProviderShared } from "./shared.js"
import { ImageInputs } from "./utils/image-input.js"
const ADAPTER = "zai-images"
const NAME = "Z.ai Images"
const PROVIDER = ProviderID.make("zai")
export const DEFAULT_BASE_URL = "https://api.z.ai/api/paas/v4"
export const PATH = "/images/generations"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type ZAIImageString<Known extends string> = Known | (string & {})
/** Provider-native options. The common `size` field lives on the request. */
export type ZAIImageOptions = {
readonly size?: ZAIImageString<
"1024x1024" | "768x1344" | "864x1152" | "1344x768" | "1152x864" | "1440x720" | "720x1440"
>
readonly quality?: ZAIImageString<"hd" | "standard">
readonly userID?: string
} & Record<string, unknown>
export type Request = ImageRequestFor<ZAIImageOptions>
// ---------------------------------------------------------------------------
// 2. Response schema
// ---------------------------------------------------------------------------
type ZAIImageBody = Record<string, unknown> & {
readonly model: string
readonly prompt: string
}
const ZAIImageResponse = Schema.Struct({
created: Schema.optional(Schema.Int),
@@ -45,81 +40,84 @@ const ZAIImageResponse = Schema.Struct({
),
})
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
export interface ModelInput {
readonly id: string
readonly auth: AuthDefinition
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions
}
const nativeOptions = (options: ZAIImageOptions | undefined) => {
if (!options) return undefined
const { userID, ...native } = options
return { user_id: userID, ...native }
return {
user_id: userID,
...native,
}
}
const fromRequest = Effect.fn("ZAIImages.fromRequest")(function* (request: Request) {
return MediaProtocol.json(
mergeJsonRecords(
{ model: request.model.id, prompt: request.prompt, size: request.size },
nativeOptions(request.providerOptions),
request.http?.body,
) ?? {},
)
})
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
if (!query) return url
const next = new URL(url)
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
return next.toString()
}
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeResponse = Effect.fn("ZAIImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, ZAIImageResponse)(response)
const decoded = output.value
if (decoded.data.length === 0) return yield* output.invalid(`${NAME} returned no images`)
const filters = decoded.content_filter ?? []
return new ImageResponse({
// Z.ai returns only URLs and no content type; the media type resolves when the asset is materialized.
images: decoded.data.map((item) => Media.url(item.url)),
// Z.ai reports applied content filters alongside a successful result; surface them instead of dropping them.
notices:
filters.length === 0
? undefined
: filters.map((filter) => ({
type: "moderated" as const,
message: `${NAME} applied a content filter${filter.role === undefined ? "" : ` for ${filter.role}`}${
filter.level === undefined ? "" : ` at level ${filter.level}`
}`,
providerMetadata: { zai: filter },
})),
providerMetadata: {
zai: {
created: decoded.created,
id: decoded.id,
requestID: decoded.request_id,
contentFilter: decoded.content_filter,
},
},
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
unsupported: ["images", "mask", "n", "aspectRatio", "seed", "format"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<ZAIImageOptions>(
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
export const model = (input: ModelInput) => {
const route: ImageRoute<ZAIImageOptions> = {
id: ADAPTER,
generate: Effect.fn("ZAIImages.generate")(function* (request: ImageRequestFor<ZAIImageOptions>, execute) {
if ((request.images?.length ?? 0) > 0)
return yield* ImageInputs.invalid("Z.ai hosted image generation does not support image inputs")
const http = mergeHttpOptions(request.model.http, request.http)
const requestBody = mergeJsonRecords(
{ model: request.model.id, prompt: request.prompt },
nativeOptions(request.options),
http?.body,
) as ZAIImageBody
const text = ProviderShared.encodeJson(requestBody)
const url = applyQuery(`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}${PATH}`, http?.query)
const headers = yield* Auth.toEffect(input.auth)({
request,
method: "POST",
url,
body: text,
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
})
const response = yield* execute(
HttpClientRequest.post(url).pipe(
HttpClientRequest.setHeaders(headers),
HttpClientRequest.bodyText(text, "application/json"),
),
)
const output = yield* ProviderShared.imageResponse(ADAPTER, "Z.ai Images", response)
const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(ZAIImageResponse))(output.body).pipe(
Effect.mapError((cause) => output.invalid("Z.ai Images returned an invalid response", cause)),
)
if (decoded.data.length === 0) return yield* output.invalid("Z.ai Images returned no images")
return new ImageResponse({
images: decoded.data.map(
(item) =>
new GeneratedImage({
mediaType: "application/octet-stream",
data: item.url,
}),
),
providerMetadata: {
zai: {
created: decoded.created,
id: decoded.id,
requestID: decoded.request_id,
contentFilter: decoded.content_filter,
},
},
})
}),
}
return ImageModel.make<ZAIImageOptions>({ id: input.id, provider: "zai", route, http: input.http })
}
export const ZAIImages = {
protocol,
model,
} as const
+1 -6
View File
@@ -44,12 +44,7 @@ export const configure = (input: LanguageModelOptions = {}) => {
model: (modelID: string | ModelID) =>
configured.model<OpenAIProviderOptionsInput>({
id: modelID,
compatibility: {
maxTokensField: "max_tokens",
reasoningField: "reasoning",
supportsStore: false,
supportsPromptCacheKey: true,
},
compatibility: { maxTokensField: "max_tokens", reasoningField: "reasoning", supportsStore: false, supportsPromptCacheKey: true },
}),
configure,
}
+1 -6
View File
@@ -47,12 +47,7 @@ export const configure = (input: LanguageModelOptions = {}) => {
model: (modelID: string | ModelID) =>
configured.model<OpenAIProviderOptionsInput>({
id: modelID,
compatibility: {
maxTokensField: "max_tokens",
reasoningField: "reasoning_content",
supportsStore: false,
supportsPromptCacheKey: true,
},
compatibility: { maxTokensField: "max_tokens", reasoningField: "reasoning_content", supportsStore: false, supportsPromptCacheKey: true },
}),
configure,
}
+11 -1
View File
@@ -38,13 +38,23 @@ export type Settings = ProviderPackage.Settings &
const fromRequest = Effect.fn("GoogleVertex.fromRequest")(function* (request: LLMRequest) {
const { serviceTier: _, ...body } = yield* Gemini.protocol.body.from(request)
// Vertex's native REST schema rejects `id` on FunctionCall/FunctionResponse parts with HTTP 400,
// unlike AI Studio, so history minted there cannot be lowered verbatim.
const contents = body.contents.map((content) => ({
...content,
parts: (content.parts ?? []).map((part) => {
if ("functionCall" in part) return { ...part, functionCall: { ...part.functionCall, id: undefined } }
if ("functionResponse" in part) return { ...part, functionResponse: { ...part.functionResponse, id: undefined } }
return part
}),
}))
const value = request.providerOptions?.labels
const labels = ProviderShared.isRecord(value)
? Object.fromEntries(
Object.entries(value).filter((entry): entry is [string, string] => typeof entry[1] === "string"),
)
: undefined
return { ...body, labels }
return { ...body, contents, labels }
})
const protocol = {
-3
View File
@@ -24,11 +24,8 @@ export * as Moonshot from "./moonshot.js"
export * as OpenAI from "./openai.js"
export * as OpenAICompatible from "./openai-compatible.js"
export * as OpenAICompatibleResponses from "./openai-compatible-responses.js"
export * as OpenCodeZen from "./opencode-zen.js"
export * as OpenRouter from "./openrouter.js"
export * as TogetherAI from "./togetherai.js"
export * as TypeSafeAI from "./typesafe-ai.js"
export * as VercelAIGateway from "./vercel-ai-gateway.js"
export * as XAI from "./xai.js"
export * as ZAI from "./zai.js"
export * as ZAICodingPlan from "./zai-coding-plan.js"
-31
View File
@@ -1,31 +0,0 @@
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { SystemOne } from "../experimental/system-one.js"
export const id = ProviderID.make("opencode")
const baseURL = "https://opencode.ai/zen/v1"
export type Options = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
export const configure = (input: Options = {}) => {
const evaluation = (modelID: string | ModelID) =>
SystemOne.model({
id: modelID,
provider: id,
providerMetadataKey: "opencode",
auth: AuthOptions.bearer(input, "OPENCODE_API_KEY"),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return { id, experimental: { evaluation }, configure }
}
export const provider = configure()
export const experimental = provider.experimental
export * as OpenCodeZen from "./opencode-zen.js"
+1 -22
View File
@@ -3,9 +3,8 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { Protocol } from "../route/protocol.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { HttpOptions, ProviderID, type CacheHint, type ModelID } from "../schema/index.js"
import { ProviderID, type CacheHint, type ModelID } from "../schema/index.js"
import type { ProviderPackage } from "../provider-package.js"
import { SystemOne } from "../experimental/system-one.js"
import { OpenAIChat } from "../protocols/openai-chat.js"
import { newBreakpoints, ttlBucket } from "../protocols/utils/cache.js"
import { isRecord } from "../protocols/shared.js"
@@ -72,14 +71,6 @@ export interface OpenRouterOptions {
export type OpenRouterProviderOptionsInput = OpenRouterOptions
export interface OpenRouterEvaluationOptions {
readonly [key: string]: unknown
readonly provider?: OpenRouterProviderRouting
readonly session_id?: string
readonly trace?: Readonly<Record<string, unknown>>
readonly user?: string
}
export type LanguageModelOptions = Omit<RouteDefaultsInput, "providerOptions"> &
ProviderAuthOption<"optional"> & {
readonly baseURL?: string
@@ -190,27 +181,15 @@ const configuredRoute = (input: LanguageModelOptions) => {
export const configure = (input: LanguageModelOptions = {}) => {
const route = configuredRoute(input)
const evaluation = (modelID: string | ModelID) =>
SystemOne.model<OpenRouterEvaluationOptions>({
id: modelID,
provider: id,
providerMetadataKey: "openrouter",
auth: AuthOptions.bearer(input, "OPENROUTER_API_KEY"),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return {
id,
model: (modelID: string | ModelID) =>
route.model<OpenRouterProviderOptionsInput>({ id: modelID, compatibility: { supportsPromptCacheKey: true } }),
experimental: { evaluation },
configure,
}
}
export const provider = configure()
export const experimental = provider.experimental
export const model: ProviderPackage.Definition<Settings, OpenRouterProviderOptionsInput>["model"] = (
modelID,
{ apiKey, baseURL, body, headers, ...providerOptions },
-31
View File
@@ -1,31 +0,0 @@
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { SystemOne } from "../experimental/system-one.js"
export const id = ProviderID.make("typesafe-ai")
const baseURL = "https://api.typesafe.ai/v1"
export type Options = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
export const configure = (input: Options = {}) => {
const evaluation = (modelID: string | ModelID) =>
SystemOne.model({
id: modelID,
provider: id,
providerMetadataKey: "typesafe",
auth: AuthOptions.bearer(input, "TYPESAFE_API_KEY"),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
return { id, experimental: { evaluation }, configure }
}
export const provider = configure()
export const experimental = provider.experimental
export * as TypeSafeAI from "./typesafe-ai.js"
@@ -1,148 +0,0 @@
import { Effect, Schema } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import {
EvaluationAnswer,
EvaluationInput,
EvaluationModel,
EvaluationQuestion,
EvaluationResponse,
EvaluationRounding,
} from "../experimental/evaluation.js"
import { Auth } from "../route/auth.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import {
AIError,
HttpContext,
HttpOptions,
InvalidProviderOutputError,
InvalidRequestError,
ModelID,
ProviderID,
ProviderMetadata,
Usage,
} from "../schema/index.js"
export const id = ProviderID.make("vercel-ai-gateway")
const baseURL = "https://ai-gateway.vercel.sh/v1"
export interface EvaluationOptions {
readonly [key: string]: unknown
readonly gateway?: Readonly<{
readonly [key: string]: unknown
readonly zeroDataRetention?: boolean
readonly only?: ReadonlyArray<string>
}>
}
export type Options = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
const Request = Schema.StructWithRest(
Schema.Struct({
model: Schema.String,
state: EvaluationInput,
questions: Schema.Record(Schema.String, EvaluationQuestion),
providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}),
[Schema.Record(Schema.String, Schema.Any)],
)
const Response = Schema.Struct({
model: Schema.optional(Schema.String),
answers: Schema.Record(Schema.String, EvaluationAnswer),
usage: Schema.optional(
Schema.Struct({
inputTokens: Schema.optional(Schema.Number),
outputTokens: Schema.optional(Schema.Number),
}),
),
rounding: Schema.optional(EvaluationRounding),
providerMetadata: Schema.optional(ProviderMetadata),
})
export const configure = (input: Options = {}) => {
const evaluation = (modelID: string | ModelID) =>
EvaluationModel.make<EvaluationOptions>({
id: modelID,
provider: id,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
route: {
id: "vercel-evaluation",
evaluate: (req, send) =>
Effect.gen(function* () {
const url = new URL(`${(input.baseURL ?? baseURL).replace(/\/$/, "")}/evaluate`)
Object.entries(req.http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value))
const body = yield* Schema.encodeUnknownEffect(Schema.fromJsonString(Request))({
...req.http?.body,
model: req.model.id,
state: req.state,
questions: req.questions,
providerOptions: req.options,
}).pipe(
Effect.mapError(
(cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) }),
),
)
const headers = yield* Auth.toEffect(
AuthOptions.bearer(input, ["AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN"]),
)({
request: req,
method: "POST",
url: url.toString(),
body,
headers: Headers.fromInput({ ...input.headers, ...req.http?.headers }),
})
const res = yield* send(
HttpClientRequest.post(url).pipe(
HttpClientRequest.setHeaders(headers),
HttpClientRequest.bodyText(body, "application/json"),
),
)
const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers })
const fail = (message: string, cause: unknown, body?: string) =>
new AIError({
reason: new InvalidProviderOutputError({
route: "vercel-evaluation",
message,
body,
http,
cause,
}),
})
const text = yield* res.text.pipe(
Effect.mapError((cause) => fail("Failed to read the Vercel AI Gateway evaluation response", cause)),
)
const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Response))(text).pipe(
Effect.mapError((cause) =>
fail("Vercel AI Gateway returned an invalid evaluation response", cause, text),
),
)
return new EvaluationResponse({
model: ModelID.make(data.model ?? req.model.id),
answers: data.answers,
usage: data.usage
? new Usage({
inputTokens: data.usage.inputTokens,
outputTokens: data.usage.outputTokens,
totalTokens:
data.usage.inputTokens === undefined && data.usage.outputTokens === undefined
? undefined
: (data.usage.inputTokens ?? 0) + (data.usage.outputTokens ?? 0),
providerMetadata: { gateway: data.usage },
})
: undefined,
rounding: data.rounding,
providerMetadata: data.providerMetadata,
})
}),
},
})
return { id, experimental: { evaluation }, configure }
}
export const provider = configure()
export const experimental = provider.experimental
export * as VercelAIGateway from "./vercel-ai-gateway.js"
+14 -20
View File
@@ -1,12 +1,12 @@
import type { LLMRequest } from "../schema/index.js"
import * as ProviderShared from "../protocols/shared.js"
export interface EndpointInput<Body, Request = LLMRequest> {
readonly request: Request
export interface EndpointInput<Body> {
readonly request: LLMRequest
readonly body: Body
}
export type EndpointPart<Body, Request = LLMRequest> = string | ((input: EndpointInput<Body, Request>) => string)
export type EndpointPart<Body> = string | ((input: EndpointInput<Body>) => string)
/**
* Declarative URL construction for one route.
@@ -17,29 +17,26 @@ export type EndpointPart<Body, Request = LLMRequest> = string | ((input: Endpoin
*
* `path` may be a string or a function of `EndpointInput`, for routes whose
* URL embeds the model id, region, or another body field (e.g. Bedrock,
* Gemini). Media routes reuse the same shape with their own request type.
* Gemini).
*/
export interface Definition<Body, Request = LLMRequest> {
export interface Definition<Body> {
readonly baseURL?: string
readonly path: EndpointPart<Body, Request>
readonly path: EndpointPart<Body>
readonly query?: Record<string, string>
}
export type EndpointPatch<Body, Request = LLMRequest> = Partial<Definition<Body, Request>>
export type EndpointPatch<Body> = Partial<Definition<Body>>
/** Construct an `Endpoint` from a path string or path function. */
export const path = <Body, Request = LLMRequest>(
value: EndpointPart<Body, Request>,
options: Omit<Definition<Body, Request>, "path"> = {},
): Definition<Body, Request> => ({
export const path = <Body>(
value: EndpointPart<Body>,
options: Omit<Definition<Body>, "path"> = {},
): Definition<Body> => ({
...options,
path: value,
})
export const merge = <Body, Request = LLMRequest>(
base: Definition<Body, Request>,
patch: EndpointPatch<Body, Request>,
): Definition<Body, Request> => ({
export const merge = <Body>(base: Definition<Body>, patch: EndpointPatch<Body>): Definition<Body> => ({
...base,
...patch,
baseURL: patch.baseURL ?? base.baseURL,
@@ -47,13 +44,10 @@ export const merge = <Body, Request = LLMRequest>(
query: patch.query === undefined ? base.query : { ...base.query, ...patch.query },
})
const renderPart = <Body, Request>(part: EndpointPart<Body, Request>, input: EndpointInput<Body, Request>) =>
const renderPart = <Body>(part: EndpointPart<Body>, input: EndpointInput<Body>) =>
typeof part === "function" ? part(input) : part
export const render = <Body, Request = LLMRequest>(
endpoint: Definition<Body, Request>,
input: EndpointInput<Body, Request>,
) => {
export const render = <Body>(endpoint: Definition<Body>, input: EndpointInput<Body>) => {
const url = new URL(`${ProviderShared.trimBaseUrl(endpoint.baseURL ?? "")}${renderPart(endpoint.path, input)}`)
for (const [key, value] of Object.entries(endpoint.query ?? {})) url.searchParams.set(key, value)
return url
-22
View File
@@ -1,22 +0,0 @@
import { Context, type Effect } from "effect"
import type { HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
import type { AIError } from "../schema/errors.js"
// The service tag lives in its own leaf module so `Media.Asset` (imported by the schema layer) can require the
// executor without pulling the full executor implementation, and therefore the schema barrel, into a cycle.
export interface Interface {
readonly execute: (
request: HttpClientRequest.HttpClientRequest,
middleware?: HttpMiddleware,
) => Effect.Effect<HttpClientResponse.HttpClientResponse, AIError>
}
export type HttpHandler = (
request: HttpClientRequest.HttpClientRequest,
) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error>
export type HttpMiddleware = (
request: HttpClientRequest.HttpClientRequest,
handler: HttpHandler,
) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error>
export class Service extends Context.Service<Service, Interface>()("@opencode/AI/RequestExecutor") {}
+17 -4
View File
@@ -1,4 +1,4 @@
import { Cause, Effect, Layer, Option, Schema, Stream } from "effect"
import { Cause, Context, Effect, Layer, Option, Schema, Stream } from "effect"
import {
FetchHttpClient,
Headers,
@@ -9,10 +9,23 @@ import {
} from "effect/unstable/http"
import { HttpContext, HttpRateLimitDetails, AIError, TransportError } from "../schema/index.js"
import { classifyProviderFailure } from "../provider-error.js"
import { Service, type HttpMiddleware, type Interface } from "./executor-service.js"
export { Service } from "./executor-service.js"
export type { HttpHandler, HttpMiddleware, Interface } from "./executor-service.js"
export interface Interface {
readonly execute: (
request: HttpClientRequest.HttpClientRequest,
middleware?: HttpMiddleware,
) => Effect.Effect<HttpClientResponse.HttpClientResponse, AIError>
}
export type HttpHandler = (
request: HttpClientRequest.HttpClientRequest,
) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error>
export type HttpMiddleware = (
request: HttpClientRequest.HttpClientRequest,
handler: HttpHandler,
) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error>
export class Service extends Context.Service<Service, Interface>()("@opencode/AI/RequestExecutor") {}
const headerDetails = (headers: Headers.Headers) =>
Object.fromEntries(Object.entries(headers).map(([name, value]) => [name, String(value)]))
-2
View File
@@ -20,8 +20,6 @@ export * from "./executor.js"
export { Auth } from "./auth.js"
export { AuthOptions } from "./auth-options.js"
export { Endpoint } from "./endpoint.js"
export { MediaRoute } from "./media.js"
export { MediaProtocol } from "./media-protocol.js"
export { Framing } from "./framing.js"
export { Protocol } from "./protocol.js"
export { HttpTransport, WebSocketTransport } from "./transport/index.js"
-95
View File
@@ -1,95 +0,0 @@
import { Effect, Schema } from "effect"
import { HttpClientResponse } from "effect/unstable/http"
import { AIError, HttpContext, InvalidProviderOutputError } from "../schema/index.js"
// ---------------------------------------------------------------------------
// Bodies
// ---------------------------------------------------------------------------
export type Body =
| { readonly type: "json"; readonly value: Record<string, unknown> }
| { readonly type: "multipart"; readonly value: FormData }
export const json = (value: Record<string, unknown>): Body => ({ type: "json", value })
export const multipart = (value: FormData): Body => ({ type: "multipart", value })
// ---------------------------------------------------------------------------
// Protocol kinds
// ---------------------------------------------------------------------------
export interface DecodeContext<Request> {
readonly request: Request
readonly body: Body
}
/** One request, one response. JSON or multipart in; JSON or raw bytes out. */
export interface Inline<Request, Response> {
readonly kind: "inline"
readonly id: string
readonly name: string
/** Common request fields this protocol cannot lower; the route rejects them before `body.from` runs. */
readonly unsupported?: ReadonlyArray<keyof Request & string>
readonly body: { readonly from: (request: Request) => Effect.Effect<Body, AIError> }
readonly response: {
readonly decode: (
response: HttpClientResponse.HttpClientResponse,
context: DecodeContext<Request>,
) => Effect.Effect<Response, AIError>
}
}
export const inline = <Request, Response>(
input: Omit<Inline<Request, Response>, "kind">,
): Inline<Request, Response> => ({
kind: "inline",
...input,
})
// ---------------------------------------------------------------------------
// Response helpers
// ---------------------------------------------------------------------------
const context = (response: HttpClientResponse.HttpClientResponse) =>
new HttpContext({ url: response.request.url, status: response.status, headers: response.headers })
/** Read a text body while retaining the original payload and HTTP context on every downstream error. */
export const text = Effect.fn("MediaProtocol.text")(function* (
route: string,
name: string,
response: HttpClientResponse.HttpClientResponse,
) {
const http = context(response)
const body = yield* response.text.pipe(
Effect.mapError(
(cause) =>
new AIError({
reason: new InvalidProviderOutputError({
route,
message: `Failed to read the ${name} response`,
http,
cause,
}),
}),
),
)
return {
body,
http,
invalid: (message: string, cause?: unknown) =>
new AIError({ reason: new InvalidProviderOutputError({ route, message, body, http, cause }) }),
}
})
/** Read and Schema-decode a JSON body. Decode failures keep the raw body as `reason.body`. */
export const decodeJson = <A>(route: string, name: string, schema: Schema.Codec<A, unknown>) => {
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
return Effect.fn("MediaProtocol.decodeJson")(function* (response: HttpClientResponse.HttpClientResponse) {
const output = yield* text(route, name, response)
const value = yield* decode(output.body).pipe(
Effect.mapError((cause) => output.invalid(`${name} returned an invalid response`, cause)),
)
return { ...output, value }
})
}
export * as MediaProtocol from "./media-protocol.js"
-116
View File
@@ -1,116 +0,0 @@
import { Effect } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { Auth } from "./auth.js"
import { Endpoint } from "./endpoint.js"
import type { Interface } from "./executor-service.js"
import { MediaProtocol } from "./media-protocol.js"
import { ProviderShared } from "../protocols/shared.js"
import { AIError, HttpOptions, ProviderID, mergeHttpOptions } from "../schema/index.js"
import { sanitizeSurrogates } from "../utils/sanitize.js"
export type Execute = Interface["execute"]
/** The minimum a media request must carry for the route to build a transport request. */
export interface MediaRequest {
readonly model: { readonly id: string; readonly provider: ProviderID; readonly http?: HttpOptions }
readonly http?: HttpOptions
}
/** Deployment inputs every media model factory accepts; provider facades fill these from `configure(...)`. */
export interface ModelInput {
readonly id: string
readonly auth: Auth.Definition
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions
}
// ---------------------------------------------------------------------------
// Route
// ---------------------------------------------------------------------------
export interface Route<Request extends MediaRequest, Response> {
readonly id: string
readonly provider: ProviderID
readonly protocol: string
readonly generate: (request: Request, execute: Execute) => Effect.Effect<Response, AIError>
}
export interface MakeInput<Request extends MediaRequest, Response> {
readonly id: string
readonly provider: string | ProviderID
readonly protocol: MediaProtocol.Inline<Request, Response>
readonly endpoint: Endpoint.Definition<MediaProtocol.Body, Request>
readonly auth: Auth.Definition
/** Deployment headers applied before transport authentication. */
readonly headers?: Record<string, string>
}
/**
* Compose an inline media protocol with an endpoint and auth into a runnable route. The route owns the transport
* plumbing every media protocol would otherwise duplicate: option merging, surrogate sanitizing, unsupported-field rejection, URL and query
* rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol for decoding.
*/
export const make = <Request extends MediaRequest, Response>(
input: MakeInput<Request, Response>,
): Route<Request, Response> => {
const provider = ProviderID.make(input.provider)
const routeHttp = input.headers === undefined ? undefined : new HttpOptions({ headers: input.headers })
const authorize = Auth.toEffect(input.auth)
return {
id: input.id,
provider,
protocol: input.protocol.id,
generate: Effect.fn(`MediaRoute.generate`)(function* (request: Request, execute: Execute) {
yield* rejectUnsupported(input.id, provider, request, input.protocol.unsupported)
const http = mergeHttpOptions(routeHttp, request.model.http, request.http)
// Sanitize after merging so model-level overlays are covered; the model value is restored, not sanitized.
const resolved: Request = { ...sanitizeSurrogates({ ...request, http }), model: request.model }
const body = yield* input.protocol.body.from(resolved)
const url = Endpoint.render(input.endpoint, { request: resolved, body })
for (const [key, value] of Object.entries(http?.query ?? {})) url.searchParams.set(key, value)
const encoded = body.type === "json" ? ProviderShared.encodeJson(body.value) : "[multipart/form-data]"
const baseHeaders = Headers.fromInput(http?.headers)
const headers = yield* authorize({
request: resolved,
method: "POST",
url: url.toString(),
body: encoded,
// The HTTP client sets the multipart boundary; a caller-supplied content-type would corrupt it.
headers: body.type === "multipart" ? Headers.remove(baseHeaders, "content-type") : baseHeaders,
})
const transport = HttpClientRequest.post(url.toString()).pipe(
HttpClientRequest.setHeaders(headers),
body.type === "json"
? HttpClientRequest.bodyText(encoded, "application/json")
: HttpClientRequest.bodyFormData(body.value),
)
const response = yield* execute(transport)
return yield* input.protocol.response.decode(response, { request: resolved, body })
}),
}
}
/** Common fields are never silently dropped: a present field the protocol declared unsupported fails typed. */
const rejectUnsupported = <Request extends object>(
route: string,
provider: ProviderID,
request: Request,
unsupported: ReadonlyArray<keyof Request & string> | undefined,
): Effect.Effect<void, AIError> => {
const present = (unsupported ?? []).filter((field) => {
const value = request[field]
return Array.isArray(value) ? value.length > 0 : value !== undefined
})
if (present.length === 0) return Effect.void
return Effect.fail(
ProviderShared.unsupportedOperation({
operation: `media.${present[0]}`,
provider,
route,
message: `${provider}/${route} does not support ${present.join(", ")}`,
}),
)
}
export * as MediaRoute from "./media.js"
-7
View File
@@ -133,12 +133,6 @@ export class UnknownProviderError extends Schema.TaggedError<UnknownProviderErro
ReasonFields,
) {}
/** A caller-supplied deadline elapsed, such as `Generation.await` polling past its `Poll.timeout`. */
export class TimeoutError extends Schema.TaggedError<TimeoutError>("AI.Error.Timeout")("Timeout", {
...ReasonFields,
timeoutMs: Schema.optional(Schema.Number),
}) {}
export const AIErrorReason = Schema.Union([
InvalidRequestError,
UnsupportedOperationError,
@@ -151,7 +145,6 @@ export const AIErrorReason = Schema.Union([
TransportError,
InvalidProviderOutputError,
UnknownProviderError,
TimeoutError,
]).pipe(Schema.toTaggedUnion("_tag"))
export type AIErrorReason = Schema.Schema.Type<typeof AIErrorReason>
+2 -41
View File
@@ -4,18 +4,18 @@ import { ContentBlockID, ToolCallID } from "./ids.js"
import {
Message,
CompactionPart,
ProviderMetadata,
ToolCallPart,
ToolOutput,
ToolResultPart,
ToolResultValue,
type ContentPart,
} from "./messages.js"
import { ProviderMetadata } from "./options.js"
import { ProviderFailureClassification } from "./errors.js"
import { Media } from "../media.js"
export const FinishReason = LLM.FinishReason
export type FinishReason = Schema.Schema.Type<typeof FinishReason>
export { ProviderMetadata } from "./messages.js"
/**
* Token usage reported by an LLM provider.
@@ -91,27 +91,6 @@ export class Usage extends Schema.Class<Usage>("AI.Usage")({
export type UsageInput = Usage | ConstructorParameters<typeof Usage>[0]
/**
* Usage reported by media routes. Providers bill images, video, speech, and transcription in different units, so
* each response carries the unit it was actually metered in instead of forcing everything into tokens.
*/
export const MediaUsage = Schema.Union([
Schema.Struct({
type: Schema.Literal("tokens"),
input: Schema.optional(Schema.Number),
output: Schema.optional(Schema.Number),
total: Schema.optional(Schema.Number),
details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}),
Schema.Struct({ type: Schema.Literal("seconds"), seconds: Schema.Number }),
Schema.Struct({ type: Schema.Literal("characters"), characters: Schema.Number }),
Schema.Struct({ type: Schema.Literal("credits"), credits: Schema.Number }),
Schema.Struct({ type: Schema.Literal("compute"), seconds: Schema.Number }),
])
.pipe(Schema.toTaggedUnion("type"))
.annotate({ identifier: "AI.MediaUsage" })
export type MediaUsage = Schema.Schema.Type<typeof MediaUsage>
/** A replacement context window, not an assistant message to append to prior history. */
export class CompactionResponse extends Schema.Class<CompactionResponse>("LLM.CompactionResponse")({
replacement: Schema.Array(Message),
@@ -284,14 +263,6 @@ export const Finish = Schema.Struct({
}).annotate({ identifier: "LLM.Event.Finish" })
export type Finish = Schema.Schema.Type<typeof Finish>
/** A generated media asset (image, audio, …) emitted by the model as first-class output rather than a tool result. */
export const MediaEvent = Schema.Struct({
type: Schema.tag("media"),
media: Media.AssetSchema,
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Event.Media" })
export type MediaEvent = Schema.Schema.Type<typeof MediaEvent>
export const ProviderErrorEvent = Schema.Struct({
type: Schema.tag("provider-error"),
message: Schema.String,
@@ -316,7 +287,6 @@ const llmEventTagged = Schema.Union([
ToolCall,
ToolResult,
ToolError,
MediaEvent,
StepFinish,
Finish,
ProviderErrorEvent,
@@ -362,7 +332,6 @@ export const LLMEvent = Object.assign(llmEventTagged, {
output: input.output === undefined ? undefined : ToolOutput.make(input.output.structured, input.output.content),
}),
toolError: (input: WithID<ToolError, ToolCallID>) => ToolError.make({ ...input, id: toolCallID(input.id) }),
media: MediaEvent.make,
stepFinish: (input: WithUsage<StepFinish>) =>
StepFinish.make({
...input,
@@ -390,7 +359,6 @@ export const LLMEvent = Object.assign(llmEventTagged, {
toolCall: llmEventTagged.guards["tool-call"],
toolResult: llmEventTagged.guards["tool-result"],
toolError: llmEventTagged.guards["tool-error"],
media: llmEventTagged.guards.media,
stepFinish: llmEventTagged.guards["step-finish"],
finish: llmEventTagged.guards.finish,
providerError: llmEventTagged.guards["provider-error"],
@@ -666,13 +634,6 @@ const reduceResponseState = (state: ResponseState, event: LLMEvent): ResponseSta
return reduceToolCall(next, event)
case "tool-result":
return appendContent(next, toolResultContent(event))
case "media":
return appendContent(
next,
event.providerMetadata === undefined
? { type: "media", media: event.media }
: { type: "media", media: event.media, providerMetadata: event.providerMetadata },
)
default:
return next
}
+7 -9
View File
@@ -8,16 +8,19 @@ import {
JsonSchema,
LanguageModelSchema,
type LanguageModel,
ProviderMetadata,
ProviderOptions,
ReasoningEffort,
} from "./options.js"
import { ProviderID } from "./ids.js"
import { Media } from "../media.js"
export const MessageRole = Schema.Literals(["system", "user", "assistant", "tool"])
export type MessageRole = Schema.Schema.Type<typeof MessageRole>
export const ProviderMetadata = Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown)).annotate({
identifier: "LLM.ProviderMetadata",
})
export type ProviderMetadata = Schema.Schema.Type<typeof ProviderMetadata>
const systemPartSchema = Schema.Struct({
type: Schema.Literal("text"),
text: Schema.String,
@@ -47,7 +50,8 @@ export type TextPart = Schema.Schema.Type<typeof TextPart>
export const MediaPart = Schema.Struct({
type: Schema.Literal("media"),
media: Media.AssetSchema,
mediaType: Schema.String,
data: Schema.Union([Schema.String, Schema.Uint8Array]),
filename: Schema.optional(Schema.String),
cache: Schema.optional(CacheHint),
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
@@ -251,12 +255,6 @@ export namespace Message {
export const text = (value: string): ContentPart => ({ type: "text", text: value })
export const media = (asset: Media.Asset, options?: Omit<MediaPart, "type" | "media">): MediaPart => ({
type: "media",
media: asset,
...options,
})
export const content = (input: ContentInput) =>
typeof input === "string" ? [text(input)] : Array.isArray(input) ? [...input] : [input]
-5
View File
@@ -39,11 +39,6 @@ const mergeStringRecords = (
export const ProviderOptions = Schema.Record(Schema.String, Schema.Unknown)
export type ProviderOptions = Schema.Schema.Type<typeof ProviderOptions>
export const ProviderMetadata = Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown)).annotate({
identifier: "LLM.ProviderMetadata",
})
export type ProviderMetadata = Schema.Schema.Type<typeof ProviderMetadata>
export const mergeProviderOptions = (
...items: ReadonlyArray<ProviderOptions | undefined>
): ProviderOptions | undefined => mergeJsonRecords(...items)
-46
View File
@@ -1,46 +0,0 @@
const ascii = (bytes: Uint8Array, start: number, end: number) => String.fromCharCode(...bytes.subarray(start, end))
const startsWith = (bytes: Uint8Array, prefix: ReadonlyArray<number>) =>
bytes.length >= prefix.length && prefix.every((value, index) => bytes[index] === value)
/**
* Sniff a media type from leading magic bytes. Covers the containers media routes commonly return; anything else is
* `undefined` so callers can fall back to a provider-declared type or `application/octet-stream`.
*/
export const detectMediaType = (bytes: Uint8Array): string | undefined => {
if (startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) return "image/png"
if (startsWith(bytes, [0xff, 0xd8, 0xff])) return "image/jpeg"
if (startsWith(bytes, [0x47, 0x49, 0x46, 0x38])) return "image/gif"
if (bytes.length >= 12 && ascii(bytes, 0, 4) === "RIFF") {
const riffType = ascii(bytes, 8, 12)
if (riffType === "WEBP") return "image/webp"
if (riffType === "WAVE") return "audio/wav"
}
if (startsWith(bytes, [0x25, 0x50, 0x44, 0x46])) return "application/pdf"
if (bytes.length >= 12 && ascii(bytes, 4, 8) === "ftyp") return "video/mp4"
if (startsWith(bytes, [0x1a, 0x45, 0xdf, 0xa3])) return "video/webm"
if (startsWith(bytes, [0x49, 0x44, 0x33]) || startsWith(bytes, [0xff, 0xfb]) || startsWith(bytes, [0xff, 0xf3]))
return "audio/mpeg"
if (startsWith(bytes, [0x4f, 0x67, 0x67, 0x53])) return "audio/ogg"
return undefined
}
const EXTENSIONS: Readonly<Record<string, string>> = {
png: "image/png",
jpg: "image/jpeg",
jpeg: "image/jpeg",
gif: "image/gif",
webp: "image/webp",
pdf: "application/pdf",
mp4: "video/mp4",
webm: "video/webm",
mp3: "audio/mpeg",
wav: "audio/wav",
ogg: "audio/ogg",
txt: "text/plain",
md: "text/markdown",
csv: "text/csv",
}
export const extensionMediaType = (path: string): string | undefined =>
EXTENSIONS[path.slice(path.lastIndexOf(".") + 1).toLowerCase()]
+1 -3
View File
@@ -1,11 +1,9 @@
import { Media } from "../media.js"
import { isRecord } from "./record.js"
export const sanitizeSurrogates = <T>(value: T): T => {
if (typeof value === "string") return value.toWellFormed() as T
if (Array.isArray(value)) return value.map(sanitizeSurrogates) as T
// Media assets carry binary or base64 payloads and a lazy byte cache; flattening them into a record would drop both.
if (value instanceof Uint8Array || value instanceof Error || value instanceof Media.Asset) return value
if (value instanceof Uint8Array || value instanceof Error) return value
if (isRecord(value))
return Object.fromEntries(
Object.entries(value).map(([key, entry]) => [key.toWellFormed(), sanitizeSurrogates(entry)]),
+3 -3
View File
@@ -87,9 +87,9 @@ testEffect(fixedResponse("")).effect(
prompt: "hello",
})
expect(LLMClient.canCompact(request)).toBe(false)
const error = yield* LLMClient.compact(request as unknown as Parameters<typeof LLMClient.compact>[0]).pipe(
Effect.flip,
)
const error = yield* LLMClient.compact(
request as unknown as Parameters<typeof LLMClient.compact>[0],
).pipe(Effect.flip)
expect(error.reason._tag).toBe("UnsupportedOperation")
expect(error.message).toContain("does not support explicit compaction")
if (error.reason._tag === "UnsupportedOperation") {
+1 -2
View File
@@ -1,6 +1,5 @@
import {
LLM,
Media,
Message,
ToolCallPart,
ToolDefinition,
@@ -60,7 +59,7 @@ export function continuationRequest(input: {
if (features.has("user-text")) firstUser.push({ type: "text", text: "What is shown here?" })
if (features.has("user-image"))
firstUser.push({ type: "media", media: Media.base64(input.image ?? "AAECAw==", "image/png") })
firstUser.push({ type: "media", mediaType: "image/png", data: input.image ?? "AAECAw==" })
if (firstUser.length > 0) messages.push(Message.user(firstUser))
if (features.has("assistant-reasoning"))
-302
View File
@@ -1,302 +0,0 @@
import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { Evaluation, EvaluationClient } from "../src/experimental.js"
import { OpenCodeZen, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js"
import { it } from "./lib/effect.js"
import { dynamicResponse } from "./lib/http.js"
describe("experimental Evaluation", () => {
it.effect("evaluates typed questions through System One", () =>
Effect.gen(function* () {
const response = yield* Evaluation.run({
model: TypeSafeAI.configure({
apiKey: "test",
baseURL: "https://typesafe.test/v1/",
headers: { "x-default": "yes" },
http: { body: { deployment: "test" }, query: { api: "v1" } },
}).experimental.evaluation("jev-latest"),
state: { ticket: "Please refund the duplicate charge." },
questions: {
department: {
type: "choice",
instructions: "Which team should handle this?",
criteria: { billing: "Payments and refunds", technical: "Bugs and outages" },
},
urgency: {
type: "score",
instructions: "How urgent is this?",
criteria: ["Can wait", "Needs attention", "Blocking"],
},
refund: { type: "boolean", instructions: "Is the customer asking for a refund?" },
},
options: { trace: { enabled: true } },
http: { body: { request_metadata: "value" }, headers: { "x-request": "yes" }, query: { trace: "1" } },
})
expect(response.model).toBe("jev-1.13.0")
expect(response.answers.department).toEqual({
type: "choice",
choice: "billing",
probabilities: { billing: 0.9, technical: 0.1 },
})
expect(response.answers.urgency).toEqual({
type: "score",
score: 1.2,
probabilities: { "0": 0, "1": 0.8, "2": 0.2 },
})
expect(response.answers.refund).toEqual({ type: "boolean", probability: 0.97 })
expect(response.usage?.totalTokens).toBe(36)
expect(response.providerMetadata).toEqual({
typesafe: {
confidence: { department: 0.8, urgency: 0.6 },
legend: { urgency: { "0": "Can wait", "1": "Needs attention", "2": "Blocking" } },
},
})
}).pipe(
Effect.provide(
EvaluationClient.layer.pipe(
Layer.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
expect(request.url).toBe("https://typesafe.test/v1/systemone?api=v1&trace=1")
expect(request.headers.get("authorization")).toBe("Bearer test")
expect(request.headers.get("x-default")).toBe("yes")
expect(request.headers.get("x-request")).toBe("yes")
expect(JSON.parse(input.text)).toEqual({
deployment: "test",
request_metadata: "value",
trace: { enabled: true },
model: "jev-latest",
state: { ticket: "Please refund the duplicate charge." },
questions: {
department: {
type: "choice",
instructions: "Which team should handle this?",
criteria: { billing: "Payments and refunds", technical: "Bugs and outages" },
},
urgency: {
type: "score",
instructions: "How urgent is this?",
criteria: ["Can wait", "Needs attention", "Blocking"],
},
refund: { type: "noul", instructions: "Is the customer asking for a refund?" },
},
})
return input.respond(
JSON.stringify({
model: "jev-1.13.0",
answers: {
department: {
type: "choice",
choice: "billing",
probabilities: { billing: 0.9, technical: 0.1 },
confidence: 0.8,
},
urgency: {
type: "score",
score: 1.2,
probabilities: { "0": 0, "1": 0.8, "2": 0.2 },
legend: { "0": "Can wait", "1": "Needs attention", "2": "Blocking" },
confidence: 0.6,
},
refund: { type: "noul", noul: 0.97 },
},
usage: { input_tokens: 30, output_tokens: 6 },
}),
{ headers: { "content-type": "application/json" } },
)
}),
),
),
),
),
),
)
it.effect("configures the OpenCode Zen System One endpoint", () =>
Evaluation.run({
model: OpenCodeZen.configure({ apiKey: "zen-key", baseURL: "https://zen.test/v1" }).experimental.evaluation(
"jev-1.13",
),
state: "hello",
questions: { greeting: { type: "boolean", instructions: "Is this a greeting?" } },
}).pipe(
Effect.tap((response) =>
Effect.sync(() => {
expect(response.answers.greeting.probability).toBe(0.99)
expect(response.usage?.providerMetadata).toEqual({
opencode: { input_tokens: 10, output_tokens: 2 },
})
}),
),
Effect.provide(
EvaluationClient.layer.pipe(
Layer.provide(
dynamicResponse((input) => {
expect(input.request.url).toBe("https://zen.test/v1/systemone")
expect(input.request.headers.authorization).toBe("Bearer zen-key")
return Effect.succeed(
input.respond(
JSON.stringify({
model: "jev-1.13.0",
answers: { greeting: { type: "noul", noul: 0.99 } },
usage: { input_tokens: 10, output_tokens: 2 },
}),
{ headers: { "content-type": "application/json" } },
),
)
}),
),
),
),
),
)
it.effect("evaluates through OpenRouter System One", () =>
Evaluation.run({
model: OpenRouter.configure({
apiKey: "openrouter-key",
baseURL: "https://openrouter.test/api/v1",
}).experimental.evaluation("typesafe/jev-1.13"),
state: "refund",
questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } },
options: { user: "user-1", session_id: "session-1" },
}).pipe(
Effect.tap((response) =>
Effect.sync(() => {
expect(response.answers.refund.probability).toBe(0.98)
expect(response.providerMetadata?.openrouter).toMatchObject({ responseId: "gen-1", provider: "TypeSafe" })
expect(response.usage?.providerMetadata?.openrouter).toEqual({
input_tokens: 10,
output_tokens: 2,
cost: 0.0001,
})
}),
),
Effect.provide(
EvaluationClient.layer.pipe(
Layer.provide(
dynamicResponse((input) => {
expect(input.request.url).toBe("https://openrouter.test/api/v1/systemone")
expect(input.request.headers.authorization).toBe("Bearer openrouter-key")
expect(JSON.parse(input.text)).toMatchObject({
model: "typesafe/jev-1.13",
user: "user-1",
session_id: "session-1",
questions: { refund: { type: "noul" } },
})
return Effect.succeed(
input.respond(
JSON.stringify({
id: "gen-1",
model: "typesafe/jev-1.13-20260917",
provider: "TypeSafe",
answers: { refund: { type: "noul", noul: 0.98 } },
usage: { input_tokens: 10, output_tokens: 2, cost: 0.0001 },
}),
{ headers: { "content-type": "application/json" } },
),
)
}),
),
),
),
),
)
it.effect("evaluates through Vercel AI Gateway", () =>
Evaluation.run({
model: VercelAIGateway.configure({
apiKey: "gateway-key",
baseURL: "https://gateway.test/v1/",
}).experimental.evaluation("typesafe-ai/jev"),
state: "refund",
questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } },
options: { gateway: { zeroDataRetention: true, only: ["typesafe-ai"] } },
}).pipe(
Effect.tap((response) =>
Effect.sync(() => {
expect(response.answers.refund.probability).toBe(0.98)
expect(response.usage?.totalTokens).toBe(12)
expect(response.providerMetadata?.gateway).toMatchObject({ generationId: "gen-1", cost: "0.0001" })
}),
),
Effect.provide(
EvaluationClient.layer.pipe(
Layer.provide(
dynamicResponse((input) => {
expect(input.request.url).toBe("https://gateway.test/v1/evaluate")
expect(input.request.headers.authorization).toBe("Bearer gateway-key")
expect(JSON.parse(input.text)).toEqual({
model: "typesafe-ai/jev",
state: "refund",
questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } },
providerOptions: { gateway: { zeroDataRetention: true, only: ["typesafe-ai"] } },
})
return Effect.succeed(
input.respond(
JSON.stringify({
model: "typesafe-ai/jev",
answers: { refund: { type: "boolean", probability: 0.98 } },
usage: { inputTokens: 10, outputTokens: 2 },
providerMetadata: { gateway: { generationId: "gen-1", cost: "0.0001" } },
}),
{ headers: { "content-type": "application/json" } },
),
)
}),
),
),
),
),
)
it.effect("rejects answers that do not match their questions", () =>
Evaluation.run({
model: VercelAIGateway.configure({
apiKey: "gateway-key",
baseURL: "https://gateway.test/v1",
}).experimental.evaluation("typesafe-ai/jev"),
state: "refund",
questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } },
}).pipe(
Effect.flip,
Effect.tap((error) => Effect.sync(() => expect(error.reason._tag).toBe("InvalidProviderOutput"))),
Effect.provide(
EvaluationClient.layer.pipe(
Layer.provide(
dynamicResponse((input) =>
Effect.succeed(
input.respond(
JSON.stringify({
answers: { refund: { type: "choice", choice: "yes", probabilities: { yes: 1 } } },
}),
{ headers: { "content-type": "application/json" } },
),
),
),
),
),
),
),
)
it.effect("rejects malformed questions before network I/O", () =>
Effect.gen(function* () {
const error = yield* Evaluation.run({
model: TypeSafeAI.experimental.evaluation("jev-latest"),
state: "hello",
questions: { score: { type: "score", instructions: "How much?", criteria: ["only"] } },
}).pipe(Effect.flip)
expect(error.reason._tag).toBe("InvalidRequest")
}).pipe(
Effect.provide(
EvaluationClient.layer.pipe(
Layer.provide(dynamicResponse(() => Effect.die("invalid evaluation reached the network"))),
),
),
),
)
})
-88
View File
@@ -1,88 +0,0 @@
import { Effect } from "effect"
import { Evaluation, EvaluationClient, EvaluationModel, type EvaluationRoute } from "../src/experimental.js"
import type { Service } from "../src/experimental/evaluation-client.js"
import { OpenCodeZen, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js"
type Requirements<T> = T extends Effect.Effect<infer _A, infer _E, infer R> ? R : never
type Success<T> = T extends Effect.Effect<infer A, infer _E, infer _R> ? A : never
type Equal<A, B> = [A, B] extends [B, A] ? true : false
type Assert<T extends true> = T
const model = TypeSafeAI.configure({ apiKey: "test" }).experimental.evaluation("jev-latest")
const request = Evaluation.request({
model,
state: { ticket: "refund" },
questions: {
topic: {
type: "choice",
instructions: "Which team?",
criteria: { billing: null, support: { includes: ["help"] } },
},
severity: { type: "score", instructions: "How severe?", criteria: ["Low", "High"] },
refund: { type: "boolean", instructions: "Refund?" },
},
})
const result = EvaluationClient.evaluate(request)
type Result = Success<typeof result>
type Choice = Assert<Equal<Result["answers"]["topic"]["choice"], "billing" | "support">>
type ClientRequirements = Assert<Equal<Requirements<typeof result>, Service>>
void (true satisfies Choice)
void (true satisfies ClientRequirements)
Effect.gen(function* () {
const response = yield* Evaluation.run({
model: OpenCodeZen.experimental.evaluation("jev-1.13"),
state: ["hello"],
questions: { greeting: { type: "boolean", instructions: "Greeting?" } },
})
response.answers.greeting.probability satisfies number
// @ts-expect-error Boolean answers do not contain a selected choice.
response.answers.greeting.choice
// @ts-expect-error Unknown question IDs are not exposed.
response.answers.missing
})
declare const route: EvaluationRoute<{ readonly temperature?: number }>
const custom = EvaluationModel.make({ id: "custom", provider: "custom", route })
Evaluation.run({
model: custom,
state: "hello",
questions: { ok: { type: "boolean", instructions: "OK?" } },
options: { temperature: 0.5 },
})
// @ts-expect-error Selected evaluation models retain their request option types.
Evaluation.run({
model: custom,
state: "hello",
questions: { ok: { type: "boolean", instructions: "OK?" } },
options: { temperature: "high" },
})
Evaluation.run({
model: OpenRouter.experimental.evaluation("typesafe/jev-1.13"),
state: "hello",
questions: { ok: { type: "boolean", instructions: "OK?" } },
options: { provider: { zdr: true }, session_id: "session-1", user: "user-1" },
})
// @ts-expect-error OpenRouter session IDs are strings.
Evaluation.run({
model: OpenRouter.experimental.evaluation("typesafe/jev-1.13"),
state: "hello",
questions: { ok: { type: "boolean", instructions: "OK?" } },
options: { session_id: 1 },
})
Evaluation.run({
model: VercelAIGateway.experimental.evaluation("typesafe-ai/jev"),
state: "hello",
questions: { ok: { type: "boolean", instructions: "OK?" } },
options: { gateway: { zeroDataRetention: true, only: ["typesafe-ai"] } },
})
// @ts-expect-error Vercel zero-data-retention controls are boolean.
Evaluation.run({
model: VercelAIGateway.experimental.evaluation("typesafe-ai/jev"),
state: "hello",
questions: { ok: { type: "boolean", instructions: "OK?" } },
options: { gateway: { zeroDataRetention: "yes" } },
})
+2 -13
View File
@@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test"
import { AIError, LanguageModel, LLM, LLMClient, Media, Provider } from "@opencode/ai"
import { AIError, ImageInput, LanguageModel, LLM, LLMClient, Provider } from "@opencode/ai"
import { Route, Protocol, WebSocketTransport } from "@opencode/ai/route"
import { Provider as ProviderSubpath } from "@opencode/ai/provider"
import {
@@ -8,12 +8,9 @@ import {
CloudflareWorkersAI,
DeepSeek,
Fireworks,
OpenCodeZen,
OpenAI,
OpenAICompatible,
OpenRouter,
TypeSafeAI,
VercelAIGateway,
XAI,
} from "@opencode/ai/providers"
import {
@@ -26,7 +23,6 @@ import {
} from "@opencode/ai/protocols"
import * as AnthropicMessages from "@opencode/ai/protocols/anthropic-messages"
import { TestLLM } from "@opencode/ai/testing"
import { Evaluation, EvaluationClient } from "@opencode/ai/experimental"
describe("public exports", () => {
test("root exposes app-facing runtime APIs", () => {
@@ -35,15 +31,12 @@ describe("public exports", () => {
expect(LLMClient.layer).toBeDefined()
expect(AIError).toBeFunction()
expect(LanguageModel.make).toBeFunction()
expect(Media.bytes).toBeFunction()
expect(ImageInput.bytes).toBeFunction()
expect(Provider.make).toBeFunction()
expect(ProviderSubpath.make).toBe(Provider.make)
expect(TestLLM.layer).toBeFunction()
expect(TestLLM.testLayer).toBeFunction()
expect(TestLLM.Test.of).toBeFunction()
expect(Evaluation.run).toBeFunction()
expect(EvaluationClient.layer).toBeDefined()
expect(EvaluationClient.fetchLayer).toBeDefined()
})
test("route barrel exposes route-authoring APIs", () => {
@@ -73,10 +66,6 @@ describe("public exports", () => {
expect(CloudflareWorkersAI.configure).toBeFunction()
expect(CloudflareWorkersAI.configure({ accountId: "fixture", apiKey: "fixture" }).model).toBeFunction()
expect(OpenRouter.model).toBeFunction()
expect(OpenRouter.experimental.evaluation).toBeFunction()
expect(TypeSafeAI.experimental.evaluation).toBeFunction()
expect(OpenCodeZen.experimental.evaluation).toBeFunction()
expect(VercelAIGateway.experimental.evaluation).toBeFunction()
expect(XAI.model).toBeFunction()
expect(XAI.provider.responses).toBe(XAI.responses)
expect(XAI.provider.chat).toBe(XAI.chat)
@@ -1,13 +1,9 @@
{
"version": 1,
"metadata": {
"tags": [
"prefix:google-vertex",
"provider:google-vertex",
"protocol:gemini"
],
"tags": ["prefix:google-vertex", "provider:google-vertex", "protocol:gemini"],
"name": "google-vertex/calls-a-tool",
"recordedAt": "2026-09-22T03:46:29.725Z"
"recordedAt": "2026-08-23T17:21:51.036Z"
},
"interactions": [
{
@@ -25,7 +21,7 @@
"headers": {
"content-type": "text/event-stream"
},
"body": "data: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"functionCall\": {\"name\": \"lookup_weather\",\"args\": {\"city\": \"Paris\"},\"id\": \"call_28936\"},\"thoughtSignature\": \"AY89a1/MVeBxtwu9l/96clFobOrrd7Q5MTf5o8/A22fc++1EsFYVeCFx9WCGmJ+D3yW3FWlRXKtJECYfprSoxixCSOYBNiV9g8/IkejbAxF26k1vxTsHkoqts0O2s8GqqBSRAavJfKQ/taTRbPuIq+b+RKZ8SDJFoWDnFjXpdS5S164uJTjmgs4ALd8oB1oQRycMsTbAY51TmkcNUnhARpjKQfqiizz4KfSRFjx7xDnza0v4jj73m2GV86eLDhVT7trIT+Dl5DsJt6RBLOE2Amse54mcpIbsYaANlY1YDU6FKuUBqnuwpl0OfhMXwWuYMogv1tH/0rTXsqNwmHyt/b9ZZXqYCdFXpMGt8udmoZ0d\"}]}}],\"usageMetadata\": {\"trafficType\": \"ON_DEMAND\"},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-09-22T03:46:28.953809Z\",\"responseId\": \"FPqxatGbOsSorb8PtY2z8Q4\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"text\": \"\"}]},\"finishReason\": \"STOP\"}],\"usageMetadata\": {\"promptTokenCount\": 39,\"candidatesTokenCount\": 16,\"totalTokenCount\": 93,\"trafficType\": \"ON_DEMAND\",\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 39}],\"candidatesTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 16}],\"thoughtsTokenCount\": 38},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-09-22T03:46:28.953809Z\",\"responseId\": \"FPqxatGbOsSorb8PtY2z8Q4\"}\r\n\r\n"
"body": "data: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"functionCall\": {\"name\": \"lookup_weather\",\"args\": {\"city\": \"Paris\"},\"id\": \"call_425130\"},\"thoughtSignature\": \"AY89a1+1fXnLgYhHMuN3Ak6LBhT6PcrYOW7iPav4LfsacvG/Z6l1yJ+AsU7vWhFj/JyPIbsJJQ+GjohM9sCIZ6nqUOIg3reo/7osmrCvFrVHedTHQcwiPzoz2Kp3gb+uWjFAXxk1EX4IRAKcu0ox1W/Z9PpuZvHkTerGO2a82e02N6MAF1YhhtbXFvSdqLRih2Os68rdOk5/Bcld7ol8qUgeyIZ3CtI3OJ5jwRcD8LjvK33A7ZFzH5Bxp/peUmXvqnu5iNhnGBxZaJy/vupCtxRZxjaS+ojG0/UhyrnRiKIpbzQ0FBkxePPn8GCX/LOe2y3GUc98co8lN8OOuCd9ZmEdx5AjHmQkPO9fAV9SxG6Bda6SDWVL8o/Uz3WSQYoUEfAdoajEWIBvcisoeCJjb7zgmRRZ9VQSPl3RXj5LFRvX8jn0YKV1CahYbc24jA==\"}]}}],\"usageMetadata\": {\"trafficType\": \"ON_DEMAND\"},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-08-23T17:21:50.308576Z\",\"responseId\": \"LiyLauDqErCErb8Pj8aWkAs\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"text\": \"\"}]},\"finishReason\": \"STOP\"}],\"usageMetadata\": {\"promptTokenCount\": 39,\"candidatesTokenCount\": 16,\"totalTokenCount\": 102,\"trafficType\": \"ON_DEMAND\",\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 39}],\"candidatesTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 16}],\"thoughtsTokenCount\": 47},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-08-23T17:21:50.308576Z\",\"responseId\": \"LiyLauDqErCErb8Pj8aWkAs\"}\r\n\r\n"
}
}
]
@@ -1,13 +1,9 @@
{
"version": 1,
"metadata": {
"tags": [
"prefix:google-vertex",
"provider:google-vertex",
"protocol:gemini"
],
"tags": ["prefix:google-vertex", "provider:google-vertex", "protocol:gemini"],
"name": "google-vertex/continues-after-a-tool-result",
"recordedAt": "2026-09-22T03:46:30.691Z"
"recordedAt": "2026-08-23T17:21:51.853Z"
},
"interactions": [
{
@@ -18,14 +14,14 @@
"headers": {
"content-type": "application/json"
},
"body": "{\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"What is the weather in Paris?\"}]},{\"role\":\"model\",\"parts\":[{\"functionCall\":{\"id\":\"call_paris_1\",\"name\":\"lookup_weather\",\"args\":{\"city\":\"Paris\"}},\"thoughtSignature\":\"skip_thought_signature_validator\"}]},{\"role\":\"user\",\"parts\":[{\"functionResponse\":{\"id\":\"call_paris_1\",\"name\":\"lookup_weather\",\"response\":{\"name\":\"lookup_weather\",\"content\":\"18C, light rain\"}}}]}],\"tools\":[{\"functionDeclarations\":[{\"name\":\"lookup_weather\",\"description\":\"Look up the current weather for a city\",\"parameters\":{\"required\":[\"city\"],\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}}}}]}]}"
"body": "{\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"What is the weather in Paris?\"}]},{\"role\":\"model\",\"parts\":[{\"functionCall\":{\"name\":\"lookup_weather\",\"args\":{\"city\":\"Paris\"}},\"thoughtSignature\":\"skip_thought_signature_validator\"}]},{\"role\":\"user\",\"parts\":[{\"functionResponse\":{\"name\":\"lookup_weather\",\"response\":{\"name\":\"lookup_weather\",\"content\":\"18C, light rain\"}}}]}],\"tools\":[{\"functionDeclarations\":[{\"name\":\"lookup_weather\",\"description\":\"Look up the current weather for a city\",\"parameters\":{\"required\":[\"city\"],\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}}}}]}]}"
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream"
},
"body": "data: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"text\": \"The weather\"}]}}],\"usageMetadata\": {\"trafficType\": \"ON_DEMAND\"},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-09-22T03:46:29.830958Z\",\"responseId\": \"Ffqxau7bMoCOrb8P_Iu5sA0\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"text\": \" in Paris is currently 18°C with light rain.\"}]}}],\"usageMetadata\": {\"trafficType\": \"ON_DEMAND\"},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-09-22T03:46:29.830958Z\",\"responseId\": \"Ffqxau7bMoCOrb8P_Iu5sA0\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"text\": \"\",\"thoughtSignature\": \"AY89a18aVI8Uxr2VaOI822u0DyuQdy4B00uhLnjqYb5Qb6Mkscccm018knLtYThB5UX8dRv1VFsORSQ0Qo6Gx9RCng2AK9EPce7p\"}]},\"finishReason\": \"STOP\"}],\"usageMetadata\": {\"promptTokenCount\": 59,\"candidatesTokenCount\": 15,\"totalTokenCount\": 74,\"trafficType\": \"ON_DEMAND\",\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 59}],\"candidatesTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 15}]},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-09-22T03:46:29.830958Z\",\"responseId\": \"Ffqxau7bMoCOrb8P_Iu5sA0\"}\r\n\r\n"
"body": "data: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"text\": \"The weather in Paris is currently 18°C with light rain.\"}]}}],\"usageMetadata\": {\"trafficType\": \"ON_DEMAND\"},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-08-23T17:21:51.220919Z\",\"responseId\": \"LyyLave9DbWnrb8P1IjLmQQ\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"text\": \"\",\"thoughtSignature\": \"AY89a197c+fpHJftPtcufnqMAyoRQKVEQK+KeG+RVHVx2wKil3L4jP4YWvfVbcuOFr2jio4Kre/hCrDANAoMFSvaZrdaPeo1b5bXQSmJKMH03yM5M6q6ME6JiBvXym143U4exIde4UbOh2tMeyXMvB3aWxcavIHd78g5G5QPLreo6A3LO5871cYYVeRwteY+/zbEdqfaAq1hlk6WYpWkNljYpjMyKwr15YC8rFLh3HYayS9tTN++GGrk/reZn6C3OEPlzPou/pXRATzcEAGVl/TW\"}]},\"finishReason\": \"STOP\"}],\"usageMetadata\": {\"promptTokenCount\": 59,\"candidatesTokenCount\": 15,\"totalTokenCount\": 98,\"trafficType\": \"ON_DEMAND\",\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 59}],\"candidatesTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 15}],\"thoughtsTokenCount\": 24},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-08-23T17:21:51.220919Z\",\"responseId\": \"LyyLave9DbWnrb8P1IjLmQQ\"}\r\n\r\n"
}
}
]
@@ -1,13 +1,9 @@
{
"version": 1,
"metadata": {
"tags": [
"prefix:google-vertex",
"provider:google-vertex",
"protocol:gemini"
],
"tags": ["prefix:google-vertex", "provider:google-vertex", "protocol:gemini"],
"name": "google-vertex/streams-text",
"recordedAt": "2026-09-22T03:46:28.840Z"
"recordedAt": "2026-08-23T17:21:50.112Z"
},
"interactions": [
{
@@ -25,7 +21,7 @@
"headers": {
"content-type": "text/event-stream"
},
"body": "data: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"text\": \"Hello\"}]}}],\"usageMetadata\": {\"trafficType\": \"ON_DEMAND\"},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-09-22T03:46:26.748531Z\",\"responseId\": \"EvqxavPXLbqerb8P1eSViQs\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"text\": \"\",\"thoughtSignature\": \"AY89a18msz0aRAZc4U/Zl7rEXjdTJsE0EWee1+hD2dT7h711q6uIRem7KWCLlji7JVSu2hDV1L8uFh8SGAO5jU4MBSZ+mq661EVXf9DZBF1CV+fetSpCUggurbYHWGGGvLRxjI96W0wR93EMoV/1pLHa4fgGVcFupoFsz/3n9J3wCkT4pRvHjn+hC7V11aSMI07t/OUeo1ssAziob9apuB+HUm2/EO17ph9TZvg3rydE5J0nmJ+DnYGLl5FfQJHgnnsjR5qaKs5wVNIKNA811lv/K5Iee/O6gMNXI3GAcGUvPKHMD5oQSs5B343blFUiqw47H1WZ9KWBoLXhVcnss6hLPmpDchInVhr3ObAHQhZIaBDBnhQR6nuc5BtgoFnwWw3w5G1yNNFt3qT3c75Rd/8NamdtveyskDbrHxQ5GkUo7ICuE/3SQG0yqWhVDjakYgVqjLG3BTKRg6I/dT5+VCeQbdxTZ+ScWkJfXYDEnlZlTjMLz2FS+B+9Zwez/1Dy/pm9H51jvuXI942kv31DfDVcTTwvKmb3SqIB9D4aomb16gDAywC7I4L/+ZwsqmcQoUvVtHGEe5NL3r6fgWtWzKht+60ZgBalmzdBfhZHd/bDXvTAX7SVpHraXjfpcdGrtv1LhU6Kr3gol94TR5Xh9dWKNhwR0s6at7MNCpj3eG9ogrWkmYSiqNeVT1ovscWvH70tDCtCNwY5sIUJVUSWdHc2OdzFxNIPkzh9OuHIn0o9Dbmup6obFoQOgi587HA/Sqvre9pDVqVMzn7w57naDmg+lsR9wS67BxbV4tVrvvfeVQAQQRft5SmSu7WCBnaWuM2/o890nE5ynFG+M2FqApJLgdfXozOguo/hROrD3WxLUyT8+FDsoCY8ky6YYTaF3OlOHpuD5Zmw4UGSdDbcEzhc6cRqBAYfMpxzN58VwNRrjG/MGTY3jrvOLJDk3sF7URa7Tn5YuRLp/2jL5YKWF2QftWiW9jl1YeORYQwNdLBuRK9L4/WOxNRWhYZToAJfwT4byh5dKzY8OnjeFcJy8RVvuRpoYeQY1P63ie/K1D3TgJEwKQXylLdPwtZRGM/P5sm2dcf5F+lRfZGufmv+Q++unrqWr6uv3IlgVGKPa8zTh+ipSaik7taKd+jertOSYkgjrrUfjC0/oyg5giuNqXh+ebIxQC7TMX8bManc5N3aomFuYXmIJ4iT/euQctq8N3gPPWmUNmrnXztdEZgpQdV2ahj+yRr1yxCriOfRo3Az3oPMPEaXxrqDc8RoWt/lOgzzp7KfBGoOVzYdvaDGr8MhsZ8Pd7HD0vZW8TNSl8R7cEK3G5EotNA+s6lF/n/RH5ewmg3cQK8bnnZ4oNjqYo3pGtw+H2rbefMMzAk4mybGe9uDqSdhWJgBzRGsMXRW+0atn9tuJTu6vPNn9asNvwXS0lcvpQskbe5sK3yhzyFzqwrJxs7Ji6yiu6kOtsQ5Girzk1835A2Zun6ZegGtQH8DbgIPdSBMkh6kypndBu4ns2gg97g9yB8XFGbv3c60sxaXPRuycl9IjBL81Yy6n6PCOBx7Pqm9xrzPn7QzgtetKxCmNUMYJvF+myiQFrMdL1w+1DD7FoZX9VNBDbNWgcfhgnL8tlHSmTb36KnScJOe8D+3n3+aDfkWpiHjrLrGJsHlaek/Ji2Cgwa0dIC6EhQitNttskIKVMa+yD6D1T6Dvz/eolnutjYwbGb3y9mtpQzmVwf2Zp2dzxYRWZk=\"}]},\"finishReason\": \"STOP\"}],\"usageMetadata\": {\"promptTokenCount\": 7,\"candidatesTokenCount\": 1,\"totalTokenCount\": 285,\"trafficType\": \"ON_DEMAND\",\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 7}],\"candidatesTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 1}],\"thoughtsTokenCount\": 277},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-09-22T03:46:26.748531Z\",\"responseId\": \"EvqxavPXLbqerb8P1eSViQs\"}\r\n\r\n"
"body": "data: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"text\": \"Hello\"}]}}],\"usageMetadata\": {\"trafficType\": \"ON_DEMAND\"},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-08-23T17:21:48.528714Z\",\"responseId\": \"LCyLasqiIO6crb8P1sDboQc\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"role\": \"model\",\"parts\": [{\"text\": \"\",\"thoughtSignature\": \"AY89a1+BGsRqlGpfT0psLB4jeTkT5rDV2HFOlrRuF7aVxDOjqNVUku6t4azeSnxpd+msHWuwXj4RS+7gmVlzVs+JNi8uj+iZWTBCi71vSh9kdK9ed/sHv9J7uL9ZWSOcgbhX/hxdXaUp5yVbQzHFXPjR9A/IkEkHV8VKarDZVFE1T1uASia74lkmyBeZZz+DQmRsLwbUHzFUKlF3qnk/SliLo21ZgASd7itlALQ0PBLJZwgeI3g7tDscDSE18hnB11Fky8q7MLd3HY16zbDvHBEMb18pmmPelPI01KdrCIwMSou/01/u5jiSUCc3pFksZawUj3tAHocHSC3ZKAQQQuUXGe5tm61C2E40/NANBeePc1S4HYE6Yo/vtX6tE02LDky5IQWX09H6+DZ7fpopP5nCUfcKPHa3hVjYquWYYMtZgXO4ZpxfVd3lt1VUDuJNN3BMMCZapjBoJZFPXPJ5t/yg9Rnd791+msGH77b4wztz1vtsPrT9oV9g6SDo9ZUH6BaOcbK7fw8FaXcGw+55malEwQy6zpRLGecooBu70p6RwhaAUyKIMX49y+F2hkNxQxDeBUNckJnu6n4w+KLyjP+bR0gqPJbGjVfteHm+QujqjJdBBT/m1u9kPo1nIbzdEs/PIADBdbuV7TkD/HoRFKpLnNmM2no8ioTtFEjKBDz4ippGi15r8pGgA6wIb/1HAvOGh+PVERdGcbelVTgfONwBqjQ7B1wmEizCfyYuMIskfwjxDGayfKlpDxrnNeogtEct9u5/DjEKlURlg9MtmW1B9P8BXYJ+7SCiRJWwW6bzB+5C+MLCnETl/mljDizoJMHK8DKIhI4oxBsrWXEuoHFwEwGIeOZq0BofH2Jz/l6+KIboV/zd581Kk0zPg/rlI6acfjUEtXtbF+t0+jzoJN7006x4i2tqXeJZ+4e5yisSArEsfJ0YzNWoJtBHG9V9/euDcEP3+jsr98efaQaQbLMPvT/Hb7CYQ7ChhGfcGxQ=\"}]},\"finishReason\": \"STOP\"}],\"usageMetadata\": {\"promptTokenCount\": 7,\"candidatesTokenCount\": 1,\"totalTokenCount\": 150,\"trafficType\": \"ON_DEMAND\",\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 7}],\"candidatesTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 1}],\"thoughtsTokenCount\": 142},\"modelVersion\": \"gemini-3.5-flash\",\"createTime\": \"2026-08-23T17:21:48.528714Z\",\"responseId\": \"LCyLasqiIO6crb8P1sDboQc\"}\r\n\r\n"
}
}
]
@@ -1,33 +0,0 @@
{
"version": 1,
"metadata": {
"model": "jev-1.13-free",
"tags": [
"prefix:opencode-zen-evaluation",
"provider:opencode",
"protocol:system-one"
],
"name": "opencode-zen-evaluation/evaluates-choice-score-and-boolean-questions",
"recordedAt": "2026-09-22T03:44:57.511Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://opencode.ai/zen/v1/systemone",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"jev-1.13-free\",\"state\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"questions\":{\"department\":{\"type\":\"choice\",\"instructions\":\"Which team should handle this support request?\",\"criteria\":{\"billing\":\"Payments, invoices, refunds, or failed charges\",\"technical\":\"Bugs, outages, or integrations\",\"sales\":\"Pricing, upgrades, or new accounts\"}},\"urgency\":{\"type\":\"score\",\"instructions\":\"How urgent is this support request?\",\"criteria\":[\"Can wait for normal support\",\"Needs prompt attention\",\"Actively blocking revenue\"]},\"refund\":{\"type\":\"noul\",\"instructions\":\"Is the customer asking for a refund?\"}}}"
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"jev-1.13-free\",\"answers\":{\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":1,\"probabilities\":{\"technical\":0,\"sales\":0,\"billing\":1}},\"urgency\":{\"type\":\"score\",\"score\":1.02,\"confidence\":0.95,\"legend\":{\"0\":\"Can wait for normal support\",\"1\":\"Needs prompt attention\",\"2\":\"Actively blocking revenue\"},\"probabilities\":{\"0\":0,\"1\":0.97,\"2\":0.03}},\"refund\":{\"type\":\"noul\",\"noul\":0.99}},\"usage\":{\"input_tokens\":422,\"output_tokens\":69}}"
}
}
]
}
@@ -1,33 +0,0 @@
{
"version": 1,
"metadata": {
"model": "typesafe/jev-1.13",
"tags": [
"prefix:openrouter-evaluation",
"provider:openrouter",
"protocol:system-one"
],
"name": "openrouter-evaluation/evaluates-choice-score-and-boolean-questions",
"recordedAt": "2026-09-22T14:54:46.656Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://openrouter.ai/api/v1/systemone",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"typesafe/jev-1.13\",\"state\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"questions\":{\"department\":{\"type\":\"choice\",\"instructions\":\"Which team should handle this support request?\",\"criteria\":{\"billing\":\"Payments, invoices, refunds, or failed charges\",\"technical\":\"Bugs, outages, or integrations\",\"sales\":\"Pricing, upgrades, or new accounts\"}},\"urgency\":{\"type\":\"score\",\"instructions\":\"How urgent is this support request?\",\"criteria\":[\"Can wait for normal support\",\"Needs prompt attention\",\"Actively blocking revenue\"]},\"refund\":{\"type\":\"noul\",\"instructions\":\"Is the customer asking for a refund?\"}}}"
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"typesafe/jev-1.13-20260917\",\"answers\":{\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"probabilities\":{\"billing\":1,\"sales\":0,\"technical\":0},\"confidence\":1},\"urgency\":{\"type\":\"score\",\"score\":1.02,\"legend\":{\"0\":\"Can wait for normal support\",\"1\":\"Needs prompt attention\",\"2\":\"Actively blocking revenue\"},\"probabilities\":{\"0\":0.01,\"1\":0.96,\"2\":0.03},\"confidence\":0.95},\"refund\":{\"type\":\"noul\",\"noul\":0.99}},\"usage\":{\"input_tokens\":422,\"output_tokens\":69,\"cost\":0.000017724},\"id\":\"gen-dec-1790088886-JRlffOLUwQdx4Lltu2se\",\"provider\":\"TypeSafe\"}"
}
}
]
}
@@ -1,33 +0,0 @@
{
"version": 1,
"metadata": {
"model": "jev-latest",
"tags": [
"prefix:typesafe-evaluation",
"provider:typesafe-ai",
"protocol:system-one"
],
"name": "typesafe-evaluation/evaluates-choice-score-and-boolean-questions",
"recordedAt": "2026-09-22T03:44:28.626Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.typesafe.ai/v1/systemone",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"jev-latest\",\"state\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"questions\":{\"department\":{\"type\":\"choice\",\"instructions\":\"Which team should handle this support request?\",\"criteria\":{\"billing\":\"Payments, invoices, refunds, or failed charges\",\"technical\":\"Bugs, outages, or integrations\",\"sales\":\"Pricing, upgrades, or new accounts\"}},\"urgency\":{\"type\":\"score\",\"instructions\":\"How urgent is this support request?\",\"criteria\":[\"Can wait for normal support\",\"Needs prompt attention\",\"Actively blocking revenue\"]},\"refund\":{\"type\":\"noul\",\"instructions\":\"Is the customer asking for a refund?\"}}}"
},
"response": {
"status": 200,
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"jev-1.13.0\",\"answers\":{\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":1.0,\"probabilities\":{\"billing\":1.0,\"sales\":0.0,\"technical\":0.0}},\"urgency\":{\"type\":\"score\",\"score\":1.01,\"confidence\":0.97,\"legend\":{\"0\":\"Can wait for normal support\",\"1\":\"Needs prompt attention\",\"2\":\"Actively blocking revenue\"},\"probabilities\":{\"0\":0.0,\"1\":0.98,\"2\":0.02}},\"refund\":{\"type\":\"noul\",\"noul\":0.99}},\"usage\":{\"input_tokens\":422,\"output_tokens\":69}}"
}
}
]
}
-116
View File
@@ -1,116 +0,0 @@
import { describe, expect } from "bun:test"
import { Effect, Fiber, Ref, Stream } from "effect"
import * as TestClock from "effect/testing/TestClock"
import {
AIError,
InvalidProviderOutputError,
Generation,
type GenerationRoute,
type GenerationSnapshot,
type GenerationStatus,
} from "../src/index.js"
import { it } from "./lib/effect.js"
/** In-memory generation route whose status advances through `statuses` on every poll. */
const scriptedRoute = (statuses: ReadonlyArray<GenerationStatus>, result: string) =>
Effect.gen(function* () {
const polls = yield* Ref.make(0)
const cancelled = yield* Ref.make(false)
// `count` is the number of polls so far; the first poll observes `statuses[0]`.
const snapshot = (count: number): GenerationSnapshot => ({
id: "gen_1",
status: statuses[Math.min(Math.max(count - 1, 0), statuses.length - 1)],
progress: count / statuses.length,
})
const route: GenerationRoute<string> = {
status: () => Ref.updateAndGet(polls, (count) => count + 1).pipe(Effect.map(snapshot)),
result: (token) =>
Effect.gen(function* () {
const count = yield* Ref.get(polls)
const status = snapshot(count).status
if (status === "completed") return `${result}:${String(token)}`
return yield* new AIError({ reason: new InvalidProviderOutputError({ message: `Generation ended ${status}` }) })
}),
cancel: () => Ref.set(cancelled, true),
}
return { route, polls, cancelled }
})
describe("Generation", () => {
it.effect("polls queued → running → completed and returns the result", () =>
Effect.gen(function* () {
const scripted = yield* scriptedRoute(["queued", "running", "completed"], "done")
const generation = new Generation(scripted.route, { op: "token_1" }, { id: "gen_1", status: "queued" })
expect(generation.terminal).toBe(false)
const fiber = yield* Effect.forkChild(generation.await({ poll: { interval: "1 second", timeout: "1 minute" } }))
yield* TestClock.adjust("3 seconds")
const result = yield* Fiber.join(fiber)
expect(result).toBe("done:[object Object]")
expect(yield* Ref.get(scripted.polls)).toBe(3)
}),
)
it.effect("returns immediately for an already terminal generation", () =>
Effect.gen(function* () {
const scripted = yield* scriptedRoute(["completed"], "done")
yield* Ref.set(scripted.polls, 1)
const generation = new Generation(scripted.route, "t", { id: "gen_1", status: "completed" })
expect(yield* generation.await()).toBe("done:t")
expect(yield* Ref.get(scripted.polls)).toBe(1)
}),
)
it.effect("fails with a Timeout reason when the generation never finishes", () =>
Effect.gen(function* () {
const scripted = yield* scriptedRoute(["running"], "never")
const generation = new Generation(scripted.route, "t", { id: "gen_1", status: "queued" })
const fiber = yield* Effect.forkChild(
generation.await({ poll: { interval: "1 second", timeout: "5 seconds" } }).pipe(Effect.flip),
)
yield* TestClock.adjust("6 seconds")
const error = yield* Fiber.join(fiber)
expect(error).toBeInstanceOf(AIError)
expect(error.reason._tag).toBe("Timeout")
expect(error.message).toContain("gen_1")
expect(yield* Ref.get(scripted.polls)).toBeGreaterThan(1)
}),
)
it.effect("surfaces the route failure body for failed generations", () =>
Effect.gen(function* () {
const scripted = yield* scriptedRoute(["running", "failed"], "unused")
const generation = new Generation(scripted.route, "t", { id: "gen_1", status: "queued" })
const fiber = yield* Effect.forkChild(generation.await({ poll: { interval: "1 second" } }).pipe(Effect.flip))
yield* TestClock.adjust("2 seconds")
const error = yield* Fiber.join(fiber)
expect(error.reason._tag).toBe("InvalidProviderOutput")
expect(error.message).toContain("Generation ended failed")
}),
)
it.effect("streams status events until the first terminal observation and cancels through the route", () =>
Effect.gen(function* () {
const scripted = yield* scriptedRoute(["queued", "running", "completed"], "done")
const generation = new Generation(scripted.route, "t", { id: "gen_1", status: "queued" })
const fiber = yield* Effect.forkChild(generation.events({ poll: { interval: "1 second" } }).pipe(Stream.runCollect))
yield* TestClock.adjust("3 seconds")
const events = Array.from(yield* Fiber.join(fiber))
expect(events).toEqual([
{ type: "generation-queued", id: "gen_1", position: undefined },
{ type: "generation-progress", id: "gen_1", progress: 2 / 3 },
{ type: "generation-finished", id: "gen_1", status: "completed" },
])
yield* generation.cancel()
expect(yield* Ref.get(scripted.cancelled)).toBe(true)
}),
)
})
+40 -131
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { Image, ImageClient, Media } from "../src/index.js"
import { Image, ImageClient, ImageInput } from "../src/index.js"
import { Google, OpenAI, XAI, ZAI } from "../src/providers.js"
import { it } from "./lib/effect.js"
import { dynamicResponse } from "./lib/http.js"
@@ -48,11 +48,11 @@ describe("Image", () => {
http: { body: { deployment: "test" }, headers: { "x-default": "yes" } },
}).image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
n: 2,
size: "2048x2048",
format: "jpeg",
providerOptions: {
options: {
n: 2,
size: "2048x2048",
quality: "future-quality",
outputFormat: "jpeg",
output_format: "avif",
outputCompression: 30,
output_compression: 40,
@@ -68,15 +68,14 @@ describe("Image", () => {
})
expect(response.images).toHaveLength(2)
expect(response.image.mediaType).toBe("image/webp")
expect(response.image.source).toEqual({ type: "bytes", data: Uint8Array.from([1, 2, 3]), mediaType: "image/webp" })
expect(yield* response.image.bytes()).toEqual(Uint8Array.from([1, 2, 3]))
expect(response.image.providerMetadata).toEqual({ openai: { revisedPrompt: "A precise robot" } })
expect(response.usage).toMatchObject({ type: "tokens", total: 12 })
expect(response.image?.mediaType).toBe("image/webp")
expect(response.image?.data).toEqual(Uint8Array.from([1, 2, 3]))
expect(response.image?.providerMetadata).toEqual({ openai: { revisedPrompt: "A precise robot" } })
expect(response.usage?.totalTokens).toBe(12)
}).pipe(
Effect.provide(
ImageClient.layer.pipe(
Layer.provideMerge(
Layer.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
@@ -114,32 +113,6 @@ describe("Image", () => {
),
)
it.effect("sends only model and prompt by default and decodes OpenAI bytes as png", () =>
Effect.gen(function* () {
const openai = OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" })
expect(openai.image("gpt-image-2").route.id).toBe("openai-images")
const response = yield* Image.generate({ model: openai.image("gpt-image-2"), prompt: "A lighthouse" }).pipe(
Effect.provide(
ImageClient.layer.pipe(
Layer.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
expect(web.url).toBe("https://openai.test/v1/images/generations")
expect(JSON.parse(input.text)).toEqual({ model: "gpt-image-2", prompt: "A lighthouse" })
return input.respond(JSON.stringify({ data: [{ b64_json: "AQID" }] }), {
headers: { "content-type": "application/json" },
})
}),
),
),
),
),
)
expect(response.image.source).toEqual({ type: "bytes", data: Uint8Array.from([1, 2, 3]), mediaType: "image/png" })
}),
)
it.effect("preserves native snake_case and unknown request options", () =>
Image.generate({
model: OpenAI.configure({
@@ -147,8 +120,8 @@ describe("Image", () => {
baseURL: "https://api.openai.test/v1",
}).image("future-image-model"),
prompt: "A lighthouse in fog",
format: "jpeg",
providerOptions: {
options: {
outputFormat: "jpeg",
output_format: "avif",
outputCompression: 30,
output_compression: 40,
@@ -157,7 +130,7 @@ describe("Image", () => {
}).pipe(
Effect.tap((response) =>
Effect.sync(() => {
expect(response.image.mediaType).toBe("image/avif")
expect(response.image?.mediaType).toBe("image/avif")
}),
),
Effect.provide(
@@ -187,9 +160,12 @@ describe("Image", () => {
Image.generate({
model: OpenAI.configure({ apiKey: "test", baseURL: "https://api.openai.test/v1" }).image("future-model"),
prompt: "Combine these images",
images: [Media.bytes(Uint8Array.from([1, 2, 3]), "image/png"), Media.fromDataUrl("data:image/jpeg;base64,BAUG")],
mask: Media.bytes(Uint8Array.from([7, 8, 9]), "image/png"),
providerOptions: {
images: [
ImageInput.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
ImageInput.url("data:image/jpeg;base64,BAUG"),
],
options: {
mask: ImageInput.bytes(Uint8Array.from([7, 8, 9]), "image/png"),
quality: "high",
future_option: true,
},
@@ -227,8 +203,8 @@ describe("Image", () => {
Image.generate({
model: OpenAI.configure({ apiKey: "test", baseURL: "https://api.openai.test/v1" }).image("future-model"),
prompt: "Combine these images",
images: [Media.url("https://example.test/source.png"), Media.ref("openai", "file_123")],
mask: Media.ref("openai", "file_mask"),
images: [ImageInput.url("https://example.test/source.png"), ImageInput.file("file_123")],
options: { mask: ImageInput.file("file_mask") },
http: { body: { future_option: true } },
}).pipe(
Effect.provide(
@@ -259,9 +235,9 @@ describe("Image", () => {
model: XAI.configure({ apiKey: "test", baseURL: "https://api.xai.test/v1" }).image("future-model"),
prompt: "Combine these images",
images: [
Media.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
Media.url("https://example.test/source.jpg"),
Media.ref("xai", "file_123"),
ImageInput.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
ImageInput.url("https://example.test/source.jpg"),
ImageInput.file("file_123"),
],
}).pipe(
Effect.provide(
@@ -293,7 +269,7 @@ describe("Image", () => {
Image.generate({
model: XAI.configure({ apiKey: "test", baseURL: "https://api.xai.test/v1" }).image("future-model"),
prompt: "Edit this image",
images: [Media.ref("xai", "file_123")],
images: [ImageInput.file("file_123")],
}).pipe(
Effect.provide(
ImageClient.layer.pipe(
@@ -321,9 +297,9 @@ describe("Image", () => {
model: Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).image("future-model"),
prompt: "Combine these images",
images: [
Media.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
Media.fromDataUrl("data:image/jpeg;base64,BAUG"),
Media.ref("google", "https://generativelanguage.googleapis.com/v1beta/files/123", "image/webp"),
ImageInput.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
ImageInput.url("data:image/jpeg;base64,BAUG"),
ImageInput.fileUri("https://generativelanguage.googleapis.com/v1beta/files/123", "image/webp"),
],
}).pipe(
Effect.provide(
@@ -362,20 +338,18 @@ describe("Image", () => {
Image.generate({
model: Google.configure({ apiKey: "test" }).image("model"),
prompt: "edit",
images: [Media.url("https://example.test/image.png")],
images: [ImageInput.url("https://example.test/image.png")],
}),
Image.generate({
model: ZAI.configure({ apiKey: "test" }).image("model"),
prompt: "edit",
images: [Media.bytes(Uint8Array.from([1]), "image/png")],
images: [ImageInput.bytes(Uint8Array.from([1]), "image/png")],
}),
]
yield* Effect.forEach(cases, (program) =>
program.pipe(
Effect.flip,
Effect.tap((error) =>
Effect.sync(() => expect(["InvalidRequest", "UnsupportedOperation"]).toContain(error.reason._tag)),
),
Effect.tap((error) => Effect.sync(() => expect(error.reason._tag).toBe("InvalidRequest"))),
),
)
}).pipe(
@@ -397,10 +371,10 @@ describe("Image", () => {
http: { body: { labels: { deployment: "test" } }, query: { api: "v1" } },
}).image("any-model-id"),
prompt: "A robot tending a rooftop garden",
aspectRatio: "16:9",
seed: 42,
providerOptions: {
options: {
aspectRatio: "16:9",
imageSize: "2K",
seed: 42,
thinkingLevel: "HIGH",
includeThoughts: true,
futureOption: true,
@@ -423,7 +397,7 @@ describe("Image", () => {
})
expect(response.images).toHaveLength(3)
expect(yield* Effect.forEach(response.images, (image) => image.bytes())).toEqual([
expect(response.images.map((image) => image.data)).toEqual([
Uint8Array.from([1, 2, 3]),
Uint8Array.from([4, 5, 6]),
Uint8Array.from([7, 8, 9]),
@@ -434,12 +408,10 @@ describe("Image", () => {
google: { candidateIndex: 0, partIndex: 3, finishReason: "STOP" },
})
expect(response.images[2].providerMetadata).toMatchObject({ google: { candidateIndex: 7, partIndex: 0 } })
expect(response.usage).toMatchObject({
type: "tokens",
input: 5,
output: 10,
details: { reasoningTokens: 3, google: { serviceTier: "STANDARD" } },
})
expect(response.usage?.inputTokens).toBe(5)
expect(response.usage?.outputTokens).toBe(10)
expect(response.usage?.reasoningTokens).toBe(3)
expect(response.usage?.providerMetadata).toMatchObject({ google: { serviceTier: "STANDARD" } })
expect(response.providerMetadata).toEqual({
google: {
modelVersion: "gemini-3.1-flash-image",
@@ -497,7 +469,7 @@ describe("Image", () => {
}).pipe(
Effect.provide(
ImageClient.layer.pipe(
Layer.provideMerge(
Layer.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
@@ -577,69 +549,6 @@ describe("Image", () => {
),
)
it.effect("surfaces filtered Google candidates as notices next to the returned image", () =>
Image.generate({
model: Google.configure({ apiKey: "test", baseURL: "https://generativelanguage.test/v1beta" }).image(
"gemini-3.1-flash-image",
),
prompt: "A robot tending a rooftop garden",
}).pipe(
Effect.tap((response) =>
Effect.sync(() => {
expect(response.images).toHaveLength(1)
expect(response.notices).toEqual([
{
type: "filtered",
message: "Google Images reported prompt feedback",
providerMetadata: { google: { promptFeedback: { blockReason: "OTHER" } } },
},
{
type: "filtered",
message: "Google Images candidate 1 finished with IMAGE_SAFETY: Blocked.",
providerMetadata: {
google: {
candidateIndex: 1,
finishReason: "IMAGE_SAFETY",
finishMessage: "Blocked.",
safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }],
},
},
},
])
}),
),
Effect.provide(
ImageClient.layer.pipe(
Layer.provide(
dynamicResponse((input) =>
Effect.succeed(
input.respond(
JSON.stringify({
promptFeedback: { blockReason: "OTHER" },
candidates: [
{
content: { parts: [{ inlineData: { mimeType: "image/png", data: "AQID" } }] },
finishReason: "STOP",
},
{
index: 1,
content: { parts: [{ text: "blocked" }] },
finishReason: "IMAGE_SAFETY",
finishMessage: "Blocked.",
safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }],
},
],
}),
{ headers: { "content-type": "application/json" } },
),
),
),
),
),
),
),
)
it.effect("includes Google diagnostics when no final image is returned", () =>
Image.generate({
model: Google.configure({ apiKey: "test", baseURL: "https://generativelanguage.test/v1beta" }).image(
+58 -65
View File
@@ -2,51 +2,50 @@ import { Effect } from "effect"
import {
Image,
ImageClient,
ImageInput,
ImageModel,
Media,
type ImageModelOptions,
type ImageOptions,
type ImageRequestFor,
type ImageRoute,
} from "../src/index.js"
import type { Service } from "../src/image-client.js"
import { Anthropic, Google, OpenAI, XAI, ZAI } from "../src/providers.js"
import { Google, OpenAI, XAI, ZAI } from "../src/providers.js"
type Requirements<T> = T extends Effect.Effect<infer _A, infer _E, infer R> ? R : never
type Equal<A, B> = [A, B] extends [B, A] ? true : false
type Assert<T extends true> = T
type GoogleLikeOptions = {
readonly aspectRatio?: "1:1" | "16:9"
readonly imageSize?: "1K" | "2K"
readonly thinkingLevel?: "LOW" | "HIGH"
} & Record<string, unknown>
declare const route: ImageRoute<GoogleLikeOptions>
const google = ImageModel.make<GoogleLikeOptions>({ id: "gemini-image", provider: "google", route })
// @ts-expect-error Extracted model options retain known provider fields.
const invalidGoogleOptions: ImageModelOptions<typeof google> = { imageSize: "8K" }
const invalidGoogleOptions: ImageModelOptions<typeof google> = { aspectRatio: "wide" }
void invalidGoogleOptions
Image.generate({
model: google,
prompt: "A lighthouse",
images: [
Media.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
Media.fromDataUrl("data:image/jpeg;base64,AQID"),
Media.ref("google", "https://generativelanguage.googleapis.com/v1beta/files/example", "image/webp"),
ImageInput.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
ImageInput.url("data:image/jpeg;base64,AQID"),
ImageInput.fileUri("https://generativelanguage.googleapis.com/v1beta/files/example", "image/webp"),
],
aspectRatio: "16:9",
seed: 7,
providerOptions: { imageSize: "2K", thinkingLevel: "HIGH", futureOption: true },
options: { aspectRatio: "16:9", imageSize: "2K", futureOption: true },
})
const googleProvider = Google.configure({ apiKey: "test" }).image("any-model-id")
Image.generate({
model: googleProvider,
prompt: "A lighthouse",
aspectRatio: "16:9",
providerOptions: {
options: {
aspectRatio: "16:9",
imageSize: "2K",
seed: 42,
thinkingLevel: "HIGH",
includeThoughts: true,
futureOption: true,
@@ -55,65 +54,57 @@ Image.generate({
Image.generate({
model: googleProvider,
prompt: "A lighthouse",
providerOptions: { imageSize: "8K", thinkingLevel: "FUTURE" },
options: { aspectRatio: "future-ratio", imageSize: "8K", thinkingLevel: "FUTURE" },
})
// @ts-expect-error Image generation options are request-scoped, not provider configuration.
Google.configure({ image: { providerOptions: { imageSize: "2K" } } })
// @ts-expect-error Known Google string options retain their value kind.
Image.generate({ model: googleProvider, prompt: "A lighthouse", providerOptions: { imageSize: 2 } })
Image.generate({ model: googleProvider, prompt: "A lighthouse", options: { imageSize: 2 } })
// @ts-expect-error Known Google numeric options retain their value kind.
Image.generate({ model: googleProvider, prompt: "A lighthouse", options: { seed: "42" } })
// @ts-expect-error Known Google boolean options retain their value kind.
Image.generate({ model: googleProvider, prompt: "A lighthouse", providerOptions: { includeThoughts: "yes" } })
Image.generate({ model: googleProvider, prompt: "A lighthouse", options: { includeThoughts: "yes" } })
const openai = OpenAI.image("gpt-image-2")
// @ts-expect-error Image generation options are request-scoped, not provider configuration.
OpenAI.configure({ image: { providerOptions: { quality: "medium" } } })
OpenAI.configure({ image: { options: { quality: "medium" } } })
const futureOpenAIOptions: ImageModelOptions<typeof openai> = { quality: "future-quality" }
void futureOpenAIOptions
Image.generate({
model: openai,
prompt: "A lighthouse",
images: [Media.url("https://example.com/source.png"), Media.ref("openai", "file_123")],
mask: Media.bytes(Uint8Array.from([1]), "image/png"),
n: 2,
size: "2048x2048",
format: "webp",
providerOptions: {
images: [ImageInput.url("https://example.com/source.png"), ImageInput.file("file_123")],
options: {
mask: ImageInput.bytes(Uint8Array.from([1]), "image/png"),
quality: "hd",
background: "transparent",
outputFormat: "webp",
size: "2048x2048",
future_option: true,
},
})
Image.generate({
model: openai,
prompt: "A lighthouse",
size: "256x256",
providerOptions: { quality: "future-quality" },
})
Image.generate({ model: openai, prompt: "A lighthouse", format: "future-format" })
Image.generate({ model: openai, prompt: "A lighthouse", providerOptions: { native_future_option: true } })
Image.generate({ model: openai, prompt: "A lighthouse", options: { quality: "future-quality", size: "256x256" } })
Image.generate({ model: openai, prompt: "A lighthouse", options: { size: "1792x1024" } })
Image.generate({ model: openai, prompt: "A lighthouse", options: { native_future_option: true } })
// @ts-expect-error Known OpenAI string options retain their value kind.
Image.generate({ model: openai, prompt: "A lighthouse", providerOptions: { quality: 1 } })
Image.generate({ model: openai, prompt: "A lighthouse", options: { quality: 1 } })
// @ts-expect-error Known OpenAI numeric options retain their value kind.
Image.generate({ model: openai, prompt: "A lighthouse", providerOptions: { outputCompression: "80" } })
Image.generate({ model: openai, prompt: "A lighthouse", options: { outputCompression: "80" } })
OpenAI.imageGeneration({ action: "future-action", quality: "future-quality", size: "2048x2048" })
// @ts-expect-error Hosted image generation numeric options retain their value kind.
OpenAI.imageGeneration({ partialImages: "2" })
// @ts-expect-error Known Google-like options are inferred from the selected model.
Image.generate({ model: google, prompt: "A lighthouse", providerOptions: { imageSize: "8K" } })
// @ts-expect-error Language models cannot be used for image requests.
Image.generate({ model: Anthropic.configure({ apiKey: "test" }).model("claude-sonnet-4-5"), prompt: "A lighthouse" })
Image.generate({ model: google, prompt: "A lighthouse", options: { aspectRatio: "wide" } })
const xai = XAI.configure({ apiKey: "test" }).image("any-model-id")
// @ts-expect-error Image generation options are request-scoped, not provider configuration.
XAI.configure({ image: { providerOptions: { resolution: "1k" } } })
XAI.configure({ image: { options: { resolution: "1k" } } })
Image.generate({
model: xai,
prompt: "A lighthouse",
images: [Media.fromDataUrl("data:image/png;base64,AQID"), Media.ref("xai", "file_123")],
n: 2,
aspectRatio: "16:9",
providerOptions: {
images: [ImageInput.url("data:image/png;base64,AQID"), ImageInput.file("file_123")],
options: {
n: 2,
aspectRatio: "future-ratio",
resolution: "future-resolution",
responseFormat: "future-format",
future_option: true,
@@ -122,43 +113,43 @@ Image.generate({
Image.generate({
model: xai,
prompt: "A lighthouse",
providerOptions: { response_format: "b64_json", native_future_option: true },
options: { aspect_ratio: "16:9", response_format: "b64_json", native_future_option: true },
})
// @ts-expect-error Common count is numeric.
Image.generate({ model: xai, prompt: "A lighthouse", n: "2" })
// @ts-expect-error Known xAI numeric options retain their value kind.
Image.generate({ model: xai, prompt: "A lighthouse", options: { n: "2" } })
// @ts-expect-error Known xAI string options retain their value kind.
Image.generate({ model: xai, prompt: "A lighthouse", providerOptions: { resolution: 2 } })
Image.generate({ model: xai, prompt: "A lighthouse", options: { resolution: 2 } })
const zai = ZAI.configure({ apiKey: "test" }).image("any-model-id")
// @ts-expect-error Image generation options are request-scoped, not provider configuration.
ZAI.configure({ image: { providerOptions: { quality: "hd" } } })
ZAI.configure({ image: { options: { quality: "hd" } } })
Image.generate({
model: zai,
prompt: "A lighthouse",
providerOptions: { quality: "future-quality", userID: "user-123", future_option: true },
options: { quality: "future-quality", userID: "user-123", future_option: true },
})
Image.generate({ model: zai, prompt: "A lighthouse", providerOptions: { user_id: "raw-user" } })
Image.generate({ model: zai, prompt: "A lighthouse", options: { user_id: "raw-user" } })
// @ts-expect-error Known Z.ai string options retain their value kind.
Image.generate({ model: zai, prompt: "A lighthouse", providerOptions: { quality: 1 } })
Image.generate({ model: zai, prompt: "A lighthouse", options: { quality: 1 } })
// @ts-expect-error Known Z.ai user IDs retain their value kind.
Image.generate({ model: zai, prompt: "A lighthouse", providerOptions: { userID: 1 } })
Image.generate({ model: zai, prompt: "A lighthouse", options: { userID: 1 } })
declare const generic: ImageModel<ImageOptions>
Image.generate({ model: generic, prompt: "A lighthouse", providerOptions: { arbitrary: true } })
const explicitAsset: Media.Asset = Media.url("https://example.com/image.png")
void explicitAsset
Image.generate({ model: generic, prompt: "A lighthouse", options: { arbitrary: true } })
const explicitImageInput: ImageInput = ImageInput.url("https://example.com/image.png")
void explicitImageInput
// @ts-expect-error Raw strings are ambiguous and are not media assets.
// @ts-expect-error Raw strings are ambiguous and are not image inputs.
Image.generate({ model: openai, prompt: "A lighthouse", images: ["AQID"] })
// @ts-expect-error Plain source objects must be lifted into `Media.Asset` first.
// @ts-expect-error Byte image inputs require an explicit MIME type.
Image.generate({ model: openai, prompt: "A lighthouse", images: [{ type: "bytes", data: new Uint8Array() }] })
// @ts-expect-error Masks are media assets, not strings.
Image.generate({ model: openai, prompt: "A lighthouse", mask: "https://example.com/mask.png" })
// @ts-expect-error File URIs require an explicit MIME type for Gemini fileData.
Image.generate({ model: google, prompt: "A lighthouse", images: [{ type: "file-uri", uri: "files/123" }] })
const request = Image.request({
model: google,
prompt: "A lighthouse",
providerOptions: { imageSize: "1K", futureOption: true },
options: { aspectRatio: "1:1", futureOption: true },
})
const typedRequest: ImageRequestFor<GoogleLikeOptions> = request
void typedRequest
@@ -166,13 +157,15 @@ const generated = ImageClient.generate(request)
type GenerateRequirements = Assert<Equal<Requirements<typeof generated>, Service>>
void (true satisfies GenerateRequirements)
// @ts-expect-error Image requests use `n`, not `count`.
// @ts-expect-error Image requests no longer expose a common count option.
Image.generate({ model: openai, prompt: "A lighthouse", count: 2 })
// @ts-expect-error Image sizes are `${width}x${height}` strings.
// @ts-expect-error Image requests no longer expose a common size option.
Image.generate({ model: openai, prompt: "A lighthouse", size: { width: 1024, height: 1024 } })
// @ts-expect-error Aspect ratios are `${w}:${h}` strings.
Image.generate({ model: openai, prompt: "A lighthouse", aspectRatio: "wide" })
// @ts-expect-error Image requests no longer expose a common aspectRatio option.
Image.generate({ model: openai, prompt: "A lighthouse", aspectRatio: "16:9" })
// @ts-expect-error Image requests no longer expose a common seed option.
Image.generate({ model: openai, prompt: "A lighthouse", seed: 1 })
// @ts-expect-error Image requests do not expose metadata.
Image.generate({ model: openai, prompt: "A lighthouse", metadata: { trace: true } })
// @ts-expect-error `options` was renamed to `providerOptions`.
Image.generate({ model: openai, prompt: "A lighthouse", options: { quality: "hd" } })
// @ts-expect-error Masks are provider options, not a common image request field.
Image.generate({ model: openai, prompt: "A lighthouse", mask: ImageInput.url("https://example.com/mask.png") })
+1 -1
View File
@@ -17,7 +17,7 @@ export type Handler = (
input: HandlerInput,
) => Effect.Effect<HttpClientResponse.HttpClientResponse, HttpClientError.HttpClientError>
export const handlerLayer = (handler: Handler): Layer.Layer<HttpClient.HttpClient> =>
const handlerLayer = (handler: Handler): Layer.Layer<HttpClient.HttpClient> =>
Layer.succeed(
HttpClient.HttpClient,
HttpClient.make((request) =>
-180
View File
@@ -1,180 +0,0 @@
import { describe, expect } from "bun:test"
import { NodeFileSystem } from "@effect/platform-node"
import { Effect, Ref, Schema } from "effect"
import { FileSystem } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { Media, Message } from "../src/index.js"
import { it } from "./lib/effect.js"
import { dynamicResponse, scriptedResponses } from "./lib/http.js"
const PNG = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0])
const JPEG = Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0, 0])
const GIF = Uint8Array.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61])
const WEBP = Uint8Array.from([0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x45, 0x42, 0x50])
const PDF = Uint8Array.from([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31])
describe("Media", () => {
it.effect("round-trips every source through Media.from(asset.source)", () =>
Effect.sync(() => {
const assets = [
Media.bytes(PNG),
Media.base64("AQID", "image/png"),
Media.url("https://example.test/a.png", { mediaType: "image/png", expiresAt: 123, headers: { a: "b" } }),
Media.ref("openai", "file_123", "image/webp"),
]
for (const asset of assets) {
const copy = Media.from(asset.source)
expect(copy.source).toEqual(asset.source)
expect(copy.mediaType).toBe(asset.mediaType)
expect(copy.kind).toBe(asset.kind)
expect(copy.expiresAt).toBe(asset.expiresAt)
}
expect(assets.map((asset) => asset.kind)).toEqual(["image", "image", "image", "image"])
expect(Media.url("https://example.test/unknown").mediaType).toBe("application/octet-stream")
expect(Media.url("https://example.test/unknown").kind).toBe("other")
expect(Media.base64("AQID", "application/pdf").kind).toBe("document")
expect(Media.base64("AQID", "video/mp4").kind).toBe("video")
expect(Media.base64("AQID", "audio/wav").kind).toBe("audio")
}),
)
it.effect("parses data URLs and rejects malformed ones", () =>
Effect.sync(() => {
const asset = Media.fromDataUrl("data:image/jpeg;base64,/9j/")
expect(asset.source).toEqual({ type: "base64", data: "/9j/", mediaType: "image/jpeg" })
expect(asset.mediaType).toBe("image/jpeg")
expect(Media.fromDataUrl("data:text/plain;charset=utf-8;base64,aGk=").source).toEqual({
type: "base64",
data: "aGk=",
mediaType: "text/plain",
})
expect(() => Media.fromDataUrl("https://example.test/a.png")).toThrow(
"Media data URLs must contain a MIME type and base64 data",
)
expect(() => Media.fromDataUrl("data:image/png,rawtext")).toThrow()
}),
)
it.effect("detects media types from magic bytes", () =>
Effect.sync(() => {
expect(Media.detectMediaType(PNG)).toBe("image/png")
expect(Media.detectMediaType(JPEG)).toBe("image/jpeg")
expect(Media.detectMediaType(GIF)).toBe("image/gif")
expect(Media.detectMediaType(WEBP)).toBe("image/webp")
expect(Media.detectMediaType(PDF)).toBe("application/pdf")
expect(Media.detectMediaType(Uint8Array.from([1, 2, 3]))).toBeUndefined()
expect(Media.bytes(PNG).mediaType).toBe("image/png")
expect(Media.bytes(Uint8Array.from([1, 2, 3])).mediaType).toBe("application/octet-stream")
expect(Media.bytes(Uint8Array.from([1, 2, 3]), "image/x-custom").mediaType).toBe("image/x-custom")
}),
)
it.effect("materializes url assets through the request executor once", () =>
Effect.gen(function* () {
const requests = yield* Ref.make<Array<HttpClientRequest.HttpClientRequest>>([])
const asset = Media.url("https://cdn.example.test/generated", {
expiresAt: 42,
headers: { authorization: "Bearer media" },
providerMetadata: { example: { id: "gen_1" } },
})
const program = Effect.gen(function* () {
const first = yield* asset.bytes()
const second = yield* asset.bytes()
expect(second).toBe(first)
expect(first).toEqual(PNG)
expect(yield* asset.base64()).toBe(Buffer.from(PNG).toString("base64"))
expect(yield* asset.dataUrl()).toBe(
`data:application/octet-stream;base64,${Buffer.from(PNG).toString("base64")}`,
)
const owned = yield* asset.materialize()
expect(owned.source).toEqual({ type: "bytes", data: PNG, mediaType: "image/png" })
expect(owned.mediaType).toBe("image/png")
expect(owned.kind).toBe("image")
expect(owned.expiresAt).toBeUndefined()
expect(owned.providerMetadata).toEqual({ example: { id: "gen_1" } })
expect(yield* owned.materialize()).toBe(owned)
const seen = yield* Ref.get(requests)
expect(seen).toHaveLength(1)
expect(seen[0].url).toBe("https://cdn.example.test/generated")
expect(seen[0].headers.authorization).toBe("Bearer media")
})
yield* program.pipe(
Effect.provide(
dynamicResponse((input) =>
Ref.update(requests, (all) => [...all, input.request]).pipe(
Effect.map(() => input.respond(PNG, { headers: { "content-type": "image/png" } })),
),
),
),
)
}),
)
it.effect("caches decoded base64 bytes and encodes owned bytes lazily", () =>
Effect.gen(function* () {
const fromBase64 = Media.base64("AQID", "image/png")
const decoded = yield* fromBase64.bytes()
expect(decoded).toEqual(Uint8Array.from([1, 2, 3]))
expect(yield* fromBase64.bytes()).toBe(decoded)
expect(yield* fromBase64.base64()).toBe("AQID")
const fromBytes = Media.bytes(Uint8Array.from([1, 2, 3]), "image/png")
const encoded = yield* fromBytes.base64()
expect(encoded).toBe("AQID")
expect(yield* fromBytes.base64()).toBe(encoded)
expect(yield* fromBytes.dataUrl()).toBe("data:image/png;base64,AQID")
const invalid = yield* Media.base64("not base64!", "image/png").bytes().pipe(Effect.flip)
expect(invalid.reason._tag).toBe("InvalidRequest")
const ref = yield* Media.ref("openai", "file_1").bytes().pipe(Effect.flip)
expect(ref.reason._tag).toBe("InvalidRequest")
}).pipe(Effect.provide(scriptedResponses(["unused"]))),
)
it.effect("serializes assets inside messages and restores them as Media.Asset", () =>
Effect.sync(() => {
const codec = Schema.fromJsonString(Message)
const message = Message.user([
Message.media(Media.base64("AQID", "image/png", { info: { width: 1, height: 1 } }), { filename: "a.png" }),
Message.media(Media.url("https://example.test/b.pdf", { mediaType: "application/pdf", expiresAt: 7 })),
])
const json = Schema.encodeSync(codec)(message)
expect(json).not.toContain("null")
const restored = Schema.decodeSync(codec)(json)
const parts = restored.content.filter((part) => part.type === "media")
expect(parts).toHaveLength(2)
expect(parts[0].media).toBeInstanceOf(Media.Asset)
expect(parts[0].media.source).toEqual({ type: "base64", data: "AQID", mediaType: "image/png" })
expect(parts[0].media.info).toEqual({ width: 1, height: 1 })
expect(parts[0].filename).toBe("a.png")
expect(parts[1].media.kind).toBe("document")
expect(parts[1].media.expiresAt).toBe(7)
}),
)
it.effect("reads files with sniffed media types and writes materialized assets", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem
const dir = yield* fs.makeTempDirectoryScoped()
const source = `${dir}/source.bin`
yield* fs.writeFile(source, PNG)
const asset = yield* Media.file(source)
expect(asset.mediaType).toBe("image/png")
expect(asset.source).toEqual({ type: "bytes", data: PNG, mediaType: "image/png" })
const plain = `${dir}/notes.md`
yield* fs.writeFile(plain, new TextEncoder().encode("# hi"))
expect((yield* Media.file(plain)).mediaType).toBe("text/markdown")
const target = `${dir}/copy.png`
yield* Media.write(Media.base64("AQID", "image/png"), target)
expect(yield* fs.readFile(target)).toEqual(Uint8Array.from([1, 2, 3]))
const missing = yield* Media.file(`${dir}/missing.png`).pipe(Effect.flip)
expect(missing.reason._tag).toBe("InvalidRequest")
}).pipe(Effect.provide(NodeFileSystem.layer), Effect.provide(scriptedResponses(["unused"]))),
)
})
-102
View File
@@ -1,102 +0,0 @@
import { describe, expect, test } from "bun:test"
import { Effect, Layer } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { AIError, LLMEvent, Media } from "../src/index.js"
import { RequestExecutor } from "../src/route.js"
import { AI } from "../src/promise.js"
import { OpenAI } from "../src/providers.js"
import { handlerLayer } from "./lib/http.js"
import { sseEvents } from "./lib/sse.js"
const openai = OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" })
const chatBody = sseEvents(
{ choices: [{ delta: { content: "Hello" } }] },
{ choices: [{ delta: { content: " world" } }] },
{ choices: [{ delta: {}, finish_reason: "stop" }] },
)
/** Executor layer that answers chat completions with SSE text and image generations with one base64 PNG. */
const executor = (seen: Array<string>) =>
RequestExecutor.layer.pipe(
Layer.provide(
handlerLayer((input) =>
Effect.gen(function* () {
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
seen.push(web.url)
if (web.url.endsWith("/images/generations"))
return input.respond(JSON.stringify({ data: [{ b64_json: "AQID" }], output_format: "png" }), {
headers: { "content-type": "application/json" },
})
if (web.url.endsWith("/chat/completions"))
return input.respond(chatBody, { headers: { "content-type": "text/event-stream" } })
return input.respond(JSON.stringify({ error: { message: "not found" } }), {
status: 404,
headers: { "content-type": "application/json" },
})
}),
),
),
)
describe("AI promise client", () => {
test("generates text, images, and streams over one managed runtime", async () => {
const seen: Array<string> = []
const ai = AI.make({ layer: executor(seen) })
const text = await ai.llm.generate({ model: openai.chat("gpt-4o-mini"), prompt: "Say hello." })
expect(text.text).toBe("Hello world")
const image = await ai.image.generate({ model: openai.image("gpt-image-2"), prompt: "A lighthouse" })
expect(image.image).toBeInstanceOf(Media.Asset)
expect(image.image.mediaType).toBe("image/png")
expect(await ai.run(image.image.bytes())).toEqual(Uint8Array.from([1, 2, 3]))
const deltas: Array<string> = []
for await (const event of ai.llm.stream({ model: openai.chat("gpt-4o-mini"), prompt: "Say hello." })) {
if (LLMEvent.is.textDelta(event)) deltas.push(event.text)
}
expect(deltas).toEqual(["Hello", " world"])
const imageEvents: Array<string> = []
for await (const event of ai.image.stream({ model: openai.image("gpt-image-2"), prompt: "A lighthouse" })) {
imageEvents.push(event.type)
}
expect(imageEvents).toEqual(["image", "finish"])
expect(seen).toEqual([
"https://openai.test/v1/chat/completions",
"https://openai.test/v1/images/generations",
"https://openai.test/v1/chat/completions",
"https://openai.test/v1/images/generations",
])
await ai.dispose()
})
test("rethrows AIError unchanged and honors abort signals", async () => {
const ai = AI.make({ layer: executor([]) })
const failure = await ai.llm
.generate({ model: openai.responses("gpt-5"), prompt: "Hello" })
.then(() => undefined)
.catch((error: unknown) => error)
expect(failure).toBeInstanceOf(AIError)
expect(failure instanceof AIError && failure.reason.http?.status).toBe(404)
const controller = new AbortController()
controller.abort()
const aborted = await ai.llm
.generate({ model: openai.chat("gpt-4o-mini"), prompt: "Hello" }, { signal: controller.signal })
.then(() => "completed")
.catch(() => "aborted")
expect(aborted).toBe("aborted")
await ai.dispose()
})
test("the default client is created lazily and can be disposed", async () => {
expect(typeof AI.ai.llm.generate).toBe("function")
expect(typeof AI.ai.image.generate).toBe("function")
await AI.ai.dispose()
})
})
@@ -1,6 +1,6 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM, LLMEvent, LLMRequest, Message, ToolDefinition, Media } from "../../src/index.js"
import { LLM, LLMEvent, LLMRequest, Message, ToolDefinition } from "../../src/index.js"
import { Alibaba } from "../../src/providers.js"
import { LLMClient } from "../../src/route.js"
import { compileRequest } from "../../src/route/client.js"
@@ -65,7 +65,7 @@ for (const api of ["chat", "messages", "responses"] as const) {
messages: [
Message.user([
{ type: "text", text: "Read the three words in this image. Reply with only the words in order." },
{ type: "media", media: Media.bytes(bytes, "image/png") },
{ type: "media", mediaType: "image/png", data: bytes },
]),
],
generation: { maxTokens: 4096 },
@@ -1,17 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import {
CacheHint,
LLM,
AIError,
LLMRequest,
Message,
ToolCallPart,
ToolDefinition,
Usage,
Media,
} from "../../src/index.js"
import { CacheHint, LLM, AIError, LLMRequest, Message, ToolCallPart, ToolDefinition, Usage } from "../../src/index.js"
import { Auth, Endpoint, LLMClient, Route } from "../../src/route.js"
import { compileRequest } from "../../src/route/client.js"
import * as AnthropicMessages from "../../src/protocols/anthropic-messages.js"
@@ -376,7 +366,7 @@ describe("Anthropic Messages route", () => {
model: opus48,
messages: [
Message.user("Before."),
Message.make({ role: "system", content: { type: "media", media: Media.base64("AAECAw==", "image/png") } }),
Message.make({ role: "system", content: { type: "media", mediaType: "image/png", data: "AAECAw==" } }),
],
}),
).pipe(Effect.flip)
@@ -2124,8 +2114,8 @@ describe("Anthropic Messages route", () => {
messages: [
Message.user([
{ type: "text", text: "What is in this image?" },
{ type: "media", media: Media.base64("AAECAw==", "image/png") },
{ type: "media", media: Media.base64("JVBERi0xLjQ=", "application/pdf"), filename: "report.pdf" },
{ type: "media", mediaType: "image/png", data: "AAECAw==" },
{ type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=", filename: "report.pdf" },
]),
],
}),
@@ -4,7 +4,6 @@ import { describe, expect } from "bun:test"
import { Effect, Encoding, Ref, Schema, Stream } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import {
Media,
CacheHint,
GenerationOptions,
type LanguageModel,
@@ -1679,10 +1678,10 @@ describe("Bedrock Converse route", () => {
messages: [
Message.user([
{ type: "text", text: "What is in this image?" },
{ type: "media", media: Media.base64("AAAA", "image/png") },
{ type: "media", media: Media.base64("BBBB", "image/jpeg") },
{ type: "media", media: Media.base64("CCCC", "image/jpg") },
{ type: "media", media: Media.base64("DDDD", "image/webp") },
{ type: "media", mediaType: "image/png", data: "AAAA" },
{ type: "media", mediaType: "image/jpeg", data: "BBBB" },
{ type: "media", mediaType: "image/jpg", data: "CCCC" },
{ type: "media", mediaType: "image/webp", data: "DDDD" },
]),
],
cache: "none",
@@ -1713,9 +1712,7 @@ describe("Bedrock Converse route", () => {
LLM.request({
id: "req_image_bytes",
model,
messages: [
Message.user([{ type: "media", media: Media.bytes(new Uint8Array([1, 2, 3, 4, 5]), "image/png") }]),
],
messages: [Message.user([{ type: "media", mediaType: "image/png", data: new Uint8Array([1, 2, 3, 4, 5]) }])],
}),
)
@@ -1736,7 +1733,7 @@ describe("Bedrock Converse route", () => {
const error = yield* compileRequest(
LLM.request({
model,
messages: [Message.user({ type: "media", media: Media.base64("not base64!", "image/png") })],
messages: [Message.user({ type: "media", mediaType: "image/png", data: "https://example.test/image.png" })],
}),
).pipe(Effect.flip)
@@ -1745,25 +1742,6 @@ describe("Bedrock Converse route", () => {
}),
)
it.effect("rejects remote image URLs that were not materialized", () =>
Effect.gen(function* () {
const error = yield* compileRequest(
LLM.request({
model,
messages: [
Message.user({
type: "media",
media: Media.url("https://example.test/image.png", { mediaType: "image/png" }),
}),
],
}),
).pipe(Effect.flip)
expect(error).toMatchObject({ reason: { _tag: "InvalidRequest" } })
expect(error.message).toContain("requires inline media")
}),
)
it.effect("lowers document media into Bedrock document blocks with format and name", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
@@ -1774,8 +1752,8 @@ describe("Bedrock Converse route", () => {
messages: [
Message.user([
{ type: "text", text: "Summarize these documents." },
{ type: "media", media: Media.base64("UERGREFUQQ==", "application/pdf"), filename: "report.pdf" },
{ type: "media", media: Media.base64("Q1NWREFUQQ==", "text/csv"), filename: "data.csv" },
{ type: "media", mediaType: "application/pdf", data: "UERGREFUQQ==", filename: "report.pdf" },
{ type: "media", mediaType: "text/csv", data: "Q1NWREFUQQ==", filename: "data.csv" },
]),
],
}),
@@ -1850,7 +1828,7 @@ describe("Bedrock Converse route", () => {
messages: [
Message.user([
{ type: "text", text: "Read this document" },
{ type: "media", media: Media.base64("UERGREFUQQ==", "application/pdf"), filename: item.filename },
{ type: "media", mediaType: "application/pdf", data: "UERGREFUQQ==", filename: item.filename },
]),
Message.assistant([ToolCallPart.make({ id: "call_read", name: "read", input: {} })]),
Message.tool({
@@ -1900,7 +1878,8 @@ describe("Bedrock Converse route", () => {
{ type: "text", text: "Read these documents" },
...["report_v1.txt", "report#v1.txt", "report v1 2.txt", "report v1.txt"].map((filename) => ({
type: "media" as const,
media: Media.base64("SGVsbG8=", "text/plain"),
mediaType: "text/plain",
data: "SGVsbG8=",
filename,
})),
]),
@@ -1929,7 +1908,8 @@ describe("Bedrock Converse route", () => {
messages: [
Message.user({
type: "media",
media: Media.base64("UERGREFUQQ==", "application/pdf"),
mediaType: "application/pdf",
data: "UERGREFUQQ==",
filename: "report.pdf",
}),
],
@@ -1959,7 +1939,8 @@ describe("Bedrock Converse route", () => {
{ type: "text", text: "Read these documents" },
...["report", undefined, 'report "final"\n.pdf'].map((filename) => ({
type: "media" as const,
media: Media.base64("SGVsbG8=", "text/plain"),
mediaType: "text/plain",
data: "SGVsbG8=",
filename,
})),
]),
@@ -2056,7 +2037,7 @@ describe("Bedrock Converse route", () => {
).pipe(Effect.flip)
expect(error).toMatchObject({ reason: { _tag: "InvalidRequest" } })
expect(error.message).toContain("requires inline media")
expect(error.message).toContain("Bedrock Converse media data must be valid base64")
}),
)
@@ -2066,7 +2047,7 @@ describe("Bedrock Converse route", () => {
LLM.request({
id: "req_bad_image",
model,
messages: [Message.user([{ type: "media", media: Media.base64("x", "image/svg+xml") }])],
messages: [Message.user([{ type: "media", mediaType: "image/svg+xml", data: "x" }])],
}),
).pipe(Effect.flip)
@@ -2080,9 +2061,7 @@ describe("Bedrock Converse route", () => {
LLM.request({
id: "req_bad_doc",
model,
messages: [
Message.user([{ type: "media", media: Media.base64("x", "application/x-tar"), filename: "a.tar" }]),
],
messages: [Message.user([{ type: "media", mediaType: "application/x-tar", data: "x", filename: "a.tar" }])],
}),
).pipe(Effect.flip)
@@ -1,96 +0,0 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Evaluation, EvaluationModel, type EvaluationOptions } from "../../src/experimental.js"
import { OpenCodeZen, OpenRouter, TypeSafeAI } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
const questions = {
department: {
type: "choice",
instructions: "Which team should handle this support request?",
criteria: {
billing: "Payments, invoices, refunds, or failed charges",
technical: "Bugs, outages, or integrations",
sales: "Pricing, upgrades, or new accounts",
},
},
urgency: {
type: "score",
instructions: "How urgent is this support request?",
criteria: ["Can wait for normal support", "Needs prompt attention", "Actively blocking revenue"],
},
refund: { type: "boolean", instructions: "Is the customer asking for a refund?" },
} as const
const state = "I was charged twice for the same invoice. Please refund the duplicate payment today."
const typesafe = recordedTests({
prefix: "typesafe-evaluation",
provider: "typesafe-ai",
protocol: "system-one",
requires: ["TYPESAFE_API_KEY"],
metadata: { model: "jev-latest" },
})
const zen = recordedTests({
prefix: "opencode-zen-evaluation",
provider: "opencode",
protocol: "system-one",
requires: ["OPENCODE_API_KEY"],
metadata: { model: "jev-1.13-free" },
})
const openrouter = recordedTests({
prefix: "openrouter-evaluation",
provider: "openrouter",
protocol: "system-one",
requires: ["OPENROUTER_API_KEY"],
metadata: { model: "typesafe/jev-1.13" },
})
describe("experimental Evaluation recorded", () => {
typesafe.effect("evaluates choice score and boolean questions", () =>
assertEvaluation(
TypeSafeAI.configure({ apiKey: process.env.TYPESAFE_API_KEY ?? "fixture" }).experimental.evaluation("jev-latest"),
"typesafe",
),
)
zen.effect("evaluates choice score and boolean questions", () =>
assertEvaluation(
OpenCodeZen.configure({ apiKey: process.env.OPENCODE_API_KEY ?? "fixture" }).experimental.evaluation(
"jev-1.13-free",
),
"opencode",
),
)
openrouter.effect("evaluates choice score and boolean questions", () =>
assertEvaluation(
OpenRouter.configure({ apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" }).experimental.evaluation(
"typesafe/jev-1.13",
),
"openrouter",
),
)
})
const assertEvaluation = <Options extends EvaluationOptions>(
model: EvaluationModel<Options>,
metadataKey: "typesafe" | "opencode" | "openrouter",
) =>
Effect.gen(function* () {
const response = yield* Evaluation.run({ model, state, questions })
expect(response.model).toContain("jev-")
expect(response.answers.department.type).toBe("choice")
expect(response.answers.department.choice).toBe("billing")
expect(response.answers.department.probabilities?.billing).toBeGreaterThan(0.5)
expect(response.answers.urgency.type).toBe("score")
expect(response.answers.urgency.score).toBeGreaterThanOrEqual(0)
expect(response.answers.urgency.score).toBeLessThanOrEqual(2)
expect(response.answers.refund.type).toBe("boolean")
expect(response.answers.refund.probability).toBeGreaterThan(0.5)
expect(response.usage?.inputTokens).toBeGreaterThan(0)
expect(response.usage?.outputTokens).toBeGreaterThan(0)
expect(response.providerMetadata?.[metadataKey]?.confidence).toBeDefined()
})
+15 -30
View File
@@ -1,6 +1,6 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM, AIError, LLMRequest, Message, ToolCallPart, ToolDefinition, Usage, Media } from "../../src/index.js"
import { LLM, AIError, LLMRequest, Message, ToolCallPart, ToolDefinition, Usage } from "../../src/index.js"
import { Auth, LLMClient } from "../../src/route.js"
import { compileRequest } from "../../src/route/client.js"
import * as Gemini from "../../src/protocols/gemini.js"
@@ -340,8 +340,8 @@ describe("Gemini route", () => {
messages: [
Message.user([
{ type: "text", text: "What is in this image?" },
{ type: "media", media: Media.base64("AAECAw==", "image/png") },
{ type: "media", media: Media.base64("JVBERi0xLjQ=", "application/pdf") },
{ type: "media", mediaType: "image/png", data: "AAECAw==" },
{ type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=" },
]),
Message.assistant([ToolCallPart.make({ id: "call_1", name: "lookup", input: { query: "weather" } })]),
Message.tool({ id: "call_1", name: "lookup", result: { forecast: "sunny" } }),
@@ -446,7 +446,7 @@ describe("Gemini route", () => {
LLM.request({
model,
messages: [
Message.user({ type: "media", media: Media.fromDataUrl("data:image/png;base64,AAEC") }),
Message.user({ type: "media", mediaType: "image/png", data: "data:image/png;base64,AAEC" }),
Message.tool({
id: "call_image",
name: "read",
@@ -636,9 +636,9 @@ describe("Gemini route", () => {
model,
messages: [
Message.user([
{ type: "media", media: Media.base64("%%%=", "image/png") },
{ type: "media", media: Media.fromDataUrl("data:image/jpeg;base64,/9j/") },
{ type: "media", media: Media.base64("PHN2Zz4=", "image/svg+xml") },
{ type: "media", mediaType: "image/png", data: "%%%=" },
{ type: "media", mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" },
{ type: "media", mediaType: "image/svg+xml", data: "PHN2Zz4=" },
]),
],
}),
@@ -648,7 +648,7 @@ describe("Gemini route", () => {
role: "user",
parts: [
{ inlineData: { mimeType: "image/png", data: "%%%=" } },
{ inlineData: { mimeType: "image/jpeg", data: "/9j/" } },
{ inlineData: { mimeType: "image/png", data: "/9j/" } },
{ inlineData: { mimeType: "image/svg+xml", data: "PHN2Zz4=" } },
],
},
@@ -1773,34 +1773,19 @@ describe("Gemini route", () => {
}),
)
it.effect("replays generated assistant media as model inline data", () =>
it.effect("rejects unsupported assistant media content", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
const error = yield* compileRequest(
LLM.request({
id: "req_media",
model,
messages: [
Message.user("Draw a cat"),
Message.assistant([
{ type: "text", text: "Here you go." },
{
type: "media",
media: Media.base64("AAECAw==", "image/png"),
providerMetadata: { google: { thoughtSignature: "sig_1" } },
},
]),
Message.user("Now make it orange"),
],
messages: [Message.assistant({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
}),
)
).pipe(Effect.flip)
expect(prepared.body.contents[1]).toEqual({
role: "model",
parts: [
{ text: "Here you go." },
{ inlineData: { mimeType: "image/png", data: "AAECAw==" }, thoughtSignature: "sig_1" },
],
})
expect(error.message).toContain(
"Gemini assistant messages only support text, reasoning, and tool-call content for now",
)
}),
)
})
@@ -1,6 +1,6 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Image, Media } from "../../src/index.js"
import { Image, ImageInput } from "../../src/index.js"
import { Google } from "../../src/providers.js"
import { dimensions } from "../lib/image.js"
import { recordedTests } from "../recorded-test.js"
@@ -22,12 +22,13 @@ describe("Google Images recorded", () => {
const response = yield* Image.generate({
model,
prompt: "A simple flat blue circle centered on a plain white background.",
aspectRatio: "1:1",
options: { aspectRatio: "1:1" },
})
expect(response.images).toHaveLength(1)
expect(response.image.mediaType).toMatch(/^image\//)
expect((yield* response.image.bytes()).length).toBeGreaterThan(0)
expect(response.image?.mediaType).toMatch(/^image\//)
expect(response.image?.data).toBeInstanceOf(Uint8Array)
expect(response.image?.data.length).toBeGreaterThan(0)
}),
)
@@ -38,16 +39,18 @@ describe("Google Images recorded", () => {
prompt:
"Transform this minimal source into a bright orange sun icon with eight rounded rays on a pale blue background.",
images: [
Media.bytes(
ImageInput.bytes(
yield* Effect.promise(() => Bun.file("test/fixtures/images/edit-source.jpg").bytes()),
"image/jpeg",
),
],
aspectRatio: "1:1",
options: { aspectRatio: "1:1" },
})
expect(response.image.mediaType).toBe("image/jpeg")
expect(dimensions(yield* response.image.bytes())).toEqual({ width: 1024, height: 1024 })
expect(response.image?.mediaType).toBe("image/jpeg")
expect(response.image?.data).toBeInstanceOf(Uint8Array)
if (!(response.image?.data instanceof Uint8Array)) throw new Error("Expected owned Google image bytes")
expect(dimensions(response.image.data)).toEqual({ width: 1024, height: 1024 })
}),
)
})
@@ -114,7 +114,7 @@ describe("Google Vertex providers", () => {
}),
)
it.effect("preserves function call ids in lowered Vertex bodies", () =>
it.effect("strips function call ids Vertex does not accept from lowered bodies", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
@@ -142,14 +142,15 @@ describe("Google Vertex providers", () => {
}),
)
expect(JSON.stringify(prepared.body.contents)).not.toContain('"id"')
expect(prepared.body.contents).toMatchObject([
{ role: "model", parts: [{ functionCall: { id: "call_1", name: "lookup", args: { query: "weather" } } }] },
{ role: "model", parts: [{ functionCall: { id: undefined, name: "lookup", args: { query: "weather" } } }] },
{
role: "user",
parts: [
{
functionResponse: {
id: "call_1",
id: undefined,
name: "lookup",
response: { name: "lookup", content: "sunny" },
},
@@ -234,23 +235,12 @@ describe("Google Vertex providers", () => {
parts: [
{ text: "Thinking.", thought: true, thoughtSignature: "reasoning_sig" },
{ text: "Checking.", thoughtSignature: "text_sig" },
{
functionCall: { id: "provider_call_1", name: "lookup", args: { query: "weather" } },
thoughtSignature: "tool_sig",
},
{ functionCall: { name: "lookup", args: { query: "weather" } }, thoughtSignature: "tool_sig" },
],
},
{
role: "user",
parts: [
{
functionResponse: {
id: "provider_call_1",
name: "lookup",
response: { name: "lookup", content: "sunny" },
},
},
],
parts: [{ functionResponse: { name: "lookup", response: { name: "lookup", content: "sunny" } } }],
},
])
}),
@@ -1,6 +1,6 @@
import { expect } from "bun:test"
import { Effect, Schema } from "effect"
import { LLM, Media, Message } from "../../src/index.js"
import { LLM, Message } from "../../src/index.js"
import { OpenAI, Azure, XAI } from "../../src/providers.js"
import { compileRequest } from "../../src/route/client.js"
import { it } from "../lib/effect.js"
@@ -16,7 +16,8 @@ for (const model of [
const message = Message.user(
details.map((detail) => ({
type: "media",
media: Media.url("https://example.com/image.png", { mediaType: "image/png" }),
mediaType: "image/png",
data: "https://example.com/image.png",
providerMetadata:
detail === undefined ? undefined : { [model.route.providerMetadataKey ?? model.provider]: { detail } },
})),
@@ -47,7 +48,8 @@ it.effect("rejects malformed image detail instead of silently discarding it", ()
messages: [
Message.user({
type: "media",
media: Media.url("https://example.com/image.png", { mediaType: "image/png" }),
mediaType: "image/png",
data: "https://example.com/image.png",
providerMetadata: { openai: { detail: 42 } },
}),
],
@@ -1,7 +1,7 @@
import { expect } from "bun:test"
import { Effect, Layer } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { Image, ImageClient, LLM, LLMEvent, LLMRequest, Media, Message, ToolDefinition } from "../../src/index.js"
import { Image, ImageClient, ImageInput, LLM, LLMEvent, LLMRequest, Message, ToolDefinition } from "../../src/index.js"
import { Meta } from "../../src/providers/meta.js"
import { MetaMessages } from "../../src/protocols/meta-messages.js"
import { AnthropicMessages } from "../../src/protocols/anthropic-messages.js"
@@ -73,9 +73,9 @@ it.effect("Meta Images preserves request overlays, bearer auth, JSON edit inputs
headers: { "x-client": "test" },
}).image("muse-image-1.0"),
prompt: "Edit",
images: [Media.bytes(Uint8Array.from([1, 2, 3]), "image/png")],
format: "webp",
providerOptions: {
images: [ImageInput.bytes(Uint8Array.from([1, 2, 3]), "image/png")],
options: {
outputFormat: "webp",
responseFormat: "url",
reasoningStrength: "low",
toolEnablement: { enable_web_search: false },
@@ -83,12 +83,8 @@ it.effect("Meta Images preserves request overlays, bearer auth, JSON edit inputs
},
http: { body: { output_format: "jpeg", future_option: true }, query: { trace: "1" } },
})
expect(response.image.mediaType).toBe("image/jpeg")
expect(response.image.source).toEqual({
type: "url",
url: "https://images.example/result.jpg",
mediaType: "image/jpeg",
})
expect(response.image?.mediaType).toBe("image/jpeg")
expect(response.image?.data).toBe("https://images.example/result.jpg")
}).pipe(
Effect.provide(
ImageClient.layer.pipe(
@@ -125,7 +121,7 @@ it.effect("Meta Images validates the final output format before sending the requ
const error = yield* Image.generate({
model: Meta.configure({ apiKey: "fixture" }).image("muse-image-1.0"),
prompt: "Draw",
format: "png",
options: { outputFormat: "png" },
http: { body: { output_format: 42 } },
}).pipe(Effect.flip)
expect(error.reason._tag).toBe("InvalidRequest")
@@ -1,6 +1,6 @@
import { expect } from "bun:test"
import { Effect } from "effect"
import { Image, LLM, LLMEvent, LLMRequest, Media, Message } from "../../src/index.js"
import { Image, ImageInput, LLM, LLMEvent, LLMRequest, Message } from "../../src/index.js"
import { Meta } from "../../src/providers/meta.js"
import { LLMClient } from "../../src/route.js"
import { compileRequest } from "../../src/route/client.js"
@@ -28,16 +28,15 @@ recorded.effect.with(
const response = yield* Image.generate({
model: meta.image(modelID),
prompt: "A flat black square centered on a plain white background. No text.",
n: 1,
size: "256x256",
providerOptions: controls,
options: { ...controls, n: 1, size: "256x256" },
})
expect(response.images).toHaveLength(1)
expect(response.image.mediaType).toBe("image/webp")
const bytes = yield* response.image.bytes()
expect(new TextDecoder().decode(bytes.slice(0, 4))).toBe("RIFF")
expect(new TextDecoder().decode(bytes.slice(8, 12))).toBe("WEBP")
expect(response.usage?.type === "tokens" ? response.usage.output : undefined).toBeGreaterThan(0)
expect(response.image?.mediaType).toBe("image/webp")
expect(response.image?.data).toBeInstanceOf(Uint8Array)
if (!(response.image?.data instanceof Uint8Array)) throw new Error("Expected image bytes")
expect(new TextDecoder().decode(response.image.data.slice(0, 4))).toBe("RIFF")
expect(new TextDecoder().decode(response.image.data.slice(8, 12))).toBe("WEBP")
expect(response.usage?.outputTokens).toBeGreaterThan(0)
}),
180_000,
)
@@ -51,18 +50,16 @@ recorded.effect.with(
model: meta.image(modelID),
prompt: "Change the shape to bright purple. Keep the plain white background.",
images: [
Media.bytes(
ImageInput.bytes(
yield* Effect.promise(() => Bun.file("test/fixtures/images/edit-source.jpg").bytes()),
"image/jpeg",
),
],
n: 1,
size: "256x256",
format: "png",
providerOptions: controls,
options: { ...controls, n: 1, outputFormat: "png", size: "256x256" },
})
expect(response.image.mediaType).toBe("image/png")
expect(Array.from((yield* response.image.bytes()).slice(0, 8))).toEqual([137, 80, 78, 71, 13, 10, 26, 10])
expect(response.image?.mediaType).toBe("image/png")
if (!(response.image?.data instanceof Uint8Array)) throw new Error("Expected image bytes")
expect(Array.from(response.image.data.slice(0, 8))).toEqual([137, 80, 78, 71, 13, 10, 26, 10])
}),
180_000,
)
@@ -1,7 +1,7 @@
import { describe, expect, test } from "bun:test"
import { ConfigProvider, Effect } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { LLM, LLMEvent, Message, ToolDefinition, Media } from "../../src/index.js"
import { LLM, LLMEvent, Message, ToolDefinition } from "../../src/index.js"
import { Mistral } from "../../src/providers/index.js"
import { MistralChat } from "../../src/protocols/index.js"
import { LLMClient } from "../../src/route.js"
@@ -46,8 +46,8 @@ describe("Mistral Chat", () => {
Message.system("Updated"),
Message.user([
{ type: "text", text: "Inspect" },
{ type: "media", media: Media.base64("aW1hZ2U=", "image/png") },
{ type: "media", media: Media.base64("cGRm", "application/pdf") },
{ type: "media", mediaType: "image/png", data: "aW1hZ2U=" },
{ type: "media", mediaType: "application/pdf", data: "cGRm" },
]),
Message.assistant([
{ type: "reasoning", text: "Think" },
@@ -230,7 +230,8 @@ describe("Mistral Chat", () => {
messages: [
Message.user({
type: "media",
media: Media.url("https://assets.example.test/input.png", { mediaType: "image/png" }),
mediaType: "image/png",
data: "https://assets.example.test/input.png",
}),
Message.tool({
id: "Ab12Cd34E",
@@ -1,6 +1,6 @@
import { expect } from "bun:test"
import { Effect } from "effect"
import { LLM, LLMEvent, Message, ToolDefinition, Media } from "../../src/index.js"
import { LLM, LLMEvent, Message, ToolDefinition } from "../../src/index.js"
import { Moonshot } from "../../src/providers.js"
import { LLMClient } from "../../src/route.js"
import { recordedTests } from "../recorded-test.js"
@@ -85,7 +85,7 @@ for (const api of ["chat", "messages", "responses"] as const) {
messages: [
Message.user([
{ type: "text", text: "Read the three words in this image. Reply only with those words in order." },
{ type: "media", media: Media.bytes(image, "image/png") },
{ type: "media", mediaType: "image/png", data: image },
]),
],
generation: { maxTokens: 4096 },
@@ -2,7 +2,6 @@ import { describe, expect } from "bun:test"
import { Effect, Ref, Schema, Stream } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import {
Media,
HttpOptions,
LLM,
AIError,
@@ -687,7 +686,7 @@ describe("OpenAI Chat route", () => {
LLM.request({
model,
messages: [
Message.user({ type: "media", media: Media.base64("AAEC", "image/png") }),
Message.user({ type: "media", mediaType: "image/png", data: "AAEC" }),
Message.system("Keep the image."),
],
}),
@@ -711,9 +710,9 @@ describe("OpenAI Chat route", () => {
model,
messages: [
Message.user([
{ type: "media", media: Media.base64("not-base64", "image/png") },
{ type: "media", media: Media.fromDataUrl("data:image/jpeg;base64,/9j/") },
{ type: "media", media: Media.base64("PHN2Zz4=", "image/svg+xml") },
{ type: "media", mediaType: "image/png", data: "not-base64" },
{ type: "media", mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" },
{ type: "media", mediaType: "image/svg+xml", data: "PHN2Zz4=" },
]),
],
}),
@@ -737,7 +736,7 @@ describe("OpenAI Chat route", () => {
const prepared = yield* compileRequest(
LLM.request({
model,
prompt: urls.map((url) => Message.media(Media.url(url, { mediaType: "image/png" }))),
prompt: urls.map((data) => ({ type: "media" as const, mediaType: "image/png", data })),
}),
)
expect(prepared.body.messages).toEqual([
@@ -784,7 +783,7 @@ describe("OpenAI Chat route", () => {
const error = yield* compileRequest(
LLM.request({
model,
messages: [Message.user({ type: "media", media: Media.base64("AAECAw==", "audio/mpeg") })],
messages: [Message.user({ type: "media", mediaType: "audio/mpeg", data: "AAECAw==" })],
}),
).pipe(Effect.flip)
expect(error.message).toContain("OpenAI Chat does not support media type audio/mpeg")
@@ -799,8 +798,8 @@ describe("OpenAI Chat route", () => {
model,
messages: [
Message.user([
{ type: "media", media: Media.base64("AAECAw==", "image/png") },
{ type: "media", media: Media.fromDataUrl("data:image/jpeg;base64,/9j/") },
{ type: "media", mediaType: "image/png", data: "AAECAw==" },
{ type: "media", mediaType: "image/jpeg", data: "data:image/jpeg;base64,/9j/" },
]),
],
}),
@@ -1,6 +1,6 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM, LLMEvent, Message, ToolDefinition, Media } from "../../src/index.js"
import { LLM, LLMEvent, Message, ToolDefinition } from "../../src/index.js"
import { configure } from "../../src/providers/openai-compatible-responses.js"
import { OpenAI } from "../../src/providers.js"
import { OpenResponses } from "../../src/protocols/open-responses.js"
@@ -129,7 +129,7 @@ describe("Open Responses-compatible route", () => {
LLM.request({
model,
messages: [
Message.user([{ type: "media", media: Media.fromDataUrl(pdf), filename: "input.pdf" }]),
Message.user([{ type: "media", mediaType: "application/pdf", data: pdf, filename: "input.pdf" }]),
Message.assistant({ type: "tool-call", id: "call_1", name: "read", input: {} }),
Message.tool({
id: "call_1",
@@ -1,6 +1,6 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Image, Media } from "../../src/index.js"
import { Image, ImageInput } from "../../src/index.js"
import { OpenAI } from "../../src/providers.js"
import { dimensions } from "../lib/image.js"
import { recordedTests } from "../recorded-test.js"
@@ -22,14 +22,13 @@ describe("OpenAI Images recorded", () => {
const response = yield* Image.generate({
model,
prompt: "A simple flat black circle centered on a plain white background.",
size: "1024x1024",
format: "jpeg",
providerOptions: { quality: "low", outputCompression: 10 },
options: { quality: "low", outputFormat: "jpeg", outputCompression: 10, size: "1024x1024" },
})
expect(response.images).toHaveLength(1)
expect(response.image.mediaType).toBe("image/jpeg")
expect((yield* response.image.bytes()).length).toBeGreaterThan(0)
expect(response.image?.mediaType).toBe("image/jpeg")
expect(response.image?.data).toBeInstanceOf(Uint8Array)
expect(response.image?.data.length).toBeGreaterThan(0)
}),
)
@@ -46,18 +45,18 @@ describe("OpenAI Images recorded", () => {
model,
prompt: "Keep the simple shape and change it from black to bright green.",
images: [
Media.bytes(
ImageInput.bytes(
yield* Effect.promise(() => Bun.file("test/fixtures/images/edit-source.jpg").bytes()),
"image/jpeg",
),
],
size: "1024x1024",
format: "jpeg",
providerOptions: { quality: "low", outputCompression: 10 },
options: { quality: "low", outputFormat: "jpeg", outputCompression: 10, size: "1024x1024" },
})
expect(response.image.mediaType).toBe("image/jpeg")
expect(dimensions(yield* response.image.bytes())).toEqual({ width: 1024, height: 1024 })
expect(response.image?.mediaType).toBe("image/jpeg")
expect(response.image?.data).toBeInstanceOf(Uint8Array)
if (!(response.image?.data instanceof Uint8Array)) throw new Error("Expected owned OpenAI image bytes")
expect(dimensions(response.image.data)).toEqual({ width: 1024, height: 1024 })
}),
)
})
@@ -2,7 +2,6 @@ import { describe, expect } from "bun:test"
import { ConfigProvider, Effect, Layer, Logger, Ref, Schema, Stream } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import {
Media,
LLM,
AIError,
HttpContext,
@@ -4617,30 +4616,6 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("replays assistant media parts as portable user image input", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model,
messages: [
Message.user("Draw a cat"),
Message.assistant([
{ type: "text", text: "Here it is." },
{ type: "media", media: Media.base64("AQID", "image/png") },
]),
Message.user("Make it orange"),
],
}),
)
expect(prepared.body.input).toMatchObject([
{ type: "message", role: "user" },
{ type: "message", role: "assistant", content: [{ type: "output_text", text: "Here it is." }] },
{ type: "message", role: "user", content: [{ type: "input_image", image_url: "data:image/png;base64,AQID" }] },
{ type: "message", role: "user" },
])
}),
)
it.effect("replays hosted image results as portable content regardless of storage", () =>
Effect.gen(function* () {
const item = {
@@ -4802,8 +4777,8 @@ describe("OpenAI Responses route", () => {
model,
messages: [
Message.user([
{ type: "media", media: Media.base64("AAECAw==", "image/png") },
{ type: "media", media: Media.base64("JVBERi0xLjQ=", "application/pdf"), filename: "report.pdf" },
{ type: "media", mediaType: "image/png", data: "AAECAw==" },
{ type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=", filename: "report.pdf" },
]),
],
}),
@@ -4834,7 +4809,8 @@ describe("OpenAI Responses route", () => {
messages: [
Message.user({
type: "media",
media: Media.fromDataUrl("data:application/pdf;base64,JVBERi0xLjQ="),
mediaType: "application/pdf",
data: "data:application/pdf;base64,JVBERi0xLjQ=",
filename: "report.pdf",
}),
],
@@ -4863,7 +4839,7 @@ describe("OpenAI Responses route", () => {
LLM.request({
id: "req_media",
model,
messages: [Message.user({ type: "media", media: Media.base64("AAECAw==", "application/x-tar") })],
messages: [Message.user({ type: "media", mediaType: "application/x-tar", data: "AAECAw==" })],
}),
)
@@ -4890,10 +4866,11 @@ describe("OpenAI Responses route", () => {
model,
messages: [
Message.user([
{ type: "media", media: Media.url("https://example.com/image.png", { mediaType: "image/png" }) },
{ type: "media", mediaType: "image/png", data: "https://example.com/image.png" },
{
type: "media",
media: Media.url("https://example.com/report.pdf", { mediaType: "application/pdf" }),
mediaType: "application/pdf",
data: "https://example.com/report.pdf",
filename: "report.pdf",
},
]),
@@ -1,6 +1,6 @@
import { describe, expect } from "bun:test"
import { Effect, Schema, Stream } from "effect"
import { LLM, LLMResponse, Message, ToolDefinition, type LanguageModel, Media } from "../../src/index.js"
import { LLM, LLMResponse, Message, ToolDefinition, type LanguageModel } from "../../src/index.js"
import { AmazonBedrock, Anthropic, Google, OpenAI, XAI } from "../../src/providers.js"
import { LLMClient } from "../../src/route.js"
import { Tool } from "../../src/tool.js"
@@ -131,7 +131,7 @@ describe("PDF recorded", () => {
generation: { maxTokens: target.maxTokens, temperature: 0 },
messages: [
Message.user([
{ type: "media", media: Media.base64(PDF, "application/pdf"), filename: target.filename },
{ type: "media", mediaType: "application/pdf", data: PDF, filename: target.filename },
{ type: "text", text: prompt },
]),
],
@@ -1,6 +1,6 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Image, Media } from "../../src/index.js"
import { Image, ImageInput } from "../../src/index.js"
import { XAI } from "../../src/providers.js"
import { dimensions } from "../lib/image.js"
import { recordedTests } from "../recorded-test.js"
@@ -22,13 +22,13 @@ describe("xAI Images recorded", () => {
const response = yield* Image.generate({
model,
prompt: "A simple flat black diamond centered on a plain white background.",
aspectRatio: "1:1",
providerOptions: { resolution: "1k", responseFormat: "b64_json" },
options: { aspectRatio: "1:1", resolution: "1k", responseFormat: "b64_json" },
})
expect(response.images).toHaveLength(1)
expect(response.image.mediaType.startsWith("image/")).toBe(true)
expect((yield* response.image.bytes()).length).toBeGreaterThan(0)
expect(response.image?.mediaType.startsWith("image/")).toBe(true)
expect(response.image?.data).toBeInstanceOf(Uint8Array)
expect(response.image?.data.length).toBeGreaterThan(0)
}),
)
@@ -38,17 +38,18 @@ describe("xAI Images recorded", () => {
model,
prompt: "Keep the simple shape and change it from black to bright purple.",
images: [
Media.bytes(
ImageInput.bytes(
yield* Effect.promise(() => Bun.file("test/fixtures/images/edit-source.jpg").bytes()),
"image/jpeg",
),
],
aspectRatio: "1:1",
providerOptions: { resolution: "1k", responseFormat: "b64_json" },
options: { aspectRatio: "1:1", resolution: "1k", responseFormat: "b64_json" },
})
expect(response.image.mediaType).toMatch(/^image\/(jpeg|png)$/)
expect(dimensions(yield* response.image.bytes())).toEqual({ width: 1024, height: 1024 })
expect(response.image?.mediaType).toMatch(/^image\/(jpeg|png)$/)
expect(response.image?.data).toBeInstanceOf(Uint8Array)
if (!(response.image?.data instanceof Uint8Array)) throw new Error("Expected owned xAI image bytes")
expect(dimensions(response.image.data)).toEqual({ width: 1024, height: 1024 })
}),
)
})
+9 -9
View File
@@ -17,9 +17,9 @@ describe("xAI Images", () => {
http: { body: { configured: true }, headers: { "x-default": "yes" } },
}).image("grok-imagine-image"),
prompt: "A robot tending a rooftop garden",
n: 2,
aspectRatio: "16:9",
providerOptions: {
options: {
n: 2,
aspectRatio: "16:9",
aspect_ratio: "4:3",
resolution: "1k",
responseFormat: "url",
@@ -34,16 +34,16 @@ describe("xAI Images", () => {
})
expect(response.images).toHaveLength(2)
expect(response.image.mediaType).toBe("image/jpeg")
expect(yield* response.image.bytes()).toEqual(Uint8Array.from([1, 2, 3]))
expect(response.images[1].mediaType).toBe("application/octet-stream")
expect(response.images[1].source).toEqual({ type: "url", url: "https://api.xai.test/image.jpg" })
expect(response.usage).toBeUndefined()
expect(response.image?.mediaType).toBe("image/jpeg")
expect(response.image?.data).toEqual(Uint8Array.from([1, 2, 3]))
expect(response.images[1]?.mediaType).toBe("application/octet-stream")
expect(response.images[1]?.data).toBe("https://api.xai.test/image.jpg")
expect(response.usage?.providerMetadata).toEqual({ xai: { num_images: 2 } })
expect(response.providerMetadata).toEqual({ xai: { usage: { num_images: 2 } } })
}).pipe(
Effect.provide(
ImageClient.layer.pipe(
Layer.provideMerge(
Layer.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
@@ -1,6 +1,6 @@
import { expect } from "bun:test"
import { Effect } from "effect"
import { LLM, LLMEvent, Message, ToolDefinition, Media } from "../../src/index.js"
import { LLM, LLMEvent, Message, ToolDefinition } from "../../src/index.js"
import { ZAI } from "../../src/providers.js"
import { LLMClient } from "../../src/route.js"
import { recordedTests } from "../recorded-test.js"
@@ -92,7 +92,7 @@ recorded.effect.with(
messages: [
Message.user([
{ type: "text", text: "Read the three words in this image. Reply with only the words in order." },
{ type: "media", media: Media.bytes(bytes, "image/png") },
{ type: "media", mediaType: "image/png", data: bytes },
]),
],
generation: { maxTokens: 4096 },
@@ -19,14 +19,13 @@ describe("Z.ai Images recorded", () => {
const response = yield* Image.generate({
model,
prompt: "A simple flat red circle centered on a plain white background.",
size: "1024x1024",
providerOptions: { quality: "standard", userID: "opencode-image-test" },
options: { size: "1024x1024", quality: "standard", userID: "opencode-image-test" },
})
expect(response.images).toHaveLength(1)
expect(response.image.mediaType).toBe("application/octet-stream")
expect(response.image.source.type).toBe("url")
expect(response.image.source.type === "url" && response.image.source.url).toStartWith("https://")
expect(response.image?.mediaType).toBe("application/octet-stream")
expect(response.image?.data).toBeString()
expect(response.image?.data).toStartWith("https://")
expect(response.providerMetadata?.zai).toBeDefined()
}),
)
+4 -11
View File
@@ -17,7 +17,7 @@ describe("Z.ai Images", () => {
http: { body: { configured: true, quality: "configured" }, query: { trace: "default" } },
}).image("glm-image"),
prompt: "A red circle on a white background",
providerOptions: {
options: {
quality: "hd",
userID: "alias-user",
user_id: "raw-user",
@@ -31,15 +31,8 @@ describe("Z.ai Images", () => {
})
expect(response.images).toHaveLength(1)
expect(response.image.mediaType).toBe("application/octet-stream")
expect(response.image.source).toEqual({ type: "url", url: "https://cdn.z.ai/generated.png" })
expect(response.notices).toEqual([
{
type: "moderated",
message: "Z.ai Images applied a content filter for future-role at level 4.5",
providerMetadata: { zai: { role: "future-role", level: 4.5 } },
},
])
expect(response.image?.mediaType).toBe("application/octet-stream")
expect(response.image?.data).toBe("https://cdn.z.ai/generated.png")
expect(response.providerMetadata).toEqual({
zai: {
created: 1_760_335_349,
@@ -116,7 +109,7 @@ describe("Z.ai Images", () => {
Image.generate({
model: ZAI.configure({ apiKey: "test" }).image("model"),
prompt: "test",
providerOptions: { quality: "future-quality", userID: "x", user_id: "raw-user" },
options: { quality: "future-quality", userID: "x", user_id: "raw-user" },
}).pipe(
Effect.provide(
ImageClient.layer.pipe(
+1 -2
View File
@@ -1,7 +1,6 @@
import { expect } from "bun:test"
import { Effect, Schema } from "effect"
import {
Media,
LLM,
LLMEvent,
LLMRequest,
@@ -331,7 +330,7 @@ const runImageScenario = (context: GoldenScenarioContext) =>
type: "text",
text: "The image contains exactly three lowercase English words. Read them left to right and reply with only those words.",
},
{ type: "media", media: Media.base64(yield* restroomImage(), "image/png") },
{ type: "media", mediaType: "image/png", data: yield* restroomImage() },
]),
assistant.expectText(/.+/, {
system: "Read images carefully. Reply only with the visible text.",
+1 -9
View File
@@ -6,8 +6,6 @@ import * as path from "node:path"
import { fileURLToPath } from "node:url"
import { LLMClient, RequestExecutor } from "../src/route.js"
import { ImageClient } from "../src/image-client.js"
import { EvaluationClient } from "../src/experimental/evaluation-client.js"
import type { Service as EvaluationClientService } from "../src/experimental/evaluation-client.js"
import type { Service as ImageClientService } from "../src/image-client.js"
import type { Service as LLMClientService } from "../src/route/client.js"
import type { Service as RequestExecutorService } from "../src/route/executor.js"
@@ -20,12 +18,7 @@ import {
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const FIXTURES_DIR = path.resolve(__dirname, "fixtures", "recordings")
type RecordedEnv =
| RequestExecutorService
| LLMClientService
| ImageClientService
| EvaluationClientService
| Socket.WebSocketConstructor
type RecordedEnv = RequestExecutorService | LLMClientService | ImageClientService | Socket.WebSocketConstructor
type RecordedTestsOptions = RecordedGroupOptions & {
readonly options?: HttpRecorder.RecorderOptions
@@ -99,7 +92,6 @@ export const recordedTests = (options: RecordedTestsOptions) =>
requestExecutor,
LLMClient.layer.pipe(Layer.provide(requestExecutor)),
ImageClient.layer.pipe(Layer.provide(requestExecutor)),
EvaluationClient.layer.pipe(Layer.provide(requestExecutor)),
webSocket,
)
},
+2 -2
View File
@@ -95,7 +95,7 @@ In Vite development mode, `origin` uses `VITE_OPENCODE_SERVER_HOST` / `VITE_OPEN
from storage. Desktop provides the local server it discovers or starts through native initialization.
With no configured servers, the app shows a full-screen connection form. Enter a server address and password,
or choose **Scan QR code** to open the camera and read the pairing code from `opencode pair`.
or choose **Scan QR code** to open the camera and read the JSON pairing code from `opencode pair`.
Scanning fills the form and immediately attempts to connect. Failed connections leave the details available
to edit and retry with **Connect**. Credentials are checked before saving the server. Camera access requires
HTTPS (or localhost) and browser permission. Saved offline servers continue to use the normal app UI.
@@ -103,7 +103,7 @@ HTTPS (or localhost) and browser permission. Saved offline servers continue to u
When the service is exposed through an HTTPS reverse proxy, advertise its external address at runtime:
```bash
opencode pair --url https://opencode.example.com
opencode pair --url https://your-machine.your-tailnet.ts.net
```
This replaces the addresses printed and encoded in the QR code while retaining the local service password.
@@ -122,6 +122,35 @@ story("does not mask or pad controls when they fit", async ({ mount }) => {
await expect(controls).toHaveCSS("padding-inline-end", "0px")
})
story("grows suggestions while preserving visible timeline context", async ({ mount }) => {
const component = await mount("opencode-composer-flow--constrained-command-suggestions")
const boundary = component.locator('[data-slot="composer-suggestion-boundary-story"]')
const suggestions = component.locator('[data-component="composer-suggestions"]')
await expect(suggestions).toHaveCSS("max-height", "166px")
await expect(suggestions).toHaveCSS("scroll-padding-bottom", "18px")
await expect.poll(() => suggestions.evaluate((element) => element.scrollHeight > element.clientHeight)).toBe(true)
await expect
.poll(async () => {
const menu = await suggestions.boundingBox()
const items = await suggestions.locator("[data-suggestion-id]").evaluateAll((elements) =>
elements.map((element) => {
const rect = element.getBoundingClientRect()
return { top: rect.top, bottom: rect.bottom }
}),
)
if (!menu) return false
const bottom = menu.y + menu.height
return items.some((item) => item.top < bottom && item.bottom > bottom)
})
.toBe(true)
await boundary.evaluate((element) => {
element.style.height = "400px"
})
await expect(suggestions).toHaveCSS("max-height", "306px")
})
// ThemeProvider writes resolved token values into a <style> block, so toggling data-color-scheme by hand
// leaves every --v2-* variable at its previous value. Switch themes through the Storybook global instead.
for (const [theme, background] of [
@@ -1,145 +0,0 @@
import { base64Encode } from "@opencode/util/encode"
import { expect, test } from "@playwright/test"
import { mockOpenCodeServer } from "../utils/mock-server"
import { expectSessionTitle } from "../utils/waits"
const directory = "C:/OpenCode/BtwSidebar"
const projectID = "proj_btw_sidebar"
const sessionID = "ses_btw_sidebar"
const otherSessionID = "ses_btw_sidebar_other"
const title = "Side question session"
const otherTitle = "Other side question session"
const server = `http://${process.env.PLAYWRIGHT_SERVER_HOST ?? "127.0.0.1"}:${process.env.PLAYWRIGHT_SERVER_PORT ?? "4096"}`
const sessionHref = (id: string) => `/server/${base64Encode(server)}/session/${id}`
test.use({ viewport: { width: 1440, height: 900 } })
test("answers /btw in the side panel without admitting a prompt", async ({ page }) => {
const generations: { sessionID: string; prompt: string }[] = []
const prompts: unknown[] = []
const generated = Promise.withResolvers<void>()
await mockOpenCodeServer(page, {
directory,
project: {
id: projectID,
worktree: directory,
vcs: "git",
name: "btw-sidebar",
time: { created: 1700000000000, updated: 1700000000000 },
sandboxes: [],
},
provider: {
all: [
{
id: "opencode",
name: "OpenCode",
models: { test: { id: "test", name: "Test", limit: { context: 200_000 } } },
},
],
connected: ["opencode"],
default: { providerID: "opencode", modelID: "test" },
},
sessions: [
{
id: sessionID,
slug: sessionID,
projectID,
directory,
title,
version: "dev",
time: { created: 1700000000000, updated: 1700000000000 },
},
{
id: otherSessionID,
slug: otherSessionID,
projectID,
directory,
title: otherTitle,
version: "dev",
time: { created: 1700000001000, updated: 1700000001000 },
},
],
pageMessages: () => ({ items: [] }),
vcsDiff: [],
onPrompt: (input) => prompts.push(input),
generate: async (input) => {
generations.push(input)
if (input.sessionID === otherSessionID) return { text: "This answer belongs to the **other session**." }
await generated.promise
return {
text: "The retry loop uses **exponential backoff** and stops after three attempts.\n\n```ts\nconst delay = 2 ** attempt\n```",
}
},
})
await page.addInitScript(
({ directory, server, sessionID, otherSessionID }) => {
localStorage.setItem(
"opencode.global.dat:server",
JSON.stringify({
projects: { local: [{ worktree: directory, expanded: true }] },
lastProject: { local: directory },
}),
)
localStorage.setItem(
"opencode.window.browser.dat:tabs",
JSON.stringify([
{ type: "session", server, sessionId: sessionID },
{ type: "session", server, sessionId: otherSessionID },
]),
)
},
{ directory, server, sessionID, otherSessionID },
)
await page.goto(sessionHref(sessionID))
await expectSessionTitle(page, title)
const editor = page.locator('[data-component="composer-editor"]')
await expect(editor).toBeEditable()
await editor.fill("/btw")
const suggestion = page.locator('[data-suggestion-id="session.btw"]')
await expect(suggestion).toBeVisible()
await suggestion.click()
await expect(editor).toHaveText("/btw ")
await editor.press("Enter")
const panel = page.locator('[data-slot="session-btw-panel"]')
await expect(panel).toBeHidden()
await expect(page.getByText("Add a question after /btw", { exact: true })).toBeVisible()
expect(generations).toEqual([])
expect(prompts).toEqual([])
await editor.fill("/btw how does the retry loop work?")
await editor.press("Enter")
const tab = page.getByRole("tab", { name: "/btw" })
await expect(panel).toBeVisible()
await expect(panel.getByRole("textbox")).toHaveCount(0)
await expect(panel.getByRole("status")).toContainText("Working")
await expect(tab).toHaveAttribute("data-selected", "")
generated.resolve()
await expect(panel.getByText("how does the retry loop work?", { exact: true })).toBeVisible()
await expect(panel.getByText("exponential backoff", { exact: false })).toBeVisible()
await expect(panel.getByText("const delay = 2 ** attempt", { exact: true })).toBeVisible()
expect(generations).toHaveLength(1)
expect(generations[0]?.sessionID).toBe(sessionID)
expect(generations[0]?.prompt).toContain("how does the retry loop work?")
expect(prompts).toEqual([])
await expect(editor).toHaveText("")
await page.locator(`[data-titlebar-tab-link][href="${sessionHref(otherSessionID)}"]`).click()
await expectSessionTitle(page, otherTitle)
await editor.fill("/btw what belongs here?")
await editor.press("Enter")
await expect(panel.getByText("other session", { exact: false })).toBeVisible()
await page.locator(`[data-titlebar-tab-link][href="${sessionHref(sessionID)}"]`).click()
await expectSessionTitle(page, title)
await expect(panel.getByText("exponential backoff", { exact: false })).toBeVisible()
await expect(panel.getByText("other session", { exact: false })).toHaveCount(0)
await page.reload()
await expectSessionTitle(page, title)
await expect(page.getByRole("tab", { name: "/btw" })).toHaveCount(0)
await expect(page.locator('[data-slot="session-btw-panel"]')).toHaveCount(0)
})
@@ -11,6 +11,7 @@ import {
textPart,
userMessage,
} from "../performance/timeline-stability/fixture"
import { reportVisualStability, startVisualProbe, stopVisualProbe, visualPlan } from "../utils/visual-stability"
// Compositor prediction can add 2025px to discrete CDP moves even in a plain
// scrollport. Disable it so the visual assertion measures the supplied gesture.
@@ -26,6 +27,91 @@ for (const device of ["Pixel 7", "iPhone 13"]) {
viewport: { width: 390, height: 844 },
})
for (const direction of ["ltr", "rtl"]) {
test(`session text follows each 50px finger movement (${direction})`, async ({ page }, testInfo) => {
const messages = Array.from({ length: 40 }, (_, index) => {
const id = `msg_${String(index).padStart(4, "0")}_mobile`
return [
userMessage(undefined, { id: `${id}_user`, created: 1690000000000 + index * 10_000 }),
assistantMessage(
[
textPart(
`prt_mobile_${index}`,
`Answer ${index}. ${"Mobile history content. ".repeat(10 + (index % 4) * 20)}`,
),
],
{ id: `${id}_assistant`, parentID: `${id}_user`, created: 1690000001000 + index * 10_000 },
),
]
}).flat()
await setupTimeline(page, { messages, viewport: { width: 390, height: 844 } })
await page.evaluate((direction) => (document.documentElement.dir = direction), direction)
const timeline = page.locator('[data-slot="session-timeline-scroll"]')
const scroller = timeline.getByRole("region", { name: "scrollable content", exact: true })
await page.evaluate(() => document.fonts.ready)
await expect(timeline.locator("[data-timeline-virtual-content]")).toBeVisible()
await expect(page.getByText("Answer 39.", { exact: false })).toBeInViewport()
await expect(timeline.locator('[data-component="markdown"]:not([data-markdown-ready])')).toHaveCount(0)
const devtools = await page.context().newCDPSession(page)
for (const [index, sign] of [1, 1, 1, -1, -1, -1].entries()) {
const bounds = await scroller.boundingBox()
expect(bounds).not.toBeNull()
if (!bounds) return
const partID = await scroller.evaluate((root, sign) => {
const view = root.getBoundingClientRect()
const line = sign > 0 ? view.top + 40 : view.bottom - 40
return [...root.querySelectorAll<HTMLElement>("[data-timeline-part-id]")]
.filter((part) => part.querySelector("p"))
.map((part) => ({ part, rect: part.getBoundingClientRect() }))
.filter(({ rect }) => rect.bottom > view.top && rect.top < view.bottom)
.sort((a, b) => Math.abs(a.rect.top - line) - Math.abs(b.rect.top - line))[0]?.part.dataset.timelinePartId
}, sign)
expect(partID).toBeTruthy()
const selector = `[data-timeline-part-id="${partID}"] p`
const anchor = scroller.locator(selector)
await expect(anchor).toHaveCount(1)
const regions = { text: { selector } }
await startVisualProbe(page, regions)
const x = bounds.x + bounds.width / 3
const y = bounds.y + (sign > 0 ? 60 : bounds.height - 60)
await devtools.send("Input.dispatchTouchEvent", { type: "touchStart", touchPoints: [{ x, y }] })
// Cross the browser's touch slop before measuring one-to-one dragging.
await devtools.send("Input.dispatchTouchEvent", {
type: "touchMove",
touchPoints: [{ x, y: y + sign * 30 }],
})
await expect(timeline.locator('[data-orientation="vertical"][data-visible="true"]')).toHaveCount(1)
await page.screenshot()
const origin = await anchor.boundingBox()
expect(origin).not.toBeNull()
if (!origin) return
for (let step = 1; step <= 8; step++) {
await devtools.send("Input.dispatchTouchEvent", {
type: "touchMove",
touchPoints: [{ x, y: y + sign * (30 + step * 50) }],
})
await expect.poll(async () => (await anchor.boundingBox())?.y).toBeCloseTo(origin.y + sign * step * 50, 0)
}
await devtools.send("Input.dispatchTouchEvent", { type: "touchEnd", touchPoints: [] })
// Include release corrections through the scroll indicator's idle state.
await expect(timeline.locator('[data-orientation="vertical"][data-visible="false"]')).toHaveCount(1)
await testInfo.attach(`swipe-${index}.png`, { body: await page.screenshot(), contentType: "image/png" })
// Native momentum may continue after release, including past this row's
// virtual window. It must not move the text against the gesture.
const finalTop = await anchor.evaluateAll((elements) => elements[0]?.getBoundingClientRect().top)
if (finalTop !== undefined) expect((finalTop - origin.y) * sign).toBeGreaterThanOrEqual(399.5)
const trace = await stopVisualProbe(page)
await reportVisualStability(
testInfo,
`swipe-${index}`,
trace,
visualPlan(regions, [{ type: "motion", regions: "all", maxPositionReversals: 0 }]),
)
}
})
}
test("reversing a touch drag stops following streamed output", async ({ page }, testInfo) => {
const partID = "prt_mobile_stream"
const content = Array.from({ length: 60 }, (_, index) => `Reading earlier output ${index}.\n\n`).join("")

Some files were not shown because too many files have changed in this diff Show More