mirror of
https://github.com/anomalyco/opencode.git
synced 2026-09-16 05:46:23 +00:00
Compare commits
37
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7f3a08a64f | ||
|
|
9bf5faa978 | ||
|
|
a062f4e803 | ||
|
|
fc49b16584 | ||
|
|
9b091ebd37 | ||
|
|
12201ee507 | ||
|
|
73bbeb7ce1 | ||
|
|
370b9d484a | ||
|
|
796c423df5 | ||
|
|
978bc53eba | ||
|
|
f7444aba27 | ||
|
|
c987590c5f | ||
|
|
1a3bd3458c | ||
|
|
67a6d52015 | ||
|
|
b1f6d21ef3 | ||
|
|
fa3721fad6 | ||
|
|
02b12aa60e | ||
|
|
1b6a07dda3 | ||
|
|
547d60183d | ||
|
|
12e53ee6d4 | ||
|
|
b629b458f7 | ||
|
|
24dff6929d | ||
|
|
7e2ca5fe26 | ||
|
|
aaed6eaa45 | ||
|
|
01443d5e00 | ||
|
|
70fe186df4 | ||
|
|
fc731d6013 | ||
|
|
4cd6093e31 | ||
|
|
b693c222da | ||
|
|
e31ad14c1d | ||
|
|
6c9115eb75 | ||
|
|
d9dfc5c1d6 | ||
|
|
c921067eb8 | ||
|
|
b76e56eca3 | ||
|
|
3c3ef00b32 | ||
|
|
d54fc305db | ||
|
|
a4aaf18855 |
+83
-83
@@ -29,15 +29,15 @@ Review endpoints in document order. For each endpoint, select one disposition an
|
||||
## Progress
|
||||
|
||||
- [x] Group 1: Foundation and placement (4)
|
||||
- [ ] Group 2: Configuration and capability catalogs (16)
|
||||
- [ ] Group 3: Credentials, integrations, MCP, and web search (22)
|
||||
- [ ] Group 4: Session lifecycle (12)
|
||||
- [ ] Group 5: Session execution and inputs (11)
|
||||
- [ ] Group 6: Session history and recovery (13)
|
||||
- [ ] Group 7: Inbox, permissions, and forms (19)
|
||||
- [ ] Group 8: Filesystem, worktrees, and VCS (12)
|
||||
- [ ] Group 9: PTYs, persistent terminals, and shells (24)
|
||||
- [ ] Group 10: Events, RPC, and experimental operations (6)
|
||||
- [x] Group 2: Configuration and capability catalogs (16)
|
||||
- [x] Group 3: Credentials, integrations, MCP, and web search (22)
|
||||
- [x] Group 4: Session lifecycle (12)
|
||||
- [x] Group 5: Session execution and inputs (11)
|
||||
- [x] Group 6: Session history and recovery (13)
|
||||
- [x] Group 7: Inbox, permissions, and forms (19)
|
||||
- [x] Group 8: Filesystem, worktrees, and VCS (12)
|
||||
- [x] Group 9: PTYs, persistent terminals, and shells (24)
|
||||
- [x] Group 10: Events, RPC, and experimental operations (6)
|
||||
|
||||
## Resolved during audit
|
||||
|
||||
@@ -100,7 +100,7 @@ Review endpoints in document order. For each endpoint, select one disposition an
|
||||
| [x] 021 | `GET` | `/api/config` | `config.get` | Keep | Compatibility entries removed; response now contains only documents and OpenCode directories. |
|
||||
| [x] 022 | `GET` | `/api/config/preferences` | `config.preferences` | Remove | Redundant special projection of global config. |
|
||||
| [x] 023 | `PATCH` | `/api/config/preferences` | `config.updatePreferences` | Remove | Redundant field-specific config mutation API. |
|
||||
| [ ] 024 | `GET` | `/api/config/shell` | `config.shells` | | |
|
||||
| [x] 024 | `GET` | `/api/config/shell` | `config.shells` | Keep | Required by the server Terminal shell setting. |
|
||||
| [x] 024a | `PATCH` | `/api/experimental/config` | `experimental.config.update` | Change | Experimental global config mutation; initially accepts only `shell`. |
|
||||
|
||||
## Group 3: Credentials, integrations, MCP, and web search
|
||||
@@ -111,21 +111,21 @@ Review endpoints in document order. For each endpoint, select one disposition an
|
||||
|---|---|---|---|---|---|
|
||||
| [x] 025 | `GET` | `/api/integration` | `integration.list` | Keep | Full integration inventory is consumed by authentication and integration-selection clients. |
|
||||
| [x] 026 | `GET` | `/api/integration/{integrationID}` | `integration.get` | Change | Missing integration now returns typed `404` instead of optional data. |
|
||||
| [x] | `POST` | `/api/experimental/integration/wellknown` | `experimental.integration.wellknown.add` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [ ] 028 | `POST` | `/api/integration/{integrationID}/connect/key` | `integration.connect.key` | | |
|
||||
| [ ] 029 | `POST` | `/api/integration/{integrationID}/connect/oauth` | `integration.oauth.connect` | | |
|
||||
| [ ] 030 | `GET` | `/api/integration/{integrationID}/connect/oauth/{attemptID}` | `integration.oauth.status` | | |
|
||||
| [ ] 031 | `DELETE` | `/api/integration/{integrationID}/connect/oauth/{attemptID}` | `integration.oauth.cancel` | | |
|
||||
| [ ] 032 | `POST` | `/api/integration/{integrationID}/connect/oauth/{attemptID}/complete` | `integration.oauth.complete` | | |
|
||||
| [ ] 033 | `POST` | `/api/integration/{integrationID}/connect/command` | `integration.command.connect` | | |
|
||||
| [ ] 034 | `GET` | `/api/integration/{integrationID}/connect/command/{attemptID}` | `integration.command.status` | | |
|
||||
| [ ] 035 | `DELETE` | `/api/integration/{integrationID}/connect/command/{attemptID}` | `integration.command.cancel` | | |
|
||||
| [x] 027 | `POST` | `/api/experimental/integration/wellknown` | `experimental.integration.wellknown.add` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 028 | `POST` | `/api/integration/{integrationID}/connect/key` | `integration.connect.key` | Change | Missing integration returns typed `404`; key form answers retained. |
|
||||
| [x] 029 | `POST` | `/api/integration/{integrationID}/connect/oauth` | `integration.oauth.connect` | Keep | OAuth connection start contract retained. |
|
||||
| [x] 030 | `GET` | `/api/integration/{integrationID}/connect/oauth/{attemptID}` | `integration.oauth.status` | Change | Missing integration or OAuth attempt returns typed `404`. |
|
||||
| [x] 031 | `DELETE` | `/api/integration/{integrationID}/connect/oauth/{attemptID}` | `integration.oauth.cancel` | Keep | Idempotent cancellation remains a no-op for unavailable or terminal attempts. |
|
||||
| [x] 032 | `POST` | `/api/integration/{integrationID}/connect/oauth/{attemptID}/complete` | `integration.oauth.complete` | Change | Missing integration or OAuth attempt returns typed `404`; code remains mode-dependent. |
|
||||
| [x] 033 | `POST` | `/api/integration/{integrationID}/connect/command` | `integration.command.connect` | Change | Missing integration or command method returns typed `404`. |
|
||||
| [x] 034 | `GET` | `/api/integration/{integrationID}/connect/command/{attemptID}` | `integration.command.status` | Change | Missing integration or command attempt returns typed `404`. |
|
||||
| [x] 035 | `DELETE` | `/api/integration/{integrationID}/connect/command/{attemptID}` | `integration.command.cancel` | Keep | Idempotent cancellation remains a no-op for unavailable or terminal attempts. |
|
||||
| [x] 036 | `GET` | `/api/mcp` | `mcp.list` | Keep | MCP inventory and connection status retained. |
|
||||
| [x] 037 | `PUT` | `/api/experimental/mcp/{server}` | `experimental.mcp.add` | Experimental-only | Runtime-only MCP override; does not persist configuration. |
|
||||
| [x] 038 | `DELETE` | `/api/experimental/mcp/{server}` | `experimental.mcp.remove` | Experimental-only | Runtime removal override; missing server returns `404`. |
|
||||
| [x] 039 | `POST` | `/api/experimental/mcp/{server}/connect` | `experimental.mcp.connect` | Experimental-only | Runtime connection override retained outside the stable API. |
|
||||
| [x] 040 | `POST` | `/api/experimental/mcp/{server}/disconnect` | `experimental.mcp.disconnect` | Experimental-only | Runtime disconnection override retained outside the stable API. |
|
||||
| [ ] 041 | `GET` | `/api/mcp/resource` | `mcp.resource.catalog` | | Deferred for later review. |
|
||||
| [x] 041 | `GET` | `/api/mcp/resource` | `mcp.resource.catalog` | Keep | Reviewed separately by coworker. |
|
||||
| [x] 042 | `PATCH` | `/api/credential/{credentialID}` | `credential.update` | Change | Removed redundant location query; credentials and events are global. |
|
||||
| [x] 043 | `DELETE` | `/api/credential/{credentialID}` | `credential.remove` | Change | Removed redundant location query; credentials and events are global. |
|
||||
| [x] 044 | `POST` | `/api/credential/{credentialID}/activate` | `credential.activate` | Change | Removed redundant location query; credentials and events are global. |
|
||||
@@ -147,7 +147,7 @@ Review endpoints in document order. For each endpoint, select one disposition an
|
||||
| [x] 053 | `POST` | `/api/session/{sessionID}/fork` | `session.fork` | Change | Request now accepts optional branded `before` message ID; omission copies full history. |
|
||||
| [x] 054 | `POST` | `/api/session/{sessionID}/agent` | `session.switchAgent` | Keep | Subsequent-execution agent selection retained. |
|
||||
| [x] 055 | `POST` | `/api/session/{sessionID}/model` | `session.switchModel` | Keep | Subsequent-execution model and optional variant selection retained. |
|
||||
| [x] 056 | `PATCH` | `/api/session/{sessionID}` | `session.rename` | Change | Title-only rename now uses the session resource path. |
|
||||
| [x] 056 | `PATCH` | `/api/session/{sessionID}` | `session.update` | Change | General session patch updates title and permissions; rules emit `session.permissions`. |
|
||||
| [x] 057 | `POST` | `/api/session/{sessionID}/move` | `session.move` | Change | Removed inaccurate local-change transfer claim; delivery behavior retained. |
|
||||
| [x] 058 | `POST` | `/api/session/{sessionID}/background` | `session.background` | Keep | Backgroundable foreground tools transition to background observation; idle requests remain no-ops. |
|
||||
|
||||
@@ -166,7 +166,7 @@ Review endpoints in document order. For each endpoint, select one disposition an
|
||||
| [x] 065 | `POST` | `/api/experimental/session/{sessionID}/wait` | `experimental.session.wait` | Experimental-only | Race-free idle barrier retained outside the stable API. |
|
||||
| [x] 066 | `POST` | `/api/session/{sessionID}/generate` | `session.generate` | Keep | Transient generation from session context retained. |
|
||||
| [x] 067 | `POST` | `/api/session/{sessionID}/interrupt` | `session.interrupt` | Change | Renamed `continue` to `resume` across public and internal interruption APIs. |
|
||||
| [x] 068 | `PUT` | `/api/experimental/session/{sessionID}/environment` | `experimental.session.environment` | Experimental-only | Process-local environment replacement retained outside the stable API. |
|
||||
| [x] 068 | `PUT` | `/api/session/{sessionID}/environment` | `session.environment` | Keep | Process-local environment replacement retained in the stable API. |
|
||||
| [x] 069 | `POST` | `/api/session/{sessionID}/view` | `session.view` | Change | Idle watermark now uses the standard epoch-millisecond timestamp schema. |
|
||||
|
||||
## Group 6: Session history and recovery
|
||||
@@ -185,7 +185,7 @@ Review endpoints in document order. For each endpoint, select one disposition an
|
||||
| [x] 077 | `GET` | `/api/experimental/session/{sessionID}/instructions/entries` | `experimental.session.instructions.entry.list` | Experimental-only | API-managed durable context entries retained outside the stable API. |
|
||||
| [x] 078 | `PUT` | `/api/experimental/session/{sessionID}/instructions/entries/{key}` | `experimental.session.instructions.entry.put` | Experimental-only | API-managed durable context entries retained outside the stable API. |
|
||||
| [x] 079 | `DELETE` | `/api/experimental/session/{sessionID}/instructions/entries/{key}` | `experimental.session.instructions.entry.remove` | Experimental-only | API-managed durable context entries retained outside the stable API. |
|
||||
| [x] | `GET` | `/api/experimental/session/{sessionID}/log` | `session.log` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 080 | `GET` | `/api/experimental/session/{sessionID}/log` | `session.log` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 081 | `GET` | `/api/session/{sessionID}/message/{messageID}` | `session.message.get` | Change | Normalized specific-message operation ID. |
|
||||
| [x] 082 | `GET` | `/api/session/{sessionID}/message` | `session.message.list` | Change | Normalized session-scoped message-list operation ID. |
|
||||
|
||||
@@ -196,24 +196,24 @@ Review endpoints in document order. For each endpoint, select one disposition an
|
||||
| Done | Method | Path | Operation ID | Decision | Notes |
|
||||
|---|---|---|---|---|---|
|
||||
| [x] 083 | `GET` | `/api/session/{sessionID}/inbox` | `session.inbox.list` | Change | Inbox timestamps now use the standard nested `time.created` shape. |
|
||||
| [ ] 084 | `DELETE` | `/api/session/{sessionID}/inbox/{inboxID}` | `session.inbox.cancel` | | |
|
||||
| [ ] 085 | `POST` | `/api/session/{sessionID}/inbox/{inboxID}/steer` | `session.inbox.steer` | | |
|
||||
| [ ] 086 | `POST` | `/api/session/{sessionID}/inbox/{inboxID}/queue` | `session.inbox.queue` | | |
|
||||
| [ ] 087 | `GET` | `/api/form/request` | `form.request.list` | | |
|
||||
| [ ] 088 | `GET` | `/api/session/{sessionID}/form` | `session.form.list` | | |
|
||||
| [ ] 089 | `POST` | `/api/session/{sessionID}/form` | `session.form.create` | | |
|
||||
| [ ] 090 | `GET` | `/api/session/{sessionID}/form/{formID}` | `session.form.get` | | |
|
||||
| [ ] 091 | `GET` | `/api/session/{sessionID}/form/{formID}/state` | `session.form.state` | | |
|
||||
| [ ] 092 | `POST` | `/api/session/{sessionID}/form/{formID}/reply` | `session.form.reply` | | |
|
||||
| [ ] 093 | `POST` | `/api/session/{sessionID}/form/{formID}/cancel` | `session.form.cancel` | | |
|
||||
| [ ] 094 | `GET` | `/api/permission/request` | `permission.request.list` | | |
|
||||
| [ ] 095 | `GET` | `/api/permission/saved` | `permission.saved.list` | | |
|
||||
| [ ] 096 | `DELETE` | `/api/permission/saved/{id}` | `permission.saved.remove` | | |
|
||||
| [ ] 097 | `POST` | `/api/session/{sessionID}/permission` | `session.permission.create` | | |
|
||||
| [ ] 098 | `GET` | `/api/session/{sessionID}/permission` | `session.permission.list` | | |
|
||||
| [ ] 099 | `GET` | `/api/session/{sessionID}/permission/{requestID}` | `session.permission.get` | | |
|
||||
| [ ] 100 | `POST` | `/api/session/{sessionID}/permission/{requestID}/reply` | `session.permission.reply` | | |
|
||||
| [ ] 101 | `PUT` | `/api/session/{sessionID}/permission/rules` | `session.permission.rules` | | |
|
||||
| [x] 084 | `DELETE` | `/api/session/{sessionID}/inbox/{inboxID}` | `session.inbox.cancel` | Change | Cancellation is idempotent and returns `204` when the session exists. |
|
||||
| [x] 085 | `PATCH` | `/api/session/{sessionID}/inbox/{inboxID}` | `session.inbox.update` | Change | Consolidated delivery mutation with `delivery: "steer" | "queue"`. |
|
||||
| [x] 086 | — | — | — | Remove | Replaced by `session.inbox.update`. |
|
||||
| [x] 087 | `GET` | `/api/form` | `form.list` | Change | Removed redundant `request` path and operation namespace. |
|
||||
| [x] 088 | `GET` | `/api/session/{sessionID}/form` | `session.form.list` | Keep | Pending session form list retained with temporary MCP sentinel compatibility. |
|
||||
| [x] 089 | `POST` | `/api/session/{sessionID}/form` | `session.form.create` | Keep | External form creation and temporary MCP sentinel ownership retained. |
|
||||
| [x] 090 | `GET` | `/api/session/{sessionID}/form/{formID}` | `session.form.get` | Change | Form definition and lifecycle state are now returned together. |
|
||||
| [x] 091 | — | — | — | Remove | State is included by `session.form.get`. |
|
||||
| [x] 092 | `POST` | `/api/session/{sessionID}/form/{formID}/reply` | `session.form.reply` | Keep | One-shot validated form reply retained. |
|
||||
| [x] 093 | `DELETE` | `/api/session/{sessionID}/form/{formID}` | `session.form.cancel` | Change | Form cancellation now deletes the pending form resource. |
|
||||
| [x] 094 | `GET` | `/api/permission/request` | `permission.request.list` | Keep | Pending-request namespace retained alongside saved permissions. |
|
||||
| [x] 095 | `GET` | `/api/permission/saved` | `permission.saved.list` | Change | Added persisted creation and update timestamps under `time`. |
|
||||
| [x] 096 | `DELETE` | `/api/permission/saved/{id}` | `permission.saved.remove` | Keep | Idempotent saved-permission deletion retained. |
|
||||
| [x] 097 | `POST` | `/api/session/{sessionID}/permission` | `session.permission.create` | Keep | Non-blocking permission evaluation and pending-request creation retained. |
|
||||
| [x] 098 | `GET` | `/api/session/{sessionID}/permission` | `session.permission.list` | Keep | Pending session permission list retained. |
|
||||
| [x] 099 | `GET` | `/api/session/{sessionID}/permission/{requestID}` | `session.permission.get` | Keep | Specific pending permission read with ownership validation retained. |
|
||||
| [x] 100 | `POST` | `/api/session/{sessionID}/permission/{requestID}/reply` | `session.permission.reply` | Change | Renamed request field from `reply` to `decision`. |
|
||||
| [x] 101 | — | — | — | Remove | Permission rules are updated through `session.update`. |
|
||||
|
||||
## Group 8: Filesystem, worktrees, and VCS
|
||||
|
||||
@@ -221,18 +221,18 @@ Review endpoints in document order. For each endpoint, select one disposition an
|
||||
|
||||
| Done | Method | Path | Operation ID | Decision | Notes |
|
||||
|---|---|---|---|---|---|
|
||||
| [ ] 102 | `GET` | `/api/fs/read/*` | `fs.read` | | |
|
||||
| [ ] 103 | `GET` | `/api/fs/list` | `fs.list` | | |
|
||||
| [ ] 104 | `GET` | `/api/fs/find` | `fs.find` | | |
|
||||
| [ ] 105 | `GET` | `/api/worktree` | `worktree.list` | | |
|
||||
| [ ] 106 | `POST` | `/api/worktree` | `worktree.create` | | |
|
||||
| [ ] 107 | `DELETE` | `/api/worktree` | `worktree.remove` | | |
|
||||
| [ ] 108 | `POST` | `/api/worktree/refresh` | `worktree.refresh` | | |
|
||||
| [ ] 109 | `GET` | `/api/vcs` | `vcs.get` | | |
|
||||
| [ ] 110 | `GET` | `/api/vcs/base` | `vcs.base` | | |
|
||||
| [ ] 111 | `GET` | `/api/vcs/status` | `vcs.status` | | |
|
||||
| [ ] 112 | `GET` | `/api/vcs/branches` | `vcs.branches` | | |
|
||||
| [ ] 113 | `GET` | `/api/vcs/diff` | `vcs.diff` | | |
|
||||
| [x] 102 | `GET` | `/api/fs/read/*` | `fs.read` | Keep | Relative wildcard file reads and raw byte responses retained. |
|
||||
| [x] 103 | `GET` | `/api/fs/list` | `fs.list` | Keep | Existing path scope and minimal entry metadata retained. |
|
||||
| [x] 104 | `GET` | `/api/fs/find` | `fs.find` | Keep | Existing ranked filesystem search retained. |
|
||||
| [x] 105 | `GET` | `/api/worktree` | `worktree.list` | Keep | Reviewed separately by coworker. |
|
||||
| [x] 106 | `POST` | `/api/worktree` | `worktree.create` | Keep | Reviewed separately by coworker. |
|
||||
| [x] 107 | `DELETE` | `/api/worktree` | `worktree.remove` | Keep | Reviewed separately by coworker. |
|
||||
| [x] 108 | `POST` | `/api/worktree/refresh` | `worktree.refresh` | Keep | Reviewed separately by coworker. |
|
||||
| [x] 109 | `GET` | `/api/vcs` | `vcs.get` | Change | Preserved branch nesting and added selected VCS provider ID. |
|
||||
| [x] 110 | `GET` | `/api/vcs/base` | `vcs.base` | Keep | Review-base inference and nullable unavailable state retained. |
|
||||
| [x] 111 | `GET` | `/api/vcs/status` | `vcs.status` | Keep | Existing working-copy status shape retained for now. |
|
||||
| [x] 112 | `GET` | `/api/vcs/branch` | `vcs.branch.list` | Change | Singular collection path and normalized operation ID. |
|
||||
| [x] 113 | `GET` | `/api/vcs/diff` | `vcs.diff` | Keep | Existing working, branch, and committed comparison modes retained. |
|
||||
|
||||
## Group 9: PTYs, persistent terminals, and shells
|
||||
|
||||
@@ -240,30 +240,30 @@ Review endpoints in document order. For each endpoint, select one disposition an
|
||||
|
||||
| Done | Method | Path | Operation ID | Decision | Notes |
|
||||
|---|---|---|---|---|---|
|
||||
| [ ] 114 | `GET` | `/api/pty` | `pty.list` | | |
|
||||
| [ ] 115 | `POST` | `/api/pty` | `pty.create` | | |
|
||||
| [ ] 116 | `GET` | `/api/pty/{ptyID}` | `pty.get` | | |
|
||||
| [ ] 117 | `PUT` | `/api/pty/{ptyID}` | `pty.update` | | |
|
||||
| [ ] 118 | `DELETE` | `/api/pty/{ptyID}` | `pty.remove` | | |
|
||||
| [ ] 119 | `POST` | `/api/pty/{ptyID}/connect-token` | `pty.connect.token` | | |
|
||||
| [ ] 120 | `GET` | `/api/pty/{ptyID}/connect` | `pty.connect` | | |
|
||||
| [x] | `GET` | `/api/experimental/session/{sessionID}/terminal/read` | `server.experimental.persistentPty.read` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] | `GET` | `/api/experimental/session/{sessionID}/terminal` | `server.experimental.persistentPty.list` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] | `POST` | `/api/experimental/session/{sessionID}/terminal` | `server.experimental.persistentPty.create` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] | `POST` | `/api/experimental/persistent-pty/shutdown` | `server.experimental.persistentPty.shutdown` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] | `POST` | `/api/experimental/persistent-pty/handoff` | `server.experimental.persistentPty.handoff` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] | `GET` | `/api/experimental/persistent-pty/{ptyID}` | `server.experimental.persistentPty.get` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] | `PUT` | `/api/experimental/persistent-pty/{ptyID}` | `server.experimental.persistentPty.update` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] | `DELETE` | `/api/experimental/persistent-pty/{ptyID}` | `server.experimental.persistentPty.remove` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] | `GET` | `/api/experimental/persistent-pty/{ptyID}/snapshot` | `server.experimental.persistentPty.snapshot` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] | `POST` | `/api/experimental/persistent-pty/{ptyID}/connect-token` | `server.experimental.persistentPty.connectToken` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] | `GET` | `/api/experimental/persistent-pty/{ptyID}/connect` | `persistentPty.connect` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [ ] 132 | `GET` | `/api/shell` | `shell.list` | | |
|
||||
| [ ] 133 | `POST` | `/api/shell` | `shell.create` | | |
|
||||
| [ ] 134 | `GET` | `/api/shell/{id}` | `shell.get` | | |
|
||||
| [ ] 135 | `DELETE` | `/api/shell/{id}` | `shell.remove` | | |
|
||||
| [ ] 136 | `PATCH` | `/api/shell/{id}/timeout` | `shell.timeout` | | |
|
||||
| [ ] 137 | `GET` | `/api/shell/{id}/output` | `shell.output` | | |
|
||||
| [x] 114 | `GET` | `/api/pty` | `pty.list` | Keep | PTY endpoints reviewed together and retained. |
|
||||
| [x] 115 | `POST` | `/api/pty` | `pty.create` | Keep | PTY endpoints reviewed together and retained. |
|
||||
| [x] 116 | `GET` | `/api/pty/{ptyID}` | `pty.get` | Keep | PTY endpoints reviewed together and retained. |
|
||||
| [x] 117 | `PUT` | `/api/pty/{ptyID}` | `pty.update` | Keep | PTY endpoints reviewed together and retained. |
|
||||
| [x] 118 | `DELETE` | `/api/pty/{ptyID}` | `pty.remove` | Keep | PTY endpoints reviewed together and retained. |
|
||||
| [x] 119 | `POST` | `/api/pty/{ptyID}/connect-token` | `pty.connect.token` | Keep | PTY endpoints reviewed together and retained. |
|
||||
| [x] 120 | `GET` | `/api/pty/{ptyID}/connect` | `pty.connect` | Keep | PTY endpoints reviewed together and retained. |
|
||||
| [x] 121 | `GET` | `/api/experimental/session/{sessionID}/terminal/read` | `server.experimental.persistentPty.read` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 122 | `GET` | `/api/experimental/session/{sessionID}/terminal` | `server.experimental.persistentPty.list` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 123 | `POST` | `/api/experimental/session/{sessionID}/terminal` | `server.experimental.persistentPty.create` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 124 | `POST` | `/api/experimental/persistent-pty/shutdown` | `server.experimental.persistentPty.shutdown` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 125 | `POST` | `/api/experimental/persistent-pty/handoff` | `server.experimental.persistentPty.handoff` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 126 | `GET` | `/api/experimental/persistent-pty/{ptyID}` | `server.experimental.persistentPty.get` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 127 | `PUT` | `/api/experimental/persistent-pty/{ptyID}` | `server.experimental.persistentPty.update` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 128 | `DELETE` | `/api/experimental/persistent-pty/{ptyID}` | `server.experimental.persistentPty.remove` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 129 | `GET` | `/api/experimental/persistent-pty/{ptyID}/snapshot` | `server.experimental.persistentPty.snapshot` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 130 | `POST` | `/api/experimental/persistent-pty/{ptyID}/connect-token` | `server.experimental.persistentPty.connectToken` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 131 | `GET` | `/api/experimental/persistent-pty/{ptyID}/connect` | `persistentPty.connect` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 132 | `GET` | `/api/shell` | `shell.list` | Change | Stable shell inventory retained; numeric timestamps documented as epoch milliseconds. |
|
||||
| [x] 133 | `POST` | `/api/shell` | `shell.create` | Change | Timeout is optional and defaults to zero; caller metadata retained. |
|
||||
| [x] 134 | `GET` | `/api/shell/{id}` | `shell.get` | Keep | Specific running or retained shell read retained. |
|
||||
| [x] 135 | `DELETE` | `/api/shell/{id}` | `shell.remove` | Change | Shell deletion is idempotent and returns `204` when already absent. |
|
||||
| [x] 136 | — | — | — | Remove | Timeout mutation remains an internal Core shell operation. |
|
||||
| [x] 137 | `GET` | `/api/shell/{id}/output` | `shell.output` | Keep | Existing byte-cursor text output paging retained. |
|
||||
|
||||
## Group 10: Events, RPC, and experimental operations
|
||||
|
||||
@@ -271,9 +271,9 @@ Review endpoints in document order. For each endpoint, select one disposition an
|
||||
|
||||
| Done | Method | Path | Operation ID | Decision | Notes |
|
||||
|---|---|---|---|---|---|
|
||||
| [ ] 138 | `POST` | `/api/generate` | `generate.text` | | |
|
||||
| [ ] 139 | `POST` | `/api/rpc/{rpcID}/{method}` | `rpc.call` | | |
|
||||
| [ ] 140 | `GET` | `/api/event` | `event.subscribe` | | |
|
||||
| [ ] 141 | `GET` | `/api/debug/location` | `debug.location.list` | | |
|
||||
| [ ] 142 | `DELETE` | `/api/debug/location` | `debug.location.evict` | | |
|
||||
| [x] | `GET` | `/api/experimental/migration/v1` | `experimental.migration.v1.status` | Experimental-only | Retained outside the stable API commitment. |
|
||||
| [x] 138 | `POST` | `/api/experimental/generate` | `experimental.generate.text` | Experimental-only | Stateless generation retained alongside session generation. |
|
||||
| [x] 139 | `POST` | `/api/rpc/{rpcID}/{method}` | `rpc.call` | Keep | Generic typed-error plugin RPC transport retained. |
|
||||
| [x] 140 | `GET` | `/api/event` | `event.subscribe` | Keep | Unified native and dynamic plugin event stream retained. |
|
||||
| [x] 141 | `GET` | `/api/debug/location` | `debug.location.list` | Keep | Loaded-location debug inventory retained. |
|
||||
| [x] 142 | `DELETE` | `/api/debug/location` | `debug.location.evict` | Keep | Idempotent loaded-location eviction retained. |
|
||||
| [x] 143 | `GET` | `/api/experimental/migration/v1` | `experimental.migration.v1.status` | Experimental-only | Retained outside the stable API commitment. |
|
||||
|
||||
@@ -353,6 +353,7 @@
|
||||
"@ff-labs/fff-node": "0.10.5",
|
||||
"@lydell/node-pty": "catalog:",
|
||||
"@modelcontextprotocol/client": "2.0.0",
|
||||
"@modelcontextprotocol/core": "2.0.0",
|
||||
"@opencode-ai/pty": "0.1.13",
|
||||
"@opencode/ai": "workspace:*",
|
||||
"@opencode/codemode": "workspace:*",
|
||||
|
||||
@@ -168,7 +168,7 @@ else
|
||||
|
||||
package_scope="@opencode"
|
||||
if [ -z "$requested_version" ]; then
|
||||
metadata=$(curl -fsSL https://opencode.ai/update/api/beta/cli/npm || true)
|
||||
metadata=$(curl -fsSL https://opencode.ai/update/api/latest/cli/npm || true)
|
||||
specific_version=$(echo "$metadata" | sed -n 's/.*"version":"\([^"]*\)".*/\1/p')
|
||||
package=$(echo "$metadata" | sed -n 's/.*"package":"\([^"]*\)".*/\1/p')
|
||||
|
||||
|
||||
+4
-4
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"nodeModules": {
|
||||
"x86_64-linux": "sha256-E5T4o3wNivOg8q4wRV7yE4CUNaUq4QNPxZsnvLaGUcg=",
|
||||
"aarch64-linux": "sha256-xQQi7LgxInZQVCznASfa0Pm+cNGBo5j4Tfp0E/bKnNw=",
|
||||
"aarch64-darwin": "sha256-qxb371dCf7WG09VvQE+HZ/x3EURpFIr11bdnQwGTHhw=",
|
||||
"x86_64-darwin": "sha256-xi1qiYr41hzbgAQl+I3xdtnkiPx2SIcA/0OMpN78qXw="
|
||||
"x86_64-linux": "sha256-U9IuP/ev6w4urvogOwQyl3rdumY6W4YaY18NkFaOVHU=",
|
||||
"aarch64-linux": "sha256-Wc8OT2DRZpVo56KaoGE0Hsj1NDknakbWXO9w2qy6j+0=",
|
||||
"aarch64-darwin": "sha256-wAea8+jajnMDxZ6XJL+Hsrf0621hwtBtWyD1+dS45dE=",
|
||||
"x86_64-darwin": "sha256-g8PCNBSV6rO+VQjKU9AtYqj+r18o+fhLDXEQq+X2EZ4="
|
||||
}
|
||||
}
|
||||
|
||||
+2
-15
@@ -10,15 +10,7 @@
|
||||
|
||||
## Conventions
|
||||
|
||||
Per-type constructors live on the type, not as top-level re-exports. Use `Message.system(...)`, `Message.user(...)`, `Message.assistant(...)`, `Message.tool(...)`, `Message.media(...)`, `LanguageModel.make(...)`, `ToolDefinition.make(...)`, `ToolCallPart.make(...)`, `ToolResultPart.make(...)`, `ToolChoice.make(...)`, `ToolChoice.named(...)`, `SystemPart.make(...)`, and `GenerationOptions.make(...)` directly. The top-level `LLM` namespace is reserved for request-shaped call APIs: `LLM.request`, `LLM.generate`, `LLM.stream`, and `LLM.generateObject`. Use `LLMRequest.update(...)` when deriving canonical request data; do not add a duplicate `LLM.updateRequest(...)` path. Two ways to construct the same thing is one too many.
|
||||
|
||||
Modality namespaces mirror `LLM` exactly: `Image.request`, `Image.generate`, `Image.stream` (later `Video`, `Speech`, `Transcription`). Common request fields (`images`, `mask`, `n`, `size`, `aspectRatio`, `seed`, `format`) lower natively or fail with a typed `AIError`; provider-native controls always live under `providerOptions`, never under a modality-specific `options` key.
|
||||
|
||||
Media payloads are always `Media.Asset` (`src/media.ts`). Construct them with `Media.bytes`, `Media.base64`, `Media.url`, `Media.ref`, `Media.fromDataUrl`, or `Media.file`; never introduce a parallel `data: string | Uint8Array` shape. `MediaPart.media`, `ImageRequest.images`/`mask`, `ImageResponse.images`, and the `media` `LLMEvent` all share it. Protocols branch on `asset.source.type` and `asset.kind` and use `ProviderShared.inlineMedia` / `requireInlineMedia` / `mediaUrl` / `MediaInput.refID` rather than re-deriving base64 or URL handling.
|
||||
|
||||
`schema/messages.ts → media.ts → route/executor-service.ts` is an accepted runtime dependency from the schema layer on the executor service tag: `Media.Asset.bytes()` must be able to download `url` sources, and the tag lives in that leaf module precisely so the schema barrel never imports the executor implementation (which imports the schema barrel back). Do not move the tag into `route/executor.ts` or import `route/executor.ts` from `src/schema/*` or `src/media.ts`.
|
||||
|
||||
Nothing in `src/*` except `src/promise.ts` may know about Promises. `@opencode/ai/promise` (`AI.make({ layer? })`, default `ai`) is the single Promise/`AsyncIterable` surface for LLM and media; it runs the Effect APIs in one `ManagedRuntime` and rethrows `AIError` unchanged.
|
||||
Per-type constructors live on the type, not as top-level re-exports. Use `Message.system(...)`, `Message.user(...)`, `Message.assistant(...)`, `Message.tool(...)`, `LanguageModel.make(...)`, `ToolDefinition.make(...)`, `ToolCallPart.make(...)`, `ToolResultPart.make(...)`, `ToolChoice.make(...)`, `ToolChoice.named(...)`, `SystemPart.make(...)`, and `GenerationOptions.make(...)` directly. The top-level `LLM` namespace is reserved for request-shaped call APIs: `LLM.request`, `LLM.generate`, `LLM.stream`, and `LLM.generateObject`. Use `LLMRequest.update(...)` when deriving canonical request data; do not add a duplicate `LLM.updateRequest(...)` path. Two ways to construct the same thing is one too many.
|
||||
|
||||
- Keep provider-defined string enums forward-compatible. Expose known values for autocomplete while accepting future values with `Known | (string & {})`; use `Schema.String` at runtime unless rejecting unknown values is required for correctness.
|
||||
- Order reasoning-effort values from lowest to highest: `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`. Provider-specific subsets follow the same relative order in types, schemas, option lists, and tests.
|
||||
@@ -94,10 +86,6 @@ The four-axis decomposition is the reason DeepSeek, TogetherAI, Cerebras, Basete
|
||||
|
||||
When a provider supports multiple physical transports, selection remains execution policy below its semantic route. `OpenResponsesChannel.transport(...)` owns the provider-neutral Responses WebSocket concept: it prepares one final request, executes HTTP by default, strips WebSocket-disallowed fields, and passes a generic channel exchange to a per-call `WebSocketChannelExecutor` when supplied. Provider-specific Responses routes opt in with handshake and connection-age policy. `Route.streamPrepared` owns decoding and acknowledges channel completion only after successful full consumption.
|
||||
|
||||
### Media Routes
|
||||
|
||||
Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute.make(...)` (`src/route/media.ts`) composes a `MediaProtocol` kind with `Endpoint` and `Auth` and owns the transport plumbing: `http` option merging, URL/query rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol. `MediaProtocol.inline` (`src/route/media-protocol.ts`) is `body.from(request)` plus `response.decode(response, context)`; use `MediaProtocol.decodeJson` / `text` / `bytes` so decode failures retain the raw body and HTTP context. `MediaProtocol.JobProtocol` is the queued shape and `Job` (`src/job.ts`) is the provider-neutral handle over a `JobRoute` (`status`, `result`, `cancel`, `pollHint`); the first video route implements it. Image protocol files follow the same section order as LLM protocols and declare unsupported common fields once through `MediaInput.rejectUnsupported`.
|
||||
|
||||
### URL Construction
|
||||
|
||||
`Endpoint` owns `{ baseURL, path, query }`. Each protocol route includes a canonical endpoint when the provider has one (e.g. `https://api.openai.com/v1`); provider helpers override endpoint fields by configuring the route before selecting a model. Generic OpenAI-compatible routes have no canonical URL and require configuration before execution.
|
||||
@@ -106,11 +94,10 @@ For providers where the URL is derived from typed inputs (Azure resource name, B
|
||||
|
||||
### Provider Facades
|
||||
|
||||
Provider-facing APIs are configured facades over route values. Endpoint/auth/resource/API-version setup happens before model selection, and model selectors accept only a model or deployment id. Every facade is wrapped in `ModelRef.facade(...)`, which makes it callable: `openai("gpt-4o-mini")` returns a `ModelRef` with lazy per-modality routes derived from the facade's own selectors (`model` → `llm`, `image` → `image`). `LLM.request` and `Image.request` resolve their route from the ref at request time, so one value names a model for every modality while named selectors stay the single source of truth:
|
||||
Provider-facing APIs are configured facades over route values. Endpoint/auth/resource/API-version setup happens before model selection, and model selectors accept only a model or deployment id:
|
||||
|
||||
```ts
|
||||
const openai = OpenAI.configure({ apiKey, baseURL })
|
||||
const ref = openai("gpt-4o-mini") // ModelRef: LLM.request → .responses, Image.request → .image
|
||||
const model = openai.responses("gpt-4o-mini")
|
||||
|
||||
const azure = Azure.configure({ resourceName, apiKey, apiVersion: "v1" })
|
||||
|
||||
+54
-104
@@ -8,10 +8,10 @@ import { LLM, LLMClient } from "@opencode/ai"
|
||||
import { RequestExecutor } from "@opencode/ai/route"
|
||||
import { OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
|
||||
const model = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY }).responses("gpt-4o-mini")
|
||||
|
||||
const request = LLM.request({
|
||||
model: openai("gpt-4o-mini"), // the request namespace picks the route; `.responses(...)` / `.chat(...)` stay explicit
|
||||
model,
|
||||
system: "You are concise.",
|
||||
prompt: "Say hello in one short sentence.",
|
||||
generation: { maxTokens: 40 },
|
||||
@@ -29,43 +29,6 @@ await Effect.runPromise(program.pipe(Effect.provide(llmLayer)))
|
||||
|
||||
Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
|
||||
|
||||
The same configured facade names image models. `Image.request` resolves the provider's image route from the ref and
|
||||
returns `Media.Asset`s with lazily decoded bytes:
|
||||
|
||||
```ts
|
||||
import { NodeFileSystem } from "@effect/platform-node"
|
||||
import { Image, ImageClient, Media } from "@opencode/ai"
|
||||
|
||||
const image = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: openai("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
size: "1024x1024",
|
||||
providerOptions: { quality: "high" }, // typed per image model
|
||||
})
|
||||
yield* Media.write(response.image, "./garden.png")
|
||||
})
|
||||
|
||||
// `asset.bytes()` / `Media.write` also need the executor, so merge it into the environment instead of hiding it.
|
||||
const imageLayer = ImageClient.layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer))
|
||||
|
||||
await Effect.runPromise(image.pipe(Effect.provide(imageLayer), Effect.provide(NodeFileSystem.layer)))
|
||||
```
|
||||
|
||||
Prefer promises? `@opencode/ai/promise` exposes the same LLM and image APIs over one managed runtime:
|
||||
|
||||
```ts
|
||||
import { AI } from "@opencode/ai/promise"
|
||||
|
||||
const ai = AI.make()
|
||||
const text = await ai.llm.generate({ model: openai("gpt-4o-mini"), prompt: "Say hello." })
|
||||
const generated = await ai.image.generate({ model: openai("gpt-image-2"), prompt: "A lighthouse" })
|
||||
for await (const event of ai.llm.stream({ model: openai("gpt-4o-mini"), prompt: "Stream hello." })) {
|
||||
// LLMEvent
|
||||
}
|
||||
await ai.dispose()
|
||||
```
|
||||
|
||||
## Alibaba Cloud Model Studio
|
||||
|
||||
`Alibaba` provides standard Model Studio inference. Configure a region explicitly, then select
|
||||
@@ -351,25 +314,23 @@ citations or separate result blocks. Retain `response.message` for either API's
|
||||
Use `Image.generate` for one-off generation or editing:
|
||||
|
||||
```ts
|
||||
import { Image, Media } from "@opencode/ai"
|
||||
import { Image, ImageInput } from "@opencode/ai"
|
||||
|
||||
const generation = Image.generate({
|
||||
model: meta("muse-image-1.0"),
|
||||
model: meta.image("muse-image-1.0"),
|
||||
prompt: "A flat black square on a white background.",
|
||||
n: 1,
|
||||
providerOptions: { reasoningStrength: "low" },
|
||||
options: { n: 1, reasoningStrength: "low" },
|
||||
})
|
||||
|
||||
const edit = Image.generate({
|
||||
model: meta("muse-image-1.0"),
|
||||
model: meta.image("muse-image-1.0"),
|
||||
prompt: "Make the square purple.",
|
||||
images: [Media.bytes(imageBytes, "image/webp")],
|
||||
format: "png",
|
||||
providerOptions: { reasoningStrength: "low" },
|
||||
images: [ImageInput.bytes(imageBytes, "image/webp")],
|
||||
options: { outputFormat: "png", reasoningStrength: "low" },
|
||||
})
|
||||
```
|
||||
|
||||
The default image format is WEBP; `format` also accepts PNG/JPEG and `responseFormat: "url"`
|
||||
The default image format is WEBP; `outputFormat` also accepts PNG/JPEG and `responseFormat: "url"`
|
||||
returns a signed URL. `size` is an aspect-ratio hint. For conversational images, select
|
||||
`meta.responses("muse-image-1.0")` with `tools: [Meta.imageGeneration({ reasoningStrength: "low" })]`.
|
||||
Generated images are provider-executed tool results with file content. Retain `response.message` to
|
||||
@@ -380,40 +341,29 @@ Meta Responses is explicitly HTTP/SSE-only and does not use WebSockets, even whe
|
||||
|
||||
## Image generation
|
||||
|
||||
Use `Image.generate` with an image model for direct asset generation. `Image.request` mirrors `LLM.request`: the
|
||||
model is a callable-facade ref (`openai("gpt-image-2")`) or an explicit `.image(...)` model, common fields
|
||||
(`images`, `mask`, `n`, `size`, `aspectRatio`, `seed`, `format`) lower natively or fail typed, and
|
||||
`providerOptions` is inferred from the selected model:
|
||||
Use `Image.generate` with an image model for direct asset generation:
|
||||
|
||||
```ts
|
||||
import { Image, Media } from "@opencode/ai"
|
||||
import { Image, ImageInput } from "@opencode/ai"
|
||||
import { OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
|
||||
|
||||
const program = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: openai("gpt-image-2"),
|
||||
model: OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY }).image("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
n: 2,
|
||||
size: "1024x1024",
|
||||
format: "webp",
|
||||
providerOptions: {
|
||||
options: {
|
||||
n: 2,
|
||||
size: "1024x1024",
|
||||
quality: "high", // inferred from the OpenAI image model
|
||||
outputFormat: "webp",
|
||||
future_option: true, // unknown native options pass through unchanged
|
||||
},
|
||||
})
|
||||
|
||||
return response.images // Media.Asset[] with owned bytes or a provider URL
|
||||
return response.images // GeneratedImage[] with owned bytes or a provider URL
|
||||
})
|
||||
```
|
||||
|
||||
`Media.Asset` is the one asset type shared by image requests, image responses, LLM messages, and tool results.
|
||||
`asset.source` is the serializable `Media.Source` (`bytes`, `base64`, `url`, or `ref`); `asset.bytes()`,
|
||||
`asset.base64()`, and `asset.dataUrl()` decode or download lazily and cache; `asset.materialize()` pulls a `url`
|
||||
asset into owned bytes before the provider URL expires. Construct assets with `Media.bytes`, `Media.base64`,
|
||||
`Media.url`, `Media.ref(provider, id)`, `Media.fromDataUrl`, or `Media.file(path)`.
|
||||
|
||||
Pass ordered image inputs to the same method for editing, composition, or image-conditioned generation:
|
||||
|
||||
```ts
|
||||
@@ -423,45 +373,49 @@ const response =
|
||||
model,
|
||||
prompt: "Combine these product photos into one studio scene",
|
||||
images: [
|
||||
Media.bytes(firstBytes, "image/png"),
|
||||
Media.url("https://example.com/second.webp"),
|
||||
Media.ref("openai", "file_123"),
|
||||
ImageInput.bytes(firstBytes, "image/png"),
|
||||
ImageInput.url("https://example.com/second.webp"),
|
||||
ImageInput.file("file_123"),
|
||||
],
|
||||
providerOptions,
|
||||
options,
|
||||
http,
|
||||
})
|
||||
```
|
||||
|
||||
`Media.ref(provider, id)` represents provider file handles such as OpenAI file IDs or Gemini Files URIs; routes
|
||||
only forward refs that belong to their own provider. Raw strings are not accepted as image inputs, avoiding
|
||||
ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a
|
||||
non-empty array selects the provider's edit behavior without enforcing provider image-count limits locally. OpenAI
|
||||
uses multipart for byte/data-URL edits and its JSON reference body for URL or file-ID edits. The common `mask`
|
||||
field selects inpainting; routes that cannot honor it fail with `UnsupportedOperation`:
|
||||
`ImageInput.fileUri(uri, mediaType)` represents provider file URIs such as Gemini Files. Raw strings are not
|
||||
accepted as image inputs, avoiding ambiguity between base64, URLs, and provider IDs. Empty or omitted `images`
|
||||
uses text-to-image generation; a non-empty array selects the provider's edit behavior without enforcing provider
|
||||
image-count limits locally. `images` is the only common image-editing field. OpenAI uses multipart for byte/data-URL
|
||||
edits and its JSON reference body for URL or file-ID edits. Its provider-specific `options.mask` accepts an
|
||||
`ImageInput` for inpainting:
|
||||
|
||||
```ts
|
||||
yield *
|
||||
Image.generate({
|
||||
model: openai("gpt-image-2"),
|
||||
model: OpenAI.configure({ apiKey }).image("gpt-image-2"),
|
||||
prompt,
|
||||
images: [Media.bytes(sourceBytes, "image/png")],
|
||||
mask: Media.bytes(maskBytes, "image/png"),
|
||||
images: [ImageInput.bytes(sourceBytes, "image/png")],
|
||||
options: { mask: ImageInput.bytes(maskBytes, "image/png") },
|
||||
})
|
||||
```
|
||||
|
||||
On multipart requests, `http.body` can override option fields but not structural `model`, `prompt`, `image[]`,
|
||||
or `mask` fields, and the transport owns the multipart `Content-Type` boundary. For JSON requests, `http.body`
|
||||
remains the final raw-native overlay. Gemini does not fetch public HTTP URLs, and hosted Z.ai image generation does
|
||||
not accept image inputs. These cases fail with a typed `AIError` before network I/O.
|
||||
The OpenAI adapter extracts this helper value into the edit request's native `mask` field rather than passing the
|
||||
tagged `ImageInput` object through as an ordinary option. On multipart requests, `http.body` can override option
|
||||
fields but not structural `model`, `prompt`, `image[]`, or `mask` fields, and the transport owns the multipart
|
||||
`Content-Type` boundary. For JSON requests, `http.body` remains the final raw-native overlay. Gemini does not fetch
|
||||
public HTTP URLs, and hosted Z.ai image generation does not accept image inputs. These cases fail with
|
||||
`InvalidRequest` before network I/O.
|
||||
|
||||
Provider-native image options belong to each request. Raw `http.body` fields have final precedence over them:
|
||||
|
||||
```ts
|
||||
const model = OpenAI.configure({ apiKey }).image("gpt-image-2")
|
||||
|
||||
yield *
|
||||
Image.generate({
|
||||
model: openai("gpt-image-2"),
|
||||
model,
|
||||
prompt,
|
||||
providerOptions: { quality: "medium" },
|
||||
options: { quality: "medium" },
|
||||
http,
|
||||
})
|
||||
```
|
||||
@@ -471,11 +425,11 @@ xAI image models use the same request API with xAI-native controls:
|
||||
```ts
|
||||
yield *
|
||||
Image.generate({
|
||||
model: XAI.configure({ apiKey })("any-model-id"),
|
||||
model: XAI.configure({ apiKey }).image("any-model-id"),
|
||||
prompt,
|
||||
n: 2,
|
||||
aspectRatio: "16:9",
|
||||
providerOptions: {
|
||||
options: {
|
||||
n: 2,
|
||||
aspectRatio: "16:9",
|
||||
resolution: "1k",
|
||||
responseFormat: "b64_json",
|
||||
future_option: true,
|
||||
@@ -491,12 +445,12 @@ import { Google } from "@opencode/ai/providers"
|
||||
|
||||
const googleProgram = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: Google.configure({ apiKey })("any-model-id"),
|
||||
model: Google.configure({ apiKey }).image("any-model-id"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
aspectRatio: "16:9",
|
||||
seed: 42,
|
||||
providerOptions: {
|
||||
options: {
|
||||
aspectRatio: "16:9",
|
||||
imageSize: "2K",
|
||||
seed: 42,
|
||||
thinkingLevel: "HIGH",
|
||||
includeThoughts: true,
|
||||
futureOption: true,
|
||||
@@ -518,9 +472,9 @@ Z.ai image models infer open Z.ai-native options from the selected model:
|
||||
```ts
|
||||
yield *
|
||||
Image.generate({
|
||||
model: ZAI.configure({ apiKey })("any-model-id"),
|
||||
model: ZAI.configure({ apiKey }).image("any-model-id"),
|
||||
prompt,
|
||||
providerOptions: {
|
||||
options: {
|
||||
quality: "hd",
|
||||
userID: "user-123",
|
||||
future_option: true,
|
||||
@@ -530,8 +484,8 @@ yield *
|
||||
```
|
||||
|
||||
Z.ai does not include trustworthy MIME metadata for output URLs, so generated images use
|
||||
`application/octet-stream` until materialized. Output URLs expire after 30 days; call `asset.materialize()` and
|
||||
persist the bytes promptly if they must remain available.
|
||||
`application/octet-stream`. Output URLs expire after 30 days; download and persist them promptly if they must
|
||||
remain available.
|
||||
|
||||
Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool:
|
||||
|
||||
@@ -549,7 +503,7 @@ const program = Effect.gen(function* () {
|
||||
})
|
||||
```
|
||||
|
||||
The hosted result is represented as a provider-executed tool call and tool result, and the generated image is also emitted as a first-class `media` `LLMEvent` (`response.message` then carries a `media` part). Gemini image-capable models emit the same `media` event for inline image output. Retaining `response.message` preserves the generated image for continuation on both routes.
|
||||
The hosted result is represented as a provider-executed tool call and tool result. Its image is a `file` content item with a data URI, so retaining `response.message` preserves the generated image for continuation.
|
||||
|
||||
## Public API
|
||||
|
||||
@@ -558,12 +512,8 @@ The hosted result is represented as a provider-executed tool call and tool resul
|
||||
- **`Message.user(...)` / `Message.assistant(...)` / `Message.tool(...)`** — message constructors from the canonical schema model.
|
||||
- **`LanguageModel.make(...)` / `ToolCallPart.make(...)` / `ToolResultPart.make(...)` / `ToolDefinition.make(...)`** — model and tool-related constructors from the canonical schema model.
|
||||
- **`LLMEvent.is.*`** — typed guards (`is.textDelta`, `is.toolCall`, `is.finish`, …) for filtering streams.
|
||||
- **`Image.request` / `Image.generate` / `Image.stream`** — generate images through a provider-neutral image request and response model.
|
||||
- **`Image.generate({...})`** — generate images through a provider-neutral image request and response model.
|
||||
- **`ImageClient`** — Effect service and layer for image execution, parallel to `LLMClient`.
|
||||
- **`Media`** — the shared asset type (`Media.Asset`, `Media.Source`) and constructors used by messages, tool results, and media requests.
|
||||
- **`ModelRef`** — the value returned by calling a configured facade (`openai("gpt-5")`); each request namespace resolves its own route from it.
|
||||
- **`Job`** — provider-neutral async job handle (`await`, `refresh`, `cancel`, `events`) used by queued media routes.
|
||||
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm` and `image` as Promise / `AsyncIterable` APIs.
|
||||
|
||||
## Testing
|
||||
|
||||
|
||||
@@ -1,342 +0,0 @@
|
||||
# Media generation in `@opencode/ai` — public API direction
|
||||
|
||||
Status: proposal. Branch `media-support`.
|
||||
|
||||
## Goal
|
||||
|
||||
`@opencode/ai` becomes the one package you reach for to generate anything: text, images, video, speech, transcripts, and later music and realtime. The LLM surface already exists and is shaped by three constraints: Effect-first, used by OpenCode Core, usable externally. Media has a different priority order: **external DX first**, Effect and Promise as peers, Core as one consumer among many.
|
||||
|
||||
The design below is derived from a survey of the raw provider APIs (OpenAI, Gemini/Veo/Imagen, xAI, Stability, BFL, fal, Replicate, Runway, Luma, Kling, MiniMax, ElevenLabs, Deepgram, Cartesia, AssemblyAI, Lyria) and from the Vercel AI SDK v7 (`generateImage`, `generateSpeech`, `transcribe`, `experimental_generateVideo`, `ImageModelV4`/`SpeechModelV4`/`TranscriptionModelV4`/`Experimental_VideoModelV4`).
|
||||
|
||||
## What the survey forces
|
||||
|
||||
1. **Three execution shapes, everywhere.** Inline sync (OpenAI images, all TTS, Gemini), async job with polling or webhook (every video provider, BFL, fal, Replicate, AssemblyAI), and bidirectional streams (ElevenLabs/Cartesia/Deepgram WS, realtime). Video has no sync provider at all.
|
||||
2. **Output is never just bytes.** base64, signed URLs with TTLs from 10 minutes (BFL) to 2 days (Veo), URLs that need auth plus redirect (Veo), separate download endpoints (Sora `/content?variant=`), raw bodies (Stability, TTS). Multi-output is the norm.
|
||||
3. **Inputs have roles.** First/last frame, mask, style/subject reference, source video for edit/extend, reference audio, prior generation id, provider-side file handles (`file_id`, `gs://`, `runway://`, `mm_file://`).
|
||||
4. **Partial streaming is modality-specific.** Images: a few whole partial frames. Audio: ordered chunks plus timestamp events. Jobs: status/progress/logs. Video: none.
|
||||
5. **Usage is a union**: tokens, seconds, characters (often only in headers), credits, compute time.
|
||||
6. **Moderation can be partial success** (Veo strips audio but returns video). Deprecations are constant (Sora API shuts down 2026-09-24, Imagen on Gemini API 2026-08-17).
|
||||
|
||||
## Where Vercel is weak and we should not be
|
||||
|
||||
- No streaming TTS at all.
|
||||
- Video job handle is experimental and only `start`/`getStatus`; polling loop is inside `generateVideo` with an injectable `delay`.
|
||||
- Unsupported inputs become silent `warnings` arrays, so a request can succeed while dropping your mask.
|
||||
- `n` is fanned out into hidden parallel calls (`maxImagesPerCall`), which obscures cost and idempotency.
|
||||
- Each modality has its own bespoke result type; the file abstraction is a lazy base64/bytes pair with no URL, expiry, or provider ref.
|
||||
- Effect's own `unstable/ai` has no media generation. Nothing in the Effect ecosystem owns this.
|
||||
|
||||
## Design principles
|
||||
|
||||
- **Same shape as LLM.** `X.request(...)` → Schema class; `X.generate(request)` / `X.stream(request)`; `XClient.Service` + `layer`; typed `AIError`. If you know `LLM`, you know `Video`.
|
||||
- **Execution shape is route policy, not API shape.** `Image.generate` returns an image whether the provider is inline or queued. Job control is available uniformly when you want it.
|
||||
- **Errors, not warnings.** Unsupported common fields fail at the protocol boundary with a typed `AIError`, as the LLM routes do today. Provider-side partial results (filtered audio, moderated sample) surface as `notices` on the response, never as silent drops.
|
||||
- **One asset type in, one asset type out**, shared with LLM messages and tool results.
|
||||
- **Typed per-model options**, no hidden fan-out, no implicit retries that spend money.
|
||||
- **Promise API is one mechanism for the whole package**, not a media-only wrapper.
|
||||
- **The request namespace names the modality; the model does not repeat it.** `Image.request({ model: openai("gpt-image-2") })`, not `openai.image("gpt-image-2")`.
|
||||
|
||||
## Public API
|
||||
|
||||
### Model selection — `ModelRef`
|
||||
|
||||
Today a model value is built as `OpenAI.configure({ apiKey }).responses("gpt-5")` or `.image("gpt-image-2")`: `configure` fixes credentials, endpoint, and defaults; the selector fixes which of the provider's APIs to hit and binds the typed `providerOptions` generic. The selector exists because OpenAI has two LLM APIs. For media that is the exception, and the request namespace already names the modality, so repeating it in the model is ceremony.
|
||||
|
||||
A configured facade is callable and returns a `ModelRef`. Each request namespace resolves its own route from the ref.
|
||||
|
||||
```ts
|
||||
import { OpenAI, Google } from "@opencode/ai/providers"
|
||||
|
||||
const openai = OpenAI.configure({ apiKey }) // OpenAI(...) alone uses env auth (OPENAI_API_KEY)
|
||||
|
||||
LLM.request({ model: openai("gpt-5"), prompt }) // → routes.llm (default: responses)
|
||||
Image.request({ model: openai("gpt-image-2"), prompt }) // → routes.image
|
||||
Video.request({ model: openai("sora-2"), prompt }) // → routes.video
|
||||
Speech.request({ model: openai("gpt-4o-mini-tts"), text }) // → routes.speech
|
||||
Transcription.request({ model: openai("gpt-4o-transcribe"), audio })
|
||||
|
||||
// Explicit selectors remain only where a provider has two APIs for one modality.
|
||||
LLM.request({ model: openai.chat("gpt-4o"), prompt })
|
||||
Image.request({ model: google.imagen("imagen-4.0-generate-001"), prompt }) // default is Gemini-native image
|
||||
```
|
||||
|
||||
Mechanics:
|
||||
|
||||
- `ModelRef<Routes>` carries `{ id, provider, routes }` where `Routes = { llm?: Route<…>; image?: ImageRoute<Opts>; video?: VideoRoute<Opts>; speech?: …; transcription?: … }`. Routes are lazy; unused protocols are not constructed.
|
||||
- `X.request<M extends XModel | ModelRef<{ x: XRoute<any> }>>` infers `providerOptions` from `M["routes"]["x"]`. Passing a ref whose provider has no `x` route is a compile error. Model-id validity stays a runtime provider error, as today.
|
||||
- Explicit selectors (`openai.chat`, `google.imagen`) return the concrete `XModel` directly, exactly as `.responses(...)` does now. `.model(id)` stays as an alias of the callable for LLM compatibility.
|
||||
- Provider package entrypoints keep `model(modelID, settings)` and gain the same resolution: `@opencode/ai/providers/openai` `model(...)` returns a ref; `@opencode/ai/providers/openai/responses` returns the concrete LLM model.
|
||||
- One default per modality per provider is part of the facade definition (OpenAI image → Images API, Google image → Gemini-native since Imagen on the Gemini API shuts down 2026-08-17).
|
||||
|
||||
This applies to LLM in the same pass so the package has one way to name a model.
|
||||
|
||||
### `Media` — the asset type
|
||||
|
||||
Replaces `MediaPart.data: string | Uint8Array`, `ImageInput`, `GeneratedImage`, and aligns `Tool.FileContent`.
|
||||
|
||||
```ts
|
||||
import { Media } from "@opencode/ai"
|
||||
|
||||
Media.Source =
|
||||
| { type: "bytes"; data: Uint8Array; mediaType: string }
|
||||
| { type: "base64"; data: string; mediaType: string }
|
||||
| { type: "url"; url: string; mediaType?: string; expiresAt?: number; headers?: Record<string, string> }
|
||||
| { type: "ref"; provider: ProviderID; id: string; mediaType?: string } // file_id, gs://, runway://, prior generation
|
||||
|
||||
class Media.Asset {
|
||||
readonly source: Media.Source
|
||||
readonly mediaType: string // always resolved (sniffed when the provider omits it)
|
||||
readonly kind: "image" | "video" | "audio" | "document" | "other"
|
||||
readonly info?: { width?; height?; durationSeconds?; sampleRate?; channels?; encoding?; format? }
|
||||
readonly expiresAt?: number
|
||||
readonly providerMetadata?: ProviderMetadata
|
||||
|
||||
bytes(): Effect<Uint8Array, AIError, RequestExecutor.Service> // downloads/decodes lazily, cached
|
||||
base64(): Effect<string, AIError, RequestExecutor.Service>
|
||||
dataUrl(): Effect<string, AIError, RequestExecutor.Service>
|
||||
materialize(): Effect<Media.Asset, AIError, RequestExecutor.Service> // url/ref → bytes, before the URL dies
|
||||
}
|
||||
|
||||
Media.bytes(data, mediaType?) Media.base64(data, mediaType?)
|
||||
Media.url(url, options?) Media.ref(provider, id)
|
||||
Media.file(path) // Bun/Node: reads + sniffs; Effect FileSystem variant for layers
|
||||
Media.write(asset, path) // convenience, uses FileSystem
|
||||
```
|
||||
|
||||
Raw-PCM outputs (Gemini TTS, Cartesia raw, Deepgram WS) carry `info.encoding/sampleRate/channels` because there is no container header.
|
||||
|
||||
### Modality namespaces
|
||||
|
||||
Each namespace mirrors `LLM` exactly.
|
||||
|
||||
```ts
|
||||
import { Image, Video, Speech, Transcription } from "@opencode/ai"
|
||||
import { OpenAI, Google, ElevenLabs, Fal } from "@opencode/ai/providers"
|
||||
```
|
||||
|
||||
#### Image
|
||||
|
||||
```ts
|
||||
const request = Image.request({
|
||||
model: openai("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
images: [Media.file("./ref.png")], // references / edit sources
|
||||
mask: Media.file("./mask.png"),
|
||||
n: 2,
|
||||
size: "1536x1024", // or aspectRatio: "3:2"
|
||||
seed: 7,
|
||||
format: "webp",
|
||||
providerOptions: { quality: "high", background: "transparent" }, // typed per model
|
||||
})
|
||||
|
||||
const response = yield* Image.generate(request) // ImageResponse
|
||||
response.image // Media.Asset (first)
|
||||
response.images // Media.Asset[]
|
||||
response.usage // Usage union (see below)
|
||||
response.notices // moderation / partial-result notices
|
||||
|
||||
yield* Image.stream(request) // Stream<ImageEvent>
|
||||
// ImageEvent: job-queued | job-progress | image-partial { index, image } | image { index, image } | finish { usage }
|
||||
```
|
||||
|
||||
Editing is not a separate function; `images`/`mask` on the request select the edit path in the route (OpenAI `/images/edits`, Gemini multimodal parts, xAI `/images/edits`). Routes that cannot honor `mask` fail with `Unsupported`.
|
||||
|
||||
#### Video
|
||||
|
||||
```ts
|
||||
const request = Video.request({
|
||||
model: google("veo-3.1-generate-preview"),
|
||||
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
|
||||
frames: { first: Media.file("./start.png"), last: Media.file("./end.png") },
|
||||
references: [Media.url("https://…/style.png")],
|
||||
video: Media.ref("openai", "video_123"), // edit / extend / remix source
|
||||
durationSeconds: 8,
|
||||
aspectRatio: "16:9",
|
||||
resolution: "1080p",
|
||||
audio: true,
|
||||
n: 1,
|
||||
providerOptions: { personGeneration: "dont_allow", negativePrompt: "text, watermark" },
|
||||
})
|
||||
|
||||
// Simple: wait for it.
|
||||
const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } })
|
||||
response.video // Media.Asset (url with expiresAt, or bytes when the route downloads)
|
||||
yield* response.video.materialize() // pull bytes before the URL expires
|
||||
|
||||
// Explicit job control.
|
||||
const job = yield* Video.start(request) // Job<VideoResponse>
|
||||
job.id; job.status; job.progress; job.token // token is serializable JSON
|
||||
yield* job.await({ poll }) // VideoResponse
|
||||
yield* job.cancel()
|
||||
|
||||
// Resume from another process.
|
||||
const resumed = yield* Video.resume(model, token) // Job<VideoResponse>
|
||||
|
||||
// Progress as a stream.
|
||||
yield* Video.stream(request) // Stream<VideoEvent>: job-queued { position } | job-progress { progress, logs } | video { index, video } | finish
|
||||
```
|
||||
|
||||
Webhooks: `Video.complete(model, token, webhook)` finishes a job from a webhook payload without polling. Token shape is route-owned and opaque (Veo operation name, fal `response_url`, Runway task id).
|
||||
|
||||
#### Speech (TTS)
|
||||
|
||||
```ts
|
||||
const request = Speech.request({
|
||||
model: elevenlabs("eleven_v3"),
|
||||
text: "Hello from OpenCode.",
|
||||
voice: "JBFqnCBsd6RMkjVDRZzb", // name, uuid, or { id } — provider-normalized
|
||||
format: "mp3", // mp3 | wav | pcm | opus | aac | flac | (string & {})
|
||||
speed: 1.0,
|
||||
language: "en",
|
||||
instructions: "Warm, unhurried.",
|
||||
providerOptions: { stability: 0.5 },
|
||||
})
|
||||
|
||||
const response = yield* Speech.generate(request) // SpeechResponse: audio: Media.Asset, timestamps?, usage
|
||||
yield* Speech.stream(request) // Stream<SpeechEvent>: audio-delta { chunk } | timestamps { words } | finish
|
||||
```
|
||||
|
||||
Streaming TTS is first-class on day one: OpenAI `stream_format: sse`, ElevenLabs `/stream`, Cartesia SSE, Deepgram chunked. Input-streaming TTS (WS, text arrives incrementally) is a later `Speech.session(...)` scoped resource, not part of `generate`.
|
||||
|
||||
#### Transcription (STT)
|
||||
|
||||
```ts
|
||||
const request = Transcription.request({
|
||||
model: openai("gpt-4o-transcribe"),
|
||||
audio: Media.file("./call.wav"),
|
||||
language: "en",
|
||||
prompt: "Names: Shoubhit, OpenCode.",
|
||||
timestamps: "word", // none | segment | word
|
||||
diarize: true,
|
||||
providerOptions: { chunkingStrategy: "auto" },
|
||||
})
|
||||
|
||||
const response = yield* Transcription.generate(request)
|
||||
response.text; response.segments; response.words; response.language; response.durationSeconds
|
||||
yield* Transcription.stream(request) // Stream<TranscriptionEvent>: text-delta | segment | finish
|
||||
```
|
||||
|
||||
Realtime STT over WebSocket is the same future `session` shape as input-streaming TTS.
|
||||
|
||||
### `Job` — shared async execution
|
||||
|
||||
```ts
|
||||
class Job<Response> {
|
||||
readonly id: string
|
||||
readonly model: MediaModel
|
||||
readonly token: unknown // route-owned serializable JSON
|
||||
readonly status: "queued" | "running" | "completed" | "failed" | "cancelled" | "expired"
|
||||
readonly progress?: number // 0..1, normalized
|
||||
readonly position?: number
|
||||
readonly expiresAt?: number
|
||||
refresh(): Effect<Job<Response>, AIError>
|
||||
await(options?: { poll?: Poll }): Effect<Response, AIError>
|
||||
cancel(): Effect<void, AIError>
|
||||
events(options?): Stream<JobEvent, AIError>
|
||||
}
|
||||
|
||||
Poll = { interval?: Duration; timeout?: Duration; schedule?: Schedule } // route may override from provider hints (`openai-poll-after-ms`)
|
||||
```
|
||||
|
||||
`Job` is not video-specific. Image routes on BFL, fal, and Replicate are jobs; `Image.start` exists for them. A route declares itself `inline` or `job`; `generate` on a job route is `start` then `await`.
|
||||
|
||||
### Usage
|
||||
|
||||
```ts
|
||||
Usage =
|
||||
| { type: "tokens"; input; output; total; details? }
|
||||
| { type: "seconds"; seconds }
|
||||
| { type: "characters"; characters }
|
||||
| { type: "credits"; credits }
|
||||
| { type: "compute"; seconds }
|
||||
```
|
||||
|
||||
Header-only usage (ElevenLabs `character-cost`, Deepgram `dg-char-count`) is lifted into `usage` by the route.
|
||||
|
||||
### Promise API — `@opencode/ai/promise`
|
||||
|
||||
Mirrors the `packages/plugin/src/effect` and `packages/plugin/src/promise` split that already exists in this repo. One mechanism for LLM and media.
|
||||
|
||||
```ts
|
||||
import { AI } from "@opencode/ai/promise"
|
||||
|
||||
const ai = AI.make() // ManagedRuntime over RequestExecutor.fetchLayer + all clients
|
||||
// AI.make({ layer }) to inject a custom executor / recorder / middleware
|
||||
|
||||
const image = await ai.image.generate({ model, prompt })
|
||||
await image.image.bytes()
|
||||
|
||||
for await (const event of ai.speech.stream({ model, text, voice })) { … }
|
||||
|
||||
const job = await ai.video.start({ model, prompt })
|
||||
const video = await job.await({ poll: { interval: 10_000 }, signal })
|
||||
const resumed = ai.video.resume(model, JSON.parse(saved))
|
||||
|
||||
const text = await ai.llm.generate({ model, prompt }) // closes today's gap: LLM has no promise API either
|
||||
for await (const event of ai.llm.stream(request)) { … }
|
||||
|
||||
await ai.dispose()
|
||||
```
|
||||
|
||||
Streams become `AsyncIterable` via `Stream.toAsyncIterable`. `AIError` is thrown as-is. `AbortSignal` maps to interruption. Nothing in `src/*` except this entrypoint knows about promises.
|
||||
|
||||
### Providers
|
||||
|
||||
Existing facades gain media routes behind the callable `ModelRef`; the modality routes each facade provides:
|
||||
|
||||
| Facade | llm | image | video | speech | transcription | other |
|
||||
|---|---|---|---|---|---|---|
|
||||
| `OpenAI` | responses (default), chat | Images API | Sora (deprecated 2026-09-24) | ✓ | ✓ | |
|
||||
| `Google` | Gemini | Gemini-native (default), `imagen` | Veo | Gemini TTS | Gemini transcribe | |
|
||||
| `XAI` | ✓ | ✓ | ✓ | | | |
|
||||
| `ElevenLabs` | | | | ✓ | Scribe | soundEffect, music |
|
||||
| `Fal` | | ✓ | ✓ | | | |
|
||||
| `Replicate`, `Runway`, `Luma`, `Kling`, `MiniMax`, `Deepgram`, `Cartesia`, `AssemblyAI`, `BlackForestLabs`, `Stability` | | per provider | | | | |
|
||||
|
||||
New facades follow the existing one-file-per-provider rule. Package entrypoints: `@opencode/ai/providers/openai` `model(id, settings)` returns the ref; modality-specific entrypoints such as `@opencode/ai/providers/openai/images` return the concrete model.
|
||||
|
||||
`ImageModel<Options>` already gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, `TranscriptionModel` follow the same generic and `ModelRef` infers through to them. A shared `MediaModel` union is what `Job` and the promise client key on.
|
||||
|
||||
### Routes and protocols
|
||||
|
||||
Media does not fit the LLM four-axis route (SSE frames → event state machine) except for streaming TTS/STT. Reuse `Endpoint`, `Auth`, `Framing`, `RequestExecutor`, and add media protocol kinds:
|
||||
|
||||
- `MediaProtocol.inline` — `body.from(request)` (JSON, multipart, or query), `response.decode(response)` (JSON, or binary body → `Media.Asset`).
|
||||
- `MediaProtocol.job` — `start`, `status`, `result`, `cancel`, optional `download`, `pollHint`, `token` schema.
|
||||
- `MediaProtocol.stream` — framing + `step` state machine emitting modality events, same discipline as LLM protocols.
|
||||
|
||||
`Route.make` for media composes one protocol kind with endpoint/auth. The existing `ImageRoute { generate(request, execute) }` is the ad-hoc version of `inline` and gets folded in.
|
||||
|
||||
### LLM integration
|
||||
|
||||
- `MediaPart` becomes `{ type: "media"; media: Media.Asset; … }` so protocols branch on `kind` and can pass `url`/`ref` sources through natively (OpenAI `image_url`, Gemini `fileData`).
|
||||
- New `LLMEvent`s: `media { media: Media.Asset }` so Gemini inline image output is first-class instead of dropped. OpenAI Responses `image_generation_call` keeps its single carrier — the provider-executed `tool-result` with `file` content — because Core consumes hosted tool-result content today and has no `media` event handling yet; it switches to the `media` carrier when Core adopts the event, so the image is never emitted twice.
|
||||
- `Message.assistant([...])` accepts media parts; Gemini multi-turn image editing replays them.
|
||||
- `Tool.FileContent` aligns with `Media.Source`.
|
||||
|
||||
## Decisions
|
||||
|
||||
All settled:
|
||||
|
||||
1. **Callable facades + `ModelRef`** replace per-modality selectors as the primary way to name a model, for LLM and media alike. Explicit selectors stay only for providers with two APIs in one modality.
|
||||
2. **`providerOptions` everywhere** (rename current `Image.options`) for consistency with LLM.
|
||||
3. **No hidden `n` fan-out.** `n` lowers natively; routes that cannot do `n > 1` fail typed. Callers use `Effect.all` / `Promise.all` explicitly.
|
||||
4. **Errors over warnings** for unsupported common fields; `notices` for provider-side partial results only.
|
||||
5. **`Media.Asset` is a class** (lazy bytes, cached) with `Media.Source` as the serializable Schema for wire/persistence. `Asset.from(source)` / `asset.source` round-trip losslessly. Same pattern as `LanguageModel` today.
|
||||
6. **Promise entrypoint**: `@opencode/ai/promise` exporting `AI.make(options?: { layer? })` plus a module-level default `ai` for scripts, covering LLM too.
|
||||
7. **Modality set for v1**: `Image`, `Video`, `Speech`, `Transcription`. `Music`/`SoundEffect` and `session` (bidirectional WS, realtime) are designed-for but deferred.
|
||||
8. **Sora is skipped** (API shuts down 2026-09-24). Video launches with Veo, xAI, fal, Runway.
|
||||
|
||||
## Build order
|
||||
|
||||
Foundation + Image ship together as the reference implementation, serially. Video, Speech, and Transcription then proceed in parallel on separate branches. Image jobs and partial streaming come last, after Video has hardened `Job`.
|
||||
|
||||
## Phasing
|
||||
|
||||
1. **Foundation** — `ModelRef` + callable facades (LLM included, `.responses`/`.chat`/`.model` kept), `Media`, `Job`, `Poll`, `Usage` union, `MediaProtocol` kinds, `@opencode/ai/promise` with `llm` + `image`. Port the five existing image protocols onto it. Unify `MediaPart` and add the `media` LLM event (fixes Gemini image output being dropped).
|
||||
2. **Video** — Veo, xAI, fal, Runway first. Then Luma, Kling, MiniMax, Replicate.
|
||||
3. **Speech + Transcription** — OpenAI, ElevenLabs, Gemini TTS, Deepgram, Cartesia, AssemblyAI. Streaming TTS from the start.
|
||||
4. **Image jobs and partials** — BFL, fal, Replicate, Stability; OpenAI `partial_images` streaming.
|
||||
5. **Later** — ElevenLabs music/SFX, Lyria, `Speech.session` / `Transcription.session`, realtime.
|
||||
|
||||
Core adoption (session attachments beyond png/jpeg/gif/webp/pdf, image-generation tool, TUI rendering) comes after phase 1 and is a Core concern.
|
||||
@@ -1,17 +1,5 @@
|
||||
import { Config, Effect, Formatter, Layer, Schema, Stream } from "effect"
|
||||
import { NodeFileSystem } from "@effect/platform-node"
|
||||
import {
|
||||
Image,
|
||||
ImageClient,
|
||||
LLM,
|
||||
LLMClient,
|
||||
LLMRequest,
|
||||
Media,
|
||||
Message,
|
||||
ProviderID,
|
||||
Tool,
|
||||
ToolRuntime,
|
||||
} from "@opencode/ai"
|
||||
import { LLM, LLMClient, LLMRequest, Message, ProviderID, Tool, ToolRuntime } from "@opencode/ai"
|
||||
import { Route, Auth, Endpoint, Framing, Protocol, RequestExecutor } from "@opencode/ai/route"
|
||||
import { OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
@@ -28,20 +16,15 @@ import { OpenAI } from "@opencode/ai/providers"
|
||||
|
||||
const apiKey = Config.redacted("OPENAI_API_KEY")
|
||||
|
||||
// 1. Configure a provider. The configured facade records provider identity,
|
||||
// deployment options, authentication, and defaults. Calling it with a model id
|
||||
// returns a `ModelRef`: `LLM.request` resolves the default LLM route
|
||||
// (Responses for OpenAI) and `Image.request` resolves the image route, so one
|
||||
// value names a model for every modality. `.responses(...)` / `.chat(...)`
|
||||
// remain available when you need a specific API.
|
||||
const openai = OpenAI.configure({
|
||||
// 1. Pick a model. The provider helper records provider identity, protocol
|
||||
// choice, capabilities, deployment options, authentication, and defaults.
|
||||
const model = OpenAI.configure({
|
||||
apiKey,
|
||||
generation: { maxTokens: 160 },
|
||||
providerOptions: {
|
||||
store: false,
|
||||
},
|
||||
})
|
||||
const model = openai("gpt-4o-mini")
|
||||
}).model("gpt-4o-mini")
|
||||
|
||||
// 2. Build a provider-neutral request. This is useful when reusing one request
|
||||
// across generate and stream examples.
|
||||
@@ -226,39 +209,18 @@ const FakeEcho = {
|
||||
}),
|
||||
}
|
||||
|
||||
// 8. Image generation uses the same facade and the same request/generate shape.
|
||||
// `response.image` is a `Media.Asset`: bytes decode lazily and are cached, and
|
||||
// `Media.write` persists them through the Effect `FileSystem`.
|
||||
const generateImage = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: openai("gpt-image-1-mini"),
|
||||
prompt: "A flat black circle centered on a plain white background.",
|
||||
size: "1024x1024",
|
||||
format: "jpeg",
|
||||
providerOptions: { quality: "low" },
|
||||
})
|
||||
|
||||
console.log("\n== image ==")
|
||||
console.log("media type:", response.image.mediaType)
|
||||
console.log("bytes:", (yield* response.image.bytes()).byteLength)
|
||||
console.log("usage", Formatter.formatJson(response.usage, { space: 2 }))
|
||||
yield* Media.write(response.image, "tutorial-image.jpg").pipe(Effect.provide(NodeFileSystem.layer))
|
||||
})
|
||||
|
||||
// Provide the LLM runtime and the HTTP request executor once. Keep one path
|
||||
// enabled at a time so the tutorial can demonstrate generate, stream, or
|
||||
// tool-loop behavior without spending tokens on every example.
|
||||
const requestExecutorLayer = RequestExecutor.fetchLayer
|
||||
const llmClientLayer = LLMClient.layer.pipe(Layer.provide(requestExecutorLayer))
|
||||
const imageClientLayer = ImageClient.layer.pipe(Layer.provide(requestExecutorLayer))
|
||||
|
||||
const program = Effect.gen(function* () {
|
||||
// yield* generateOnce
|
||||
// yield* streamText
|
||||
// yield* generateStructuredObject
|
||||
// yield* generateDynamicObject.pipe(Effect.andThen((response) => Effect.sync(() => console.log(response.object))))
|
||||
// yield* generateImage
|
||||
yield* streamWithTools
|
||||
}).pipe(Effect.provide(Layer.mergeAll(requestExecutorLayer, llmClientLayer, imageClientLayer)))
|
||||
}).pipe(Effect.provide(Layer.mergeAll(requestExecutorLayer, llmClientLayer)))
|
||||
|
||||
Effect.runPromise(program)
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
],
|
||||
"exports": {
|
||||
".": "./src/index.ts",
|
||||
"./promise": "./src/promise.ts",
|
||||
"./testing": "./src/testing.ts",
|
||||
"./*": "./src/*.ts"
|
||||
},
|
||||
|
||||
@@ -1,21 +1,15 @@
|
||||
import { Context, Effect, Layer, Stream } from "effect"
|
||||
import { Context, Effect, Layer } from "effect"
|
||||
import { RequestExecutor } from "./route/executor.js"
|
||||
import type { AIError } from "./schema/index.js"
|
||||
import {
|
||||
responseEvents,
|
||||
type ImageEvent,
|
||||
type ImageOptions,
|
||||
type ImageRequestFor,
|
||||
type ImageResponse,
|
||||
} from "./image.js"
|
||||
import { mergeHttpOptions, type AIError } from "./schema/index.js"
|
||||
import { sanitizeSurrogates } from "./utils/sanitize.js"
|
||||
import type { ImageOptions, ImageRequest, ImageRequestFor, ImageResponse } from "./image.js"
|
||||
|
||||
export type Execute = RequestExecutor.Interface["execute"]
|
||||
|
||||
export interface Interface {
|
||||
readonly generate: <Options extends ImageOptions>(
|
||||
request: ImageRequestFor<Options>,
|
||||
) => Effect.Effect<ImageResponse, AIError>
|
||||
readonly stream: <Options extends ImageOptions>(
|
||||
request: ImageRequestFor<Options>,
|
||||
) => Stream.Stream<ImageEvent, AIError>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/ImageClient") {}
|
||||
@@ -28,27 +22,23 @@ export const generate = <Options extends ImageOptions>(
|
||||
return yield* client.generate(request)
|
||||
})
|
||||
|
||||
export const stream = <Options extends ImageOptions>(
|
||||
request: ImageRequestFor<Options>,
|
||||
): Stream.Stream<ImageEvent, AIError, Service> =>
|
||||
Stream.unwrap(
|
||||
Effect.gen(function* () {
|
||||
const client = yield* Service
|
||||
return client.stream(request)
|
||||
}),
|
||||
)
|
||||
|
||||
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
const generate = <Options extends ImageOptions>(request: ImageRequestFor<Options>) =>
|
||||
request.model.route.generate(request, executor.execute)
|
||||
return Service.of({
|
||||
generate,
|
||||
// Inline routes have no partial frames yet; the stream is the completed response expanded into events.
|
||||
stream: (request) =>
|
||||
Stream.unwrap(generate(request).pipe(Effect.map((response) => Stream.fromIterable(responseEvents(response))))),
|
||||
generate: (request) =>
|
||||
request.model.route.generate(
|
||||
{
|
||||
...sanitizeSurrogates({
|
||||
...request,
|
||||
model: undefined,
|
||||
http: mergeHttpOptions(request.model.http, request.http),
|
||||
}),
|
||||
model: request.model,
|
||||
},
|
||||
executor.execute,
|
||||
),
|
||||
})
|
||||
}),
|
||||
)
|
||||
@@ -57,5 +47,4 @@ export const ImageClient = {
|
||||
Service,
|
||||
layer,
|
||||
generate,
|
||||
stream,
|
||||
} as const
|
||||
|
||||
+68
-168
@@ -1,32 +1,22 @@
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import { Media } from "./media.js"
|
||||
import { ModelRef, type ResolveImageModel } from "./model-ref.js"
|
||||
import { Endpoint } from "./route/endpoint.js"
|
||||
import { MediaRoute } from "./route/media.js"
|
||||
import type { MediaProtocol } from "./route/media-protocol.js"
|
||||
import { Effect, Schema } from "effect"
|
||||
import {
|
||||
AIError,
|
||||
HttpOptions,
|
||||
InvalidRequestError,
|
||||
MediaUsage,
|
||||
AIError,
|
||||
ModelID,
|
||||
ProviderID,
|
||||
ProviderMetadata,
|
||||
UnsupportedOperationError,
|
||||
Usage,
|
||||
} from "./schema/index.js"
|
||||
import { ImageClient, Service } from "./image-client.js"
|
||||
import { ImageClient, Service, type Execute as ImageExecute } from "./image-client.js"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Model
|
||||
// ---------------------------------------------------------------------------
|
||||
export interface ImageRoute<Options extends ImageOptions = ImageOptions> {
|
||||
readonly id: string
|
||||
readonly generate: (request: ImageRequestFor<Options>, execute: ImageExecute) => Effect.Effect<ImageResponse, AIError>
|
||||
}
|
||||
|
||||
export type ImageOptions = Record<string, unknown>
|
||||
|
||||
export type ImageRoute<Options extends ImageOptions = ImageOptions> = MediaRoute.Route<
|
||||
ImageRequestFor<Options>,
|
||||
ImageResponse
|
||||
>
|
||||
|
||||
export class ImageModel<Options extends ImageOptions = ImageOptions> {
|
||||
declare protected readonly _Options: (options: Options) => Options
|
||||
readonly id: ModelID
|
||||
@@ -49,26 +39,6 @@ export class ImageModel<Options extends ImageOptions = ImageOptions> {
|
||||
http: input.http,
|
||||
})
|
||||
}
|
||||
|
||||
/** Compose an inline image protocol with its canonical path into a model for one deployment. */
|
||||
static fromRoute<Options extends ImageOptions = ImageOptions>(
|
||||
route: ImageModel.RouteInput<Options>,
|
||||
input: MediaRoute.ModelInput,
|
||||
) {
|
||||
return ImageModel.make<Options>({
|
||||
id: input.id,
|
||||
provider: route.provider,
|
||||
http: input.http,
|
||||
route: MediaRoute.make({
|
||||
id: route.id,
|
||||
provider: route.provider,
|
||||
protocol: route.protocol,
|
||||
endpoint: Endpoint.path(route.path, { baseURL: input.baseURL ?? route.baseURL }),
|
||||
auth: input.auth,
|
||||
headers: input.headers,
|
||||
}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export namespace ImageModel {
|
||||
@@ -84,87 +54,81 @@ export namespace ImageModel {
|
||||
readonly id: string | ModelID
|
||||
readonly provider: string | ProviderID
|
||||
}
|
||||
|
||||
export interface RouteInput<Options extends ImageOptions = ImageOptions> {
|
||||
readonly id: string
|
||||
readonly provider: string | ProviderID
|
||||
readonly protocol: MediaProtocol.Inline<ImageRequestFor<Options>, ImageResponse>
|
||||
readonly path: Endpoint.EndpointPart<MediaProtocol.Body, ImageRequestFor<Options>>
|
||||
/** Canonical base URL; `ModelInput.baseURL` overrides it per deployment. */
|
||||
readonly baseURL?: string
|
||||
}
|
||||
}
|
||||
|
||||
export const ImageModelSchema = Schema.declare((value): value is ImageModel => value instanceof ImageModel, {
|
||||
expected: "Image.Model",
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Request
|
||||
// ---------------------------------------------------------------------------
|
||||
const ImageBytesInput = Schema.Struct({
|
||||
type: Schema.Literal("bytes"),
|
||||
data: Schema.Uint8Array,
|
||||
mediaType: Schema.String,
|
||||
})
|
||||
const ImageUrlInput = Schema.Struct({
|
||||
type: Schema.Literal("url"),
|
||||
url: Schema.String,
|
||||
})
|
||||
const ImageFileIDInput = Schema.Struct({
|
||||
type: Schema.Literal("file-id"),
|
||||
id: Schema.String,
|
||||
})
|
||||
const ImageFileURIInput = Schema.Struct({
|
||||
type: Schema.Literal("file-uri"),
|
||||
uri: Schema.String,
|
||||
mediaType: Schema.String,
|
||||
})
|
||||
|
||||
export type ImageSize = `${number}x${number}`
|
||||
export const ImageSize = Schema.declare<ImageSize>(
|
||||
(value): value is ImageSize => typeof value === "string" && /^\d+x\d+$/.test(value),
|
||||
{ title: "ImageSize" },
|
||||
)
|
||||
export const ImageInputSchema = Schema.Union([
|
||||
ImageBytesInput,
|
||||
ImageUrlInput,
|
||||
ImageFileIDInput,
|
||||
ImageFileURIInput,
|
||||
]).pipe(Schema.toTaggedUnion("type"))
|
||||
export type ImageInput = Schema.Schema.Type<typeof ImageInputSchema>
|
||||
|
||||
export type ImageAspectRatio = `${number}:${number}`
|
||||
export const ImageAspectRatio = Schema.declare<ImageAspectRatio>(
|
||||
(value): value is ImageAspectRatio => typeof value === "string" && /^\d+(?:\.\d+)?:\d+(?:\.\d+)?$/.test(value),
|
||||
{ title: "ImageAspectRatio" },
|
||||
)
|
||||
|
||||
export type ImageFormat = "png" | "jpeg" | "webp" | (string & {})
|
||||
export const ImageInput = {
|
||||
bytes: (data: Uint8Array, mediaType: string): ImageInput => ({ type: "bytes", data, mediaType }),
|
||||
url: (url: string): ImageInput => ({ type: "url", url }),
|
||||
file: (id: string): ImageInput => ({ type: "file-id", id }),
|
||||
fileUri: (uri: string, mediaType: string): ImageInput => ({ type: "file-uri", uri, mediaType }),
|
||||
} as const
|
||||
|
||||
export class ImageRequest extends Schema.Class<ImageRequest>("Image.Request")({
|
||||
model: ImageModelSchema,
|
||||
prompt: Schema.String,
|
||||
/** Edit sources or style/subject references, in order. */
|
||||
images: Schema.optional(Schema.Array(Media.AssetSchema)),
|
||||
/** Inpainting mask; routes that cannot honor it fail with `UnsupportedOperation`. */
|
||||
mask: Schema.optional(Media.AssetSchema),
|
||||
n: Schema.optional(Schema.Int),
|
||||
size: Schema.optional(ImageSize),
|
||||
aspectRatio: Schema.optional(ImageAspectRatio),
|
||||
seed: Schema.optional(Schema.Number),
|
||||
format: Schema.optional(Schema.String),
|
||||
providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
images: Schema.optional(Schema.Array(ImageInputSchema)),
|
||||
options: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
http: Schema.optional(HttpOptions),
|
||||
}) {
|
||||
declare protected readonly _ImageRequest: void
|
||||
}
|
||||
|
||||
export type ImageRequestFor<Options extends ImageOptions = ImageOptions> = Omit<
|
||||
ImageRequest,
|
||||
"model" | "providerOptions"
|
||||
> & {
|
||||
export type ImageRequestFor<Options extends ImageOptions = ImageOptions> = Omit<ImageRequest, "model" | "options"> & {
|
||||
readonly model: ImageModel<Options>
|
||||
readonly providerOptions?: Options
|
||||
readonly options?: Options
|
||||
}
|
||||
|
||||
export type ImageModelInput = ImageModel | ModelRef.WithImage
|
||||
export type ImageModelOptions<Model> = Model extends ImageModel<infer Options> ? Options : never
|
||||
|
||||
export type ImageModelOptions<Model> = ResolveImageModel<Model> extends ImageModel<infer Options> ? Options : never
|
||||
|
||||
export type ImageRequestInput<Model extends ImageModelInput = ImageModelInput> = Omit<
|
||||
export type ImageRequestInput<Model extends object = ImageModel> = Omit<
|
||||
ConstructorParameters<typeof ImageRequest>[0],
|
||||
"model" | "providerOptions" | "http"
|
||||
"model" | "options" | "http"
|
||||
> & {
|
||||
readonly model: Model
|
||||
readonly format?: ImageFormat
|
||||
readonly providerOptions?: NoInfer<ImageModelOptions<Model>>
|
||||
readonly options?: NoInfer<ImageModelOptions<Model>>
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
} & (Model extends ImageModel<ImageModelOptions<Model>> ? unknown : never)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response and events
|
||||
// ---------------------------------------------------------------------------
|
||||
export class GeneratedImage extends Schema.Class<GeneratedImage>("Image.Generated")({
|
||||
mediaType: Schema.String,
|
||||
data: Schema.Union([Schema.String, Schema.Uint8Array]),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}) {}
|
||||
|
||||
export class ImageResponse extends Schema.Class<ImageResponse>("Image.Response")({
|
||||
images: Schema.Array(Media.AssetSchema),
|
||||
usage: Schema.optional(MediaUsage),
|
||||
notices: Schema.optional(Schema.Array(Media.Notice)),
|
||||
images: Schema.Array(GeneratedImage),
|
||||
usage: Schema.optional(Usage),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}) {
|
||||
get image() {
|
||||
@@ -172,57 +136,7 @@ export class ImageResponse extends Schema.Class<ImageResponse>("Image.Response")
|
||||
}
|
||||
}
|
||||
|
||||
export const ImageOutputEvent = Schema.Struct({
|
||||
type: Schema.tag("image"),
|
||||
index: Schema.Number,
|
||||
image: Media.AssetSchema,
|
||||
}).annotate({ identifier: "Image.Event.Image" })
|
||||
|
||||
export const ImageFinishEvent = Schema.Struct({
|
||||
type: Schema.tag("finish"),
|
||||
usage: Schema.optional(MediaUsage),
|
||||
notices: Schema.optional(Schema.Array(Media.Notice)),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}).annotate({ identifier: "Image.Event.Finish" })
|
||||
|
||||
const imageEventTagged = Schema.Union([ImageOutputEvent, ImageFinishEvent]).pipe(Schema.toTaggedUnion("type"))
|
||||
export const ImageEvent = Object.assign(imageEventTagged, {
|
||||
is: {
|
||||
image: imageEventTagged.guards.image,
|
||||
finish: imageEventTagged.guards.finish,
|
||||
},
|
||||
})
|
||||
export type ImageEvent = Schema.Schema.Type<typeof imageEventTagged>
|
||||
|
||||
/** Inline routes produce every image at once; expand the response into the streaming event shape. */
|
||||
export const responseEvents = (response: ImageResponse): ReadonlyArray<ImageEvent> => [
|
||||
...response.images.map((image, index) => ImageOutputEvent.make({ index, image })),
|
||||
ImageFinishEvent.make({
|
||||
usage: response.usage,
|
||||
notices: response.notices,
|
||||
providerMetadata: response.providerMetadata,
|
||||
}),
|
||||
]
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Request-shaped call API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const resolveModel = (model: ImageModelInput): ImageModel => {
|
||||
if (model instanceof ImageModel) return model
|
||||
// Typed callers cannot reach the throw; it guards JS callers passing a ref without an image selector.
|
||||
const selected = model.facade.image?.(model.id)
|
||||
if (selected) return selected
|
||||
throw new AIError({
|
||||
reason: new UnsupportedOperationError({
|
||||
operation: "image",
|
||||
provider: model.provider,
|
||||
message: `${model.provider} does not expose an image route for ${model.id}`,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
export function request<const Model extends ImageModelInput>(
|
||||
export function request<const Model extends object>(
|
||||
input: ImageRequestInput<Model>,
|
||||
): ImageRequestFor<ImageModelOptions<Model>>
|
||||
export function request(input: ImageRequest): ImageRequest
|
||||
@@ -230,43 +144,29 @@ export function request(input: ImageRequest | ImageRequestInput) {
|
||||
if (input instanceof ImageRequest) return input
|
||||
return new ImageRequest({
|
||||
...input,
|
||||
model: resolveModel(input.model),
|
||||
model: input.model as unknown as ImageModel,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
})
|
||||
}
|
||||
|
||||
const requestEffect = (input: ImageRequest | ImageRequestInput) =>
|
||||
Effect.try({
|
||||
try: () => request(input),
|
||||
catch: (error) =>
|
||||
error instanceof AIError
|
||||
? error
|
||||
: new AIError({
|
||||
reason: new InvalidRequestError({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
cause: error,
|
||||
}),
|
||||
}),
|
||||
})
|
||||
|
||||
export function generate<const Model extends ImageModelInput>(
|
||||
export function generate<const Model extends object>(
|
||||
input: ImageRequestInput<Model>,
|
||||
): Effect.Effect<ImageResponse, AIError, Service>
|
||||
export function generate(input: ImageRequest): Effect.Effect<ImageResponse, AIError, Service>
|
||||
export function generate(input: ImageRequest | ImageRequestInput) {
|
||||
return requestEffect(input).pipe(Effect.flatMap((request) => ImageClient.generate(request)))
|
||||
}
|
||||
|
||||
export function stream<const Model extends ImageModelInput>(
|
||||
input: ImageRequestInput<Model>,
|
||||
): Stream.Stream<ImageEvent, AIError, Service>
|
||||
export function stream(input: ImageRequest): Stream.Stream<ImageEvent, AIError, Service>
|
||||
export function stream(input: ImageRequest | ImageRequestInput) {
|
||||
return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => ImageClient.stream(request))))
|
||||
return Effect.try({
|
||||
try: () => (input instanceof ImageRequest ? input : request(input)),
|
||||
catch: (error) =>
|
||||
new AIError({
|
||||
reason: new InvalidRequestError({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
cause: error,
|
||||
}),
|
||||
}),
|
||||
}).pipe(Effect.flatMap((request) => ImageClient.generate(request as unknown as ImageRequestFor<ImageOptions>)))
|
||||
}
|
||||
|
||||
export const Image = {
|
||||
request,
|
||||
generate,
|
||||
stream,
|
||||
} as const
|
||||
|
||||
@@ -11,30 +11,9 @@ export type {
|
||||
Service as LLMClientService,
|
||||
} from "./route/client.js"
|
||||
export * from "./schema/index.js"
|
||||
export {
|
||||
ImageAspectRatio,
|
||||
ImageEvent,
|
||||
ImageModel,
|
||||
ImageModelSchema,
|
||||
ImageRequest,
|
||||
ImageResponse,
|
||||
ImageSize,
|
||||
} from "./image.js"
|
||||
export type {
|
||||
ImageFormat,
|
||||
ImageModelInput,
|
||||
ImageModelOptions,
|
||||
ImageOptions,
|
||||
ImageRequestFor,
|
||||
ImageRequestInput,
|
||||
ImageRoute,
|
||||
} from "./image.js"
|
||||
export { GeneratedImage, ImageInput, ImageInputSchema, ImageModel, ImageRequest, ImageResponse } from "./image.js"
|
||||
export type { ImageModelOptions, ImageOptions, ImageRequestFor, ImageRequestInput, ImageRoute } from "./image.js"
|
||||
export { Image } from "./image.js"
|
||||
export { Media } from "./media.js"
|
||||
export { ModelRef } from "./model-ref.js"
|
||||
export type { ResolveImageModel, ResolveLanguageModel } from "./model-ref.js"
|
||||
export { Job } from "./job.js"
|
||||
export type { Event as JobEvent, Poll, Route as JobRoute, Snapshot as JobSnapshot, Status as JobStatus } from "./job.js"
|
||||
export { Tool, ToolFailure, toDefinitions } from "./tool.js"
|
||||
export { ToolRuntime } from "./tool-runtime.js"
|
||||
export type { DispatchResult as ToolDispatchResult, ToolSettlement } from "./tool-runtime.js"
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
import { Duration, Effect, Schedule, Schema, Stream } from "effect"
|
||||
import { AIError, TimeoutError } from "./schema/errors.js"
|
||||
|
||||
export const Status = Schema.Literals(["queued", "running", "completed", "failed", "cancelled", "expired"])
|
||||
export type Status = Schema.Schema.Type<typeof Status>
|
||||
|
||||
/** Provider-neutral view of one job observation. */
|
||||
export interface Snapshot {
|
||||
readonly id: string
|
||||
readonly status: Status
|
||||
/** Normalized 0..1 when the provider reports progress. */
|
||||
readonly progress?: number
|
||||
readonly position?: number
|
||||
readonly expiresAt?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Route-owned job operations. `token` is the route's serializable handle (operation name, task id, response URL)
|
||||
* so a job can be resumed from another process; its shape is opaque to `Job`.
|
||||
*/
|
||||
export interface Route<Response> {
|
||||
readonly status: (token: unknown) => Effect.Effect<Snapshot, AIError>
|
||||
readonly result: (token: unknown) => Effect.Effect<Response, AIError>
|
||||
readonly cancel?: (token: unknown) => Effect.Effect<void, AIError>
|
||||
/** Provider polling hint (e.g. `openai-poll-after-ms`) that overrides the default interval for the next poll. */
|
||||
readonly pollHint?: (snapshot: Snapshot) => Duration.Duration | undefined
|
||||
}
|
||||
|
||||
export interface Poll {
|
||||
readonly interval?: Duration.Input
|
||||
readonly timeout?: Duration.Input
|
||||
/** Full override of the polling schedule; `interval` and `pollHint` are ignored when supplied. */
|
||||
readonly schedule?: Schedule.Schedule<unknown, Snapshot>
|
||||
}
|
||||
|
||||
export const DEFAULT_POLL_INTERVAL = Duration.seconds(5)
|
||||
export const DEFAULT_POLL_TIMEOUT = Duration.minutes(10)
|
||||
|
||||
export type Event =
|
||||
| { readonly type: "job-queued"; readonly id: string; readonly position?: number }
|
||||
| { readonly type: "job-progress"; readonly id: string; readonly progress?: number }
|
||||
| { readonly type: "job-finished"; readonly id: string; readonly status: Status }
|
||||
|
||||
const TERMINAL: ReadonlySet<Status> = new Set(["completed", "failed", "cancelled", "expired"])
|
||||
|
||||
export class Job<Response> {
|
||||
readonly id: string
|
||||
readonly status: Status
|
||||
readonly progress?: number
|
||||
readonly position?: number
|
||||
readonly expiresAt?: number
|
||||
|
||||
constructor(
|
||||
readonly route: Route<Response>,
|
||||
readonly token: unknown,
|
||||
snapshot: Snapshot,
|
||||
) {
|
||||
this.id = snapshot.id
|
||||
this.status = snapshot.status
|
||||
this.progress = snapshot.progress
|
||||
this.position = snapshot.position
|
||||
this.expiresAt = snapshot.expiresAt
|
||||
}
|
||||
|
||||
get snapshot(): Snapshot {
|
||||
return {
|
||||
id: this.id,
|
||||
status: this.status,
|
||||
progress: this.progress,
|
||||
position: this.position,
|
||||
expiresAt: this.expiresAt,
|
||||
}
|
||||
}
|
||||
|
||||
get terminal() {
|
||||
return TERMINAL.has(this.status)
|
||||
}
|
||||
|
||||
refresh(): Effect.Effect<Job<Response>, AIError> {
|
||||
return this.route.status(this.token).pipe(Effect.map((snapshot) => new Job(this.route, this.token, snapshot)))
|
||||
}
|
||||
|
||||
/** Poll until the job reaches a terminal status, then fetch the result. Fails with a `Timeout` reason on deadline. */
|
||||
await(options?: { readonly poll?: Poll }): Effect.Effect<Response, AIError> {
|
||||
const timeout = Duration.fromInputUnsafe(options?.poll?.timeout ?? DEFAULT_POLL_TIMEOUT)
|
||||
const settled = this.terminal ? Effect.succeed(this) : this.poll(options?.poll)
|
||||
return settled.pipe(
|
||||
// Non-completed terminal states also go through `result` so the route can surface its provider failure body.
|
||||
Effect.flatMap((job) => job.route.result(job.token)),
|
||||
Effect.timeoutOrElse({
|
||||
duration: timeout,
|
||||
orElse: () =>
|
||||
new AIError({
|
||||
reason: new TimeoutError({
|
||||
message: `Job ${this.id} did not finish within ${Duration.format(timeout)}`,
|
||||
timeoutMs: Duration.toMillis(timeout),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
cancel(): Effect.Effect<void, AIError> {
|
||||
return this.route.cancel?.(this.token) ?? Effect.void
|
||||
}
|
||||
|
||||
/** Status observations as a stream, ending after the first terminal observation. */
|
||||
events(options?: { readonly poll?: Poll }): Stream.Stream<Event, AIError> {
|
||||
const observations = this.terminal
|
||||
? Stream.make(this)
|
||||
: Stream.fromEffectSchedule(this.refresh(), this.schedule(options?.poll)).pipe(
|
||||
Stream.takeUntil((job) => job.terminal),
|
||||
)
|
||||
return observations.pipe(
|
||||
Stream.map((job): Event => {
|
||||
if (job.terminal) return { type: "job-finished", id: job.id, status: job.status }
|
||||
if (job.status === "queued") return { type: "job-queued", id: job.id, position: job.position }
|
||||
return { type: "job-progress", id: job.id, progress: job.progress }
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
private poll(poll: Poll | undefined) {
|
||||
return this.refresh().pipe(Effect.repeat({ schedule: this.schedule(poll), until: (job) => job.terminal }))
|
||||
}
|
||||
|
||||
private schedule(poll: Poll | undefined): Schedule.Schedule<unknown, Job<Response>> {
|
||||
if (poll?.schedule) return poll.schedule.pipe(Schedule.setInputType<Job<Response>>())
|
||||
const interval = poll?.interval ?? DEFAULT_POLL_INTERVAL
|
||||
const pollHint = this.route.pollHint
|
||||
const spaced = Schedule.spaced(interval).pipe(Schedule.setInputType<Job<Response>>())
|
||||
if (!pollHint) return spaced
|
||||
return spaced.pipe(
|
||||
Schedule.modifyDelay((metadata) => Effect.succeed(pollHint(metadata.input.snapshot) ?? interval)),
|
||||
)
|
||||
}
|
||||
}
|
||||
+17
-26
@@ -18,24 +18,20 @@ import {
|
||||
type ToolEntryInput,
|
||||
} from "./schema/index.js"
|
||||
import { make as makeTool, toDefinitions, type ToolSchema } from "./tool.js"
|
||||
import { ModelRef, type ResolveLanguageModel } from "./model-ref.js"
|
||||
|
||||
/** A concrete language model or a callable-facade ref whose provider exposes an LLM route. */
|
||||
export type LanguageModelInput = LanguageModel | ModelRef
|
||||
|
||||
/** Input accepted by `LLM.request`, normalized into the canonical `LLMRequest` class. */
|
||||
export type RequestInput<SelectedModel extends LanguageModelInput = LanguageModelInput> = Omit<
|
||||
export type RequestInput<SelectedLanguageModel extends LanguageModel = LanguageModel> = Omit<
|
||||
ConstructorParameters<typeof LLMRequest>[0],
|
||||
"model" | "system" | "messages" | "tools" | "toolChoice" | "generation" | "http" | "providerOptions"
|
||||
> & {
|
||||
readonly model: SelectedModel
|
||||
readonly model: SelectedLanguageModel
|
||||
readonly system?: string | SystemPart | ReadonlyArray<SystemPart>
|
||||
readonly prompt?: string | ContentPart | ReadonlyArray<ContentPart>
|
||||
readonly messages?: ReadonlyArray<Message | Message.Input>
|
||||
readonly tools?: ReadonlyArray<ToolEntryInput>
|
||||
readonly toolChoice?: ToolChoice.Input
|
||||
readonly generation?: GenerationOptions.Input
|
||||
readonly providerOptions?: NoInfer<LanguageModelProviderOptions<ResolveLanguageModel<SelectedModel>>>
|
||||
readonly providerOptions?: NoInfer<LanguageModelProviderOptions<SelectedLanguageModel>>
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
@@ -43,14 +39,10 @@ export const generate = LLMClient.generate
|
||||
|
||||
export const stream = LLMClient.stream
|
||||
|
||||
// The stored request keeps the concrete model; refs are resolved once here so nothing downstream knows about them.
|
||||
const resolveModel = (model: LanguageModelInput) => (model instanceof ModelRef ? model.facade.model(model.id) : model)
|
||||
|
||||
export const request = <const SelectedModel extends LanguageModelInput>(
|
||||
input: RequestInput<SelectedModel>,
|
||||
): LLMRequest<ResolveLanguageModel<SelectedModel>> => {
|
||||
export const request = <const SelectedLanguageModel extends LanguageModel>(
|
||||
input: RequestInput<SelectedLanguageModel>,
|
||||
) => {
|
||||
const {
|
||||
model,
|
||||
system: requestSystem,
|
||||
prompt,
|
||||
messages,
|
||||
@@ -61,9 +53,8 @@ export const request = <const SelectedModel extends LanguageModelInput>(
|
||||
http: requestHttp,
|
||||
...rest
|
||||
} = input
|
||||
return new LLMRequest<ResolveLanguageModel<SelectedModel>>({
|
||||
return new LLMRequest({
|
||||
...rest,
|
||||
model: resolveModel(model) as ResolveLanguageModel<SelectedModel>,
|
||||
system: SystemPart.content(requestSystem),
|
||||
messages: [...(messages?.map(Message.make) ?? []), ...(prompt === undefined ? [] : [Message.user(prompt)])],
|
||||
tools: tools?.map(ToolEntry.make) ?? [],
|
||||
@@ -78,8 +69,8 @@ const GENERATE_OBJECT_TOOL_NAME = "generate_object"
|
||||
|
||||
const GENERATE_OBJECT_TOOL_DESCRIPTION = "Return the structured result by calling this tool."
|
||||
|
||||
type GenerateObjectBase<SelectedModel extends LanguageModelInput = LanguageModelInput> = Omit<
|
||||
RequestInput<SelectedModel>,
|
||||
type GenerateObjectBase<SelectedLanguageModel extends LanguageModel = LanguageModel> = Omit<
|
||||
RequestInput<SelectedLanguageModel>,
|
||||
"tools" | "toolChoice"
|
||||
>
|
||||
|
||||
@@ -100,13 +91,13 @@ export class GenerateObjectResponse<T> {
|
||||
|
||||
export interface GenerateObjectOptions<
|
||||
S extends ToolSchema<any>,
|
||||
SelectedModel extends LanguageModelInput = LanguageModelInput,
|
||||
> extends GenerateObjectBase<SelectedModel> {
|
||||
SelectedLanguageModel extends LanguageModel = LanguageModel,
|
||||
> extends GenerateObjectBase<SelectedLanguageModel> {
|
||||
readonly schema: S
|
||||
}
|
||||
|
||||
export interface GenerateObjectDynamicOptions<SelectedModel extends LanguageModelInput = LanguageModelInput>
|
||||
extends GenerateObjectBase<SelectedModel> {
|
||||
export interface GenerateObjectDynamicOptions<SelectedLanguageModel extends LanguageModel = LanguageModel>
|
||||
extends GenerateObjectBase<SelectedLanguageModel> {
|
||||
/** Raw JSON Schema object describing the expected output shape. */
|
||||
readonly jsonSchema: JsonSchema.JsonSchema
|
||||
}
|
||||
@@ -156,11 +147,11 @@ const runGenerateObject = Effect.fn("LLM.generateObject")(function* (
|
||||
* 2. `jsonSchema: JsonSchema.JsonSchema` — `.object` is `unknown`. Use when
|
||||
* the schema is only available at runtime (MCP, plugin manifests). Caller validates.
|
||||
*/
|
||||
export function generateObject<const SelectedModel extends LanguageModelInput, S extends ToolSchema<any>>(
|
||||
options: GenerateObjectOptions<S, SelectedModel>,
|
||||
export function generateObject<const SelectedLanguageModel extends LanguageModel, S extends ToolSchema<any>>(
|
||||
options: GenerateObjectOptions<S, SelectedLanguageModel>,
|
||||
): Effect.Effect<GenerateObjectResponse<Schema.Schema.Type<S>>, AIError, Service>
|
||||
export function generateObject<const SelectedModel extends LanguageModelInput>(
|
||||
options: GenerateObjectDynamicOptions<SelectedModel>,
|
||||
export function generateObject<const SelectedLanguageModel extends LanguageModel>(
|
||||
options: GenerateObjectDynamicOptions<SelectedLanguageModel>,
|
||||
): Effect.Effect<GenerateObjectResponse<unknown>, AIError, Service>
|
||||
export function generateObject(options: GenerateObjectOptions<ToolSchema<any>> | GenerateObjectDynamicOptions) {
|
||||
if ("schema" in options) {
|
||||
|
||||
@@ -1,307 +0,0 @@
|
||||
export * as Media from "./media.js"
|
||||
|
||||
import { Effect, Encoding, FileSystem, Schema, SchemaGetter } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { ProviderID } from "./schema/ids.js"
|
||||
import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "./schema/errors.js"
|
||||
import { ProviderMetadata } from "./schema/options.js"
|
||||
import { Service } from "./route/executor-service.js"
|
||||
import { detectMediaType, extensionMediaType } from "./utils/media-type.js"
|
||||
|
||||
export { detectMediaType } from "./utils/media-type.js"
|
||||
|
||||
const OCTET_STREAM = "application/octet-stream"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Source — the serializable wire/persistence form of a media asset
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const BytesSource = Schema.Struct({
|
||||
type: Schema.Literal("bytes"),
|
||||
data: Schema.Uint8Array,
|
||||
mediaType: Schema.String,
|
||||
})
|
||||
|
||||
const Base64Source = Schema.Struct({
|
||||
type: Schema.Literal("base64"),
|
||||
data: Schema.String,
|
||||
mediaType: Schema.String,
|
||||
})
|
||||
|
||||
const UrlSource = Schema.Struct({
|
||||
type: Schema.Literal("url"),
|
||||
url: Schema.String,
|
||||
mediaType: Schema.optional(Schema.String),
|
||||
/** Epoch milliseconds after which the provider no longer serves the URL. */
|
||||
expiresAt: Schema.optional(Schema.Number),
|
||||
/** Headers required to fetch the URL, such as provider auth for Veo downloads. */
|
||||
headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
|
||||
})
|
||||
|
||||
/** A provider-side handle: OpenAI `file_id`, Gemini file URI, `gs://`, `runway://`, or a prior generation id. */
|
||||
const RefSource = Schema.Struct({
|
||||
type: Schema.Literal("ref"),
|
||||
provider: ProviderID,
|
||||
id: Schema.String,
|
||||
mediaType: Schema.optional(Schema.String),
|
||||
})
|
||||
|
||||
export const Source = Schema.Union([BytesSource, Base64Source, UrlSource, RefSource])
|
||||
.pipe(Schema.toTaggedUnion("type"))
|
||||
.annotate({ identifier: "Media.Source" })
|
||||
export type Source = Schema.Schema.Type<typeof Source>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Kind, Info, Notice
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const Kind = Schema.Literals(["image", "video", "audio", "document", "other"])
|
||||
export type Kind = Schema.Schema.Type<typeof Kind>
|
||||
|
||||
export const kindOf = (mediaType: string): Kind => {
|
||||
const lower = mediaType.toLowerCase()
|
||||
if (lower.startsWith("image/")) return "image"
|
||||
if (lower.startsWith("video/")) return "video"
|
||||
if (lower.startsWith("audio/")) return "audio"
|
||||
if (lower === "application/pdf" || lower.startsWith("text/")) return "document"
|
||||
return "other"
|
||||
}
|
||||
|
||||
/** Container-independent facts about the payload; raw PCM audio relies on these because it has no header. */
|
||||
export const Info = Schema.Struct({
|
||||
width: Schema.optional(Schema.Number),
|
||||
height: Schema.optional(Schema.Number),
|
||||
durationSeconds: Schema.optional(Schema.Number),
|
||||
sampleRate: Schema.optional(Schema.Number),
|
||||
channels: Schema.optional(Schema.Number),
|
||||
encoding: Schema.optional(Schema.String),
|
||||
format: Schema.optional(Schema.String),
|
||||
}).annotate({ identifier: "Media.Info" })
|
||||
export type Info = Schema.Schema.Type<typeof Info>
|
||||
|
||||
/** A provider-side partial result such as stripped audio or a moderated sample; never a silent drop. */
|
||||
export const Notice = Schema.Struct({
|
||||
type: Schema.Literals(["moderated", "filtered", "other"]),
|
||||
message: Schema.String,
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}).annotate({ identifier: "Media.Notice" })
|
||||
export type Notice = Schema.Schema.Type<typeof Notice>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Asset
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const invalid = (message: string, cause?: unknown) =>
|
||||
new AIError({ reason: new InvalidRequestError({ message, cause }) })
|
||||
|
||||
/** Synchronous view of an inline payload; `undefined` for `url` and `ref` sources, which carry no local bytes. */
|
||||
export interface Inline {
|
||||
readonly mime: string
|
||||
readonly base64: string
|
||||
readonly dataUrl: string
|
||||
}
|
||||
|
||||
export class Asset {
|
||||
readonly source: Source
|
||||
/** Derived from the source: declared type, sniffed magic bytes, then `application/octet-stream`. */
|
||||
readonly mediaType: string
|
||||
readonly kind: Kind
|
||||
readonly info?: Info
|
||||
/** Epoch milliseconds after which a `url` source stops resolving. */
|
||||
readonly expiresAt?: number
|
||||
readonly providerMetadata?: ProviderMetadata
|
||||
|
||||
// Derived payload forms are cached on the instance because every protocol lowering re-reads the same payload. The
|
||||
// cache is check-then-set (concurrent first reads of a `url` source may both download) and is never observable
|
||||
// through `source`, so round-tripping through `Media.from(asset.source)` stays lossless.
|
||||
#bytes: Uint8Array | undefined
|
||||
#base64: string | undefined
|
||||
|
||||
constructor(input: Asset.Input) {
|
||||
this.source = input.source
|
||||
this.mediaType =
|
||||
input.source.mediaType ??
|
||||
(input.source.type === "bytes" ? detectMediaType(input.source.data) : undefined) ??
|
||||
OCTET_STREAM
|
||||
this.kind = kindOf(this.mediaType)
|
||||
this.info = input.info
|
||||
this.expiresAt = input.source.type === "url" ? input.source.expiresAt : undefined
|
||||
this.providerMetadata = input.providerMetadata
|
||||
}
|
||||
|
||||
/** Inline payload without effects, for protocols that embed base64 or data URLs directly. */
|
||||
inline(): Inline | undefined {
|
||||
const source = this.source
|
||||
if (source.type !== "bytes" && source.type !== "base64") return undefined
|
||||
const base64 = source.type === "base64" ? source.data : (this.#base64 ??= Encoding.encodeBase64(source.data))
|
||||
const mime = this.mediaType.toLowerCase()
|
||||
return { mime, base64, dataUrl: `data:${mime};base64,${base64}` }
|
||||
}
|
||||
|
||||
/** Decoded payload; downloads `url` sources through the request executor and caches the result. */
|
||||
bytes(): Effect.Effect<Uint8Array, AIError, Service> {
|
||||
return Effect.suspend(() => {
|
||||
const source = this.source
|
||||
if (source.type === "bytes") return Effect.succeed(source.data)
|
||||
if (this.#bytes !== undefined) return Effect.succeed(this.#bytes)
|
||||
if (source.type === "ref")
|
||||
return Effect.fail(invalid(`Cannot materialize provider ref ${source.provider}:${source.id}`))
|
||||
const decoded =
|
||||
source.type === "base64"
|
||||
? Effect.fromResult(Encoding.decodeBase64(source.data)).pipe(
|
||||
Effect.mapError((cause) => invalid(`Media asset contains invalid base64 data`, cause)),
|
||||
)
|
||||
: download(source)
|
||||
return decoded.pipe(Effect.tap((data) => Effect.sync(() => (this.#bytes = data))))
|
||||
})
|
||||
}
|
||||
|
||||
base64(): Effect.Effect<string, AIError, Service> {
|
||||
return Effect.suspend(() => {
|
||||
const source = this.source
|
||||
if (source.type === "base64") return Effect.succeed(source.data)
|
||||
if (this.#base64 !== undefined) return Effect.succeed(this.#base64)
|
||||
return this.bytes().pipe(Effect.map((data) => (this.#base64 = Encoding.encodeBase64(data))))
|
||||
})
|
||||
}
|
||||
|
||||
dataUrl(): Effect.Effect<string, AIError, Service> {
|
||||
return this.base64().pipe(Effect.map((data) => `data:${this.mediaType};base64,${data}`))
|
||||
}
|
||||
|
||||
/**
|
||||
* The `AssetEncoded` JSON form with `bytes` sources as base64, matching `Schema.toCodecJson(AssetSchema)`, so a
|
||||
* plain `JSON.stringify` of messages or events stays lossless and decodes back through the JSON codec.
|
||||
*/
|
||||
toJSON() {
|
||||
const source = this.source
|
||||
return {
|
||||
source: source.type === "bytes" ? { ...source, data: Encoding.encodeBase64(source.data) } : source,
|
||||
info: this.info,
|
||||
providerMetadata: this.providerMetadata,
|
||||
}
|
||||
}
|
||||
|
||||
/** Pull `url` sources into owned bytes before the URL expires. Inline sources return themselves. */
|
||||
materialize(): Effect.Effect<Asset, AIError, Service> {
|
||||
if (this.source.type === "bytes" || this.source.type === "base64") return Effect.succeed(this)
|
||||
return this.bytes().pipe(
|
||||
Effect.map((data) =>
|
||||
bytes(data, this.source.mediaType, { info: this.info, providerMetadata: this.providerMetadata }),
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export namespace Asset {
|
||||
export interface Input {
|
||||
readonly source: Source
|
||||
readonly info?: Info
|
||||
readonly providerMetadata?: ProviderMetadata
|
||||
}
|
||||
}
|
||||
|
||||
/** JSON form of an asset: the serializable `Source` plus caller-supplied metadata. `bytes` sources encode as base64. */
|
||||
export const AssetEncoded = Schema.Struct({
|
||||
source: Source,
|
||||
info: Schema.optional(Info),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}).annotate({ identifier: "Media.AssetEncoded" })
|
||||
|
||||
const encodeAsset = (asset: Asset): typeof AssetEncoded.Type => ({
|
||||
source: asset.source,
|
||||
info: asset.info,
|
||||
providerMetadata: asset.providerMetadata,
|
||||
})
|
||||
|
||||
const AssetInstance = Schema.declare((value): value is Asset => value instanceof Asset, {
|
||||
expected: "Media.Asset",
|
||||
})
|
||||
|
||||
/** `Asset` in the type domain and `AssetEncoded` on the wire, so messages and events holding assets serialize. */
|
||||
export const AssetSchema = AssetEncoded.pipe(
|
||||
Schema.decodeTo(AssetInstance, {
|
||||
decode: SchemaGetter.transform((encoded) => new Asset(encoded)),
|
||||
encode: SchemaGetter.transform(encodeAsset),
|
||||
}),
|
||||
)
|
||||
|
||||
const download = Effect.fn("Media.download")(function* (source: Extract<Source, { readonly type: "url" }>) {
|
||||
const executor = yield* Service
|
||||
const response = yield* executor.execute(
|
||||
HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(source.headers ?? {})),
|
||||
)
|
||||
const buffer = yield* response.arrayBuffer.pipe(
|
||||
Effect.mapError(
|
||||
(cause) =>
|
||||
new AIError({
|
||||
reason: new InvalidProviderOutputError({
|
||||
message: `Failed to read media from ${source.url}`,
|
||||
http: new HttpContext({ url: response.request.url, status: response.status, headers: response.headers }),
|
||||
cause,
|
||||
}),
|
||||
}),
|
||||
),
|
||||
)
|
||||
return new Uint8Array(buffer)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constructors
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type AssetOptions = Omit<Asset.Input, "source">
|
||||
|
||||
export const from = (source: Source, options?: AssetOptions) => new Asset({ ...options, source })
|
||||
|
||||
export const bytes = (data: Uint8Array, mediaType?: string, options?: AssetOptions) =>
|
||||
from({ type: "bytes", data, mediaType: mediaType ?? detectMediaType(data) ?? OCTET_STREAM }, options)
|
||||
|
||||
export const base64 = (data: string, mediaType: string, options?: AssetOptions) =>
|
||||
from({ type: "base64", data, mediaType }, options)
|
||||
|
||||
export const url = (
|
||||
value: string,
|
||||
options?: AssetOptions & Omit<Extract<Source, { readonly type: "url" }>, "type" | "url">,
|
||||
) => {
|
||||
const { mediaType, expiresAt, headers, ...rest } = options ?? {}
|
||||
return from({ type: "url", url: value, mediaType, expiresAt, headers }, rest)
|
||||
}
|
||||
|
||||
export const ref = (provider: string | ProviderID, id: string, mediaType?: string, options?: AssetOptions) =>
|
||||
from({ type: "ref", provider: ProviderID.make(provider), id, mediaType }, options)
|
||||
|
||||
const DATA_URL = /^data:([^;,]+)(?:;[^,]*)*;base64,(.*)$/s
|
||||
|
||||
/** Parse a `data:<mime>;base64,<data>` URL, or `undefined` when the value is not a base64 data URL. */
|
||||
export const parseDataUrl = (value: string, options?: AssetOptions) => {
|
||||
const match = DATA_URL.exec(value)
|
||||
return match === null ? undefined : base64(match[2], match[1], options)
|
||||
}
|
||||
|
||||
/** Parse a `data:<mime>;base64,<data>` URL. Malformed input throws a typed `AIError` because constructors are sync. */
|
||||
export const fromDataUrl = (dataUrl: string, options?: AssetOptions) => {
|
||||
const asset = parseDataUrl(dataUrl, options)
|
||||
if (asset === undefined) throw invalid("Media data URLs must contain a MIME type and base64 data")
|
||||
return asset
|
||||
}
|
||||
|
||||
/** Read a file through `FileSystem` and sniff its media type from magic bytes, then the extension. */
|
||||
export const file = (path: string, options?: AssetOptions): Effect.Effect<Asset, AIError, FileSystem.FileSystem> =>
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* FileSystem.FileSystem
|
||||
const data = yield* fs
|
||||
.readFile(path)
|
||||
.pipe(Effect.mapError((cause) => invalid(`Failed to read media file ${path}`, cause)))
|
||||
return bytes(data, detectMediaType(data) ?? extensionMediaType(path), options)
|
||||
})
|
||||
|
||||
/** Materialize an asset and write its bytes through `FileSystem`. */
|
||||
export const write = (asset: Asset, path: string): Effect.Effect<void, AIError, FileSystem.FileSystem | Service> =>
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* FileSystem.FileSystem
|
||||
const data = yield* asset.bytes()
|
||||
yield* fs
|
||||
.writeFile(path, data)
|
||||
.pipe(Effect.mapError((cause) => invalid(`Failed to write media file ${path}`, cause)))
|
||||
})
|
||||
@@ -1,54 +0,0 @@
|
||||
import { ModelID, type ProviderID } from "./schema/ids.js"
|
||||
import type { LanguageModel } from "./schema/options.js"
|
||||
import type { ImageModel } from "./image.js"
|
||||
|
||||
/**
|
||||
* A provider-scoped model name bound to its configured facade. Callable facades return one of these so
|
||||
* `LLM.request({ model: openai("gpt-5") })` and `Image.request({ model: openai("gpt-image-2") })` name a model once
|
||||
* and let the request namespace pick the selector (`facade.model`, `facade.image`, …) on demand.
|
||||
*/
|
||||
export class ModelRef<S extends ModelRef.Selectors = ModelRef.Selectors> {
|
||||
constructor(
|
||||
readonly id: ModelID,
|
||||
readonly facade: S,
|
||||
) {}
|
||||
|
||||
get provider() {
|
||||
return this.facade.id
|
||||
}
|
||||
}
|
||||
|
||||
export namespace ModelRef {
|
||||
/** The named selectors a configured provider facade exposes; `model` is the default LLM route. */
|
||||
export interface Selectors {
|
||||
readonly id: ProviderID
|
||||
// oxlint-disable-next-line typescript-eslint/no-explicit-any
|
||||
readonly model: (id: ModelID) => LanguageModel<any, any>
|
||||
// oxlint-disable-next-line typescript-eslint/no-explicit-any
|
||||
readonly image?: (id: ModelID) => ImageModel<any>
|
||||
}
|
||||
|
||||
/** A ref whose provider exposes an image route. */
|
||||
// oxlint-disable-next-line typescript-eslint/no-explicit-any
|
||||
export type WithImage = ModelRef<Selectors & { readonly image: (id: ModelID) => ImageModel<any> }>
|
||||
|
||||
/** Make a configured facade callable: `openai("gpt-5")` returns a `ModelRef` over the facade's own selectors. */
|
||||
export const facade = <const S extends Selectors>(selectors: S) =>
|
||||
Object.assign((id: string | ModelID) => new ModelRef(ModelID.make(id), selectors), selectors)
|
||||
}
|
||||
|
||||
/** The concrete `LanguageModel` a request will run against: either the model itself or the ref's `model` selector. */
|
||||
// oxlint-disable-next-line typescript-eslint/no-explicit-any
|
||||
export type ResolveLanguageModel<Model> =
|
||||
Model extends LanguageModel<any, any> ? Model : Model extends ModelRef<infer S> ? ReturnType<S["model"]> : never
|
||||
|
||||
/** The concrete `ImageModel` a request will run against: either the model itself or the ref's `image` selector. */
|
||||
// oxlint-disable-next-line typescript-eslint/no-explicit-any
|
||||
export type ResolveImageModel<Model> =
|
||||
Model extends ImageModel<any>
|
||||
? Model
|
||||
: Model extends ModelRef<infer S>
|
||||
? S extends { readonly image: (id: ModelID) => infer Selected }
|
||||
? Selected
|
||||
: never
|
||||
: never
|
||||
@@ -1,100 +0,0 @@
|
||||
import { Effect, Layer, ManagedRuntime, Stream } from "effect"
|
||||
import { Image, ImageRequest, type ImageModelInput, type ImageRequestInput } from "./image.js"
|
||||
import { ImageClient } from "./image-client.js"
|
||||
import { LLM } from "./index.js"
|
||||
import { LLMClient } from "./route/client.js"
|
||||
import { RequestExecutor } from "./route/executor.js"
|
||||
import { LLMRequest } from "./schema/index.js"
|
||||
import type { LanguageModelInput, RequestInput } from "./llm.js"
|
||||
|
||||
/**
|
||||
* Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM and image clients
|
||||
* over a request executor; every method runs the corresponding Effect API and rethrows `AIError` unchanged.
|
||||
*/
|
||||
export interface Options {
|
||||
/** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or middleware here. */
|
||||
readonly layer?: Layer.Layer<RequestExecutor.Service>
|
||||
}
|
||||
|
||||
export interface RunOptions {
|
||||
readonly signal?: AbortSignal
|
||||
}
|
||||
|
||||
export type Services =
|
||||
| Layer.Success<typeof LLMClient.layer>
|
||||
| Layer.Success<typeof ImageClient.layer>
|
||||
| RequestExecutor.Service
|
||||
|
||||
const abortEffect = (signal: AbortSignal | undefined) =>
|
||||
signal === undefined
|
||||
? Effect.never
|
||||
: Effect.callback<void>((resume) => {
|
||||
if (signal.aborted) {
|
||||
resume(Effect.void)
|
||||
return
|
||||
}
|
||||
const onAbort = () => resume(Effect.void)
|
||||
signal.addEventListener("abort", onAbort, { once: true })
|
||||
return Effect.sync(() => signal.removeEventListener("abort", onAbort))
|
||||
})
|
||||
|
||||
export const make = (options: Options = {}) => {
|
||||
const runtime = ManagedRuntime.make(
|
||||
Layer.mergeAll(LLMClient.layer, ImageClient.layer).pipe(
|
||||
Layer.provideMerge(options.layer ?? RequestExecutor.fetchLayer),
|
||||
),
|
||||
)
|
||||
|
||||
/** Run any package Effect (for example `asset.bytes()`) inside this runtime. */
|
||||
const run = <A, E>(effect: Effect.Effect<A, E, Services>, options?: RunOptions) =>
|
||||
runtime.runPromise(effect, { signal: options?.signal })
|
||||
|
||||
const iterate = <A, E>(stream: Stream.Stream<A, E, Services>, options?: RunOptions): AsyncIterable<A> =>
|
||||
Stream.toAsyncIterable(
|
||||
Stream.unwrap(
|
||||
runtime.contextEffect.pipe(
|
||||
Effect.map(
|
||||
(context): Stream.Stream<A, E> =>
|
||||
stream.pipe(Stream.interruptWhen(abortEffect(options?.signal)), Stream.provideContext(context)),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
// The typed `generate`/`stream` overloads take a concrete input or a request, not the union; normalize once here.
|
||||
const llmRequest = (input: RequestInput | LLMRequest) => (input instanceof LLMRequest ? input : LLM.request(input))
|
||||
const imageRequest = (input: ImageRequestInput | ImageRequest) =>
|
||||
input instanceof ImageRequest ? input : Image.request(input)
|
||||
|
||||
return {
|
||||
run,
|
||||
llm: {
|
||||
request: LLM.request,
|
||||
generate: <const Model extends LanguageModelInput>(
|
||||
input: RequestInput<Model> | LLMRequest,
|
||||
options?: RunOptions,
|
||||
) => run(LLM.generate(llmRequest(input)), options),
|
||||
stream: <const Model extends LanguageModelInput>(input: RequestInput<Model> | LLMRequest, options?: RunOptions) =>
|
||||
iterate(LLM.stream(llmRequest(input)), options),
|
||||
},
|
||||
image: {
|
||||
request: Image.request,
|
||||
generate: <const Model extends ImageModelInput>(
|
||||
input: ImageRequestInput<Model> | ImageRequest,
|
||||
options?: RunOptions,
|
||||
) => run(Image.generate(imageRequest(input)), options),
|
||||
stream: <const Model extends ImageModelInput>(
|
||||
input: ImageRequestInput<Model> | ImageRequest,
|
||||
options?: RunOptions,
|
||||
) => iterate(Image.stream(imageRequest(input)), options),
|
||||
},
|
||||
dispose: () => runtime.dispose(),
|
||||
}
|
||||
}
|
||||
|
||||
export type Client = ReturnType<typeof make>
|
||||
|
||||
/** Default client over `RequestExecutor.fetchLayer` for scripts; the runtime builds its layer on first use. */
|
||||
export const ai = make()
|
||||
|
||||
export * as AI from "./promise.js"
|
||||
@@ -660,7 +660,7 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (
|
||||
part: MediaPart,
|
||||
breakpoints?: Cache.Breakpoints,
|
||||
) {
|
||||
const mime = part.media.mediaType.toLowerCase()
|
||||
const mime = part.mediaType.toLowerCase()
|
||||
const cacheControlValue = breakpoints ? cacheControl(breakpoints, part.cache) : undefined
|
||||
const fileId = fileIdFromMetadata(part.metadata)
|
||||
|
||||
@@ -689,9 +689,9 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (
|
||||
} satisfies AnthropicDocumentBlock
|
||||
}
|
||||
|
||||
const rawString = ProviderShared.mediaUrl(part.media)?.trim()
|
||||
const rawString = typeof part.data === "string" ? part.data.trim() : undefined
|
||||
// SDK URL sources: URLImageSource:3817 / URLPDFSource:3823 {type:"url", url}
|
||||
if (rawString && isHttpUrl(rawString)) {
|
||||
if (rawString && isHttpUrl(rawString) && !rawString.startsWith("data:")) {
|
||||
if (mime.startsWith("image/"))
|
||||
return {
|
||||
type: "image" as const,
|
||||
@@ -716,11 +716,20 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (
|
||||
} satisfies AnthropicDocumentBlock
|
||||
}
|
||||
|
||||
const media = yield* ProviderShared.requireInlineMedia("Anthropic Messages", part.media)
|
||||
|
||||
// SDK PlainTextSource:2716 {type:"text", media_type:"text/plain", data}
|
||||
if (mime === "text/plain") {
|
||||
const textData = Buffer.from(media.base64, "base64").toString("utf8")
|
||||
const textData =
|
||||
typeof part.data !== "string"
|
||||
? Buffer.from(part.data).toString("utf8")
|
||||
: part.data.startsWith("data:")
|
||||
? (() => {
|
||||
const comma = part.data.indexOf(",")
|
||||
const payload = comma >= 0 ? part.data.slice(comma + 1) : part.data
|
||||
return part.data.includes(";base64")
|
||||
? Buffer.from(payload, "base64").toString("utf8")
|
||||
: decodeURIComponent(payload)
|
||||
})()
|
||||
: part.data
|
||||
return {
|
||||
type: "document" as const,
|
||||
source: { type: "text" as const, media_type: "text/plain" as const, data: textData },
|
||||
@@ -735,6 +744,7 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (
|
||||
} satisfies AnthropicDocumentBlock
|
||||
}
|
||||
|
||||
const media = ProviderShared.normalizeMedia(part)
|
||||
if (media.mime === "application/pdf")
|
||||
return {
|
||||
type: "document" as const,
|
||||
@@ -753,7 +763,7 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (
|
||||
: { citations: citationsFromMetadata(part.metadata)! }),
|
||||
} satisfies AnthropicDocumentBlock
|
||||
if (!media.mime.startsWith("image/"))
|
||||
return yield* invalid(`Anthropic Messages does not support media type ${part.media.mediaType}`)
|
||||
return yield* invalid(`Anthropic Messages does not support media type ${part.mediaType}`)
|
||||
return {
|
||||
type: "image" as const,
|
||||
source: {
|
||||
@@ -772,7 +782,7 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (
|
||||
// content instead of JSON-stringifying base64 into a prompt string.
|
||||
const lowerToolResultContentItem = Effect.fnUntraced(function* (item: Tool.Content) {
|
||||
if (item.type === "text") return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock
|
||||
return yield* lowerMedia(ProviderShared.toolFileMedia(item))
|
||||
return yield* lowerMedia({ type: "media", mediaType: item.mime, data: item.uri, filename: item.name })
|
||||
})
|
||||
|
||||
const lowerToolResultContent = Effect.fnUntraced(function* (part: ToolResultPart) {
|
||||
|
||||
@@ -303,7 +303,15 @@ const lowerToolResultContent = Effect.fn("BedrockConverse.lowerToolResultContent
|
||||
content.push({ text: item.text })
|
||||
continue
|
||||
}
|
||||
const media = yield* BedrockMedia.lower(ProviderShared.toolFileMedia(item), documentNames)
|
||||
const media = yield* BedrockMedia.lower(
|
||||
{
|
||||
type: "media",
|
||||
mediaType: item.mime,
|
||||
data: item.uri,
|
||||
filename: item.name,
|
||||
},
|
||||
documentNames,
|
||||
)
|
||||
content.push(...media)
|
||||
}
|
||||
return content
|
||||
|
||||
@@ -19,7 +19,6 @@ import {
|
||||
type ToolDefinition,
|
||||
} from "../schema/index.js"
|
||||
import { classifyProviderFailure } from "../provider-error.js"
|
||||
import { Media } from "../media.js"
|
||||
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
|
||||
import { GeminiToolSchema } from "./utils/gemini-tool-schema.js"
|
||||
import { Lifecycle } from "./utils/lifecycle.js"
|
||||
@@ -101,18 +100,9 @@ const GeminiInlineDataPart = Schema.Struct({
|
||||
mimeType: Schema.String,
|
||||
data: Schema.String,
|
||||
}),
|
||||
thoughtSignature: optionalNull(Schema.String),
|
||||
})
|
||||
type GeminiInlineDataPart = Schema.Schema.Type<typeof GeminiInlineDataPart>
|
||||
|
||||
/** Gemini Files API reference; the only remote input Gemini accepts. */
|
||||
const GeminiFileDataPart = Schema.Struct({
|
||||
fileData: Schema.Struct({
|
||||
mimeType: Schema.String,
|
||||
fileUri: Schema.String,
|
||||
}),
|
||||
})
|
||||
|
||||
const GeminiFunctionCallPart = Schema.Struct({
|
||||
functionCall: Schema.Struct({
|
||||
id: optionalNull(Schema.String),
|
||||
@@ -134,7 +124,6 @@ const GeminiFunctionResponsePart = Schema.Struct({
|
||||
const GeminiContentPart = Schema.Union([
|
||||
GeminiTextPart,
|
||||
GeminiInlineDataPart,
|
||||
GeminiFileDataPart,
|
||||
GeminiFunctionCallPart,
|
||||
GeminiFunctionResponsePart,
|
||||
])
|
||||
@@ -298,13 +287,9 @@ const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
|
||||
tool: (name) => ({ functionCallingConfig: { mode: "ANY" as const, allowedFunctionNames: [name] } }),
|
||||
})
|
||||
|
||||
// Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs.
|
||||
const lowerContentPart = Effect.fn("Gemini.lowerContentPart")(function* (part: TextPart | MediaPart) {
|
||||
const lowerUserPart = Effect.fn("Gemini.lowerUserPart")(function* (part: TextPart | MediaPart) {
|
||||
if (part.type === "text") return { text: part.text }
|
||||
const source = part.media.source
|
||||
if (source.type === "ref" && source.provider === "google")
|
||||
return { fileData: { mimeType: part.media.mediaType, fileUri: source.id } }
|
||||
const media = yield* ProviderShared.requireInlineMedia("Gemini", part.media)
|
||||
const media = ProviderShared.normalizeMedia(part)
|
||||
return { inlineData: { mimeType: media.mime, data: media.base64 } }
|
||||
})
|
||||
|
||||
@@ -352,7 +337,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
|
||||
for (const part of message.content) {
|
||||
if (!ProviderShared.supportsContent(part, ["text", "media"]))
|
||||
return yield* ProviderShared.unsupportedContent("Gemini", "user", ["text", "media"])
|
||||
parts.push(yield* lowerContentPart(part))
|
||||
parts.push(yield* lowerUserPart(part))
|
||||
}
|
||||
contents.push({ role: "user", parts })
|
||||
continue
|
||||
@@ -363,23 +348,12 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
|
||||
// Parallel Gemini 3 calls may carry one signature on the first call; unsigned sibling calls are valid.
|
||||
let hasSignedToolCall = false
|
||||
for (const part of message.content) {
|
||||
if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call", "media"]))
|
||||
return yield* ProviderShared.unsupportedContent("Gemini", "assistant", [
|
||||
"text",
|
||||
"reasoning",
|
||||
"tool-call",
|
||||
"media",
|
||||
])
|
||||
if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call"]))
|
||||
return yield* ProviderShared.unsupportedContent("Gemini", "assistant", ["text", "reasoning", "tool-call"])
|
||||
if (part.type === "text") {
|
||||
parts.push({ text: part.text, thoughtSignature: thoughtSignature(part.providerMetadata, metadataKey) })
|
||||
continue
|
||||
}
|
||||
// Generated images replay as model-role inline data so multi-turn image editing keeps the prior output.
|
||||
if (part.type === "media") {
|
||||
const lowered = yield* lowerContentPart(part)
|
||||
parts.push({ ...lowered, thoughtSignature: thoughtSignature(part.providerMetadata, metadataKey) })
|
||||
continue
|
||||
}
|
||||
if (part.type === "reasoning") {
|
||||
parts.push({
|
||||
text: part.text,
|
||||
@@ -429,7 +403,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
|
||||
const media: GeminiInlineDataPart[] = []
|
||||
for (const item of content) {
|
||||
if (item.type === "text") continue
|
||||
const value = yield* ProviderShared.requireInlineMedia("Gemini", ProviderShared.toolFileMedia(item).media)
|
||||
const value = ProviderShared.normalizeToolFile(item)
|
||||
media.push({ inlineData: { mimeType: value.mime, data: value.base64 } })
|
||||
}
|
||||
if (legacyToolMedia && media.length > 0) (pendingMedia ??= []).push(...media)
|
||||
@@ -711,19 +685,6 @@ const step = (state: ParserState, event: GeminiEvent) => {
|
||||
// each block kind must retain the signature attached to its own parts.
|
||||
if (signature !== undefined && "thought" in part && part.thought) reasoningSignature = signature
|
||||
else if (signature !== undefined && "text" in part) textSignature = signature
|
||||
// Image-capable Gemini models return generated images as inline data parts; surface them as first-class output.
|
||||
if ("inlineData" in part) {
|
||||
lifecycle = Lifecycle.stepStart(lifecycle, events)
|
||||
events.push(
|
||||
LLMEvent.media({
|
||||
media: Media.base64(part.inlineData.data, part.inlineData.mimeType),
|
||||
providerMetadata: signature
|
||||
? providerMetadata(state.providerMetadataKey, { thoughtSignature: signature })
|
||||
: undefined,
|
||||
}),
|
||||
)
|
||||
continue
|
||||
}
|
||||
if ("text" in part && part.text.length > 0) {
|
||||
if (part.thought) {
|
||||
if (textId !== undefined) {
|
||||
|
||||
@@ -1,36 +1,40 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
|
||||
import { Effect, Encoding, Schema } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import {
|
||||
GeneratedImage,
|
||||
ImageModel,
|
||||
ImageResponse,
|
||||
type ImageInput,
|
||||
type ImageRequestFor,
|
||||
type ImageRoute,
|
||||
} from "../image.js"
|
||||
import { Auth, type Definition as AuthDefinition } from "../route/auth.js"
|
||||
import { AIError, Usage, mergeHttpOptions, mergeJsonRecords, type HttpOptions } from "../schema/index.js"
|
||||
import { ProviderShared } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
import { ImageInputs } from "./utils/image-input.js"
|
||||
|
||||
const ADAPTER = "google-images"
|
||||
const NAME = "Google Images"
|
||||
const PROVIDER = ProviderID.make("google")
|
||||
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type GoogleImageString<Known extends string> = Known | (string & {})
|
||||
|
||||
/** Provider-native options. Common fields (`aspectRatio`, `seed`, `images`) live on the request. */
|
||||
export type GoogleImageOptions = {
|
||||
readonly aspectRatio?: GoogleImageString<
|
||||
"1:1" | "2:3" | "3:2" | "3:4" | "4:3" | "4:5" | "5:4" | "9:16" | "16:9" | "21:9"
|
||||
>
|
||||
readonly imageSize?: GoogleImageString<"1K" | "2K" | "4K">
|
||||
readonly seed?: number
|
||||
readonly thinkingLevel?: GoogleImageString<"MINIMAL" | "LOW" | "MEDIUM" | "HIGH">
|
||||
readonly includeThoughts?: boolean
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = ImageRequestFor<GoogleImageOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Response schema
|
||||
// ---------------------------------------------------------------------------
|
||||
export type GoogleImageBody = Record<string, unknown> & {
|
||||
readonly contents: ReadonlyArray<{
|
||||
readonly role: "user"
|
||||
readonly parts: ReadonlyArray<Record<string, unknown>>
|
||||
}>
|
||||
readonly generationConfig: Record<string, unknown>
|
||||
}
|
||||
|
||||
const GoogleUsage = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
@@ -81,20 +85,30 @@ const GoogleImageResponse = Schema.Struct({
|
||||
promptFeedback: Schema.optional(Schema.Unknown),
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
export interface ModelInput {
|
||||
readonly id: string
|
||||
readonly auth: AuthDefinition
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions
|
||||
}
|
||||
|
||||
const generationConfig = (request: Request) => {
|
||||
const { imageSize, thinkingLevel, includeThoughts, ...native } = request.providerOptions ?? {}
|
||||
const imageConfig = { aspectRatio: request.aspectRatio, imageSize }
|
||||
const thinkingConfig = { thinkingLevel, includeThoughts }
|
||||
const nativeOptions = (options: GoogleImageOptions | undefined) => {
|
||||
const { aspectRatio, imageSize, seed, thinkingLevel, includeThoughts, ...native } = options ?? {}
|
||||
const image = {
|
||||
aspectRatio,
|
||||
imageSize,
|
||||
}
|
||||
const thinkingConfig = {
|
||||
thinkingLevel,
|
||||
includeThoughts,
|
||||
}
|
||||
return (
|
||||
mergeJsonRecords(
|
||||
{
|
||||
responseModalities: ["IMAGE"],
|
||||
imageConfig: Object.values(imageConfig).some((value) => value !== undefined) ? imageConfig : undefined,
|
||||
seed: request.seed,
|
||||
imageConfig: Object.values(image).some((value) => value !== undefined) ? image : undefined,
|
||||
seed,
|
||||
thinkingConfig: Object.values(thinkingConfig).some((value) => value !== undefined) ? thinkingConfig : undefined,
|
||||
},
|
||||
native,
|
||||
@@ -102,208 +116,176 @@ const generationConfig = (request: Request) => {
|
||||
)
|
||||
}
|
||||
|
||||
// Gemini does not fetch public URLs; inline payloads or Gemini Files references are the only accepted inputs.
|
||||
const imagePart = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
|
||||
const inline = asset.inline()
|
||||
if (inline) return Effect.succeed({ inlineData: { mimeType: inline.mime, data: inline.base64 } })
|
||||
const id = MediaInput.refID(asset, PROVIDER)
|
||||
if (id) return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri: id } })
|
||||
if (asset.source.type === "ref")
|
||||
return Effect.fail(
|
||||
ProviderShared.invalidRequest(
|
||||
"Google generateContent requires Gemini file references rather than other providers' file IDs",
|
||||
),
|
||||
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
|
||||
if (!query) return url
|
||||
const next = new URL(url)
|
||||
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
|
||||
return next.toString()
|
||||
}
|
||||
|
||||
export const model = (input: ModelInput) => {
|
||||
const route: ImageRoute<GoogleImageOptions> = {
|
||||
id: ADAPTER,
|
||||
generate: Effect.fn("GoogleImages.generate")(function* (request: ImageRequestFor<GoogleImageOptions>, execute) {
|
||||
const imageParts = yield* Effect.forEach(request.images ?? [], googleImagePart)
|
||||
const http = mergeHttpOptions(request.model.http, request.http)
|
||||
const requestBody = mergeJsonRecords(
|
||||
{
|
||||
contents: [{ role: "user", parts: [{ text: request.prompt }, ...imageParts] }],
|
||||
generationConfig: nativeOptions(request.options),
|
||||
},
|
||||
http?.body,
|
||||
) as GoogleImageBody
|
||||
const text = ProviderShared.encodeJson(requestBody)
|
||||
const url = applyQuery(
|
||||
`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}/models/${request.model.id}:generateContent`,
|
||||
http?.query,
|
||||
)
|
||||
const headers = yield* Auth.toEffect(input.auth)({
|
||||
request,
|
||||
method: "POST",
|
||||
url,
|
||||
body: text,
|
||||
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
|
||||
})
|
||||
const response = yield* execute(
|
||||
HttpClientRequest.post(url).pipe(
|
||||
HttpClientRequest.setHeaders(headers),
|
||||
HttpClientRequest.bodyText(text, "application/json"),
|
||||
),
|
||||
)
|
||||
const output = yield* ProviderShared.imageResponse(ADAPTER, "Google Images", response)
|
||||
const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(GoogleImageResponse))(output.body).pipe(
|
||||
Effect.mapError((cause) => output.invalid("Google Images returned an invalid response", cause)),
|
||||
)
|
||||
const candidates = decoded.candidates ?? []
|
||||
const candidateMetadata = candidates.map((candidate, candidateIndex) => ({
|
||||
index: candidate.index ?? candidateIndex,
|
||||
finishReason: candidate.finishReason,
|
||||
finishMessage: candidate.finishMessage,
|
||||
safetyRatings: candidate.safetyRatings,
|
||||
citationMetadata: candidate.citationMetadata,
|
||||
groundingMetadata: candidate.groundingMetadata,
|
||||
parts: (candidate.content?.parts ?? []).map((part) =>
|
||||
part.inlineData === undefined
|
||||
? {
|
||||
type: "text",
|
||||
text: part.text,
|
||||
thought: part.thought,
|
||||
thoughtSignature: part.thoughtSignature,
|
||||
}
|
||||
: {
|
||||
type: "inlineData",
|
||||
mediaType: part.inlineData.mimeType,
|
||||
thought: part.thought,
|
||||
thoughtSignature: part.thoughtSignature,
|
||||
},
|
||||
),
|
||||
}))
|
||||
const encoded = candidates.flatMap((candidate, candidateIndex) =>
|
||||
(candidate.content?.parts ?? []).flatMap((part, partIndex) =>
|
||||
part.inlineData === undefined || part.thought === true
|
||||
? []
|
||||
: [{ candidate, candidateIndex, partIndex, inlineData: part.inlineData }],
|
||||
),
|
||||
)
|
||||
const images = yield* Effect.forEach(encoded, (item) =>
|
||||
Effect.fromResult(Encoding.decodeBase64(item.inlineData.data)).pipe(
|
||||
Effect.mapError((cause) =>
|
||||
output.invalid(
|
||||
`Google Images candidate ${item.candidateIndex} part ${item.partIndex} contains invalid base64 data`,
|
||||
cause,
|
||||
),
|
||||
),
|
||||
Effect.map(
|
||||
(data) =>
|
||||
new GeneratedImage({
|
||||
mediaType: item.inlineData.mimeType,
|
||||
data,
|
||||
providerMetadata: {
|
||||
google: {
|
||||
candidateIndex: item.candidate.index ?? item.candidateIndex,
|
||||
partIndex: item.partIndex,
|
||||
finishReason: item.candidate.finishReason,
|
||||
safetyRatings: item.candidate.safetyRatings,
|
||||
citationMetadata: item.candidate.citationMetadata,
|
||||
groundingMetadata: item.candidate.groundingMetadata,
|
||||
thoughtSignature: item.candidate.content?.parts[item.partIndex]?.thoughtSignature,
|
||||
},
|
||||
},
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
if (images.length === 0) {
|
||||
const finishReasons = candidates.flatMap((candidate) =>
|
||||
candidate.finishReason === undefined ? [] : [candidate.finishReason],
|
||||
)
|
||||
return yield* output.invalid(
|
||||
`Google Images returned no final images${
|
||||
finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})`
|
||||
}; inspect body for prompt feedback and candidate details`,
|
||||
)
|
||||
}
|
||||
const usage = decoded.usageMetadata
|
||||
const outputTokens =
|
||||
usage?.candidatesTokenCount === undefined
|
||||
? undefined
|
||||
: usage.candidatesTokenCount + (usage.thoughtsTokenCount ?? 0)
|
||||
return new ImageResponse({
|
||||
images,
|
||||
usage:
|
||||
usage === undefined
|
||||
? undefined
|
||||
: new Usage({
|
||||
inputTokens: usage.promptTokenCount,
|
||||
outputTokens,
|
||||
nonCachedInputTokens: ProviderShared.subtractTokens(
|
||||
usage.promptTokenCount,
|
||||
usage.cachedContentTokenCount,
|
||||
),
|
||||
cacheReadInputTokens: usage.cachedContentTokenCount,
|
||||
reasoningTokens: usage.thoughtsTokenCount,
|
||||
totalTokens: ProviderShared.totalTokens(usage.promptTokenCount, outputTokens, usage.totalTokenCount),
|
||||
providerMetadata: { google: usage },
|
||||
}),
|
||||
providerMetadata: {
|
||||
google: {
|
||||
modelVersion: decoded.modelVersion,
|
||||
responseId: decoded.responseId,
|
||||
promptFeedback: decoded.promptFeedback,
|
||||
candidates: candidateMetadata,
|
||||
},
|
||||
},
|
||||
})
|
||||
}),
|
||||
}
|
||||
return ImageModel.make<GoogleImageOptions>({ id: input.id, provider: "google", route, http: input.http })
|
||||
}
|
||||
|
||||
const googleImagePart = (image: ImageInput): Effect.Effect<Record<string, unknown>, AIError> => {
|
||||
if (image.type === "bytes")
|
||||
return Effect.succeed({ inlineData: { mimeType: image.mediaType, data: Encoding.encodeBase64(image.data) } })
|
||||
if (image.type === "file-uri") return Effect.succeed({ fileData: { mimeType: image.mediaType, fileUri: image.uri } })
|
||||
if (image.type === "url")
|
||||
return ImageInputs.decodeDataUrl(image.url).pipe(
|
||||
Effect.flatMap((decoded) => {
|
||||
if (decoded === undefined)
|
||||
return Effect.fail(
|
||||
ImageInputs.invalid(
|
||||
"Google generateContent does not fetch public image URLs; use bytes, a data URL, or a Gemini file URI",
|
||||
),
|
||||
)
|
||||
return Effect.succeed({
|
||||
inlineData: { mimeType: decoded.mediaType, data: Encoding.encodeBase64(decoded.data) },
|
||||
})
|
||||
}),
|
||||
)
|
||||
return Effect.fail(
|
||||
ProviderShared.invalidRequest(
|
||||
"Google generateContent does not fetch public image URLs; use bytes, a data URL, or a Gemini file reference",
|
||||
),
|
||||
ImageInputs.invalid("Google generateContent requires Gemini file URIs rather than provider file IDs"),
|
||||
)
|
||||
}
|
||||
|
||||
const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request: Request) {
|
||||
if (request.n !== undefined && request.n > 1)
|
||||
return yield* ProviderShared.unsupportedOperation({
|
||||
operation: "image.n",
|
||||
provider: PROVIDER,
|
||||
route: ADAPTER,
|
||||
message: `${NAME} generates one image per request; call it once per image instead of n=${request.n}`,
|
||||
})
|
||||
const parts = yield* Effect.forEach(request.images ?? [], imagePart)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
contents: [{ role: "user", parts: [{ text: request.prompt }, ...parts] }],
|
||||
generationConfig: generationConfig(request),
|
||||
},
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, GoogleImageResponse)(response)
|
||||
const decoded = output.value
|
||||
const candidates = decoded.candidates ?? []
|
||||
const candidateMetadata = candidates.map((candidate, candidateIndex) => ({
|
||||
index: candidate.index ?? candidateIndex,
|
||||
finishReason: candidate.finishReason,
|
||||
finishMessage: candidate.finishMessage,
|
||||
safetyRatings: candidate.safetyRatings,
|
||||
citationMetadata: candidate.citationMetadata,
|
||||
groundingMetadata: candidate.groundingMetadata,
|
||||
parts: (candidate.content?.parts ?? []).map((part) =>
|
||||
part.inlineData === undefined
|
||||
? { type: "text", text: part.text, thought: part.thought, thoughtSignature: part.thoughtSignature }
|
||||
: {
|
||||
type: "inlineData",
|
||||
mediaType: part.inlineData.mimeType,
|
||||
thought: part.thought,
|
||||
thoughtSignature: part.thoughtSignature,
|
||||
},
|
||||
),
|
||||
}))
|
||||
// Thought parts are drafts; only non-thought inline data is a final image.
|
||||
const encoded = candidates.flatMap((candidate, candidateIndex) =>
|
||||
(candidate.content?.parts ?? []).flatMap((part, partIndex) =>
|
||||
part.inlineData === undefined || part.thought === true
|
||||
? []
|
||||
: [
|
||||
{
|
||||
candidate,
|
||||
candidateIndex,
|
||||
partIndex,
|
||||
inlineData: part.inlineData,
|
||||
thoughtSignature: part.thoughtSignature,
|
||||
},
|
||||
],
|
||||
),
|
||||
)
|
||||
const images = yield* Effect.forEach(encoded, (item) =>
|
||||
MediaInput.decodedAsset(
|
||||
output.invalid,
|
||||
`${NAME} candidate ${item.candidateIndex} part ${item.partIndex}`,
|
||||
item.inlineData.data,
|
||||
item.inlineData.mimeType,
|
||||
{
|
||||
providerMetadata: {
|
||||
google: {
|
||||
candidateIndex: item.candidate.index ?? item.candidateIndex,
|
||||
partIndex: item.partIndex,
|
||||
finishReason: item.candidate.finishReason,
|
||||
safetyRatings: item.candidate.safetyRatings,
|
||||
citationMetadata: item.candidate.citationMetadata,
|
||||
groundingMetadata: item.candidate.groundingMetadata,
|
||||
thoughtSignature: item.thoughtSignature,
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
if (images.length === 0) {
|
||||
const finishReasons = candidates.flatMap((candidate) =>
|
||||
candidate.finishReason === undefined ? [] : [candidate.finishReason],
|
||||
)
|
||||
return yield* output.invalid(
|
||||
`${NAME} returned no final images${
|
||||
finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})`
|
||||
}; inspect body for prompt feedback and candidate details`,
|
||||
)
|
||||
}
|
||||
// Candidates that stopped for a safety or policy reason are partial results, not a silent drop.
|
||||
const notices = [
|
||||
...(decoded.promptFeedback === undefined
|
||||
? []
|
||||
: [
|
||||
{
|
||||
type: "filtered" as const,
|
||||
message: `${NAME} reported prompt feedback`,
|
||||
providerMetadata: { google: { promptFeedback: decoded.promptFeedback } },
|
||||
},
|
||||
]),
|
||||
...candidates.flatMap((candidate, index) =>
|
||||
candidate.finishReason === undefined || candidate.finishReason === "STOP"
|
||||
? []
|
||||
: [
|
||||
{
|
||||
type: "filtered" as const,
|
||||
message: `${NAME} candidate ${candidate.index ?? index} finished with ${candidate.finishReason}${
|
||||
candidate.finishMessage === undefined ? "" : `: ${candidate.finishMessage}`
|
||||
}`,
|
||||
providerMetadata: {
|
||||
google: {
|
||||
candidateIndex: candidate.index ?? index,
|
||||
finishReason: candidate.finishReason,
|
||||
finishMessage: candidate.finishMessage,
|
||||
safetyRatings: candidate.safetyRatings,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
),
|
||||
]
|
||||
const usage = decoded.usageMetadata
|
||||
const outputTokens =
|
||||
usage?.candidatesTokenCount === undefined ? undefined : usage.candidatesTokenCount + (usage.thoughtsTokenCount ?? 0)
|
||||
return new ImageResponse({
|
||||
images,
|
||||
notices: notices.length === 0 ? undefined : notices,
|
||||
usage:
|
||||
usage === undefined
|
||||
? undefined
|
||||
: {
|
||||
type: "tokens",
|
||||
input: usage.promptTokenCount,
|
||||
output: outputTokens,
|
||||
total: ProviderShared.totalTokens(usage.promptTokenCount, outputTokens, usage.totalTokenCount),
|
||||
details: {
|
||||
reasoningTokens: usage.thoughtsTokenCount,
|
||||
cacheReadInputTokens: usage.cachedContentTokenCount,
|
||||
google: usage,
|
||||
},
|
||||
},
|
||||
providerMetadata: {
|
||||
google: {
|
||||
modelVersion: decoded.modelVersion,
|
||||
responseId: decoded.responseId,
|
||||
promptFeedback: decoded.promptFeedback,
|
||||
candidates: candidateMetadata,
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
|
||||
id: ADAPTER,
|
||||
name: NAME,
|
||||
unsupported: ["mask", "size", "format"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<GoogleImageOptions>(
|
||||
{
|
||||
id: ADAPTER,
|
||||
provider: PROVIDER,
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) => `/models/${request.model.id}:generateContent`,
|
||||
},
|
||||
input,
|
||||
)
|
||||
|
||||
export const GoogleImages = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
|
||||
@@ -1,25 +1,17 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
|
||||
import { Effect, Encoding, Schema } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import { GeneratedImage, ImageModel, ImageResponse, type ImageRequestFor, type ImageRoute } from "../image.js"
|
||||
import { Auth } from "../route/auth.js"
|
||||
import { Usage, mergeHttpOptions, mergeJsonRecords, type HttpOptions } from "../schema/index.js"
|
||||
import { JsonObject, ProviderShared, optionalNull } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
const ADAPTER = "meta-images"
|
||||
const NAME = "Meta Images"
|
||||
const PROVIDER = ProviderID.make("meta")
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
import { ImageInputs } from "./utils/image-input.js"
|
||||
|
||||
type OpenString<Known extends string> = Known | (string & {})
|
||||
|
||||
/** Provider-native options. Common fields (`n`, `size`, `format`, `images`) live on the request. */
|
||||
export type ImageOptions = {
|
||||
readonly n?: number
|
||||
/** Aspect ratio hint, not an exact output resolution. */
|
||||
readonly size?: string
|
||||
readonly outputFormat?: OpenString<"webp" | "png" | "jpeg">
|
||||
readonly responseFormat?: OpenString<"b64_json" | "url">
|
||||
readonly reasoningStrength?: OpenString<"low" | "high">
|
||||
readonly toolEnablement?: {
|
||||
@@ -30,19 +22,12 @@ export type ImageOptions = {
|
||||
readonly [key: string]: unknown
|
||||
}
|
||||
|
||||
export type Request = ImageRequestFor<ImageOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Request body and response schemas
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const Body = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
model: Schema.String,
|
||||
prompt: Schema.String,
|
||||
images: Schema.optional(Schema.Array(JsonObject)),
|
||||
n: Schema.optional(Schema.Number),
|
||||
/** Aspect ratio hint, not an exact output resolution. */
|
||||
size: Schema.optional(Schema.String),
|
||||
output_format: Schema.optional(Schema.String),
|
||||
response_format: Schema.optional(Schema.String),
|
||||
@@ -64,102 +49,85 @@ const Response = Schema.Struct({
|
||||
),
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
|
||||
|
||||
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
|
||||
const inline = asset.inline()
|
||||
if (inline) return Effect.succeed({ image_url: inline.dataUrl })
|
||||
const url = ProviderShared.mediaUrl(asset)
|
||||
if (url) return Effect.succeed({ image_url: url })
|
||||
return Effect.fail(ProviderShared.invalidRequest(`${NAME} accepts image bytes and URLs`))
|
||||
export const model = (input: {
|
||||
readonly id: string
|
||||
readonly auth: Auth.Definition
|
||||
readonly baseURL: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions
|
||||
}) => {
|
||||
const route: ImageRoute<ImageOptions> = {
|
||||
id: "meta-images",
|
||||
generate: Effect.fn("MetaImages.generate")(function* (request: ImageRequestFor<ImageOptions>, execute) {
|
||||
const http = mergeHttpOptions(request.model.http, request.http)
|
||||
const images = yield* Effect.forEach(request.images ?? [], (image) => {
|
||||
if (image.type === "bytes") return Effect.succeed({ image_url: ImageInputs.dataUrl(image) })
|
||||
if (image.type === "url") return Effect.succeed({ image_url: image.url })
|
||||
return ImageInputs.invalid("Meta Images accepts image bytes and URLs")
|
||||
})
|
||||
const { outputFormat, responseFormat, reasoningStrength, toolEnablement, ...native } = request.options ?? {}
|
||||
const payload = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(Body))(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
model: request.model.id,
|
||||
prompt: request.prompt,
|
||||
images: images.length === 0 ? undefined : images,
|
||||
output_format: outputFormat,
|
||||
response_format: responseFormat,
|
||||
reasoning_strength: reasoningStrength,
|
||||
tool_enablement: toolEnablement,
|
||||
},
|
||||
native,
|
||||
http?.body,
|
||||
),
|
||||
)
|
||||
const body = ProviderShared.encodeJson(payload)
|
||||
const url = new URL(`${input.baseURL.replace(/\/$/, "")}/images/${images.length === 0 ? "generations" : "edits"}`)
|
||||
Object.entries(http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value))
|
||||
const headers = yield* Auth.toEffect(input.auth)({
|
||||
request,
|
||||
method: "POST",
|
||||
url: url.toString(),
|
||||
body,
|
||||
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
|
||||
})
|
||||
const response = yield* execute(
|
||||
HttpClientRequest.post(url.toString()).pipe(
|
||||
HttpClientRequest.setHeaders(headers),
|
||||
HttpClientRequest.bodyText(body, "application/json"),
|
||||
),
|
||||
)
|
||||
const output = yield* ProviderShared.imageResponse("meta-images", "Meta Images", response)
|
||||
const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Response))(output.body).pipe(
|
||||
Effect.mapError((cause) => output.invalid("Meta Images returned an invalid response", cause)),
|
||||
)
|
||||
const format = decoded.output_format ?? payload.output_format ?? "webp"
|
||||
const generated = yield* Effect.forEach(decoded.data, (item, index) => {
|
||||
if (item.b64_json)
|
||||
return Effect.fromResult(Encoding.decodeBase64(item.b64_json)).pipe(
|
||||
Effect.mapError((cause) => output.invalid(`Meta Images result ${index} contains invalid base64`, cause)),
|
||||
Effect.map((data) => new GeneratedImage({ mediaType: `image/${format}`, data })),
|
||||
)
|
||||
if (item.url) return Effect.succeed(new GeneratedImage({ mediaType: `image/${format}`, data: item.url }))
|
||||
return output.invalid(`Meta Images result ${index} has neither image data nor a URL`)
|
||||
})
|
||||
if (generated.length === 0) return yield* output.invalid("Meta Images returned no images")
|
||||
return new ImageResponse({
|
||||
images: generated,
|
||||
usage:
|
||||
decoded.usage === undefined
|
||||
? undefined
|
||||
: new Usage({
|
||||
inputTokens: decoded.usage.input_tokens,
|
||||
outputTokens: decoded.usage.output_tokens,
|
||||
totalTokens: decoded.usage.total_tokens,
|
||||
providerMetadata: { meta: decoded.usage },
|
||||
}),
|
||||
providerMetadata: { meta: { outputFormat: format } },
|
||||
})
|
||||
}),
|
||||
}
|
||||
return ImageModel.make<ImageOptions>({ id: input.id, provider: "meta", route, http: input.http })
|
||||
}
|
||||
|
||||
const fromRequest = Effect.fn("MetaImages.fromRequest")(function* (request: Request) {
|
||||
const images = yield* Effect.forEach(request.images ?? [], reference)
|
||||
const { responseFormat, reasoningStrength, toolEnablement, ...native } = request.providerOptions ?? {}
|
||||
const payload = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(Body))(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
model: request.model.id,
|
||||
prompt: request.prompt,
|
||||
images: images.length === 0 ? undefined : images,
|
||||
n: request.n,
|
||||
size: request.size,
|
||||
output_format: request.format,
|
||||
response_format: responseFormat,
|
||||
reasoning_strength: reasoningStrength,
|
||||
tool_enablement: toolEnablement,
|
||||
},
|
||||
native,
|
||||
request.http?.body,
|
||||
),
|
||||
)
|
||||
return MediaProtocol.json(payload)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeResponse = Effect.fn("MetaImages.decodeResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.DecodeContext<Request>,
|
||||
) {
|
||||
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, Response)(response)
|
||||
const decoded = output.value
|
||||
const requested = context.body.type === "json" ? context.body.value.output_format : undefined
|
||||
const format = decoded.output_format ?? (typeof requested === "string" ? requested : "webp")
|
||||
const mediaType = `image/${format}`
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) => {
|
||||
if (item.b64_json)
|
||||
return MediaInput.decodedAsset(output.invalid, `${NAME} result ${index}`, item.b64_json, mediaType, {
|
||||
info: { format },
|
||||
})
|
||||
if (item.url) return Effect.succeed(Media.url(item.url, { mediaType, info: { format } }))
|
||||
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
|
||||
})
|
||||
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
|
||||
return new ImageResponse({
|
||||
images,
|
||||
usage:
|
||||
decoded.usage === undefined
|
||||
? undefined
|
||||
: {
|
||||
type: "tokens",
|
||||
input: decoded.usage.input_tokens,
|
||||
output: decoded.usage.output_tokens,
|
||||
total: decoded.usage.total_tokens,
|
||||
details: { meta: decoded.usage },
|
||||
},
|
||||
providerMetadata: { meta: { outputFormat: format } },
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
|
||||
id: ADAPTER,
|
||||
name: NAME,
|
||||
unsupported: ["mask", "aspectRatio", "seed"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput & { readonly baseURL: string }) =>
|
||||
ImageModel.fromRoute<ImageOptions>(
|
||||
{
|
||||
id: ADAPTER,
|
||||
provider: PROVIDER,
|
||||
protocol,
|
||||
path: ({ request }) => `/images/${isEdit(request) ? "edits" : "generations"}`,
|
||||
},
|
||||
input,
|
||||
)
|
||||
|
||||
export * as MetaImages from "./meta-images.js"
|
||||
|
||||
@@ -224,13 +224,11 @@ type MistralEvent = Schema.Schema.Type<typeof MistralEvent>
|
||||
const MistralStreamEvent = Schema.Union([Schema.Literal(DONE), Protocol.jsonEvent(MistralEvent)])
|
||||
|
||||
const lowerMedia = Effect.fn("MistralChat.lowerMedia")(function* (part: MediaPart) {
|
||||
const mime = part.media.mediaType.toLowerCase()
|
||||
const url =
|
||||
ProviderShared.mediaUrl(part.media) ??
|
||||
(yield* ProviderShared.requireInlineMedia("Mistral Chat", part.media)).dataUrl
|
||||
if (mime.startsWith("image/")) return { type: "image_url" as const, image_url: url }
|
||||
if (mime === "application/pdf") return { type: "document_url" as const, document_url: url }
|
||||
return yield* ProviderShared.invalidRequest(`Mistral Chat does not support media type ${part.media.mediaType}`)
|
||||
const media = ProviderShared.normalizeMedia(part)
|
||||
const url = typeof part.data === "string" && /^(?:https?:|data:)/.test(part.data) ? part.data : media.dataUrl
|
||||
if (media.mime.startsWith("image/")) return { type: "image_url" as const, image_url: url }
|
||||
if (media.mime === "application/pdf") return { type: "document_url" as const, document_url: url }
|
||||
return yield* ProviderShared.invalidRequest(`Mistral Chat does not support media type ${part.mediaType}`)
|
||||
})
|
||||
|
||||
const lowerUser = Effect.fn("MistralChat.lowerUser")(function* (message: LLMRequest["messages"][number]) {
|
||||
@@ -318,7 +316,7 @@ const lowerToolResults = Effect.fn("MistralChat.lowerToolResults")(function* (
|
||||
content.push({ type: "text", text: item.text })
|
||||
continue
|
||||
}
|
||||
content.push(yield* lowerMedia(ProviderShared.toolFileMedia(item)))
|
||||
content.push(yield* lowerMedia({ type: "media", mediaType: item.mime, data: item.uri, filename: item.name }))
|
||||
}
|
||||
output.push({
|
||||
role: "tool",
|
||||
|
||||
@@ -18,7 +18,6 @@ import {
|
||||
type ToolDefinition,
|
||||
type ToolResultPart,
|
||||
} from "../schema/index.js"
|
||||
import type { Media } from "../media.js"
|
||||
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
|
||||
import { classifyProviderFailure } from "../provider-error.js"
|
||||
import { effortUpdate } from "../effort-updates.js"
|
||||
@@ -450,7 +449,7 @@ export interface ProviderAdapter {
|
||||
) => Effect.Effect<{ readonly type: string }, AIError>
|
||||
readonly lowerMedia?: (input: {
|
||||
readonly part: MediaPart
|
||||
readonly media: Media.Inline | undefined
|
||||
readonly media: ProviderShared.NormalizedMedia
|
||||
readonly request: LLMRequest
|
||||
}) => MediaInput | undefined
|
||||
readonly restoreHostedToolItem?: (item: unknown) => HostedToolReplayItem | undefined
|
||||
@@ -557,28 +556,29 @@ const lowerMedia = Effect.fn("OpenResponses.lowerMedia")(function* (
|
||||
adapter: ProviderAdapter,
|
||||
target: "message" | "tool-result",
|
||||
) {
|
||||
const media = part.media.inline()
|
||||
const media = ProviderShared.normalizeMedia(part)
|
||||
const providerMedia = adapter.lowerMedia?.({ part, media, request })
|
||||
if (providerMedia) return providerMedia
|
||||
const detail = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(OpenResponsesInputImage.fields.detail))(
|
||||
part.providerMetadata?.[metadataKey(request.model)]?.detail,
|
||||
)
|
||||
const mime = part.media.mediaType.toLowerCase()
|
||||
const url = ProviderShared.mediaUrl(part.media)
|
||||
const location = url ?? (yield* ProviderShared.requireInlineMedia(adapter.name, part.media)).dataUrl
|
||||
if (part.media.kind !== "image") {
|
||||
if (target === "tool-result" && part.media.kind === "video")
|
||||
return { type: "input_video" as const, video_url: location }
|
||||
const url =
|
||||
typeof part.data === "string" && (part.data.startsWith("https://") || part.data.startsWith("http://"))
|
||||
? part.data
|
||||
: undefined
|
||||
if (!media.mime.startsWith("image/")) {
|
||||
if (target === "tool-result" && media.mime.startsWith("video/"))
|
||||
return { type: "input_video" as const, video_url: url ?? media.dataUrl }
|
||||
return {
|
||||
type: "input_file" as const,
|
||||
filename: part.filename ?? (mime === "application/pdf" ? "document.pdf" : "file"),
|
||||
filename: part.filename ?? (media.mime === "application/pdf" ? "document.pdf" : "file"),
|
||||
detail,
|
||||
...(url ? { file_url: url } : { file_data: location }),
|
||||
...(url ? { file_url: url } : { file_data: media.dataUrl }),
|
||||
}
|
||||
}
|
||||
return {
|
||||
type: "input_image" as const,
|
||||
image_url: location,
|
||||
image_url: url ?? media.dataUrl,
|
||||
detail,
|
||||
}
|
||||
})
|
||||
@@ -608,7 +608,12 @@ const lowerToolResultContentItem = Effect.fnUntraced(function* (
|
||||
adapter: ProviderAdapter,
|
||||
) {
|
||||
if (item.type === "text") return { type: "input_text" as const, text: item.text }
|
||||
return yield* lowerMedia(ProviderShared.toolFileMedia(item), request, adapter, "tool-result")
|
||||
return yield* lowerMedia(
|
||||
{ type: "media", mediaType: item.mime, data: item.uri, filename: item.name },
|
||||
request,
|
||||
adapter,
|
||||
"tool-result",
|
||||
)
|
||||
})
|
||||
|
||||
const lowerHostedToolResultContentItem = Effect.fnUntraced(function* (
|
||||
@@ -617,7 +622,11 @@ const lowerHostedToolResultContentItem = Effect.fnUntraced(function* (
|
||||
adapter: ProviderAdapter,
|
||||
) {
|
||||
if (item.type === "text") return { type: "input_text" as const, text: item.text }
|
||||
return yield* lowerMessageMedia(ProviderShared.toolFileMedia(item), request, adapter)
|
||||
return yield* lowerMessageMedia(
|
||||
{ type: "media", mediaType: item.mime, data: item.uri, filename: item.name },
|
||||
request,
|
||||
adapter,
|
||||
)
|
||||
})
|
||||
|
||||
const lowerToolResultOutput = Effect.fnUntraced(function* (
|
||||
@@ -763,22 +772,11 @@ const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (part.type === "media") {
|
||||
flushText()
|
||||
// Responses has no assistant-authored image item; replay generated media (e.g. from Gemini) as user input.
|
||||
input.push({
|
||||
type: "message",
|
||||
role: "user",
|
||||
content: [yield* lowerMessageMedia(part, request, adapter)],
|
||||
})
|
||||
continue
|
||||
}
|
||||
return yield* ProviderShared.unsupportedContent(adapter.name, "assistant", [
|
||||
"text",
|
||||
"reasoning",
|
||||
"tool-call",
|
||||
"tool-result",
|
||||
"media",
|
||||
])
|
||||
}
|
||||
flushText()
|
||||
|
||||
@@ -320,10 +320,13 @@ const lowerToolCall = (part: ToolCallPart, options: LoweringOptions): OpenAIChat
|
||||
})
|
||||
|
||||
const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart) {
|
||||
if (part.media.kind !== "image")
|
||||
return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.media.mediaType}`)
|
||||
const media = ProviderShared.normalizeMedia(part)
|
||||
if (!media.mime.startsWith("image/"))
|
||||
return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.mediaType}`)
|
||||
const url =
|
||||
ProviderShared.mediaUrl(part.media) ?? (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl
|
||||
typeof part.data === "string" && (part.data.startsWith("https://") || part.data.startsWith("http://"))
|
||||
? part.data
|
||||
: media.dataUrl
|
||||
return { type: "image_url" as const, image_url: { url } }
|
||||
})
|
||||
|
||||
@@ -457,7 +460,11 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (
|
||||
cache_control: options.cacheControl?.(part.cache),
|
||||
})
|
||||
const files = content.filter((item) => item.type === "file")
|
||||
images.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item)))))
|
||||
images.push(
|
||||
...(yield* Effect.forEach(files, (item) =>
|
||||
lowerMedia({ type: "media", mediaType: item.mime, data: item.uri, filename: item.name }),
|
||||
)),
|
||||
)
|
||||
}
|
||||
return { messages, images }
|
||||
})
|
||||
@@ -711,8 +718,7 @@ const lowerOptions = (request: LLMRequest, supportsStore: boolean) => {
|
||||
// Default off: strict providers 400 on unknown body fields, so only send
|
||||
// the key where compatibility explicitly allows it. Header-based affinity
|
||||
// (x-session-affinity, x-grok-conv-id, ...) is unaffected.
|
||||
const cacheKey =
|
||||
(request.model.compatibility?.supportsPromptCacheKey ?? false) ? ProviderShared.promptCacheKey(request) : undefined
|
||||
const cacheKey = (request.model.compatibility?.supportsPromptCacheKey ?? false) ? ProviderShared.promptCacheKey(request) : undefined
|
||||
return {
|
||||
...(supportsStore && options.store !== undefined ? { store: options.store } : {}),
|
||||
// For providers that support `store`, ensure stateless `store:false` is sent
|
||||
|
||||
@@ -1,39 +1,43 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
|
||||
import { Effect, Encoding, Schema } from "effect"
|
||||
import { Headers, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import {
|
||||
ImageModel,
|
||||
GeneratedImage,
|
||||
ImageResponse,
|
||||
type ImageInput,
|
||||
type ImageRequestFor,
|
||||
type ImageRoute,
|
||||
} from "../image.js"
|
||||
import { Auth, type Definition as AuthDefinition } from "../route/auth.js"
|
||||
import { Usage, mergeHttpOptions, mergeJsonRecords, type HttpOptions } from "../schema/index.js"
|
||||
import { ProviderShared } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
import { ImageInputs } from "./utils/image-input.js"
|
||||
import { OpenAIImage } from "./utils/openai-image.js"
|
||||
|
||||
const ADAPTER = "openai-images"
|
||||
const NAME = "OpenAI Images"
|
||||
const PROVIDER = ProviderID.make("openai")
|
||||
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
||||
export const PATH = "/images/generations"
|
||||
export const EDIT_PATH = "/images/edits"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type OpenAIImageString<Known extends string> = Known | (string & {})
|
||||
|
||||
/** Provider-native options. Common fields (`n`, `size`, `format`, `images`, `mask`) live on the request. */
|
||||
export type OpenAIImageOptions = {
|
||||
readonly mask?: ImageInput
|
||||
readonly n?: number
|
||||
readonly size?: OpenAIImageString<
|
||||
"auto" | "256x256" | "512x512" | "1024x1024" | "1536x1024" | "1024x1536" | "1792x1024" | "1024x1792"
|
||||
>
|
||||
readonly quality?: OpenAIImageString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
|
||||
readonly background?: OpenAIImageString<"auto" | "opaque" | "transparent">
|
||||
readonly moderation?: OpenAIImageString<"auto" | "low">
|
||||
readonly outputFormat?: OpenAIImageString<"png" | "jpeg" | "webp">
|
||||
readonly outputCompression?: number
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = ImageRequestFor<OpenAIImageOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Response schema
|
||||
// ---------------------------------------------------------------------------
|
||||
export type OpenAIImageBody = Record<string, unknown> & {
|
||||
readonly model: string
|
||||
readonly prompt: string
|
||||
}
|
||||
|
||||
const OpenAIImageResponse = Schema.Struct({
|
||||
data: Schema.Array(
|
||||
@@ -55,153 +59,196 @@ const OpenAIImageResponse = Schema.Struct({
|
||||
),
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Multipart field names the route owns; `http.body` overlays cannot smuggle replacements for them. */
|
||||
const RESERVED_FORM_FIELDS = new Set(["model", "prompt", "image", "image[]", "images", "mask"])
|
||||
export interface ModelInput {
|
||||
readonly id: string
|
||||
readonly auth: AuthDefinition
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions
|
||||
}
|
||||
|
||||
const nativeOptions = (options: OpenAIImageOptions | undefined) => {
|
||||
if (!options) return undefined
|
||||
const { outputCompression, ...native } = options
|
||||
return { output_compression: outputCompression, ...native }
|
||||
}
|
||||
|
||||
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
|
||||
|
||||
const isInline = (asset: Media.Asset) => asset.inline() !== undefined
|
||||
|
||||
const blob = (data: Uint8Array, mediaType: string) => {
|
||||
const buffer = new ArrayBuffer(data.byteLength)
|
||||
new Uint8Array(buffer).set(data)
|
||||
return new Blob([buffer], { type: mediaType })
|
||||
}
|
||||
|
||||
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
|
||||
const inline = asset.inline()
|
||||
if (inline) return Effect.succeed({ image_url: inline.dataUrl })
|
||||
const url = ProviderShared.mediaUrl(asset)
|
||||
if (url) return Effect.succeed({ image_url: url })
|
||||
const id = MediaInput.refID(asset, PROVIDER)
|
||||
if (id) return Effect.succeed({ file_id: id })
|
||||
return Effect.fail(
|
||||
ProviderShared.invalidRequest("OpenAI Images accepts image URLs, data URLs, bytes, and OpenAI file IDs"),
|
||||
)
|
||||
}
|
||||
|
||||
const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Request) {
|
||||
const images = request.images ?? []
|
||||
const mask = request.mask
|
||||
if (mask !== undefined && images.length === 0)
|
||||
return yield* ProviderShared.invalidRequest("An OpenAI image mask requires at least one input image")
|
||||
const fields = mergeJsonRecords(
|
||||
{ n: request.n, size: request.size, output_format: request.format },
|
||||
nativeOptions(request.providerOptions),
|
||||
request.http?.body,
|
||||
)
|
||||
|
||||
// Owned bytes go through multipart edits; remote URLs and file IDs use the JSON edits body instead.
|
||||
if (images.length > 0 && images.every(isInline) && (mask === undefined || isInline(mask))) {
|
||||
const form = new FormData()
|
||||
form.append("model", request.model.id)
|
||||
form.append("prompt", request.prompt)
|
||||
Object.entries(fields ?? {}).forEach(([key, value]) => {
|
||||
if (RESERVED_FORM_FIELDS.has(key)) return
|
||||
form.append(key, typeof value === "string" ? value : ProviderShared.encodeJson(value))
|
||||
})
|
||||
const uploads = yield* Effect.forEach(images, (image) => MediaInput.inlineBytes(ADAPTER, image))
|
||||
uploads.forEach((data, index) => form.append("image[]", blob(data, images[index].mediaType), `image-${index}`))
|
||||
if (mask !== undefined)
|
||||
form.append("mask", blob(yield* MediaInput.inlineBytes(ADAPTER, mask), mask.mediaType), "mask")
|
||||
return MediaProtocol.multipart(form)
|
||||
const { mask: _, outputFormat, outputCompression, ...native } = options
|
||||
return {
|
||||
output_format: outputFormat,
|
||||
output_compression: outputCompression,
|
||||
...native,
|
||||
}
|
||||
|
||||
const references = yield* Effect.forEach(images, reference)
|
||||
const maskReference = mask === undefined ? undefined : yield* reference(mask)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
model: request.model.id,
|
||||
prompt: request.prompt,
|
||||
images: references.length === 0 ? undefined : references,
|
||||
mask: maskReference,
|
||||
},
|
||||
fields,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const requestedFormat = (body: MediaProtocol.Body) => {
|
||||
const value = body.type === "json" ? body.value.output_format : body.value.get("output_format")
|
||||
return typeof value === "string" ? value : undefined
|
||||
}
|
||||
|
||||
const decodeResponse = Effect.fn("OpenAIImages.decodeResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: MediaProtocol.DecodeContext<Request>,
|
||||
) {
|
||||
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, OpenAIImageResponse)(response)
|
||||
const decoded = output.value
|
||||
const format = decoded.output_format ?? requestedFormat(context.body) ?? "png"
|
||||
const mediaType = `image/${format}`
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) => {
|
||||
const providerMetadata =
|
||||
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } }
|
||||
if (item.b64_json)
|
||||
return MediaInput.decodedAsset(output.invalid, `${NAME} result ${index}`, item.b64_json, mediaType, {
|
||||
info: { format },
|
||||
providerMetadata,
|
||||
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
|
||||
if (!query) return url
|
||||
const next = new URL(url)
|
||||
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
|
||||
return next.toString()
|
||||
}
|
||||
|
||||
export const model = (input: ModelInput) => {
|
||||
const route: ImageRoute<OpenAIImageOptions> = {
|
||||
id: ADAPTER,
|
||||
generate: Effect.fn("OpenAIImages.generate")(function* (request: ImageRequestFor<OpenAIImageOptions>, execute) {
|
||||
const mask = request.options?.mask
|
||||
if (mask !== undefined && (request.images?.length ?? 0) === 0)
|
||||
return yield* ImageInputs.invalid("An OpenAI image mask requires at least one input image")
|
||||
const http = mergeHttpOptions(request.model.http, request.http)
|
||||
const sourceImages = request.images ?? []
|
||||
const multipartImages = yield* Effect.forEach(sourceImages, (image) => {
|
||||
if (image.type === "bytes") return Effect.succeed({ data: image.data, mediaType: image.mediaType })
|
||||
if (image.type === "url") return ImageInputs.decodeDataUrl(image.url)
|
||||
return Effect.undefined
|
||||
})
|
||||
if (item.url) return Effect.succeed(Media.url(item.url, { mediaType, info: { format }, providerMetadata }))
|
||||
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
|
||||
const multipartMask =
|
||||
mask === undefined
|
||||
? undefined
|
||||
: mask.type === "bytes"
|
||||
? { data: mask.data, mediaType: mask.mediaType }
|
||||
: mask.type === "url"
|
||||
? yield* ImageInputs.decodeDataUrl(mask.url)
|
||||
: undefined
|
||||
const useMultipart =
|
||||
sourceImages.length > 0 &&
|
||||
multipartImages.every((image) => image !== undefined) &&
|
||||
(mask === undefined || multipartMask !== undefined)
|
||||
const path = sourceImages.length === 0 ? PATH : EDIT_PATH
|
||||
const url = applyQuery(`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}${path}`, http?.query)
|
||||
|
||||
if (useMultipart) {
|
||||
const form = new FormData()
|
||||
form.append("model", request.model.id)
|
||||
form.append("prompt", request.prompt)
|
||||
Object.entries(mergeJsonRecords(nativeOptions(request.options), http?.body) ?? {}).forEach(([key, value]) => {
|
||||
if (["model", "prompt", "image", "image[]", "images", "mask"].includes(key)) return
|
||||
form.append(key, typeof value === "string" ? value : ProviderShared.encodeJson(value))
|
||||
})
|
||||
multipartImages.forEach((image, index) => {
|
||||
if (image === undefined) return
|
||||
form.append("image[]", imageBlob(image.data, image.mediaType), `image-${index}`)
|
||||
})
|
||||
if (multipartMask !== undefined)
|
||||
form.append("mask", imageBlob(multipartMask.data, multipartMask.mediaType), "mask")
|
||||
const headers = yield* Auth.toEffect(input.auth)({
|
||||
request,
|
||||
method: "POST",
|
||||
url,
|
||||
body: "[multipart/form-data]",
|
||||
headers: Headers.remove(Headers.fromInput({ ...input.headers, ...http?.headers }), "content-type"),
|
||||
})
|
||||
const response = yield* execute(
|
||||
HttpClientRequest.post(url).pipe(HttpClientRequest.setHeaders(headers), HttpClientRequest.bodyFormData(form)),
|
||||
)
|
||||
return yield* parseResponse(response, request.options, http?.body)
|
||||
}
|
||||
|
||||
const references = sourceImages.map((image) => {
|
||||
if (image.type === "bytes") return { image_url: ImageInputs.dataUrl(image) }
|
||||
if (image.type === "url") return { image_url: image.url }
|
||||
if (image.type === "file-id") return { file_id: image.id }
|
||||
return undefined
|
||||
})
|
||||
if (references.some((image) => image === undefined))
|
||||
return yield* ImageInputs.invalid("OpenAI Images accepts image URLs, data URLs, bytes, and file IDs")
|
||||
const maskReference =
|
||||
mask === undefined
|
||||
? undefined
|
||||
: mask.type === "bytes"
|
||||
? { image_url: ImageInputs.dataUrl(mask) }
|
||||
: mask.type === "url"
|
||||
? { image_url: mask.url }
|
||||
: mask.type === "file-id"
|
||||
? { file_id: mask.id }
|
||||
: undefined
|
||||
if (mask !== undefined && maskReference === undefined)
|
||||
return yield* ImageInputs.invalid("OpenAI Images accepts masks as URLs, data URLs, bytes, or file IDs")
|
||||
const requestBody = mergeJsonRecords(
|
||||
{
|
||||
model: request.model.id,
|
||||
prompt: request.prompt,
|
||||
images: references.length === 0 ? undefined : references,
|
||||
mask: maskReference,
|
||||
},
|
||||
nativeOptions(request.options),
|
||||
http?.body,
|
||||
) as OpenAIImageBody
|
||||
const text = ProviderShared.encodeJson(requestBody)
|
||||
const headers = yield* Auth.toEffect(input.auth)({
|
||||
request,
|
||||
method: "POST",
|
||||
url,
|
||||
body: text,
|
||||
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
|
||||
})
|
||||
const response = yield* execute(
|
||||
HttpClientRequest.post(url).pipe(
|
||||
HttpClientRequest.setHeaders(headers),
|
||||
HttpClientRequest.bodyText(text, "application/json"),
|
||||
),
|
||||
)
|
||||
return yield* parseResponse(response, request.options, http?.body)
|
||||
}),
|
||||
}
|
||||
return ImageModel.make<OpenAIImageOptions>({ id: input.id, provider: "openai", route, http: input.http })
|
||||
}
|
||||
|
||||
const parseResponse = Effect.fn("OpenAIImages.parseResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
options: OpenAIImageOptions | undefined,
|
||||
overlay: Record<string, unknown> | undefined,
|
||||
) {
|
||||
const output = yield* ProviderShared.imageResponse(ADAPTER, "OpenAI Images", response)
|
||||
const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(OpenAIImageResponse))(output.body).pipe(
|
||||
Effect.mapError((cause) => output.invalid("OpenAI Images returned an invalid response", cause)),
|
||||
)
|
||||
const requestBody = mergeJsonRecords(nativeOptions(options), overlay)
|
||||
const format =
|
||||
decoded.output_format ?? (typeof requestBody?.output_format === "string" ? requestBody.output_format : "png")
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) => {
|
||||
if (item.b64_json)
|
||||
return Effect.fromResult(Encoding.decodeBase64(item.b64_json)).pipe(
|
||||
Effect.mapError((cause) => output.invalid(`OpenAI Images result ${index} contains invalid base64 data`, cause)),
|
||||
Effect.map(
|
||||
(data) =>
|
||||
new GeneratedImage({
|
||||
mediaType: `image/${format}`,
|
||||
data,
|
||||
providerMetadata:
|
||||
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } },
|
||||
}),
|
||||
),
|
||||
)
|
||||
if (item.url)
|
||||
return Effect.succeed(
|
||||
new GeneratedImage({
|
||||
mediaType: `image/${format}`,
|
||||
data: item.url,
|
||||
providerMetadata:
|
||||
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } },
|
||||
}),
|
||||
)
|
||||
return Effect.fail(output.invalid(`OpenAI Images result ${index} has neither image data nor a URL`))
|
||||
})
|
||||
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
|
||||
if (images.length === 0) return yield* output.invalid("OpenAI Images returned no images")
|
||||
return new ImageResponse({
|
||||
images,
|
||||
usage:
|
||||
decoded.usage === undefined
|
||||
? undefined
|
||||
: {
|
||||
type: "tokens",
|
||||
input: decoded.usage.input_tokens,
|
||||
output: decoded.usage.output_tokens,
|
||||
total: decoded.usage.total_tokens,
|
||||
details: { openai: decoded.usage },
|
||||
},
|
||||
: new Usage({
|
||||
inputTokens: decoded.usage.input_tokens,
|
||||
outputTokens: decoded.usage.output_tokens,
|
||||
totalTokens: decoded.usage.total_tokens,
|
||||
providerMetadata: { openai: decoded.usage },
|
||||
}),
|
||||
providerMetadata: { openai: { outputFormat: format } },
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
|
||||
id: ADAPTER,
|
||||
name: NAME,
|
||||
unsupported: ["aspectRatio", "seed"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<OpenAIImageOptions>(
|
||||
{
|
||||
id: ADAPTER,
|
||||
provider: PROVIDER,
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
|
||||
},
|
||||
input,
|
||||
)
|
||||
const imageBlob = (data: Uint8Array, mediaType: string) => {
|
||||
const buffer = new ArrayBuffer(data.byteLength)
|
||||
new Uint8Array(buffer).set(data)
|
||||
return new Blob([buffer], { type: mediaType })
|
||||
}
|
||||
|
||||
export const OpenAIImages = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
import { Buffer } from "node:buffer"
|
||||
import { Tool } from "@opencode/schema/tool"
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import * as Sse from "effect/unstable/encoding/Sse"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import { Media } from "../media.js"
|
||||
import { Headers, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import {
|
||||
InvalidProviderOutputError,
|
||||
InvalidRequestError,
|
||||
UnsupportedOperationError,
|
||||
AIError,
|
||||
HttpContext,
|
||||
LLMRequest,
|
||||
Message,
|
||||
ToolDefinition,
|
||||
@@ -168,34 +169,25 @@ export const wrappedSystemUpdate = Effect.fn("ProviderShared.wrappedSystemUpdate
|
||||
export const parseToolInput = (route: string, name: string, raw: string) =>
|
||||
parseJson(route, raw || "{}", `Invalid JSON input for ${route} tool call ${name}`)
|
||||
|
||||
/** Inline view or a typed `InvalidRequest` for routes that cannot fetch URLs or dereference provider refs. */
|
||||
export const requireInlineMedia = (route: string, asset: Media.Asset): Effect.Effect<Media.Inline, AIError> => {
|
||||
const inline = asset.inline()
|
||||
return inline ? Effect.succeed(inline) : Effect.fail(inlineRequired(route, asset))
|
||||
export interface NormalizedMedia {
|
||||
readonly mime: string
|
||||
readonly base64: string
|
||||
readonly dataUrl: string
|
||||
}
|
||||
|
||||
export const inlineRequired = (route: string, asset: Media.Asset) =>
|
||||
invalidRequest(
|
||||
`${route} requires inline media (bytes or base64); ${asset.source.type} sources must be materialized first`,
|
||||
)
|
||||
|
||||
/** The remote URL of a `url` asset, for protocols that accept `http(s)` references natively. */
|
||||
export const mediaUrl = (asset: Media.Asset) => (asset.source.type === "url" ? asset.source.url : undefined)
|
||||
|
||||
/**
|
||||
* Lift a tool-result file into a `MediaPart`. Tool files carry either a data URL, an `http(s)` URL, or raw base64 in
|
||||
* `uri`; the declared `mime` wins over any data-URL prefix so tool authors control the type the model sees.
|
||||
*/
|
||||
export const toolFileMedia = (item: Tool.FileContent): MediaPart => {
|
||||
const parsed = Media.parseDataUrl(item.uri)
|
||||
const asset = parsed
|
||||
? Media.from({ ...parsed.source, mediaType: item.mime })
|
||||
: /^https?:\/\//.test(item.uri)
|
||||
? Media.url(item.uri, { mediaType: item.mime })
|
||||
: Media.base64(item.uri, item.mime)
|
||||
return Message.media(asset, { filename: item.name })
|
||||
export const normalizeMedia = (part: MediaPart): NormalizedMedia => {
|
||||
const mime = part.mediaType.toLowerCase()
|
||||
if (typeof part.data !== "string") {
|
||||
const base64 = Buffer.from(part.data).toString("base64")
|
||||
return { mime, base64, dataUrl: `data:${mime};base64,${base64}` }
|
||||
}
|
||||
if (!part.data.startsWith("data:")) return { mime, base64: part.data, dataUrl: `data:${mime};base64,${part.data}` }
|
||||
return { mime, base64: part.data.slice(part.data.indexOf(",") + 1), dataUrl: part.data }
|
||||
}
|
||||
|
||||
export const normalizeToolFile = (part: Tool.FileContent) =>
|
||||
normalizeMedia({ type: "media", mediaType: part.mime, data: part.uri, filename: part.name })
|
||||
|
||||
export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
|
||||
|
||||
export const toolResultText = (part: ToolResultPart) => {
|
||||
@@ -323,6 +315,34 @@ export const flattenToolRequest = (request: LLMRequest) => {
|
||||
}
|
||||
}
|
||||
|
||||
export const imageResponse = Effect.fn("ProviderShared.imageResponse")(function* (
|
||||
route: string,
|
||||
name: string,
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const http = new HttpContext({ url: response.request.url, status: response.status, headers: response.headers })
|
||||
const body = yield* response.text.pipe(
|
||||
Effect.mapError(
|
||||
(cause) =>
|
||||
new AIError({
|
||||
reason: new InvalidProviderOutputError({
|
||||
route,
|
||||
message: `Failed to read the ${name} response`,
|
||||
http,
|
||||
cause,
|
||||
}),
|
||||
}),
|
||||
),
|
||||
)
|
||||
return {
|
||||
body,
|
||||
invalid: (message: string, cause?: unknown) =>
|
||||
new AIError({
|
||||
reason: new InvalidProviderOutputError({ route, message, body, http, cause }),
|
||||
}),
|
||||
}
|
||||
})
|
||||
|
||||
export const matchToolChoice = <Auto, None, Required, Tool>(
|
||||
route: string,
|
||||
toolChoice: NonNullable<LLMRequest["toolChoice"]>,
|
||||
|
||||
@@ -77,7 +77,7 @@ function documentName(filename: string | undefined, names: Set<string>) {
|
||||
}
|
||||
|
||||
const mediaBase64 = Effect.fn("BedrockMedia.mediaBase64")(function* (part: MediaPart) {
|
||||
const media = yield* ProviderShared.requireInlineMedia("Bedrock Converse", part.media)
|
||||
const media = ProviderShared.normalizeMedia(part)
|
||||
const bytes = yield* Effect.fromResult(Encoding.decodeBase64(media.base64)).pipe(
|
||||
Effect.mapError((cause) =>
|
||||
ProviderShared.invalidRequest("Bedrock Converse media data must be valid base64", cause),
|
||||
@@ -92,15 +92,13 @@ const mediaBase64 = Effect.fn("BedrockMedia.mediaBase64")(function* (part: Media
|
||||
// get an image-specific error so the caller knows it's a format-support issue,
|
||||
// not a kind-detection issue.
|
||||
export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart, documentNames: Set<string>) {
|
||||
const mime = part.media.mediaType.toLowerCase()
|
||||
const mime = part.mediaType.toLowerCase()
|
||||
const imageFormat = IMAGE_FORMATS[mime as keyof typeof IMAGE_FORMATS]
|
||||
if (imageFormat) {
|
||||
return [{ image: { format: imageFormat, source: { bytes: yield* mediaBase64(part) } } } satisfies ImageBlock]
|
||||
}
|
||||
if (mime.startsWith("image/"))
|
||||
return yield* ProviderShared.invalidRequest(
|
||||
`Bedrock Converse does not support image media type ${part.media.mediaType}`,
|
||||
)
|
||||
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`)
|
||||
const documentFormat = DOCUMENT_FORMATS[mime as keyof typeof DOCUMENT_FORMATS]
|
||||
if (documentFormat) {
|
||||
const name = documentName(part.filename, documentNames)
|
||||
@@ -114,7 +112,7 @@ export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart,
|
||||
]
|
||||
: [block]
|
||||
}
|
||||
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.media.mediaType}`)
|
||||
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
|
||||
})
|
||||
|
||||
export * as BedrockMedia from "./bedrock-media.js"
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { Effect, Encoding } from "effect"
|
||||
import type { ImageInput } from "../../image.js"
|
||||
import { InvalidRequestError, AIError } from "../../schema/index.js"
|
||||
|
||||
const invalid = (message: string, cause?: unknown) =>
|
||||
new AIError({
|
||||
reason: new InvalidRequestError({ message, cause }),
|
||||
})
|
||||
|
||||
export const dataUrl = (input: Extract<ImageInput, { readonly type: "bytes" }>) =>
|
||||
`data:${input.mediaType};base64,${Encoding.encodeBase64(input.data)}`
|
||||
|
||||
export const decodeDataUrl = (
|
||||
url: string,
|
||||
): Effect.Effect<{ readonly mediaType: string; readonly data: Uint8Array } | undefined, AIError> => {
|
||||
if (!url.startsWith("data:")) return Effect.undefined
|
||||
const match = /^data:([^;,]+);base64,(.*)$/s.exec(url)
|
||||
if (!match) return Effect.fail(invalid("Image data URLs must contain a MIME type and base64 data"))
|
||||
return Effect.fromResult(Encoding.decodeBase64(match[2])).pipe(
|
||||
Effect.mapError((cause) => invalid("Image data URL contains invalid base64 data", cause)),
|
||||
Effect.map((data) => ({ mediaType: match[1], data })),
|
||||
)
|
||||
}
|
||||
|
||||
export const invalidImageInput = invalid
|
||||
|
||||
export const ImageInputs = {
|
||||
dataUrl,
|
||||
decodeDataUrl,
|
||||
invalid: invalidImageInput,
|
||||
} as const
|
||||
@@ -1,33 +0,0 @@
|
||||
import { Effect, Encoding } from "effect"
|
||||
import { Media } from "../../media.js"
|
||||
import type { AIError, ProviderID } from "../../schema/index.js"
|
||||
import { ProviderShared } from "../shared.js"
|
||||
|
||||
/** Owned bytes for multipart uploads; decodes `base64` sources and rejects remote sources. */
|
||||
export const inlineBytes = (route: string, asset: Media.Asset): Effect.Effect<Uint8Array, AIError> => {
|
||||
if (asset.source.type === "bytes") return Effect.succeed(asset.source.data)
|
||||
const inline = asset.inline()
|
||||
if (!inline) return Effect.fail(ProviderShared.inlineRequired(route, asset))
|
||||
return Effect.fromResult(Encoding.decodeBase64(inline.base64)).pipe(
|
||||
Effect.mapError((cause) => ProviderShared.invalidRequest(`${route} media contains invalid base64 data`, cause)),
|
||||
)
|
||||
}
|
||||
|
||||
/** Provider file handle when the ref belongs to this provider; refs from other providers are never forwarded. */
|
||||
export const refID = (asset: Media.Asset, provider: ProviderID) =>
|
||||
asset.source.type === "ref" && asset.source.provider === provider ? asset.source.id : undefined
|
||||
|
||||
/** Decode a provider's base64 output once into an owned `bytes` asset, sniffing the type when it is not declared. */
|
||||
export const decodedAsset = (
|
||||
invalid: (message: string, cause?: unknown) => AIError,
|
||||
label: string,
|
||||
data: string,
|
||||
mediaType: string | undefined,
|
||||
options?: Media.AssetOptions,
|
||||
) =>
|
||||
Effect.fromResult(Encoding.decodeBase64(data)).pipe(
|
||||
Effect.mapError((cause) => invalid(`${label} contains invalid base64 data`, cause)),
|
||||
Effect.map((bytes) => Media.bytes(bytes, mediaType, options)),
|
||||
)
|
||||
|
||||
export * as MediaInput from "./media-input.js"
|
||||
@@ -17,7 +17,6 @@ import { RequestExecutor } from "../../route/executor.js"
|
||||
import { HttpTransport } from "../../route/transport/index.js"
|
||||
import { OpenResponses } from "../open-responses.js"
|
||||
import { JsonObject, optionalNull, ProviderShared } from "../shared.js"
|
||||
import { Media } from "../../media.js"
|
||||
|
||||
const Body = Schema.Struct({
|
||||
model: Schema.String,
|
||||
@@ -158,22 +157,20 @@ function toMessage(item: (typeof Response.Type.output)[number], model: LLMReques
|
||||
if (part.type === "input_image")
|
||||
return {
|
||||
type: "media",
|
||||
media: replayMedia(part.image_url, "image/*"),
|
||||
data: part.image_url,
|
||||
mediaType: /^data:([^;,]+)/.exec(part.image_url)?.[1] ?? "image/*",
|
||||
providerMetadata: part.detail === undefined ? undefined : { [key]: { detail: part.detail } },
|
||||
}
|
||||
const data = part.file_url === undefined ? part.file_data : part.file_url
|
||||
return {
|
||||
type: "media",
|
||||
media: replayMedia(part.file_url === undefined ? part.file_data : part.file_url, "application/octet-stream"),
|
||||
data,
|
||||
filename: part.filename,
|
||||
mediaType: /^data:([^;,]+)/.exec(data)?.[1] ?? "application/octet-stream",
|
||||
providerMetadata: part.detail === undefined ? undefined : { [key]: { detail: part.detail } },
|
||||
}
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/** Replayed compaction items carry either a data URL or a remote URL; the data URL's own type wins when present. */
|
||||
const replayMedia = (value: string, fallbackType: string) =>
|
||||
Media.parseDataUrl(value) ??
|
||||
(/^https?:\/\//.test(value) ? Media.url(value, { mediaType: fallbackType }) : Media.base64(value, fallbackType))
|
||||
|
||||
export * as ResponsesCompaction from "./responses-compaction.js"
|
||||
|
||||
@@ -1,38 +1,61 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
|
||||
import { Effect, Encoding, Schema } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import { GeneratedImage, ImageModel, ImageResponse, type ImageRequestFor, type ImageRoute } from "../image.js"
|
||||
import { Auth, type Definition as AuthDefinition } from "../route/auth.js"
|
||||
import { Usage, mergeHttpOptions, mergeJsonRecords, type HttpOptions } from "../schema/index.js"
|
||||
import { ProviderShared, optionalNull } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
import { ImageInputs } from "./utils/image-input.js"
|
||||
|
||||
const ADAPTER = "xai-images"
|
||||
const NAME = "xAI Images"
|
||||
const PROVIDER = ProviderID.make("xai")
|
||||
export const DEFAULT_BASE_URL = "https://api.x.ai/v1"
|
||||
export const PATH = "/images/generations"
|
||||
export const EDIT_PATH = "/images/edits"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type XAIImageString<Known extends string> = Known | (string & {})
|
||||
|
||||
/** Provider-native options. Common fields (`n`, `aspectRatio`, `images`) live on the request. */
|
||||
export type XAIImageOptions = {
|
||||
readonly n?: number
|
||||
readonly aspectRatio?: XAIImageString<
|
||||
| "1:1"
|
||||
| "3:4"
|
||||
| "4:3"
|
||||
| "9:16"
|
||||
| "16:9"
|
||||
| "2:3"
|
||||
| "3:2"
|
||||
| "9:19.5"
|
||||
| "19.5:9"
|
||||
| "9:20"
|
||||
| "20:9"
|
||||
| "1:2"
|
||||
| "2:1"
|
||||
| "auto"
|
||||
>
|
||||
readonly aspect_ratio?: XAIImageString<
|
||||
| "1:1"
|
||||
| "3:4"
|
||||
| "4:3"
|
||||
| "9:16"
|
||||
| "16:9"
|
||||
| "2:3"
|
||||
| "3:2"
|
||||
| "9:19.5"
|
||||
| "19.5:9"
|
||||
| "9:20"
|
||||
| "20:9"
|
||||
| "1:2"
|
||||
| "2:1"
|
||||
| "auto"
|
||||
>
|
||||
readonly resolution?: XAIImageString<"1k" | "2k">
|
||||
readonly responseFormat?: XAIImageString<"url" | "b64_json">
|
||||
readonly response_format?: XAIImageString<"url" | "b64_json">
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = ImageRequestFor<XAIImageOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Response schema
|
||||
// ---------------------------------------------------------------------------
|
||||
type XAIImageBody = Record<string, unknown> & {
|
||||
readonly model: string
|
||||
readonly prompt: string
|
||||
}
|
||||
|
||||
const XAIImageResponse = Schema.Struct({
|
||||
data: Schema.Array(
|
||||
@@ -46,109 +69,120 @@ const XAIImageResponse = Schema.Struct({
|
||||
usage: Schema.optional(Schema.Unknown),
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
export interface ModelInput {
|
||||
readonly id: string
|
||||
readonly auth: AuthDefinition
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions
|
||||
}
|
||||
|
||||
const nativeOptions = (options: XAIImageOptions | undefined) => {
|
||||
if (!options) return undefined
|
||||
const { responseFormat, ...native } = options
|
||||
return { response_format: responseFormat, ...native }
|
||||
const { aspectRatio, responseFormat, ...native } = options
|
||||
return {
|
||||
aspect_ratio: aspectRatio,
|
||||
response_format: responseFormat,
|
||||
...native,
|
||||
}
|
||||
}
|
||||
|
||||
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
|
||||
|
||||
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
|
||||
const inline = asset.inline()
|
||||
if (inline) return Effect.succeed({ url: inline.dataUrl, type: "image_url" as const })
|
||||
const url = ProviderShared.mediaUrl(asset)
|
||||
if (url) return Effect.succeed({ url, type: "image_url" as const })
|
||||
const id = MediaInput.refID(asset, PROVIDER)
|
||||
if (id) return Effect.succeed({ file_id: id })
|
||||
return Effect.fail(ProviderShared.invalidRequest(`${NAME} accepts image URLs, data URLs, bytes, and xAI file IDs`))
|
||||
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
|
||||
if (!query) return url
|
||||
const next = new URL(url)
|
||||
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
|
||||
return next.toString()
|
||||
}
|
||||
|
||||
const fromRequest = Effect.fn("XAIImages.fromRequest")(function* (request: Request) {
|
||||
const references = yield* Effect.forEach(request.images ?? [], reference)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
model: request.model.id,
|
||||
prompt: request.prompt,
|
||||
// xAI takes one edit source as `image` and several as `images`.
|
||||
image: references.length === 1 ? references[0] : undefined,
|
||||
images: references.length > 1 ? references : undefined,
|
||||
n: request.n,
|
||||
aspect_ratio: request.aspectRatio,
|
||||
},
|
||||
nativeOptions(request.providerOptions),
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, XAIImageResponse)(response)
|
||||
const decoded = output.value
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) => {
|
||||
const providerMetadata =
|
||||
item.revised_prompt === undefined || item.revised_prompt === null
|
||||
? undefined
|
||||
: { xai: { revisedPrompt: item.revised_prompt } }
|
||||
if (item.b64_json)
|
||||
return MediaInput.decodedAsset(
|
||||
output.invalid,
|
||||
`${NAME} result ${index}`,
|
||||
item.b64_json,
|
||||
item.mime_type ?? undefined,
|
||||
export const model = (input: ModelInput) => {
|
||||
const route: ImageRoute<XAIImageOptions> = {
|
||||
id: ADAPTER,
|
||||
generate: Effect.fn("XAIImages.generate")(function* (request: ImageRequestFor<XAIImageOptions>, execute) {
|
||||
const http = mergeHttpOptions(request.model.http, request.http)
|
||||
const imageReferences = (request.images ?? []).map((image) => {
|
||||
if (image.type === "bytes") return { url: ImageInputs.dataUrl(image), type: "image_url" as const }
|
||||
if (image.type === "url") return { url: image.url, type: "image_url" as const }
|
||||
if (image.type === "file-id") return { file_id: image.id }
|
||||
return undefined
|
||||
})
|
||||
if (imageReferences.some((image) => image === undefined))
|
||||
return yield* ImageInputs.invalid("xAI Images accepts image URLs, data URLs, bytes, and file IDs")
|
||||
const requestBody = mergeJsonRecords(
|
||||
{
|
||||
providerMetadata,
|
||||
model: request.model.id,
|
||||
prompt: request.prompt,
|
||||
image: imageReferences.length === 1 ? imageReferences[0] : undefined,
|
||||
images: imageReferences.length > 1 ? imageReferences : undefined,
|
||||
},
|
||||
nativeOptions(request.options),
|
||||
http?.body,
|
||||
) as XAIImageBody
|
||||
const text = ProviderShared.encodeJson(requestBody)
|
||||
const url = applyQuery(
|
||||
`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}${imageReferences.length === 0 ? PATH : EDIT_PATH}`,
|
||||
http?.query,
|
||||
)
|
||||
if (item.url)
|
||||
return Effect.succeed(Media.url(item.url, { mediaType: item.mime_type ?? undefined, providerMetadata }))
|
||||
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
|
||||
})
|
||||
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
|
||||
const usage = ProviderShared.isRecord(decoded.usage) ? decoded.usage : undefined
|
||||
// xAI reports image counts rather than tokens, seconds, or credits; the raw record stays in provider metadata.
|
||||
return new ImageResponse({
|
||||
images,
|
||||
providerMetadata: usage === undefined ? undefined : { xai: { usage } },
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
|
||||
id: ADAPTER,
|
||||
name: NAME,
|
||||
unsupported: ["mask", "size", "seed", "format"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<XAIImageOptions>(
|
||||
{
|
||||
id: ADAPTER,
|
||||
provider: PROVIDER,
|
||||
protocol,
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
|
||||
},
|
||||
input,
|
||||
)
|
||||
const headers = yield* Auth.toEffect(input.auth)({
|
||||
request,
|
||||
method: "POST",
|
||||
url,
|
||||
body: text,
|
||||
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
|
||||
})
|
||||
const response = yield* execute(
|
||||
HttpClientRequest.post(url).pipe(
|
||||
HttpClientRequest.setHeaders(headers),
|
||||
HttpClientRequest.bodyText(text, "application/json"),
|
||||
),
|
||||
)
|
||||
const output = yield* ProviderShared.imageResponse(ADAPTER, "xAI Images", response)
|
||||
const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(XAIImageResponse))(output.body).pipe(
|
||||
Effect.mapError((cause) => output.invalid("xAI Images returned an invalid response", cause)),
|
||||
)
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) => {
|
||||
const mediaType = item.mime_type ?? "application/octet-stream"
|
||||
if (item.b64_json)
|
||||
return Effect.fromResult(Encoding.decodeBase64(item.b64_json)).pipe(
|
||||
Effect.mapError((cause) =>
|
||||
output.invalid(`xAI Images result ${index} contains invalid base64 data`, cause),
|
||||
),
|
||||
Effect.map(
|
||||
(data) =>
|
||||
new GeneratedImage({
|
||||
mediaType,
|
||||
data,
|
||||
providerMetadata:
|
||||
item.revised_prompt === undefined || item.revised_prompt === null
|
||||
? undefined
|
||||
: { xai: { revisedPrompt: item.revised_prompt } },
|
||||
}),
|
||||
),
|
||||
)
|
||||
if (item.url)
|
||||
return Effect.succeed(
|
||||
new GeneratedImage({
|
||||
mediaType,
|
||||
data: item.url,
|
||||
providerMetadata:
|
||||
item.revised_prompt === undefined || item.revised_prompt === null
|
||||
? undefined
|
||||
: { xai: { revisedPrompt: item.revised_prompt } },
|
||||
}),
|
||||
)
|
||||
return Effect.fail(output.invalid(`xAI Images result ${index} has neither image data nor a URL`))
|
||||
})
|
||||
if (images.length === 0) return yield* output.invalid("xAI Images returned no images")
|
||||
const usage = ProviderShared.isRecord(decoded.usage) ? decoded.usage : undefined
|
||||
return new ImageResponse({
|
||||
images,
|
||||
usage: usage === undefined ? undefined : new Usage({ providerMetadata: { xai: usage } }),
|
||||
providerMetadata: usage === undefined ? undefined : { xai: { usage } },
|
||||
})
|
||||
}),
|
||||
}
|
||||
return ImageModel.make<XAIImageOptions>({ id: input.id, provider: "xai", route, http: input.http })
|
||||
}
|
||||
|
||||
export const XAIImages = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
|
||||
@@ -1,34 +1,29 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import type { HttpClientResponse } from "effect/unstable/http"
|
||||
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
|
||||
import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import { GeneratedImage, ImageModel, ImageResponse, type ImageRequestFor, type ImageRoute } from "../image.js"
|
||||
import { Auth, type Definition as AuthDefinition } from "../route/auth.js"
|
||||
import { mergeHttpOptions, mergeJsonRecords, type HttpOptions } from "../schema/index.js"
|
||||
import { ProviderShared } from "./shared.js"
|
||||
import { ImageInputs } from "./utils/image-input.js"
|
||||
|
||||
const ADAPTER = "zai-images"
|
||||
const NAME = "Z.ai Images"
|
||||
const PROVIDER = ProviderID.make("zai")
|
||||
export const DEFAULT_BASE_URL = "https://api.z.ai/api/paas/v4"
|
||||
export const PATH = "/images/generations"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Public model input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type ZAIImageString<Known extends string> = Known | (string & {})
|
||||
|
||||
/** Provider-native options. The common `size` field lives on the request. */
|
||||
export type ZAIImageOptions = {
|
||||
readonly size?: ZAIImageString<
|
||||
"1024x1024" | "768x1344" | "864x1152" | "1344x768" | "1152x864" | "1440x720" | "720x1440"
|
||||
>
|
||||
readonly quality?: ZAIImageString<"hd" | "standard">
|
||||
readonly userID?: string
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type Request = ImageRequestFor<ZAIImageOptions>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Response schema
|
||||
// ---------------------------------------------------------------------------
|
||||
type ZAIImageBody = Record<string, unknown> & {
|
||||
readonly model: string
|
||||
readonly prompt: string
|
||||
}
|
||||
|
||||
const ZAIImageResponse = Schema.Struct({
|
||||
created: Schema.optional(Schema.Int),
|
||||
@@ -45,81 +40,84 @@ const ZAIImageResponse = Schema.Struct({
|
||||
),
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Request body construction
|
||||
// ---------------------------------------------------------------------------
|
||||
export interface ModelInput {
|
||||
readonly id: string
|
||||
readonly auth: AuthDefinition
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions
|
||||
}
|
||||
|
||||
const nativeOptions = (options: ZAIImageOptions | undefined) => {
|
||||
if (!options) return undefined
|
||||
const { userID, ...native } = options
|
||||
return { user_id: userID, ...native }
|
||||
return {
|
||||
user_id: userID,
|
||||
...native,
|
||||
}
|
||||
}
|
||||
|
||||
const fromRequest = Effect.fn("ZAIImages.fromRequest")(function* (request: Request) {
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{ model: request.model.id, prompt: request.prompt, size: request.size },
|
||||
nativeOptions(request.providerOptions),
|
||||
request.http?.body,
|
||||
) ?? {},
|
||||
)
|
||||
})
|
||||
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
|
||||
if (!query) return url
|
||||
const next = new URL(url)
|
||||
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
|
||||
return next.toString()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Response decoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const decodeResponse = Effect.fn("ZAIImages.decodeResponse")(function* (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, ZAIImageResponse)(response)
|
||||
const decoded = output.value
|
||||
if (decoded.data.length === 0) return yield* output.invalid(`${NAME} returned no images`)
|
||||
const filters = decoded.content_filter ?? []
|
||||
return new ImageResponse({
|
||||
// Z.ai returns only URLs and no content type; the media type resolves when the asset is materialized.
|
||||
images: decoded.data.map((item) => Media.url(item.url)),
|
||||
// Z.ai reports applied content filters alongside a successful result; surface them instead of dropping them.
|
||||
notices:
|
||||
filters.length === 0
|
||||
? undefined
|
||||
: filters.map((filter) => ({
|
||||
type: "moderated" as const,
|
||||
message: `${NAME} applied a content filter${filter.role === undefined ? "" : ` for ${filter.role}`}${
|
||||
filter.level === undefined ? "" : ` at level ${filter.level}`
|
||||
}`,
|
||||
providerMetadata: { zai: filter },
|
||||
})),
|
||||
providerMetadata: {
|
||||
zai: {
|
||||
created: decoded.created,
|
||||
id: decoded.id,
|
||||
requestID: decoded.request_id,
|
||||
contentFilter: decoded.content_filter,
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
|
||||
id: ADAPTER,
|
||||
name: NAME,
|
||||
unsupported: ["images", "mask", "n", "aspectRatio", "seed", "format"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
ImageModel.fromRoute<ZAIImageOptions>(
|
||||
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
|
||||
input,
|
||||
)
|
||||
export const model = (input: ModelInput) => {
|
||||
const route: ImageRoute<ZAIImageOptions> = {
|
||||
id: ADAPTER,
|
||||
generate: Effect.fn("ZAIImages.generate")(function* (request: ImageRequestFor<ZAIImageOptions>, execute) {
|
||||
if ((request.images?.length ?? 0) > 0)
|
||||
return yield* ImageInputs.invalid("Z.ai hosted image generation does not support image inputs")
|
||||
const http = mergeHttpOptions(request.model.http, request.http)
|
||||
const requestBody = mergeJsonRecords(
|
||||
{ model: request.model.id, prompt: request.prompt },
|
||||
nativeOptions(request.options),
|
||||
http?.body,
|
||||
) as ZAIImageBody
|
||||
const text = ProviderShared.encodeJson(requestBody)
|
||||
const url = applyQuery(`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}${PATH}`, http?.query)
|
||||
const headers = yield* Auth.toEffect(input.auth)({
|
||||
request,
|
||||
method: "POST",
|
||||
url,
|
||||
body: text,
|
||||
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
|
||||
})
|
||||
const response = yield* execute(
|
||||
HttpClientRequest.post(url).pipe(
|
||||
HttpClientRequest.setHeaders(headers),
|
||||
HttpClientRequest.bodyText(text, "application/json"),
|
||||
),
|
||||
)
|
||||
const output = yield* ProviderShared.imageResponse(ADAPTER, "Z.ai Images", response)
|
||||
const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(ZAIImageResponse))(output.body).pipe(
|
||||
Effect.mapError((cause) => output.invalid("Z.ai Images returned an invalid response", cause)),
|
||||
)
|
||||
if (decoded.data.length === 0) return yield* output.invalid("Z.ai Images returned no images")
|
||||
return new ImageResponse({
|
||||
images: decoded.data.map(
|
||||
(item) =>
|
||||
new GeneratedImage({
|
||||
mediaType: "application/octet-stream",
|
||||
data: item.url,
|
||||
}),
|
||||
),
|
||||
providerMetadata: {
|
||||
zai: {
|
||||
created: decoded.created,
|
||||
id: decoded.id,
|
||||
requestID: decoded.request_id,
|
||||
contentFilter: decoded.content_filter,
|
||||
},
|
||||
},
|
||||
})
|
||||
}),
|
||||
}
|
||||
return ImageModel.make<ZAIImageOptions>({ id: input.id, provider: "zai", route, http: input.http })
|
||||
}
|
||||
|
||||
export const ZAIImages = {
|
||||
protocol,
|
||||
model,
|
||||
} as const
|
||||
|
||||
@@ -59,7 +59,15 @@ export const isContextOverflowFailure = (failure: unknown) =>
|
||||
: Schema.is(ProviderErrorEvent)(failure) && failure.classification === "context-overflow"
|
||||
|
||||
const decodeJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown))
|
||||
const QUOTA_CODES = new Set(["insufficient_quota", "usage_not_included", "billing_error"])
|
||||
// OpenCode Zen reports account caps as typed 429/402 errors that are not throttles.
|
||||
const QUOTA_CODES = new Set([
|
||||
"insufficient_quota",
|
||||
"usage_not_included",
|
||||
"billing_error",
|
||||
"gousagelimiterror",
|
||||
"freeusagelimiterror",
|
||||
"creditlimitexceeded",
|
||||
])
|
||||
const AUTH_CODES = new Set(["authentication_error", "permission_error"])
|
||||
const SERVER_CODES = new Set([
|
||||
"api_error",
|
||||
@@ -87,7 +95,8 @@ const CONTENT_POLICY_CODES = new Set([
|
||||
// as a `[code]` label at the start of the rewritten message.
|
||||
const GATEWAY_CODE_LABEL = /^[^:\n]+: \[([A-Za-z0-9_.-]+)\]/
|
||||
const RATE_LIMIT_TEXT = /rate increased too quickly|rate[-_\s]?limit|too[_\s]?many[_\s]?requests/i
|
||||
const QUOTA_TEXT = /insufficient[-_\s]?quota|quota[-_\s]?exceeded/i
|
||||
// Only consulted on 429, where throttles and account caps share a status.
|
||||
const QUOTA_TEXT = /insufficient[-_\s]?quota|quota[-_\s]?exceeded|budget exceeded|usage limit/i
|
||||
// Policy rejections without a dedicated code, matched against the provider's own
|
||||
// explanation only. OpenAI reuses `invalid_prompt` for usage-policy rejections while
|
||||
// Bedrock Mantle reuses it for schema validation; Anthropic reports blocked output
|
||||
@@ -143,7 +152,11 @@ export function classifyProviderFailure(input: ProviderFailure): AIError["reason
|
||||
return new InvalidRequestError({ ...details, classification: "payload-too-large" })
|
||||
if (codes.some((code) => CONTENT_POLICY_CODES.has(code)) || (clientScoped && CONTENT_POLICY_TEXT.test(input.message)))
|
||||
return new ContentPolicyError(details)
|
||||
if (codes.some((code) => QUOTA_CODES.has(code)) || (input.status === 429 && QUOTA_TEXT.test(text)))
|
||||
if (
|
||||
input.status === 402 ||
|
||||
codes.some((code) => QUOTA_CODES.has(code)) ||
|
||||
(input.status === 429 && QUOTA_TEXT.test(text))
|
||||
)
|
||||
return new QuotaExceededError(details)
|
||||
if (input.status === 401 || input.status === 403 || codes.some((code) => AUTH_CODES.has(code)))
|
||||
return new AuthenticationError(details)
|
||||
@@ -163,10 +176,12 @@ export function classifyProviderFailure(input: ProviderFailure): AIError["reason
|
||||
input.status === 408 ||
|
||||
input.status === 409 ||
|
||||
(input.status !== undefined && input.status >= 500) ||
|
||||
// Server codes and phrasing only decide when no HTTP status contradicts them:
|
||||
// gateways such as OpenCode Zen substitute `server_error` for codes they do
|
||||
// not forward, so a 4xx with a server code is still a rejected request.
|
||||
((input.status === undefined || input.status < 400) &&
|
||||
!codes.some((code) => INVALID_REQUEST_CODES.has(code)) &&
|
||||
SERVER_ERROR_TEXT.test(text)) ||
|
||||
codes.some((code) => SERVER_CODES.has(code) || code.includes("exhausted") || code.includes("unavailable"))
|
||||
((!codes.some((code) => INVALID_REQUEST_CODES.has(code)) && SERVER_ERROR_TEXT.test(text)) ||
|
||||
codes.some((code) => SERVER_CODES.has(code) || code.includes("exhausted") || code.includes("unavailable"))))
|
||||
)
|
||||
return new ProviderInternalError({
|
||||
...details,
|
||||
|
||||
@@ -8,7 +8,6 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { ProviderConfigurationError, ProviderID, ToolDefinition, type ModelID } from "../schema/index.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("alibaba")
|
||||
|
||||
@@ -104,7 +103,7 @@ export const configure = (input: Config) => {
|
||||
})
|
||||
.model<MessagesOptionsInput>({ id, compatibility: { requireSignature: false } })
|
||||
const responses = (id: string | ModelID) => responsesRoute.with(common).model<ResponsesOptionsInput>({ id })
|
||||
return ModelRef.facade({ id, model: chat, chat, messages, responses, configure })
|
||||
return { id, model: chat, chat, messages, responses, configure }
|
||||
}
|
||||
|
||||
export const provider = { id, configure }
|
||||
|
||||
@@ -6,7 +6,6 @@ import { OpenResponses } from "../protocols/open-responses.js"
|
||||
import { BedrockAuth, type Credentials } from "../protocols/utils/bedrock-auth.js"
|
||||
import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("amazon-bedrock")
|
||||
|
||||
@@ -98,13 +97,13 @@ export const configure = (input: Config = {}) => {
|
||||
.with(withOpenAIOptions(modelID, modelDefaults))
|
||||
.model<OpenAIProviderOptionsInput>({ id: modelID })
|
||||
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: responses,
|
||||
chat,
|
||||
responses,
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -4,7 +4,6 @@ import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/
|
||||
import * as BedrockConverse from "../protocols/bedrock-converse.js"
|
||||
import type { BedrockCredentials } from "../protocols/bedrock-converse.js"
|
||||
import { BedrockAuth } from "../protocols/utils/bedrock-auth.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("amazon-bedrock")
|
||||
|
||||
@@ -55,11 +54,11 @@ const configuredRoute = (input: Config) => {
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const route = configuredRoute(input)
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => route.model({ id: modelID }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -4,7 +4,6 @@ import { Auth } from "../route/auth.js"
|
||||
import type { ProviderAuthOption } from "../route/auth-options.js"
|
||||
import type { RouteDefaultsInput } from "../route/client.js"
|
||||
import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export type AnthropicOptionsInput = AnthropicMessages.OptionsInput
|
||||
export type AnthropicProviderOptionsInput = AnthropicMessages.ProviderOptionsInput
|
||||
@@ -50,11 +49,11 @@ export const configure = (input: Config) => {
|
||||
endpoint: { baseURL },
|
||||
auth: auth(input),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id: ProviderID.make(provider),
|
||||
model: (modelID: string | ModelID) => route.model<AnthropicMessages.ProviderOptionsInput>({ id: modelID }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = {
|
||||
|
||||
@@ -5,7 +5,6 @@ import type { ProviderPackage } from "../provider-package.js"
|
||||
import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { AnthropicMessages } from "../protocols/anthropic-messages.js"
|
||||
import { AnthropicCompatible } from "./anthropic-compatible.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export type AnthropicOptionsInput = AnthropicMessages.OptionsInput
|
||||
export type AnthropicProviderOptionsInput = AnthropicMessages.ProviderOptionsInput
|
||||
@@ -45,11 +44,11 @@ export const configure = (input: Config = {}) => {
|
||||
baseURL: baseURL ?? AnthropicMessages.DEFAULT_BASE_URL,
|
||||
provider: id,
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => compatible.model(modelID),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -7,7 +7,6 @@ import * as OpenAIChat from "../protocols/openai-chat.js"
|
||||
import * as OpenAIResponses from "../protocols/openai-responses.js"
|
||||
import { ProviderShared } from "../protocols/shared.js"
|
||||
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("azure")
|
||||
const routeAuth = Auth.remove("authorization")
|
||||
@@ -132,13 +131,13 @@ export const configure = (input: Config) => {
|
||||
.with(withOpenAIOptions(modelID, modelDefaults))
|
||||
.model<OpenAIProviderOptionsInput>({ id: modelID, compatibility: { supportsPromptCacheKey: true } })
|
||||
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: responses,
|
||||
responses,
|
||||
chat,
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = {
|
||||
|
||||
@@ -5,7 +5,6 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("baseten")
|
||||
const baseURL = "https://inference.baseten.co/v1"
|
||||
@@ -40,11 +39,11 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
endpoint: { baseURL: endpoint ?? baseURL },
|
||||
auth: AuthOptions.bearer(input, "BASETEN_API_KEY"),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => configured.model<OpenAIProviderOptionsInput>({ id: modelID }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -5,7 +5,6 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("cerebras")
|
||||
const baseURL = "https://api.cerebras.ai/v1"
|
||||
@@ -40,20 +39,15 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
endpoint: { baseURL: endpoint ?? baseURL },
|
||||
auth: AuthOptions.bearer(input, "CEREBRAS_API_KEY"),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) =>
|
||||
configured.model<OpenAIProviderOptionsInput>({
|
||||
id: modelID,
|
||||
compatibility: {
|
||||
maxTokensField: "max_tokens",
|
||||
reasoningField: "reasoning",
|
||||
supportsStore: false,
|
||||
supportsPromptCacheKey: true,
|
||||
},
|
||||
compatibility: { maxTokensField: "max_tokens", reasoningField: "reasoning", supportsStore: false, supportsPromptCacheKey: true },
|
||||
}),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -6,7 +6,6 @@ import type { AtLeastOne, ProviderAuthOption } from "../route/auth-options.js"
|
||||
import type { RouteDefaultsInput } from "../route/client.js"
|
||||
import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("cloudflare-ai-gateway")
|
||||
export const authEnvVars = ["CLOUDFLARE_API_TOKEN", "CF_AIG_TOKEN"] as const
|
||||
@@ -109,7 +108,7 @@ export const configure = (input: LanguageModelOptions) => {
|
||||
const responses = responsesRoute.with(defaults)
|
||||
const messages = messagesRoute.with(defaults)
|
||||
const chat = route.with(defaults)
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (input: string | ModelID) => {
|
||||
const wire = modelID(input)
|
||||
@@ -118,7 +117,7 @@ export const configure = (input: LanguageModelOptions) => {
|
||||
return chat.model<OpenAIProviderOptionsInput>({ id: wire })
|
||||
},
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = { id, configure }
|
||||
|
||||
@@ -5,7 +5,6 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("cloudflare-workers-ai")
|
||||
export const authEnvVars = ["CLOUDFLARE_API_KEY", "CLOUDFLARE_WORKERS_AI_TOKEN", "CLOUDFLARE_API_TOKEN"] as const
|
||||
@@ -55,11 +54,11 @@ export const configure = (input: LanguageModelOptions) => {
|
||||
endpoint: { baseURL: baseURL(input) },
|
||||
auth: AuthOptions.bearer(input, authEnvVars),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => configured.model<OpenAIProviderOptionsInput>({ id: modelID }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = { id, configure }
|
||||
|
||||
@@ -5,7 +5,6 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("deepinfra")
|
||||
const baseURL = "https://api.deepinfra.com/v1/openai"
|
||||
@@ -43,20 +42,15 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
},
|
||||
auth: AuthOptions.bearer(input, "DEEPINFRA_API_KEY"),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) =>
|
||||
configured.model<OpenAIProviderOptionsInput>({
|
||||
id: modelID,
|
||||
compatibility: {
|
||||
maxTokensField: "max_tokens",
|
||||
reasoningField: "reasoning_content",
|
||||
supportsStore: false,
|
||||
supportsPromptCacheKey: true,
|
||||
},
|
||||
compatibility: { maxTokensField: "max_tokens", reasoningField: "reasoning_content", supportsStore: false, supportsPromptCacheKey: true },
|
||||
}),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -5,7 +5,6 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("deepseek")
|
||||
const baseURL = "https://api.deepseek.com/v1"
|
||||
@@ -40,7 +39,7 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
endpoint: { baseURL: endpoint ?? baseURL },
|
||||
auth: AuthOptions.bearer(input, "DEEPSEEK_API_KEY"),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) =>
|
||||
configured.model<OpenAIProviderOptionsInput>({
|
||||
@@ -48,7 +47,7 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
compatibility: { maxTokensField: "max_tokens", supportsStore: false },
|
||||
}),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -5,7 +5,6 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("fireworks")
|
||||
const baseURL = "https://api.fireworks.ai/inference/v1"
|
||||
@@ -40,12 +39,12 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
endpoint: { baseURL: endpoint ?? baseURL },
|
||||
auth: AuthOptions.bearer(input, "FIREWORKS_API_KEY"),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) =>
|
||||
configured.model<OpenAIProviderOptionsInput>({ id: modelID, compatibility: { supportsPromptCacheKey: true } }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -5,7 +5,6 @@ import { Endpoint } from "../route/endpoint.js"
|
||||
import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { GoogleVertexShared } from "./google-vertex-shared.js"
|
||||
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("google-vertex")
|
||||
|
||||
@@ -63,11 +62,11 @@ const configuredRoute = (input: Config) => {
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const route = configuredRoute(input)
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => route.model<OpenAIProviderOptionsInput>({ id: modelID }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = {
|
||||
|
||||
@@ -7,7 +7,6 @@ import { Endpoint } from "../route/endpoint.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { GoogleVertexShared } from "./google-vertex-shared.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export type AnthropicOptionsInput = AnthropicMessages.OptionsInput
|
||||
export type AnthropicProviderOptionsInput = AnthropicMessages.ProviderOptionsInput
|
||||
@@ -92,11 +91,11 @@ const configuredRoute = (input: Config) => {
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const route = configuredRoute(input)
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => route.model<AnthropicMessages.ProviderOptionsInput>({ id: modelID }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = {
|
||||
|
||||
@@ -5,7 +5,6 @@ import { Endpoint } from "../route/endpoint.js"
|
||||
import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { GoogleVertexShared } from "./google-vertex-shared.js"
|
||||
import type { OpenResponsesProviderOptionsInput } from "./open-responses-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("google-vertex")
|
||||
|
||||
@@ -64,11 +63,11 @@ const configuredRoute = (input: Config) => {
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const route = configuredRoute(input)
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => route.model<OpenResponsesProviderOptionsInput>({ id: modelID }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = {
|
||||
|
||||
@@ -8,7 +8,6 @@ import { Endpoint } from "../route/endpoint.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { ProviderConfigurationError, ProviderID, type LLMRequest, type ModelID } from "../schema/index.js"
|
||||
import { GoogleVertexShared } from "./google-vertex-shared.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export interface GeminiOptionsInput extends Gemini.OptionsInput {
|
||||
readonly labels?: Readonly<Record<string, string>>
|
||||
@@ -117,12 +116,12 @@ const configuredRoute = (input: Config, modelID: string | ModelID) => {
|
||||
}
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) =>
|
||||
configuredRoute(input, modelID).model<GeminiProviderOptionsInput>({ id: modelID }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = {
|
||||
|
||||
@@ -5,7 +5,6 @@ import type { ProviderPackage } from "../provider-package.js"
|
||||
import { HttpOptions, ProviderID, mergeHttpOptions, type ModelID } from "../schema/index.js"
|
||||
import { Gemini } from "../protocols/gemini.js"
|
||||
import { GoogleImages } from "../protocols/google-images.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export type { GoogleImageOptions } from "../protocols/google-images.js"
|
||||
export type GeminiOptionsInput = Gemini.OptionsInput
|
||||
@@ -49,12 +48,12 @@ export const configure = (input: Config = {}) => {
|
||||
headers: input.headers,
|
||||
http: mergeHttpOptions(input.http === undefined ? undefined : HttpOptions.make(input.http)),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => route.model<Gemini.ProviderOptionsInput>({ id: modelID }),
|
||||
image,
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -8,7 +8,6 @@ import { Endpoint } from "../route/endpoint.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { ProviderID, type ModelID, type LLMRequest } from "../schema/index.js"
|
||||
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("groq")
|
||||
const baseURL = "https://api.groq.com/openai/v1"
|
||||
@@ -85,7 +84,7 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
endpoint: { baseURL: endpoint ?? baseURL },
|
||||
auth: AuthOptions.bearer(input, "GROQ_API_KEY"),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) =>
|
||||
configured.model<ProviderOptions>({
|
||||
@@ -99,7 +98,7 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
},
|
||||
}),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -9,7 +9,6 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { HttpOptions, ProviderID, ToolDefinition, type ModelID } from "../schema/index.js"
|
||||
import type { OpenResponsesProviderOptionsInput } from "./open-responses-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("meta")
|
||||
const baseURL = "https://api.meta.ai/v1"
|
||||
@@ -148,7 +147,7 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
headers: input.headers,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
})
|
||||
return ModelRef.facade({ id, model: responses, responses, chat, messages, image, configure })
|
||||
return { id, model: responses, responses, chat, messages, image, configure }
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -10,7 +10,6 @@ import { Endpoint } from "../route/endpoint.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { ProviderID, type LLMRequest, type ModelID } from "../schema/index.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("minimax")
|
||||
|
||||
@@ -121,7 +120,7 @@ export const configure = (input: Config = {}) => {
|
||||
})
|
||||
const responses = (modelID: string | ModelID) =>
|
||||
responsesRoute.with(defaults).model<ResponsesOptionsInput>({ id: modelID })
|
||||
return ModelRef.facade({ id, model: messages, messages, chat, responses, configure })
|
||||
return { id, model: messages, messages, chat, responses, configure }
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -3,7 +3,6 @@ import { MistralChat } from "../protocols/mistral-chat.js"
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import type { RouteDefaultsInput } from "../route/client.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("mistral")
|
||||
|
||||
@@ -31,11 +30,11 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
endpoint: { baseURL: baseURL ?? MistralChat.DEFAULT_BASE_URL },
|
||||
auth: AuthOptions.bearer(input, "MISTRAL_API_KEY"),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => configured.model<ProviderOptions>({ id: modelID }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -10,7 +10,6 @@ import { Endpoint } from "../route/endpoint.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { ProviderID, type LLMRequest, type ModelID } from "../schema/index.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("moonshotai")
|
||||
|
||||
@@ -127,7 +126,7 @@ export const configure = (input: Config = {}) => {
|
||||
responsesRoute
|
||||
.with(defaults)
|
||||
.model<ResponsesOptionsInput>({ id: modelID, compatibility: { toolSchema: "moonshot" } })
|
||||
return ModelRef.facade({ id, model: chat, chat, messages, responses, configure })
|
||||
return { id, model: chat, chat, messages, responses, configure }
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -4,7 +4,6 @@ import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import type { RouteDefaultsInput } from "../route/client.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import type { OpenResponsesProviderOptionsInput } from "./open-responses-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export type { OpenResponsesOptionsInput, OpenResponsesProviderOptionsInput } from "./open-responses-options.js"
|
||||
|
||||
@@ -35,11 +34,11 @@ export const configure = (input: Config) => {
|
||||
endpoint: { baseURL },
|
||||
auth: AuthOptions.bearer(input, []),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id: ProviderID.make(provider),
|
||||
model: (modelID: string | ModelID) => route.model<OpenResponsesProviderOptionsInput>({ id: modelID }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = {
|
||||
|
||||
@@ -4,7 +4,6 @@ import type { RouteDefaultsInput } from "../route/client.js"
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("openai-compatible")
|
||||
|
||||
@@ -33,12 +32,12 @@ export const configure = (input: GenericModelOptions) => {
|
||||
endpoint: { baseURL },
|
||||
auth: AuthOptions.bearer(input, []),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id: ProviderID.make(provider),
|
||||
model: (modelID: string | ModelID) =>
|
||||
route.model<OpenAIProviderOptionsInput>({ id: modelID, provider: ProviderID.make(provider) }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = {
|
||||
|
||||
@@ -6,7 +6,6 @@ import * as OpenAIChat from "../protocols/openai-chat.js"
|
||||
import * as OpenAIResponses from "../protocols/openai-responses.js"
|
||||
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { OpenAIImages, type OpenAIImageString } from "../protocols/openai-images.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export type { OpenAIOptionsInput, OpenAIResponseIncludable } from "./openai-options.js"
|
||||
export type { OpenAIImageOptions } from "../protocols/openai-images.js"
|
||||
@@ -108,14 +107,14 @@ export const configure = (input: Config = {}) => {
|
||||
),
|
||||
})
|
||||
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: responses,
|
||||
responses,
|
||||
chat,
|
||||
image,
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -8,7 +8,6 @@ import type { ProviderPackage } from "../provider-package.js"
|
||||
import { OpenAIChat } from "../protocols/openai-chat.js"
|
||||
import { newBreakpoints, ttlBucket } from "../protocols/utils/cache.js"
|
||||
import { isRecord } from "../protocols/shared.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("openrouter")
|
||||
const baseURL = "https://openrouter.ai/api/v1"
|
||||
@@ -182,12 +181,12 @@ const configuredRoute = (input: LanguageModelOptions) => {
|
||||
|
||||
export const configure = (input: LanguageModelOptions = {}) => {
|
||||
const route = configuredRoute(input)
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) =>
|
||||
route.model<OpenRouterProviderOptionsInput>({ id: modelID, compatibility: { supportsPromptCacheKey: true } }),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -5,7 +5,6 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("togetherai")
|
||||
const baseURL = "https://api.together.xyz/v1"
|
||||
@@ -40,7 +39,7 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
endpoint: { baseURL: endpoint ?? baseURL },
|
||||
auth: AuthOptions.bearer(input, ["TOGETHER_API_KEY", "TOGETHER_AI_API_KEY"]),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) =>
|
||||
configured.model<OpenAIProviderOptionsInput>({
|
||||
@@ -48,7 +47,7 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
compatibility: { maxTokensField: "max_tokens", supportsStore: false, supportsStrictMode: false },
|
||||
}),
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -8,7 +8,6 @@ import { XAIResponses } from "../protocols/xai-responses.js"
|
||||
import { XAIImages } from "../protocols/xai-images.js"
|
||||
import type { OpenAIOptionsInput } from "./openai-options.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("xai")
|
||||
const baseURL = "https://api.x.ai/v1"
|
||||
@@ -96,14 +95,14 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
headers: input.headers,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
})
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: responses,
|
||||
responses,
|
||||
chat,
|
||||
image,
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -9,7 +9,6 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("zai-coding-plan")
|
||||
|
||||
@@ -73,7 +72,7 @@ export const configure = (input: Config = {}) => {
|
||||
.model<MessagesOptionsInput>({ id: modelID, compatibility: { requireSignature: false } })
|
||||
const responses = (modelID: string | ModelID) =>
|
||||
responsesRoute.with(defaults).model<ResponsesOptionsInput>({ id: modelID })
|
||||
return ModelRef.facade({ id, model: chat, chat, messages, responses, configure })
|
||||
return { id, model: chat, chat, messages, responses, configure }
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -6,7 +6,6 @@ import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import { ModelRef } from "../model-ref.js"
|
||||
|
||||
export const id = ProviderID.make("zai")
|
||||
|
||||
@@ -58,13 +57,13 @@ export const configure = (input: Config = {}) => {
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
})
|
||||
|
||||
return ModelRef.facade({
|
||||
return {
|
||||
id,
|
||||
model: chat,
|
||||
chat,
|
||||
image,
|
||||
configure,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import type { LLMRequest } from "../schema/index.js"
|
||||
import * as ProviderShared from "../protocols/shared.js"
|
||||
|
||||
export interface EndpointInput<Body, Request = LLMRequest> {
|
||||
readonly request: Request
|
||||
export interface EndpointInput<Body> {
|
||||
readonly request: LLMRequest
|
||||
readonly body: Body
|
||||
}
|
||||
|
||||
export type EndpointPart<Body, Request = LLMRequest> = string | ((input: EndpointInput<Body, Request>) => string)
|
||||
export type EndpointPart<Body> = string | ((input: EndpointInput<Body>) => string)
|
||||
|
||||
/**
|
||||
* Declarative URL construction for one route.
|
||||
@@ -17,29 +17,26 @@ export type EndpointPart<Body, Request = LLMRequest> = string | ((input: Endpoin
|
||||
*
|
||||
* `path` may be a string or a function of `EndpointInput`, for routes whose
|
||||
* URL embeds the model id, region, or another body field (e.g. Bedrock,
|
||||
* Gemini). Media routes reuse the same shape with their own request type.
|
||||
* Gemini).
|
||||
*/
|
||||
export interface Definition<Body, Request = LLMRequest> {
|
||||
export interface Definition<Body> {
|
||||
readonly baseURL?: string
|
||||
readonly path: EndpointPart<Body, Request>
|
||||
readonly path: EndpointPart<Body>
|
||||
readonly query?: Record<string, string>
|
||||
}
|
||||
|
||||
export type EndpointPatch<Body, Request = LLMRequest> = Partial<Definition<Body, Request>>
|
||||
export type EndpointPatch<Body> = Partial<Definition<Body>>
|
||||
|
||||
/** Construct an `Endpoint` from a path string or path function. */
|
||||
export const path = <Body, Request = LLMRequest>(
|
||||
value: EndpointPart<Body, Request>,
|
||||
options: Omit<Definition<Body, Request>, "path"> = {},
|
||||
): Definition<Body, Request> => ({
|
||||
export const path = <Body>(
|
||||
value: EndpointPart<Body>,
|
||||
options: Omit<Definition<Body>, "path"> = {},
|
||||
): Definition<Body> => ({
|
||||
...options,
|
||||
path: value,
|
||||
})
|
||||
|
||||
export const merge = <Body, Request = LLMRequest>(
|
||||
base: Definition<Body, Request>,
|
||||
patch: EndpointPatch<Body, Request>,
|
||||
): Definition<Body, Request> => ({
|
||||
export const merge = <Body>(base: Definition<Body>, patch: EndpointPatch<Body>): Definition<Body> => ({
|
||||
...base,
|
||||
...patch,
|
||||
baseURL: patch.baseURL ?? base.baseURL,
|
||||
@@ -47,13 +44,10 @@ export const merge = <Body, Request = LLMRequest>(
|
||||
query: patch.query === undefined ? base.query : { ...base.query, ...patch.query },
|
||||
})
|
||||
|
||||
const renderPart = <Body, Request>(part: EndpointPart<Body, Request>, input: EndpointInput<Body, Request>) =>
|
||||
const renderPart = <Body>(part: EndpointPart<Body>, input: EndpointInput<Body>) =>
|
||||
typeof part === "function" ? part(input) : part
|
||||
|
||||
export const render = <Body, Request = LLMRequest>(
|
||||
endpoint: Definition<Body, Request>,
|
||||
input: EndpointInput<Body, Request>,
|
||||
) => {
|
||||
export const render = <Body>(endpoint: Definition<Body>, input: EndpointInput<Body>) => {
|
||||
const url = new URL(`${ProviderShared.trimBaseUrl(endpoint.baseURL ?? "")}${renderPart(endpoint.path, input)}`)
|
||||
for (const [key, value] of Object.entries(endpoint.query ?? {})) url.searchParams.set(key, value)
|
||||
return url
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
import { Context, type Effect } from "effect"
|
||||
import type { HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import type { AIError } from "../schema/errors.js"
|
||||
|
||||
// The service tag lives in its own leaf module so `Media.Asset` (imported by the schema layer) can require the
|
||||
// executor without pulling the full executor implementation, and therefore the schema barrel, into a cycle.
|
||||
export interface Interface {
|
||||
readonly execute: (
|
||||
request: HttpClientRequest.HttpClientRequest,
|
||||
middleware?: HttpMiddleware,
|
||||
) => Effect.Effect<HttpClientResponse.HttpClientResponse, AIError>
|
||||
}
|
||||
|
||||
export type HttpHandler = (
|
||||
request: HttpClientRequest.HttpClientRequest,
|
||||
) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error>
|
||||
export type HttpMiddleware = (
|
||||
request: HttpClientRequest.HttpClientRequest,
|
||||
handler: HttpHandler,
|
||||
) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error>
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/AI/RequestExecutor") {}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Cause, Effect, Layer, Option, Schema, Stream } from "effect"
|
||||
import { Cause, Context, Effect, Layer, Option, Schema, Stream } from "effect"
|
||||
import {
|
||||
FetchHttpClient,
|
||||
Headers,
|
||||
@@ -9,10 +9,23 @@ import {
|
||||
} from "effect/unstable/http"
|
||||
import { HttpContext, HttpRateLimitDetails, AIError, TransportError } from "../schema/index.js"
|
||||
import { classifyProviderFailure } from "../provider-error.js"
|
||||
import { Service, type HttpMiddleware, type Interface } from "./executor-service.js"
|
||||
|
||||
export { Service } from "./executor-service.js"
|
||||
export type { HttpHandler, HttpMiddleware, Interface } from "./executor-service.js"
|
||||
export interface Interface {
|
||||
readonly execute: (
|
||||
request: HttpClientRequest.HttpClientRequest,
|
||||
middleware?: HttpMiddleware,
|
||||
) => Effect.Effect<HttpClientResponse.HttpClientResponse, AIError>
|
||||
}
|
||||
|
||||
export type HttpHandler = (
|
||||
request: HttpClientRequest.HttpClientRequest,
|
||||
) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error>
|
||||
export type HttpMiddleware = (
|
||||
request: HttpClientRequest.HttpClientRequest,
|
||||
handler: HttpHandler,
|
||||
) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error>
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/AI/RequestExecutor") {}
|
||||
|
||||
const headerDetails = (headers: Headers.Headers) =>
|
||||
Object.fromEntries(Object.entries(headers).map(([name, value]) => [name, String(value)]))
|
||||
|
||||
@@ -20,8 +20,6 @@ export * from "./executor.js"
|
||||
export { Auth } from "./auth.js"
|
||||
export { AuthOptions } from "./auth-options.js"
|
||||
export { Endpoint } from "./endpoint.js"
|
||||
export { MediaRoute } from "./media.js"
|
||||
export { MediaProtocol } from "./media-protocol.js"
|
||||
export { Framing } from "./framing.js"
|
||||
export { Protocol } from "./protocol.js"
|
||||
export { HttpTransport, WebSocketTransport } from "./transport/index.js"
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { HttpClientResponse } from "effect/unstable/http"
|
||||
import { AIError, HttpContext, InvalidProviderOutputError } from "../schema/index.js"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bodies
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type Body =
|
||||
| { readonly type: "json"; readonly value: Record<string, unknown> }
|
||||
| { readonly type: "multipart"; readonly value: FormData }
|
||||
|
||||
export const json = (value: Record<string, unknown>): Body => ({ type: "json", value })
|
||||
export const multipart = (value: FormData): Body => ({ type: "multipart", value })
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Protocol kinds
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface DecodeContext<Request> {
|
||||
readonly request: Request
|
||||
readonly body: Body
|
||||
}
|
||||
|
||||
/** One request, one response. JSON or multipart in; JSON or raw bytes out. */
|
||||
export interface Inline<Request, Response> {
|
||||
readonly kind: "inline"
|
||||
readonly id: string
|
||||
readonly name: string
|
||||
/** Common request fields this protocol cannot lower; the route rejects them before `body.from` runs. */
|
||||
readonly unsupported?: ReadonlyArray<keyof Request & string>
|
||||
readonly body: { readonly from: (request: Request) => Effect.Effect<Body, AIError> }
|
||||
readonly response: {
|
||||
readonly decode: (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
context: DecodeContext<Request>,
|
||||
) => Effect.Effect<Response, AIError>
|
||||
}
|
||||
}
|
||||
|
||||
export const inline = <Request, Response>(
|
||||
input: Omit<Inline<Request, Response>, "kind">,
|
||||
): Inline<Request, Response> => ({
|
||||
kind: "inline",
|
||||
...input,
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const context = (response: HttpClientResponse.HttpClientResponse) =>
|
||||
new HttpContext({ url: response.request.url, status: response.status, headers: response.headers })
|
||||
|
||||
/** Read a text body while retaining the original payload and HTTP context on every downstream error. */
|
||||
export const text = Effect.fn("MediaProtocol.text")(function* (
|
||||
route: string,
|
||||
name: string,
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
) {
|
||||
const http = context(response)
|
||||
const body = yield* response.text.pipe(
|
||||
Effect.mapError(
|
||||
(cause) =>
|
||||
new AIError({
|
||||
reason: new InvalidProviderOutputError({
|
||||
route,
|
||||
message: `Failed to read the ${name} response`,
|
||||
http,
|
||||
cause,
|
||||
}),
|
||||
}),
|
||||
),
|
||||
)
|
||||
return {
|
||||
body,
|
||||
http,
|
||||
invalid: (message: string, cause?: unknown) =>
|
||||
new AIError({ reason: new InvalidProviderOutputError({ route, message, body, http, cause }) }),
|
||||
}
|
||||
})
|
||||
|
||||
/** Read and Schema-decode a JSON body. Decode failures keep the raw body as `reason.body`. */
|
||||
export const decodeJson = <A>(route: string, name: string, schema: Schema.Codec<A, unknown>) => {
|
||||
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
|
||||
return Effect.fn("MediaProtocol.decodeJson")(function* (response: HttpClientResponse.HttpClientResponse) {
|
||||
const output = yield* text(route, name, response)
|
||||
const value = yield* decode(output.body).pipe(
|
||||
Effect.mapError((cause) => output.invalid(`${name} returned an invalid response`, cause)),
|
||||
)
|
||||
return { ...output, value }
|
||||
})
|
||||
}
|
||||
|
||||
export * as MediaProtocol from "./media-protocol.js"
|
||||
@@ -1,116 +0,0 @@
|
||||
import { Effect } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import { Auth } from "./auth.js"
|
||||
import { Endpoint } from "./endpoint.js"
|
||||
import type { Interface } from "./executor-service.js"
|
||||
import { MediaProtocol } from "./media-protocol.js"
|
||||
import { ProviderShared } from "../protocols/shared.js"
|
||||
import { AIError, HttpOptions, ProviderID, mergeHttpOptions } from "../schema/index.js"
|
||||
import { sanitizeSurrogates } from "../utils/sanitize.js"
|
||||
|
||||
export type Execute = Interface["execute"]
|
||||
|
||||
/** The minimum a media request must carry for the route to build a transport request. */
|
||||
export interface MediaRequest {
|
||||
readonly model: { readonly id: string; readonly provider: ProviderID; readonly http?: HttpOptions }
|
||||
readonly http?: HttpOptions
|
||||
}
|
||||
|
||||
/** Deployment inputs every media model factory accepts; provider facades fill these from `configure(...)`. */
|
||||
export interface ModelInput {
|
||||
readonly id: string
|
||||
readonly auth: Auth.Definition
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface Route<Request extends MediaRequest, Response> {
|
||||
readonly id: string
|
||||
readonly provider: ProviderID
|
||||
readonly protocol: string
|
||||
readonly generate: (request: Request, execute: Execute) => Effect.Effect<Response, AIError>
|
||||
}
|
||||
|
||||
export interface MakeInput<Request extends MediaRequest, Response> {
|
||||
readonly id: string
|
||||
readonly provider: string | ProviderID
|
||||
readonly protocol: MediaProtocol.Inline<Request, Response>
|
||||
readonly endpoint: Endpoint.Definition<MediaProtocol.Body, Request>
|
||||
readonly auth: Auth.Definition
|
||||
/** Deployment headers applied before transport authentication. */
|
||||
readonly headers?: Record<string, string>
|
||||
}
|
||||
|
||||
/**
|
||||
* Compose an inline media protocol with an endpoint and auth into a runnable route. The route owns the transport
|
||||
* plumbing every media protocol would otherwise duplicate: option merging, surrogate sanitizing, unsupported-field rejection, URL and query
|
||||
* rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol for decoding.
|
||||
*/
|
||||
export const make = <Request extends MediaRequest, Response>(
|
||||
input: MakeInput<Request, Response>,
|
||||
): Route<Request, Response> => {
|
||||
const provider = ProviderID.make(input.provider)
|
||||
const routeHttp = input.headers === undefined ? undefined : new HttpOptions({ headers: input.headers })
|
||||
const authorize = Auth.toEffect(input.auth)
|
||||
return {
|
||||
id: input.id,
|
||||
provider,
|
||||
protocol: input.protocol.id,
|
||||
generate: Effect.fn(`MediaRoute.generate`)(function* (request: Request, execute: Execute) {
|
||||
yield* rejectUnsupported(input.id, provider, request, input.protocol.unsupported)
|
||||
const http = mergeHttpOptions(routeHttp, request.model.http, request.http)
|
||||
// Sanitize after merging so model-level overlays are covered; the model value is restored, not sanitized.
|
||||
const resolved: Request = { ...sanitizeSurrogates({ ...request, http }), model: request.model }
|
||||
const body = yield* input.protocol.body.from(resolved)
|
||||
const url = Endpoint.render(input.endpoint, { request: resolved, body })
|
||||
for (const [key, value] of Object.entries(http?.query ?? {})) url.searchParams.set(key, value)
|
||||
const encoded = body.type === "json" ? ProviderShared.encodeJson(body.value) : "[multipart/form-data]"
|
||||
const baseHeaders = Headers.fromInput(http?.headers)
|
||||
const headers = yield* authorize({
|
||||
request: resolved,
|
||||
method: "POST",
|
||||
url: url.toString(),
|
||||
body: encoded,
|
||||
// The HTTP client sets the multipart boundary; a caller-supplied content-type would corrupt it.
|
||||
headers: body.type === "multipart" ? Headers.remove(baseHeaders, "content-type") : baseHeaders,
|
||||
})
|
||||
const transport = HttpClientRequest.post(url.toString()).pipe(
|
||||
HttpClientRequest.setHeaders(headers),
|
||||
body.type === "json"
|
||||
? HttpClientRequest.bodyText(encoded, "application/json")
|
||||
: HttpClientRequest.bodyFormData(body.value),
|
||||
)
|
||||
const response = yield* execute(transport)
|
||||
return yield* input.protocol.response.decode(response, { request: resolved, body })
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/** Common fields are never silently dropped: a present field the protocol declared unsupported fails typed. */
|
||||
const rejectUnsupported = <Request extends object>(
|
||||
route: string,
|
||||
provider: ProviderID,
|
||||
request: Request,
|
||||
unsupported: ReadonlyArray<keyof Request & string> | undefined,
|
||||
): Effect.Effect<void, AIError> => {
|
||||
const present = (unsupported ?? []).filter((field) => {
|
||||
const value = request[field]
|
||||
return Array.isArray(value) ? value.length > 0 : value !== undefined
|
||||
})
|
||||
if (present.length === 0) return Effect.void
|
||||
return Effect.fail(
|
||||
ProviderShared.unsupportedOperation({
|
||||
operation: `media.${present[0]}`,
|
||||
provider,
|
||||
route,
|
||||
message: `${provider}/${route} does not support ${present.join(", ")}`,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
export * as MediaRoute from "./media.js"
|
||||
@@ -133,12 +133,6 @@ export class UnknownProviderError extends Schema.TaggedError<UnknownProviderErro
|
||||
ReasonFields,
|
||||
) {}
|
||||
|
||||
/** A caller-supplied deadline elapsed, such as `Job.await` polling past its `Poll.timeout`. */
|
||||
export class TimeoutError extends Schema.TaggedError<TimeoutError>("AI.Error.Timeout")("Timeout", {
|
||||
...ReasonFields,
|
||||
timeoutMs: Schema.optional(Schema.Number),
|
||||
}) {}
|
||||
|
||||
export const AIErrorReason = Schema.Union([
|
||||
InvalidRequestError,
|
||||
UnsupportedOperationError,
|
||||
@@ -151,7 +145,6 @@ export const AIErrorReason = Schema.Union([
|
||||
TransportError,
|
||||
InvalidProviderOutputError,
|
||||
UnknownProviderError,
|
||||
TimeoutError,
|
||||
]).pipe(Schema.toTaggedUnion("_tag"))
|
||||
export type AIErrorReason = Schema.Schema.Type<typeof AIErrorReason>
|
||||
|
||||
|
||||
@@ -4,18 +4,18 @@ import { ContentBlockID, ToolCallID } from "./ids.js"
|
||||
import {
|
||||
Message,
|
||||
CompactionPart,
|
||||
ProviderMetadata,
|
||||
ToolCallPart,
|
||||
ToolOutput,
|
||||
ToolResultPart,
|
||||
ToolResultValue,
|
||||
type ContentPart,
|
||||
} from "./messages.js"
|
||||
import { ProviderMetadata } from "./options.js"
|
||||
import { ProviderFailureClassification } from "./errors.js"
|
||||
import { Media } from "../media.js"
|
||||
|
||||
export const FinishReason = LLM.FinishReason
|
||||
export type FinishReason = Schema.Schema.Type<typeof FinishReason>
|
||||
export { ProviderMetadata } from "./messages.js"
|
||||
|
||||
/**
|
||||
* Token usage reported by an LLM provider.
|
||||
@@ -91,27 +91,6 @@ export class Usage extends Schema.Class<Usage>("AI.Usage")({
|
||||
|
||||
export type UsageInput = Usage | ConstructorParameters<typeof Usage>[0]
|
||||
|
||||
/**
|
||||
* Usage reported by media routes. Providers bill images, video, speech, and transcription in different units, so
|
||||
* each response carries the unit it was actually metered in instead of forcing everything into tokens.
|
||||
*/
|
||||
export const MediaUsage = Schema.Union([
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("tokens"),
|
||||
input: Schema.optional(Schema.Number),
|
||||
output: Schema.optional(Schema.Number),
|
||||
total: Schema.optional(Schema.Number),
|
||||
details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
}),
|
||||
Schema.Struct({ type: Schema.Literal("seconds"), seconds: Schema.Number }),
|
||||
Schema.Struct({ type: Schema.Literal("characters"), characters: Schema.Number }),
|
||||
Schema.Struct({ type: Schema.Literal("credits"), credits: Schema.Number }),
|
||||
Schema.Struct({ type: Schema.Literal("compute"), seconds: Schema.Number }),
|
||||
])
|
||||
.pipe(Schema.toTaggedUnion("type"))
|
||||
.annotate({ identifier: "AI.MediaUsage" })
|
||||
export type MediaUsage = Schema.Schema.Type<typeof MediaUsage>
|
||||
|
||||
/** A replacement context window, not an assistant message to append to prior history. */
|
||||
export class CompactionResponse extends Schema.Class<CompactionResponse>("LLM.CompactionResponse")({
|
||||
replacement: Schema.Array(Message),
|
||||
@@ -284,14 +263,6 @@ export const Finish = Schema.Struct({
|
||||
}).annotate({ identifier: "LLM.Event.Finish" })
|
||||
export type Finish = Schema.Schema.Type<typeof Finish>
|
||||
|
||||
/** A generated media asset (image, audio, …) emitted by the model as first-class output rather than a tool result. */
|
||||
export const MediaEvent = Schema.Struct({
|
||||
type: Schema.tag("media"),
|
||||
media: Media.AssetSchema,
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}).annotate({ identifier: "LLM.Event.Media" })
|
||||
export type MediaEvent = Schema.Schema.Type<typeof MediaEvent>
|
||||
|
||||
export const ProviderErrorEvent = Schema.Struct({
|
||||
type: Schema.tag("provider-error"),
|
||||
message: Schema.String,
|
||||
@@ -316,7 +287,6 @@ const llmEventTagged = Schema.Union([
|
||||
ToolCall,
|
||||
ToolResult,
|
||||
ToolError,
|
||||
MediaEvent,
|
||||
StepFinish,
|
||||
Finish,
|
||||
ProviderErrorEvent,
|
||||
@@ -362,7 +332,6 @@ export const LLMEvent = Object.assign(llmEventTagged, {
|
||||
output: input.output === undefined ? undefined : ToolOutput.make(input.output.structured, input.output.content),
|
||||
}),
|
||||
toolError: (input: WithID<ToolError, ToolCallID>) => ToolError.make({ ...input, id: toolCallID(input.id) }),
|
||||
media: MediaEvent.make,
|
||||
stepFinish: (input: WithUsage<StepFinish>) =>
|
||||
StepFinish.make({
|
||||
...input,
|
||||
@@ -390,7 +359,6 @@ export const LLMEvent = Object.assign(llmEventTagged, {
|
||||
toolCall: llmEventTagged.guards["tool-call"],
|
||||
toolResult: llmEventTagged.guards["tool-result"],
|
||||
toolError: llmEventTagged.guards["tool-error"],
|
||||
media: llmEventTagged.guards.media,
|
||||
stepFinish: llmEventTagged.guards["step-finish"],
|
||||
finish: llmEventTagged.guards.finish,
|
||||
providerError: llmEventTagged.guards["provider-error"],
|
||||
@@ -666,13 +634,6 @@ const reduceResponseState = (state: ResponseState, event: LLMEvent): ResponseSta
|
||||
return reduceToolCall(next, event)
|
||||
case "tool-result":
|
||||
return appendContent(next, toolResultContent(event))
|
||||
case "media":
|
||||
return appendContent(
|
||||
next,
|
||||
event.providerMetadata === undefined
|
||||
? { type: "media", media: event.media }
|
||||
: { type: "media", media: event.media, providerMetadata: event.providerMetadata },
|
||||
)
|
||||
default:
|
||||
return next
|
||||
}
|
||||
|
||||
@@ -8,16 +8,19 @@ import {
|
||||
JsonSchema,
|
||||
LanguageModelSchema,
|
||||
type LanguageModel,
|
||||
ProviderMetadata,
|
||||
ProviderOptions,
|
||||
ReasoningEffort,
|
||||
} from "./options.js"
|
||||
import { ProviderID } from "./ids.js"
|
||||
import { Media } from "../media.js"
|
||||
|
||||
export const MessageRole = Schema.Literals(["system", "user", "assistant", "tool"])
|
||||
export type MessageRole = Schema.Schema.Type<typeof MessageRole>
|
||||
|
||||
export const ProviderMetadata = Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown)).annotate({
|
||||
identifier: "LLM.ProviderMetadata",
|
||||
})
|
||||
export type ProviderMetadata = Schema.Schema.Type<typeof ProviderMetadata>
|
||||
|
||||
const systemPartSchema = Schema.Struct({
|
||||
type: Schema.Literal("text"),
|
||||
text: Schema.String,
|
||||
@@ -47,7 +50,8 @@ export type TextPart = Schema.Schema.Type<typeof TextPart>
|
||||
|
||||
export const MediaPart = Schema.Struct({
|
||||
type: Schema.Literal("media"),
|
||||
media: Media.AssetSchema,
|
||||
mediaType: Schema.String,
|
||||
data: Schema.Union([Schema.String, Schema.Uint8Array]),
|
||||
filename: Schema.optional(Schema.String),
|
||||
cache: Schema.optional(CacheHint),
|
||||
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
@@ -251,12 +255,6 @@ export namespace Message {
|
||||
|
||||
export const text = (value: string): ContentPart => ({ type: "text", text: value })
|
||||
|
||||
export const media = (asset: Media.Asset, options?: Omit<MediaPart, "type" | "media">): MediaPart => ({
|
||||
type: "media",
|
||||
media: asset,
|
||||
...options,
|
||||
})
|
||||
|
||||
export const content = (input: ContentInput) =>
|
||||
typeof input === "string" ? [text(input)] : Array.isArray(input) ? [...input] : [input]
|
||||
|
||||
|
||||
@@ -39,11 +39,6 @@ const mergeStringRecords = (
|
||||
export const ProviderOptions = Schema.Record(Schema.String, Schema.Unknown)
|
||||
export type ProviderOptions = Schema.Schema.Type<typeof ProviderOptions>
|
||||
|
||||
export const ProviderMetadata = Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown)).annotate({
|
||||
identifier: "LLM.ProviderMetadata",
|
||||
})
|
||||
export type ProviderMetadata = Schema.Schema.Type<typeof ProviderMetadata>
|
||||
|
||||
export const mergeProviderOptions = (
|
||||
...items: ReadonlyArray<ProviderOptions | undefined>
|
||||
): ProviderOptions | undefined => mergeJsonRecords(...items)
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
const ascii = (bytes: Uint8Array, start: number, end: number) => String.fromCharCode(...bytes.subarray(start, end))
|
||||
|
||||
const startsWith = (bytes: Uint8Array, prefix: ReadonlyArray<number>) =>
|
||||
bytes.length >= prefix.length && prefix.every((value, index) => bytes[index] === value)
|
||||
|
||||
/**
|
||||
* Sniff a media type from leading magic bytes. Covers the containers media routes commonly return; anything else is
|
||||
* `undefined` so callers can fall back to a provider-declared type or `application/octet-stream`.
|
||||
*/
|
||||
export const detectMediaType = (bytes: Uint8Array): string | undefined => {
|
||||
if (startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) return "image/png"
|
||||
if (startsWith(bytes, [0xff, 0xd8, 0xff])) return "image/jpeg"
|
||||
if (startsWith(bytes, [0x47, 0x49, 0x46, 0x38])) return "image/gif"
|
||||
if (bytes.length >= 12 && ascii(bytes, 0, 4) === "RIFF") {
|
||||
const riffType = ascii(bytes, 8, 12)
|
||||
if (riffType === "WEBP") return "image/webp"
|
||||
if (riffType === "WAVE") return "audio/wav"
|
||||
}
|
||||
if (startsWith(bytes, [0x25, 0x50, 0x44, 0x46])) return "application/pdf"
|
||||
if (bytes.length >= 12 && ascii(bytes, 4, 8) === "ftyp") return "video/mp4"
|
||||
if (startsWith(bytes, [0x1a, 0x45, 0xdf, 0xa3])) return "video/webm"
|
||||
if (startsWith(bytes, [0x49, 0x44, 0x33]) || startsWith(bytes, [0xff, 0xfb]) || startsWith(bytes, [0xff, 0xf3]))
|
||||
return "audio/mpeg"
|
||||
if (startsWith(bytes, [0x4f, 0x67, 0x67, 0x53])) return "audio/ogg"
|
||||
return undefined
|
||||
}
|
||||
|
||||
const EXTENSIONS: Readonly<Record<string, string>> = {
|
||||
png: "image/png",
|
||||
jpg: "image/jpeg",
|
||||
jpeg: "image/jpeg",
|
||||
gif: "image/gif",
|
||||
webp: "image/webp",
|
||||
pdf: "application/pdf",
|
||||
mp4: "video/mp4",
|
||||
webm: "video/webm",
|
||||
mp3: "audio/mpeg",
|
||||
wav: "audio/wav",
|
||||
ogg: "audio/ogg",
|
||||
txt: "text/plain",
|
||||
md: "text/markdown",
|
||||
csv: "text/csv",
|
||||
}
|
||||
|
||||
export const extensionMediaType = (path: string): string | undefined =>
|
||||
EXTENSIONS[path.slice(path.lastIndexOf(".") + 1).toLowerCase()]
|
||||
@@ -1,11 +1,9 @@
|
||||
import { Media } from "../media.js"
|
||||
import { isRecord } from "./record.js"
|
||||
|
||||
export const sanitizeSurrogates = <T>(value: T): T => {
|
||||
if (typeof value === "string") return value.toWellFormed() as T
|
||||
if (Array.isArray(value)) return value.map(sanitizeSurrogates) as T
|
||||
// Media assets carry binary or base64 payloads and a lazy byte cache; flattening them into a record would drop both.
|
||||
if (value instanceof Uint8Array || value instanceof Error || value instanceof Media.Asset) return value
|
||||
if (value instanceof Uint8Array || value instanceof Error) return value
|
||||
if (isRecord(value))
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, entry]) => [key.toWellFormed(), sanitizeSurrogates(entry)]),
|
||||
|
||||
@@ -87,9 +87,9 @@ testEffect(fixedResponse("")).effect(
|
||||
prompt: "hello",
|
||||
})
|
||||
expect(LLMClient.canCompact(request)).toBe(false)
|
||||
const error = yield* LLMClient.compact(request as unknown as Parameters<typeof LLMClient.compact>[0]).pipe(
|
||||
Effect.flip,
|
||||
)
|
||||
const error = yield* LLMClient.compact(
|
||||
request as unknown as Parameters<typeof LLMClient.compact>[0],
|
||||
).pipe(Effect.flip)
|
||||
expect(error.reason._tag).toBe("UnsupportedOperation")
|
||||
expect(error.message).toContain("does not support explicit compaction")
|
||||
if (error.reason._tag === "UnsupportedOperation") {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import {
|
||||
LLM,
|
||||
Media,
|
||||
Message,
|
||||
ToolCallPart,
|
||||
ToolDefinition,
|
||||
@@ -60,7 +59,7 @@ export function continuationRequest(input: {
|
||||
|
||||
if (features.has("user-text")) firstUser.push({ type: "text", text: "What is shown here?" })
|
||||
if (features.has("user-image"))
|
||||
firstUser.push({ type: "media", media: Media.base64(input.image ?? "AAECAw==", "image/png") })
|
||||
firstUser.push({ type: "media", mediaType: "image/png", data: input.image ?? "AAECAw==" })
|
||||
if (firstUser.length > 0) messages.push(Message.user(firstUser))
|
||||
|
||||
if (features.has("assistant-reasoning"))
|
||||
|
||||
@@ -309,7 +309,7 @@ describe("RequestExecutor", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("classifies provider overloads hidden behind HTTP 400", () =>
|
||||
it.effect("does not let server codes override a 4xx rejection", () =>
|
||||
Effect.gen(function* () {
|
||||
const classify = (body: string) =>
|
||||
Effect.gen(function* () {
|
||||
@@ -317,11 +317,11 @@ describe("RequestExecutor", () => {
|
||||
const error = yield* executor.execute(request).pipe(Effect.flip)
|
||||
|
||||
expectAIError(error)
|
||||
expect(error.reason).toMatchObject({ _tag: "ProviderInternal" })
|
||||
expect(error.reason).toMatchObject({ _tag: "InvalidRequest" })
|
||||
}).pipe(Effect.provide(fixedResponse(body, { status: 400 })))
|
||||
|
||||
yield* classify('{"code":"resource_exhausted"}')
|
||||
yield* classify('{"code":"service_unavailable"}')
|
||||
yield* classify('{"error":{"type":"server_error","message":"Upstream request failed: Model is unavailable."}}')
|
||||
}),
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { AIError, LanguageModel, LLM, LLMClient, Media, Provider } from "@opencode/ai"
|
||||
import { AIError, ImageInput, LanguageModel, LLM, LLMClient, Provider } from "@opencode/ai"
|
||||
import { Route, Protocol, WebSocketTransport } from "@opencode/ai/route"
|
||||
import { Provider as ProviderSubpath } from "@opencode/ai/provider"
|
||||
import {
|
||||
@@ -31,7 +31,7 @@ describe("public exports", () => {
|
||||
expect(LLMClient.layer).toBeDefined()
|
||||
expect(AIError).toBeFunction()
|
||||
expect(LanguageModel.make).toBeFunction()
|
||||
expect(Media.bytes).toBeFunction()
|
||||
expect(ImageInput.bytes).toBeFunction()
|
||||
expect(Provider.make).toBeFunction()
|
||||
expect(ProviderSubpath.make).toBe(Provider.make)
|
||||
expect(TestLLM.layer).toBeFunction()
|
||||
|
||||
+40
-104
@@ -1,7 +1,7 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { Image, ImageClient, Media } from "../src/index.js"
|
||||
import { Image, ImageClient, ImageInput } from "../src/index.js"
|
||||
import { Google, OpenAI, XAI, ZAI } from "../src/providers.js"
|
||||
import { it } from "./lib/effect.js"
|
||||
import { dynamicResponse } from "./lib/http.js"
|
||||
@@ -48,11 +48,11 @@ describe("Image", () => {
|
||||
http: { body: { deployment: "test" }, headers: { "x-default": "yes" } },
|
||||
}).image("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
n: 2,
|
||||
size: "2048x2048",
|
||||
format: "jpeg",
|
||||
providerOptions: {
|
||||
options: {
|
||||
n: 2,
|
||||
size: "2048x2048",
|
||||
quality: "future-quality",
|
||||
outputFormat: "jpeg",
|
||||
output_format: "avif",
|
||||
outputCompression: 30,
|
||||
output_compression: 40,
|
||||
@@ -68,14 +68,14 @@ describe("Image", () => {
|
||||
})
|
||||
|
||||
expect(response.images).toHaveLength(2)
|
||||
expect(response.image.mediaType).toBe("image/webp")
|
||||
expect(yield* response.image.bytes()).toEqual(Uint8Array.from([1, 2, 3]))
|
||||
expect(response.image.providerMetadata).toEqual({ openai: { revisedPrompt: "A precise robot" } })
|
||||
expect(response.usage).toMatchObject({ type: "tokens", total: 12 })
|
||||
expect(response.image?.mediaType).toBe("image/webp")
|
||||
expect(response.image?.data).toEqual(Uint8Array.from([1, 2, 3]))
|
||||
expect(response.image?.providerMetadata).toEqual({ openai: { revisedPrompt: "A precise robot" } })
|
||||
expect(response.usage?.totalTokens).toBe(12)
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
Layer.provideMerge(
|
||||
Layer.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
@@ -120,8 +120,8 @@ describe("Image", () => {
|
||||
baseURL: "https://api.openai.test/v1",
|
||||
}).image("future-image-model"),
|
||||
prompt: "A lighthouse in fog",
|
||||
format: "jpeg",
|
||||
providerOptions: {
|
||||
options: {
|
||||
outputFormat: "jpeg",
|
||||
output_format: "avif",
|
||||
outputCompression: 30,
|
||||
output_compression: 40,
|
||||
@@ -130,7 +130,7 @@ describe("Image", () => {
|
||||
}).pipe(
|
||||
Effect.tap((response) =>
|
||||
Effect.sync(() => {
|
||||
expect(response.image.mediaType).toBe("image/avif")
|
||||
expect(response.image?.mediaType).toBe("image/avif")
|
||||
}),
|
||||
),
|
||||
Effect.provide(
|
||||
@@ -160,9 +160,12 @@ describe("Image", () => {
|
||||
Image.generate({
|
||||
model: OpenAI.configure({ apiKey: "test", baseURL: "https://api.openai.test/v1" }).image("future-model"),
|
||||
prompt: "Combine these images",
|
||||
images: [Media.bytes(Uint8Array.from([1, 2, 3]), "image/png"), Media.fromDataUrl("data:image/jpeg;base64,BAUG")],
|
||||
mask: Media.bytes(Uint8Array.from([7, 8, 9]), "image/png"),
|
||||
providerOptions: {
|
||||
images: [
|
||||
ImageInput.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
|
||||
ImageInput.url("data:image/jpeg;base64,BAUG"),
|
||||
],
|
||||
options: {
|
||||
mask: ImageInput.bytes(Uint8Array.from([7, 8, 9]), "image/png"),
|
||||
quality: "high",
|
||||
future_option: true,
|
||||
},
|
||||
@@ -200,8 +203,8 @@ describe("Image", () => {
|
||||
Image.generate({
|
||||
model: OpenAI.configure({ apiKey: "test", baseURL: "https://api.openai.test/v1" }).image("future-model"),
|
||||
prompt: "Combine these images",
|
||||
images: [Media.url("https://example.test/source.png"), Media.ref("openai", "file_123")],
|
||||
mask: Media.ref("openai", "file_mask"),
|
||||
images: [ImageInput.url("https://example.test/source.png"), ImageInput.file("file_123")],
|
||||
options: { mask: ImageInput.file("file_mask") },
|
||||
http: { body: { future_option: true } },
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
@@ -232,9 +235,9 @@ describe("Image", () => {
|
||||
model: XAI.configure({ apiKey: "test", baseURL: "https://api.xai.test/v1" }).image("future-model"),
|
||||
prompt: "Combine these images",
|
||||
images: [
|
||||
Media.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
|
||||
Media.url("https://example.test/source.jpg"),
|
||||
Media.ref("xai", "file_123"),
|
||||
ImageInput.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
|
||||
ImageInput.url("https://example.test/source.jpg"),
|
||||
ImageInput.file("file_123"),
|
||||
],
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
@@ -266,7 +269,7 @@ describe("Image", () => {
|
||||
Image.generate({
|
||||
model: XAI.configure({ apiKey: "test", baseURL: "https://api.xai.test/v1" }).image("future-model"),
|
||||
prompt: "Edit this image",
|
||||
images: [Media.ref("xai", "file_123")],
|
||||
images: [ImageInput.file("file_123")],
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
@@ -294,9 +297,9 @@ describe("Image", () => {
|
||||
model: Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).image("future-model"),
|
||||
prompt: "Combine these images",
|
||||
images: [
|
||||
Media.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
|
||||
Media.fromDataUrl("data:image/jpeg;base64,BAUG"),
|
||||
Media.ref("google", "https://generativelanguage.googleapis.com/v1beta/files/123", "image/webp"),
|
||||
ImageInput.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
|
||||
ImageInput.url("data:image/jpeg;base64,BAUG"),
|
||||
ImageInput.fileUri("https://generativelanguage.googleapis.com/v1beta/files/123", "image/webp"),
|
||||
],
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
@@ -335,20 +338,18 @@ describe("Image", () => {
|
||||
Image.generate({
|
||||
model: Google.configure({ apiKey: "test" }).image("model"),
|
||||
prompt: "edit",
|
||||
images: [Media.url("https://example.test/image.png")],
|
||||
images: [ImageInput.url("https://example.test/image.png")],
|
||||
}),
|
||||
Image.generate({
|
||||
model: ZAI.configure({ apiKey: "test" }).image("model"),
|
||||
prompt: "edit",
|
||||
images: [Media.bytes(Uint8Array.from([1]), "image/png")],
|
||||
images: [ImageInput.bytes(Uint8Array.from([1]), "image/png")],
|
||||
}),
|
||||
]
|
||||
yield* Effect.forEach(cases, (program) =>
|
||||
program.pipe(
|
||||
Effect.flip,
|
||||
Effect.tap((error) =>
|
||||
Effect.sync(() => expect(["InvalidRequest", "UnsupportedOperation"]).toContain(error.reason._tag)),
|
||||
),
|
||||
Effect.tap((error) => Effect.sync(() => expect(error.reason._tag).toBe("InvalidRequest"))),
|
||||
),
|
||||
)
|
||||
}).pipe(
|
||||
@@ -370,10 +371,10 @@ describe("Image", () => {
|
||||
http: { body: { labels: { deployment: "test" } }, query: { api: "v1" } },
|
||||
}).image("any-model-id"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
aspectRatio: "16:9",
|
||||
seed: 42,
|
||||
providerOptions: {
|
||||
options: {
|
||||
aspectRatio: "16:9",
|
||||
imageSize: "2K",
|
||||
seed: 42,
|
||||
thinkingLevel: "HIGH",
|
||||
includeThoughts: true,
|
||||
futureOption: true,
|
||||
@@ -396,7 +397,7 @@ describe("Image", () => {
|
||||
})
|
||||
|
||||
expect(response.images).toHaveLength(3)
|
||||
expect(yield* Effect.forEach(response.images, (image) => image.bytes())).toEqual([
|
||||
expect(response.images.map((image) => image.data)).toEqual([
|
||||
Uint8Array.from([1, 2, 3]),
|
||||
Uint8Array.from([4, 5, 6]),
|
||||
Uint8Array.from([7, 8, 9]),
|
||||
@@ -407,12 +408,10 @@ describe("Image", () => {
|
||||
google: { candidateIndex: 0, partIndex: 3, finishReason: "STOP" },
|
||||
})
|
||||
expect(response.images[2].providerMetadata).toMatchObject({ google: { candidateIndex: 7, partIndex: 0 } })
|
||||
expect(response.usage).toMatchObject({
|
||||
type: "tokens",
|
||||
input: 5,
|
||||
output: 10,
|
||||
details: { reasoningTokens: 3, google: { serviceTier: "STANDARD" } },
|
||||
})
|
||||
expect(response.usage?.inputTokens).toBe(5)
|
||||
expect(response.usage?.outputTokens).toBe(10)
|
||||
expect(response.usage?.reasoningTokens).toBe(3)
|
||||
expect(response.usage?.providerMetadata).toMatchObject({ google: { serviceTier: "STANDARD" } })
|
||||
expect(response.providerMetadata).toEqual({
|
||||
google: {
|
||||
modelVersion: "gemini-3.1-flash-image",
|
||||
@@ -470,7 +469,7 @@ describe("Image", () => {
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
Layer.provideMerge(
|
||||
Layer.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
@@ -550,69 +549,6 @@ describe("Image", () => {
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("surfaces filtered Google candidates as notices next to the returned image", () =>
|
||||
Image.generate({
|
||||
model: Google.configure({ apiKey: "test", baseURL: "https://generativelanguage.test/v1beta" }).image(
|
||||
"gemini-3.1-flash-image",
|
||||
),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
}).pipe(
|
||||
Effect.tap((response) =>
|
||||
Effect.sync(() => {
|
||||
expect(response.images).toHaveLength(1)
|
||||
expect(response.notices).toEqual([
|
||||
{
|
||||
type: "filtered",
|
||||
message: "Google Images reported prompt feedback",
|
||||
providerMetadata: { google: { promptFeedback: { blockReason: "OTHER" } } },
|
||||
},
|
||||
{
|
||||
type: "filtered",
|
||||
message: "Google Images candidate 1 finished with IMAGE_SAFETY: Blocked.",
|
||||
providerMetadata: {
|
||||
google: {
|
||||
candidateIndex: 1,
|
||||
finishReason: "IMAGE_SAFETY",
|
||||
finishMessage: "Blocked.",
|
||||
safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }],
|
||||
},
|
||||
},
|
||||
},
|
||||
])
|
||||
}),
|
||||
),
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
Layer.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.succeed(
|
||||
input.respond(
|
||||
JSON.stringify({
|
||||
promptFeedback: { blockReason: "OTHER" },
|
||||
candidates: [
|
||||
{
|
||||
content: { parts: [{ inlineData: { mimeType: "image/png", data: "AQID" } }] },
|
||||
finishReason: "STOP",
|
||||
},
|
||||
{
|
||||
index: 1,
|
||||
content: { parts: [{ text: "blocked" }] },
|
||||
finishReason: "IMAGE_SAFETY",
|
||||
finishMessage: "Blocked.",
|
||||
safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
{ headers: { "content-type": "application/json" } },
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("includes Google diagnostics when no final image is returned", () =>
|
||||
Image.generate({
|
||||
model: Google.configure({ apiKey: "test", baseURL: "https://generativelanguage.test/v1beta" }).image(
|
||||
|
||||
@@ -2,51 +2,50 @@ import { Effect } from "effect"
|
||||
import {
|
||||
Image,
|
||||
ImageClient,
|
||||
ImageInput,
|
||||
ImageModel,
|
||||
Media,
|
||||
type ImageModelOptions,
|
||||
type ImageOptions,
|
||||
type ImageRequestFor,
|
||||
type ImageRoute,
|
||||
} from "../src/index.js"
|
||||
import type { Service } from "../src/image-client.js"
|
||||
import { Anthropic, Google, OpenAI, XAI, ZAI } from "../src/providers.js"
|
||||
import { Google, OpenAI, XAI, ZAI } from "../src/providers.js"
|
||||
|
||||
type Requirements<T> = T extends Effect.Effect<infer _A, infer _E, infer R> ? R : never
|
||||
type Equal<A, B> = [A, B] extends [B, A] ? true : false
|
||||
type Assert<T extends true> = T
|
||||
|
||||
type GoogleLikeOptions = {
|
||||
readonly aspectRatio?: "1:1" | "16:9"
|
||||
readonly imageSize?: "1K" | "2K"
|
||||
readonly thinkingLevel?: "LOW" | "HIGH"
|
||||
} & Record<string, unknown>
|
||||
|
||||
declare const route: ImageRoute<GoogleLikeOptions>
|
||||
const google = ImageModel.make<GoogleLikeOptions>({ id: "gemini-image", provider: "google", route })
|
||||
// @ts-expect-error Extracted model options retain known provider fields.
|
||||
const invalidGoogleOptions: ImageModelOptions<typeof google> = { imageSize: "8K" }
|
||||
const invalidGoogleOptions: ImageModelOptions<typeof google> = { aspectRatio: "wide" }
|
||||
void invalidGoogleOptions
|
||||
|
||||
Image.generate({
|
||||
model: google,
|
||||
prompt: "A lighthouse",
|
||||
images: [
|
||||
Media.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
|
||||
Media.fromDataUrl("data:image/jpeg;base64,AQID"),
|
||||
Media.ref("google", "https://generativelanguage.googleapis.com/v1beta/files/example", "image/webp"),
|
||||
ImageInput.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
|
||||
ImageInput.url("data:image/jpeg;base64,AQID"),
|
||||
ImageInput.fileUri("https://generativelanguage.googleapis.com/v1beta/files/example", "image/webp"),
|
||||
],
|
||||
aspectRatio: "16:9",
|
||||
seed: 7,
|
||||
providerOptions: { imageSize: "2K", thinkingLevel: "HIGH", futureOption: true },
|
||||
options: { aspectRatio: "16:9", imageSize: "2K", futureOption: true },
|
||||
})
|
||||
|
||||
const googleProvider = Google.configure({ apiKey: "test" }).image("any-model-id")
|
||||
Image.generate({
|
||||
model: googleProvider,
|
||||
prompt: "A lighthouse",
|
||||
aspectRatio: "16:9",
|
||||
providerOptions: {
|
||||
options: {
|
||||
aspectRatio: "16:9",
|
||||
imageSize: "2K",
|
||||
seed: 42,
|
||||
thinkingLevel: "HIGH",
|
||||
includeThoughts: true,
|
||||
futureOption: true,
|
||||
@@ -55,70 +54,57 @@ Image.generate({
|
||||
Image.generate({
|
||||
model: googleProvider,
|
||||
prompt: "A lighthouse",
|
||||
providerOptions: { imageSize: "8K", thinkingLevel: "FUTURE" },
|
||||
options: { aspectRatio: "future-ratio", imageSize: "8K", thinkingLevel: "FUTURE" },
|
||||
})
|
||||
// @ts-expect-error Image generation options are request-scoped, not provider configuration.
|
||||
Google.configure({ image: { providerOptions: { imageSize: "2K" } } })
|
||||
// @ts-expect-error Known Google string options retain their value kind.
|
||||
Image.generate({ model: googleProvider, prompt: "A lighthouse", providerOptions: { imageSize: 2 } })
|
||||
Image.generate({ model: googleProvider, prompt: "A lighthouse", options: { imageSize: 2 } })
|
||||
// @ts-expect-error Known Google numeric options retain their value kind.
|
||||
Image.generate({ model: googleProvider, prompt: "A lighthouse", options: { seed: "42" } })
|
||||
// @ts-expect-error Known Google boolean options retain their value kind.
|
||||
Image.generate({ model: googleProvider, prompt: "A lighthouse", providerOptions: { includeThoughts: "yes" } })
|
||||
Image.generate({ model: googleProvider, prompt: "A lighthouse", options: { includeThoughts: "yes" } })
|
||||
|
||||
const openai = OpenAI.image("gpt-image-2")
|
||||
// @ts-expect-error Image generation options are request-scoped, not provider configuration.
|
||||
OpenAI.configure({ image: { providerOptions: { quality: "medium" } } })
|
||||
OpenAI.configure({ image: { options: { quality: "medium" } } })
|
||||
const futureOpenAIOptions: ImageModelOptions<typeof openai> = { quality: "future-quality" }
|
||||
void futureOpenAIOptions
|
||||
Image.generate({
|
||||
model: openai,
|
||||
prompt: "A lighthouse",
|
||||
images: [Media.url("https://example.com/source.png"), Media.ref("openai", "file_123")],
|
||||
mask: Media.bytes(Uint8Array.from([1]), "image/png"),
|
||||
n: 2,
|
||||
size: "2048x2048",
|
||||
format: "webp",
|
||||
providerOptions: {
|
||||
images: [ImageInput.url("https://example.com/source.png"), ImageInput.file("file_123")],
|
||||
options: {
|
||||
mask: ImageInput.bytes(Uint8Array.from([1]), "image/png"),
|
||||
quality: "hd",
|
||||
background: "transparent",
|
||||
outputFormat: "webp",
|
||||
size: "2048x2048",
|
||||
future_option: true,
|
||||
},
|
||||
})
|
||||
Image.generate({
|
||||
model: openai,
|
||||
prompt: "A lighthouse",
|
||||
size: "256x256",
|
||||
providerOptions: { quality: "future-quality" },
|
||||
})
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", format: "future-format" })
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", providerOptions: { native_future_option: true } })
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", options: { quality: "future-quality", size: "256x256" } })
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", options: { size: "1792x1024" } })
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", options: { native_future_option: true } })
|
||||
// @ts-expect-error Known OpenAI string options retain their value kind.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", providerOptions: { quality: 1 } })
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", options: { quality: 1 } })
|
||||
// @ts-expect-error Known OpenAI numeric options retain their value kind.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", providerOptions: { outputCompression: "80" } })
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", options: { outputCompression: "80" } })
|
||||
OpenAI.imageGeneration({ action: "future-action", quality: "future-quality", size: "2048x2048" })
|
||||
// @ts-expect-error Hosted image generation numeric options retain their value kind.
|
||||
OpenAI.imageGeneration({ partialImages: "2" })
|
||||
// @ts-expect-error Known Google-like options are inferred from the selected model.
|
||||
Image.generate({ model: google, prompt: "A lighthouse", providerOptions: { imageSize: "8K" } })
|
||||
|
||||
// Callable facades resolve the image route from the ref and infer its options.
|
||||
const openaiRef = OpenAI.configure({ apiKey: "test" })("gpt-image-2")
|
||||
Image.generate({ model: openaiRef, prompt: "A lighthouse", providerOptions: { quality: "high" } })
|
||||
// @ts-expect-error Ref-resolved OpenAI options retain their value kind.
|
||||
Image.generate({ model: openaiRef, prompt: "A lighthouse", providerOptions: { quality: 1 } })
|
||||
// @ts-expect-error Providers without an image route cannot be used for image requests.
|
||||
Image.generate({ model: Anthropic.configure({ apiKey: "test" })("claude-sonnet-4-5"), prompt: "A lighthouse" })
|
||||
Image.generate({ model: google, prompt: "A lighthouse", options: { aspectRatio: "wide" } })
|
||||
|
||||
const xai = XAI.configure({ apiKey: "test" }).image("any-model-id")
|
||||
// @ts-expect-error Image generation options are request-scoped, not provider configuration.
|
||||
XAI.configure({ image: { providerOptions: { resolution: "1k" } } })
|
||||
XAI.configure({ image: { options: { resolution: "1k" } } })
|
||||
Image.generate({
|
||||
model: xai,
|
||||
prompt: "A lighthouse",
|
||||
images: [Media.fromDataUrl("data:image/png;base64,AQID"), Media.ref("xai", "file_123")],
|
||||
n: 2,
|
||||
aspectRatio: "16:9",
|
||||
providerOptions: {
|
||||
images: [ImageInput.url("data:image/png;base64,AQID"), ImageInput.file("file_123")],
|
||||
options: {
|
||||
n: 2,
|
||||
aspectRatio: "future-ratio",
|
||||
resolution: "future-resolution",
|
||||
responseFormat: "future-format",
|
||||
future_option: true,
|
||||
@@ -127,43 +113,43 @@ Image.generate({
|
||||
Image.generate({
|
||||
model: xai,
|
||||
prompt: "A lighthouse",
|
||||
providerOptions: { response_format: "b64_json", native_future_option: true },
|
||||
options: { aspect_ratio: "16:9", response_format: "b64_json", native_future_option: true },
|
||||
})
|
||||
// @ts-expect-error Common count is numeric.
|
||||
Image.generate({ model: xai, prompt: "A lighthouse", n: "2" })
|
||||
// @ts-expect-error Known xAI numeric options retain their value kind.
|
||||
Image.generate({ model: xai, prompt: "A lighthouse", options: { n: "2" } })
|
||||
// @ts-expect-error Known xAI string options retain their value kind.
|
||||
Image.generate({ model: xai, prompt: "A lighthouse", providerOptions: { resolution: 2 } })
|
||||
Image.generate({ model: xai, prompt: "A lighthouse", options: { resolution: 2 } })
|
||||
|
||||
const zai = ZAI.configure({ apiKey: "test" }).image("any-model-id")
|
||||
// @ts-expect-error Image generation options are request-scoped, not provider configuration.
|
||||
ZAI.configure({ image: { providerOptions: { quality: "hd" } } })
|
||||
ZAI.configure({ image: { options: { quality: "hd" } } })
|
||||
Image.generate({
|
||||
model: zai,
|
||||
prompt: "A lighthouse",
|
||||
providerOptions: { quality: "future-quality", userID: "user-123", future_option: true },
|
||||
options: { quality: "future-quality", userID: "user-123", future_option: true },
|
||||
})
|
||||
Image.generate({ model: zai, prompt: "A lighthouse", providerOptions: { user_id: "raw-user" } })
|
||||
Image.generate({ model: zai, prompt: "A lighthouse", options: { user_id: "raw-user" } })
|
||||
// @ts-expect-error Known Z.ai string options retain their value kind.
|
||||
Image.generate({ model: zai, prompt: "A lighthouse", providerOptions: { quality: 1 } })
|
||||
Image.generate({ model: zai, prompt: "A lighthouse", options: { quality: 1 } })
|
||||
// @ts-expect-error Known Z.ai user IDs retain their value kind.
|
||||
Image.generate({ model: zai, prompt: "A lighthouse", providerOptions: { userID: 1 } })
|
||||
Image.generate({ model: zai, prompt: "A lighthouse", options: { userID: 1 } })
|
||||
|
||||
declare const generic: ImageModel<ImageOptions>
|
||||
Image.generate({ model: generic, prompt: "A lighthouse", providerOptions: { arbitrary: true } })
|
||||
const explicitAsset: Media.Asset = Media.url("https://example.com/image.png")
|
||||
void explicitAsset
|
||||
Image.generate({ model: generic, prompt: "A lighthouse", options: { arbitrary: true } })
|
||||
const explicitImageInput: ImageInput = ImageInput.url("https://example.com/image.png")
|
||||
void explicitImageInput
|
||||
|
||||
// @ts-expect-error Raw strings are ambiguous and are not media assets.
|
||||
// @ts-expect-error Raw strings are ambiguous and are not image inputs.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", images: ["AQID"] })
|
||||
// @ts-expect-error Plain source objects must be lifted into `Media.Asset` first.
|
||||
// @ts-expect-error Byte image inputs require an explicit MIME type.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", images: [{ type: "bytes", data: new Uint8Array() }] })
|
||||
// @ts-expect-error Masks are media assets, not strings.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", mask: "https://example.com/mask.png" })
|
||||
// @ts-expect-error File URIs require an explicit MIME type for Gemini fileData.
|
||||
Image.generate({ model: google, prompt: "A lighthouse", images: [{ type: "file-uri", uri: "files/123" }] })
|
||||
|
||||
const request = Image.request({
|
||||
model: google,
|
||||
prompt: "A lighthouse",
|
||||
providerOptions: { imageSize: "1K", futureOption: true },
|
||||
options: { aspectRatio: "1:1", futureOption: true },
|
||||
})
|
||||
const typedRequest: ImageRequestFor<GoogleLikeOptions> = request
|
||||
void typedRequest
|
||||
@@ -171,13 +157,15 @@ const generated = ImageClient.generate(request)
|
||||
type GenerateRequirements = Assert<Equal<Requirements<typeof generated>, Service>>
|
||||
void (true satisfies GenerateRequirements)
|
||||
|
||||
// @ts-expect-error Image requests use `n`, not `count`.
|
||||
// @ts-expect-error Image requests no longer expose a common count option.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", count: 2 })
|
||||
// @ts-expect-error Image sizes are `${width}x${height}` strings.
|
||||
// @ts-expect-error Image requests no longer expose a common size option.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", size: { width: 1024, height: 1024 } })
|
||||
// @ts-expect-error Aspect ratios are `${w}:${h}` strings.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", aspectRatio: "wide" })
|
||||
// @ts-expect-error Image requests no longer expose a common aspectRatio option.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", aspectRatio: "16:9" })
|
||||
// @ts-expect-error Image requests no longer expose a common seed option.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", seed: 1 })
|
||||
// @ts-expect-error Image requests do not expose metadata.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", metadata: { trace: true } })
|
||||
// @ts-expect-error `options` was renamed to `providerOptions`.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", options: { quality: "hd" } })
|
||||
// @ts-expect-error Masks are provider options, not a common image request field.
|
||||
Image.generate({ model: openai, prompt: "A lighthouse", mask: ImageInput.url("https://example.com/mask.png") })
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect, Fiber, Ref, Stream } from "effect"
|
||||
import * as TestClock from "effect/testing/TestClock"
|
||||
import {
|
||||
AIError,
|
||||
InvalidProviderOutputError,
|
||||
Job,
|
||||
type JobRoute,
|
||||
type JobSnapshot,
|
||||
type JobStatus,
|
||||
} from "../src/index.js"
|
||||
import { it } from "./lib/effect.js"
|
||||
|
||||
/** In-memory job route whose status advances through `statuses` on every poll. */
|
||||
const scriptedRoute = (statuses: ReadonlyArray<JobStatus>, result: string) =>
|
||||
Effect.gen(function* () {
|
||||
const polls = yield* Ref.make(0)
|
||||
const cancelled = yield* Ref.make(false)
|
||||
// `count` is the number of polls so far; the first poll observes `statuses[0]`.
|
||||
const snapshot = (count: number): JobSnapshot => ({
|
||||
id: "job_1",
|
||||
status: statuses[Math.min(Math.max(count - 1, 0), statuses.length - 1)],
|
||||
progress: count / statuses.length,
|
||||
})
|
||||
const route: JobRoute<string> = {
|
||||
status: () => Ref.updateAndGet(polls, (count) => count + 1).pipe(Effect.map(snapshot)),
|
||||
result: (token) =>
|
||||
Effect.gen(function* () {
|
||||
const count = yield* Ref.get(polls)
|
||||
const status = snapshot(count).status
|
||||
if (status === "completed") return `${result}:${String(token)}`
|
||||
return yield* new AIError({ reason: new InvalidProviderOutputError({ message: `Job ended ${status}` }) })
|
||||
}),
|
||||
cancel: () => Ref.set(cancelled, true),
|
||||
}
|
||||
return { route, polls, cancelled }
|
||||
})
|
||||
|
||||
describe("Job", () => {
|
||||
it.effect("polls queued → running → completed and returns the result", () =>
|
||||
Effect.gen(function* () {
|
||||
const scripted = yield* scriptedRoute(["queued", "running", "completed"], "done")
|
||||
const job = new Job(scripted.route, { op: "token_1" }, { id: "job_1", status: "queued" })
|
||||
expect(job.terminal).toBe(false)
|
||||
|
||||
const fiber = yield* Effect.forkChild(job.await({ poll: { interval: "1 second", timeout: "1 minute" } }))
|
||||
yield* TestClock.adjust("3 seconds")
|
||||
const result = yield* Fiber.join(fiber)
|
||||
|
||||
expect(result).toBe("done:[object Object]")
|
||||
expect(yield* Ref.get(scripted.polls)).toBe(3)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("returns immediately for an already terminal job", () =>
|
||||
Effect.gen(function* () {
|
||||
const scripted = yield* scriptedRoute(["completed"], "done")
|
||||
yield* Ref.set(scripted.polls, 1)
|
||||
const job = new Job(scripted.route, "t", { id: "job_1", status: "completed" })
|
||||
expect(yield* job.await()).toBe("done:t")
|
||||
expect(yield* Ref.get(scripted.polls)).toBe(1)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails with a Timeout reason when the job never finishes", () =>
|
||||
Effect.gen(function* () {
|
||||
const scripted = yield* scriptedRoute(["running"], "never")
|
||||
const job = new Job(scripted.route, "t", { id: "job_1", status: "queued" })
|
||||
|
||||
const fiber = yield* Effect.forkChild(
|
||||
job.await({ poll: { interval: "1 second", timeout: "5 seconds" } }).pipe(Effect.flip),
|
||||
)
|
||||
yield* TestClock.adjust("6 seconds")
|
||||
const error = yield* Fiber.join(fiber)
|
||||
|
||||
expect(error).toBeInstanceOf(AIError)
|
||||
expect(error.reason._tag).toBe("Timeout")
|
||||
expect(error.message).toContain("job_1")
|
||||
expect(yield* Ref.get(scripted.polls)).toBeGreaterThan(1)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("surfaces the route failure body for failed jobs", () =>
|
||||
Effect.gen(function* () {
|
||||
const scripted = yield* scriptedRoute(["running", "failed"], "unused")
|
||||
const job = new Job(scripted.route, "t", { id: "job_1", status: "queued" })
|
||||
|
||||
const fiber = yield* Effect.forkChild(job.await({ poll: { interval: "1 second" } }).pipe(Effect.flip))
|
||||
yield* TestClock.adjust("2 seconds")
|
||||
const error = yield* Fiber.join(fiber)
|
||||
|
||||
expect(error.reason._tag).toBe("InvalidProviderOutput")
|
||||
expect(error.message).toContain("Job ended failed")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("streams status events until the first terminal observation and cancels through the route", () =>
|
||||
Effect.gen(function* () {
|
||||
const scripted = yield* scriptedRoute(["queued", "running", "completed"], "done")
|
||||
const job = new Job(scripted.route, "t", { id: "job_1", status: "queued" })
|
||||
|
||||
const fiber = yield* Effect.forkChild(job.events({ poll: { interval: "1 second" } }).pipe(Stream.runCollect))
|
||||
yield* TestClock.adjust("3 seconds")
|
||||
const events = Array.from(yield* Fiber.join(fiber))
|
||||
|
||||
expect(events).toEqual([
|
||||
{ type: "job-queued", id: "job_1", position: undefined },
|
||||
{ type: "job-progress", id: "job_1", progress: 2 / 3 },
|
||||
{ type: "job-finished", id: "job_1", status: "completed" },
|
||||
])
|
||||
|
||||
yield* job.cancel()
|
||||
expect(yield* Ref.get(scripted.cancelled)).toBe(true)
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -17,7 +17,7 @@ export type Handler = (
|
||||
input: HandlerInput,
|
||||
) => Effect.Effect<HttpClientResponse.HttpClientResponse, HttpClientError.HttpClientError>
|
||||
|
||||
export const handlerLayer = (handler: Handler): Layer.Layer<HttpClient.HttpClient> =>
|
||||
const handlerLayer = (handler: Handler): Layer.Layer<HttpClient.HttpClient> =>
|
||||
Layer.succeed(
|
||||
HttpClient.HttpClient,
|
||||
HttpClient.make((request) =>
|
||||
|
||||
@@ -1,180 +0,0 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { NodeFileSystem } from "@effect/platform-node"
|
||||
import { Effect, Ref, Schema } from "effect"
|
||||
import { FileSystem } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { Media, Message } from "../src/index.js"
|
||||
import { it } from "./lib/effect.js"
|
||||
import { dynamicResponse, scriptedResponses } from "./lib/http.js"
|
||||
|
||||
const PNG = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0])
|
||||
const JPEG = Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0, 0])
|
||||
const GIF = Uint8Array.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61])
|
||||
const WEBP = Uint8Array.from([0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x45, 0x42, 0x50])
|
||||
const PDF = Uint8Array.from([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31])
|
||||
|
||||
describe("Media", () => {
|
||||
it.effect("round-trips every source through Media.from(asset.source)", () =>
|
||||
Effect.sync(() => {
|
||||
const assets = [
|
||||
Media.bytes(PNG),
|
||||
Media.base64("AQID", "image/png"),
|
||||
Media.url("https://example.test/a.png", { mediaType: "image/png", expiresAt: 123, headers: { a: "b" } }),
|
||||
Media.ref("openai", "file_123", "image/webp"),
|
||||
]
|
||||
for (const asset of assets) {
|
||||
const copy = Media.from(asset.source)
|
||||
expect(copy.source).toEqual(asset.source)
|
||||
expect(copy.mediaType).toBe(asset.mediaType)
|
||||
expect(copy.kind).toBe(asset.kind)
|
||||
expect(copy.expiresAt).toBe(asset.expiresAt)
|
||||
}
|
||||
expect(assets.map((asset) => asset.kind)).toEqual(["image", "image", "image", "image"])
|
||||
expect(Media.url("https://example.test/unknown").mediaType).toBe("application/octet-stream")
|
||||
expect(Media.url("https://example.test/unknown").kind).toBe("other")
|
||||
expect(Media.base64("AQID", "application/pdf").kind).toBe("document")
|
||||
expect(Media.base64("AQID", "video/mp4").kind).toBe("video")
|
||||
expect(Media.base64("AQID", "audio/wav").kind).toBe("audio")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("parses data URLs and rejects malformed ones", () =>
|
||||
Effect.sync(() => {
|
||||
const asset = Media.fromDataUrl("data:image/jpeg;base64,/9j/")
|
||||
expect(asset.source).toEqual({ type: "base64", data: "/9j/", mediaType: "image/jpeg" })
|
||||
expect(asset.mediaType).toBe("image/jpeg")
|
||||
expect(Media.fromDataUrl("data:text/plain;charset=utf-8;base64,aGk=").source).toEqual({
|
||||
type: "base64",
|
||||
data: "aGk=",
|
||||
mediaType: "text/plain",
|
||||
})
|
||||
expect(() => Media.fromDataUrl("https://example.test/a.png")).toThrow(
|
||||
"Media data URLs must contain a MIME type and base64 data",
|
||||
)
|
||||
expect(() => Media.fromDataUrl("data:image/png,rawtext")).toThrow()
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("detects media types from magic bytes", () =>
|
||||
Effect.sync(() => {
|
||||
expect(Media.detectMediaType(PNG)).toBe("image/png")
|
||||
expect(Media.detectMediaType(JPEG)).toBe("image/jpeg")
|
||||
expect(Media.detectMediaType(GIF)).toBe("image/gif")
|
||||
expect(Media.detectMediaType(WEBP)).toBe("image/webp")
|
||||
expect(Media.detectMediaType(PDF)).toBe("application/pdf")
|
||||
expect(Media.detectMediaType(Uint8Array.from([1, 2, 3]))).toBeUndefined()
|
||||
expect(Media.bytes(PNG).mediaType).toBe("image/png")
|
||||
expect(Media.bytes(Uint8Array.from([1, 2, 3])).mediaType).toBe("application/octet-stream")
|
||||
expect(Media.bytes(Uint8Array.from([1, 2, 3]), "image/x-custom").mediaType).toBe("image/x-custom")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("materializes url assets through the request executor once", () =>
|
||||
Effect.gen(function* () {
|
||||
const requests = yield* Ref.make<Array<HttpClientRequest.HttpClientRequest>>([])
|
||||
const asset = Media.url("https://cdn.example.test/generated", {
|
||||
expiresAt: 42,
|
||||
headers: { authorization: "Bearer media" },
|
||||
providerMetadata: { example: { id: "gen_1" } },
|
||||
})
|
||||
const program = Effect.gen(function* () {
|
||||
const first = yield* asset.bytes()
|
||||
const second = yield* asset.bytes()
|
||||
expect(second).toBe(first)
|
||||
expect(first).toEqual(PNG)
|
||||
expect(yield* asset.base64()).toBe(Buffer.from(PNG).toString("base64"))
|
||||
expect(yield* asset.dataUrl()).toBe(
|
||||
`data:application/octet-stream;base64,${Buffer.from(PNG).toString("base64")}`,
|
||||
)
|
||||
|
||||
const owned = yield* asset.materialize()
|
||||
expect(owned.source).toEqual({ type: "bytes", data: PNG, mediaType: "image/png" })
|
||||
expect(owned.mediaType).toBe("image/png")
|
||||
expect(owned.kind).toBe("image")
|
||||
expect(owned.expiresAt).toBeUndefined()
|
||||
expect(owned.providerMetadata).toEqual({ example: { id: "gen_1" } })
|
||||
expect(yield* owned.materialize()).toBe(owned)
|
||||
|
||||
const seen = yield* Ref.get(requests)
|
||||
expect(seen).toHaveLength(1)
|
||||
expect(seen[0].url).toBe("https://cdn.example.test/generated")
|
||||
expect(seen[0].headers.authorization).toBe("Bearer media")
|
||||
})
|
||||
yield* program.pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Ref.update(requests, (all) => [...all, input.request]).pipe(
|
||||
Effect.map(() => input.respond(PNG, { headers: { "content-type": "image/png" } })),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("caches decoded base64 bytes and encodes owned bytes lazily", () =>
|
||||
Effect.gen(function* () {
|
||||
const fromBase64 = Media.base64("AQID", "image/png")
|
||||
const decoded = yield* fromBase64.bytes()
|
||||
expect(decoded).toEqual(Uint8Array.from([1, 2, 3]))
|
||||
expect(yield* fromBase64.bytes()).toBe(decoded)
|
||||
expect(yield* fromBase64.base64()).toBe("AQID")
|
||||
|
||||
const fromBytes = Media.bytes(Uint8Array.from([1, 2, 3]), "image/png")
|
||||
const encoded = yield* fromBytes.base64()
|
||||
expect(encoded).toBe("AQID")
|
||||
expect(yield* fromBytes.base64()).toBe(encoded)
|
||||
expect(yield* fromBytes.dataUrl()).toBe("data:image/png;base64,AQID")
|
||||
|
||||
const invalid = yield* Media.base64("not base64!", "image/png").bytes().pipe(Effect.flip)
|
||||
expect(invalid.reason._tag).toBe("InvalidRequest")
|
||||
const ref = yield* Media.ref("openai", "file_1").bytes().pipe(Effect.flip)
|
||||
expect(ref.reason._tag).toBe("InvalidRequest")
|
||||
}).pipe(Effect.provide(scriptedResponses(["unused"]))),
|
||||
)
|
||||
|
||||
it.effect("serializes assets inside messages and restores them as Media.Asset", () =>
|
||||
Effect.sync(() => {
|
||||
const codec = Schema.fromJsonString(Message)
|
||||
const message = Message.user([
|
||||
Message.media(Media.base64("AQID", "image/png", { info: { width: 1, height: 1 } }), { filename: "a.png" }),
|
||||
Message.media(Media.url("https://example.test/b.pdf", { mediaType: "application/pdf", expiresAt: 7 })),
|
||||
])
|
||||
const json = Schema.encodeSync(codec)(message)
|
||||
expect(json).not.toContain("null")
|
||||
const restored = Schema.decodeSync(codec)(json)
|
||||
const parts = restored.content.filter((part) => part.type === "media")
|
||||
expect(parts).toHaveLength(2)
|
||||
expect(parts[0].media).toBeInstanceOf(Media.Asset)
|
||||
expect(parts[0].media.source).toEqual({ type: "base64", data: "AQID", mediaType: "image/png" })
|
||||
expect(parts[0].media.info).toEqual({ width: 1, height: 1 })
|
||||
expect(parts[0].filename).toBe("a.png")
|
||||
expect(parts[1].media.kind).toBe("document")
|
||||
expect(parts[1].media.expiresAt).toBe(7)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("reads files with sniffed media types and writes materialized assets", () =>
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* FileSystem.FileSystem
|
||||
const dir = yield* fs.makeTempDirectoryScoped()
|
||||
const source = `${dir}/source.bin`
|
||||
yield* fs.writeFile(source, PNG)
|
||||
|
||||
const asset = yield* Media.file(source)
|
||||
expect(asset.mediaType).toBe("image/png")
|
||||
expect(asset.source).toEqual({ type: "bytes", data: PNG, mediaType: "image/png" })
|
||||
|
||||
const plain = `${dir}/notes.md`
|
||||
yield* fs.writeFile(plain, new TextEncoder().encode("# hi"))
|
||||
expect((yield* Media.file(plain)).mediaType).toBe("text/markdown")
|
||||
|
||||
const target = `${dir}/copy.png`
|
||||
yield* Media.write(Media.base64("AQID", "image/png"), target)
|
||||
expect(yield* fs.readFile(target)).toEqual(Uint8Array.from([1, 2, 3]))
|
||||
|
||||
const missing = yield* Media.file(`${dir}/missing.png`).pipe(Effect.flip)
|
||||
expect(missing.reason._tag).toBe("InvalidRequest")
|
||||
}).pipe(Effect.provide(NodeFileSystem.layer), Effect.provide(scriptedResponses(["unused"]))),
|
||||
)
|
||||
})
|
||||
@@ -1,108 +0,0 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { Image, ImageClient, ImageModel, LLM, ModelID, ModelRef, ProviderID, type LanguageModel } from "../src/index.js"
|
||||
import { Anthropic, Google, OpenAI } from "../src/providers.js"
|
||||
import { compileRequest } from "../src/route/client.js"
|
||||
import { it } from "./lib/effect.js"
|
||||
import { dynamicResponse } from "./lib/http.js"
|
||||
|
||||
describe("ModelRef", () => {
|
||||
it.effect("callable facades return refs that resolve to the default LLM route", () =>
|
||||
Effect.gen(function* () {
|
||||
const openai = OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" })
|
||||
const ref = openai("gpt-5")
|
||||
expect(ref).toBeInstanceOf(ModelRef)
|
||||
expect(ref.id).toBe(ModelID.make("gpt-5"))
|
||||
expect(ref.provider).toBe(ProviderID.make("openai"))
|
||||
|
||||
const request = LLM.request({ model: ref, prompt: "Hello", providerOptions: { reasoningEffort: "high" } })
|
||||
expect(request.model.id).toBe(ModelID.make("gpt-5"))
|
||||
expect(request.model.route.id).toBe("openai-responses")
|
||||
expect(request.model.provider).toBe(openai.responses("gpt-5").provider)
|
||||
expect(LLM.request({ model: openai.chat("gpt-4o"), prompt: "Hello" }).model.route.id).toBe("openai-chat")
|
||||
|
||||
const prepared = yield* compileRequest(request)
|
||||
expect(prepared.route).toBe("openai-responses")
|
||||
expect(prepared.body).toMatchObject({ model: "gpt-5", reasoning: { effort: "high" } })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("resolves the image route for image requests", () =>
|
||||
Effect.gen(function* () {
|
||||
const openai = OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" })
|
||||
const request = Image.request({
|
||||
model: openai("gpt-image-2"),
|
||||
prompt: "A lighthouse",
|
||||
providerOptions: { quality: "high" },
|
||||
})
|
||||
expect(request.model).toBeInstanceOf(ImageModel)
|
||||
expect(request.model.id).toBe(ModelID.make("gpt-image-2"))
|
||||
expect(request.model.route.id).toBe("openai-images")
|
||||
expect(
|
||||
Image.request({ model: Google.configure({ apiKey: "test" })("gemini-image"), prompt: "x" }).model.route.id,
|
||||
).toBe("google-images")
|
||||
|
||||
const response = yield* Image.generate({ model: openai("gpt-image-2"), prompt: "A lighthouse" }).pipe(
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
Layer.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.url).toBe("https://openai.test/v1/images/generations")
|
||||
expect(JSON.parse(input.text)).toEqual({ model: "gpt-image-2", prompt: "A lighthouse" })
|
||||
return input.respond(JSON.stringify({ data: [{ b64_json: "AQID" }] }), {
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
expect(response.image.source).toEqual({ type: "bytes", data: Uint8Array.from([1, 2, 3]), mediaType: "image/png" })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails typed when a provider has no route for the requested modality", () =>
|
||||
Effect.gen(function* () {
|
||||
const ref = Anthropic.configure({ apiKey: "test" })("claude-sonnet-4-5")
|
||||
expect("image" in ref.facade).toBe(false)
|
||||
const error = yield* Image.generate({
|
||||
// The type system rejects this; the runtime must still fail with a typed error.
|
||||
model: ref as unknown as ModelRef.WithImage,
|
||||
prompt: "A lighthouse",
|
||||
}).pipe(
|
||||
Effect.flip,
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(Layer.provide(dynamicResponse(() => Effect.die("unrouted request reached HTTP")))),
|
||||
),
|
||||
)
|
||||
expect(error.reason._tag).toBe("UnsupportedOperation")
|
||||
expect(error.message).toContain("anthropic does not expose an image route")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("resolves selectors lazily and keeps the facade as the source of truth", () =>
|
||||
Effect.sync(() => {
|
||||
let built = 0
|
||||
const model = OpenAI.configure({ apiKey: "test" }).responses("gpt-5")
|
||||
const facade = ModelRef.facade({
|
||||
id: model.provider,
|
||||
model: (id): LanguageModel => {
|
||||
built += 1
|
||||
return OpenAI.configure({ apiKey: "test" }).responses(id)
|
||||
},
|
||||
configure: () => undefined,
|
||||
})
|
||||
const ref = facade("gpt-5")
|
||||
expect(built).toBe(0)
|
||||
expect(ref.facade.model).toBe(facade.model)
|
||||
expect(ref.provider).toBe(model.provider)
|
||||
expect(LLM.request({ model: ref, prompt: "Hi" }).model.id).toBe(ModelID.make("gpt-5"))
|
||||
expect(built).toBe(1)
|
||||
expect(facade.model(ModelID.make("gpt-4o")).id).toBe(ModelID.make("gpt-4o"))
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -1,90 +0,0 @@
|
||||
import { Image, ImageModel, LanguageModel, LLM, LLMRequest, ModelRef, type ResolveLanguageModel } from "../src/index.js"
|
||||
import type { OpenAIImageOptions } from "../src/protocols/openai-images.js"
|
||||
import type { GoogleImageOptions } from "../src/protocols/google-images.js"
|
||||
import type { OpenAIProviderOptionsInput } from "../src/providers/openai-options.js"
|
||||
import { Anthropic, Google, OpenAI } from "../src/providers.js"
|
||||
|
||||
type Equal<A, B> = [A, B] extends [B, A] ? true : false
|
||||
type Assert<T extends true> = T
|
||||
|
||||
const openai = OpenAI.configure({ apiKey: "test" })
|
||||
const google = Google.configure({ apiKey: "test" })
|
||||
const anthropic = Anthropic.configure({ apiKey: "test" })
|
||||
|
||||
// The callable facade keeps its named selectors.
|
||||
openai.responses("gpt-5")
|
||||
openai.chat("gpt-4o")
|
||||
openai.image("gpt-image-2")
|
||||
openai.model("gpt-5")
|
||||
openai.configure({ apiKey: "other" })
|
||||
|
||||
// Refs carry lazy routes and resolve per request namespace.
|
||||
const ref = openai("gpt-5")
|
||||
type RefIsModelRef = Assert<Equal<typeof ref extends ModelRef ? true : false, true>>
|
||||
type RefResolvesResponses = Assert<Equal<ResolveLanguageModel<typeof ref>, ReturnType<typeof openai.responses>>>
|
||||
type RefOptions = Assert<
|
||||
Equal<
|
||||
// oxlint-disable-next-line typescript-eslint/no-explicit-any
|
||||
ResolveLanguageModel<typeof ref> extends LanguageModel<infer Options, any> ? Options : never,
|
||||
OpenAIProviderOptionsInput
|
||||
>
|
||||
>
|
||||
void (true satisfies RefOptions)
|
||||
void (true satisfies RefIsModelRef)
|
||||
void (true satisfies RefResolvesResponses)
|
||||
|
||||
// LLM.request infers providerOptions from the ref's llm route.
|
||||
const llmRequest = LLM.request({ model: ref, prompt: "Hello", providerOptions: { reasoningEffort: "high" } })
|
||||
type LLMRequestIsTyped = Assert<Equal<typeof llmRequest extends LLMRequest ? true : false, true>>
|
||||
void (true satisfies LLMRequestIsTyped)
|
||||
LLM.request({
|
||||
model: ref,
|
||||
prompt: "Hello",
|
||||
// @ts-expect-error Known OpenAI options retain their value kinds through the ref.
|
||||
providerOptions: { reasoningEffort: 1 },
|
||||
})
|
||||
LLM.request({
|
||||
model: anthropic("claude-sonnet-4-5"),
|
||||
prompt: "Hello",
|
||||
providerOptions: { thinking: { type: "enabled", budgetTokens: 1024 } },
|
||||
})
|
||||
LLM.generate(LLM.request({ model: google("gemini-2.5-pro"), prompt: "Hello" }))
|
||||
|
||||
// Image.request infers providerOptions from the ref's image route.
|
||||
const imageRequest = Image.request({
|
||||
model: openai("gpt-image-2"),
|
||||
prompt: "A lighthouse",
|
||||
providerOptions: { quality: "high" },
|
||||
})
|
||||
type ImageRequestOptions = Assert<Equal<typeof imageRequest.providerOptions, OpenAIImageOptions | undefined>>
|
||||
type ImageRequestModel = Assert<Equal<typeof imageRequest.model, ImageModel<OpenAIImageOptions>>>
|
||||
void (true satisfies ImageRequestOptions)
|
||||
void (true satisfies ImageRequestModel)
|
||||
// @ts-expect-error Known OpenAI image options retain their value kinds through the ref.
|
||||
Image.request({ model: openai("gpt-image-2"), prompt: "A lighthouse", providerOptions: { outputCompression: "80" } })
|
||||
const googleImage = Image.request({
|
||||
model: google("gemini-image"),
|
||||
prompt: "A lighthouse",
|
||||
providerOptions: { imageSize: "2K" },
|
||||
})
|
||||
type GoogleImageRequestOptions = Assert<Equal<typeof googleImage.providerOptions, GoogleImageOptions | undefined>>
|
||||
void (true satisfies GoogleImageRequestOptions)
|
||||
// @ts-expect-error Known Google image options retain their value kinds through the ref.
|
||||
Image.request({ model: google("gemini-image"), prompt: "A lighthouse", providerOptions: { includeThoughts: "yes" } })
|
||||
|
||||
// Providers without an image route are rejected at compile time.
|
||||
// @ts-expect-error Anthropic exposes no image route.
|
||||
Image.request({ model: anthropic("claude-sonnet-4-5"), prompt: "A lighthouse" })
|
||||
// @ts-expect-error Anthropic exposes no image route.
|
||||
Image.generate({ model: anthropic("claude-sonnet-4-5"), prompt: "A lighthouse" })
|
||||
type AnthropicFacade = ReturnType<typeof anthropic>["facade"]
|
||||
type AnthropicHasNoImage = Assert<Equal<AnthropicFacade extends { readonly image: unknown } ? true : false, false>>
|
||||
void (true satisfies AnthropicHasNoImage)
|
||||
|
||||
// Concrete models keep working everywhere a ref is accepted.
|
||||
LLM.request({ model: openai.responses("gpt-5"), prompt: "Hello" })
|
||||
Image.request({ model: openai.image("gpt-image-2"), prompt: "A lighthouse" })
|
||||
// @ts-expect-error A language model is not an image model.
|
||||
Image.request({ model: openai.responses("gpt-5"), prompt: "A lighthouse" })
|
||||
// @ts-expect-error An image model is not a language model.
|
||||
LLM.request({ model: openai.image("gpt-image-2"), prompt: "Hello" })
|
||||
@@ -1,102 +0,0 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { AIError, LLMEvent, Media } from "../src/index.js"
|
||||
import { RequestExecutor } from "../src/route.js"
|
||||
import { AI } from "../src/promise.js"
|
||||
import { OpenAI } from "../src/providers.js"
|
||||
import { handlerLayer } from "./lib/http.js"
|
||||
import { sseEvents } from "./lib/sse.js"
|
||||
|
||||
const openai = OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" })
|
||||
|
||||
const chatBody = sseEvents(
|
||||
{ choices: [{ delta: { content: "Hello" } }] },
|
||||
{ choices: [{ delta: { content: " world" } }] },
|
||||
{ choices: [{ delta: {}, finish_reason: "stop" }] },
|
||||
)
|
||||
|
||||
/** Executor layer that answers chat completions with SSE text and image generations with one base64 PNG. */
|
||||
const executor = (seen: Array<string>) =>
|
||||
RequestExecutor.layer.pipe(
|
||||
Layer.provide(
|
||||
handlerLayer((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
seen.push(web.url)
|
||||
if (web.url.endsWith("/images/generations"))
|
||||
return input.respond(JSON.stringify({ data: [{ b64_json: "AQID" }], output_format: "png" }), {
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
if (web.url.endsWith("/chat/completions"))
|
||||
return input.respond(chatBody, { headers: { "content-type": "text/event-stream" } })
|
||||
return input.respond(JSON.stringify({ error: { message: "not found" } }), {
|
||||
status: 404,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
describe("AI promise client", () => {
|
||||
test("generates text, images, and streams over one managed runtime", async () => {
|
||||
const seen: Array<string> = []
|
||||
const ai = AI.make({ layer: executor(seen) })
|
||||
|
||||
const text = await ai.llm.generate({ model: openai.chat("gpt-4o-mini"), prompt: "Say hello." })
|
||||
expect(text.text).toBe("Hello world")
|
||||
|
||||
const image = await ai.image.generate({ model: openai("gpt-image-2"), prompt: "A lighthouse" })
|
||||
expect(image.image).toBeInstanceOf(Media.Asset)
|
||||
expect(image.image.mediaType).toBe("image/png")
|
||||
expect(await ai.run(image.image.bytes())).toEqual(Uint8Array.from([1, 2, 3]))
|
||||
|
||||
const deltas: Array<string> = []
|
||||
for await (const event of ai.llm.stream({ model: openai.chat("gpt-4o-mini"), prompt: "Say hello." })) {
|
||||
if (LLMEvent.is.textDelta(event)) deltas.push(event.text)
|
||||
}
|
||||
expect(deltas).toEqual(["Hello", " world"])
|
||||
|
||||
const imageEvents: Array<string> = []
|
||||
for await (const event of ai.image.stream({ model: openai("gpt-image-2"), prompt: "A lighthouse" })) {
|
||||
imageEvents.push(event.type)
|
||||
}
|
||||
expect(imageEvents).toEqual(["image", "finish"])
|
||||
|
||||
expect(seen).toEqual([
|
||||
"https://openai.test/v1/chat/completions",
|
||||
"https://openai.test/v1/images/generations",
|
||||
"https://openai.test/v1/chat/completions",
|
||||
"https://openai.test/v1/images/generations",
|
||||
])
|
||||
await ai.dispose()
|
||||
})
|
||||
|
||||
test("rethrows AIError unchanged and honors abort signals", async () => {
|
||||
const ai = AI.make({ layer: executor([]) })
|
||||
|
||||
const failure = await ai.llm
|
||||
.generate({ model: openai.responses("gpt-5"), prompt: "Hello" })
|
||||
.then(() => undefined)
|
||||
.catch((error: unknown) => error)
|
||||
expect(failure).toBeInstanceOf(AIError)
|
||||
expect(failure instanceof AIError && failure.reason.http?.status).toBe(404)
|
||||
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
const aborted = await ai.llm
|
||||
.generate({ model: openai.chat("gpt-4o-mini"), prompt: "Hello" }, { signal: controller.signal })
|
||||
.then(() => "completed")
|
||||
.catch(() => "aborted")
|
||||
expect(aborted).toBe("aborted")
|
||||
|
||||
await ai.dispose()
|
||||
})
|
||||
|
||||
test("the default client is created lazily and can be disposed", async () => {
|
||||
expect(typeof AI.ai.llm.generate).toBe("function")
|
||||
expect(typeof AI.ai.image.generate).toBe("function")
|
||||
await AI.ai.dispose()
|
||||
})
|
||||
})
|
||||
@@ -249,10 +249,54 @@ describe("provider error classification", () => {
|
||||
|
||||
test("classifies any remaining 4xx status as an invalid request", () => {
|
||||
expect(
|
||||
[400, 402, 404, 418, 422, 451].map(
|
||||
(status) => classifyProviderFailure({ message: `HTTP ${status}`, status })._tag,
|
||||
[400, 404, 418, 422, 451].map((status) => classifyProviderFailure({ message: `HTTP ${status}`, status })._tag),
|
||||
).toEqual(Array(5).fill("InvalidRequest"))
|
||||
})
|
||||
|
||||
test("classifies 402 as exhausted quota", () => {
|
||||
expect(classifyProviderFailure({ message: "Payment Required", status: 402 })._tag).toBe("QuotaExceeded")
|
||||
})
|
||||
|
||||
test("classifies OpenCode Zen account limits as quota rather than throttling", () => {
|
||||
const typed = (type: string, message: string) => ({ type: "error", error: { type, message } })
|
||||
const substituted = (message: string) => ({
|
||||
error: { type: "server_error", message: `Upstream request failed: ${message}` },
|
||||
})
|
||||
const cases: ReadonlyArray<[number, { error: { message: string } }]> = [
|
||||
[429, typed("GoUsageLimitError", "Go usage limit exceeded")],
|
||||
[429, typed("FreeUsageLimitError", "Rate limit exceeded. Please try again later.")],
|
||||
[402, typed("CreditLimitExceeded", "Credit limit exceeded.")],
|
||||
[402, substituted("Insufficient account funds")],
|
||||
[402, substituted("Account invoice is overdue")],
|
||||
[429, substituted("Account budget exceeded")],
|
||||
]
|
||||
expect(
|
||||
cases.map(
|
||||
([status, body]) =>
|
||||
classifyProviderFailure({ message: body.error.message, status, rawBody: JSON.stringify(body) })._tag,
|
||||
),
|
||||
).toEqual(Array(6).fill("InvalidRequest"))
|
||||
).toEqual(Array(6).fill("QuotaExceeded"))
|
||||
})
|
||||
|
||||
test("does not let substituted server codes make a 4xx retryable", () => {
|
||||
const openai = { error: { type: "server_error", message: "Upstream request failed: Model is unavailable." } }
|
||||
const anthropic = {
|
||||
type: "error",
|
||||
error: { type: "api_error", message: "Upstream request failed: Model is unavailable." },
|
||||
}
|
||||
expect(
|
||||
[openai, anthropic].map(
|
||||
(body) =>
|
||||
classifyProviderFailure({ message: body.error.message, status: 400, rawBody: JSON.stringify(body) })._tag,
|
||||
),
|
||||
).toEqual(["InvalidRequest", "InvalidRequest"])
|
||||
// Without a contradicting status the same codes still mark provider trouble.
|
||||
expect(classifyProviderFailure({ message: openai.error.message, rawBody: JSON.stringify(openai) })._tag).toBe(
|
||||
"ProviderInternal",
|
||||
)
|
||||
expect(
|
||||
classifyProviderFailure({ message: openai.error.message, status: 200, rawBody: JSON.stringify(openai) })._tag,
|
||||
).toBe("ProviderInternal")
|
||||
})
|
||||
|
||||
test("classifies nested provider codes when a top-level code is also present", () => {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { LLM, LLMEvent, LLMRequest, Message, ToolDefinition, Media } from "../../src/index.js"
|
||||
import { LLM, LLMEvent, LLMRequest, Message, ToolDefinition } from "../../src/index.js"
|
||||
import { Alibaba } from "../../src/providers.js"
|
||||
import { LLMClient } from "../../src/route.js"
|
||||
import { compileRequest } from "../../src/route/client.js"
|
||||
@@ -65,7 +65,7 @@ for (const api of ["chat", "messages", "responses"] as const) {
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "text", text: "Read the three words in this image. Reply with only the words in order." },
|
||||
{ type: "media", media: Media.bytes(bytes, "image/png") },
|
||||
{ type: "media", mediaType: "image/png", data: bytes },
|
||||
]),
|
||||
],
|
||||
generation: { maxTokens: 4096 },
|
||||
|
||||
@@ -1,17 +1,7 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import {
|
||||
CacheHint,
|
||||
LLM,
|
||||
AIError,
|
||||
LLMRequest,
|
||||
Message,
|
||||
ToolCallPart,
|
||||
ToolDefinition,
|
||||
Usage,
|
||||
Media,
|
||||
} from "../../src/index.js"
|
||||
import { CacheHint, LLM, AIError, LLMRequest, Message, ToolCallPart, ToolDefinition, Usage } from "../../src/index.js"
|
||||
import { Auth, Endpoint, LLMClient, Route } from "../../src/route.js"
|
||||
import { compileRequest } from "../../src/route/client.js"
|
||||
import * as AnthropicMessages from "../../src/protocols/anthropic-messages.js"
|
||||
@@ -294,7 +284,7 @@ describe("Anthropic Messages route", () => {
|
||||
model: opus48,
|
||||
messages: [
|
||||
Message.user("Before."),
|
||||
Message.make({ role: "system", content: { type: "media", media: Media.base64("AAECAw==", "image/png") } }),
|
||||
Message.make({ role: "system", content: { type: "media", mediaType: "image/png", data: "AAECAw==" } }),
|
||||
],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
@@ -2042,8 +2032,8 @@ describe("Anthropic Messages route", () => {
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "text", text: "What is in this image?" },
|
||||
{ type: "media", media: Media.base64("AAECAw==", "image/png") },
|
||||
{ type: "media", media: Media.base64("JVBERi0xLjQ=", "application/pdf"), filename: "report.pdf" },
|
||||
{ type: "media", mediaType: "image/png", data: "AAECAw==" },
|
||||
{ type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=", filename: "report.pdf" },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
|
||||
@@ -4,7 +4,6 @@ import { describe, expect } from "bun:test"
|
||||
import { Effect, Encoding, Ref, Schema, Stream } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import {
|
||||
Media,
|
||||
CacheHint,
|
||||
GenerationOptions,
|
||||
type LanguageModel,
|
||||
@@ -1679,10 +1678,10 @@ describe("Bedrock Converse route", () => {
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "text", text: "What is in this image?" },
|
||||
{ type: "media", media: Media.base64("AAAA", "image/png") },
|
||||
{ type: "media", media: Media.base64("BBBB", "image/jpeg") },
|
||||
{ type: "media", media: Media.base64("CCCC", "image/jpg") },
|
||||
{ type: "media", media: Media.base64("DDDD", "image/webp") },
|
||||
{ type: "media", mediaType: "image/png", data: "AAAA" },
|
||||
{ type: "media", mediaType: "image/jpeg", data: "BBBB" },
|
||||
{ type: "media", mediaType: "image/jpg", data: "CCCC" },
|
||||
{ type: "media", mediaType: "image/webp", data: "DDDD" },
|
||||
]),
|
||||
],
|
||||
cache: "none",
|
||||
@@ -1713,9 +1712,7 @@ describe("Bedrock Converse route", () => {
|
||||
LLM.request({
|
||||
id: "req_image_bytes",
|
||||
model,
|
||||
messages: [
|
||||
Message.user([{ type: "media", media: Media.bytes(new Uint8Array([1, 2, 3, 4, 5]), "image/png") }]),
|
||||
],
|
||||
messages: [Message.user([{ type: "media", mediaType: "image/png", data: new Uint8Array([1, 2, 3, 4, 5]) }])],
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -1736,7 +1733,7 @@ describe("Bedrock Converse route", () => {
|
||||
const error = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [Message.user({ type: "media", media: Media.base64("not base64!", "image/png") })],
|
||||
messages: [Message.user({ type: "media", mediaType: "image/png", data: "https://example.test/image.png" })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
@@ -1745,25 +1742,6 @@ describe("Bedrock Converse route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects remote image URLs that were not materialized", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
Message.user({
|
||||
type: "media",
|
||||
media: Media.url("https://example.test/image.png", { mediaType: "image/png" }),
|
||||
}),
|
||||
],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error).toMatchObject({ reason: { _tag: "InvalidRequest" } })
|
||||
expect(error.message).toContain("requires inline media")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers document media into Bedrock document blocks with format and name", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
@@ -1774,8 +1752,8 @@ describe("Bedrock Converse route", () => {
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "text", text: "Summarize these documents." },
|
||||
{ type: "media", media: Media.base64("UERGREFUQQ==", "application/pdf"), filename: "report.pdf" },
|
||||
{ type: "media", media: Media.base64("Q1NWREFUQQ==", "text/csv"), filename: "data.csv" },
|
||||
{ type: "media", mediaType: "application/pdf", data: "UERGREFUQQ==", filename: "report.pdf" },
|
||||
{ type: "media", mediaType: "text/csv", data: "Q1NWREFUQQ==", filename: "data.csv" },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
@@ -1850,7 +1828,7 @@ describe("Bedrock Converse route", () => {
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "text", text: "Read this document" },
|
||||
{ type: "media", media: Media.base64("UERGREFUQQ==", "application/pdf"), filename: item.filename },
|
||||
{ type: "media", mediaType: "application/pdf", data: "UERGREFUQQ==", filename: item.filename },
|
||||
]),
|
||||
Message.assistant([ToolCallPart.make({ id: "call_read", name: "read", input: {} })]),
|
||||
Message.tool({
|
||||
@@ -1900,7 +1878,8 @@ describe("Bedrock Converse route", () => {
|
||||
{ type: "text", text: "Read these documents" },
|
||||
...["report_v1.txt", "report#v1.txt", "report v1 2.txt", "report v1.txt"].map((filename) => ({
|
||||
type: "media" as const,
|
||||
media: Media.base64("SGVsbG8=", "text/plain"),
|
||||
mediaType: "text/plain",
|
||||
data: "SGVsbG8=",
|
||||
filename,
|
||||
})),
|
||||
]),
|
||||
@@ -1929,7 +1908,8 @@ describe("Bedrock Converse route", () => {
|
||||
messages: [
|
||||
Message.user({
|
||||
type: "media",
|
||||
media: Media.base64("UERGREFUQQ==", "application/pdf"),
|
||||
mediaType: "application/pdf",
|
||||
data: "UERGREFUQQ==",
|
||||
filename: "report.pdf",
|
||||
}),
|
||||
],
|
||||
@@ -1959,7 +1939,8 @@ describe("Bedrock Converse route", () => {
|
||||
{ type: "text", text: "Read these documents" },
|
||||
...["report", undefined, 'report "final"\n.pdf'].map((filename) => ({
|
||||
type: "media" as const,
|
||||
media: Media.base64("SGVsbG8=", "text/plain"),
|
||||
mediaType: "text/plain",
|
||||
data: "SGVsbG8=",
|
||||
filename,
|
||||
})),
|
||||
]),
|
||||
@@ -2056,7 +2037,7 @@ describe("Bedrock Converse route", () => {
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error).toMatchObject({ reason: { _tag: "InvalidRequest" } })
|
||||
expect(error.message).toContain("requires inline media")
|
||||
expect(error.message).toContain("Bedrock Converse media data must be valid base64")
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -2066,7 +2047,7 @@ describe("Bedrock Converse route", () => {
|
||||
LLM.request({
|
||||
id: "req_bad_image",
|
||||
model,
|
||||
messages: [Message.user([{ type: "media", media: Media.base64("x", "image/svg+xml") }])],
|
||||
messages: [Message.user([{ type: "media", mediaType: "image/svg+xml", data: "x" }])],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
@@ -2080,9 +2061,7 @@ describe("Bedrock Converse route", () => {
|
||||
LLM.request({
|
||||
id: "req_bad_doc",
|
||||
model,
|
||||
messages: [
|
||||
Message.user([{ type: "media", media: Media.base64("x", "application/x-tar"), filename: "a.tar" }]),
|
||||
],
|
||||
messages: [Message.user([{ type: "media", mediaType: "application/x-tar", data: "x", filename: "a.tar" }])],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { LLM, AIError, LLMRequest, Message, ToolCallPart, ToolDefinition, Usage, Media } from "../../src/index.js"
|
||||
import { LLM, AIError, LLMRequest, Message, ToolCallPart, ToolDefinition, Usage } from "../../src/index.js"
|
||||
import { Auth, LLMClient } from "../../src/route.js"
|
||||
import { compileRequest } from "../../src/route/client.js"
|
||||
import * as Gemini from "../../src/protocols/gemini.js"
|
||||
@@ -340,8 +340,8 @@ describe("Gemini route", () => {
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "text", text: "What is in this image?" },
|
||||
{ type: "media", media: Media.base64("AAECAw==", "image/png") },
|
||||
{ type: "media", media: Media.base64("JVBERi0xLjQ=", "application/pdf") },
|
||||
{ type: "media", mediaType: "image/png", data: "AAECAw==" },
|
||||
{ type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=" },
|
||||
]),
|
||||
Message.assistant([ToolCallPart.make({ id: "call_1", name: "lookup", input: { query: "weather" } })]),
|
||||
Message.tool({ id: "call_1", name: "lookup", result: { forecast: "sunny" } }),
|
||||
@@ -446,7 +446,7 @@ describe("Gemini route", () => {
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
Message.user({ type: "media", media: Media.fromDataUrl("data:image/png;base64,AAEC") }),
|
||||
Message.user({ type: "media", mediaType: "image/png", data: "data:image/png;base64,AAEC" }),
|
||||
Message.tool({
|
||||
id: "call_image",
|
||||
name: "read",
|
||||
@@ -636,9 +636,9 @@ describe("Gemini route", () => {
|
||||
model,
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "media", media: Media.base64("%%%=", "image/png") },
|
||||
{ type: "media", media: Media.fromDataUrl("data:image/jpeg;base64,/9j/") },
|
||||
{ type: "media", media: Media.base64("PHN2Zz4=", "image/svg+xml") },
|
||||
{ type: "media", mediaType: "image/png", data: "%%%=" },
|
||||
{ type: "media", mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" },
|
||||
{ type: "media", mediaType: "image/svg+xml", data: "PHN2Zz4=" },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
@@ -648,7 +648,7 @@ describe("Gemini route", () => {
|
||||
role: "user",
|
||||
parts: [
|
||||
{ inlineData: { mimeType: "image/png", data: "%%%=" } },
|
||||
{ inlineData: { mimeType: "image/jpeg", data: "/9j/" } },
|
||||
{ inlineData: { mimeType: "image/png", data: "/9j/" } },
|
||||
{ inlineData: { mimeType: "image/svg+xml", data: "PHN2Zz4=" } },
|
||||
],
|
||||
},
|
||||
@@ -1773,34 +1773,19 @@ describe("Gemini route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("replays generated assistant media as model inline data", () =>
|
||||
it.effect("rejects unsupported assistant media content", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
const error = yield* compileRequest(
|
||||
LLM.request({
|
||||
id: "req_media",
|
||||
model,
|
||||
messages: [
|
||||
Message.user("Draw a cat"),
|
||||
Message.assistant([
|
||||
{ type: "text", text: "Here you go." },
|
||||
{
|
||||
type: "media",
|
||||
media: Media.base64("AAECAw==", "image/png"),
|
||||
providerMetadata: { google: { thoughtSignature: "sig_1" } },
|
||||
},
|
||||
]),
|
||||
Message.user("Now make it orange"),
|
||||
],
|
||||
messages: [Message.assistant({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
|
||||
}),
|
||||
)
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(prepared.body.contents[1]).toEqual({
|
||||
role: "model",
|
||||
parts: [
|
||||
{ text: "Here you go." },
|
||||
{ inlineData: { mimeType: "image/png", data: "AAECAw==" }, thoughtSignature: "sig_1" },
|
||||
],
|
||||
})
|
||||
expect(error.message).toContain(
|
||||
"Gemini assistant messages only support text, reasoning, and tool-call content for now",
|
||||
)
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { Image, Media } from "../../src/index.js"
|
||||
import { Image, ImageInput } from "../../src/index.js"
|
||||
import { Google } from "../../src/providers.js"
|
||||
import { dimensions } from "../lib/image.js"
|
||||
import { recordedTests } from "../recorded-test.js"
|
||||
@@ -22,12 +22,13 @@ describe("Google Images recorded", () => {
|
||||
const response = yield* Image.generate({
|
||||
model,
|
||||
prompt: "A simple flat blue circle centered on a plain white background.",
|
||||
aspectRatio: "1:1",
|
||||
options: { aspectRatio: "1:1" },
|
||||
})
|
||||
|
||||
expect(response.images).toHaveLength(1)
|
||||
expect(response.image.mediaType).toMatch(/^image\//)
|
||||
expect((yield* response.image.bytes()).length).toBeGreaterThan(0)
|
||||
expect(response.image?.mediaType).toMatch(/^image\//)
|
||||
expect(response.image?.data).toBeInstanceOf(Uint8Array)
|
||||
expect(response.image?.data.length).toBeGreaterThan(0)
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -38,16 +39,18 @@ describe("Google Images recorded", () => {
|
||||
prompt:
|
||||
"Transform this minimal source into a bright orange sun icon with eight rounded rays on a pale blue background.",
|
||||
images: [
|
||||
Media.bytes(
|
||||
ImageInput.bytes(
|
||||
yield* Effect.promise(() => Bun.file("test/fixtures/images/edit-source.jpg").bytes()),
|
||||
"image/jpeg",
|
||||
),
|
||||
],
|
||||
aspectRatio: "1:1",
|
||||
options: { aspectRatio: "1:1" },
|
||||
})
|
||||
|
||||
expect(response.image.mediaType).toBe("image/jpeg")
|
||||
expect(dimensions(yield* response.image.bytes())).toEqual({ width: 1024, height: 1024 })
|
||||
expect(response.image?.mediaType).toBe("image/jpeg")
|
||||
expect(response.image?.data).toBeInstanceOf(Uint8Array)
|
||||
if (!(response.image?.data instanceof Uint8Array)) throw new Error("Expected owned Google image bytes")
|
||||
expect(dimensions(response.image.data)).toEqual({ width: 1024, height: 1024 })
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { expect } from "bun:test"
|
||||
import { Effect, Schema } from "effect"
|
||||
import { LLM, Media, Message } from "../../src/index.js"
|
||||
import { LLM, Message } from "../../src/index.js"
|
||||
import { OpenAI, Azure, XAI } from "../../src/providers.js"
|
||||
import { compileRequest } from "../../src/route/client.js"
|
||||
import { it } from "../lib/effect.js"
|
||||
@@ -16,7 +16,8 @@ for (const model of [
|
||||
const message = Message.user(
|
||||
details.map((detail) => ({
|
||||
type: "media",
|
||||
media: Media.url("https://example.com/image.png", { mediaType: "image/png" }),
|
||||
mediaType: "image/png",
|
||||
data: "https://example.com/image.png",
|
||||
providerMetadata:
|
||||
detail === undefined ? undefined : { [model.route.providerMetadataKey ?? model.provider]: { detail } },
|
||||
})),
|
||||
@@ -47,7 +48,8 @@ it.effect("rejects malformed image detail instead of silently discarding it", ()
|
||||
messages: [
|
||||
Message.user({
|
||||
type: "media",
|
||||
media: Media.url("https://example.com/image.png", { mediaType: "image/png" }),
|
||||
mediaType: "image/png",
|
||||
data: "https://example.com/image.png",
|
||||
providerMetadata: { openai: { detail: 42 } },
|
||||
}),
|
||||
],
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { expect } from "bun:test"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { Image, ImageClient, LLM, LLMEvent, LLMRequest, Media, Message, ToolDefinition } from "../../src/index.js"
|
||||
import { Image, ImageClient, ImageInput, LLM, LLMEvent, LLMRequest, Message, ToolDefinition } from "../../src/index.js"
|
||||
import { Meta } from "../../src/providers/meta.js"
|
||||
import { MetaMessages } from "../../src/protocols/meta-messages.js"
|
||||
import { AnthropicMessages } from "../../src/protocols/anthropic-messages.js"
|
||||
@@ -73,9 +73,9 @@ it.effect("Meta Images preserves request overlays, bearer auth, JSON edit inputs
|
||||
headers: { "x-client": "test" },
|
||||
}).image("muse-image-1.0"),
|
||||
prompt: "Edit",
|
||||
images: [Media.bytes(Uint8Array.from([1, 2, 3]), "image/png")],
|
||||
format: "webp",
|
||||
providerOptions: {
|
||||
images: [ImageInput.bytes(Uint8Array.from([1, 2, 3]), "image/png")],
|
||||
options: {
|
||||
outputFormat: "webp",
|
||||
responseFormat: "url",
|
||||
reasoningStrength: "low",
|
||||
toolEnablement: { enable_web_search: false },
|
||||
@@ -83,12 +83,8 @@ it.effect("Meta Images preserves request overlays, bearer auth, JSON edit inputs
|
||||
},
|
||||
http: { body: { output_format: "jpeg", future_option: true }, query: { trace: "1" } },
|
||||
})
|
||||
expect(response.image.mediaType).toBe("image/jpeg")
|
||||
expect(response.image.source).toEqual({
|
||||
type: "url",
|
||||
url: "https://images.example/result.jpg",
|
||||
mediaType: "image/jpeg",
|
||||
})
|
||||
expect(response.image?.mediaType).toBe("image/jpeg")
|
||||
expect(response.image?.data).toBe("https://images.example/result.jpg")
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
@@ -125,7 +121,7 @@ it.effect("Meta Images validates the final output format before sending the requ
|
||||
const error = yield* Image.generate({
|
||||
model: Meta.configure({ apiKey: "fixture" }).image("muse-image-1.0"),
|
||||
prompt: "Draw",
|
||||
format: "png",
|
||||
options: { outputFormat: "png" },
|
||||
http: { body: { output_format: 42 } },
|
||||
}).pipe(Effect.flip)
|
||||
expect(error.reason._tag).toBe("InvalidRequest")
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { Image, LLM, LLMEvent, LLMRequest, Media, Message } from "../../src/index.js"
|
||||
import { Image, ImageInput, LLM, LLMEvent, LLMRequest, Message } from "../../src/index.js"
|
||||
import { Meta } from "../../src/providers/meta.js"
|
||||
import { LLMClient } from "../../src/route.js"
|
||||
import { compileRequest } from "../../src/route/client.js"
|
||||
@@ -28,16 +28,15 @@ recorded.effect.with(
|
||||
const response = yield* Image.generate({
|
||||
model: meta.image(modelID),
|
||||
prompt: "A flat black square centered on a plain white background. No text.",
|
||||
n: 1,
|
||||
size: "256x256",
|
||||
providerOptions: controls,
|
||||
options: { ...controls, n: 1, size: "256x256" },
|
||||
})
|
||||
expect(response.images).toHaveLength(1)
|
||||
expect(response.image.mediaType).toBe("image/webp")
|
||||
const bytes = yield* response.image.bytes()
|
||||
expect(new TextDecoder().decode(bytes.slice(0, 4))).toBe("RIFF")
|
||||
expect(new TextDecoder().decode(bytes.slice(8, 12))).toBe("WEBP")
|
||||
expect(response.usage?.type === "tokens" ? response.usage.output : undefined).toBeGreaterThan(0)
|
||||
expect(response.image?.mediaType).toBe("image/webp")
|
||||
expect(response.image?.data).toBeInstanceOf(Uint8Array)
|
||||
if (!(response.image?.data instanceof Uint8Array)) throw new Error("Expected image bytes")
|
||||
expect(new TextDecoder().decode(response.image.data.slice(0, 4))).toBe("RIFF")
|
||||
expect(new TextDecoder().decode(response.image.data.slice(8, 12))).toBe("WEBP")
|
||||
expect(response.usage?.outputTokens).toBeGreaterThan(0)
|
||||
}),
|
||||
180_000,
|
||||
)
|
||||
@@ -51,18 +50,16 @@ recorded.effect.with(
|
||||
model: meta.image(modelID),
|
||||
prompt: "Change the shape to bright purple. Keep the plain white background.",
|
||||
images: [
|
||||
Media.bytes(
|
||||
ImageInput.bytes(
|
||||
yield* Effect.promise(() => Bun.file("test/fixtures/images/edit-source.jpg").bytes()),
|
||||
"image/jpeg",
|
||||
),
|
||||
],
|
||||
n: 1,
|
||||
size: "256x256",
|
||||
format: "png",
|
||||
providerOptions: controls,
|
||||
options: { ...controls, n: 1, outputFormat: "png", size: "256x256" },
|
||||
})
|
||||
expect(response.image.mediaType).toBe("image/png")
|
||||
expect(Array.from((yield* response.image.bytes()).slice(0, 8))).toEqual([137, 80, 78, 71, 13, 10, 26, 10])
|
||||
expect(response.image?.mediaType).toBe("image/png")
|
||||
if (!(response.image?.data instanceof Uint8Array)) throw new Error("Expected image bytes")
|
||||
expect(Array.from(response.image.data.slice(0, 8))).toEqual([137, 80, 78, 71, 13, 10, 26, 10])
|
||||
}),
|
||||
180_000,
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { ConfigProvider, Effect } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { LLM, LLMEvent, Message, ToolDefinition, Media } from "../../src/index.js"
|
||||
import { LLM, LLMEvent, Message, ToolDefinition } from "../../src/index.js"
|
||||
import { Mistral } from "../../src/providers/index.js"
|
||||
import { MistralChat } from "../../src/protocols/index.js"
|
||||
import { LLMClient } from "../../src/route.js"
|
||||
@@ -46,8 +46,8 @@ describe("Mistral Chat", () => {
|
||||
Message.system("Updated"),
|
||||
Message.user([
|
||||
{ type: "text", text: "Inspect" },
|
||||
{ type: "media", media: Media.base64("aW1hZ2U=", "image/png") },
|
||||
{ type: "media", media: Media.base64("cGRm", "application/pdf") },
|
||||
{ type: "media", mediaType: "image/png", data: "aW1hZ2U=" },
|
||||
{ type: "media", mediaType: "application/pdf", data: "cGRm" },
|
||||
]),
|
||||
Message.assistant([
|
||||
{ type: "reasoning", text: "Think" },
|
||||
@@ -230,7 +230,8 @@ describe("Mistral Chat", () => {
|
||||
messages: [
|
||||
Message.user({
|
||||
type: "media",
|
||||
media: Media.url("https://assets.example.test/input.png", { mediaType: "image/png" }),
|
||||
mediaType: "image/png",
|
||||
data: "https://assets.example.test/input.png",
|
||||
}),
|
||||
Message.tool({
|
||||
id: "Ab12Cd34E",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { LLM, LLMEvent, Message, ToolDefinition, Media } from "../../src/index.js"
|
||||
import { LLM, LLMEvent, Message, ToolDefinition } from "../../src/index.js"
|
||||
import { Moonshot } from "../../src/providers.js"
|
||||
import { LLMClient } from "../../src/route.js"
|
||||
import { recordedTests } from "../recorded-test.js"
|
||||
@@ -85,7 +85,7 @@ for (const api of ["chat", "messages", "responses"] as const) {
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "text", text: "Read the three words in this image. Reply only with those words in order." },
|
||||
{ type: "media", media: Media.bytes(image, "image/png") },
|
||||
{ type: "media", mediaType: "image/png", data: image },
|
||||
]),
|
||||
],
|
||||
generation: { maxTokens: 4096 },
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user