Compare commits

...
Author SHA1 Message Date
Aiden Cline f6cb97672e feat(core): fit output limits to the context window 2026-09-26 00:06:15 -05:00
opencode-agent[bot]andrekram1-node 37049a5a13 fix(tui): skip model selection after MCP connection (#51448)
Co-authored-by: rekram1-node <rekram1-node@users.noreply.github.com>
2026-09-26 00:00:25 -05:00
Aiden Cline a64bc3616e refactor(core): cleanup session compaction (#51447) 2026-09-26 00:00:11 -05:00
opencode 39021dfd67 sync release versions for v2.0.18 2026-09-25 23:57:39 +00:00
opencode-agent[bot] 709ddc0d79 chore: update nix node_modules hashes 2026-09-25 23:23:45 +00:00
Kit LangtonandAiden Cline 041885d838 fix(core): decode legacy media in compaction checkpoints (#51409)
Co-authored-by: Aiden Cline <aidenpcline@gmail.com>
2026-09-25 18:08:16 -05:00
Aiden Cline 29ce49db0f refactor(util): share a browser opener across cli, tui, and core (#51412) 2026-09-25 18:00:32 -05:00
opencode 00738c5b2d sync release versions for v2.0.17 2026-09-25 21:09:12 +00:00
Aiden Cline 6ec8ca920f feat(core): name Copilot sessions with the free utility model (#51237) 2026-09-25 14:53:16 -05:00
Shoubhit Dash 6585bb7105 fix(ai): keep OpenAI image output settings and Z.ai URL expiry (#51380) 2026-09-25 22:51:46 +05:30
Shoubhit Dash f954688fbb fix(ai): harden OpenAI transcription stream parsing (#51378) 2026-09-25 22:51:22 +05:30
Shoubhit Dash 1463dabde9 fix(ai): tighten media error consistency (#51374) 2026-09-25 22:46:04 +05:30
Shoubhit Dash 29ea6ee05b test(ai): cover media facade selectors and url asset edges (#51376) 2026-09-25 22:37:20 +05:30
Shoubhit Dash b170904731 docs(ai): describe transcription speakers as a constraint (#51382) 2026-09-25 22:36:02 +05:30
Shoubhit Dash 88e1fa9304 fix(ai): accept timestamps: false on speech routes without timestamps (#51375) 2026-09-25 22:33:59 +05:30
opencode-agent[bot]andvimtor ff1bf315ed docs(www): hide Console Usage API documentation (#51358)
Co-authored-by: vimtor <36263538+vimtor@users.noreply.github.com>
2026-09-25 19:01:48 +02:00
Shoubhit Dash 144ce00e00 fix(ai): bound generation event poll sleep by the deadline (#51372) 2026-09-25 22:31:35 +05:30
Shoubhit Dash ad504094f0 fix(ai): never delete a finished Runway task on cancel (#51373) 2026-09-25 22:24:16 +05:30
Shoubhit Dash bad6834a3e fix(ai): size fal Kontext by aspect ratio and decode sync_mode data URIs (#51371) 2026-09-25 22:23:14 +05:30
Shoubhit Dash 0c4bbc3cd1 feat(ai): keep prompt cache across effort switches on GPT-6 Sol and Luna (#51339) 2026-09-25 22:15:23 +05:30
Shoubhit Dash ae7dd82126 fix(ai): report Black Forest Labs submit cost as image usage (#51370) 2026-09-25 22:13:55 +05:30
Shoubhit Dash 65d5123ead fix(ai): enable AssemblyAI speaker labels when speakers is set (#51369) 2026-09-25 22:11:44 +05:30
Shoubhit Dash 4eb46a8885 fix(core): revert always-thinking variants for Claude Opus 5.5 (#51359) 2026-09-25 22:03:12 +05:30
Jack 1986e92842 docs(go): show permanent DeepSeek $60 allowance (#51363) 2026-09-26 00:17:08 +08:00
Shoubhit Dash 14fc63ba9e fix(core): keep thinking on for Claude Opus 5.5 variants (#51338) 2026-09-25 18:32:41 +05:30
opencode-agent[bot]andnexxeln c34ffa117e fix(ai): preserve Gemini 3.8 TTS WAV output (#51300)
Co-authored-by: nexxeln <95541290+nexxeln@users.noreply.github.com>
2026-09-25 18:11:36 +05:30
beeb14e910 feat(prompt): undo queued prompts back into the input (#51124)
Co-authored-by: vimtor <36263538+vimtor@users.noreply.github.com>
Co-authored-by: vimtor <vn4varro@gmail.com>
2026-09-25 14:36:29 +02:00
opencode-agent[bot] aae42e2e75 chore(core): refresh bundled models.dev snapshot 2026-09-25 12:21:04 +00:00
cc9011c1ae fix(tui): virtualize large added-file diffs (#51122)
Co-authored-by: vimtor <36263538+vimtor@users.noreply.github.com>
Co-authored-by: vimtor <vn4varro@gmail.com>
2026-09-25 13:51:00 +02:00
Victor Navarro 6cd938e1e9 feat(core): register Console-hosted MCP servers (#51325) 2026-09-25 13:16:04 +02:00
Jack 7de6b3fc15 docs(console): document Qwen3.8 Max (#51320) 2026-09-25 19:13:45 +08:00
opencode-agent[bot] c1c9a13993 chore: update nix node_modules hashes 2026-09-25 08:37:39 +00:00
Simon Klee 917d904f18 tui: update OpenTUI v0.5.12 (#50567) 2026-09-25 08:17:14 +00:00
opencode-agent[bot]andBrendonovich ee5b67eb84 fix(app): unify session project icon resolution (#51288)
Co-authored-by: Brendonovich <14191578+Brendonovich@users.noreply.github.com>
2026-09-25 07:47:12 +00:00
Aiden Cline 048a47e89e docs: focus compaction page on user-facing behavior (#51270) 2026-09-25 00:21:17 -05:00
Aiden Cline 5335347e80 feat(codemode): add WeakMap and WeakSet (#51257) 2026-09-25 00:15:19 -05:00
Aiden Cline 16b18dff13 Revert "fix(core): fit model limits and recover compaction overflow" (#51273) 2026-09-25 00:12:14 -05:00
Aiden Cline 61c2349cef fix(core): fit model limits and recover compaction overflow (#51238) 2026-09-25 00:11:40 -05:00
opencode-agent[bot]andBrendonovich 684721efb8 feat(app): add provider account switching (#51266)
Co-authored-by: Brendonovich <Brendonovich@users.noreply.github.com>
2026-09-25 05:03:33 +00:00
Aiden Cline 962c14a49c fix(codemode): honor thisArg, program toString in computed keys, and ToPrimitive in String and Number arguments (#51264) 2026-09-24 23:46:15 -05:00
opencode-agent[bot]andrekram1-node 85b98e7da4 fix(tui): handle storage watcher errors after startup (#51243)
Co-authored-by: rekram1-node <rekram1-node@users.noreply.github.com>
2026-09-24 20:25:06 -05:00
Aiden Cline 8061220b08 test(codemode): vendor every eligible test262 directory and bound unsupported globals (#51242) 2026-09-24 20:19:24 -05:00
Luke Parker b02cc35f13 fix(desktop): keep browser page visible under floating content (#51240) 2026-09-25 10:35:17 +10:00
Aiden Cline e23d89c9a9 fix(codemode): destructure object patterns from primitives and convert Date components through ToPrimitive (#51239) 2026-09-24 19:32:19 -05:00
e8b3e19e85 fix(tui): don't crash when fs.watch throws (e.g. ENOSPC) (#51210)
Co-authored-by: Alireza Haghdoost <haghdoost@uber.com>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-24 19:16:43 -05:00
Aiden Cline 5256f30957 feat(codemode): honor program valueOf and toString in operators and conversions (#50837) 2026-09-24 18:45:48 -05:00
Aiden Cline 61ecf404b9 fix(core): apply GPT verbosity defaults at request time (#51166) 2026-09-24 17:13:39 -05:00
Shoubhit Dash 92d2b1700f refactor(ai): one media client shape and route types erased over options (#51226) 2026-09-25 03:15:45 +05:30
Aiden Cline 56262121ee feat(codemode): bind this and arguments in functions, add Function.prototype.call, apply, and bind (#50831) 2026-09-24 16:34:46 -05:00
Aiden Cline e3b588e7d2 refactor(ai): apply tool schema rules once per request (#51162) 2026-09-24 16:27:30 -05:00
opencode-agent[bot]andnexxeln 1de648cb13 feat(ai): restore direct LLM input overloads (#51211)
Co-authored-by: nexxeln <95541290+nexxeln@users.noreply.github.com>
2026-09-25 02:49:07 +05:30
Aiden Cline 03be7f385b fix(core): accept 128-character tool names (#51207) 2026-09-24 14:59:57 -05:00
Aiden Cline 8118690839 fix(core): show line counts when tool output is truncated (#51200) 2026-09-24 14:50:55 -05:00
Aiden Cline a16eedfed7 fix(codemode): make derived error constructors inherit from Error (#51045) 2026-09-24 14:03:10 -05:00
Aiden Cline e796f2f9a5 fix(core): read provider errors from plain AI SDK stream errors (#51194) 2026-09-24 13:52:32 -05:00
Aiden Cline 20610e6645 fix(ai): fit Claude thinking budgets on Bedrock Converse (#51190) 2026-09-24 13:48:39 -05:00
Aiden Cline 7f245b0968 fix(ai): fit OpenRouter reasoning budgets to the output limit (#51189) 2026-09-24 13:42:10 -05:00
Shoubhit Dash 7013e925f5 refactor(ai): keep LLM calls request-only in the promise client (#51180) 2026-09-24 23:47:42 +05:30
Aiden Cline 499c2feaa3 fix(core): cap Alibaba thinking budget variants at 64k (#51154) 2026-09-24 13:14:49 -05:00
Aiden Cline 03af821aa5 fix(core): restore the shell no-output placeholder (#51187) 2026-09-24 13:12:42 -05:00
Aiden Cline c903774556 fix(ai): fit thinking budgets to the output limit (#51157) 2026-09-24 12:58:54 -05:00
James Long 14aaf91e65 fix(tui): mark failed groups with a plain ✗ (#51175) 2026-09-24 13:52:14 -04:00
James Long c832432d89 refactor(tui): drop unused yellow alias from opencode theme (#51177) 2026-09-24 13:17:21 -04:00
opencode-agent[bot]andjlongster 1d431a80df fix(cli): reuse core declarations during typecheck (#51165)
Co-authored-by: jlongster <jlongster@users.noreply.github.com>
2026-09-24 13:16:46 -04:00
Shoubhit Dash 19a9e41c28 refactor(ai): lift shared media protocol plumbing and drop dead generation surface (#51138) 2026-09-24 21:53:43 +05:30
Aiden Cline d932cad09c fix(ai): declare tool schema roots as objects (#51153) 2026-09-24 11:22:38 -05:00
Aiden Cline e22c1622e0 fix(core): report shell commands killed by a signal (#51145) 2026-09-24 11:21:31 -05:00
Aiden Cline 0e4ad0ce76 fix(core): send Grok reasoning effort on Bedrock Converse (#51151) 2026-09-24 11:06:05 -05:00
Aiden Cline 6afcc887c8 fix(ai): omit maxTokens for Nova 2 at high reasoning effort (#51150) 2026-09-24 10:45:43 -05:00
James Long c1f50659a7 feat(tui): add transcript verbosity levels (#51131) 2026-09-24 11:21:15 -04:00
James Long 32d3535f66 fix(core): publish batched deltas before the next block starts (#51105) 2026-09-24 09:55:27 -04:00
Shoubhit Dash 9810d98bc2 feat(ai): polish media DX with promise file helpers and one client layer (#50978) 2026-09-24 18:34:14 +05:30
Shoubhit Dash 2caba90a63 feat(app): replace password pairing with one-time links (#50972) 2026-09-24 18:28:48 +05:30
Shoubhit Dash eccf0b3b7b feat(server): pair with one-time connect links (#50970) 2026-09-24 18:28:48 +05:30
opencode-agent[bot] 808588e9b9 chore(core): refresh bundled models.dev snapshot 2026-09-24 12:20:56 +00:00
Victor Navarro c35c211460 feat(tui): expose model variant selection to plugins (#51101) 2026-09-24 14:19:27 +02:00
opencode-agent[bot]andvimtor efb8e23dcc fix(tui): move session when recovering missing directory (#51096)
Co-authored-by: vimtor <36263538+vimtor@users.noreply.github.com>
2026-09-24 13:47:23 +02:00
opencode-agent[bot]andBrendonovich ad53e39d2e fix(app): group nested tabs and show full session path (#51064)
Co-authored-by: Brendonovich <14191578+Brendonovich@users.noreply.github.com>
2026-09-24 08:10:15 +00:00
opencode-agent[bot] 3ddfc15df5 feat(app): show bottom navigation setting on mobile production (#51067) 2026-09-24 16:09:44 +08:00
opencode ac2426e103 sync release versions for v2.0.16 2026-09-24 06:34:17 +00:00
opencode-agent[bot] 0bc8b8dbeb feat(session-ui): show GitHub mark on Markdown links (#51000) 2026-09-24 14:08:32 +08:00
Luke Parker dca73ba9e3 fix(app): show composer attachment remove buttons on touch devices (#51047) 2026-09-24 15:58:30 +10:00
Aiden Cline 318a8c1aba feat(codemode): add Object.freeze/seal/create/getPrototypeOf and structuredClone (#50834) 2026-09-24 00:50:54 -05:00
opencode-agent[bot]andiamdavidhill bee5014f89 feat(app): simplify experimental settings (#49373)
Co-authored-by: iamdavidhill <1879069+iamdavidhill@users.noreply.github.com>
2026-09-24 15:48:41 +10:00
opencode-agent[bot]andDax 8652729fdb fix(desktop): identify updater requests (#50445)
Co-authored-by: Dax <826656+thdxr@users.noreply.github.com>
2026-09-24 15:44:20 +10:00
usrnk1 bfcaf1b0cd feat(desktop): reveal session workspace metadata (#49793) 2026-09-24 15:42:21 +10:00
opencode-agent[bot]andjayair f24fdeca70 fix(app): keep session summary as an overlay (#50386)
Co-authored-by: jayair <53023+jayair@users.noreply.github.com>
2026-09-24 15:38:06 +10:00
Aiden Cline dc48655743 fix(codemode): compare with == by IsLooselyEqual and stop deep-walking operands for opaque references (#50830) 2026-09-24 00:36:41 -05:00
Luke Parker ab52135b4e test(app): match settings server param to the e2e server (#51039) 2026-09-24 15:35:10 +10:00
usrnk1andLukeParkerDev e31f78af78 feat(desktop): refine tool count spacing (#50356)
Co-authored-by: LukeParkerDev <10430890+Hona@users.noreply.github.com>
2026-09-24 15:29:27 +10:00
usrnk1andLukeParkerDev bc3669b97f feat(desktop): adapt composer command menu height (#49343)
Co-authored-by: LukeParkerDev <10430890+Hona@users.noreply.github.com>
2026-09-24 15:28:07 +10:00
Luke Parker 7460d855ac fix(app): reread files when opening a preview (#51036) 2026-09-24 15:23:46 +10:00
Aiden Cline 683d470fe4 fix(core): report read length and cap limits and long lines (#51011) 2026-09-24 00:20:52 -05:00
Aiden Cline ef4938d344 test(ai): remove orphaned Gemini cassettes (#51034) 2026-09-24 00:17:10 -05:00
Aiden Cline b098b0e06f fix(ai): apply Gemini and Kimi schema handling by model name (#51017) 2026-09-24 00:16:48 -05:00
usrnk1andLukeParkerDev 27dfac3d36 fix(app): preserve settings route on refresh (#50364)
Co-authored-by: LukeParkerDev <10430890+Hona@users.noreply.github.com>
2026-09-24 14:52:21 +10:00
opencode-agent[bot]andHona 5253661d36 feat(i18n): complete current V2 translation coverage (#50204)
Co-authored-by: Hona <10430890+Hona@users.noreply.github.com>
2026-09-24 14:44:59 +10:00
Aiden Cline ff6b8c21e7 feat(core): MCP Code Mode defaults, turn off PostHog's server-side Code Mode (#51029) 2026-09-23 23:30:12 -05:00
opencode-agent[bot] 8e39040eed chore: update nix node_modules hashes 2026-09-24 04:24:26 +00:00
95cfca73b0 feat(desktop): improve settings project management (#49172)
Co-authored-by: David Hill <iamdavidhill@gmail.com>
Co-authored-by: LukeParkerDev <10430890+Hona@users.noreply.github.com>
2026-09-24 14:23:15 +10:00
Vladimir GlafirovandAiden Cline f0aff6cfe8 feat(core): restore GitLab workflow discovery and add OAuth login (#50422)
Co-authored-by: Aiden Cline <aidenpcline@gmail.com>
2026-09-23 23:00:52 -05:00
usrnk1andLukeParkerDev b7d707cd64 feat(app): sign in to OpenCode Console and Go through the device flow (#48501)
Co-authored-by: LukeParkerDev <10430890+Hona@users.noreply.github.com>
2026-09-24 03:51:35 +00:00
Omkar Chandorkar b486652067 fix(ui): fix desktop theme background seeds, light contrast, and token typos (#50517)
Signed-off-by: Omkar Chandorkar <gotenksIN@aospa.co>
2026-09-24 13:51:18 +10:00
Aiden Cline 7d1e212fb5 fix(core): route responses-only Copilot models to the Responses API (#51024) 2026-09-23 22:30:57 -05:00
Aiden Clineandrekram1-node 2c369a21c9 feat(core): note renamed legacy tools when migrating V1 sessions (#50188)
Co-authored-by: rekram1-node <rekram1-node@users.noreply.github.com>
2026-09-23 21:29:20 -05:00
opencode-agent[bot]andrekram1-node f2e73dd03f fix(tui): open execute details after tool failure (#50921)
Co-authored-by: rekram1-node <rekram1-node@users.noreply.github.com>
2026-09-23 21:10:18 -05:00
Aiden Cline 88a9688c9c fix(ai): send Gemini tool schemas as parametersJsonSchema (#51009) 2026-09-23 20:28:33 -05:00
Luke Parker 8d5eb3bf9c fix(app): start MCP sign-in from row click (#51001) 2026-09-24 10:29:32 +10:00
Aiden Clineandrekram1-node 6e0857cc0e fix: Update condition for OpenAIPlugin to check 'astra' (#50989)
Co-authored-by: rekram1-node <rekram1-node@users.noreply.github.com>
2026-09-23 17:43:12 -05:00
opencode-agent[bot]andrekram1-node 757e565c23 fix(core): ignore blank subagent options (#50977)
Co-authored-by: rekram1-node <rekram1-node@users.noreply.github.com>
2026-09-23 16:29:08 -05:00
James Long f53cc41657 fix(tui): revert transcript verbosity levels (#50983) 2026-09-23 17:20:21 -04:00
James Long 1b6acc3347 fix(tui): revert the Verbosity setting rename (#50982) 2026-09-23 17:18:22 -04:00
James Long 7d126eb1e5 fix(tui): rename the transcript verbosity setting to Verbosity (#50981) 2026-09-23 17:07:55 -04:00
James Long bbbac2507b feat(tui): add transcript verbosity levels (#50941) 2026-09-23 16:54:42 -04:00
Frank 7f93d0018c fix(www): widen docs content and keep navigation visible (#50974) 2026-09-23 16:35:35 -04:00
Filip 33b686feb9 feat: list & read resources as codemode tools (#50680) 2026-09-23 22:08:03 +02:00
Aiden Cline fb4422ef62 fix(ai): add types to untyped enums in Moonshot tool schemas (#50961) 2026-09-23 15:04:48 -05:00
Jack d31ce33ca5 docs(go): add GPT 6 Luna to V2 guide (#50946) 2026-09-24 03:25:23 +08:00
Shoubhit Dash 56db9053e8 feat(ai): add queued image routes, partial image streaming, and BFL, fal, Replicate, Stability (#50959) 2026-09-24 00:54:37 +05:30
opencode-agent[bot]andrekram1-node c98d9c65dc fix(cli): redact credentials in debug config (#50957)
Co-authored-by: rekram1-node <rekram1-node@users.noreply.github.com>
2026-09-23 14:04:22 -05:00
Aiden Cline b47824fb63 fix(core): recover reads with quote and accent variants (#50953) 2026-09-23 14:02:31 -05:00
Aiden Cline f51dc62721 fix(tui): remove the stale MCPFailed error message (#50958) 2026-09-23 14:02:07 -05:00
Aiden Cline ad15173aa7 fix(ai): ignore Vertex keepalives sent as SSE data (#50954) 2026-09-23 13:25:24 -05:00
Jack e64e4d0fbb docs: add Space Bunny to V2 console docs (#50940) 2026-09-24 01:03:39 +08:00
opencode-agent[bot]andMrMushrooooom 7f39ae9ecb fix(stats): attribute Hy4 preview traffic to Tencent (#50935)
Co-authored-by: MrMushrooooom <19261047+MrMushrooooom@users.noreply.github.com>
2026-09-24 00:56:42 +08:00
James Long b3b030856b feat(tui): budget transcript mounting by rendered entries (#50936) 2026-09-23 12:36:10 -04:00
Aiden Cline a117ebb408 fix(client): include the detail in client error messages (#50929) 2026-09-23 11:33:49 -05:00
Aiden Cline f2d9ebb886 fix(ai): send PDFs as file parts on OpenAI Chat routes (#50933) 2026-09-23 11:27:33 -05:00
James Long 0332a26be6 fix(tui): load history until the oldest group is complete (#50930) 2026-09-23 11:58:44 -04:00
opencode-agent[bot] bb7876dfa8 chore: update nix node_modules hashes 2026-09-23 15:48:10 +00:00
Shoubhit Dash dbaa57a21b feat(ai): add transcription across inline, streamed, and queued routes (#50910) 2026-09-23 21:15:50 +05:30
James Long b6bc55764c refactor(tui): persist group expansion and exact scroll anchors (#48489) 2026-09-23 11:33:19 -04:00
Aiden Cline 14a3311a61 feat(codemode): add Uint8Array callback methods and coerce indexOf start indexes (#50829) 2026-09-23 10:27:40 -05:00
opencode-agent[bot]andnexxeln dddb5eb96f fix(util): update OpenTelemetry to patched core (#50867)
Co-authored-by: nexxeln <95541290+nexxeln@users.noreply.github.com>
2026-09-23 20:57:16 +05:30
Shoubhit Dash affa57e40f fix(core): relax websocket idle timeout and honor chunkTimeout (#50914) 2026-09-23 20:22:41 +05:30
opencode-agent[bot] 681ea07a92 chore(core): refresh bundled models.dev snapshot 2026-09-23 12:20:47 +00:00
Adam 6e89d9e2c3 feat(core): identify the client on Console requests (#50896) 2026-09-23 07:09:19 -05:00
Shoubhit Dash f526727178 feat(ai): add speech generation with streaming media routes (#50883) 2026-09-23 17:35:23 +05:30
Luke Parker 9c8a63e852 fix(app): keep composer focus after attaching files (#50864) 2026-09-23 18:04:39 +10:00
Luke Parker fabf56781c fix(app): keep timeline scroll when switching models (#50865) 2026-09-23 18:03:42 +10:00
opencode a25d304201 sync release versions for v2.0.15 2026-09-23 07:15:55 +00:00
Shoubhit Dash cc8886c8bb feat(ai): add video generation with queued media routes (#50703) 2026-09-23 12:27:38 +05:30
Luke Parker 8ce629be22 fix(cli): keep Windows upgrades and uninstalls from fighting the running binary (#50819) 2026-09-23 06:35:40 +00:00
Aiden Cline 68b28bdb98 fix(codemode): coerce match/search patterns, allow any for...in target, bind the last duplicate parameter (#50802) 2026-09-22 23:32:27 -05:00
opencode-agent[bot] 150dc69e4b chore: update nix node_modules hashes 2026-09-23 04:29:53 +00:00
Vladimir Glafirov d5d4461e67 chore: bump gitlab-ai-provider to 6.16.0 (#50318) 2026-09-22 23:13:04 -05:00
Aiden Cline d56ce74373 fix(client): throw declared API errors as Error instances (#50788) 2026-09-22 21:25:28 -05:00
Aiden Cline 17abc5906b feat(codemode): add tagged templates and String.raw (#50791) 2026-09-22 21:25:21 -05:00
Aiden Cline 8656838a5b fix(ai): ignore bare null SSE frames (#50793) 2026-09-22 21:23:09 -05:00
Dax 53179daefa feat(core): order projects by recent activity (#50790) 2026-09-23 01:59:32 +00:00
Aiden Cline bab26d63ea docs(codemode): correct the interpreter support matrix (#50789) 2026-09-22 20:49:54 -05:00
Aiden Cline f0381e5da3 fix(core): normalize AI SDK fragment boundaries (#50685) 2026-09-22 20:27:34 -05:00
Aiden Cline 740072694d fix: show API error messages in remaining CLI and TUI paths (#50783) 2026-09-22 20:16:45 -05:00
Aiden Cline 2e4abeb25d fix(tui): show API error messages in toasts (#50778) 2026-09-22 20:02:32 -05:00
Aiden Cline 43f1dad8e1 fix(core): log error messages for MCP OAuth and credential failures (#50767) 2026-09-22 19:35:34 -05:00
OpeOginni cf4b4c2312 fix(tui): export complete session transcript (#50733) 2026-09-22 19:24:57 -05:00
Luke Parker 3bf8a5a8cf fix(app): keep Console sign-in visible when a Zen API key is stored (#50763) 2026-09-23 10:19:54 +10:00
Aiden Cline ddeb19790a fix(ai): replay Kimi reasoning details without the streaming index (#50383) 2026-09-22 19:10:57 -05:00
opencode-agent[bot]andrekram1-node fe0d1682ca fix(tui): show latest step in turn token summary (#50765)
Co-authored-by: rekram1-node <rekram1-node@users.noreply.github.com>
2026-09-22 18:56:52 -05:00
opencode-agent[bot] 1746672c42 chore: update nix node_modules hashes 2026-09-22 23:11:50 +00:00
Shoubhit Dash 3a2203eaac fix(core): install git plugins from branch subdirectories (#50754) 2026-09-23 04:21:36 +05:30
Shoubhit Dash 38c320ea4c fix(acp): forward provider retry status to clients (#50752) 2026-09-23 04:11:23 +05:30
opencode-agent[bot]andrekram1-node 5c53cfc342 feat(session): allow metadata updates (#50025)
Co-authored-by: rekram1-node <rekram1-node@users.noreply.github.com>
2026-09-22 17:27:38 -05:00
James Long 8683406690 feat(theme): support dynamic hue names (#50728) 2026-09-22 17:34:38 -04:00
Frank 126294a322 docs(www): merge Console documentation updates 2026-09-22 17:29:38 -04:00
opencode-agent[bot]andjlongster d2bbefbac8 fix(ci): avoid stale Bun dependency caches (#49845)
Co-authored-by: jlongster <17031+jlongster@users.noreply.github.com>
2026-09-22 16:30:15 -04:00
846 changed files with 67079 additions and 8173 deletions
+2 -4
View File
@@ -53,8 +53,6 @@ runs:
with:
path: ${{ steps.cache.outputs.dir }}
key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
restore-keys: |
${{ runner.os }}-bun-
- name: Install setuptools for distutils compatibility
run: python3 -m pip install setuptools || pip install setuptools || true
@@ -66,9 +64,9 @@ runs:
# e.g. ./patches/ for standard-openapi
# https://github.com/oven-sh/bun/issues/28147
if [ "$RUNNER_OS" = "Windows" ]; then
bun install --linker hoisted ${{ inputs.install-flags }}
bun install --frozen-lockfile --linker hoisted ${{ inputs.install-flags }}
else
bun install ${{ inputs.install-flags }}
bun install --frozen-lockfile ${{ inputs.install-flags }}
fi
shell: bash
+1 -1
View File
@@ -184,7 +184,7 @@ const table = sqliteTable("session", {
- Keep `SessionRunner`, model resolution, tool registry, permissions, and filesystem Location-scoped. Omitted `Location.workspaceID` means implicit-local placement; explicit workspace identity remains reserved for future placement semantics.
- Preserve one explicit `llm.stream(request)` call per Physical Attempt and reload projected history before durable continuation. A logical Step may use generic pre-output retries, one full-context retry after continuation rejection, incomplete-stream continuation, or one overflow-compaction rebuild. Generic retries retain the logical step number and do not consume another agent-step allowance. Do not delegate orchestration to an in-memory tool loop.
- Keep local Session drains process-local until clustering is implemented. `SessionRunCoordinator` joins explicit same-Session resumes, coalesces prompt wakeups, and allows different Sessions to run concurrently. A write-ahead execution claim marks a process-local busy period for restart recovery: terminal completion, failure, or user interruption releases it, while shutdown interruption and process death preserve it. Startup recovery resumes claimed top-level Sessions with durable per-execution attempt accounting. The claim is a recovery marker, not clustered ownership, fencing, or an exactly-once guarantee.
- Keep native compaction mechanisms out of `SessionCompaction`. Plugins register `native` strategies through the `SessionCompaction` editor that turn a prepared request into a replacement window (the built-in `NativeCompactionPlugin` handles `@opencode/ai` compaction operations); later registrations win. Core owns the provider-mode decision, route provenance, the retry policy, overflow recovery, interruption, usage accounting, and checkpoint persistence.
- Keep provider-specific native compaction mechanisms in `@opencode/ai` behind `LLMClient.compact`. `SessionCompaction` chooses a summary or native compaction from the model's `compaction` setting and owns route provenance, request shrinking, the retry policy, interruption, usage accounting, and checkpoint persistence.
- Keep delivery vocabulary explicit. Prompts steer by default. At safe step boundaries, steered compaction takes priority up to the first steered move control; other steers retain enqueue order. At an idle boundary, steers take priority; otherwise exactly one queued item delivers before the runner reevaluates continuation. Inbox items may be cancelled or changed between queue and steer before delivery. Promoting new user input resets the selected agent's step allowance; a batch of steers resets it once.
- One step is one logical LLM call; its durable record covers only the model-visible span. Do not write "provider turn", and do not use bare "turn" for a single call: "turn" is reserved for the future assistant-turn unit containing all steps from prompt promotion until the session would go idle.
- Keep event replay ownership separate from clustered Session execution ownership.
+110 -111
View File
@@ -32,7 +32,7 @@
},
"packages/ai": {
"name": "@opencode/ai",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@aws-sdk/credential-providers": "3.1057.0",
"@opencode/schema": "workspace:*",
@@ -54,7 +54,7 @@
},
"packages/app": {
"name": "@opencode/app",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@corvu/drawer": "catalog:",
"@dnd-kit/abstract": "0.5.0",
@@ -112,8 +112,9 @@
},
"packages/cli": {
"name": "@opencode/cli",
"version": "2.0.14",
"version": "2.0.18",
"bin": {
"opencode": "./bin/opencode.cjs",
"opencode2": "./bin/opencode2.cjs",
},
"dependencies": {
@@ -134,7 +135,6 @@
"effect": "catalog:",
"immer": "11.1.4",
"jsonc-parser": "3.3.1",
"open": "10.1.2",
"solid-js": "catalog:",
"tree-sitter-bash": "0.25.0",
"tree-sitter-powershell": "0.25.10",
@@ -176,7 +176,7 @@
},
"packages/client": {
"name": "@opencode/client",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@opencode/protocol": "workspace:*",
"@opencode/schema": "workspace:*",
@@ -202,7 +202,7 @@
},
"packages/codemode": {
"name": "@opencode/codemode",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"acorn": "8.15.0",
"effect": "catalog:",
@@ -215,7 +215,7 @@
},
"packages/console/app": {
"name": "@opencode/console-app",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@cloudflare/vite-plugin": "1.15.2",
"@ibm/plex": "6.4.1",
@@ -251,7 +251,7 @@
},
"packages/console/core": {
"name": "@opencode/console-core",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@aws-sdk/client-sts": "3.782.0",
"@jsx-email/render": "1.1.1",
@@ -278,7 +278,7 @@
},
"packages/console/function": {
"name": "@opencode/console-function",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@openauthjs/openauth": "0.0.0-20250322224806",
"@opencode/console-core": "workspace:*",
@@ -295,7 +295,7 @@
},
"packages/console/mail": {
"name": "@opencode/console-mail",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@jsx-email/all": "2.2.3",
"@jsx-email/cli": "1.4.3",
@@ -319,7 +319,7 @@
},
"packages/console/support": {
"name": "@opencode/console-support",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@cloudflare/vite-plugin": "1.15.2",
"@opencode/console-core": "workspace:*",
@@ -339,7 +339,7 @@
},
"packages/core": {
"name": "@opencode/core",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@ai-sdk/cohere": "3.0.27",
"@ai-sdk/gateway": "3.0.104",
@@ -367,7 +367,7 @@
"drizzle-orm": "catalog:",
"effect": "catalog:",
"fuzzysort": "3.1.0",
"gitlab-ai-provider": "6.12.1",
"gitlab-ai-provider": "6.18.0",
"google-auth-library": "10.5.0",
"gray-matter": "4.0.3",
"htmlparser2": "8.0.2",
@@ -407,7 +407,7 @@
},
"packages/desktop": {
"name": "@opencode/desktop",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@zip.js/zip.js": "2.7.62",
"electron-context-menu": "5.0.0",
@@ -456,7 +456,7 @@
},
"packages/enterprise": {
"name": "@opencode/enterprise",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@hono/standard-validator": "catalog:",
"@opencode-ai/sdk": "1.18.21",
@@ -493,7 +493,7 @@
},
"packages/function": {
"name": "@opencode/function",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@octokit/auth-app": "8.0.1",
"@octokit/rest": "catalog:",
@@ -509,7 +509,7 @@
},
"packages/http-recorder": {
"name": "@opencode/http-recorder",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@effect/platform-node-shared": "4.0.0-rc.112",
},
@@ -528,7 +528,7 @@
},
"packages/httpapi-codegen": {
"name": "@opencode/httpapi-codegen",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"effect": "catalog:",
"prettier": "3.6.2",
@@ -541,7 +541,7 @@
},
"packages/latex": {
"name": "@opencode/latex",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@opencode/plugin": "workspace:*",
"@opentui/core": "catalog:",
@@ -555,7 +555,7 @@
},
"packages/merman": {
"name": "@opencode/merman",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@opencode/plugin": "workspace:*",
"@opentui/core": "catalog:",
@@ -570,7 +570,7 @@
},
"packages/plugin": {
"name": "@opencode/plugin",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@ai-sdk/provider": "3.0.8",
"@opencode/ai": "workspace:*",
@@ -596,8 +596,8 @@
},
"peerDependencies": {
"@opencode/theme": "workspace:*",
"@opentui/core": ">=0.5.10",
"@opentui/solid": ">=0.5.10",
"@opentui/core": ">=0.5.12",
"@opentui/solid": ">=0.5.12",
"solid-js": ">=1.9.0",
},
"optionalPeers": [
@@ -609,7 +609,7 @@
},
"packages/plugin-browser": {
"name": "@opencode/plugin-browser",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@opencode/plugin": "workspace:*",
"@opencode/schema": "workspace:*",
@@ -639,7 +639,7 @@
},
"packages/protocol": {
"name": "@opencode/protocol",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@opencode/schema": "workspace:*",
"effect": "catalog:",
@@ -654,7 +654,7 @@
},
"packages/schema": {
"name": "@opencode/schema",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@standard-schema/spec": "catalog:",
"effect": "catalog:",
@@ -678,7 +678,7 @@
},
"packages/sdk": {
"name": "@opencode/sdk",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@opencode/client": "workspace:*",
"@opencode/core": "workspace:*",
@@ -699,7 +699,7 @@
},
"packages/server": {
"name": "@opencode/server",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@effect/platform-node": "catalog:",
"@effect/platform-node-shared": "catalog:",
@@ -721,7 +721,7 @@
},
"packages/session-ui": {
"name": "@opencode/session-ui",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@kobalte/core": "catalog:",
"@opencode/client": "workspace:*",
@@ -756,7 +756,7 @@
},
"packages/simulation": {
"name": "@opencode/simulation",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@opencode/ai": "workspace:*",
"@opencode/core": "workspace:*",
@@ -776,7 +776,7 @@
},
"packages/stats/app": {
"name": "@opencode/stats-app",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@ibm/plex": "6.4.1",
"@kobalte/core": "catalog:",
@@ -810,7 +810,7 @@
},
"packages/stats/core": {
"name": "@opencode/stats-core",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@aws-sdk/client-athena": "3.933.0",
"@planetscale/database": "1.19.0",
@@ -829,7 +829,7 @@
},
"packages/stats/server": {
"name": "@opencode/stats-server",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@aws-sdk/client-firehose": "3.933.0",
"@effect/platform-node": "catalog:",
@@ -875,7 +875,7 @@
},
"packages/theme": {
"name": "@opencode/theme",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@opentui/core": "catalog:",
"effect": "catalog:",
@@ -889,7 +889,7 @@
},
"packages/tui": {
"name": "@opencode/tui",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@opencode/client": "workspace:*",
"@opencode/core": "workspace:*",
@@ -908,7 +908,6 @@
"effect": "catalog:",
"fuzzysort": "catalog:",
"get-east-asian-width": "catalog:",
"open": "10.1.2",
"opentui-spinner": "catalog:",
"remeda": "catalog:",
"solid-js": "catalog:",
@@ -924,7 +923,7 @@
},
"packages/ui": {
"name": "@opencode/ui",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@kobalte/core": "catalog:",
"@pierre/diffs": "catalog:",
@@ -959,7 +958,7 @@
},
"packages/util": {
"name": "@opencode/util",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@effect/opentelemetry": "catalog:",
"@effect/platform-node": "catalog:",
@@ -967,16 +966,21 @@
"@npmcli/arborist": "catalog:",
"@npmcli/config": "10.8.1",
"@opentelemetry/api": "1.9.0",
"@opentelemetry/context-async-hooks": "2.6.1",
"@opentelemetry/exporter-trace-otlp-http": "0.214.0",
"@opentelemetry/sdk-trace-base": "2.6.1",
"@opentelemetry/sdk-trace-node": "2.6.1",
"@opentelemetry/api-logs": "0.219.0",
"@opentelemetry/context-async-hooks": "2.8.0",
"@opentelemetry/exporter-trace-otlp-http": "0.219.0",
"@opentelemetry/resources": "2.8.0",
"@opentelemetry/sdk-logs": "0.219.0",
"@opentelemetry/sdk-metrics": "2.8.0",
"@opentelemetry/sdk-trace-base": "2.8.0",
"@opentelemetry/sdk-trace-node": "2.8.0",
"cross-spawn": "catalog:",
"effect": "catalog:",
"glob": "13.0.5",
"mime-types": "3.0.2",
"minimatch": "10.2.5",
"npm-package-arg": "13.0.2",
"open": "11.0.4",
"pacote": "21.5.1",
},
"devDependencies": {
@@ -992,7 +996,7 @@
},
"packages/web": {
"name": "@opencode/web",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"@astrojs/cloudflare": "12.6.3",
"@astrojs/markdown-remark": "6.3.1",
@@ -1033,7 +1037,7 @@
},
"services/update": {
"name": "@opencode/update",
"version": "2.0.14",
"version": "2.0.18",
"dependencies": {
"jose": "6.0.11",
"semver": "catalog:",
@@ -1068,13 +1072,13 @@
"trustedDependencies": [
"electron",
"esbuild",
"protobufjs",
],
"patchedDependencies": {
"@pierre/trees@1.0.0-beta.4": "patches/@pierre%2Ftrees@1.0.0-beta.4.patch",
"@tanstack/virtual-core@3.17.8": "patches/@tanstack%2Fvirtual-core@3.17.8.patch",
"ghostty-web@github:anomalyco/ghostty-web#83c0a07": "patches/ghostty-web@0.3.0.patch",
"@modelcontextprotocol/client@2.0.0": "patches/@modelcontextprotocol%2Fclient@2.0.0.patch",
"pacote@21.5.1": "patches/pacote@21.5.1.patch",
"@standard-community/standard-openapi@0.2.9": "patches/@standard-community%2Fstandard-openapi@0.2.9.patch",
"@npmcli/agent@4.0.2": "patches/@npmcli%2Fagent@4.0.2.patch",
"@silvia-odwyer/photon-node@0.3.4": "patches/@silvia-odwyer%2Fphoton-node@0.3.4.patch",
@@ -1093,6 +1097,7 @@
"@types/node": "catalog:",
"bun-types": "1.4.2",
"effect": "catalog:",
"open": "11.0.4",
"solid-js": "catalog:",
},
"catalog": {
@@ -1109,9 +1114,9 @@
"@npmcli/arborist": "9.4.0",
"@octokit/rest": "22.0.0",
"@openauthjs/openauth": "0.0.0-20250322224806",
"@opentui/core": "0.5.10",
"@opentui/keymap": "0.5.10",
"@opentui/solid": "0.5.10",
"@opentui/core": "0.5.12",
"@opentui/keymap": "0.5.12",
"@opentui/solid": "0.5.12",
"@pierre/diffs": "1.2.10",
"@playwright/test": "1.59.1",
"@sentry/solid": "10.71.0",
@@ -2219,55 +2224,55 @@
"@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="],
"@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.214.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-40lSJeqYO8Uz2Yj7u94/SJWE/wONa7rmMKjI1ZcIjgf3MHNHv1OZUCrCETGuaRF62d5pQD1wKIW+L4lmSMTzZA=="],
"@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.219.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-FFx7YnaYJlIjqWW/AG/yAZ0L/NEY724PipXXXQLdtZPbLwBGbUMTGL1i/esI56TWfTUXxhLfpgrnWJCG8aUJyg=="],
"@opentelemetry/context-async-hooks": ["@opentelemetry/context-async-hooks@2.6.1", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-XHzhwRNkBpeP8Fs/qjGrAf9r9PRv67wkJQ/7ZPaBQQ68DYlTBBx5MF9LvPx7mhuXcDessKK2b+DcxqwpgkcivQ=="],
"@opentelemetry/context-async-hooks": ["@opentelemetry/context-async-hooks@2.8.0", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-/3FIraneMcng67SUJCxvyInk/oxzwsxyadufk0wwfOBLf5wqtAGX4MoQASwSbndBPeARzBryUM9Azr5kHIdWLw=="],
"@opentelemetry/core": ["@opentelemetry/core@2.6.1", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-8xHSGWpJP9wBxgBpnqGL0R3PbdWQndL1Qp50qrg71+B28zK5OQmUgcDKLJgzyAAV38t4tOyLMGDD60LneR5W8g=="],
"@opentelemetry/core": ["@opentelemetry/core@2.8.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-hd1Lfh8p545nNz+jq1Ejfz+Mn1hyLuxYn1YzTfFNrxr8urEWMNQLPf1Th8kjOH+HxwawCrtgBp8JpBUR4ZSgww=="],
"@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/exporter-trace-otlp-http@0.214.0", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/otlp-exporter-base": "0.214.0", "@opentelemetry/otlp-transformer": "0.214.0", "@opentelemetry/resources": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-kIN8nTBMgV2hXzV/a20BCFilPZdAIMYYJGSgfMMRm/Xa+07y5hRDS2Vm12A/z8Cdu3Sq++ZvJfElokX2rkgGgw=="],
"@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/exporter-trace-otlp-http@0.219.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/otlp-exporter-base": "0.219.0", "@opentelemetry/otlp-transformer": "0.219.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-9t6SvBXXBEjOBcIzgozvBbd3jWrv3Gt3ngGhl1fhdZ/zRc7oZDVOFEqbi2zlBpW9BXhgDMKv422J0DL/3iQWfw=="],
"@opentelemetry/instrumentation": ["@opentelemetry/instrumentation@0.220.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "import-in-the-middle": "^3.0.0", "require-in-the-middle": "^8.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw=="],
"@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.214.0", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/otlp-transformer": "0.214.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-u1Gdv0/E9wP+apqWf7Wv2npXmgJtxsW2XL0TEv9FZloTZRuMBKmu8cYVXwS4Hm3q/f/3FuCnPTgiwYvIqRSpRg=="],
"@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.219.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/otlp-transformer": "0.219.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-zvIxQX/AZUVKDU+hCuYx+7UkiP7GRdnk1ZbFQRYzHvYp47cAWR4j3IhoPhV9KaeXEv2xdGq3IA6PnpzDmLcmSA=="],
"@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.214.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.214.0", "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", "@opentelemetry/sdk-logs": "0.214.0", "@opentelemetry/sdk-metrics": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1", "protobufjs": "^7.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-DSaYcuBRh6uozfsWN3R8HsN0yDhCuWP7tOFdkUOVaWD1KVJg8m4qiLUsg/tNhTLS9HUYUcwNpwL2eroLtsZZ/w=="],
"@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.219.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.219.0", "@opentelemetry/core": "2.8.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-logs": "0.219.0", "@opentelemetry/sdk-metrics": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-aaYKAyXhw9VchKZVGOopD3Gw/kPsyrX2c6IQ0AW32mTjqmZOh5Y6Gf5OYqTNqVktAeBjmFinhyFaCwW6GYK9YQ=="],
"@opentelemetry/resources": ["@opentelemetry/resources@2.6.1", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-lID/vxSuKWXM55XhAKNoYXu9Cutoq5hFdkbTdI/zDKQktXzcWBVhNsOkiZFTMU9UtEWuGRNe0HUgmsFldIdxVA=="],
"@opentelemetry/resources": ["@opentelemetry/resources@2.8.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-qmXQ27ilDbUK/vGMqwL8D4/rhn76C+sherM4wTbjlfknR8Nvfc/hCxjRJPhkzZzUsPiNg16SA31NxMabwttRjg=="],
"@opentelemetry/sdk-logs": ["@opentelemetry/sdk-logs@0.214.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.214.0", "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-zf6acnScjhsaBUU22zXZ/sLWim1dfhUAbGXdMmHmNG3LfBnQ3DKsOCITb2IZwoUsNNMTogqFKBnlIPPftUgGwA=="],
"@opentelemetry/sdk-logs": ["@opentelemetry/sdk-logs@0.219.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.219.0", "@opentelemetry/core": "2.8.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-s6lTKRakaPClvKoWHRChxnXjDMkM/TQ30ff78jN6EBGf7MI7VzANE5PU3f4z9qDUudWjvZjOLHG0rBnBKYvoXA=="],
"@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@2.6.1", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-9t9hJHX15meBy2NmTJxL+NJfXmnausR2xUDvE19XQce0Qi/GBtDGamU8nS1RMbdgDmhgpm3VaOu2+fiS/SfTpQ=="],
"@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@2.8.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/resources": "2.8.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-UDBGaj6W0Rgy5rTTaoxs8gVGF/aGkAKyjurJv7se6wjRxJu7FoquTLT/vt54DZfo4crbprYfhX/SOK9+BPw1qg=="],
"@opentelemetry/sdk-trace": ["@opentelemetry/sdk-trace@2.11.0", "", { "dependencies": { "@opentelemetry/core": "2.11.0", "@opentelemetry/resources": "2.11.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA=="],
"@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.6.1", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-r86ut4T1e8vNwB35CqCcKd45yzqH6/6Wzvpk2/cZB8PsPLlZFTvrh8yfOS3CYZYcUmAx4hHTZJ8AO8Dj8nrdhw=="],
"@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.8.0", "", { "dependencies": { "@opentelemetry/core": "2.8.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-mhU4jp+vW0mGbFRd+GeXHvmfA4aDqWjBjLC3pE5XMpLs0IE2ryYb019Ts2AQrOq67gaTF25D91+fgvEHDZEnuQ=="],
"@opentelemetry/sdk-trace-node": ["@opentelemetry/sdk-trace-node@2.6.1", "", { "dependencies": { "@opentelemetry/context-async-hooks": "2.6.1", "@opentelemetry/core": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-Hh2i4FwHWRFhnO2Q/p6svMxy8MPsNCG0uuzUY3glqm0rwM0nQvbTO1dXSp9OqQoTKXcQzaz9q1f65fsurmOhNw=="],
"@opentelemetry/sdk-trace-node": ["@opentelemetry/sdk-trace-node@2.8.0", "", { "dependencies": { "@opentelemetry/context-async-hooks": "2.8.0", "@opentelemetry/core": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-nZt9OGufioAc3AfoLTqA9bsAeaMJAictYDdI2VcNQ+PmT+3rfKjAZDZvgPfd8VPX0O5Bw1hdQF6kDK8VSpZiWg=="],
"@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.43.0", "", {}, "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg=="],
"@opentui/core": ["@opentui/core@0.5.10", "", { "dependencies": { "bun-ffi-structs": "0.3.1", "diff": "9.0.0", "marked": "17.0.1", "string-width": "7.2.0", "strip-ansi": "7.1.2" }, "optionalDependencies": { "@opentui/core-darwin-arm64": "0.5.10", "@opentui/core-darwin-x64": "0.5.10", "@opentui/core-linux-arm64": "0.5.10", "@opentui/core-linux-arm64-musl": "0.5.10", "@opentui/core-linux-x64": "0.5.10", "@opentui/core-linux-x64-musl": "0.5.10", "@opentui/core-win32-arm64": "0.5.10", "@opentui/core-win32-x64": "0.5.10" }, "peerDependencies": { "web-tree-sitter": "0.25.10" } }, "sha512-C3a2UbmefeAjIxAgm4BqjuSxKT4oqutfvYFwVvUgMxmGRHkNbBc/s7sukV0JgwcxFcV3uMFrXxo+E+BQtvuOiw=="],
"@opentui/core": ["@opentui/core@0.5.12", "", { "dependencies": { "bun-ffi-structs": "0.3.1", "diff": "9.0.0", "marked": "17.0.1", "string-width": "7.2.0", "strip-ansi": "7.1.2" }, "optionalDependencies": { "@opentui/core-darwin-arm64": "0.5.12", "@opentui/core-darwin-x64": "0.5.12", "@opentui/core-linux-arm64": "0.5.12", "@opentui/core-linux-arm64-musl": "0.5.12", "@opentui/core-linux-x64": "0.5.12", "@opentui/core-linux-x64-musl": "0.5.12", "@opentui/core-win32-arm64": "0.5.12", "@opentui/core-win32-x64": "0.5.12" }, "peerDependencies": { "web-tree-sitter": "0.25.10" } }, "sha512-ZXBE5gmvdovmV8zJQrOQf6E44v1tJRDEgrM2MYhEglzgXZ+smIUp95O8zeRYGsuIzQIiMPMgQqKtTJuzvAb7BQ=="],
"@opentui/core-darwin-arm64": ["@opentui/core-darwin-arm64@0.5.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Vyb+nTbhab8ZcRy5gg1loEEGwRcIbjAeVRIBfHBcbFDqmITBOg7x2gqJ+x/TnoOy4uwMhCmICUN2wiyREw3r1Q=="],
"@opentui/core-darwin-arm64": ["@opentui/core-darwin-arm64@0.5.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-YdVnP0tAyerBNl0mIcmQEOotPeZzW1VnSXKBl5cyZ5e6nDd2Y+ui/8eRPpn1oqcamf1NCnzS4ohMgejOvna8Zg=="],
"@opentui/core-darwin-x64": ["@opentui/core-darwin-x64@0.5.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-tTFLcM7Oj1gTyhm/bUdAt3C6grZdCxPk6+/g2azcZBUlI3/62LwbeRS6HbQKFFmm+1fUmX8cq6kWrtul885mVg=="],
"@opentui/core-darwin-x64": ["@opentui/core-darwin-x64@0.5.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-uRrQJdHmLUSj3PV23QPi3WSimYTTxcXnVouxF6U4xMXlOv4N3SxnHfVwMRQkPqbGOfvVWHeLE6FdK4C+ubU0sQ=="],
"@opentui/core-linux-arm64": ["@opentui/core-linux-arm64@0.5.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-ncJXcgudhBf2GdJyF3xVQN/Ec+1F7GOL+pRrURmgBYSj2v1w6EyoDQFAACtPTK2c3R38W6fvZwL4JSLlm4EFXQ=="],
"@opentui/core-linux-arm64": ["@opentui/core-linux-arm64@0.5.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-XeKhuIaEtgipvuPHbl4qPOBj+Ut+2zObmsxMVM1jDcjz/FatG9PGeGQPx1G1SnvH2AgpT4K+eCu7DUF0+yIqoQ=="],
"@opentui/core-linux-arm64-musl": ["@opentui/core-linux-arm64-musl@0.5.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-dGMphDKexSdeYqwl0wgoFBP88Ta/cdi1Zc1mk29/ENkSCGz+74zlCHgqTHRNGLmI8W5TfuUtCyktQH11/Z+TBQ=="],
"@opentui/core-linux-arm64-musl": ["@opentui/core-linux-arm64-musl@0.5.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-VZ2sNMw1d/r1SLPjUbOP9LKscKz1CQjID8adTL6gG8Lrrq+mYcIUxutyB+P/eG0J/7oRZLPR6OMt7dUOap6RTg=="],
"@opentui/core-linux-x64": ["@opentui/core-linux-x64@0.5.10", "", { "os": "linux", "cpu": "x64" }, "sha512-5qtYaOgwVycZD1GaGshTRsi0rXPAmVExO03N1JQaHu+NYxK/vXSOc7Bu4QW0sPXx3Sp0SpzpP+FHjXABfoK66g=="],
"@opentui/core-linux-x64": ["@opentui/core-linux-x64@0.5.12", "", { "os": "linux", "cpu": "x64" }, "sha512-eZiCjEzwbb6qClPPfk32Nha9xmr9obt69Xj0+9SKsXxWLBKkjQEGOMRoh/R9ObaQF4aq8If1xV3VEY0sD9W9vg=="],
"@opentui/core-linux-x64-musl": ["@opentui/core-linux-x64-musl@0.5.10", "", { "os": "linux", "cpu": "x64" }, "sha512-Oj4H9hApuvuTKPWxh4SoZAgGJorR7vbvnrZA/cAkSMAk2VGSoHRRcqeXQbcH8IcdjVZ0KFpv8Zkl/D5Ye+2mew=="],
"@opentui/core-linux-x64-musl": ["@opentui/core-linux-x64-musl@0.5.12", "", { "os": "linux", "cpu": "x64" }, "sha512-WWW0hVBoSYZ3D6AgZ4u2Y5/u/IyIq2pDb+4yI3WgJ70Wyt6ofHy+6kRGRgbXFn1p+rPInAHjCXD2v6C7iEKSrA=="],
"@opentui/core-win32-arm64": ["@opentui/core-win32-arm64@0.5.10", "", { "os": "win32", "cpu": "arm64" }, "sha512-A9VhgvTxQoUdZ+8LmUumEng1sQNbj9QQQT3NYG9mSxI54qTANi7vOWNSphMiY6RMVsr22pgm6nUvSSvJXv7Jog=="],
"@opentui/core-win32-arm64": ["@opentui/core-win32-arm64@0.5.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-aLbm6870Ybls6CYL4zMOCImTBPLZHZMUXJFGqMI44lIWxitkAtT6zg5lYA4oRqFRzzryDclxr29+hDgT3p3Blw=="],
"@opentui/core-win32-x64": ["@opentui/core-win32-x64@0.5.10", "", { "os": "win32", "cpu": "x64" }, "sha512-u3KHa7kEeWrmKVDRJYpxSGO+g5E9cMGlrmTsPN3GVPHUmQMiREUawLXUvsU8+IHaQnqG3Q5nuE1yf4fPBzS+Qw=="],
"@opentui/core-win32-x64": ["@opentui/core-win32-x64@0.5.12", "", { "os": "win32", "cpu": "x64" }, "sha512-KTwtwpfd2zF9opVh3SyRJYDd1o3Xv4XL8OZb8Zi+CqWUel6Y2IDCiVivCv8fGJt3J7wOIXXtuZI9ZUkLyKJCiQ=="],
"@opentui/keymap": ["@opentui/keymap@0.5.10", "", { "dependencies": { "@opentui/core": "0.5.10" }, "peerDependencies": { "@opentui/react": "0.5.10", "@opentui/solid": "0.5.10", "react": ">=19.2.0", "solid-js": "1.9.12" }, "optionalPeers": ["@opentui/react", "@opentui/solid", "react", "solid-js"] }, "sha512-8vDJF+ltXscSnLEv3rgCa4m7PcoYZeUT9BngugpFCmVoNevbaRtYijjdfiUuLmXfT61lO5QbR6nEhn2RZMK8ow=="],
"@opentui/keymap": ["@opentui/keymap@0.5.12", "", { "dependencies": { "@opentui/core": "0.5.12" }, "peerDependencies": { "@opentui/react": "0.5.12", "@opentui/solid": "0.5.12", "react": ">=19.2.0", "solid-js": "1.9.12" }, "optionalPeers": ["@opentui/react", "@opentui/solid", "react", "solid-js"] }, "sha512-yWPvJjRhJTRoRSUucQq9Ua8ZW7n/2YQ/j6JxWq5Qekm4WuFiTplEkebR/Aj2/xA8tX68NOE5qv1LrY0Jk3NLNQ=="],
"@opentui/solid": ["@opentui/solid@0.5.10", "", { "dependencies": { "@babel/core": "7.28.0", "@babel/preset-typescript": "7.27.1", "@opentui/core": "0.5.10", "babel-plugin-module-resolver": "5.0.2", "babel-preset-solid": "1.9.12", "entities": "7.0.1", "s-js": "^0.4.9" }, "peerDependencies": { "solid-js": "1.9.12" } }, "sha512-KrmMIsHiKBHOABTC0brOwqWm+sGq1ZX2sGCAx6WgtBbE3STMup9n8TAy/6gUYhwcjC9zugT53ytfSVwCwVWZUg=="],
"@opentui/solid": ["@opentui/solid@0.5.12", "", { "dependencies": { "@babel/core": "7.28.0", "@babel/preset-typescript": "7.27.1", "@opentui/core": "0.5.12", "babel-plugin-module-resolver": "5.0.2", "babel-preset-solid": "1.9.12", "entities": "7.0.1", "s-js": "^0.4.9" }, "peerDependencies": { "solid-js": "1.9.12" } }, "sha512-hAiVlVMtT7AkHGblKwcW1YAuXtxkSy1XSf/RRc4j3IlG3mTNX0bhJdnGOo3Xw14EqeZMp41Mcp5WzHAzMm/DzA=="],
"@oslojs/asn1": ["@oslojs/asn1@1.0.0", "", { "dependencies": { "@oslojs/binary": "1.0.0" } }, "sha512-zw/wn0sj0j0QKbIXfIlnEcTviaCzYOY3V5rAyjR6YtOByFtJiT574+8p9Wlach0lZH9fddD4yb9laEAIl4vXQA=="],
@@ -2553,24 +2558,6 @@
"@protobuf-ts/runtime-rpc": ["@protobuf-ts/runtime-rpc@2.11.1", "", { "dependencies": { "@protobuf-ts/runtime": "^2.11.1" } }, "sha512-4CqqUmNA+/uMz00+d3CYKgElXO9VrEbucjnBFEjqI4GuDrEQ32MaI3q+9qPBvIGOlL4PmHXrzM32vBPWRhQKWQ=="],
"@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="],
"@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="],
"@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="],
"@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.1", "", {}, "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg=="],
"@protobufjs/fetch": ["@protobufjs/fetch@1.1.1", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1" } }, "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw=="],
"@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="],
"@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="],
"@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="],
"@protobufjs/utf8": ["@protobufjs/utf8@1.1.2", "", {}, "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug=="],
"@puppeteer/browsers": ["@puppeteer/browsers@3.2.1", "", { "dependencies": { "modern-tar": "^0.8.0", "yargs": "^18.0.0" }, "peerDependencies": { "proxy-agent": ">=8.0.1", "yauzl": "^2.10.0 || ^3.4.0" }, "optionalPeers": ["proxy-agent", "yauzl"], "bin": { "browsers": "lib/main-cli.js" } }, "sha512-KDz+3qDRdBAlRlMjmKyj6dEs33YHTk/xRHEENSXq6TNnhgoU15ruSHtEBeVF6OZ9tBDY55Se4P0nFMNsipzU9A=="],
"@radix-ui/colors": ["@radix-ui/colors@1.0.1", "", {}, "sha512-xySw8f0ZVsAEP+e7iLl3EvcBXX7gsIlC1Zso/sPBW9gIWerBTgz6axrjU+MZ39wD+WFi5h5zdWpsg3+hwt2Qsg=="],
@@ -4127,7 +4114,7 @@
"github-slugger": ["github-slugger@2.0.0", "", {}, "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw=="],
"gitlab-ai-provider": ["gitlab-ai-provider@6.12.1", "", { "dependencies": { "@anthropic-ai/sdk": "^0.71.0", "@anycable/core": "^0.9.2", "graphql-request": "^6.1.0", "isomorphic-ws": "^5.0.0", "openai": "^6.16.0", "socket.io-client": "^4.8.1", "vscode-jsonrpc": "^8.2.1", "zod": "^3.25.76" }, "peerDependencies": { "@ai-sdk/provider": ">=3.0.0", "@ai-sdk/provider-utils": ">=4.0.0" } }, "sha512-Qn5iHqvjG8yktI5MWaUgdRR94l7O4WtYW0CAbhsCh1Tj0Fei/DeprOYPVyf4Nht1Ix6U2PXSYM32QOHI6Z2TDw=="],
"gitlab-ai-provider": ["gitlab-ai-provider@6.18.0", "", { "dependencies": { "@anthropic-ai/sdk": "^0.71.0", "@anycable/core": "^0.9.2", "graphql-request": "^6.1.0", "isomorphic-ws": "^5.0.0", "openai": "^6.16.0", "socket.io-client": "^4.8.1", "vscode-jsonrpc": "^8.2.1", "zod": "^3.25.76" }, "peerDependencies": { "@ai-sdk/provider": ">=3.0.0", "@ai-sdk/provider-utils": ">=4.0.0" } }, "sha512-dXTXkNt1SFCL7jGlqazHL6iUJEug21qq0tx2s+Tui7jWNUqIAHEzY9WY1+PnZlhxvB56jPPBcFAT0attpjnopg=="],
"glob": ["glob@13.0.5", "", { "dependencies": { "minimatch": "^10.2.1", "minipass": "^7.1.2", "path-scurry": "^2.0.0" } }, "sha512-BzXxZg24Ibra1pbQ/zE7Kys4Ua1ks7Bn6pKLkVPZ9FZe4JQS6/Q7ef3LG1H+k7lUf5l4T3PLSyYyYJVYUvfgTw=="],
@@ -4335,7 +4322,7 @@
"is-decimal": ["is-decimal@2.0.1", "", {}, "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A=="],
"is-docker": ["is-docker@3.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="],
"is-docker": ["is-docker@4.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-LHE+wROyG/Y/0ZnbktRCoTix2c1RhgWaZraMZ8o1Q7zCh0VSrICJQO5oqIIISrcSBtrXv0o233w1IYwsWCjTzA=="],
"is-document.all": ["is-document.all@1.0.0", "", { "dependencies": { "call-bound": "^1.0.4" } }, "sha512-+XSoyS05OdBbhFuELhgTCpFNHkpBOJqtsZfUFFpe5QTw+9Sjbh8zitxhQkYAo6wV7e1Vb8cAPvpCk9jGam/82g=="],
@@ -4353,6 +4340,8 @@
"is-hexadecimal": ["is-hexadecimal@2.0.1", "", {}, "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg=="],
"is-in-ssh": ["is-in-ssh@1.0.0", "", {}, "sha512-jYa6Q9rH90kR1vKB6NM7qqd1mge3Fx4Dhw5TVlK1MUBqhEOuCagrEHMevNuCcbECmXZ0ThXkRm+Ymr51HwEPAw=="],
"is-inside-container": ["is-inside-container@1.0.0", "", { "dependencies": { "is-docker": "^3.0.0" }, "bin": { "is-inside-container": "cli.js" } }, "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA=="],
"is-map": ["is-map@2.0.3", "", {}, "sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw=="],
@@ -4399,7 +4388,7 @@
"is-whitespace": ["is-whitespace@0.3.0", "", {}, "sha512-RydPhl4S6JwAyj0JJjshWJEFG6hNye3pZFBRZaTUfZFwGHxzppNaNOVgQuS/E/SlhrApuMXrpnK1EEIXfdo3Dg=="],
"is-wsl": ["is-wsl@3.1.1", "", { "dependencies": { "is-inside-container": "^1.0.0" } }, "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw=="],
"is-wsl": ["is-wsl@2.2.0", "", { "dependencies": { "is-docker": "^2.0.0" } }, "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww=="],
"isarray": ["isarray@1.0.0", "", {}, "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="],
@@ -4863,7 +4852,7 @@
"oniguruma-to-es": ["oniguruma-to-es@4.3.6", "", { "dependencies": { "oniguruma-parser": "^0.12.2", "regex": "^6.1.0", "regex-recursion": "^6.0.2" } }, "sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA=="],
"open": ["open@10.1.2", "", { "dependencies": { "default-browser": "^5.2.1", "define-lazy-prop": "^3.0.0", "is-inside-container": "^1.0.0", "is-wsl": "^3.1.0" } }, "sha512-cxN6aIDPz6rm8hbebcP7vrQNhvRcveZoJU72Y7vskh4oIm+BZwBECnx5nTmrlres1Qapvx27Qo1Auukpf8PKXw=="],
"open": ["open@11.0.4", "", { "dependencies": { "default-browser": "^5.5.1", "define-lazy-prop": "^3.0.0", "is-in-ssh": "^1.0.0", "is-inside-container": "^1.0.0", "powershell-utils": "^0.2.1", "wsl-utils": "^1.0.0" } }, "sha512-++Zlftm0kVLPmzC06t6epuWmcRMDbI4z5P3NNX979WA/k23+NtSOynEGzsVfZwguKw2mi5umVgnBlJQMwRz4Pg=="],
"openai": ["openai@6.49.0", "", { "peerDependencies": { "@aws-sdk/credential-provider-node": ">=3.972.0 <4", "@smithy/hash-node": ">=4.3.0 <5", "@smithy/signature-v4": ">=5.4.0 <6", "ws": "^8.18.0", "zod": "^3.25 || ^4.0" }, "optionalPeers": ["@aws-sdk/credential-provider-node", "@smithy/hash-node", "@smithy/signature-v4", "ws", "zod"] }, "sha512-aYCc0C6L864eR6WSYIwQGyXriw/nIyZx0ObvhzOEVuk0zoBDpynjSbrionWI7q65B5H8jJX0DXR9snEzM6bfPg=="],
@@ -5007,6 +4996,8 @@
"postject": ["postject@1.0.0-alpha.6", "", { "dependencies": { "commander": "^9.4.0" }, "bin": { "postject": "dist/cli.js" } }, "sha512-b9Eb8h2eVqNE8edvKdwqkrY6O7kAwmI8kcnBv1NScolYJbo59XUF0noFq+lxbC1yN20bmC0WBEbDC5H/7ASb0A=="],
"powershell-utils": ["powershell-utils@0.2.1", "", {}, "sha512-C+y9x90UElAddDZmV4qOx9W53B61PO7cIqWz2dQsWlwswuq4mr8NEwytdGKboYbQlGZ3awrkTeNvcZiZNHnQ8A=="],
"preact": ["preact@11.0.0-beta.0", "", {}, "sha512-IcODoASASYwJ9kxz7+MJeiJhvLriwSb4y4mHIyxdgaRZp6kPUud7xytrk/6GZw8U3y6EFJaRb5wi9SrEK+8+lg=="],
"preact-render-to-string": ["preact-render-to-string@6.6.5", "", { "peerDependencies": { "preact": ">=10 || >= 11.0.0-0" } }, "sha512-O6MHzYNIKYaiSX3bOw0gGZfEbOmlIDtDfWwN1JJdc/T3ihzRT6tGGSEWE088dWrEDGa1u7101q+6fzQnO9XCPA=="],
@@ -5047,8 +5038,6 @@
"proto-list": ["proto-list@1.2.4", "", {}, "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA=="],
"protobufjs": ["protobufjs@7.6.5", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw=="],
"proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="],
"pump": ["pump@3.0.4", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA=="],
@@ -5829,7 +5818,7 @@
"ws": ["ws@8.21.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g=="],
"wsl-utils": ["wsl-utils@0.1.0", "", { "dependencies": { "is-wsl": "^3.1.0" } }, "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw=="],
"wsl-utils": ["wsl-utils@1.0.0", "", { "dependencies": { "is-wsl": "^3.1.0", "powershell-utils": "^0.1.0" } }, "sha512-Hl0ZOAs672vg+06kfujwRhoS6/jehvULrlFkuF2dRu6pHgA8U06h3xqNIqNNU1LTXPcedxByAR4GS6pwQK0mgA=="],
"xdg-basedir": ["xdg-basedir@5.1.0", "", {}, "sha512-GCPAHLvrIH13+c0SuacwvRYj2SxJXQ4kaVTT5xgL3kPrz56XxkF21IGhjSE1+W0aw7gpBWRGXLCPnPby6lSpmQ=="],
@@ -5925,8 +5914,6 @@
"@astrojs/telemetry/ci-info": ["ci-info@4.4.0", "", {}, "sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg=="],
"@astrojs/telemetry/is-docker": ["is-docker@4.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-LHE+wROyG/Y/0ZnbktRCoTix2c1RhgWaZraMZ8o1Q7zCh0VSrICJQO5oqIIISrcSBtrXv0o233w1IYwsWCjTzA=="],
"@aws-crypto/crc32/@aws-sdk/types": ["@aws-sdk/types@3.974.4", "", { "dependencies": { "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-dSFDNG00MEz0/xl5gxL62giLd1iYyJsTxZ1I1DOj6lC+bbgLB4TRsYClJg3b62dhXT1uATzsTNXPnC+33EJV3A=="],
"@aws-crypto/crc32c/@aws-sdk/types": ["@aws-sdk/types@3.974.4", "", { "dependencies": { "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-dSFDNG00MEz0/xl5gxL62giLd1iYyJsTxZ1I1DOj6lC+bbgLB4TRsYClJg3b62dhXT1uATzsTNXPnC+33EJV3A=="],
@@ -6245,6 +6232,8 @@
"@opencode/www/wrangler": ["wrangler@4.110.0", "", { "dependencies": { "@cloudflare/kv-asset-handler": "0.5.0", "@cloudflare/unenv-preset": "2.16.1", "blake3-wasm": "2.1.5", "esbuild": "0.28.1", "miniflare": "4.20260708.1", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", "workerd": "1.20260708.1" }, "optionalDependencies": { "fsevents": "2.3.3" }, "peerDependencies": { "@cloudflare/workers-types": "^5.20260708.1" }, "optionalPeers": ["@cloudflare/workers-types"], "bin": { "wrangler": "bin/wrangler.js", "wrangler2": "bin/wrangler.js", "cf-wrangler": "bin/cf-wrangler.js" } }, "sha512-xZeXKYi7hxQRF5anL+v77RkufJNpF9f3Eqeyqq2QBsETpLZgh0Agj0jJ6JPtkbgn6ukZdh8OK5egsGPWIditgg=="],
"@opentelemetry/api-logs/@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="],
"@opentelemetry/instrumentation/@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.220.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w=="],
"@opentelemetry/sdk-trace/@opentelemetry/core": ["@opentelemetry/core@2.11.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA=="],
@@ -6407,8 +6396,6 @@
"builder-util/js-yaml": ["js-yaml@4.3.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ=="],
"chrome-launcher/is-wsl": ["is-wsl@2.2.0", "", { "dependencies": { "is-docker": "^2.0.0" } }, "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww=="],
"chromium-bidi/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
"clean-css/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
@@ -6513,6 +6500,10 @@
"import-in-the-middle/es-module-lexer": ["es-module-lexer@2.3.2", "", {}, "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw=="],
"is-inside-container/is-docker": ["is-docker@3.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="],
"is-wsl/is-docker": ["is-docker@2.2.1", "", { "bin": { "is-docker": "cli.js" } }, "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="],
"js-beautify/glob": ["glob@10.5.0", "", { "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg=="],
"js-beautify/nopt": ["nopt@7.2.1", "", { "dependencies": { "abbrev": "^2.0.0" }, "bin": { "nopt": "bin/nopt.js" } }, "sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w=="],
@@ -6523,8 +6514,6 @@
"lighthouse/devtools-protocol": ["devtools-protocol@0.0.1663043", "", {}, "sha512-33aOY3ZnBP1dgZsshgaL+/XlsQleiFZgyUaDtdZkEa1nbZhVY1MoDeWjk+wxg25fU924l1ZJfoGNmjjeA/5s1w=="],
"lighthouse/open": ["open@8.4.2", "", { "dependencies": { "define-lazy-prop": "^2.0.0", "is-docker": "^2.1.1", "is-wsl": "^2.2.0" } }, "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ=="],
"lighthouse/ws": ["ws@7.5.13", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": "^5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-rsKI6xDBFVf4r/x8XyChGK04QR/XHroxs/jUcoWvtEZM8TPU/X/uIY9B1CsSzYws9ZJb/6bbBu7dPhFW00CAoA=="],
"md-to-react-email/marked": ["marked@7.0.4", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-t8eP0dXRJMtMvBojtkcsA7n48BkauktUKzfkPSCq85ZMTJ0v76Rke4DYz01omYpPTUh4p/f7HePgRo3ebG8+QQ=="],
@@ -6627,8 +6616,6 @@
"sst/jose": ["jose@5.2.3", "", {}, "sha512-KUXdbctm1uHVL8BYhnyHkgp3zDX5KW8ZhAKVFEfUbU2P8Alpzjb+48hHvjOdQIyPshoblhzsuqOwEEAbtHVirA=="],
"storybook/open": ["open@10.2.0", "", { "dependencies": { "default-browser": "^5.2.1", "define-lazy-prop": "^3.0.0", "is-inside-container": "^1.0.0", "wsl-utils": "^0.1.0" } }, "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA=="],
"storybook-solidjs-vite/semver": ["semver@7.8.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg=="],
"storybook-solidjs-vite/vite": ["vite@7.1.11", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-uzcxnSDVjAopEUjljkWh8EIrg6tlzrjFUfMcR1EVsRDGwf/ccef0qQPRyOrROwhrTDaApueq+ja+KLPlzR/zdg=="],
@@ -6715,6 +6702,10 @@
"write-file-atomic/signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="],
"wsl-utils/is-wsl": ["is-wsl@3.1.1", "", { "dependencies": { "is-inside-container": "^1.0.0" } }, "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw=="],
"wsl-utils/powershell-utils": ["powershell-utils@0.1.0", "", {}, "sha512-dM0jVuXJPsDN6DvRpea484tCUaMiXWjuCn++HGTqUWzGDjv5tZkEZldAJ/UMlqRYGFrD/etByo4/xOuC/snX2A=="],
"yaml-language-server/prettier": ["prettier@3.9.6", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g=="],
"yaml-language-server/request-light": ["request-light@0.5.8", "", {}, "sha512-3Zjgh+8b5fhRJBQZoy+zbVKpAQGLyka0MPgW3zruTF4dFFJ8Fqcfu9YsAvi/rvdcaTeWG3MkbZv4WKxAn/84Lg=="],
@@ -7349,8 +7340,6 @@
"builder-util/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="],
"chrome-launcher/is-wsl/is-docker": ["is-docker@2.2.1", "", { "bin": { "is-docker": "cli.js" } }, "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="],
"cliui/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
"cliui/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
@@ -7411,12 +7400,6 @@
"lazystream/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="],
"lighthouse/open/define-lazy-prop": ["define-lazy-prop@2.0.0", "", {}, "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og=="],
"lighthouse/open/is-docker": ["is-docker@2.2.1", "", { "bin": { "is-docker": "cli.js" } }, "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="],
"lighthouse/open/is-wsl": ["is-wsl@2.2.0", "", { "dependencies": { "is-docker": "^2.0.0" } }, "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww=="],
"miniflare/sharp/@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.0.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ=="],
"miniflare/sharp/@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.0.4" }, "os": "darwin", "cpu": "x64" }, "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q=="],
@@ -7687,6 +7670,10 @@
"@astrojs/starlight/@astrojs/mdx/@astrojs/markdown-remark/shiki": ["shiki@3.23.0", "", { "dependencies": { "@shikijs/core": "3.23.0", "@shikijs/engine-javascript": "3.23.0", "@shikijs/engine-oniguruma": "3.23.0", "@shikijs/langs": "3.23.0", "@shikijs/themes": "3.23.0", "@shikijs/types": "3.23.0", "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-55Dj73uq9ZXL5zyeRPzHQsK7Nbyt6Y10k5s7OjuFZGMhpp4r/rsLBH0o/0fstIzX1Lep9VxefWljK/SKCzygIA=="],
"@astrojs/starlight/astro/@astrojs/telemetry/is-docker": ["is-docker@3.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="],
"@astrojs/starlight/astro/@astrojs/telemetry/is-wsl": ["is-wsl@3.1.1", "", { "dependencies": { "is-inside-container": "^1.0.0" } }, "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw=="],
"@astrojs/starlight/astro/p-queue/p-timeout": ["p-timeout@6.1.4", "", {}, "sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg=="],
"@astrojs/starlight/astro/sharp/@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.0.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ=="],
@@ -8089,6 +8076,10 @@
"@opencode/web/@astrojs/cloudflare/wrangler/workerd": ["workerd@1.20260708.1", "", { "optionalDependencies": { "@cloudflare/workerd-darwin-64": "1.20260708.1", "@cloudflare/workerd-darwin-arm64": "1.20260708.1", "@cloudflare/workerd-linux-64": "1.20260708.1", "@cloudflare/workerd-linux-arm64": "1.20260708.1", "@cloudflare/workerd-windows-64": "1.20260708.1" }, "bin": { "workerd": "bin/workerd" } }, "sha512-WAK+Kt/VVCSldH2qSr8lx46XCJ4Q+bdlHNaFqUtOHthBEIB8C1N8HVW+VOLrxDoTCk0NGNv0zajnBeQK4JOB9w=="],
"@opencode/web/astro/@astrojs/telemetry/is-docker": ["is-docker@3.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="],
"@opencode/web/astro/@astrojs/telemetry/is-wsl": ["is-wsl@3.1.1", "", { "dependencies": { "is-inside-container": "^1.0.0" } }, "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw=="],
"@opencode/web/astro/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="],
"@opencode/web/astro/p-queue/p-timeout": ["p-timeout@6.1.4", "", {}, "sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg=="],
@@ -8229,6 +8220,10 @@
"archiver-utils/glob/path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
"astro-expressive-code/astro/@astrojs/telemetry/is-docker": ["is-docker@3.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="],
"astro-expressive-code/astro/@astrojs/telemetry/is-wsl": ["is-wsl@3.1.1", "", { "dependencies": { "is-inside-container": "^1.0.0" } }, "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw=="],
"astro-expressive-code/astro/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="],
"astro-expressive-code/astro/p-queue/p-timeout": ["p-timeout@6.1.4", "", {}, "sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg=="],
@@ -8331,6 +8326,10 @@
"temp/rimraf/glob/minimatch": ["minimatch@3.1.5", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w=="],
"toolbeam-docs-theme/astro/@astrojs/telemetry/is-docker": ["is-docker@3.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="],
"toolbeam-docs-theme/astro/@astrojs/telemetry/is-wsl": ["is-wsl@3.1.1", "", { "dependencies": { "is-inside-container": "^1.0.0" } }, "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw=="],
"toolbeam-docs-theme/astro/js-yaml/argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="],
"toolbeam-docs-theme/astro/p-queue/p-timeout": ["p-timeout@6.1.4", "", {}, "sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg=="],
+4 -4
View File
@@ -1,8 +1,8 @@
{
"nodeModules": {
"x86_64-linux": "sha256-WBGj4wU5PQs+0zbDbScFhJs0GL8qWnswIsJb/Kee5Qc=",
"aarch64-linux": "sha256-Z8hkyNMdqCpuv24q50QhO2RECINvENjBhLDY5FKcnFc=",
"aarch64-darwin": "sha256-HZHb9HcXMOc7bKR7eADQSyfWxX37GAcoRi/EMSpRiP4=",
"x86_64-darwin": "sha256-CcWCOAcW4t9ADs48fMhumSbDcQWdk91LpkG6fe/gWlk="
"x86_64-linux": "sha256-9gJjhes2ueYckAgdeGlPwZcaIDdwB3ZnqK/XHHXhWNs=",
"aarch64-linux": "sha256-Sy5YXYM9tKevIITdV++bP35SJNaFCQVKwlNJRbWsD1Q=",
"aarch64-darwin": "sha256-wiXHjKXm2VIFvalwITpSiRHaFZEWc8UJIqjQyc/0f0s=",
"x86_64-darwin": "sha256-r/mnhdNbnPIJOY3qvtuY6GQ7ed1Nauq65X+8uERhdP8="
}
}
+6 -5
View File
@@ -2,7 +2,7 @@
"$schema": "https://json.schemastore.org/package.json",
"name": "opencode",
"description": "AI-powered development tool",
"version": "2.0.14",
"version": "2.0.18",
"private": true,
"type": "module",
"packageManager": "bun@1.4.2",
@@ -52,9 +52,9 @@
"@octokit/rest": "22.0.0",
"@hono/standard-validator": "0.2.0",
"@hono/zod-validator": "0.4.2",
"@opentui/core": "0.5.10",
"@opentui/keymap": "0.5.10",
"@opentui/solid": "0.5.10",
"@opentui/core": "0.5.12",
"@opentui/keymap": "0.5.12",
"@opentui/solid": "0.5.12",
"@tanstack/solid-virtual": "3.13.37",
"@shikijs/stream": "4.4.3",
"@standard-schema/spec": "1.1.0",
@@ -162,6 +162,7 @@
"@types/node": "catalog:",
"bun-types": "1.4.2",
"effect": "catalog:",
"open": "11.0.4",
"solid-js": "catalog:"
},
"patchedDependencies": {
@@ -173,7 +174,7 @@
"solid-js@1.9.15": "patches/solid-js@1.9.15.patch",
"@ai-sdk/mistral@3.0.51": "patches/@ai-sdk%2Fmistral@3.0.51.patch",
"gcp-metadata@8.1.2": "patches/gcp-metadata@8.1.2.patch",
"pacote@21.5.0": "patches/pacote@21.5.0.patch",
"pacote@21.5.1": "patches/pacote@21.5.1.patch",
"@ai-sdk/google@3.0.73": "patches/@ai-sdk%2Fgoogle@3.0.73.patch",
"@pierre/trees@1.0.0-beta.4": "patches/@pierre%2Ftrees@1.0.0-beta.4.patch",
"@tanstack/virtual-core@3.17.8": "patches/@tanstack%2Fvirtual-core@3.17.8.patch",
+13 -6
View File
@@ -10,11 +10,11 @@
## Conventions
Per-type constructors live on the type, not as top-level re-exports. Use `Message.system(...)`, `Message.user(...)`, `Message.assistant(...)`, `Message.tool(...)`, `Message.media(...)`, `LanguageModel.make(...)`, `ToolDefinition.make(...)`, `ToolCallPart.make(...)`, `ToolResultPart.make(...)`, `ToolChoice.make(...)`, `ToolChoice.named(...)`, `SystemPart.make(...)`, and `GenerationOptions.make(...)` directly. The top-level `LLM` namespace is reserved for request-shaped call APIs: `LLM.request`, `LLM.generate`, `LLM.stream`, and `LLM.generateObject`. Use `LLMRequest.update(...)` when deriving canonical request data; do not add a duplicate `LLM.updateRequest(...)` path. Two ways to construct the same thing is one too many.
Per-type constructors live on the type, not as top-level re-exports. Use `Message.system(...)`, `Message.user(...)`, `Message.assistant(...)`, `Message.tool(...)`, `Message.media(...)`, `LanguageModel.make(...)`, `ToolDefinition.make(...)`, `ToolCallPart.make(...)`, `ToolResultPart.make(...)`, `ToolChoice.make(...)`, `ToolChoice.named(...)`, `SystemPart.make(...)`, and `GenerationOptions.make(...)` directly. The top-level `LLM` namespace is reserved for request-shaped call APIs: `LLM.request`, `LLM.generate`, `LLM.stream`, and `LLM.generateObject`. `LLM.generate`/`LLM.stream` and Promise `ai.llm.generate`/`ai.llm.stream` accept ergonomic input or an `LLMRequest`; both paths use the same canonical request. Core still builds, logs, replays, and updates that durable `LLMRequest` boundary. Use `LLMRequest.update(...)` when deriving canonical request data; do not add a duplicate `LLM.updateRequest(...)` path.
Modality namespaces mirror `LLM` exactly: `Image.request`, `Image.generate`, `Image.stream` (later `Video`, `Speech`, `Transcription`). Common request fields (`images`, `mask`, `n`, `size`, `aspectRatio`, `seed`, `format`) lower natively or fail with a typed `AIError`; provider-native controls always live under `providerOptions`, never under a modality-specific `options` key.
Modality namespaces mirror `LLM` exactly: `Image.request`, `Image.generate`, `Image.stream`, and the same for `Video`, `Speech`, and `Transcription`. Common request fields (`images`, `mask`, `n`, `size`, `aspectRatio`, `seed`, `format`) lower natively or fail with a typed `AIError`; provider-native controls always live under `providerOptions`, never under a modality-specific `options` key.
Media payloads are always `Media.Asset` (`src/media.ts`). Construct them with `Media.bytes`, `Media.base64`, `Media.url`, `Media.ref`, `Media.fromDataUrl`, or `Media.file`; never introduce a parallel `data: string | Uint8Array` shape. `MediaPart.media`, `ImageRequest.images`/`mask`, `ImageResponse.images`, and the `media` `LLMEvent` all share it. Protocols branch on `asset.source.type` and `asset.kind` and use `ProviderShared.inlineMedia` / `requireInlineMedia` / `mediaUrl` / `MediaInput.refID` rather than re-deriving base64 or URL handling.
Media payloads are always `Media.Asset` (`src/media.ts`). Construct them with `Media.bytes`, `Media.base64`, `Media.url`, `Media.ref`, `Media.fromDataUrl`, or `Media.file`; never introduce a parallel `data: string | Uint8Array` shape. `MediaPart.media`, `ImageRequest.images`/`mask`, `ImageResponse.images`, and the `media` `LLMEvent` all share it. Protocols branch on `asset.source.type` and `asset.kind` and use `ProviderShared.requireInlineMedia` / `inlineRequired` / `mediaUrl` / `mediaReference` and `MediaInput.inlineBytes` / `refID` rather than re-deriving base64 or URL handling.
`schema/messages.ts → media.ts → route/executor-service.ts` is an accepted runtime dependency from the schema layer on the executor service tag: `Media.Asset.bytes()` must be able to download `url` sources, and the tag lives in that leaf module precisely so the schema barrel never imports the executor implementation (which imports the schema barrel back). Do not move the tag into `route/executor.ts` or import `route/executor.ts` from `src/schema/*` or `src/media.ts`.
@@ -55,7 +55,7 @@ const request = LLM.request({
prompt: "Say hello.",
})
const response = yield * LLMClient.generate(request)
const response = yield * LLMClient.generate(request) // inside Effect.gen
```
`LLM.request(...)` builds an `LLMRequest`. `LLMClient.generate(...)` reads the executable route carried by `request.model.route`, builds the provider-native body, asks the route's transport for a real `HttpClientRequest.HttpClientRequest`, sends it through `RequestExecutor.Service`, parses the provider stream into common `LLMEvent`s, and finally returns an `LLMResponse`.
@@ -96,7 +96,13 @@ When a provider supports multiple physical transports, selection remains executi
### Media Routes
Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute.make(...)` (`src/route/media.ts`) composes a `MediaProtocol` kind with `Endpoint` and `Auth` and owns the transport plumbing: `http` option merging, URL/query rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol. `MediaProtocol.inline` (`src/route/media-protocol.ts`) is `body.from(request)` plus `response.decode(response, context)`; use `MediaProtocol.decodeJson` / `text` / `bytes` so decode failures retain the raw body and HTTP context. `Generation` (`src/generation.ts`) is the provider-neutral handle for a queued generation over a `GenerationRoute` (`status`, `result`, `cancel`, `pollHint`); the first video route implements it. Image protocol files follow the same section order as LLM protocols and declare unsupported common fields once through `MediaInput.rejectUnsupported`.
Media does not fit the SSE-frames-to-event-state-machine LLM route. `MediaRoute.inline(...)` / `queued(...)` / `stream(...)` (`src/route/media.ts`) compose a `MediaProtocol` kind with `Endpoint` and `Auth` and own the transport plumbing: `http` option merging, URL/query rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol. `MediaProtocol.inline` (`src/route/media-protocol.ts`) is `body.from(request)` plus `response.decode(response, context)`; each protocol declares `const route = MediaProtocol.identity({ id, name, provider })` once and decodes through `route.decodeJson` / `route.text` / `route.decodeStarted` so decode failures retain the raw body and HTTP context, raising `route.unsupported(operation, message)` for requests it cannot lower, and passes `route` as the first argument to `MediaProtocol.inline` / `queued` / `stream`. `Generation` (`src/generation.ts`) is the provider-neutral handle for a queued generation over a `GenerationRoute` (`status`, `result`, `cancel`). Image protocol files follow the same section order as LLM protocols and declare unsupported common fields once through the protocol's `unsupported` list.
`MediaProtocol.queued` is the submit-then-poll kind every video route uses: `start` (body + decode into `{ token, snapshot }`), `status`, `result`, and optional `cancel` (with `activeOnly` when the provider's cancel endpoint deletes finished work, as Runway's does: the route refreshes status first and skips terminal generations), each addressed by a route-owned `token` whose `Schema.Codec` makes it serializable. `MediaRoute.inline` and `MediaRoute.queued` compose the two kinds with `Endpoint` and `Auth`; the queued route decodes the token once at the boundary (`start` output or `resume` input) and closes over it in a token-free `GenerationRoute` (`status`/`result`/`cancel` are plain Effects), so `Generation` never sees the token's shape and only carries the encoded JSON for persistence. Polls reuse the route's auth and deployment headers plus the request's `http` overlay after `start`, and resolve relative paths against the route base URL (provider-issued absolute URLs such as fal's `status_url` pass through). `result` is always its own GET even when the provider returns output inside the status document, so `Generation.await` behaves the same after `start` and after `resume`. `PollContext.auth` carries only what `Auth` added or changed so protocols can hand download credentials to output assets as transient `Media.Asset.headers` (Veo) — never part of `source` or JSON. Status strings map through a per-protocol `STATUS` table via `MediaProtocol.status`; terminal generations without output fail through `output.ended` / `output.contentPolicy` with the provider document on `reason.body`. `GenerationAwaitOptions` (`AwaitOptions` in `src/generation.ts`, `{ poll?: Poll }`) is the one options type for `await`, `events`, `Video.generate`, and `Video.stream`.
`MediaProtocol.stream` is the incremental kind every speech route uses, with the same discipline as LLM protocols. `MediaRoute.stream` submits the caller's request as `MediaProtocol.Addressed<Request>` (`{ ...request, mode }`, `mode: "generate" | "stream"`), so one provider stays one protocol: `body.from`, the endpoint path, and `frames` read `request.mode` to pick the body, path, and framing. `frames(bytes, context)` returns frames — `Framing.sse`, `Framing.lines`, `Framing.document` (a single-document response shaped like a streamed record), or the raw `bytes` for chunked audio. `initial()` is fresh per-response parser state; `step` folds each frame into it and emits modality events; `finish(state, context)` runs once after the last frame with the request, body, and observed `http` (header-only usage lives there) and emits exactly one terminal event or fails with `route.incomplete()`. Keep parser state to real accumulators and derive anything the request or body determines in `finish`. `generate` runs the same stream and folds it with the modality's `collect`. Request-derived URL parameters go on the body's `query` (array values repeat the parameter), applied before route and caller `http.query`. Decode frames with `route.decodeFrame` and raise stream-time failures with `route.frameError` (the frame stays on `reason.body`); protocols never thread HTTP context, because the route fills `reason.http` on stream errors that lack it. Speech protocols share `protocols/utils/speech-stream.ts` for deltas, timestamps, voice ids, PCM and container descriptions, and the terminal asset.
Every modality route is the inline | stream | queued union (transcription uses all three: OpenAI and Gemini stream, Deepgram is inline, AssemblyAI is queued), every client is `MediaClient.make(Service, { modality, responseEvents })` (`src/media-client.ts`), which dispatches on the route's `kind`, and every model composes through `composeRoute`. fal queue protocols come from `protocols/utils/fal-queue.ts`, bodies are `json`, `multipart`, or `binary` (a raw upload), and a queued protocol that must upload media before submitting implements `start.prepare` (`MediaProtocol.Prepare`; AssemblyAI `/v2/upload`).
### URL Construction
@@ -106,7 +112,7 @@ For providers where the URL is derived from typed inputs (Azure resource name, B
### Provider Facades
Provider-facing APIs are configured facades over route values. Endpoint/auth/resource/API-version setup happens before model selection, and model selectors accept only a model or deployment id. Media models use per-modality selectors on the same facade (`openai.image(id)`, later `.video` / `.speech` / `.transcription`) that mirror `openai.responses(id)`; the one-word overlap with the request namespace is accepted over a second construction path:
Provider-facing APIs are configured facades over route values. Endpoint/auth/resource/API-version setup happens before model selection, and model selectors accept only a model or deployment id. Media models use per-modality selectors on the same facade (`openai.image(id)`, `.speech(id)`, `.transcription(id)`, `google.video(id)`) that mirror `openai.responses(id)`; the one-word overlap with the request namespace is accepted over a second construction path:
```ts
const openai = OpenAI.configure({ apiKey, baseURL })
@@ -269,6 +275,7 @@ Use this order for every protocol module:
### Rules
- Keep protocol files focused on the protocol. Move provider-specific projection, signing, media normalization, or other bulky transformations into `src/protocols/utils/*`.
- Send `tool.inputSchema` as given. `prepareRequest` applies the tool schema rules (`ToolSchemaProjection.tools`) once per request, including tools in namespaces. A protocol whose API needs a model family's rules for every model declares `sanitizer` instead of transforming schemas itself.
- Use `Effect.fn("Provider.fromRequest")` for request body construction entrypoints. Use `Effect.fn(...)` for event handlers that yield effects; keep purely synchronous handlers as plain functions returning a `StepResult` that the dispatcher lifts via `Effect.succeed(...)`.
- Parser state owns terminal information. The state machine records finish reason, usage, and pending tool calls; emit one terminal `finish` event (or `provider-error`) for each completed response. If a provider splits reason and usage across events, merge them in parser state before flushing.
- Emit exactly one terminal `finish` event for a completed response, normally after a matching `step-finish`. Use `stream.terminal` to stop reading when the provider has a completion sentinel; use `stream.onHalt` when the final event must be flushed after the framed stream ends.
+442 -100
View File
@@ -1,40 +1,38 @@
# @opencode/ai
Schema-first language model and image-generation APIs built with Effect.
Schema-first APIs for text, images, video, speech, and transcription, built with Effect.
```ts
import { Effect, Layer } from "effect"
import { LLM, LLMClient } from "@opencode/ai"
import { RequestExecutor } from "@opencode/ai/route"
import { Effect } from "effect"
import { AIClient, LLM } from "@opencode/ai"
import { OpenAI } from "@opencode/ai/providers"
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
const request = LLM.request({
model: openai.responses("gpt-4o-mini"), // `.chat(...)` selects the Chat Completions API instead
system: "You are concise.",
prompt: "Say hello in one short sentence.",
generation: { maxTokens: 40 },
})
const program = Effect.gen(function* () {
const response = yield* LLMClient.generate(request)
const response = yield* LLM.generate({
model: openai.responses("gpt-4o-mini"), // `.chat(...)` selects the Chat Completions API instead
system: "You are concise.",
prompt: "Say hello in one short sentence.",
generation: { maxTokens: 40 },
})
console.log(response.text)
})
const llmLayer = LLMClient.layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
await Effect.runPromise(program.pipe(Effect.provide(llmLayer)))
// Every modality client plus the HTTP request executor; `AIClient.layerWith(executor)` swaps the executor.
await Effect.runPromise(program.pipe(Effect.provide(AIClient.layer)))
```
Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
Run `LLM.stream(...)` instead of `generate` when you want incremental `LLMEvent`s. Both accept input or a prebuilt
`LLM.request(...)`. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses,
Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
The same configured facade names image models. `Image.request` resolves the provider's image route from the ref and
returns `Media.Asset`s with lazily decoded bytes:
The same configured facade names image, video, speech, and transcription models. `Image.generate` resolves the
provider's image route from the model and returns `Media.Asset`s with lazily decoded bytes:
```ts
import { NodeFileSystem } from "@effect/platform-node"
import { Image, ImageClient, Media } from "@opencode/ai"
import { Image, Media } from "@opencode/ai"
const image = Effect.gen(function* () {
const response = yield* Image.generate({
@@ -46,21 +44,38 @@ const image = Effect.gen(function* () {
yield* Media.write(response.image, "./garden.png")
})
// `asset.bytes()` / `Media.write` also need the executor, so merge it into the environment instead of hiding it.
const imageLayer = ImageClient.layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer))
await Effect.runPromise(image.pipe(Effect.provide(imageLayer), Effect.provide(NodeFileSystem.layer)))
// `Media.file` / `Media.write` use the Effect `FileSystem` service; provide your platform's layer.
await Effect.runPromise(image.pipe(Effect.provide(AIClient.layer), Effect.provide(NodeFileSystem.layer)))
```
Prefer promises? `@opencode/ai/promise` exposes the same LLM and image APIs over one managed runtime:
Advanced: each client also has its own `layer`, which requires `RequestExecutor.Service`. Compose client layers with
`Layer.provideMerge`, not `Layer.provide`: `asset.bytes()`, `Media.write`, and Gemini's `media` output parts need the
executor too, and hiding it fails type-checking with `RequestExecutorService` left in the requirements.
To share a policy such as logging across every client, wrap the executor once with `RequestExecutor.middleware`:
```ts
import { RequestExecutor } from "@opencode/ai/route"
const logged = RequestExecutor.middleware((request, next) =>
Effect.log(`${request.method} ${request.url}`).pipe(Effect.andThen(next(request))),
)
const everything = AIClient.layerWith(logged) // or AI.make({ layer: logged })
```
Prefer promises? `@opencode/ai/promise` exposes the same LLM and media APIs over one managed runtime, plus asset
helpers; `ai.file` and `ai.write` load `node:fs/promises` on first use, so no Effect `FileSystem` is needed:
```ts
import { AI } from "@opencode/ai/promise"
const ai = AI.make()
const text = await ai.llm.generate({ model: openai.responses("gpt-4o-mini"), prompt: "Say hello." })
const input = { model: openai.responses("gpt-4o-mini"), prompt: "Say hello." }
const text = await ai.llm.generate(input)
const generated = await ai.image.generate({ model: openai.image("gpt-image-2"), prompt: "A lighthouse" })
for await (const event of ai.llm.stream({ model: openai.responses("gpt-4o-mini"), prompt: "Stream hello." })) {
await ai.write(generated.image, "./lighthouse.png") // also ai.file(path), ai.bytes(asset), ai.base64(asset), ai.materialize(asset)
for await (const event of ai.llm.stream(ai.llm.request(input))) {
// LLMEvent
}
await ai.dispose()
@@ -322,10 +337,9 @@ and `moonshot/responses`; each exports `model(modelID, settings)`.
MiniMax defaults to its Messages API and reads `MINIMAX_API_KEY` when `apiKey` is omitted:
```ts
import { Effect, Layer } from "effect"
import { LLM, LLMClient } from "@opencode/ai"
import { Effect } from "effect"
import { AIClient, LLM } from "@opencode/ai"
import { MiniMax } from "@opencode/ai/providers"
import { RequestExecutor } from "@opencode/ai/route"
const minimax = MiniMax.configure({ apiKey: process.env.MINIMAX_API_KEY })
const request = LLM.request({
@@ -335,8 +349,7 @@ const request = LLM.request({
generation: { maxTokens: 1536 },
})
const layer = LLMClient.layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
const response = await Effect.runPromise(LLMClient.generate(request).pipe(Effect.provide(layer)))
const response = await Effect.runPromise(LLM.generate(request).pipe(Effect.provide(AIClient.layer)))
console.log(response.text)
```
@@ -405,14 +418,14 @@ Use `Image.generate` for one-off generation or editing:
import { Image, Media } from "@opencode/ai"
const generation = Image.generate({
model: meta("muse-image-1.0"),
model: meta.image("muse-image-1.0"),
prompt: "A flat black square on a white background.",
n: 1,
providerOptions: { reasoningStrength: "low" },
})
const edit = Image.generate({
model: meta("muse-image-1.0"),
model: meta.image("muse-image-1.0"),
prompt: "Make the square purple.",
images: [Media.bytes(imageBytes, "image/webp")],
format: "png",
@@ -459,6 +472,25 @@ const program = Effect.gen(function* () {
})
```
Common fields are portable in shape, not in support. Unsupported fields fail with a typed `AIError` before any network
call rather than being dropped, so check this table before swapping only the `model`:
| Provider | `n` | `size` | `aspectRatio` | `seed` | `format` | `images` | `mask` |
| --------------------- | --- | --------- | ------------- | ------ | -------- | -------------------------------- | ------------------- |
| OpenAI | ✓¹ | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ |
| Google (Gemini) | 1 | ✗ | ✓ | ✓ | ✗ | ✓ (no public URLs) | ✗ |
| xAI | ✓ | ✗ | ✓ | ✗ | ✗ | ✓ | ✗ |
| Z.ai | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ |
| Meta | ✓ | ✓ (hint) | ✗ | ✗ | ✓ | ✓ | ✗ |
| Black Forest Labs | 1 | per model | per model | ✓ | ✓ | per model (1–8) | `flux-pro-1.0-fill` |
| fal | ✓ | per model | per model | ✓ | ✓ | 1 (several on `/edit`, `/multi`) | ✓ |
| Replicate | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ (use `providerOptions`) | ✗ |
| Stability `image` | 1 | ✗ | ✓ | ✓ | ✓ | 1 (not on `core`) | ✗ |
| Stability `upscale()` | ✗ | ✗ | ✗ | ✓ | ✓ | exactly 1 (required) | ✗ |
✓ lowers natively; ✗ fails whenever the field is set (including `n: 1`); `1` means `n > 1` fails. ¹ `Image.stream` on OpenAI generates one image. fal
rejects `size` and `aspectRatio` together; which one a fal or BFL model takes depends on the model.
`Media.Asset` is the one asset type shared by image requests, image responses, LLM messages, and tool results.
`asset.source` is the serializable `Media.Source` (`bytes`, `base64`, `url`, or `ref`); `asset.bytes()`,
`asset.base64()`, and `asset.dataUrl()` decode or download lazily and cache; `asset.materialize()` pulls a `url`
@@ -468,9 +500,8 @@ asset into owned bytes before the provider URL expires. Construct assets with `M
Pass ordered image inputs to the same method for editing, composition, or image-conditioned generation:
```ts
const response =
yield *
Image.generate({
const composed = Effect.gen(function* () {
const response = yield* Image.generate({
model,
prompt: "Combine these product photos into one studio scene",
images: [
@@ -481,23 +512,25 @@ const response =
providerOptions,
http,
})
return response.images
})
```
`Media.ref(provider, id)` represents provider file handles such as OpenAI file IDs or Gemini Files URIs; routes
only forward refs that belong to their own provider. Raw strings are not accepted as image inputs, avoiding
ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a
non-empty array selects the provider's edit behavior without enforcing provider image-count limits locally. OpenAI
only forward refs that belong to their own provider (OpenAI, xAI, and Gemini images accept them). No shipped route
returns a ref yet, and `asset.bytes()` / `materialize()` on a ref fail by design. Raw strings are not accepted as
image inputs, avoiding ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a
non-empty array selects the provider's edit behavior (see the table above for routes that limit the count). OpenAI
uses multipart for byte/data-URL edits and its JSON reference body for URL or file-ID edits. The common `mask`
field selects inpainting; routes that cannot honor it fail with `UnsupportedOperation`:
```ts
yield *
Image.generate({
model: openai.image("gpt-image-2"),
prompt,
images: [Media.bytes(sourceBytes, "image/png")],
mask: Media.bytes(maskBytes, "image/png"),
})
const inpainted = Image.generate({
model: openai.image("gpt-image-2"),
prompt,
images: [Media.bytes(sourceBytes, "image/png")],
mask: Media.bytes(maskBytes, "image/png"),
})
```
On multipart requests, `http.body` can override option fields but not structural `model`, `prompt`, `image[]`,
@@ -508,31 +541,31 @@ not accept image inputs. These cases fail with a typed `AIError` before network
Provider-native image options belong to each request. Raw `http.body` fields have final precedence over them:
```ts
yield *
Image.generate({
model: openai.image("gpt-image-2"),
prompt,
providerOptions: { quality: "medium" },
http,
})
const medium = Image.generate({
model: openai.image("gpt-image-2"),
prompt,
providerOptions: { quality: "medium" },
http,
})
```
xAI image models use the same request API with xAI-native controls:
```ts
yield *
Image.generate({
model: XAI.configure({ apiKey })("any-model-id"),
prompt,
n: 2,
aspectRatio: "16:9",
providerOptions: {
resolution: "1k",
responseFormat: "b64_json",
future_option: true,
},
http,
})
import { XAI } from "@opencode/ai/providers"
const xai = Image.generate({
model: XAI.configure({ apiKey }).image("any-model-id"),
prompt,
n: 2,
aspectRatio: "16:9",
providerOptions: {
resolution: "1k",
responseFormat: "b64_json",
future_option: true,
},
http,
})
```
Google's current Gemini image models use the same direct API:
@@ -542,7 +575,7 @@ import { Google } from "@opencode/ai/providers"
const googleProgram = Effect.gen(function* () {
const response = yield* Image.generate({
model: Google.configure({ apiKey })("any-model-id"),
model: Google.configure({ apiKey }).image("any-model-id"),
prompt: "A robot tending a rooftop garden",
aspectRatio: "16:9",
seed: 42,
@@ -567,23 +600,88 @@ their mapped aliases, and `http.body` is the final deep overlay. The selected mo
Z.ai image models infer open Z.ai-native options from the selected model:
```ts
yield *
Image.generate({
model: ZAI.configure({ apiKey })("any-model-id"),
prompt,
providerOptions: {
quality: "hd",
userID: "user-123",
future_option: true,
},
http,
})
import { ZAI } from "@opencode/ai/providers"
const zai = Image.generate({
model: ZAI.configure({ apiKey }).image("any-model-id"),
prompt,
providerOptions: {
quality: "hd",
userID: "user-123",
future_option: true,
},
http,
})
```
Z.ai does not include trustworthy MIME metadata for output URLs, so generated images use
`application/octet-stream` until materialized. Output URLs expire after 30 days; call `asset.materialize()` and
persist the bytes promptly if they must remain available.
### Partial images
OpenAI's GPT image models stream previews. `Image.stream` sends `stream: true` with `partialImages` (0–3, default 2)
and emits `image-partial` events before each final `image`; `Image.generate` keeps the plain JSON request:
```ts
import { Stream } from "effect"
import { ImageEvent } from "@opencode/ai"
const previews = Image.stream({
model: openai.image("gpt-image-2"),
prompt: "A lighthouse at dusk",
providerOptions: { partialImages: 2 },
}).pipe(Stream.runForEach((event) => (ImageEvent.is.imagePartial(event) ? showPreview(event.image) : Effect.void)))
```
The provider may send fewer previews than requested when the final image is ready first.
### Queued image providers
Black Forest Labs, fal, Replicate, and Stability's creative upscaler are submit-then-poll routes. `Image.generate`
and `Image.stream` poll for you (pass `{ poll }` to tune the interval and timeout); `Image.start` returns a
`Generation` whose `token` is serializable JSON for `Image.resume` in another process:
```ts
import { BlackForestLabs, Stability } from "@opencode/ai/providers"
const bfl = BlackForestLabs.configure({ apiKey: process.env.BFL_API_KEY })
const submit = Effect.gen(function* () {
const generation = yield* Image.start({ model: bfl.image("flux-2-pro"), prompt, size: "1024x768" })
persist({ provider: "black-forest-labs", modelID: "flux-2-pro", token: generation.token })
})
const finish = Effect.gen(function* () {
const saved = load()
const resumed = yield* Image.resume(bfl.image(saved.modelID), saved.token)
return yield* resumed.await({ poll: { interval: "2 seconds" } })
})
```
The token carries no route identity, so persist the provider and model ID alongside it: `resume` needs the model.
- **Black Forest Labs** — results are downloaded before returning, because `result.sample` expires in 10 minutes.
- **Replicate** — inputs are model-defined, so only `prompt` lowers: sizing, count, seed, format, and files go in
`providerOptions` under the model's names, with files as `Media.Asset` (data URLs up to 256 KB, larger by URL).
Outputs are removed an hour after the prediction completes. `Prefer: wait=60` in `headers` or `http.headers` holds
the submission open so a fast prediction costs one result read.
- **Stability** — `stability.image(id)` generates inline; `stability.upscale()` is the creative upscaler, queued:
```ts
const stability = Stability.configure({ apiKey: process.env.STABILITY_API_KEY })
const upscaled = Effect.gen(function* () {
const small = yield* Media.file("./small.png")
return yield* Image.generate(
{ model: stability.upscale(), prompt: "A lighthouse", images: [small] },
{ poll: { interval: "5 seconds" } },
)
})
```
Imagen is not available: Google shut it down on the Gemini API, and Vertex discontinued the Imagen 4 models on
2026-06-30. `Google.image(...)` uses Gemini-native image models.
Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool:
```ts
@@ -600,20 +698,257 @@ const program = Effect.gen(function* () {
})
```
The hosted result is represented as a provider-executed tool call and tool result, and the generated image is also emitted as a first-class `media` `LLMEvent` (`response.message` then carries a `media` part). Gemini image-capable models emit the same `media` event for inline image output. Retaining `response.message` preserves the generated image for continuation on both routes.
The hosted result is represented as a provider-executed tool call and a tool result whose content carries the generated image as a file. Gemini image-capable models instead emit a first-class `media` `LLMEvent` for inline image output (`response.message` then carries a `media` part). Retaining `response.message` preserves the generated image for continuation on both routes.
## Video generation
Video mirrors `Image` with one difference: every provider is asynchronous, so the route is a submit-then-poll
`Generation`. Models come from `.video(...)` selectors on the `Google` (Veo), `XAI`, `Fal`, and `Runway` facades.
Common fields (`frames`, `references`, `video`, `durationSeconds`, `aspectRatio`, `resolution`, `audio`, `n`, `seed`,
`negativePrompt`) lower natively or fail with a typed `AIError` before any network call; provider-native controls live
under `providerOptions`, inferred from the selected model.
```ts
import { Video } from "@opencode/ai"
import { Google, Runway } from "@opencode/ai/providers"
const google = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY })
// Simple: submit and wait.
const program = Effect.gen(function* () {
const response = yield* Video.generate(
{
model: google.video("veo-3.1-generate-preview"),
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
aspectRatio: "16:9",
resolution: "1080p",
durationSeconds: 8,
providerOptions: { personGeneration: "allow_adult" },
},
{ poll: { interval: "10 seconds", timeout: "10 minutes" } },
)
// Veo serves files for two days behind the API key. The asset knows the deadline (`expiresAt`) and carries the
// download credentials only on the live instance (`asset.headers`), never in `source` or JSON: materialize
// before persisting, or the persisted URL cannot be fetched again.
return yield* response.video.materialize()
})
// Explicit control: keep the handle, persist the token, resume elsewhere.
const controlled = Effect.gen(function* () {
const generation = yield* Video.start({ model: google.video("veo-3.1-generate-preview"), prompt })
generation.id // provider operation / task / request id
generation.status // "queued" | "running" | "completed" | "failed" | "cancelled" | "expired"
generation.token // route-owned JSON: `{ operation }`, `{ requestID }`, `{ taskID }`, or fal's follow-up URLs
// The token carries no route identity: persist the provider and model ID alongside it, since `resume` needs the model.
const saved = JSON.stringify(generation.token)
const resumed = yield* Video.resume(google.video("veo-3.1-generate-preview"), JSON.parse(saved))
return yield* resumed.await({ poll: { interval: "10 seconds" } })
})
// Progress as a stream: generation-queued | generation-progress | video | finish.
const events = Video.stream({ model: Runway.configure({ apiKey }).video("gen4.5"), prompt }, { poll })
```
Status polls, result fetches, cancels, and asset downloads all run through the same request executor with the route's
auth. `Generation.await` and `Generation.events` fail with a
`Timeout` reason when `poll.timeout` (default 10 minutes) elapses. Failed,
cancelled, and expired generations fail typed with the provider's terminal document on `reason.body`; moderation
outcomes (Veo `raiMediaFilteredReasons`, xAI `respect_moderation`, Runway `SAFETY.*` codes) surface as `notices` when
a video is still returned and as a `ContentPolicy` reason when nothing is.
Provider notes:
- **Google Veo** takes inline bytes only (materialize `url` assets first); `frames.last` requires `frames.first`;
audio is always on, so `audio: false` fails typed; one video per request. Output URLs need the API key to
download, which the returned asset holds transiently (see above).
- **xAI** sends a `video` input to `/videos/edits`, or `/videos/extensions` with `providerOptions.mode: "extend"`.
`seed` and `negativePrompt` are not supported.
- **fal** endpoints are model-specific: `durationSeconds`, `references`, and `frames.last` fail typed and belong in
`providerOptions` under the model's own names (`duration: "8s"`, `end_image_url`, …). Auth is
`Authorization: Key <FAL_KEY>`.
- **Runway** expects pixel ratios in `aspectRatio` for most models (`"1280:720"`), pins `X-Runway-Version`, reports
`usage: { type: "credits" }`, and its output URLs expire after 24–48 hours.
The promise client exposes the same surface: `ai.video.start(...)` resolves to a handle with `await`, `events`,
`result`, `refresh`, `cancel`, and `token`; `ai.video.generate`, `ai.video.resume(model, token)`, and
`ai.video.stream` mirror the Effect API. The handle's `status` and `progress` are a snapshot from when it was
created; `refresh()` resolves to a new handle.
```ts
import { ai } from "@opencode/ai/promise"
const generation = await ai.video.start({ model, prompt })
for await (const event of generation.events({ poll: { interval: 10_000 } })) console.log(event.type)
const video = await generation.result({ signal })
await ai.write(video.video, "./kite.mp4")
```
## Speech generation
Speech (text-to-speech) is one request whose response is parsed incrementally, so every route supports both
`Speech.generate` (the whole file) and `Speech.stream` (audio chunks as they arrive). Models come from `.speech(...)`
selectors on the `OpenAI`, `Google` (Gemini TTS), `ElevenLabs`, `Cartesia`, and `Deepgram` facades. Common fields
(`voice`, `format`, `speed`, `language`, `instructions`, `timestamps`) lower natively or fail with a typed `AIError`
before any network call; provider-native controls live under `providerOptions`, inferred from the selected model.
```ts
import { Media, Speech, SpeechClient, SpeechEvent } from "@opencode/ai"
import { ElevenLabs, OpenAI } from "@opencode/ai/providers"
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
// The whole file, written to disk.
const program = Effect.gen(function* () {
const response = yield* Speech.generate({
model: openai.speech("gpt-4o-mini-tts"),
text: "Hello from OpenCode.",
voice: "coral",
format: "mp3",
instructions: "Warm and unhurried.",
})
response.audio // Media.Asset with bytes; headerless PCM carries info.encoding / sampleRate / channels
response.usage // undefined: OpenAI reports tokens only on SSE streams (Gemini: tokens; ElevenLabs: credits; Deepgram: characters)
yield* Media.write(response.audio, "hello.mp3")
})
// Chunks as they arrive: audio-delta* (interleaved with timestamps) then one finish carrying the assembled asset.
const events = Speech.stream({
model: ElevenLabs.configure({ apiKey }).speech("eleven_flash_v2_5"),
text: "Hello from OpenCode.",
voice: "JBFqnCBsd6RMkjVDRZzb",
format: "pcm",
timestamps: true,
}).pipe(
Stream.tap((event) => {
if (SpeechEvent.is.audioDelta(event)) return play(event.chunk)
if (SpeechEvent.is.timestamps(event)) return highlight(event.items) // { text, startSeconds, endSeconds }[]
return Effect.void
}),
)
```
`voice` is the provider's own identifier — a name on OpenAI and Gemini (`"coral"`, `"Kore"`), a voice id on
ElevenLabs and Cartesia. `{ id }` selects an OpenAI custom voice (`{ id: "voice_1234" }`) and means the same as the
plain string elsewhere. There is no cross-provider voice catalog. `format` is the container-level word (`mp3`, `wav`,
`pcm`, `opus`, `aac`, `flac`); sample rates and bitrates live under `providerOptions`, and a value the route cannot
produce fails as `UnsupportedOperation`. Streams buffer every chunk so `finish` can carry the whole clip.
Provider notes:
- **OpenAI** streams over SSE (`stream_format: "sse"`), which is also the only place it reports token usage; `tts-1`
and `tts-1-hd` do not support SSE and stream the raw audio body instead. `pcm` is 24 kHz 16-bit mono. `language`
and `timestamps` are not supported.
- **Gemini TTS** returns the provider's default output: WAV for Gemini 3.8 TTS `generate`, raw 16-bit PCM
(`audio/L16;codec=pcm;rate=24000`) otherwise. `pcm` is the only explicit `format` it accepts, and it fails typed on
Gemini 3.8 `generate`; the route never wraps PCM as WAV. Style is directed in the text, so `instructions` and
`speed` fail typed. Only `gemini-3.1-flash-tts-preview` and later support streaming. Two-speaker audio goes through
`providerOptions.speechConfig.multiSpeakerVoiceConfig`.
- **ElevenLabs** requires `voice` (the path voice id) and authenticates with `xi-api-key`. `format` maps to the
`output_format` query parameter (`mp3_44100_128`, `pcm_24000`, `wav_24000`, `opus_48000_64`);
`providerOptions.outputFormat` sets the exact string. WAV is only available from `generate`. `timestamps: true`
selects the `with-timestamps` endpoints and yields character-level alignment. `instructions` is not supported.
- **Cartesia** requires `voice` and pins `Cartesia-Version`. `generate` defaults to MP3 from `/tts/bytes`; streams
and `timestamps: true` (word-level) use `/tts/sse`, which only serves raw PCM. `providerOptions.sampleRate`,
`bitRate`, and `encoding` complete `output_format`. No usage is reported.
- **Deepgram** Aura's voice is the model id (`aura-2-thalia-en`), so `voice` and `language` fail typed. `format`
and `providerOptions` lower to query parameters (`encoding`, `container`, `sample_rate`, `bit_rate`); `pcm` is
`linear16` without a container. Auth is `Authorization: Token <DEEPGRAM_API_KEY>`.
The promise client mirrors the Effect API; `ai.speech.stream` is an `AsyncIterable`.
```ts
import { ai } from "@opencode/ai/promise"
const response = await ai.speech.generate({ model, text: "Hello from OpenCode.", voice: "coral" })
await ai.write(response.audio, "hello.mp3")
for await (const event of ai.speech.stream({ model, text: "Hello from OpenCode.", voice: "coral" })) {
if (event.type === "audio-delta") player.write(event.chunk)
}
```
## Transcription
Transcription (speech-to-text) is the one modality whose providers use every route kind: OpenAI and Gemini stream,
Deepgram answers inline, and AssemblyAI is queued. `Transcription.generate` and `Transcription.stream` work on all of
them; `Transcription.start` / `resume` return a `Generation` on queued routes and fail with `UnsupportedOperation`
elsewhere. Models come from `.transcription(...)` selectors on the `OpenAI`, `Google`, `Deepgram`, and `AssemblyAI`
facades. Common fields (`language`, `prompt`, `timestamps: "none" | "segment" | "word"`, `diarize`, `speakers`) lower
natively or fail with a typed `AIError` before any network call; a route may return more than asked.
```ts
import { Console, Effect, Stream } from "effect"
import { Media, Transcription, TranscriptionEvent } from "@opencode/ai"
import { AssemblyAI, Deepgram, OpenAI } from "@opencode/ai/providers"
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
const program = Effect.gen(function* () {
const audio = yield* Media.file("./call.mp3")
// Speaker-labelled segments; labels are provider-native strings ("A", "0", "spk:0").
const response = yield* Transcription.generate({
model: Deepgram.configure({ apiKey }).transcription("nova-3"),
audio,
diarize: true,
timestamps: "word",
})
response.text // "Hello from OpenCode."
response.segments // [{ text, startSeconds, endSeconds, speaker: "0" }]
response.words // [{ text, startSeconds, endSeconds, speaker, confidence }]
response.language // the provider's own value, lowercased ("en", "english", "en_us")
// Text deltas as the model transcribes, then one finish carrying the whole transcript.
yield* Transcription.stream({ model: openai.transcription("gpt-4o-mini-transcribe"), audio }).pipe(
Stream.tap((event) => (TranscriptionEvent.is.textDelta(event) ? Console.log(event.delta) : Effect.void)),
Stream.runDrain,
)
// Queued: persist the token with the provider and model ID (the token alone cannot pick the model), resume, and await.
const model = AssemblyAI.configure({ apiKey }).transcription("universal-3-5-pro")
const generation = yield* Transcription.start({ model, audio })
const resumed = yield* Transcription.resume(model, JSON.parse(JSON.stringify(generation.token)))
const transcript = yield* resumed.await({ poll: { interval: "3 seconds" } })
})
```
Inline routes emit only `finish` from `stream` (no faked deltas); queued routes emit `generation-queued` /
`generation-progress` before it.
Provider notes:
- **OpenAI** takes inline audio only; `diarize` needs `gpt-4o-transcribe-diarize`, timestamps need `whisper-1`, and `whisper-1` does not stream.
- **Gemini** needs a transcribe model (`gemini-3.5-transcribe`); `prompt` and `speakers` fail typed.
- **Deepgram** detects the language unless `language` is set; vocabulary goes in `providerOptions.keyterm`.
- **AssemblyAI** uploads inline audio before submitting and is the only route that accepts `speakers`.
The promise client mirrors the Effect API:
```ts
const audio = await ai.file("./call.mp3")
const text = (await ai.transcription.generate({ model, audio })).text
for await (const event of ai.transcription.stream({ model, audio })) if (event.type === "text-delta") write(event.delta)
const generation = await ai.transcription.start({ model: assemblyai, audio })
const transcript = await generation.await({ poll: { interval: 3_000 } })
```
## Public API
- **`LLM.request({...})`** — build a provider-neutral `LLMRequest`. Accepts ergonomic inputs (`system: string`, `prompt: string`) that normalize into the canonical Schema classes.
- **`LLM.generate` / `LLM.stream`** — re-exported from `LLMClient` for one-import use.
- **`LLM.generate` / `LLM.stream`** — run direct input or an `LLMRequest` through `LLMClient` for one-import use.
- **`Message.user(...)` / `Message.assistant(...)` / `Message.tool(...)`** — message constructors from the canonical schema model.
- **`LanguageModel.make(...)` / `ToolCallPart.make(...)` / `ToolResultPart.make(...)` / `ToolDefinition.make(...)`** — model and tool-related constructors from the canonical schema model.
- **`LLMEvent.is.*`** — typed guards (`is.textDelta`, `is.toolCall`, `is.finish`, …) for filtering streams.
- **`Image.request` / `Image.generate` / `Image.stream`** — generate images through a provider-neutral image request and response model.
- **`Image.request` / `generate` / `stream` / `start` / `resume`** — images over inline, streaming (partial previews), and queued routes through a provider-neutral request and response model.
- **`ImageClient`** — Effect service and layer for image execution, parallel to `LLMClient`.
- **`Media`** — the shared asset type (`Media.Asset`, `Media.Source`) and constructors used by messages, tool results, and media requests.
- **`Generation`** — provider-neutral handle for an in-flight media generation (`await`, `refresh`, `cancel`, `events`) used by queued media routes.
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm` and `image` as Promise / `AsyncIterable` APIs.
- **`Video.request` / `generate` / `stream` / `start` / `resume`** — queued video generation through a provider-neutral request; `VideoClient` is its Effect service and layer.
- **`Speech.request` / `Speech.generate` / `Speech.stream`** — text-to-speech through a provider-neutral request; `SpeechClient` is its Effect service and layer.
- **`Transcription.request` / `generate` / `stream` / `start` / `resume`** — speech-to-text over inline, streaming, and queued routes; `TranscriptionClient` is its Effect service and layer.
- **`AIClient.layer` / `AIClient.layerWith(executor)`** — every modality client plus the request executor in one layer.
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm`, `image`, `video`, `speech`, and `transcription` as Promise / `AsyncIterable` APIs, plus `file`, `write`, `bytes`, `base64`, and `materialize` for assets.
## Testing
@@ -676,11 +1011,13 @@ This is different from prompt caching, server-side history storage, or truncatio
Prefer this operation, where supported, when the application owns compaction policy and durable context updates.
```ts
const result = yield * LLMClient.compact(request)
const next = LLMRequest.update(request, {
messages: result.replacement,
const compacted = Effect.gen(function* () {
const result = yield* LLMClient.compact(request)
const next = LLMRequest.update(request, {
messages: result.replacement,
})
return yield* LLMClient.generate(next)
})
const response = yield * LLMClient.generate(next)
```
`replacement` replaces the complete input window. Do not append it to the original transcript or extract only the encrypted item: the provider may retain additional messages in its output. Retained user and assistant messages remain ordinary messages with typed text, media, or reasoning parts, in their original order. Provider-specific message IDs, status, and phase use `providerMetadata`, not a raw output array hidden in an assistant message. Unsupported returned item types fail explicitly.
@@ -696,16 +1033,16 @@ The input must still fit the model's context window. Explicit compaction is not
OpenAI Responses also exposes a separate, explicitly selected mechanism:
```ts
const result =
yield *
LLMClient.compact(request, {
const checkpoint = Effect.gen(function* () {
const result = yield* LLMClient.compact(request, {
mechanism: "trigger",
webSocket, // Optional: without it, the request uses HTTP/SSE.
})
result.checkpoint // Successful encrypted CompactionPart.
result.responseID
result.usage
result.checkpoint // Successful encrypted CompactionPart.
result.responseID
result.usage
})
```
This appends a native `compaction_trigger` control item to the full input and sends a normal Responses request, with tools and instructions retained, `stream: true`, `store: false`, and parallel tool calls enabled. It removes normal-answer text/output-format controls, forced tool choices, output-token/tool-call limits, and automatic `context_management`. Body overlays cannot replace `input` or supply `previous_response_id`/`conversation`; the complete canonical history is required for safe stateless replay. Request metadata, auth, headers, query parameters, service tier, and supported prompt-cache settings are preserved.
@@ -719,9 +1056,11 @@ The supplied WebSocket executor can reuse a compatible append baseline for the c
Trigger support is separate from endpoint support. Only the OpenAI Responses route advertises it; Azure, xAI, Chat, and compatible Responses routes do not inherit it. Untyped calls still fail before sending: missing route capabilities return `UnsupportedOperation`, while unknown mechanism names and invalid inputs return `InvalidRequest`. Dynamic callers must narrow for the selected mechanism:
```ts
if (LLMClient.canCompact(request, { mechanism: "trigger" })) {
const result = yield * LLMClient.compact(request, { mechanism: "trigger" })
}
const narrowed = Effect.gen(function* () {
if (LLMClient.canCompact(request, { mechanism: "trigger" })) {
const result = yield* LLMClient.compact(request, { mechanism: "trigger" })
}
})
```
This capability describes protocol implementation, **not universal availability on OpenAI API deployments**. The host application owns subscription/deployment eligibility, OAuth, endpoint selection, and deployment-specific headers. Local protocol/socket tests do not establish live provider support.
@@ -730,9 +1069,10 @@ This capability describes protocol implementation, **not universal availability
`providerOptions.contextManagement` lets the provider decide when to compact during an ordinary `generate` or `stream` call. This is an advanced option for callers that own persistence and recovery: persist the complete assistant message, including its checkpoint, before continuing. Enabling the option does not provide durable checkpoint storage, interruption recovery, or model-switch policy. Keep the prior context until a successful checkpoint has been persisted.
Inside an `Effect.gen`, enable OpenAI compaction with typed provider options:
Enable OpenAI compaction with typed provider options:
```ts
import { Effect } from "effect"
import { LLM, LLMClient, LLMRequest, Message } from "@opencode/ai"
import { OpenAI } from "@opencode/ai/providers"
@@ -743,9 +1083,11 @@ const request = LLM.request({
contextManagement: [{ type: "compaction", compactThreshold: 200_000 }],
},
})
const response = yield * LLMClient.generate(request)
const next = LLMRequest.update(request, {
messages: [...request.messages, response.message, Message.user("Continue")],
const continued = Effect.gen(function* () {
const response = yield* LLMClient.generate(request)
return LLMRequest.update(request, {
messages: [...request.messages, response.message, Message.user("Continue")],
})
})
```
@@ -973,7 +1315,7 @@ Compose a route with `Route.make({ protocol, endpoint, auth, framing, ... })`. T
## Effect
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `LLMClient.layer` for LLM dispatch and `ImageClient.layer` for image dispatch, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `AIClient.layer` (or `AIClient.layerWith(executor)`) for every modality, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
## See also
+243 -97
View File
@@ -1,6 +1,7 @@
# Media generation in `@opencode/ai` — public API direction
Status: proposal. Branch `media-support`.
Status: phases 1–4 implemented (through Image queued routes and partial images; ElevenLabs Scribe transcription
pending); phase 5 proposal.
## Goal
@@ -11,11 +12,11 @@ The design below is derived from a survey of the raw provider APIs (OpenAI, Gemi
## What the survey forces
1. **Three execution shapes, everywhere.** Inline sync (OpenAI images, all TTS, Gemini), async job with polling or webhook (every video provider, BFL, fal, Replicate, AssemblyAI), and bidirectional streams (ElevenLabs/Cartesia/Deepgram WS, realtime). Video has no sync provider at all.
2. **Output is never just bytes.** base64, signed URLs with TTLs from 10 minutes (BFL) to 2 days (Veo), URLs that need auth plus redirect (Veo), separate download endpoints (Sora `/content?variant=`), raw bodies (Stability, TTS). Multi-output is the norm.
2. **Output is never just bytes.** base64, signed URLs with TTLs from 10 minutes (BFL, so its route downloads before returning via `PollContext.materialize`) to 2 days (Veo), URLs that need auth plus redirect (Veo), separate download endpoints (Sora `/content?variant=`), raw bodies (Stability, TTS). Multi-output is the norm.
3. **Inputs have roles.** First/last frame, mask, style/subject reference, source video for edit/extend, reference audio, prior generation id, provider-side file handles (`file_id`, `gs://`, `runway://`, `mm_file://`).
4. **Partial streaming is modality-specific.** Images: a few whole partial frames. Audio: ordered chunks plus timestamp events. Jobs: status/progress/logs. Video: none.
5. **Usage is a union**: tokens, seconds, characters (often only in headers), credits, compute time.
6. **Moderation can be partial success** (Veo strips audio but returns video). Deprecations are constant (Sora API shuts down 2026-09-24, Imagen on Gemini API 2026-08-17).
6. **Moderation can be partial success** (Veo strips audio but returns video). Deprecations are constant (Sora API shuts down 2026-09-24; Imagen is shut down on the Gemini API and past its 2026-06-30 discontinuation date on Vertex).
## Where existing SDKs are weak and we should not be
@@ -40,7 +41,7 @@ The design below is derived from a survey of the raw provider APIs (OpenAI, Gemi
### Model selection
A model value is built as `OpenAI.configure({ apiKey }).responses("gpt-5")` or `.image("gpt-image-2")`: `configure` fixes credentials, endpoint, and defaults; the selector fixes which of the provider's APIs to hit and binds the typed `providerOptions` generic. Media follows the same shape with one selector per modality — `openai.image(id)` today, `.video(id)` / `.speech(id)` / `.transcription(id)` as those modalities land — mirroring `openai.responses(id)`. `Image.request` accepts `ImageModel` only, exactly as `LLM.request` accepts `LanguageModel`.
A model value is built as `OpenAI.configure({ apiKey }).responses("gpt-5")` or `.image("gpt-image-2")`: `configure` fixes credentials, endpoint, and defaults; the selector fixes which of the provider's APIs to hit and binds the typed `providerOptions` generic. Media follows the same shape with one selector per modality — `.image(id)`, `.video(id)`, `.speech(id)`, `.transcription(id)` on the facades that offer each — mirroring `openai.responses(id)`. `Image.request` accepts `ImageModel` only, exactly as `LLM.request` accepts `LanguageModel`.
```ts
import { OpenAI, Google } from "@opencode/ai/providers"
@@ -54,7 +55,7 @@ Speech.request({ model: openai.speech("gpt-4o-mini-tts"), text })
Transcription.request({ model: openai.transcription("gpt-4o-transcribe"), audio })
```
The request namespace and the selector share one word (`Image.request` + `.image(...)`). That redundancy is accepted: a callable facade returning a lazily resolved ref would be a second way to construct the same model, and the type machinery to infer `providerOptions` through it is not worth one word. Where a provider has two APIs for one modality, the selectors stay explicit (`openai.chat`, a future `google.imagen`), and one default per modality per provider is part of the facade definition (OpenAI image → Images API, Google image → Gemini-native since Imagen on the Gemini API shuts down 2026-08-17). Provider package entrypoints keep `model(modelID, settings)` per modality-specific path, e.g. `@opencode/ai/providers/openai/responses`.
The request namespace and the selector share one word (`Image.request` + `.image(...)`). That redundancy is accepted: a callable facade returning a lazily resolved ref would be a second way to construct the same model, and the type machinery to infer `providerOptions` through it is not worth one word. Where a provider has two routes for one modality, the selectors stay explicit (`openai.chat`, `stability.image` inline vs `stability.upscale()` queued), and one default per modality per provider is part of the facade definition (OpenAI image → Images API, Google image → Gemini-native; Imagen is shut down, so there is no `google.imagen`). The facade selector (`openai.image(id)`) is the public path for media models. Modality-specific package entrypoints (`model(modelID, settings)` beside today's LLM paths such as `@opencode/ai/providers/openai/responses`) are deferred until Core has a modality-aware model resolver; Core's resolver accepts only `LanguageModel` today.
### `Media` — the asset type
@@ -66,7 +67,7 @@ import { Media } from "@opencode/ai"
Media.Source =
| { type: "bytes"; data: Uint8Array; mediaType: string }
| { type: "base64"; data: string; mediaType: string }
| { type: "url"; url: string; mediaType?: string; expiresAt?: number; headers?: Record<string, string> }
| { type: "url"; url: string; mediaType?: string; expiresAt?: number }
| { type: "ref"; provider: ProviderID; id: string; mediaType?: string } // file_id, gs://, runway://, prior generation
class Media.Asset {
@@ -76,6 +77,7 @@ class Media.Asset {
readonly info?: { width?; height?; durationSeconds?; sampleRate?; channels?; encoding?; format? }
readonly expiresAt?: number
readonly providerMetadata?: ProviderMetadata
readonly headers?: Record<string, string> // transient download credentials (Veo); never in source/JSON
bytes(): Effect<Uint8Array, AIError, RequestExecutor.Service> // downloads/decodes lazily, cached
base64(): Effect<string, AIError, RequestExecutor.Service>
@@ -85,10 +87,18 @@ class Media.Asset {
Media.bytes(data, mediaType?) Media.base64(data, mediaType?)
Media.url(url, options?) Media.ref(provider, id)
Media.file(path) // Bun/Node: reads + sniffs; Effect FileSystem variant for layers
Media.write(asset, path) // convenience, uses FileSystem
Media.file(path) // Effect<Asset, AIError, FileSystem>: reads + sniffs
Media.write(asset, path) // Effect<void, AIError, FileSystem | RequestExecutor.Service>
```
`Media.file` and `Media.write` stay Effect-only: bring your platform's `FileSystem` layer. The Promise client owns the
runtime path: `ai.file(path)` and `ai.write(asset, path)` read and write through `node:fs/promises` (loaded on first
use) with the same media-type sniffing and `InvalidRequest` failures, and `ai.bytes`, `ai.base64`, and
`ai.materialize` run the asset methods in its runtime.
A `ref` source is accepted as input only by routes whose provider issues file handles. No shipped route produces one
yet, so `bytes()` and `materialize()` on a ref fail by design until a producer exists.
Raw-PCM outputs (Gemini TTS, Cartesia raw, Deepgram WS) carry `info.encoding/sampleRate/channels` because there is no container header.
### Modality namespaces
@@ -103,128 +113,253 @@ import { OpenAI, Google, ElevenLabs, Fal } from "@opencode/ai/providers"
#### Image
```ts
const request = Image.request({
model: openai.image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
images: [Media.file("./ref.png")], // references / edit sources
mask: Media.file("./mask.png"),
n: 2,
size: "1536x1024", // or aspectRatio: "3:2"
seed: 7,
format: "webp",
providerOptions: { quality: "high", background: "transparent" }, // typed per model
Effect.gen(function* () {
const request = Image.request({
model: openai.image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
images: [yield* Media.file("./ref.png")], // references / edit sources
mask: yield* Media.file("./mask.png"),
n: 2,
size: "1536x1024", // OpenAI sizes by pixels; Gemini/xAI take aspectRatio instead
format: "webp",
providerOptions: { quality: "high", background: "transparent" }, // typed per model
})
const response = yield* Image.generate(request) // ImageResponse
response.image // Media.Asset (first)
response.images // Media.Asset[]
response.usage // Usage union (see below)
response.notices // moderation / partial-result notices
Image.stream(request) // Stream<ImageEvent>
// ImageEvent: generation-queued | generation-progress | image-partial { index, image } | image { index, image } | finish { usage }
})
const response = yield* Image.generate(request) // ImageResponse
response.image // Media.Asset (first)
response.images // Media.Asset[]
response.usage // Usage union (see below)
response.notices // moderation / partial-result notices
yield* Image.stream(request) // Stream<ImageEvent>
// ImageEvent: generation-queued | generation-progress | image-partial { index, image } | image { index, image } | finish { usage }
```
`size` and `aspectRatio` are not interchangeable; each route rejects fields it cannot lower — see the portability table
in the README's Image generation section.
Editing is not a separate function; `images`/`mask` on the request select the edit path in the route (OpenAI `/images/edits`, Gemini multimodal parts, xAI `/images/edits`). Routes that cannot honor `mask` fail with `Unsupported`.
`ImageRoute` is the inline | stream | queued union, dispatched on `route.kind`, like every modality route. `Image.stream` on a streaming route emits `image-partial` previews before each `image`; on a queued route it emits `generation-queued` / `generation-progress` observations, then the result's `image` and `finish` events.
#### Video
Shipped in phase 2 (`src/video.ts`, `src/video-client.ts`, protocols `google-video`, `xai-video`, `fal-video`, `runway-video`).
```ts
const request = Video.request({
model: google.video("veo-3.1-generate-preview"),
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
frames: { first: Media.file("./start.png"), last: Media.file("./end.png") },
references: [Media.url("https://…/style.png")],
video: Media.ref("openai", "video_123"), // edit / extend / remix source
durationSeconds: 8,
aspectRatio: "16:9",
resolution: "1080p",
audio: true,
n: 1,
providerOptions: { personGeneration: "dont_allow", negativePrompt: "text, watermark" },
Effect.gen(function* () {
const request = Video.request({
model: google.video("veo-3.1-generate-preview"),
prompt: "Panning wide shot of a calico kitten sleeping in the sunshine",
frames: { first: yield* Media.file("./start.png"), last: yield* Media.file("./end.png") },
references: [yield* Media.file("./style.png")],
video: Media.bytes(previous, "video/mp4"), // edit / extend source
durationSeconds: 8,
aspectRatio: "16:9",
resolution: "1080p",
audio: true,
n: 1,
seed: 7,
negativePrompt: "text, watermark", // common, not provider-native
providerOptions: { personGeneration: "allow_adult" },
})
// Simple: wait for it.
const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } })
response.video // Media.Asset: url (expiresAt on Veo and Runway; transient `headers` for Veo downloads)
response.usage // credits on Runway; the other three report none (xAI's usage.cost_in_usd_ticks is not decoded)
response.notices // Veo raiMediaFilteredReasons → filtered, xAI respect_moderation → moderated
yield* response.video.materialize() // pull bytes before the URL expires
// Explicit generation control.
const generation = yield* Video.start(request) // Generation<VideoResponse>
generation.id; generation.status; generation.progress; generation.position; generation.token
yield* generation.await({ poll }) // VideoResponse
yield* generation.cancel() // fal PUT cancel_url, Runway DELETE /tasks/{id}; Veo and xAI succeed without a request
// Resume from another process. The token is validated against the route's codec and refreshed once. It carries no
// route identity, so persist the provider and model ID alongside it: `resume` needs the model.
const resumed = yield* Video.resume(model, JSON.parse(saved))
// Progress as a stream.
Video.stream(request, { poll }) // Stream<VideoEvent>: generation-queued { id, position } | generation-progress { id, progress } | video { index, video } | finish { usage, notices }
})
// Simple: wait for it.
const response = yield* Video.generate(request, { poll: { interval: "10 seconds", timeout: "10 minutes" } })
response.video // Media.Asset (url with expiresAt, or bytes when the route downloads)
yield* response.video.materialize() // pull bytes before the URL expires
// Explicit generation control.
const generation = yield* Video.start(request) // Generation<VideoResponse>
generation.id; generation.status; generation.progress; generation.token // token is serializable JSON
yield* generation.await({ poll }) // VideoResponse
yield* generation.cancel()
// Resume from another process.
const resumed = yield* Video.resume(model, token) // Generation<VideoResponse>
// Progress as a stream.
yield* Video.stream(request) // Stream<VideoEvent>: generation-queued { position } | generation-progress { progress, logs } | video { index, video } | finish
```
Webhooks: `Video.complete(model, token, webhook)` finishes a generation from a webhook payload without polling. Token shape is route-owned and opaque (Veo operation name, fal `response_url`, Runway task id).
Tokens are route-owned JSON: Veo `{ operation }`, xAI `{ requestID }`, Runway `{ taskID }`, fal
`{ requestID, statusURL, responseURL, cancelURL }` (fal's follow-up URLs are authoritative and absolute). Common-field
lowering per provider: Veo takes inline media only, rejects `audio: false` and `n > 1`, and requires `frames.first`
when `frames.last` is set; xAI rejects `n`, `seed`, and `negativePrompt` and routes a `video` input to edits or
(`providerOptions.mode: "extend"`) extensions; fal rejects `n`, plus `durationSeconds`, `references`, and `frames.last`
because the field names and enums differ per model; Runway passes `aspectRatio` through as its pixel `ratio` and
rejects `n`.
Deferred: `Video.complete(model, token, webhook)` (finish from a webhook payload without polling) and provider poll
hints (none of the four providers emit one). Later providers: Luma, Kling, MiniMax, Replicate.
#### Speech (TTS)
Shipped in phase 3 (`src/speech.ts`, `src/speech-client.ts`, protocols `openai-speech`, `google-speech`,
`elevenlabs-speech`, `cartesia-speech`, `deepgram-speech`; new `ElevenLabs`, `Cartesia`, and `Deepgram` facades).
```ts
const request = Speech.request({
model: elevenlabs.speech("eleven_v3"),
model: elevenlabs.speech("eleven_flash_v2_5"),
text: "Hello from OpenCode.",
voice: "JBFqnCBsd6RMkjVDRZzb", // name, uuid, or { id } — provider-normalized
voice: "JBFqnCBsd6RMkjVDRZzb", // provider-native identifier, or { id }
format: "mp3", // mp3 | wav | pcm | opus | aac | flac | (string & {})
speed: 1.0,
language: "en",
instructions: "Warm, unhurried.",
providerOptions: { stability: 0.5 },
instructions: "Warm, unhurried.", // only OpenAI; elsewhere fails typed
timestamps: true, // request alignment; routes without it fail typed
providerOptions: { voice_settings: { stability: 0.5 } },
})
const response = yield* Speech.generate(request) // SpeechResponse: audio: Media.Asset, timestamps?, usage
yield* Speech.stream(request) // Stream<SpeechEvent>: audio-delta { chunk } | timestamps { words } | finish
const response = yield* Speech.generate(request) // SpeechResponse: audio: Media.Asset, timestamps?, usage?, providerMetadata?
yield* Speech.stream(request) // Stream<SpeechEvent>: generation-queued | generation-progress | audio-delta { chunk } | timestamps { items } | finish { audio, usage? }
```
Streaming TTS is first-class on day one: OpenAI `stream_format: sse`, ElevenLabs `/stream`, Cartesia SSE, Deepgram chunked. Input-streaming TTS (WS, text arrives incrementally) is a later `Speech.session(...)` scoped resource, not part of `generate`.
Execution is `MediaProtocol.stream` for every provider: one request whose body is framed and folded by a `step`
state machine, with `generate` running the same stream and collecting it. The route submits the request with its
`mode` (`"generate" | "stream"`), which lets one provider stay one protocol — OpenAI adds `stream_format: "sse"` (except `tts-1`/`tts-1-hd`, which stream raw bytes), ElevenLabs appends
`/stream`, Cartesia switches `/tts/bytes` to `/tts/sse`, Gemini switches `generateContent` to
`streamGenerateContent`. The terminal `finish` event carries the assembled asset (every provider's stream is
concatenable chunks), so stream consumers also get the whole file and `generate` is just "take `finish`, gather
`timestamps`". The cost is memory: a stream holds every chunk until `finish`, so even a consumer that only plays deltas
keeps the whole clip in memory. That is bounded by the providers' input text limits (a few minutes of audio); a
long-form or session API would need an opt-out.
**Voice.** `voice?: string | { id: string }`. A string is passed through as the provider's native identifier — a
name on OpenAI and Gemini, a voice id on ElevenLabs (path segment) and Cartesia. `{ id }` selects an OpenAI custom
voice and is treated as the plain string on routes that do not distinguish custom from built-in. Deepgram's voice is
the model id (`aura-2-thalia-en`), so `voice` is `unsupported` there. There is no cross-provider voice catalog or
name→id resolution. Multi-speaker (Gemini `speechConfig.multiSpeakerVoiceConfig`) and per-voice settings
(ElevenLabs `voice_settings`) go through `providerOptions`.
**Format and PCM.** `format` is container-level; provider sample rates and bitrates live under `providerOptions`
(ElevenLabs `outputFormat`, Cartesia `sampleRate`/`bitRate`/`encoding`, Deepgram `encoding`/`container`/`sampleRate`/
`bitRate`). Each protocol maps `format` to its native value (ElevenLabs `mp3_44100_128`/`pcm_24000`/`wav_24000`/
`opus_48000_64`, Cartesia `{ container, encoding, sample_rate }`, Deepgram `encoding`+`container`) and declares the
asset's media type rather than sniffing, because headerless PCM can look like an MPEG frame sync. Headerless PCM
always carries `info.encoding`, `info.sampleRate`, and `info.channels`; its media type is the provider's declaration
(Gemini `audio/L16;codec=pcm;rate=24000`, Deepgram's `content-type`) or `audio/pcm`. Gemini's asset follows the
provider's declared type: WAV for Gemini 3.8 TTS `generate`, headerless PCM otherwise. The route never wraps PCM as WAV,
so `pcm` is the only explicit `format` it accepts, and not on Gemini 3.8 `generate`. Every `format` value a route cannot
produce (unknown to it, a container on Cartesia SSE, WAV on an ElevenLabs stream, anything but `pcm` on Gemini, `pcm` on
Gemini 3.8 `generate`) fails the same way as an unsupported field: `UnsupportedOperation` with
`operation: "media.format"`.
**Timestamps.** `timestamps: true` on the request asks for alignment. ElevenLabs selects the `with-timestamps`
endpoints (character-level, NDJSON when streaming); Cartesia sets `add_timestamps` on `/tts/sse` (word-level; a
`generate` with timestamps collects the SSE stream). OpenAI, Gemini, and Deepgram reject it.
Common-field lowering per provider:
| Provider | `voice` | `speed` | `language` | `instructions` | `timestamps` | Usage |
|---|---|---|---|---|---|---|
| OpenAI | `voice` (name or `{ id }`) | `speed` | unsupported | `instructions` | unsupported | `tokens` from SSE `speech.audio.done` only |
| Gemini | `prebuiltVoiceConfig.voiceName` | unsupported | `speechConfig.languageCode` | unsupported (direct in text) | unsupported | `tokens` from `usageMetadata` |
| ElevenLabs | path voice id (required) | `voice_settings.speed` | `language_code` | unsupported | `with-timestamps` | `credits` from `character-cost` header |
| Cartesia | `voice` (required) | `generation_config.speed` | `language` | unsupported | `add_timestamps` | none |
| Deepgram | unsupported (voice is the model) | `speed` query | unsupported | unsupported | unsupported | `characters` from `dg-char-count` header |
Deferred: `Speech.session(...)` — input-streaming TTS where text arrives incrementally over a WebSocket (ElevenLabs
`stream-input`, Cartesia WebSocket contexts, Deepgram WebSocket speak) — is a separate scoped resource, not part of
`generate`/`stream`, and ships with the realtime work in phase 5.
#### Transcription (STT)
Shipped as the second half of phase 3 (`src/transcription.ts`, `src/transcription-client.ts`, protocols
`openai-transcription`, `google-transcription`, `deepgram-transcription`, `assemblyai-transcription`; new `AssemblyAI`
facade).
```ts
const request = Transcription.request({
model: openai.transcription("gpt-4o-transcribe"),
audio: Media.file("./call.wav"),
language: "en",
prompt: "Names: Shoubhit, OpenCode.",
timestamps: "word", // none | segment | word
model: openai.transcription("gpt-4o-transcribe-diarize"),
audio: yield* Media.file("./call.wav"),
language: "en", // provider-native passthrough
timestamps: "segment", // none | segment | word
diarize: true,
providerOptions: { chunkingStrategy: "auto" },
speakers: 2, // exact speaker count (AssemblyAI only)
providerOptions: { known_speaker_names: ["agent"] },
})
const response = yield* Transcription.generate(request)
response.text; response.segments; response.words; response.language; response.durationSeconds
yield* Transcription.stream(request) // Stream<TranscriptionEvent>: text-delta | segment | finish
response.text; response.segments; response.words; response.language; response.durationSeconds; response.usage
yield* Transcription.stream(request) // Stream<TranscriptionEvent>: generation-queued | generation-progress | text-delta | segment | finish
const generation = yield* Transcription.start(request) // queued routes only
yield* Transcription.resume(model, token)
```
Realtime STT over WebSocket is the same future `session` shape as input-streaming TTS.
Transcription is the first modality whose providers span all three protocol kinds, and it needed no fourth kind.
Every `MediaRoute` now carries its `kind`; `TranscriptionRoute` is the union of the inline, stream, and queued routes;
`TranscriptionModel.fromRoute` is overloaded per protocol kind (arity picks the overload: `<Options>`,
`<Options, Frame, State>`, `<Options, Token>`) and composes through the shared `composeRoute` (`src/media-model.ts`),
which picks `MediaRoute.inline` / `stream` / `queued`; and `TranscriptionClient`, like every modality client, is
`MediaClient.make` (`src/media-client.ts`), which dispatches on `route.kind`. `generate` on a queued route is `start`
then `await`; `stream` on an inline route is the response as a single `finish`, and on a queued route it is the status
observations followed by `finish`. `start` / `resume` on a non-queued route fail with `UnsupportedOperation` (`transcription.start`). The
`finish` event carries the whole transcript (text, segments, words, language, duration, usage), so the stream route's
`collect` is just "take `finish`".
The route layer gained a `binary` body with array-valued `query` (Deepgram) and `Queued.start.prepare` (AssemblyAI's
upload); `packages/ai/AGENTS.md` (Media Routes) describes both.
Settled rules:
- **Timestamps.** A granularity the selected route or model cannot produce fails as `UnsupportedOperation`
(`media.timestamps`), following Speech; a route that returns more than asked (Deepgram and AssemblyAI always return
words) is not stripped. Segments always carry start and end times: Gemini times each transcription part from its
word offsets, so segment timestamps and diarization also request word offsets there.
- **Diarization.** `diarize` means segments (and words, where the provider labels them) carry `speaker`. Labels are
provider-native strings — OpenAI `A` or a known speaker name, Deepgram `0`, Gemini `spk:0`, AssemblyAI `A` — with no
cross-provider speaker model. `speakers` is the exact number of speakers to label, which AssemblyAI (`speakers_expected`, the only route that
accepts it) treats as a constraint rather than a hint.
- **Language** is passed through (`language`, OpenAI `gpt-transcribe` `languages[]`, Gemini `languageCodes`,
AssemblyAI `language_code`). `response.language` is the provider's own value, lowercased but not normalized: an
ISO code on most routes (AssemblyAI's detection returns `en`), `english` from whisper-1. Deepgram and AssemblyAI
assume English unless asked to detect, so a missing `language` enables their detection.
- **Gemini** requires a transcribe model; other model ids fail with `UnsupportedOperation` before the call, because
general models ignore `audioTranscriptionConfig` and answer conversationally. Streamed chunks carry whole speaker
turns (one part per turn), which join with a space.
- **Streaming inline providers** emit only `finish`; deltas are never faked.
- **Units.** AssemblyAI milliseconds and Gemini protobuf durations (`"0.400s"`) are normalized to seconds at the
protocol boundary.
| Provider | Kind | Audio input | `timestamps` | `diarize` | Unsupported | Usage |
|---|---|---|---|---|---|---|
| OpenAI | stream (`stream: true` in `stream` mode; `whisper-1` ignores `stream`, so it emits only `finish`) | multipart `file` (inline only) | `whisper-1` (`verbose_json`); diarize model: `segment` | `gpt-4o-transcribe-diarize` (`diarized_json`) | `speakers`; `prompt` on the diarize model | `tokens` or `seconds` |
| Gemini | stream (`generateContent` / `streamGenerateContent`) | `inlineData` or Gemini Files `fileData` | `audioTranscriptionConfig.wordTimestamp` | `audioTranscriptionConfig.diarization` | `prompt`, `speakers` | `tokens` |
| Deepgram | inline | raw body, or JSON `{ url }` | words always; `segment` → `utterances` | `diarize_model=latest` + `utterances` | `prompt`, `speakers` | `seconds` (`metadata.duration`) |
| AssemblyAI | queued (upload → submit → poll) | `/v2/upload` then `audio_url`, or a URL | words always; `segment` → `speaker_labels` | `speaker_labels` | — | `seconds` (`audio_duration`) |
Deferred: `Transcription.session(...)` — realtime STT over WebSocket (Deepgram live, AssemblyAI streaming, ElevenLabs
realtime, OpenAI realtime transcription) — is the same future scoped `session` shape as input-streaming TTS and ships
with the realtime work in phase 5. ElevenLabs Scribe is not implemented yet.
### `Generation` — shared async execution
```ts
class Generation<Response> {
readonly id: string
readonly model: MediaModel
readonly route: GenerationRoute<Response> // token-free: { status, result, cancel?: Effect } closed over the decoded token
readonly token: unknown // route-owned serializable JSON
readonly status: "queued" | "running" | "completed" | "failed" | "cancelled" | "expired"
readonly progress?: number // 0..1, normalized
readonly position?: number
readonly expiresAt?: number
refresh(): Effect<Generation<Response>, AIError>
await(options?: { poll?: Poll }): Effect<Response, AIError>
result(): Effect<Response, AIError>
await(options?: GenerationAwaitOptions): Effect<Response, AIError>
cancel(): Effect<void, AIError>
events(options?): Stream<GenerationEvent, AIError>
events(options?: GenerationAwaitOptions): Stream<GenerationEvent, AIError> // fails with Timeout past poll.timeout, checked per observation
}
Poll = { interval?: Duration; timeout?: Duration; schedule?: Schedule } // route may override from provider hints (`openai-poll-after-ms`)
GenerationAwaitOptions = { poll?: Poll }
Poll = { interval?: Duration; timeout?: Duration }
```
`Generation` is not video-specific. Image routes on BFL, fal, and Replicate are queued; `Image.start` exists for them. A route declares itself `inline` or `queued`; `generate` on a queued route is `start` then `await`.
`Generation` is not video-specific. Image routes on BFL, fal, Replicate, and Stability `upscale()` are queued; `Image.start` exists for them. A route declares itself `inline` or `queued`; `generate` on a queued route is `start` then `await`.
### Usage
@@ -250,15 +385,18 @@ const ai = AI.make() // ManagedRuntime over Reque
// AI.make({ layer }) to inject a custom executor / recorder / middleware
const image = await ai.image.generate({ model, prompt })
await image.image.bytes()
await ai.bytes(image.image) // also ai.base64, ai.materialize, ai.write(asset, path)
const reference = await ai.file("./ref.png")
for await (const event of ai.speech.stream({ model, text, voice })) { … }
const generation = await ai.video.start({ model, prompt })
const generation = await ai.video.start({ model, prompt }) // snapshot handle; refresh() returns a new one
for await (const event of generation.events({ poll: { interval: 10_000 } })) { … }
const video = await generation.await({ poll: { interval: 10_000 }, signal })
const resumed = ai.video.resume(model, JSON.parse(saved))
const resumed = await ai.video.resume(model, JSON.parse(saved)) // persist provider + model ID with the token
const text = await ai.llm.generate({ model, prompt }) // closes today's gap: LLM has no promise API either
const request = ai.llm.request({ model, prompt })
const text = await ai.llm.generate(request)
for await (const event of ai.llm.stream(request)) { … }
await ai.dispose()
@@ -268,30 +406,38 @@ Streams become `AsyncIterable` via `Stream.toAsyncIterable`. `AIError` is thrown
### Providers
Existing facades gain per-modality selectors; the modality routes each facade provides:
Existing facades gain per-modality selectors; the modality routes each facade provides (*italics* are not
implemented):
| Facade | llm | image | video | speech | transcription | other |
|---|---|---|---|---|---|---|
| `OpenAI` | responses (default), chat | Images API | Sora (deprecated 2026-09-24) | ✓ | ✓ | |
| `Google` | Gemini | Gemini-native (default), `imagen` | Veo | Gemini TTS | Gemini transcribe | |
| `OpenAI` | responses (default), chat | Images API (stream) | *Sora skipped (decision 8)* | ✓ | ✓ | |
| `Google` | Gemini | Gemini-native | Veo | Gemini TTS | `gemini-3.5-transcribe` | |
| `XAI` | ✓ | ✓ | ✓ | | | |
| `ElevenLabs` | | | | ✓ | Scribe | soundEffect, music |
| `Fal` | | ✓ | ✓ | | | |
| `Replicate`, `Runway`, `Luma`, `Kling`, `MiniMax`, `Deepgram`, `Cartesia`, `AssemblyAI`, `BlackForestLabs`, `Stability` | | per provider | | | | |
| `ElevenLabs` | | | | ✓ | *Scribe (pending)* | *soundEffect, music (phase 5)* |
| `Cartesia` | | | | ✓ | | |
| `Deepgram` | | | | Aura | ✓ | |
| `Fal` | | ✓ (queued) | ✓ | | | |
| `AssemblyAI` | | | | | ✓ (queued) | |
| `BlackForestLabs` | | ✓ (queued) | | | | |
| `Replicate` | | ✓ (queued) | | | | |
| `Stability` | | `image` (inline), `upscale()` (queued) | | | | |
| `Runway` | | | ✓ | | | |
| `Luma`, `Kling`, `MiniMax` | | *deferred* | *deferred* | | | |
New facades follow the existing one-file-per-provider rule. Package entrypoints are modality-specific, such as `@opencode/ai/providers/openai/images`, and return the concrete model.
New facades follow the existing one-file-per-provider rule. The facade selector is the public path for media models; modality-specific package entrypoints (for example `@opencode/ai/providers/openai/images`) are deferred until Core has a modality-aware model resolver.
`ImageModel<Options>` already gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, `TranscriptionModel` follow the same generic. A shared `MediaModel` union is what `Generation` and the promise client key on.
`ImageModel<Options>` gives typed `providerOptions` per model; `VideoModel`, `SpeechModel`, and `TranscriptionModel` follow the same generic. As with `LanguageModel`, the route type does not carry `Options`, so `ImageModel<OpenAIImageOptions>` is an `ImageModel` and client methods take plain `ImageRequestFor`. They share an internal `MediaModel` base class (ids, route, `http` overlays) that is not part of the public exports; `Generation` and the promise client work with the concrete modality models.
### Routes and protocols
Media does not fit the LLM four-axis route (SSE frames → event state machine) except for streaming TTS/STT. Reuse `Endpoint`, `Auth`, `Framing`, `RequestExecutor`, and add media protocol kinds:
- `MediaProtocol.inline` — `body.from(request)` (JSON, multipart, or query), `response.decode(response)` (JSON, or binary body → `Media.Asset`).
- `MediaProtocol.queued` — `start`, `status`, `result`, `cancel`, optional `download`, `pollHint`, `token` schema.
- `MediaProtocol.stream` — framing + `step` state machine emitting modality events, same discipline as LLM protocols.
- `MediaProtocol.queued` — `start` (body + decode to `{ token, snapshot }`), `status`, `result`, optional `cancel`, and a `token` codec. `result` is always a separate GET (against the status document for Veo/xAI/Runway, fal's `response_url` otherwise) so `await` after `start` and after `resume` share one path. `PollContext.auth` hands the auth headers the route sent to the protocol for output URLs that need them (Veo downloads); they become transient `Media.Asset.headers`, never part of `source`. There is no separate `download` step: `Media.Asset.bytes()` downloads through the executor with those headers. `MediaRoute.inline(...)` / `MediaRoute.queued(...)` compose each kind with endpoint and auth; the queued route decodes the token once and hands `Generation` a token-free `{ status, result, cancel? }`.
- `MediaProtocol.stream` — `body.from(request)` over the request plus its `mode`, `frames` (a function that picks the framing for the call: `Framing.sse`, `lines`, `document`, or the raw bytes), fresh per-response `initial()` state, `step` emitting modality events, and `finish(state, context)` — with the observed response for header-only usage — emitting exactly one terminal event or failing as an incomplete stream. The route fills `reason.http` on stream errors. `MediaRoute.stream(...)` exposes `stream` and `generate` (the same stream folded by the modality's `collect`).
`Route.make` for media composes one protocol kind with endpoint/auth. The existing `ImageRoute { generate(request, execute) }` is the ad-hoc version of `inline` and gets folded in.
`MediaRoute.inline` / `MediaRoute.queued` / `MediaRoute.stream` compose one protocol kind with endpoint/auth and tag the route with its `kind`; `ImageModel`/`VideoModel`/`SpeechModel`/`TranscriptionModel` share the `MediaModel` base (`src/media-model.ts`).
### LLM integration
@@ -320,9 +466,9 @@ Foundation + Image ship together as the reference implementation, serially. Vide
## Phasing
1. **Foundation** — per-modality selectors, `Media`, `Generation`, `Poll`, `Usage` union, `MediaProtocol` kinds, `@opencode/ai/promise` with `llm` + `image`. Port the five existing image protocols onto it. Unify `MediaPart` and add the `media` LLM event (fixes Gemini image output being dropped).
2. **Video** — Veo, xAI, fal, Runway first. Then Luma, Kling, MiniMax, Replicate.
3. **Speech + Transcription** — OpenAI, ElevenLabs, Gemini TTS, Deepgram, Cartesia, AssemblyAI. Streaming TTS from the start.
4. **Image queued routes and partials** — BFL, fal, Replicate, Stability; OpenAI `partial_images` streaming.
2. **Video** — ✅ Veo, xAI, fal, Runway shipped (`MediaProtocol.queued`, `Video.start/generate/resume/stream`, promise `ai.video`). Deferred: `Video.complete` (webhooks), Luma, Kling, MiniMax, Replicate.
3. **Speech + Transcription** — ✅ Speech: OpenAI, Gemini TTS, ElevenLabs, Cartesia, Deepgram shipped (`MediaProtocol.stream`, `Speech.generate/stream`, promise `ai.speech`). ✅ Transcription: OpenAI, Gemini, Deepgram, AssemblyAI shipped across all three route kinds (`Transcription.generate/stream/start/resume`, promise `ai.transcription`). Pending: ElevenLabs Scribe. Deferred: `Speech.session` and `Transcription.session` (WebSocket streaming).
4. **Image queued routes and partials** — ✅ BFL, fal, Replicate, and Stability creative upscale queued; Stability generate inline; OpenAI `partial_images` streaming (`image-partial` restored). Imagen dropped: shut down on the Gemini API and discontinued on Vertex (2026-06-30). Deferred: Stability's synchronous edit and fast/conservative upscale endpoints.
5. **Later** — ElevenLabs music/SFX, Lyria, `Speech.session` / `Transcription.session`, realtime.
Core adoption (session attachments beyond png/jpeg/gif/webp/pdf, image-generation tool, TUI rendering) comes after phase 1 and is a Core concern.
+7 -20
View File
@@ -1,18 +1,7 @@
import { Config, Effect, Formatter, Layer, Schema, Stream } from "effect"
import { Config, Effect, Formatter, Schema, Stream } from "effect"
import { NodeFileSystem } from "@effect/platform-node"
import {
Image,
ImageClient,
LLM,
LLMClient,
LLMRequest,
Media,
Message,
ProviderID,
Tool,
ToolRuntime,
} from "@opencode/ai"
import { Route, Auth, Endpoint, Framing, Protocol, RequestExecutor } from "@opencode/ai/route"
import { AIClient, Image, LLM, LLMRequest, Media, Message, ProviderID, Tool, ToolRuntime } from "@opencode/ai"
import { Route, Auth, Endpoint, Framing, Protocol } from "@opencode/ai/route"
import { OpenAI } from "@opencode/ai/providers"
/**
@@ -243,12 +232,10 @@ const generateImage = Effect.gen(function* () {
yield* Media.write(response.image, "tutorial-image.jpg").pipe(Effect.provide(NodeFileSystem.layer))
})
// Provide the LLM runtime and the HTTP request executor once. Keep one path
// enabled at a time so the tutorial can demonstrate generate, stream, or
// Provide every modality client and the HTTP request executor once with
// `AIClient.layer` (`AIClient.layerWith(executor)` swaps the executor). Keep one
// path enabled at a time so the tutorial can demonstrate generate, stream, or
// tool-loop behavior without spending tokens on every example.
const requestExecutorLayer = RequestExecutor.fetchLayer
const llmClientLayer = LLMClient.layer.pipe(Layer.provide(requestExecutorLayer))
const imageClientLayer = ImageClient.layer.pipe(Layer.provide(requestExecutorLayer))
const program = Effect.gen(function* () {
// yield* generateOnce
@@ -257,6 +244,6 @@ const program = Effect.gen(function* () {
// yield* generateDynamicObject.pipe(Effect.andThen((response) => Effect.sync(() => console.log(response.object))))
// yield* generateImage
yield* streamWithTools
}).pipe(Effect.provide(Layer.mergeAll(requestExecutorLayer, llmClientLayer, imageClientLayer)))
}).pipe(Effect.provide(AIClient.layer))
Effect.runPromise(program)
+1 -1
View File
@@ -1,6 +1,6 @@
{
"$schema": "https://json.schemastore.org/package.json",
"version": "2.0.14",
"version": "2.0.18",
"name": "@opencode/ai",
"type": "module",
"license": "MIT",
+105
View File
@@ -104,6 +104,111 @@ const PROVIDERS: ReadonlyArray<Provider> = [
vars: [{ name: "XAI_API_KEY" }],
validate: (env) => validateBearer("https://api.x.ai/v1/models", Redacted.make(env.XAI_API_KEY)),
},
{
id: "fal",
label: "fal",
tier: "canary",
note: "fal queue image and video recorded tests",
vars: [{ name: "FAL_KEY" }],
// fal has no free authenticated list endpoint; a 404 for an unknown request id proves the key was accepted.
validate: (env) =>
Effect.gen(function* () {
const http = yield* HttpClient.HttpClient
const response = yield* http.execute(
HttpClientRequest.get(
"https://queue.fal.run/fal-ai/veo3.1/requests/00000000-0000-0000-0000-000000000000/status",
).pipe(HttpClientRequest.setHeaders({ authorization: `Key ${Redacted.value(Redacted.make(env.FAL_KEY))}` })),
)
if (response.status === 404) return undefined
return yield* responseError(response)
}),
},
{
id: "black-forest-labs",
label: "Black Forest Labs",
tier: "canary",
note: "BFL FLUX queued image recorded tests",
vars: [{ name: "BFL_API_KEY" }],
validate: (env) =>
HttpClientRequest.get("https://api.bfl.ai/v1/credits").pipe(
HttpClientRequest.setHeader("x-key", Redacted.value(Redacted.make(env.BFL_API_KEY))),
executeRequest,
),
},
{
id: "replicate",
label: "Replicate",
tier: "canary",
note: "Replicate prediction image recorded tests",
vars: [{ name: "REPLICATE_API_TOKEN" }],
validate: (env) => validateBearer("https://api.replicate.com/v1/account", Redacted.make(env.REPLICATE_API_TOKEN)),
},
{
id: "stability",
label: "Stability AI",
tier: "canary",
note: "Stability inline generate and queued upscale recorded tests",
vars: [{ name: "STABILITY_API_KEY" }],
validate: (env) => validateBearer("https://api.stability.ai/v1/user/balance", Redacted.make(env.STABILITY_API_KEY)),
},
{
id: "runway",
label: "Runway",
tier: "canary",
note: "Runway task video recorded tests",
vars: [{ name: "RUNWAYML_API_SECRET" }],
validate: (env) =>
validateBearer("https://api.dev.runwayml.com/v1/organization", Redacted.make(env.RUNWAYML_API_SECRET), {
"X-Runway-Version": "2024-11-06",
}),
},
{
id: "elevenlabs",
label: "ElevenLabs",
tier: "canary",
note: "ElevenLabs text-to-speech recorded tests",
vars: [{ name: "ELEVENLABS_API_KEY" }],
validate: (env) =>
HttpClientRequest.get("https://api.elevenlabs.io/v1/models").pipe(
HttpClientRequest.setHeader("xi-api-key", Redacted.value(Redacted.make(env.ELEVENLABS_API_KEY))),
executeRequest,
),
},
{
id: "cartesia",
label: "Cartesia",
tier: "canary",
note: "Cartesia text-to-speech recorded tests",
vars: [{ name: "CARTESIA_API_KEY" }],
validate: (env) =>
validateBearer("https://api.cartesia.ai/voices?limit=1", Redacted.make(env.CARTESIA_API_KEY), {
"Cartesia-Version": "2026-08-14",
}),
},
{
id: "deepgram",
label: "Deepgram",
tier: "canary",
note: "Deepgram Aura text-to-speech and Nova transcription recorded tests",
vars: [{ name: "DEEPGRAM_API_KEY" }],
validate: (env) =>
HttpClientRequest.get("https://api.deepgram.com/v1/projects").pipe(
HttpClientRequest.setHeader("authorization", `Token ${Redacted.value(Redacted.make(env.DEEPGRAM_API_KEY))}`),
executeRequest,
),
},
{
id: "assemblyai",
label: "AssemblyAI",
tier: "canary",
note: "AssemblyAI queued transcription recorded tests",
vars: [{ name: "ASSEMBLYAI_API_KEY" }],
validate: (env) =>
HttpClientRequest.get("https://api.assemblyai.com/v2/transcript?limit=1").pipe(
HttpClientRequest.setHeader("authorization", Redacted.value(Redacted.make(env.ASSEMBLYAI_API_KEY))),
executeRequest,
),
},
{
id: "cloudflare-ai-gateway",
label: "Cloudflare AI Gateway",
+24
View File
@@ -0,0 +1,24 @@
import { Layer } from "effect"
import { ImageClient } from "./image-client.js"
import { LLMClient } from "./route/client.js"
import { RequestExecutor } from "./route/executor.js"
import { SpeechClient } from "./speech-client.js"
import { TranscriptionClient } from "./transcription-client.js"
import { VideoClient } from "./video-client.js"
/** Every modality client over `executor`, which stays in the output so `asset.bytes()` and `Media.write` resolve. */
export const layerWith = <E, R>(executor: Layer.Layer<RequestExecutor.Service, E, R>) =>
Layer.mergeAll(
LLMClient.layer,
ImageClient.layer,
VideoClient.layer,
SpeechClient.layer,
TranscriptionClient.layer,
).pipe(Layer.provideMerge(executor))
/** Every modality client plus the executor over `RequestExecutor.fetchLayer`: the one layer most programs need. */
export const layer = layerWith(RequestExecutor.fetchLayer)
export type Services = Layer.Success<typeof layer>
export * as AIClient from "./ai-client.js"
@@ -86,7 +86,7 @@ export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer
})
}),
)
export const fetchLayer = layer.pipe(Layer.provide(RequestExecutor.fetchLayer))
export const fetchLayer = layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer))
export const EvaluationClient = {
Service,
+1 -1
View File
@@ -214,7 +214,7 @@ export function request(input: EvaluationRequest | EvaluationRequestInput) {
return new EvaluationRequest({
...input,
model: input.model as unknown as EvaluationModel,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
http: HttpOptions.make(input.http),
})
}
+108 -57
View File
@@ -1,4 +1,4 @@
import { Duration, Effect, Schedule, Schema, Stream } from "effect"
import { Clock, Duration, Effect, Schedule, Schema, Stream } from "effect"
import { AIError, TimeoutError } from "./schema/errors.js"
export const Status = Schema.Literals(["queued", "running", "completed", "failed", "cancelled", "expired"])
@@ -11,47 +11,59 @@ export interface Snapshot {
/** Normalized 0..1 when the provider reports progress. */
readonly progress?: number
readonly position?: number
readonly expiresAt?: number
}
/**
* Route-owned generation operations. `token` is the route's serializable handle (operation name, task id, response URL)
* so a generation can be resumed from another process; its shape is opaque to `Generation`.
* Route-owned generation operations for one generation. The media route decodes its serializable token once (from the
* submission response or a `resume` input) and closes over it, so `Generation` never sees the token's shape.
*/
export interface Route<Response> {
readonly status: (token: unknown) => Effect.Effect<Snapshot, AIError>
readonly result: (token: unknown) => Effect.Effect<Response, AIError>
readonly cancel?: (token: unknown) => Effect.Effect<void, AIError>
/** Provider polling hint (e.g. `openai-poll-after-ms`) that overrides the default interval for the next poll. */
readonly pollHint?: (snapshot: Snapshot) => Duration.Duration | undefined
readonly status: Effect.Effect<Snapshot, AIError>
readonly result: Effect.Effect<Response, AIError>
readonly cancel?: Effect.Effect<void, AIError>
}
export interface Poll {
readonly interval?: Duration.Input
readonly timeout?: Duration.Input
/** Full override of the polling schedule; `interval` and `pollHint` are ignored when supplied. */
readonly schedule?: Schedule.Schedule<unknown, Snapshot>
}
export interface AwaitOptions {
readonly poll?: Poll
}
export const DEFAULT_POLL_INTERVAL = Duration.seconds(5)
export const DEFAULT_POLL_TIMEOUT = Duration.minutes(10)
export type Event =
| { readonly type: "generation-queued"; readonly id: string; readonly position?: number }
| { readonly type: "generation-progress"; readonly id: string; readonly progress?: number }
| { readonly type: "generation-finished"; readonly id: string; readonly status: Status }
export const QueuedEvent = Schema.Struct({
type: Schema.tag("generation-queued"),
id: Schema.String,
position: Schema.optional(Schema.Number),
}).annotate({ identifier: "Generation.Event.Queued" })
export const ProgressEvent = Schema.Struct({
type: Schema.tag("generation-progress"),
id: Schema.String,
progress: Schema.optional(Schema.Number),
}).annotate({ identifier: "Generation.Event.Progress" })
export type Observation = Schema.Schema.Type<typeof QueuedEvent> | Schema.Schema.Type<typeof ProgressEvent>
export type Event = Observation | { readonly type: "generation-finished"; readonly id: string; readonly status: Status }
const TERMINAL: ReadonlySet<Status> = new Set(["completed", "failed", "cancelled", "expired"])
export const isTerminal = (status: Status) => TERMINAL.has(status)
export class Generation<Response> {
readonly id: string
readonly status: Status
readonly progress?: number
readonly position?: number
readonly expiresAt?: number
constructor(
readonly route: Route<Response>,
/** Route-owned serializable JSON; pass it to the modality's `resume` from another process. */
readonly token: unknown,
snapshot: Snapshot,
) {
@@ -59,7 +71,6 @@ export class Generation<Response> {
this.status = snapshot.status
this.progress = snapshot.progress
this.position = snapshot.position
this.expiresAt = snapshot.expiresAt
}
get snapshot(): Snapshot {
@@ -68,70 +79,110 @@ export class Generation<Response> {
status: this.status,
progress: this.progress,
position: this.position,
expiresAt: this.expiresAt,
}
}
get terminal() {
return TERMINAL.has(this.status)
return isTerminal(this.status)
}
refresh(): Effect.Effect<Generation<Response>, AIError> {
return this.route.status(this.token).pipe(Effect.map((snapshot) => new Generation(this.route, this.token, snapshot)))
return this.route.status.pipe(Effect.map((snapshot) => new Generation(this.route, this.token, snapshot)))
}
/** Fetch the result without polling; non-completed terminal generations fail with the provider's terminal body. */
result(): Effect.Effect<Response, AIError> {
return this.route.result
}
/** Poll until the generation reaches a terminal status, then fetch the result. Fails with a `Timeout` reason on deadline. */
await(options?: { readonly poll?: Poll }): Effect.Effect<Response, AIError> {
await(options?: AwaitOptions): Effect.Effect<Response, AIError> {
const timeout = Duration.fromInputUnsafe(options?.poll?.timeout ?? DEFAULT_POLL_TIMEOUT)
const settled = this.terminal ? Effect.succeed(this) : this.poll(options?.poll)
return settled.pipe(
// Non-completed terminal states also go through `result` so the route can surface its provider failure body.
Effect.flatMap((generation) => generation.route.result(generation.token)),
Effect.timeoutOrElse({
duration: timeout,
orElse: () =>
new AIError({
reason: new TimeoutError({
message: `Generation ${this.id} did not finish within ${Duration.format(timeout)}`,
timeoutMs: Duration.toMillis(timeout),
}),
}),
}),
Effect.flatMap((generation) => generation.result()),
Effect.timeoutOrElse({ duration: timeout, orElse: () => this.timeoutError(timeout) }),
)
}
cancel(): Effect.Effect<void, AIError> {
return this.route.cancel?.(this.token) ?? Effect.void
return this.route.cancel ?? Effect.void
}
/** Status observations as a stream, ending after the first terminal observation. */
events(options?: { readonly poll?: Poll }): Stream.Stream<Event, AIError> {
const observations = this.terminal
? Stream.make(this)
: Stream.fromEffectSchedule(this.refresh(), this.schedule(options?.poll)).pipe(
Stream.takeUntil((generation) => generation.terminal),
)
return observations.pipe(
Stream.map((generation): Event => {
if (generation.terminal) return { type: "generation-finished", id: generation.id, status: generation.status }
if (generation.status === "queued") return { type: "generation-queued", id: generation.id, position: generation.position }
return { type: "generation-progress", id: generation.id, progress: generation.progress }
}),
/**
* Status observations as a stream, ending after the first terminal observation. Each poll and each sleep between polls
* is bounded by the time remaining until `poll.timeout`, so a hung status request or a long interval fails the stream at
* the deadline instead of stalling it. (`Stream.interruptWhen` would express this directly but deadlocks under
* `TestClock` when the source completes while the timer sleeps.)
*/
events(options?: AwaitOptions): Stream.Stream<Event, AIError> {
if (this.terminal) return Stream.make(this.event())
const timeout = Duration.fromInputUnsafe(options?.poll?.timeout ?? DEFAULT_POLL_TIMEOUT)
return Stream.unwrap(
Clock.currentTimeMillis.pipe(
Effect.map((start) => {
const deadline = start + Duration.toMillis(timeout)
// Fail before polling once the deadline has passed: a fast status request could otherwise win the zero-budget
// race and schedule another zero-delay poll.
const refresh = Clock.currentTimeMillis.pipe(
Effect.flatMap((now) =>
now >= deadline
? this.timeoutError(timeout)
: this.refresh().pipe(
Effect.timeoutOrElse({
duration: Duration.millis(deadline - now),
orElse: () => this.timeoutError(timeout),
}),
),
),
)
const schedule = this.schedule(options?.poll).pipe(
Schedule.modifyDelay((meta) =>
Effect.succeed(Duration.min(meta.duration, Duration.millis(Math.max(0, deadline - meta.now)))),
),
)
return Stream.fromEffectSchedule(refresh, schedule).pipe(
Stream.takeUntil((generation) => generation.terminal),
Stream.map((generation) => generation.event()),
)
}),
),
)
}
private event(): Event {
if (this.terminal) return { type: "generation-finished", id: this.id, status: this.status }
if (this.status === "queued") return { type: "generation-queued", id: this.id, position: this.position }
return { type: "generation-progress", id: this.id, progress: this.progress }
}
private timeoutError(timeout: Duration.Duration) {
return new AIError({
reason: new TimeoutError({
message: `Generation ${this.id} did not finish within ${Duration.format(timeout)}`,
timeoutMs: Duration.toMillis(timeout),
}),
})
}
private poll(poll: Poll | undefined) {
return this.refresh().pipe(Effect.repeat({ schedule: this.schedule(poll), until: (generation) => generation.terminal }))
}
private schedule(poll: Poll | undefined): Schedule.Schedule<unknown, Generation<Response>> {
if (poll?.schedule) return poll.schedule.pipe(Schedule.setInputType<Generation<Response>>())
const interval = poll?.interval ?? DEFAULT_POLL_INTERVAL
const pollHint = this.route.pollHint
const spaced = Schedule.spaced(interval).pipe(Schedule.setInputType<Generation<Response>>())
if (!pollHint) return spaced
return spaced.pipe(
Schedule.modifyDelay((metadata) => Effect.succeed(pollHint(metadata.input.snapshot) ?? interval)),
return this.refresh().pipe(
Effect.repeat({ schedule: this.schedule(poll), until: (generation) => generation.terminal }),
)
}
private schedule(poll: Poll | undefined) {
return Schedule.spaced(poll?.interval ?? DEFAULT_POLL_INTERVAL)
}
}
export const resultEvents = <Response, A>(
generation: Generation<Response>,
expand: (response: Response) => ReadonlyArray<A>,
options?: AwaitOptions,
): Stream.Stream<Observation | A, AIError> =>
generation.events(options).pipe(
Stream.filter((event): event is Observation => event.type !== "generation-finished"),
Stream.concat(Stream.fromIterableEffect(Effect.map(generation.result(), expand))),
)
+19 -50
View File
@@ -1,61 +1,30 @@
import { Context, Effect, Layer, Stream } from "effect"
import { RequestExecutor } from "./route/executor.js"
import type { AIError } from "./schema/index.js"
import { Context } from "effect"
import { MediaClient } from "./media-client.js"
import {
responseEvents,
ImageOutputEvent,
ImageFinishEvent,
type ImageEvent,
type ImageOptions,
type ImageRequestFor,
type ImageResponse,
} from "./image.js"
export interface Interface {
readonly generate: <Options extends ImageOptions>(
request: ImageRequestFor<Options>,
) => Effect.Effect<ImageResponse, AIError>
readonly stream: <Options extends ImageOptions>(
request: ImageRequestFor<Options>,
) => Stream.Stream<ImageEvent, AIError>
}
export type Interface = MediaClient.Interface<ImageRequestFor, ImageEvent, ImageResponse>
export class Service extends Context.Service<Service, Interface>()("@opencode/ImageClient") {}
export const generate = <Options extends ImageOptions>(
request: ImageRequestFor<Options>,
): Effect.Effect<ImageResponse, AIError, Service> =>
Effect.gen(function* () {
const client = yield* Service
return yield* client.generate(request)
})
export const stream = <Options extends ImageOptions>(
request: ImageRequestFor<Options>,
): Stream.Stream<ImageEvent, AIError, Service> =>
Stream.unwrap(
Effect.gen(function* () {
const client = yield* Service
return client.stream(request)
}),
)
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
Service,
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
const generate = <Options extends ImageOptions>(request: ImageRequestFor<Options>) =>
request.model.route.generate(request, executor.execute)
return Service.of({
generate,
// Inline routes have no partial frames yet; the stream is the completed response expanded into events.
stream: (request) =>
Stream.unwrap(generate(request).pipe(Effect.map((response) => Stream.fromIterable(responseEvents(response))))),
})
}),
)
export class ImageClientService extends Context.Service<ImageClientService, Interface>()("@opencode/ImageClient") {}
export const Service = ImageClientService
export type Service = ImageClientService
export const ImageClient = {
Service,
layer,
generate,
stream,
...MediaClient.make(Service, {
modality: "image",
responseEvents: (response: ImageResponse) => [
...response.images.map((image, index) => ImageOutputEvent.make({ index, image })),
ImageFinishEvent.make({
usage: response.usage,
notices: response.notices,
providerMetadata: response.providerMetadata,
}),
],
}),
} as const
+81 -111
View File
@@ -1,98 +1,48 @@
import { Effect, Schema, Stream } from "effect"
import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./generation.js"
import { Media } from "./media.js"
import { Endpoint } from "./route/endpoint.js"
import { MediaModel, composeRoute, tryRequest } from "./media-model.js"
import { MediaRoute } from "./route/media.js"
import type { MediaProtocol } from "./route/media-protocol.js"
import {
AIError,
HttpOptions,
InvalidRequestError,
MediaUsage,
ModelID,
ProviderID,
ProviderMetadata,
} from "./schema/index.js"
import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js"
import { ImageClient, Service } from "./image-client.js"
// ---------------------------------------------------------------------------
// Model
// ---------------------------------------------------------------------------
export type ImageOptions = Record<string, unknown>
export type ImageOptions = MediaModel.Options
export type ImageRoute<Options extends ImageOptions = ImageOptions> = MediaRoute.Route<
ImageRequestFor<Options>,
ImageResponse
>
export type ImageRoute = MediaRoute.AnyRoute<ImageRequestFor, ImageEvent, ImageResponse>
export class ImageModel<Options extends ImageOptions = ImageOptions> {
declare protected readonly _Options: (options: Options) => Options
readonly id: ModelID
readonly provider: ProviderID
readonly route: ImageRoute<Options>
readonly http?: HttpOptions
export class ImageModel<Options extends ImageOptions = ImageOptions> extends MediaModel<ImageRoute, Options> {
declare protected readonly _ImageModel: void
constructor(input: ImageModel.Input<Options>) {
this.id = input.id
this.provider = input.provider
this.route = input.route
this.http = input.http
}
static make<Options extends ImageOptions = ImageOptions>(input: ImageModel.MakeInput<Options>) {
return new ImageModel<Options>({
id: ModelID.make(input.id),
provider: ProviderID.make(input.provider),
route: input.route,
http: input.http,
})
}
/** Compose an inline image protocol with its canonical path into a model for one deployment. */
static fromRoute<Options extends ImageOptions = ImageOptions>(
route: ImageModel.RouteInput<Options>,
/** The number of type arguments selects the kind: `<Options>`, `<Options, Frame, State>`, or `<Options, Token>`. */
static fromRoute<Options extends ImageOptions>(
route: MediaModel.InlineRouteInput<ImageRequestFor<Options>, ImageResponse>,
input: MediaRoute.ModelInput,
): ImageModel<Options>
static fromRoute<Options extends ImageOptions, Frame, State>(
route: MediaModel.StreamRouteInput<ImageRequestFor<Options>, ImageEvent, Frame, State>,
input: MediaRoute.ModelInput,
): ImageModel<Options>
static fromRoute<Options extends ImageOptions, Token>(
route: MediaModel.QueuedRouteInput<ImageRequestFor<Options>, ImageResponse, Token>,
input: MediaRoute.ModelInput,
): ImageModel<Options>
static fromRoute<Options extends ImageOptions, Frame, State, Token>(
route: MediaModel.AnyRouteInput<ImageRequestFor<Options>, ImageEvent, ImageResponse, Frame, State, Token>,
input: MediaRoute.ModelInput,
) {
return ImageModel.make<Options>({
return new ImageModel<Options>({
id: input.id,
provider: route.provider,
provider: route.protocol.provider,
http: input.http,
route: MediaRoute.make({
id: route.id,
provider: route.provider,
protocol: route.protocol,
endpoint: Endpoint.path(route.path, { baseURL: input.baseURL ?? route.baseURL }),
auth: input.auth,
headers: input.headers,
}),
route: composeRoute(route, input, collectResponse) as ImageRoute,
})
}
}
export namespace ImageModel {
export interface Input<Options extends ImageOptions = ImageOptions> {
readonly id: ModelID
readonly provider: ProviderID
readonly route: ImageRoute<Options>
readonly http?: HttpOptions
}
export interface MakeInput<Options extends ImageOptions = ImageOptions>
extends Omit<Input<Options>, "id" | "provider"> {
readonly id: string | ModelID
readonly provider: string | ProviderID
}
export interface RouteInput<Options extends ImageOptions = ImageOptions> {
readonly id: string
readonly provider: string | ProviderID
readonly protocol: MediaProtocol.Inline<ImageRequestFor<Options>, ImageResponse>
readonly path: Endpoint.EndpointPart<MediaProtocol.Body, ImageRequestFor<Options>>
/** Canonical base URL; `ModelInput.baseURL` overrides it per deployment. */
readonly baseURL?: string
}
}
export const ImageModelSchema = Schema.declare((value): value is ImageModel => value instanceof ImageModel, {
expected: "Image.Model",
})
@@ -107,13 +57,10 @@ export const ImageSize = Schema.declare<ImageSize>(
{ title: "ImageSize" },
)
export type ImageAspectRatio = `${number}:${number}`
export const ImageAspectRatio = Schema.declare<ImageAspectRatio>(
(value): value is ImageAspectRatio => typeof value === "string" && /^\d+(?:\.\d+)?:\d+(?:\.\d+)?$/.test(value),
{ title: "ImageAspectRatio" },
)
export type ImageAspectRatio = Media.AspectRatio
export const ImageAspectRatio = Media.AspectRatio
export type ImageFormat = "png" | "jpeg" | "webp" | (string & {})
export type ImageFormat = OpenString<"png" | "jpeg" | "webp">
export class ImageRequest extends Schema.Class<ImageRequest>("Image.Request")({
model: ImageModelSchema,
@@ -168,6 +115,13 @@ export class ImageResponse extends Schema.Class<ImageResponse>("Image.Response")
}
}
/** The `index`-th lower-fidelity preview of an image still generating; the `image` event supersedes it. */
export const ImagePartialEvent = Schema.Struct({
type: Schema.tag("image-partial"),
index: Schema.Number,
image: Media.AssetSchema,
}).annotate({ identifier: "Image.Event.Partial" })
export const ImageOutputEvent = Schema.Struct({
type: Schema.tag("image"),
index: Schema.Number,
@@ -181,24 +135,37 @@ export const ImageFinishEvent = Schema.Struct({
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "Image.Event.Finish" })
const imageEventTagged = Schema.Union([ImageOutputEvent, ImageFinishEvent]).pipe(Schema.toTaggedUnion("type"))
const imageEventTagged = Schema.Union([
QueuedEvent,
ProgressEvent,
ImagePartialEvent,
ImageOutputEvent,
ImageFinishEvent,
]).pipe(Schema.toTaggedUnion("type"))
export const ImageEvent = Object.assign(imageEventTagged, {
is: {
generationQueued: imageEventTagged.guards["generation-queued"],
generationProgress: imageEventTagged.guards["generation-progress"],
imagePartial: imageEventTagged.guards["image-partial"],
image: imageEventTagged.guards.image,
finish: imageEventTagged.guards.finish,
},
})
export type ImageEvent = Schema.Schema.Type<typeof imageEventTagged>
/** Inline routes produce every image at once; expand the response into the streaming event shape. */
export const responseEvents = (response: ImageResponse): ReadonlyArray<ImageEvent> => [
...response.images.map((image, index) => ImageOutputEvent.make({ index, image })),
ImageFinishEvent.make({
usage: response.usage,
notices: response.notices,
providerMetadata: response.providerMetadata,
}),
]
const collectResponse = (events: ReadonlyArray<ImageEvent>): Effect.Effect<ImageResponse> => {
const finish = events.find(ImageEvent.is.finish)
// Every image protocol's `finish` emits the terminal event or fails, so a completed stream always has one.
if (finish === undefined) return Effect.die(new Error("The image stream completed without a finish event"))
return Effect.succeed(
new ImageResponse({
images: events.filter(ImageEvent.is.image).map((event) => event.image),
usage: finish.usage,
notices: finish.notices,
providerMetadata: finish.providerMetadata,
}),
)
}
// ---------------------------------------------------------------------------
// Request-shaped call API
@@ -212,40 +179,43 @@ export function request(input: ImageRequest | ImageRequestInput) {
if (input instanceof ImageRequest) return input
return new ImageRequest({
...input,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
http: HttpOptions.make(input.http),
})
}
const requestEffect = (input: ImageRequest | ImageRequestInput) =>
Effect.try({
try: () => request(input),
catch: (error) =>
new AIError({
reason: new InvalidRequestError({
message: error instanceof Error ? error.message : String(error),
cause: error,
}),
}),
})
const requestEffect = (input: ImageRequest | ImageRequestInput) => tryRequest(() => request(input))
export function generate<const Model extends ImageModel>(
input: ImageRequestInput<Model>,
input: ImageRequest | ImageRequestInput<Model>,
options?: AwaitOptions,
): Effect.Effect<ImageResponse, AIError, Service>
export function generate(input: ImageRequest): Effect.Effect<ImageResponse, AIError, Service>
export function generate(input: ImageRequest | ImageRequestInput) {
return requestEffect(input).pipe(Effect.flatMap((request) => ImageClient.generate(request)))
export function generate(input: ImageRequest | ImageRequestInput, options?: AwaitOptions) {
return requestEffect(input).pipe(Effect.flatMap((request) => ImageClient.generate(request, options)))
}
export function stream<const Model extends ImageModel>(
input: ImageRequestInput<Model>,
input: ImageRequest | ImageRequestInput<Model>,
options?: AwaitOptions,
): Stream.Stream<ImageEvent, AIError, Service>
export function stream(input: ImageRequest): Stream.Stream<ImageEvent, AIError, Service>
export function stream(input: ImageRequest | ImageRequestInput) {
return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => ImageClient.stream(request))))
export function stream(input: ImageRequest | ImageRequestInput, options?: AwaitOptions) {
return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => ImageClient.stream(request, options))))
}
/** Inline and streaming routes fail with `UnsupportedOperation`. */
export function start<const Model extends ImageModel>(
input: ImageRequest | ImageRequestInput<Model>,
): Effect.Effect<Generation<ImageResponse>, AIError, Service>
export function start(input: ImageRequest | ImageRequestInput) {
return requestEffect(input).pipe(Effect.flatMap((request) => ImageClient.start(request)))
}
export const resume = (model: ImageModel, token: unknown): Effect.Effect<Generation<ImageResponse>, AIError, Service> =>
ImageClient.resume(model, token)
export const Image = {
request,
generate,
stream,
start,
resume,
} as const
+67 -2
View File
@@ -1,3 +1,4 @@
export { AIClient } from "./ai-client.js"
export { LLMClient } from "./route/client.js"
export { ImageClient } from "./image-client.js"
export { Auth } from "./route/auth.js"
@@ -8,7 +9,7 @@ export type {
RouteLanguageModelInput,
RouteRoutedLanguageModelInput,
Interface as LLMClientShape,
Service as LLMClientService,
LLMClientService,
} from "./route/client.js"
export * from "./schema/index.js"
export {
@@ -29,9 +30,73 @@ export type {
ImageRoute,
} from "./image.js"
export { Image } from "./image.js"
export { VideoClient } from "./video-client.js"
export {
VideoAspectRatio,
VideoEvent,
VideoFrames,
VideoModel,
VideoModelSchema,
VideoRequest,
VideoResponse,
} from "./video.js"
export type {
VideoModelOptions,
VideoOptions,
VideoRequestFor,
VideoRequestInput,
VideoResolution,
VideoRoute,
} from "./video.js"
export { Video } from "./video.js"
export { SpeechClient } from "./speech-client.js"
export {
SpeechEvent,
SpeechModel,
SpeechModelSchema,
SpeechRequest,
SpeechResponse,
SpeechTimestamp,
SpeechVoice,
} from "./speech.js"
export type {
SpeechFormat,
SpeechModelOptions,
SpeechOptions,
SpeechRequestFor,
SpeechRequestInput,
SpeechRoute,
} from "./speech.js"
export { Speech } from "./speech.js"
export { TranscriptionClient } from "./transcription-client.js"
export {
TranscriptionEvent,
TranscriptionModel,
TranscriptionModelSchema,
TranscriptionRequest,
TranscriptionResponse,
TranscriptionSegment,
TranscriptionTimestamps,
TranscriptionWord,
} from "./transcription.js"
export type {
TranscriptionModelOptions,
TranscriptionOptions,
TranscriptionRequestFor,
TranscriptionRequestInput,
TranscriptionRoute,
} from "./transcription.js"
export { Transcription } from "./transcription.js"
export { Media } from "./media.js"
export { Generation } from "./generation.js"
export type { Event as GenerationEvent, Poll, Route as GenerationRoute, Snapshot as GenerationSnapshot, Status as GenerationStatus } from "./generation.js"
export type {
AwaitOptions as GenerationAwaitOptions,
Event as GenerationEvent,
Poll,
Route as GenerationRoute,
Snapshot as GenerationSnapshot,
Status as GenerationStatus,
} from "./generation.js"
export { Tool, ToolFailure, toDefinitions } from "./tool.js"
export { ToolRuntime } from "./tool-runtime.js"
export type { DispatchResult as ToolDispatchResult, ToolSettlement } from "./tool-runtime.js"
+23 -5
View File
@@ -1,5 +1,6 @@
import { Effect, JsonSchema, Schema } from "effect"
import { LLMClient, Service } from "./route/client.js"
import { Effect, JsonSchema, Schema, Stream } from "effect"
import { tryRequest } from "./media-model.js"
import { LLMClient, Service, type StreamOptions } from "./route/client.js"
import {
GenerationOptions,
HttpOptions,
@@ -35,9 +36,26 @@ export type RequestInput<SelectedLanguageModel extends LanguageModel = LanguageM
readonly http?: HttpOptions.Input
}
export const generate = LLMClient.generate
export function generate<const Model extends LanguageModel>(
input: RequestInput<Model>,
options?: StreamOptions,
): Effect.Effect<LLMResponse, AIError, Service>
export function generate(input: LLMRequest, options?: StreamOptions): Effect.Effect<LLMResponse, AIError, Service>
export function generate(input: RequestInput | LLMRequest, options?: StreamOptions) {
return requestEffect(input).pipe(Effect.flatMap((request) => LLMClient.generate(request, options)))
}
export const stream = LLMClient.stream
export function stream<const Model extends LanguageModel>(
input: RequestInput<Model>,
options?: StreamOptions,
): Stream.Stream<LLMEvent, AIError, Service>
export function stream(input: LLMRequest, options?: StreamOptions): Stream.Stream<LLMEvent, AIError, Service>
export function stream(input: RequestInput | LLMRequest, options?: StreamOptions) {
return Stream.unwrap(requestEffect(input).pipe(Effect.map((request) => LLMClient.stream(request, options))))
}
const requestEffect = (input: RequestInput | LLMRequest) =>
input instanceof LLMRequest ? Effect.succeed(input) : tryRequest(() => request(input))
export const request = <const SelectedLanguageModel extends LanguageModel>(
input: RequestInput<SelectedLanguageModel>,
@@ -61,7 +79,7 @@ export const request = <const SelectedLanguageModel extends LanguageModel>(
toolChoice: requestToolChoice ? ToolChoice.make(requestToolChoice) : undefined,
generation: requestGeneration === undefined ? undefined : GenerationOptions.make(requestGeneration),
providerOptions: requestProviderOptions,
http: requestHttp === undefined ? undefined : HttpOptions.make(requestHttp),
http: HttpOptions.make(requestHttp),
})
}
+77
View File
@@ -0,0 +1,77 @@
import { type Context, Effect, Layer, Stream } from "effect"
import { resultEvents, type AwaitOptions, type Generation, type Observation } from "./generation.js"
import { RequestExecutor } from "./route/executor.js"
import type { MediaRoute } from "./route/media.js"
import { AIError, UnsupportedOperationError } from "./schema/index.js"
/** A media request whose model carries the route that executes it. */
export interface RoutedRequest<Self extends MediaRoute.MediaRequest, Event, Response> extends MediaRoute.MediaRequest {
readonly model: MediaRoute.MediaRequest["model"] & { readonly route: MediaRoute.AnyRoute<Self, Event, Response> }
}
/** `start` and `resume` fail with `UnsupportedOperation` on inline and stream routes. */
export interface Interface<Req extends RoutedRequest<Req, Event, Response>, Event, Response> {
readonly generate: (request: Req, options?: AwaitOptions) => Effect.Effect<Response, AIError>
readonly stream: (request: Req, options?: AwaitOptions) => Stream.Stream<Event | Observation, AIError>
readonly start: (request: Req) => Effect.Effect<Generation<Response>, AIError>
readonly resume: (model: Req["model"], token: unknown) => Effect.Effect<Generation<Response>, AIError>
}
/** One modality's layer and service accessors, dispatching each request on its route's `kind`. */
export const make = <Self, Req extends RoutedRequest<Req, Event, Response>, Event, Response>(
service: Context.Service<Self, Interface<Req, Event, Response>>,
input: {
readonly modality: string
/** A completed response expanded into the streaming event shape. */
readonly responseEvents: (response: Response) => ReadonlyArray<Event>
},
) => ({
layer: Layer.effect(
service,
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
const notQueued = (route: MediaRoute.AnyRoute<Req, Event, Response>, operation: string) =>
new AIError({
reason: new UnsupportedOperationError({
operation: `${input.modality}.${operation}`,
provider: route.provider,
route: route.id,
message: `${route.provider}/${route.id} is not a queued route; use generate or stream`,
}),
})
const start = (request: Req) => {
const route = request.model.route
if (route.kind !== "queued") return Effect.fail(notQueued(route, "start"))
return route.start(request, executor.execute)
}
return service.of({
start,
resume: (model, token) => {
if (model.route.kind !== "queued") return Effect.fail(notQueued(model.route, "resume"))
return model.route.resume(model, token, executor.execute)
},
generate: (request, options) => {
const route = request.model.route
if (route.kind !== "queued") return route.generate(request, executor.execute)
return start(request).pipe(Effect.flatMap((generation) => generation.await(options)))
},
stream: (request, options) => {
const route = request.model.route
if (route.kind === "stream") return route.stream(request, executor.execute)
if (route.kind === "queued")
return Stream.unwrap(
start(request).pipe(Effect.map((generation) => resultEvents(generation, input.responseEvents, options))),
)
return Stream.fromIterableEffect(Effect.map(route.generate(request, executor.execute), input.responseEvents))
},
})
}),
),
generate: (request: Req, options?: AwaitOptions) => service.use((client) => client.generate(request, options)),
stream: (request: Req, options?: AwaitOptions) =>
Stream.unwrap(service.useSync((client) => client.stream(request, options))),
start: (request: Req) => service.use((client) => client.start(request)),
resume: (model: Req["model"], token: unknown) => service.use((client) => client.resume(model, token)),
})
export * as MediaClient from "./media-client.js"
+110
View File
@@ -0,0 +1,110 @@
import { Effect } from "effect"
import { Endpoint } from "./route/endpoint.js"
import { MediaRoute } from "./route/media.js"
import type { MediaProtocol } from "./route/media-protocol.js"
import { AIError, HttpOptions, InvalidRequestError, ModelID, ProviderID } from "./schema/index.js"
/**
* What every media model carries: ids, the configured route, and deployment `http` overlays. Modality classes
* (`ImageModel`, `VideoModel`, `SpeechModel`, `TranscriptionModel`) extend it with their route type and a nominal
* marker so one cannot stand in for the other in requests.
*/
export class MediaModel<Route, Options> {
// As with `LanguageModel`, the route type is erased over `Options`; `fromRoute` and the constructor trust that the
// route accepts every request this model's `Options` admit.
declare protected readonly _Options: Options
readonly id: ModelID
readonly provider: ProviderID
readonly route: Route
readonly http?: HttpOptions
constructor(input: MediaModel.Input<Route>) {
this.id = ModelID.make(input.id)
this.provider = ProviderID.make(input.provider)
this.route = input.route
this.http = input.http
}
}
export namespace MediaModel {
export type Options = Record<string, unknown>
export interface Input<Route> {
readonly id: string | ModelID
readonly provider: string | ProviderID
readonly route: Route
readonly http?: HttpOptions
}
/** A protocol plus its canonical start path; `ModelInput.baseURL` overrides `baseURL` per deployment. */
export interface RouteInput<Request extends MediaRoute.MediaRequest, Protocol> {
readonly protocol: Protocol
readonly path: Endpoint.EndpointPart<MediaProtocol.Body, Request>
readonly baseURL?: string
/** Headers the protocol requires on every call, such as a pinned API version; deployment headers win. */
readonly headers?: Record<string, string>
}
export type InlineRouteInput<Request extends MediaRoute.MediaRequest, Response> = RouteInput<
Request,
MediaProtocol.Inline<Request, Response>
>
export type StreamRouteInput<Request extends MediaRoute.MediaRequest, Event, Frame, State> = RouteInput<
MediaProtocol.Addressed<Request>,
MediaProtocol.Streamed<Request, Event, Frame, State>
>
export type QueuedRouteInput<Request extends MediaRoute.MediaRequest, Response, Token> = RouteInput<
Request,
MediaProtocol.Queued<Request, Response, Token>
>
export type AnyRouteInput<Request extends MediaRoute.MediaRequest, Event, Response, Frame, State, Token> =
| InlineRouteInput<Request, Response>
| StreamRouteInput<Request, Event, Frame, State>
| QueuedRouteInput<Request, Response, Token>
}
/** Compose a protocol route input with one deployment through `MediaRoute.inline`, `queued`, or `stream`. */
export const composeRoute = <Request extends MediaRoute.MediaRequest, Event, Response, Frame, State, Token>(
route: MediaModel.AnyRouteInput<Request, Event, Response, Frame, State, Token>,
input: MediaRoute.ModelInput,
collect: (events: ReadonlyArray<Event>) => Effect.Effect<Response, AIError>,
): MediaRoute.AnyRoute<Request, Event, Response> => {
if (isStreamInput(route)) return MediaRoute.stream({ ...composition(route, input), collect })
if (isQueuedInput(route)) return MediaRoute.queued(composition(route, input))
return MediaRoute.inline(composition(route, input))
}
const composition = <Request extends MediaRoute.MediaRequest, Protocol>(
route: MediaModel.RouteInput<Request, Protocol>,
input: MediaRoute.ModelInput,
): MediaRoute.Composition<Request> & { readonly protocol: Protocol } => ({
protocol: route.protocol,
endpoint: Endpoint.path(route.path, { baseURL: input.baseURL ?? route.baseURL }),
auth: input.auth,
headers:
route.headers === undefined && input.headers === undefined ? undefined : { ...route.headers, ...input.headers },
})
const isStreamInput = <Request extends MediaRoute.MediaRequest, Event, Response, Frame, State, Token>(
route: MediaModel.AnyRouteInput<Request, Event, Response, Frame, State, Token>,
): route is MediaModel.StreamRouteInput<Request, Event, Frame, State> => route.protocol.kind === "stream"
const isQueuedInput = <Request extends MediaRoute.MediaRequest, Event, Response, Frame, State, Token>(
route: MediaModel.AnyRouteInput<Request, Event, Response, Frame, State, Token>,
): route is MediaModel.QueuedRouteInput<Request, Response, Token> => route.protocol.kind === "queued"
/** Lift a synchronous Schema-class constructor into a typed `InvalidRequest` failure. */
export const tryRequest = <A>(make: () => A): Effect.Effect<A, AIError> =>
Effect.try({
try: make,
catch: (error) =>
new AIError({
reason: new InvalidRequestError({
message: error instanceof Error ? error.message : String(error),
cause: error,
}),
}),
})
+25 -9
View File
@@ -6,7 +6,7 @@ import { ProviderID } from "./schema/ids.js"
import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "./schema/errors.js"
import { ProviderMetadata } from "./schema/options.js"
import { Service } from "./route/executor-service.js"
import { detectMediaType, extensionMediaType } from "./utils/media-type.js"
import { detectMediaType, fileMediaType } from "./utils/media-type.js"
export { detectMediaType } from "./utils/media-type.js"
@@ -34,8 +34,6 @@ const UrlSource = Schema.Struct({
mediaType: Schema.optional(Schema.String),
/** Epoch milliseconds after which the provider no longer serves the URL. */
expiresAt: Schema.optional(Schema.Number),
/** Headers required to fetch the URL, such as provider auth for Veo downloads. */
headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
})
/** A provider-side handle: OpenAI `file_id`, Gemini file URI, `gs://`, `runway://`, or a prior generation id. */
@@ -55,6 +53,12 @@ export type Source = Schema.Schema.Type<typeof Source>
// Kind, Info, Notice
// ---------------------------------------------------------------------------
export type AspectRatio = `${number}:${number}`
export const AspectRatio = Schema.declare<AspectRatio>(
(value): value is AspectRatio => typeof value === "string" && /^\d+(?:\.\d+)?:\d+(?:\.\d+)?$/.test(value),
{ title: "Media.AspectRatio" },
)
export const Kind = Schema.Literals(["image", "video", "audio", "document", "other"])
export type Kind = Schema.Schema.Type<typeof Kind>
@@ -110,6 +114,8 @@ export class Asset {
/** Epoch milliseconds after which a `url` source stops resolving. */
readonly expiresAt?: number
readonly providerMetadata?: ProviderMetadata
/** Transient download credentials for `url` sources; see `Asset.Input.headers`. */
readonly headers?: Record<string, string>
// Derived payload forms are cached on the instance because every protocol lowering re-reads the same payload. The
// cache is check-then-set (concurrent first reads of a `url` source may both download) and is never observable
@@ -127,6 +133,7 @@ export class Asset {
this.info = input.info
this.expiresAt = input.source.type === "url" ? input.source.expiresAt : undefined
this.providerMetadata = input.providerMetadata
this.headers = input.source.type === "url" ? input.headers : undefined
}
/** Inline payload without effects, for protocols that embed base64 or data URLs directly. */
@@ -151,7 +158,7 @@ export class Asset {
? Effect.fromResult(Encoding.decodeBase64(source.data)).pipe(
Effect.mapError((cause) => invalid(`Media asset contains invalid base64 data`, cause)),
)
: download(source)
: download(source, this.headers)
return decoded.pipe(Effect.tap((data) => Effect.sync(() => (this.#bytes = data))))
})
}
@@ -198,6 +205,12 @@ export namespace Asset {
readonly source: Source
readonly info?: Info
readonly providerMetadata?: ProviderMetadata
/**
* Headers required to download a `url` source, such as the provider API key Veo demands for its file URIs.
* They are runtime-only: never part of `source`, `toJSON()`, or `AssetSchema`, so a persisted asset cannot leak
* credentials and cannot be downloaded again after a round-trip. Call `materialize()` before persisting.
*/
readonly headers?: Record<string, string>
}
}
@@ -226,10 +239,13 @@ export const AssetSchema = AssetEncoded.pipe(
}),
)
const download = Effect.fn("Media.download")(function* (source: Extract<Source, { readonly type: "url" }>) {
const download = Effect.fn("Media.download")(function* (
source: Extract<Source, { readonly type: "url" }>,
headers: Record<string, string> | undefined,
) {
const executor = yield* Service
const response = yield* executor.execute(
HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(source.headers ?? {})),
HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(headers ?? {})),
)
const buffer = yield* response.arrayBuffer.pipe(
Effect.mapError(
@@ -264,8 +280,8 @@ export const url = (
value: string,
options?: AssetOptions & Omit<Extract<Source, { readonly type: "url" }>, "type" | "url">,
) => {
const { mediaType, expiresAt, headers, ...rest } = options ?? {}
return from({ type: "url", url: value, mediaType, expiresAt, headers }, rest)
const { mediaType, expiresAt, ...rest } = options ?? {}
return from({ type: "url", url: value, mediaType, expiresAt }, rest)
}
export const ref = (provider: string | ProviderID, id: string, mediaType?: string, options?: AssetOptions) =>
@@ -293,7 +309,7 @@ export const file = (path: string, options?: AssetOptions): Effect.Effect<Asset,
const data = yield* fs
.readFile(path)
.pipe(Effect.mapError((cause) => invalid(`Failed to read media file ${path}`, cause)))
return bytes(data, detectMediaType(data) ?? extensionMediaType(path), options)
return bytes(data, fileMediaType(data, path), options)
})
/** Materialize an asset and write its bytes through `FileSystem`. */
+142 -29
View File
@@ -1,18 +1,30 @@
import { Effect, Layer, ManagedRuntime, Stream } from "effect"
import { Image, ImageModel, ImageRequest, type ImageRequestInput } from "./image.js"
import { ImageClient } from "./image-client.js"
import { AIClient } from "./ai-client.js"
import type { AwaitOptions, Event, Generation, Snapshot } from "./generation.js"
import { Image, type ImageModel, type ImageRequest, type ImageRequestInput } from "./image.js"
import { LLM } from "./index.js"
import { LLMClient } from "./route/client.js"
import { Media } from "./media.js"
import { tryRequest } from "./media-model.js"
import { RequestExecutor } from "./route/executor.js"
import { LanguageModel, LLMRequest } from "./schema/index.js"
import { AIError, InvalidRequestError, LanguageModel, LLMRequest } from "./schema/index.js"
import type { RequestInput } from "./llm.js"
import { Speech, type SpeechModel, type SpeechRequest, type SpeechRequestInput } from "./speech.js"
import {
Transcription,
type TranscriptionModel,
type TranscriptionRequest,
type TranscriptionRequestInput,
} from "./transcription.js"
import { fileMediaType } from "./utils/media-type.js"
import { Video, type VideoModel, type VideoRequest, type VideoRequestInput } from "./video.js"
/**
* Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM and image clients
* over a request executor; every method runs the corresponding Effect API and rethrows `AIError` unchanged.
* Promise-first entrypoint for scripts and non-Effect callers. One `ManagedRuntime` hosts the LLM, image, video, speech,
* and transcription clients over a request executor; every method runs the corresponding Effect API and rethrows
* `AIError` unchanged. `file` and `write` load `node:fs/promises` on first use, so importing this module does not.
*/
export interface Options {
/** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or middleware here. */
/** Executor layer; defaults to `RequestExecutor.fetchLayer`. Inject a recorder or `RequestExecutor.middleware(fn)` here. */
readonly layer?: Layer.Layer<RequestExecutor.Service>
}
@@ -20,10 +32,23 @@ export interface RunOptions {
readonly signal?: AbortSignal
}
export type Services =
| Layer.Success<typeof LLMClient.layer>
| Layer.Success<typeof ImageClient.layer>
| RequestExecutor.Service
export type Services = AIClient.Services
/**
* Promise view of a `Generation`. Its fields are a snapshot taken when the handle was created; `refresh()` resolves to a
* new handle rather than updating this one.
*/
export type GenerationHandle<Response> = Snapshot & {
/** Serializable JSON; pass it back to `resume` from another process. */
readonly token: unknown
readonly await: (options?: AwaitOptions & RunOptions) => Promise<Response>
/** Status observations until the first terminal one, polling like `await`; abort ends iteration without throwing. */
readonly events: (options?: AwaitOptions & RunOptions) => AsyncIterable<Event>
/** The result without polling; fails when the generation has not completed. */
readonly result: (options?: RunOptions) => Promise<Response>
readonly refresh: (options?: RunOptions) => Promise<GenerationHandle<Response>>
readonly cancel: (options?: RunOptions) => Promise<void>
}
const abortEffect = (signal: AbortSignal | undefined) =>
signal === undefined
@@ -39,13 +64,9 @@ const abortEffect = (signal: AbortSignal | undefined) =>
})
export const make = (options: Options = {}) => {
const runtime = ManagedRuntime.make(
Layer.mergeAll(LLMClient.layer, ImageClient.layer).pipe(
Layer.provideMerge(options.layer ?? RequestExecutor.fetchLayer),
),
)
const runtime = ManagedRuntime.make(AIClient.layerWith(options.layer ?? RequestExecutor.fetchLayer))
/** Run any package Effect (for example `asset.bytes()`) inside this runtime. */
/** Run any package Effect (for example `LLMClient.compact(...)`) inside this runtime. */
const run = <A, E>(effect: Effect.Effect<A, E, Services>, options?: RunOptions) =>
runtime.runPromise(effect, { signal: options?.signal })
@@ -61,32 +82,121 @@ export const make = (options: Options = {}) => {
),
)
// The typed `generate`/`stream` overloads take a concrete input or a request, not the union; normalize once here.
const llmRequest = (input: RequestInput | LLMRequest) => (input instanceof LLMRequest ? input : LLM.request(input))
const imageRequest = (input: ImageRequestInput | ImageRequest) =>
input instanceof ImageRequest ? input : Image.request(input)
const handle = <Response>(generation: Generation<Response>): GenerationHandle<Response> => ({
...generation.snapshot,
token: generation.token,
await: (options) => run(generation.await({ poll: options?.poll }), options),
events: (options) => iterate(generation.events({ poll: options?.poll }), options),
result: (options) => run(generation.result(), options),
refresh: (options) => run(generation.refresh(), options).then(handle),
cancel: (options) => run(generation.cancel(), options),
})
const llmRequest = (input: RequestInput | LLMRequest) =>
input instanceof LLMRequest ? Effect.succeed(input) : tryRequest(() => LLM.request(input))
return {
run,
/** Decoded asset bytes, downloading `url` sources through the executor. */
bytes: (asset: Media.Asset, options?: RunOptions) => run(asset.bytes(), options),
base64: (asset: Media.Asset, options?: RunOptions) => run(asset.base64(), options),
/** Pull a `url` asset into owned bytes before the provider URL expires. */
materialize: (asset: Media.Asset, options?: RunOptions) => run(asset.materialize(), options),
/** Read a file into an asset like `Media.file`: sniffed media type, then the extension's. */
file: async (path: string, options?: Media.AssetOptions & RunOptions) => {
const { readFile } = await import("node:fs/promises")
return run(
Effect.tryPromise({
try: (signal) => readFile(path, { signal }),
catch: (cause) => fileError(`Failed to read media file ${path}`, cause),
}).pipe(
Effect.map((buffer) => {
const data = new Uint8Array(buffer)
return Media.bytes(data, fileMediaType(data, path), options)
}),
),
options,
)
},
/** Write an asset's bytes like `Media.write`, downloading `url` sources through the executor. */
write: async (asset: Media.Asset, path: string, options?: RunOptions) => {
const { writeFile } = await import("node:fs/promises")
return run(
asset.bytes().pipe(
Effect.flatMap((data) =>
Effect.tryPromise({
try: (signal) => writeFile(path, data, { signal }),
catch: (cause) => fileError(`Failed to write media file ${path}`, cause),
}),
),
),
options,
)
},
llm: {
request: LLM.request,
generate: <const Model extends LanguageModel>(
input: RequestInput<Model> | LLMRequest,
options?: RunOptions,
) => run(LLM.generate(llmRequest(input)), options),
generate: <const Model extends LanguageModel>(input: RequestInput<Model> | LLMRequest, options?: RunOptions) =>
run(Effect.flatMap(llmRequest(input), LLM.generate), options),
stream: <const Model extends LanguageModel>(input: RequestInput<Model> | LLMRequest, options?: RunOptions) =>
iterate(LLM.stream(llmRequest(input)), options),
iterate(Stream.unwrap(Effect.map(llmRequest(input), LLM.stream)), options),
},
image: {
request: Image.request,
generate: <const Model extends ImageModel>(
input: ImageRequestInput<Model> | ImageRequest,
options?: RunOptions,
) => run(Image.generate(imageRequest(input)), options),
options?: AwaitOptions & RunOptions,
) => run(Image.generate(input, { poll: options?.poll }), options),
stream: <const Model extends ImageModel>(
input: ImageRequestInput<Model> | ImageRequest,
options?: AwaitOptions & RunOptions,
) => iterate(Image.stream(input, { poll: options?.poll }), options),
start: <const Model extends ImageModel>(input: ImageRequestInput<Model> | ImageRequest, options?: RunOptions) =>
run(Image.start(input), options).then(handle),
resume: (model: ImageModel, token: unknown, options?: RunOptions) =>
run(Image.resume(model, token), options).then(handle),
},
video: {
request: Video.request,
start: <const Model extends VideoModel>(input: VideoRequestInput<Model> | VideoRequest, options?: RunOptions) =>
run(Video.start(input), options).then(handle),
generate: <const Model extends VideoModel>(
input: VideoRequestInput<Model> | VideoRequest,
options?: AwaitOptions & RunOptions,
) => run(Video.generate(input, { poll: options?.poll }), options),
resume: (model: VideoModel, token: unknown, options?: RunOptions) =>
run(Video.resume(model, token), options).then(handle),
stream: <const Model extends VideoModel>(
input: VideoRequestInput<Model> | VideoRequest,
options?: AwaitOptions & RunOptions,
) => iterate(Video.stream(input, { poll: options?.poll }), options),
},
speech: {
request: Speech.request,
generate: <const Model extends SpeechModel>(
input: SpeechRequestInput<Model> | SpeechRequest,
options?: RunOptions,
) => iterate(Image.stream(imageRequest(input)), options),
) => run(Speech.generate(input), options),
stream: <const Model extends SpeechModel>(
input: SpeechRequestInput<Model> | SpeechRequest,
options?: RunOptions,
) => iterate(Speech.stream(input), options),
},
transcription: {
request: Transcription.request,
generate: <const Model extends TranscriptionModel>(
input: TranscriptionRequestInput<Model> | TranscriptionRequest,
options?: AwaitOptions & RunOptions,
) => run(Transcription.generate(input, { poll: options?.poll }), options),
stream: <const Model extends TranscriptionModel>(
input: TranscriptionRequestInput<Model> | TranscriptionRequest,
options?: AwaitOptions & RunOptions,
) => iterate(Transcription.stream(input, { poll: options?.poll }), options),
start: <const Model extends TranscriptionModel>(
input: TranscriptionRequestInput<Model> | TranscriptionRequest,
options?: RunOptions,
) => run(Transcription.start(input), options).then(handle),
resume: (model: TranscriptionModel, token: unknown, options?: RunOptions) =>
run(Transcription.resume(model, token), options).then(handle),
},
dispose: () => runtime.dispose(),
}
@@ -94,6 +204,9 @@ export const make = (options: Options = {}) => {
export type Client = ReturnType<typeof make>
const fileError = (message: string, cause: unknown) =>
new AIError({ reason: new InvalidRequestError({ message, cause }) })
/** Default client over `RequestExecutor.fetchLayer` for scripts; the runtime builds its layer on first use. */
export const ai = make()
+5 -1
View File
@@ -70,7 +70,11 @@ export const protocol = Protocol.make({
return {
...(yield* OpenAIChat.protocol.body.from(req)),
enable_thinking: opts.enableThinking,
thinking_budget: opts.thinkingBudget,
// Alibaba also rejects an explicit budget that is not below `max_completion_tokens`.
thinking_budget:
opts.thinkingBudget === undefined
? undefined
: ProviderShared.fitThinkingBudget(opts.thinkingBudget, req.generation?.maxTokens),
preserve_thinking: opts.preserveThinking,
clear_thinking: opts.clearThinking,
thinking: opts.thinking,
@@ -26,18 +26,21 @@ export const protocol = Protocol.make({
from: Effect.fn("AlibabaMessages.fromRequest")(function* (req) {
const opts = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(Options))(req.providerOptions ?? {})
// Model Studio accepts enabled thinking without Anthropic's mandatory token budget.
const body = yield* AnthropicMessages.protocol.body.from(
LLMRequest.update(req, {
providerOptions: { ...req.providerOptions, thinking: undefined },
}),
)
const budget = opts.thinking?.budgetTokens ?? opts.thinking?.budget_tokens
return {
...(yield* AnthropicMessages.protocol.body.from(
LLMRequest.update(req, {
providerOptions: { ...req.providerOptions, thinking: undefined },
}),
)),
...body,
thinking:
opts.thinking === undefined
? undefined
: {
type: opts.thinking.type,
budget_tokens: opts.thinking.budgetTokens ?? opts.thinking.budget_tokens,
budget_tokens:
budget === undefined ? undefined : ProviderShared.fitThinkingBudget(budget, body.max_tokens),
},
}
}),
+16 -17
View File
@@ -18,7 +18,6 @@ import {
type CacheHint,
type FinishReasonDetails,
type FinishReason,
type JsonSchema,
type MediaPart,
type ProviderMetadata,
type ProviderOptions,
@@ -31,13 +30,13 @@ import { classifyProviderFailure } from "../provider-error.js"
import { effortUpdate, resolveEffortUpdates } from "../effort-updates.js"
import * as Cache from "./utils/cache.js"
import { Lifecycle } from "./utils/lifecycle.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
import { ToolStream } from "./utils/tool-stream.js"
const ADAPTER = "anthropic-messages"
export const DEFAULT_BASE_URL = "https://api.anthropic.com/v1"
export const PATH = "/messages"
export const DEFAULT_MAX_TOKENS = 32_000
const MIN_THINKING_BUDGET = 1_024
const DEFAULT_EFFORT = "high"
const SSE_EVENTS = new Set([
@@ -524,10 +523,10 @@ const redactedDataFromMetadata = (metadata: ProviderMetadata | undefined, key: s
return typeof provider.redactedData === "string" ? provider.redactedData : undefined
}
const lowerTool = (breakpoints: Cache.Breakpoints, tool: ToolDefinition, inputSchema: JsonSchema): AnthropicTool => ({
const lowerTool = (breakpoints: Cache.Breakpoints, tool: ToolDefinition): AnthropicTool => ({
name: tool.name,
description: tool.description,
input_schema: inputSchema,
input_schema: tool.inputSchema,
cache_control: cacheControl(breakpoints, tool.cache),
})
@@ -1027,6 +1026,15 @@ const applyThinkingBindingDefault = (model: LLMRequest["model"], thinking: Anthr
}
}
// Anthropic also requires an explicit thinking budget below `max_tokens` and at or above its minimum.
const fitThinking = (thinking: AnthropicThinking | undefined, maxTokens: number) =>
thinking?.type === "enabled"
? {
...thinking,
budget_tokens: ProviderShared.fitThinkingBudget(thinking.budget_tokens, maxTokens, MIN_THINKING_BUDGET),
}
: thinking
const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (request: LLMRequest) {
const options = yield* decodeOptions(request.providerOptions ?? {})
const management = options.contextManagement
@@ -1034,22 +1042,12 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
const format = outputConfig?.format ?? undefined
const updates = resolveEffortUpdates(request, options.effort ?? outputConfig?.effort ?? undefined)
const generation = request.generation
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
// Allocate the 4-breakpoint budget in invalidation order: tools → system →
// messages. Tools live highest in the cache hierarchy, so when callers
// over-mark we keep their tool hints and shed the message-tail ones first.
const breakpoints = Cache.newBreakpoints(ANTHROPIC_BREAKPOINT_CAP)
const flattened = ProviderShared.flattenToolRequest(updates.request)
const tools =
flattened.tools.length === 0
? undefined
: flattened.tools.map((tool) =>
lowerTool(
breakpoints,
tool,
ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility),
),
)
const tools = flattened.tools.length === 0 ? undefined : flattened.tools.map((tool) => lowerTool(breakpoints, tool))
// Anthropic rejects tool_choice when tools are absent; "none" is only meaningful with tools present.
const toolChoice = tools === undefined || !request.toolChoice ? undefined : yield* lowerToolChoice(request.toolChoice)
const systemParts = request.system.filter((part) => part.text.length > 0)
@@ -1069,6 +1067,7 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
}
const output_config =
updates.effort === undefined && format === undefined ? undefined : { effort: updates.effort, format }
const maxTokens = generation?.maxTokens ?? DEFAULT_MAX_TOKENS
const body = {
model: request.model.id,
system,
@@ -1076,12 +1075,12 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
tools,
tool_choice: toolChoice,
stream: true as const,
max_tokens: generation?.maxTokens ?? DEFAULT_MAX_TOKENS,
max_tokens: maxTokens,
temperature: generation?.temperature,
top_p: generation?.topP,
top_k: generation?.topK,
stop_sequences: generation?.stop,
thinking: applyThinkingBindingDefault(request.model, options.thinking),
thinking: applyThinkingBindingDefault(request.model, fitThinking(options.thinking, maxTokens)),
output_config,
// top-level passthrough per SDK MessageCreateParamsBase:4638,4643,4649,4654,4670
cache_control: options.cache_control ?? options.cacheControl,
@@ -0,0 +1,209 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "assemblyai-transcription", name: "AssemblyAI", provider: "assemblyai" })
export const DEFAULT_BASE_URL = "https://api.assemblyai.com"
export const PATH = "/v2/transcript"
export const UPLOAD_PATH = "/v2/upload"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type AssemblyAITranscriptionOptions = {
readonly keyterms_prompt?: ReadonlyArray<string>
readonly punctuate?: boolean
readonly format_text?: boolean
readonly disfluencies?: boolean
readonly filter_profanity?: boolean
readonly temperature?: number
readonly speaker_options?: { readonly min_speakers_expected?: number; readonly max_speakers_expected?: number }
readonly language_detection_options?: {
readonly expected_languages?: ReadonlyArray<string>
readonly fallback_language?: string
readonly code_switching?: boolean
}
readonly speech_models?: ReadonlyArray<OpenString<"universal-3-5-pro" | "universal-2">>
} & Record<string, unknown>
export type Request = TranscriptionRequestFor<AssemblyAITranscriptionOptions>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
export const Token = Schema.Struct({ transcriptID: Schema.String })
export type Token = Schema.Schema.Type<typeof Token>
const Upload = Schema.Struct({ upload_url: Schema.String })
/** Word and utterance times are milliseconds. */
const Transcript = Schema.Struct({
id: Schema.String,
status: Schema.String,
text: optionalNull(Schema.String),
words: optionalNull(
Schema.Array(
Schema.Struct({
text: Schema.String,
start: Schema.Number,
end: Schema.Number,
confidence: optionalNull(Schema.Number),
speaker: optionalNull(Schema.String),
}),
),
),
utterances: optionalNull(
Schema.Array(
Schema.Struct({
text: Schema.String,
start: Schema.Number,
end: Schema.Number,
speaker: optionalNull(Schema.String),
}),
),
),
language_code: optionalNull(Schema.String),
audio_duration: optionalNull(Schema.Number),
speech_model_used: optionalNull(Schema.String),
error: optionalNull(Schema.String),
})
const STATUS = {
queued: "queued",
processing: "running",
completed: "completed",
error: "failed",
} as const satisfies Record<string, Status>
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const decodeUpload = route.decodeJson(Upload)
/** `/v2/transcript` only takes a URL, so inline audio is uploaded to `/v2/upload` first. */
const prepare = Effect.fn("AssemblyAITranscription.prepare")(function* (request: Request, send: MediaProtocol.Send) {
if (request.audio.source.type !== "bytes" && request.audio.source.type !== "base64") return request
const audio = yield* MediaInput.inlineBytes(route.id, request.audio)
const uploaded = yield* send(UPLOAD_PATH, MediaProtocol.binary(audio, "application/octet-stream")).pipe(
Effect.flatMap(decodeUpload),
)
return { ...request, audio: Media.url(uploaded.value.upload_url, { mediaType: request.audio.mediaType }) }
})
const fromRequest = Effect.fn("AssemblyAITranscription.fromRequest")(function* (request: Request) {
const audio = yield* ProviderShared.mediaReference(request.audio, route.provider, route.name)
return MediaProtocol.json(
mergeJsonRecords(
{
audio_url: audio.value,
speech_models: [request.model.id],
language_code: request.language,
language_detection: request.language === undefined ? true : undefined,
prompt: request.prompt,
// Turn-level `utterances`, the only segments AssemblyAI returns, and `speakers_expected` require speaker labels.
speaker_labels:
request.diarize === true || request.timestamps === "segment" || request.speakers !== undefined
? true
: undefined,
speakers_expected: request.speakers,
},
request.providerOptions,
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeTranscript = route.decodeJson(Transcript)
const decodeStart = Effect.fn("AssemblyAITranscription.decodeStart")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* decodeTranscript(response)
const status = yield* MediaProtocol.status(STATUS, output.value.status, output)
return { token: { transcriptID: output.value.id }, snapshot: { id: output.value.id, status } }
})
const decodeStatus = Effect.fn("AssemblyAITranscription.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeTranscript(response)
const status = yield* MediaProtocol.status(STATUS, output.value.status, output)
return { id: context.token.transcriptID, status }
})
const seconds = (milliseconds: number) => milliseconds / 1000
const decodeResult = Effect.fn("AssemblyAITranscription.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeTranscript(response)
const transcript = output.value
const status = yield* MediaProtocol.status(STATUS, transcript.status, output)
const error = transcript.error ?? undefined
if (status === "failed")
return yield* output.ended("failed", `${route.name} transcription failed${error === undefined ? "" : `: ${error}`}`)
if (status !== "completed") return yield* output.pending(context.token.transcriptID)
const duration = transcript.audio_duration ?? undefined
return new TranscriptionResponse({
text: transcript.text ?? "",
segments: transcript.utterances?.map((utterance) => ({
text: utterance.text,
startSeconds: seconds(utterance.start),
endSeconds: seconds(utterance.end),
speaker: utterance.speaker ?? undefined,
})),
words: transcript.words?.map((word) => ({
text: word.text,
startSeconds: seconds(word.start),
endSeconds: seconds(word.end),
speaker: word.speaker ?? undefined,
confidence: word.confidence ?? undefined,
})),
language: transcript.language_code?.toLowerCase(),
durationSeconds: duration,
usage: duration === undefined ? undefined : { type: "seconds", seconds: duration },
providerMetadata: {
assemblyai: { transcriptId: transcript.id, speechModel: transcript.speech_model_used ?? undefined },
},
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
const transcriptPath = (token: Token) => `${PATH}/${token.transcriptID}`
export const protocol = MediaProtocol.queued<Request, TranscriptionResponse, Token>(route, {
token: Token,
start: { prepare, body: { from: fromRequest }, decode: decodeStart },
status: { path: transcriptPath, decode: decodeStatus },
result: { path: transcriptPath, decode: decodeResult },
})
export const model = (input: MediaRoute.ModelInput) =>
TranscriptionModel.fromRoute<AssemblyAITranscriptionOptions, Token>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
export const AssemblyAITranscription = {
protocol,
model,
} as const
+52 -25
View File
@@ -9,9 +9,8 @@ import {
type CacheHint,
type FinishReason,
type FinishReasonDetails,
type JsonSchema,
type LLMRequest,
type LanguageModelToolSchemaCompatibility,
type LanguageModel,
type ProviderMetadata,
type ReasoningPart,
type ToolCallPart,
@@ -26,8 +25,8 @@ import { BedrockCache } from "./utils/bedrock-cache.js"
import { BedrockMedia } from "./utils/bedrock-media.js"
import { Lifecycle } from "./utils/lifecycle.js"
import { MistralToolID } from "./utils/mistral-tool-id.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
import { ToolStream } from "./utils/tool-stream.js"
import { concatBytes } from "../utils/bytes.js"
const ADAPTER = "bedrock-converse"
@@ -220,22 +219,18 @@ type BedrockEvent = Schema.Schema.Type<typeof BedrockEvent>
// =============================================================================
// Request Lowering
// =============================================================================
const lowerToolSpec = (tool: ToolDefinition, inputSchema: JsonSchema): BedrockToolSpec => ({
const lowerToolSpec = (tool: ToolDefinition): BedrockToolSpec => ({
toolSpec: {
name: tool.name,
...(tool.description.trim().length > 0 ? { description: tool.description } : {}),
inputSchema: { json: inputSchema },
inputSchema: { json: tool.inputSchema },
},
})
const lowerTools = (
compatibility: LanguageModelToolSchemaCompatibility | undefined,
breakpoints: BedrockCache.Breakpoints,
tools: ReadonlyArray<ToolDefinition>,
): BedrockTool[] => {
const lowerTools = (breakpoints: BedrockCache.Breakpoints, tools: ReadonlyArray<ToolDefinition>): BedrockTool[] => {
const result: BedrockTool[] = []
for (const tool of tools) {
result.push(lowerToolSpec(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, compatibility)))
result.push(lowerToolSpec(tool))
const cachePoint = BedrockCache.block(breakpoints, tool.cache)
if (cachePoint) result.push(cachePoint)
}
@@ -429,17 +424,50 @@ const lowerSystem = (breakpoints: BedrockCache.Breakpoints, system: ReadonlyArra
return content.length === 0 ? undefined : content
}
// Nova 2 rejects `maxTokens` at high reasoning effort, where its output can exceed the field's maximum. Other models
// that take `reasoningConfig`, such as Grok on Bedrock, accept it.
const isNova2 = (model: LanguageModel) => /\bamazon\.nova-2-/.test(model.id)
const isHighReasoningEffort = Schema.is(
Schema.Struct({
additionalModelRequestFields: Schema.Struct({
reasoningConfig: Schema.Struct({ maxReasoningEffort: Schema.Literal("high") }),
}),
}),
)
const Options = Schema.Struct({
thinking: Schema.optional(Schema.Struct({ type: Schema.Literal("enabled"), budgetTokens: Schema.Number })),
})
export type OptionsInput = typeof Options.Type
const decodeOptions = ProviderShared.validateWith(Schema.decodeUnknownEffect(Options))
// Claude on Bedrock requires the thinking budget below `maxTokens`, with a minimum of 1,024.
const MIN_THINKING_BUDGET = 1_024
const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request: LLMRequest) {
const toolChoice = request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined
const flattened = ProviderShared.flattenToolRequest(request)
const generation = request.generation
const options = yield* decodeOptions(request.providerOptions ?? {})
const maxTokens =
isNova2(request.model) && isHighReasoningEffort(request.http?.body) ? undefined : generation?.maxTokens
const thinking =
options.thinking === undefined
? undefined
: {
type: "enabled",
budget_tokens: ProviderShared.fitThinkingBudget(
options.thinking.budgetTokens,
maxTokens,
MIN_THINKING_BUDGET,
),
}
// Bedrock-Claude shares Anthropic's 4-breakpoint cap. Spend the budget in
// tools → system → messages order to favour the highest-impact prefixes.
const breakpoints = BedrockCache.breakpoints(request.model.id)
const toolConfig = (() => {
if (flattened.tools.length === 0) return undefined
return {
tools: lowerTools(request.model.compatibility?.toolSchema, breakpoints, flattened.tools),
tools: lowerTools(breakpoints, flattened.tools),
// Converse has no native "none". Keep definitions stable for prompt
// caching and omit only the unsupported choice.
toolChoice,
@@ -454,14 +482,14 @@ const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request:
}
const inferenceConfig = (() => {
if (
generation?.maxTokens === undefined &&
maxTokens === undefined &&
generation?.temperature === undefined &&
generation?.topP === undefined &&
(generation?.stop === undefined || generation.stop.length === 0)
)
return undefined
return {
maxTokens: generation?.maxTokens,
maxTokens,
temperature: generation?.temperature,
topP: generation?.topP,
stopSequences: generation?.stop,
@@ -473,9 +501,15 @@ const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request:
system,
inferenceConfig,
toolConfig,
// Converse's base inferenceConfig has no topK; Anthropic/Nova accept it
// as a model-specific field, so it goes through additionalModelRequestFields.
additionalModelRequestFields: generation?.topK === undefined ? undefined : { top_k: generation.topK },
// Converse's base inferenceConfig has no topK or thinking; Anthropic/Nova accept them
// as model-specific fields, so they go through additionalModelRequestFields.
additionalModelRequestFields:
generation?.topK === undefined && thinking === undefined
? undefined
: {
...(generation?.topK === undefined ? {} : { top_k: generation.topK }),
...(thinking === undefined ? {} : { thinking }),
},
}
})
@@ -524,14 +558,7 @@ interface ParserState {
readonly reasoningRedactedContent: Readonly<Record<number, ReadonlyArray<Uint8Array>>>
}
const encodeRedactedContent = (chunks: ReadonlyArray<Uint8Array>) => {
const bytes = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.length, 0))
chunks.reduce((offset, chunk) => {
bytes.set(chunk, offset)
return offset + chunk.length
}, 0)
return Encoding.encodeBase64(bytes)
}
const encodeRedactedContent = (chunks: ReadonlyArray<Uint8Array>) => Encoding.encodeBase64(concatBytes(chunks))
const step = (state: ParserState, event: BedrockEvent) =>
Effect.gen(function* () {
+220
View File
@@ -0,0 +1,220 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import type { Status } from "../generation.js"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords } from "../schema/index.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "bfl-images", name: "Black Forest Labs", provider: "black-forest-labs" })
export const DEFAULT_BASE_URL = "https://api.bfl.ai"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type BlackForestLabsImageOptions = {
readonly safety_tolerance?: number
readonly prompt_upsampling?: boolean
readonly disable_pup?: boolean
readonly raw?: boolean
readonly guidance?: number
readonly steps?: number
} & Record<string, unknown>
export type Request = ImageRequestFor<BlackForestLabsImageOptions>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
/**
* Regional clusters answer on different hosts, so the returned `polling_url` is followed verbatim. BFL reports the
* credit cost on submit, so it rides on the token; it is optional so tokens persisted before it existed still decode.
*/
export const Token = Schema.Struct({
id: Schema.String,
pollingURL: Schema.String,
cost: Schema.optionalKey(Schema.Number),
})
export type Token = Schema.Schema.Type<typeof Token>
const StartResponse = Schema.Struct({
id: Schema.String,
polling_url: Schema.String,
cost: optionalNull(Schema.Number),
})
const Result = Schema.Struct({
id: Schema.String,
status: Schema.String,
result: optionalNull(
Schema.StructWithRest(
Schema.Struct({ sample: Schema.String, seed: optionalNull(Schema.Number), prompt: optionalNull(Schema.String) }),
[Schema.Record(Schema.String, Schema.Unknown)],
),
),
cost: optionalNull(Schema.Number),
})
const STATUS = {
Pending: "running",
Reasoning: "running",
Generating: "running",
Ready: "completed",
Error: "failed",
// Moderation is terminal; `decodeResult` reports it as a content-policy failure.
"Content Moderated": "failed",
"Request Moderated": "failed",
"Task not found": "expired",
} as const satisfies Record<string, Status>
const isModerated = (status: string) => status === "Content Moderated" || status === "Request Moderated"
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
interface Capabilities {
readonly sizing: "dimensions" | "aspectRatio" | "none"
/** `input_image` numbers extra references `input_image_2`…; the other fields take one image. */
readonly imageField: "input_image" | "image_prompt" | "image"
readonly maxImages: number
readonly mask: boolean
}
const capabilities = (model: string): Capabilities => {
if (model.startsWith("flux-pro-1.0-fill")) return { sizing: "none", imageField: "image", maxImages: 1, mask: true }
if (model.startsWith("flux-pro-1.0-expand")) return { sizing: "none", imageField: "image", maxImages: 1, mask: false }
if (model.startsWith("flux-kontext"))
return { sizing: "aspectRatio", imageField: "input_image", maxImages: 4, mask: false }
if (model.startsWith("flux-pro-1.1-ultra"))
return { sizing: "aspectRatio", imageField: "image_prompt", maxImages: 1, mask: false }
if (model.startsWith("flux-pro-1.1") || model.startsWith("flux-dev"))
return { sizing: "dimensions", imageField: "image_prompt", maxImages: 1, mask: false }
if (model.startsWith("flux-2-klein"))
return { sizing: "dimensions", imageField: "input_image", maxImages: 4, mask: false }
return { sizing: "dimensions", imageField: "input_image", maxImages: 8, mask: false }
}
const validate = (request: Request, model: Capabilities) => {
const id = request.model.id
const images = request.images?.length ?? 0
if (request.n !== undefined && request.n > 1)
return Effect.fail(route.unsupported("media.n", `${id} generates one image per request; call it once per image`))
if (request.size !== undefined && model.sizing !== "dimensions")
return Effect.fail(route.unsupported("media.size", `${id} does not take size (width and height)`))
if (request.aspectRatio !== undefined && model.sizing !== "aspectRatio")
return Effect.fail(route.unsupported("media.aspectRatio", `${id} does not take aspectRatio`))
if (images > model.maxImages)
return Effect.fail(route.unsupported("media.images", `${id} takes at most ${model.maxImages} images`))
if (request.mask !== undefined && !model.mask)
return Effect.fail(route.unsupported("media.mask", `${id} does not inpaint; use flux-pro-1.0-fill`))
return Effect.void
}
const imageInput = (asset: Media.Asset) => {
const value = asset.inline()?.base64 ?? ProviderShared.mediaUrl(asset)
if (value === undefined)
return Effect.fail(ProviderShared.invalidRequest(`${route.name} accepts inline images or https URLs`))
return Effect.succeed(value)
}
const fromRequest = Effect.fn("BlackForestLabsImages.fromRequest")(function* (request: Request) {
const model = capabilities(request.model.id)
yield* validate(request, model)
const images = yield* Effect.forEach(request.images ?? [], imageInput)
const fields = images.map((image, index) => [
index === 0 ? model.imageField : `${model.imageField}_${index + 1}`,
image,
])
return MediaProtocol.json(
mergeJsonRecords(
{
prompt: request.prompt,
...(request.size === undefined ? {} : MediaInput.dimensions(request.size)),
aspect_ratio: request.aspectRatio,
seed: request.seed,
output_format: request.format,
mask: request.mask === undefined ? undefined : yield* imageInput(request.mask),
...Object.fromEntries(fields),
},
request.providerOptions,
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
token: {
id: value.id,
pollingURL: value.polling_url,
...(value.cost === undefined || value.cost === null ? {} : { cost: value.cost }),
},
snapshot: { id: value.id, status: "queued" },
}))
const decodeDocument = route.decodeJson(Result)
const decodeStatus = Effect.fn("BlackForestLabsImages.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeDocument(response)
return { id: context.token.id, status: yield* MediaProtocol.status(STATUS, output.value.status, output) }
})
const decodeResult = Effect.fn("BlackForestLabsImages.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeDocument(response)
const document = output.value
const status = yield* MediaProtocol.status(STATUS, document.status, output)
if (isModerated(document.status)) return yield* output.contentPolicy(`${route.name} moderated the generation`)
if (status === "failed" || status === "expired")
return yield* output.ended(status, `${route.name} generation ${context.token.id} ended with ${document.status}`)
if (status !== "completed") return yield* output.pending(context.token.id)
if (document.result === undefined || document.result === null)
return yield* output.invalid(`${route.name} generation ${context.token.id} has no result`)
const { sample, seed, prompt, ...rest } = document.result
// A settled `cost` on the result supersedes the submit-time cost carried on the token.
const cost = document.cost ?? context.token.cost
return new ImageResponse({
// `sample` is a signed URL that expires 10 minutes after the result is ready, so it is downloaded now.
images: [yield* context.materialize(Media.url(sample))],
usage: cost === undefined ? undefined : { type: "credits", credits: cost },
providerMetadata: {
bfl: { id: context.token.id, seed: seed ?? undefined, prompt: prompt ?? undefined, ...rest },
},
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.queued<Request, ImageResponse, Token>(route, {
token: Token,
start: { body: { from: fromRequest }, decode: decodeStart },
status: { path: (token) => token.pollingURL, decode: decodeStatus },
result: { path: (token) => token.pollingURL, decode: decodeResult },
})
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<BlackForestLabsImageOptions, Token>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => `/v1/${request.model.id}` },
input,
)
export const BlackForestLabsImages = {
protocol,
model,
} as const
@@ -0,0 +1,185 @@
import { Effect, Schema } from "effect"
import { classifyProviderFailure } from "../provider-error.js"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { AIError, mergeJsonRecords, type OpenString } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { SpeechStream } from "./utils/speech-stream.js"
const route = MediaProtocol.identity({ id: "cartesia-speech", name: "Cartesia", provider: "cartesia" })
export const DEFAULT_BASE_URL = "https://api.cartesia.ai"
export const API_VERSION = "2026-08-14"
export const BYTES_PATH = "/tts/bytes"
export const SSE_PATH = "/tts/sse"
const DEFAULT_SAMPLE_RATE = 44100
const DEFAULT_BIT_RATE = 128000
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type CartesiaEncoding = SpeechStream.PcmEncoding
export type CartesiaSpeechOptions = {
readonly sampleRate?: 8000 | 16000 | 22050 | 24000 | 44100 | 48000
readonly bitRate?: 32000 | 64000 | 96000 | 128000 | 192000
readonly encoding?: CartesiaEncoding
readonly generation_config?: {
readonly volume?: number
readonly emotion?: OpenString<"neutral" | "calm" | "angry" | "content" | "sad" | "scared">
}
readonly pronunciation_dict_id?: string
} & Record<string, unknown>
export type Request = SpeechRequestFor<CartesiaSpeechOptions>
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
/** `phoneme_timestamps` and future record types are ignored. */
const SseEvent = Schema.Struct({
type: Schema.String,
data: Schema.optional(Schema.Uint8ArrayFromBase64),
word_timestamps: Schema.optional(
Schema.Struct({
words: Schema.Array(Schema.String),
start: Schema.Array(Schema.Number),
end: Schema.Array(Schema.Number),
}),
),
status_code: Schema.optional(Schema.Number),
title: Schema.optional(Schema.String),
message: Schema.optional(Schema.String),
error_code: optionalNull(Schema.String),
})
const decodeEvent = route.decodeFrame(SseEvent)
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
interface State extends SpeechStream.Audio {
readonly done: boolean
}
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
/** Timestamps exist only on the SSE endpoint, so a `generate` that asks for them collects an SSE stream. */
const usesSse = (request: MediaProtocol.Addressed<Request>) => request.mode === "stream" || request.timestamps === true
const CONTAINERS: Readonly<Record<string, "raw" | "wav" | "mp3">> = { pcm: "raw", wav: "wav", mp3: "mp3" }
const outputFormat = Effect.fn("CartesiaSpeech.outputFormat")(function* (request: MediaProtocol.Addressed<Request>) {
const sse = usesSse(request)
const format = request.format ?? (sse ? "pcm" : "mp3")
const container = CONTAINERS[format]
if (container === undefined)
return yield* route.unsupported(
"media.format",
`${route.name} supports the pcm, wav, and mp3 formats, not "${format}"`,
)
if (sse && container !== "raw")
return yield* route.unsupported(
"media.format",
`${route.name} streams and timestamps only raw PCM; request format "pcm" instead of "${format}"`,
)
const sampleRate = request.providerOptions?.sampleRate ?? DEFAULT_SAMPLE_RATE
if (container === "mp3")
return { container, sample_rate: sampleRate, bit_rate: request.providerOptions?.bitRate ?? DEFAULT_BIT_RATE }
return { container, encoding: request.providerOptions?.encoding ?? "pcm_s16le", sample_rate: sampleRate }
})
const fromRequest = Effect.fn("CartesiaSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
const voice = SpeechStream.voiceID(request.voice)
if (voice === undefined)
return yield* ProviderShared.invalidRequest(`${route.name} requires a voice id; pass it as \`voice\``)
const { sampleRate: _sampleRate, bitRate: _bitRate, encoding: _encoding, ...native } = request.providerOptions ?? {}
return MediaProtocol.json(
mergeJsonRecords(
{
model_id: request.model.id,
transcript: request.text,
voice,
output_format: yield* outputFormat(request),
language: request.language,
generation_config: request.speed === undefined ? undefined : { speed: request.speed },
add_timestamps: request.timestamps === true ? true : undefined,
},
native,
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const onEvent = Effect.fn("CartesiaSpeech.onEvent")(function* (state: State, frame: string) {
const event = yield* decodeEvent(frame)
if (event.type === "chunk" && event.data !== undefined) return SpeechStream.delta(state, event.data)
if (event.type === "timestamps" && event.word_timestamps !== undefined) {
const words = event.word_timestamps
return [state, SpeechStream.timestamps(words.words, words.start, words.end)] as const
}
if (event.type === "done") return [{ ...state, done: true }, []] as const
if (event.type === "error")
return yield* new AIError({
reason: classifyProviderFailure({
message: `${route.name} stream failed${event.title === undefined ? "" : ` (${event.title})`}: ${event.message ?? "unknown error"}`,
status: event.status_code,
rawBody: frame,
}),
})
return [state, []] as const
})
const finish = Effect.fn("CartesiaSpeech.finish")(function* (
state: State,
context: MediaProtocol.ResponseContext<Request>,
) {
if (usesSse(context.request) && !state.done) return yield* route.incomplete()
const format = yield* outputFormat(context.request)
return yield* SpeechStream.finish(
route,
state,
format.container === "raw"
? SpeechStream.pcm(format.encoding, format.sample_rate)
: SpeechStream.container(format.container, format.sample_rate),
)
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>(route, {
unsupported: ["instructions"],
body: { from: fromRequest },
frames: (bytes, context) => (usesSse(context.request) ? Framing.sse.frame(bytes) : bytes),
initial: () => ({ chunks: [], done: false }),
step: SpeechStream.step(onEvent),
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<CartesiaSpeechOptions, string | Uint8Array, State>(
{
protocol,
baseURL: DEFAULT_BASE_URL,
headers: { "Cartesia-Version": API_VERSION },
path: ({ request }) => (usesSse(request) ? SSE_PATH : BYTES_PATH),
},
input,
)
export const CartesiaSpeech = {
protocol,
model,
} as const
@@ -0,0 +1,140 @@
import { Effect } from "effect"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { MediaInput } from "./utils/media-input.js"
import { SpeechStream } from "./utils/speech-stream.js"
const route = MediaProtocol.identity({ id: "deepgram-speech", name: "Deepgram", provider: "deepgram" })
export const DEFAULT_BASE_URL = "https://api.deepgram.com"
export const PATH = "/v1/speak"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type DeepgramEncoding = OpenString<"linear16" | "mulaw" | "alaw" | "mp3" | "opus" | "flac" | "aac">
export type DeepgramSpeechOptions = {
readonly encoding?: DeepgramEncoding
readonly container?: OpenString<"wav" | "ogg" | "none">
readonly sampleRate?: number
readonly bitRate?: number
readonly mip_opt_out?: boolean
readonly tag?: string
} & Record<string, unknown>
export type Request = SpeechRequestFor<DeepgramSpeechOptions>
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
type State = SpeechStream.Audio
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const FORMATS: Readonly<Record<string, { readonly encoding: string; readonly container?: string }>> = {
mp3: { encoding: "mp3" },
wav: { encoding: "linear16", container: "wav" },
pcm: { encoding: "linear16", container: "none" },
opus: { encoding: "opus" },
flac: { encoding: "flac" },
aac: { encoding: "aac" },
}
const audioFormat = (request: Request) => {
const format = request.format === undefined ? undefined : FORMATS[request.format]
return {
encoding: request.providerOptions?.encoding ?? format?.encoding,
container: request.providerOptions?.container ?? format?.container,
}
}
const queryParameters = (request: Request) => {
const { encoding: _encoding, container: _container, sampleRate, bitRate, ...native } = request.providerOptions ?? {}
return MediaInput.query(route.id, {
...native,
model: request.model.id,
...audioFormat(request),
sample_rate: sampleRate,
bit_rate: bitRate,
speed: request.speed,
})
}
const fromRequest = Effect.fn("DeepgramSpeech.fromRequest")(function* (request: Request) {
// Not in `unsupported`: that list would also reject `timestamps: false`, which asks for nothing.
if (request.timestamps === true)
return yield* route.unsupported("media.timestamps", `${route.name} does not return timestamps`)
if (
request.format !== undefined &&
FORMATS[request.format] === undefined &&
request.providerOptions?.encoding === undefined
)
return yield* route.unsupported(
"media.format",
`${route.name} has no encoding for format "${request.format}"; pass providerOptions.encoding`,
)
return MediaProtocol.json(
mergeJsonRecords({ text: request.text }, request.http?.body) ?? {},
yield* queryParameters(request),
)
})
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const HEADERLESS_ENCODINGS: Readonly<Record<string, SpeechStream.PcmEncoding>> = {
linear16: "pcm_s16le",
mulaw: "pcm_mulaw",
alaw: "pcm_alaw",
}
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
const headers = context.http.headers
const mediaType = headers["content-type"]
const format = audioFormat(context.request)
const encoding = HEADERLESS_ENCODINGS[format.encoding ?? ""]
const requestID = headers["dg-request-id"]
const modelName = headers["dg-model-name"]
return SpeechStream.finish(route, state, {
...(format.container === "none" && encoding !== undefined
? SpeechStream.pcm(encoding, SpeechStream.sampleRate(mediaType), mediaType)
: // Deepgram's default encoding is MP3; WAV is a container around any encoding.
{ mediaType, info: { format: format.container === "wav" ? "wav" : (format.encoding ?? "mp3") } }),
usage: SpeechStream.headerUsage("characters", headers["dg-char-count"]),
providerMetadata:
requestID === undefined && modelName === undefined
? undefined
: { deepgram: { requestId: requestID, modelName } },
})
}
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, Uint8Array, State>(route, {
unsupported: ["voice", "language", "instructions"],
body: { from: fromRequest },
frames: (bytes) => bytes,
initial: () => ({ chunks: [] }),
step: (state, frame) => Effect.succeed(SpeechStream.delta(state, frame)),
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<DeepgramSpeechOptions, Uint8Array, State>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
export const DeepgramSpeech = {
protocol,
model,
} as const
@@ -0,0 +1,186 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { TranscriptionModel, TranscriptionResponse, type TranscriptionRequestFor } from "../transcription.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "deepgram-transcription", name: "Deepgram", provider: "deepgram" })
export const DEFAULT_BASE_URL = "https://api.deepgram.com"
export const PATH = "/v1/listen"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type DeepgramTranscriptionOptions = {
readonly smart_format?: boolean
readonly punctuate?: boolean
readonly paragraphs?: boolean
readonly utterances?: boolean
readonly detect_language?: boolean | ReadonlyArray<string>
readonly keyterm?: ReadonlyArray<string>
readonly diarize_model?: OpenString<"latest" | "v1" | "v2">
readonly filler_words?: boolean
readonly numerals?: boolean
readonly mip_opt_out?: boolean
readonly tag?: string | ReadonlyArray<string>
} & Record<string, unknown>
export type Request = TranscriptionRequestFor<DeepgramTranscriptionOptions>
// ---------------------------------------------------------------------------
// 2. Response schema
// ---------------------------------------------------------------------------
const Word = Schema.Struct({
word: Schema.String,
start: Schema.Number,
end: Schema.Number,
confidence: Schema.optional(Schema.Number),
speaker: Schema.optional(Schema.Number),
punctuated_word: Schema.optional(Schema.String),
})
const ListenResponse = Schema.Struct({
metadata: Schema.optional(
Schema.Struct({ request_id: Schema.optional(Schema.String), duration: Schema.optional(Schema.Number) }),
),
results: Schema.Struct({
channels: Schema.Array(
Schema.Struct({
alternatives: Schema.optional(
Schema.Array(Schema.Struct({ transcript: Schema.String, words: Schema.optional(Schema.Array(Word)) })),
),
detected_language: Schema.optional(Schema.String),
}),
),
utterances: Schema.optional(
Schema.Array(
Schema.Struct({
start: Schema.Number,
end: Schema.Number,
transcript: Schema.String,
speaker: Schema.optional(Schema.Number),
words: Schema.optional(Schema.Array(Word)),
}),
),
),
}),
})
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const query = (request: Request) =>
MediaInput.query(
route.id,
mergeJsonRecords(
{
model: request.model.id,
smart_format: true,
language: request.language,
// Deepgram assumes English unless asked to detect, unlike the other routes' auto-detection.
detect_language: request.language === undefined ? true : undefined,
// `diarize=true` is deprecated in favor of choosing a diarization model.
diarize_model: request.diarize === true ? "latest" : undefined,
utterances: request.diarize === true || request.timestamps === "segment" ? true : undefined,
},
request.providerOptions,
) ?? {},
)
const fromRequest = Effect.fn("DeepgramTranscription.fromRequest")(function* (request: Request) {
const url = ProviderShared.mediaUrl(request.audio)
if (url !== undefined)
return MediaProtocol.json(mergeJsonRecords({ url }, request.http?.body) ?? {}, yield* query(request))
if (request.http?.body !== undefined)
return yield* ProviderShared.invalidRequest(
`${route.name} sends inline audio as the raw body, so http.body cannot apply`,
)
const audio = yield* MediaInput.inlineBytes(route.id, request.audio)
return MediaProtocol.binary(audio, request.audio.mediaType, yield* query(request))
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeListen = route.decodeJson(ListenResponse)
const speaker = (value: number | undefined) => (value === undefined ? undefined : String(value))
const wordText = (word: typeof Word.Type) => word.punctuated_word ?? word.word
// Utterances split on pauses, not speakers: the v2 diarizer labels a whole utterance with one speaker even when its
// words change speaker, so segments split each utterance at speaker changes.
const speakerTurns = (words: ReadonlyArray<typeof Word.Type>) =>
words.reduce<Array<Array<typeof Word.Type>>>((turns, word) => {
const last = turns.at(-1)
if (last === undefined || last[0].speaker !== word.speaker) return [...turns, [word]]
last.push(word)
return turns
}, [])
const decodeResponse = Effect.fn("DeepgramTranscription.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* decodeListen(response)
const channel = output.value.results.channels[0]
const alternative = channel?.alternatives?.[0]
if (alternative === undefined) return yield* output.invalid(`${route.name} returned no transcript`)
const duration = output.value.metadata?.duration
const requestID = output.value.metadata?.request_id
return new TranscriptionResponse({
text: alternative.transcript,
segments: output.value.results.utterances?.flatMap((utterance) =>
utterance.words === undefined || utterance.words.length === 0
? [
{
text: utterance.transcript,
startSeconds: utterance.start,
endSeconds: utterance.end,
speaker: speaker(utterance.speaker),
},
]
: speakerTurns(utterance.words).map((turn) => ({
text: turn.map(wordText).join(" "),
startSeconds: turn[0].start,
endSeconds: turn[turn.length - 1].end,
speaker: speaker(turn[0].speaker),
})),
),
words: alternative.words?.map((word) => ({
text: wordText(word),
startSeconds: word.start,
endSeconds: word.end,
speaker: speaker(word.speaker),
confidence: word.confidence,
})),
language: channel?.detected_language?.toLowerCase(),
durationSeconds: duration,
usage: duration === undefined ? undefined : { type: "seconds", seconds: duration },
providerMetadata: requestID === undefined ? undefined : { deepgram: { requestId: requestID } },
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, TranscriptionResponse>(route, {
unsupported: ["prompt", "speakers"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput) =>
TranscriptionModel.fromRoute<DeepgramTranscriptionOptions>({ protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, input)
export const DeepgramTranscription = {
protocol,
model,
} as const
@@ -0,0 +1,210 @@
import { Effect, Schema } from "effect"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { SpeechStream } from "./utils/speech-stream.js"
const route = MediaProtocol.identity({ id: "elevenlabs-speech", name: "ElevenLabs", provider: "elevenlabs" })
export const DEFAULT_BASE_URL = "https://api.elevenlabs.io"
export const PATH = "/v1/text-to-speech"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type ElevenLabsOutputFormat = OpenString<
| "mp3_22050_32"
| "mp3_24000_48"
| "mp3_44100_32"
| "mp3_44100_64"
| "mp3_44100_96"
| "mp3_44100_128"
| "mp3_44100_192"
| "pcm_8000"
| "pcm_16000"
| "pcm_22050"
| "pcm_24000"
| "pcm_32000"
| "pcm_44100"
| "pcm_48000"
| "wav_8000"
| "wav_16000"
| "wav_22050"
| "wav_24000"
| "wav_32000"
| "wav_44100"
| "wav_48000"
| "ulaw_8000"
| "alaw_8000"
| "opus_48000_32"
| "opus_48000_64"
| "opus_48000_96"
| "opus_48000_128"
| "opus_48000_192"
>
export type ElevenLabsSpeechOptions = {
readonly outputFormat?: ElevenLabsOutputFormat
readonly voice_settings?: {
readonly stability?: number
readonly similarity_boost?: number
readonly style?: number
readonly use_speaker_boost?: boolean
}
readonly seed?: number
readonly apply_text_normalization?: OpenString<"auto" | "on" | "off">
} & Record<string, unknown>
export type Request = SpeechRequestFor<ElevenLabsSpeechOptions>
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
const Alignment = Schema.Struct({
characters: Schema.Array(Schema.String),
character_start_times_seconds: Schema.Array(Schema.Number),
character_end_times_seconds: Schema.Array(Schema.Number),
})
const TimestampedAudio = Schema.Struct({
audio_base64: Schema.Uint8ArrayFromBase64,
alignment: optionalNull(Alignment),
})
const decodeRecord = route.decodeFrame(TimestampedAudio)
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
type State = SpeechStream.Audio
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const OUTPUT_FORMATS: Readonly<Record<string, string>> = {
mp3: "mp3_44100_128",
pcm: "pcm_24000",
wav: "wav_24000",
opus: "opus_48000_64",
}
/** WAV is served only by the non-streaming endpoints. */
const outputFormat = Effect.fn("ElevenLabsSpeech.outputFormat")(function* (request: MediaProtocol.Addressed<Request>) {
const format = request.providerOptions?.outputFormat ?? OUTPUT_FORMATS[request.format ?? "mp3"]
if (format === undefined)
return yield* route.unsupported(
"media.format",
`${route.name} has no default output format for "${request.format}"; pass providerOptions.outputFormat`,
)
if (request.mode === "stream" && format.startsWith("wav_"))
return yield* route.unsupported(
"media.format",
`${route.name} streams mp3, pcm, opus, ulaw, and alaw but not "${format}"; use generate for WAV`,
)
return format
})
const fromRequest = Effect.fn("ElevenLabsSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
if (request.voice === undefined)
return yield* ProviderShared.invalidRequest(`${route.name} requires a voice id; pass it as \`voice\``)
const { outputFormat: _outputFormat, ...native } = request.providerOptions ?? {}
return MediaProtocol.json(
mergeJsonRecords(
{
text: request.text,
model_id: request.model.id,
language_code: request.language,
voice_settings: request.speed === undefined ? undefined : { speed: request.speed },
},
native,
request.http?.body,
) ?? {},
{ output_format: yield* outputFormat(request) },
)
})
const path = (request: MediaProtocol.Addressed<Request>) =>
`${PATH}/${encodeURIComponent(SpeechStream.voiceID(request.voice) ?? "")}${request.mode === "stream" ? "/stream" : ""}${
request.timestamps === true ? "/with-timestamps" : ""
}`
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const onRecord = Effect.fn("ElevenLabsSpeech.onRecord")(function* (state: State, frame: string) {
const record = yield* decodeRecord(frame)
const [next, events] = SpeechStream.delta(state, record.audio_base64)
const alignment = record.alignment
if (!alignment) return [next, events] as const
return [
next,
[
...events,
...SpeechStream.timestamps(
alignment.characters,
alignment.character_start_times_seconds,
alignment.character_end_times_seconds,
),
],
] as const
})
const PCM_CODECS: Readonly<Record<string, SpeechStream.PcmEncoding>> = {
pcm: "pcm_s16le",
ulaw: "pcm_mulaw",
alaw: "pcm_alaw",
}
const describeOutput = (format: string) => {
const [codec = format, rate] = format.split("_")
const sampleRate = rate === undefined ? undefined : Number(rate)
const encoding = PCM_CODECS[codec]
return encoding === undefined ? SpeechStream.container(codec, sampleRate) : SpeechStream.pcm(encoding, sampleRate)
}
const finish = Effect.fn("ElevenLabsSpeech.finish")(function* (
state: State,
context: MediaProtocol.ResponseContext<Request>,
) {
const requestID = context.http.headers["request-id"]
return yield* SpeechStream.finish(route, state, {
...describeOutput(yield* outputFormat(context.request)),
// `character-cost` is billed credits, not a character count (3 for 20 characters on `eleven_flash_v2_5`).
usage: SpeechStream.headerUsage("credits", context.http.headers["character-cost"]),
providerMetadata: requestID === undefined ? undefined : { elevenlabs: { requestId: requestID } },
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>(route, {
unsupported: ["instructions"],
body: { from: fromRequest },
frames: (bytes, context) => {
if (context.request.timestamps !== true) return bytes
return context.request.mode === "stream" ? Framing.lines.frame(bytes) : Framing.document.frame(bytes)
},
initial: () => ({ chunks: [] }),
step: SpeechStream.step(onRecord),
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<ElevenLabsSpeechOptions, string | Uint8Array, State>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => path(request) },
input,
)
export const ElevenLabsSpeech = {
protocol,
model,
} as const
+156
View File
@@ -0,0 +1,156 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { FalQueue } from "./utils/fal-queue.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "fal-images", name: "fal Images", provider: "fal" })
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type FalImageOptions = {
readonly image_size?: OpenString<
"square_hd" | "square" | "portrait_4_3" | "portrait_16_9" | "landscape_4_3" | "landscape_16_9"
>
readonly enable_safety_checker?: boolean
} & Record<string, unknown>
export type Request = ImageRequestFor<FalImageOptions>
// ---------------------------------------------------------------------------
// 2. Response schema
// ---------------------------------------------------------------------------
const QueueResult = Schema.StructWithRest(
Schema.Struct({
images: Schema.Array(
Schema.Struct({
url: Schema.String,
width: optionalNull(Schema.Number),
height: optionalNull(Schema.Number),
content_type: optionalNull(Schema.String),
}),
),
seed: optionalNull(Schema.Number),
has_nsfw_concepts: optionalNull(Schema.Array(Schema.Boolean)),
}),
[Schema.Record(Schema.String, Schema.Unknown)],
)
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const sizing = (model: string) => {
if (/^fal-ai\/(nano-banana|flux-pro\/(v1\.1-ultra|kontext))/.test(model)) return "aspect_ratio"
if (model.startsWith("fal-ai/flux")) return "image_size"
return undefined
}
const validate = (request: Request) => {
const id = request.model.id
const field = sizing(id)
if (request.size !== undefined && request.aspectRatio !== undefined)
return Effect.fail(ProviderShared.invalidRequest(`${route.name} accepts either size or aspectRatio, not both`))
if (request.size !== undefined && field === "aspect_ratio")
return Effect.fail(route.unsupported("media.size", `${id} sizes by aspectRatio`))
if (request.aspectRatio !== undefined && field === "image_size")
return Effect.fail(route.unsupported("media.aspectRatio", `${id} sizes by size (image_size)`))
if ((request.images?.length ?? 0) > 1 && !takesImageList(id))
return Effect.fail(
route.unsupported(
"media.images",
`${id} takes one image_url; use an /edit or /multi endpoint for several images`,
),
)
return Effect.void
}
// `/edit` and `/multi` (Kontext) endpoints take an `image_urls` list; image-to-image, fill, and Ultra take one
// `image_url` (beside `mask_url`).
const takesImageList = (model: string) => model.endsWith("/edit") || model.endsWith("/multi")
const fromRequest = Effect.fn("FalImages.fromRequest")(function* (request: Request) {
yield* validate(request)
const images = yield* Effect.forEach(request.images ?? [], (image) => FalQueue.mediaUrl(image, route.name))
const list = takesImageList(request.model.id)
return MediaProtocol.json(
mergeJsonRecords(
{
prompt: request.prompt,
num_images: request.n,
seed: request.seed,
image_size: request.size === undefined ? undefined : MediaInput.dimensions(request.size),
aspect_ratio: request.aspectRatio,
output_format: request.format,
image_urls: list && images.length > 0 ? images : undefined,
image_url: list ? undefined : images[0],
mask_url: request.mask === undefined ? undefined : yield* FalQueue.mediaUrl(request.mask, route.name),
},
request.providerOptions,
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeQueueResult = route.decodeJson(QueueResult)
const decodeResult = Effect.fn("FalImages.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<FalQueue.Token>,
) {
const output = yield* decodeQueueResult(response)
const { images, seed, has_nsfw_concepts, ...rest } = output.value
if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`)
// With the safety checker on, flagged images come back blacked out rather than omitted.
const flagged = (has_nsfw_concepts ?? []).flatMap((value, index) => (value ? [index] : []))
return new ImageResponse({
images: images.map((image) => {
const info = { width: image.width ?? undefined, height: image.height ?? undefined }
// `sync_mode: true` returns data URIs instead of hosted URLs.
return (
Media.parseDataUrl(image.url, { info }) ??
Media.url(image.url, { mediaType: image.content_type ?? undefined, info })
)
}),
notices:
flagged.length === 0
? undefined
: flagged.map((index) => ({
type: "moderated" as const,
message: `${route.name} flagged image ${index} as NSFW`,
})),
providerMetadata: { fal: { requestId: context.token.requestID, seed: seed ?? undefined, ...rest } },
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = FalQueue.protocol<Request, ImageResponse>(route, {
from: fromRequest,
decodeResult,
})
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<FalImageOptions, FalQueue.Token>(
{ protocol, baseURL: FalQueue.DEFAULT_BASE_URL, path: ({ request }) => `/${request.model.id}` },
input,
)
export const FalImages = {
protocol,
model,
} as const
+121
View File
@@ -0,0 +1,121 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
import { optionalNull } from "./shared.js"
import { FalQueue } from "./utils/fal-queue.js"
const route = MediaProtocol.identity({ id: "fal-video", name: "fal Video", provider: "fal" })
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
/**
* Provider-native input. fal video endpoints are model-specific: `duration` is a string enum whose values differ per
* model (`"8s"` for Veo, `"5"` for Kling), and last-frame fields are named per model (`end_image_url`,
* `last_frame_url`, `tail_image_url`), so those pass through here instead of lowering from common fields.
*/
export type FalVideoOptions = {
readonly duration?: OpenString<"4s" | "6s" | "8s" | "5" | "10">
} & Record<string, unknown>
export type Request = VideoRequestFor<FalVideoOptions>
// ---------------------------------------------------------------------------
// 2. Response schema
// ---------------------------------------------------------------------------
const QueueResult = Schema.StructWithRest(
Schema.Struct({
video: Schema.Struct({
url: Schema.String,
content_type: optionalNull(Schema.String),
file_name: optionalNull(Schema.String),
file_size: optionalNull(Schema.Number),
}),
seed: optionalNull(Schema.Number),
}),
[Schema.Record(Schema.String, Schema.Unknown)],
)
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const fromRequest = Effect.fn("FalVideo.fromRequest")(function* (request: Request) {
if (request.frames?.last !== undefined)
return yield* route.unsupported(
"video.frames.last",
`${route.name} names the last frame per model; pass it through providerOptions (e.g. end_image_url) instead of frames.last`,
)
const imageUrl =
request.frames?.first === undefined ? undefined : yield* FalQueue.mediaUrl(request.frames.first, route.name)
const videoUrl = request.video === undefined ? undefined : yield* FalQueue.mediaUrl(request.video, route.name)
return MediaProtocol.json(
mergeJsonRecords(
{
prompt: request.prompt,
negative_prompt: request.negativePrompt,
seed: request.seed,
aspect_ratio: request.aspectRatio,
resolution: request.resolution,
generate_audio: request.audio,
image_url: imageUrl,
video_url: videoUrl,
},
request.providerOptions,
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeQueueResult = route.decodeJson(QueueResult)
const decodeResult = Effect.fn("FalVideo.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<FalQueue.Token>,
) {
const output = yield* decodeQueueResult(response)
const { video, seed, ...rest } = output.value
return new VideoResponse({
videos: [Media.url(video.url, { mediaType: video.content_type ?? "video/mp4" })],
providerMetadata: {
fal: {
requestId: context.token.requestID,
seed: seed ?? undefined,
fileName: video.file_name ?? undefined,
fileSize: video.file_size ?? undefined,
...rest,
},
},
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = FalQueue.protocol<Request, VideoResponse>(route, {
unsupported: ["n", "durationSeconds", "references"],
from: fromRequest,
decodeResult,
})
export const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<FalVideoOptions, FalQueue.Token>(
{ protocol, baseURL: FalQueue.DEFAULT_BASE_URL, path: ({ request }) => `/${request.model.id}` },
input,
)
export const FalVideo = {
protocol,
model,
} as const
+24 -40
View File
@@ -10,7 +10,6 @@ import {
LLMEvent,
Usage,
type FinishReason,
type JsonSchema,
type LLMRequest,
type MediaPart,
type ProviderMetadata,
@@ -22,13 +21,14 @@ import {
import { classifyProviderFailure } from "../provider-error.js"
import { Media } from "../media.js"
import { JsonObject, knownString, lenient, optionalArray, optionalNull, ProviderShared } from "./shared.js"
import { GeminiToolSchema } from "./utils/gemini-tool-schema.js"
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
import { Lifecycle } from "./utils/lifecycle.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
const ADAPTER = "gemini"
// Google documents this sentinel for replaying Gemini 3 function calls after their original signature was lost.
const SKIP_THOUGHT_SIGNATURE_VALIDATOR = "skip_thought_signature_validator"
// Gemini 2.5 rejects a budget under the model's minimum: 512 on Flash-Lite, the highest, and 128 on Pro.
const MIN_THINKING_BUDGET = 512
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
// Gemini 3 rejects replayed function calls without a thought signature. Google's SDKs avoid that in normal chats by
@@ -132,7 +132,7 @@ const GeminiSystemInstruction = Schema.Struct({
const GeminiFunctionDeclaration = Schema.Struct({
name: Schema.String,
description: Schema.String,
parameters: Schema.optional(JsonObject),
parametersJsonSchema: JsonObject,
})
const GeminiTool = Schema.Struct({
@@ -265,36 +265,14 @@ interface ParserState {
readonly seenCallIds?: ReadonlySet<string>
}
// =============================================================================
// Tool Schema Conversion
// =============================================================================
// Tool-schema conversion has two distinct concerns:
//
// 1. Sanitize — fix common authoring mistakes Gemini rejects: integer/number
// enums (must be strings), `required` entries that don't match a property,
// untyped arrays (`items` must be present), and `properties`/`required`
// keys on non-object scalars. Mirrors OpenCode's historical Gemini rules.
//
// 2. Project — lossy mapping from JSON Schema to Gemini's schema dialect:
// drop empty root parameter schemas while preserving nested empty objects,
// expand type arrays into `anyOf`, derive `nullable: true` from null members,
// coerce `const` to `[const]` enum, recurse properties/items, and propagate
// only an allowlisted set of keys (description, required, format, type,
// nullable, enum, properties, items, allOf, anyOf, oneOf, minLength).
// Anything outside the allowlist (e.g. `additionalProperties`, `$ref`) is
// silently dropped.
//
// Sanitize runs first, then project. The implementation lives in
// `utils/gemini-tool-schema` so this protocol keeps the same shape as the other
// provider protocols.
// =============================================================================
// Request Lowering
// =============================================================================
const lowerTool = (tool: ToolDefinition, inputSchema: JsonSchema) => ({
// Tool schemas go in `parametersJsonSchema`, which accepts standard JSON Schema.
const lowerTool = (tool: ToolDefinition) => ({
name: tool.name,
description: tool.description,
parameters: GeminiToolSchema.convert(inputSchema),
parametersJsonSchema: tool.inputSchema,
})
const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
@@ -305,14 +283,9 @@ const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
tool: (name) => ({ functionCallingConfig: { mode: "ANY" as const, allowedFunctionNames: [name] } }),
})
// Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs.
const lowerContentPart = Effect.fn("Gemini.lowerContentPart")(function* (part: TextPart | MediaPart) {
if (part.type === "text") return { text: part.text }
const source = part.media.source
if (source.type === "ref" && source.provider === "google")
return { fileData: { mimeType: part.media.mediaType, fileUri: source.id } }
const media = yield* ProviderShared.requireInlineMedia("Gemini", part.media)
return { inlineData: { mimeType: media.mime, data: media.base64 } }
return yield* GeminiGenerateContent.mediaPart("Gemini", part.media)
})
const providerMetadata = (key: string, metadata: Record<string, unknown>): ProviderMetadata => ({ [key]: metadata })
@@ -469,7 +442,6 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
const hasTools = flattened.tools.length > 0
const generation = request.generation
const options = yield* decodeOptions(request.providerOptions ?? {})
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
const generationConfig = {
maxOutputTokens: generation?.maxTokens,
temperature: generation?.temperature,
@@ -479,10 +451,22 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
presencePenalty: generation?.presencePenalty,
seed: generation?.seed,
stopSequences: generation?.stop,
// Gemini accepts a budget above `maxOutputTokens`, but thinking then leaves the answer empty.
thinkingConfig:
options.thinkingConfig === undefined
? undefined
: { ...options.thinkingConfig, includeThoughts: options.thinkingConfig.includeThoughts ?? true },
: {
...options.thinkingConfig,
includeThoughts: options.thinkingConfig.includeThoughts ?? true,
thinkingBudget:
options.thinkingConfig.thinkingBudget === undefined
? undefined
: ProviderShared.fitThinkingBudget(
options.thinkingConfig.thinkingBudget,
generation?.maxTokens,
MIN_THINKING_BUDGET,
),
},
}
return {
@@ -495,9 +479,7 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
tools: hasTools
? [
{
functionDeclarations: flattened.tools.map((tool) =>
lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility)),
),
functionDeclarations: flattened.tools.map(lowerTool),
},
]
: undefined,
@@ -833,6 +815,8 @@ export const protocol = Protocol.make({
schema: GeminiBody,
from: fromRequest,
},
// Gemini's schema rules are this API's default, including for tuned endpoints whose IDs do not name Gemini.
sanitizer: "gemini",
stream: {
event: Protocol.jsonEvent(GeminiEvent),
initial: (request) => ({
+20 -45
View File
@@ -1,28 +1,24 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderShared } from "./shared.js"
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
import { MediaInput } from "./utils/media-input.js"
const ADAPTER = "google-images"
const NAME = "Google Images"
const PROVIDER = ProviderID.make("google")
const route = MediaProtocol.identity({ id: "google-images", name: "Google Images", provider: "google" })
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type GoogleImageString<Known extends string> = Known | (string & {})
/** Provider-native options. Common fields (`aspectRatio`, `seed`, `images`) live on the request. */
export type GoogleImageOptions = {
readonly imageSize?: GoogleImageString<"1K" | "2K" | "4K">
readonly thinkingLevel?: GoogleImageString<"MINIMAL" | "LOW" | "MEDIUM" | "HIGH">
readonly imageSize?: OpenString<"1K" | "2K" | "4K">
readonly thinkingLevel?: OpenString<"MINIMAL" | "LOW" | "MEDIUM" | "HIGH">
readonly includeThoughts?: boolean
} & Record<string, unknown>
@@ -102,34 +98,15 @@ const generationConfig = (request: Request) => {
)
}
// Gemini does not fetch public URLs; inline payloads or Gemini Files references are the only accepted inputs.
const imagePart = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ inlineData: { mimeType: inline.mime, data: inline.base64 } })
const id = MediaInput.refID(asset, PROVIDER)
if (id) return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri: id } })
if (asset.source.type === "ref")
return Effect.fail(
ProviderShared.invalidRequest(
"Google generateContent requires Gemini file references rather than other providers' file IDs",
),
)
return Effect.fail(
ProviderShared.invalidRequest(
"Google generateContent does not fetch public image URLs; use bytes, a data URL, or a Gemini file reference",
),
)
}
const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request: Request) {
if (request.n !== undefined && request.n > 1)
return yield* ProviderShared.unsupportedOperation({
operation: "image.n",
provider: PROVIDER,
route: ADAPTER,
message: `${NAME} generates one image per request; call it once per image instead of n=${request.n}`,
})
const parts = yield* Effect.forEach(request.images ?? [], imagePart)
return yield* route.unsupported(
"media.n",
`${route.name} generates one image per request; call it once per image instead of n=${request.n}`,
)
const parts = yield* Effect.forEach(request.images ?? [], (image) =>
GeminiGenerateContent.mediaPart(route.name, image),
)
return MediaProtocol.json(
mergeJsonRecords(
{
@@ -145,10 +122,12 @@ const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request: Re
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeDocument = route.decodeJson(GoogleImageResponse)
const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, GoogleImageResponse)(response)
const output = yield* decodeDocument(response)
const decoded = output.value
const candidates = decoded.candidates ?? []
const candidateMetadata = candidates.map((candidate, candidateIndex) => ({
@@ -188,7 +167,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
const images = yield* Effect.forEach(encoded, (item) =>
MediaInput.decodedAsset(
output.invalid,
`${NAME} candidate ${item.candidateIndex} part ${item.partIndex}`,
`${route.name} candidate ${item.candidateIndex} part ${item.partIndex}`,
item.inlineData.data,
item.inlineData.mimeType,
{
@@ -211,7 +190,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
candidate.finishReason === undefined ? [] : [candidate.finishReason],
)
return yield* output.invalid(
`${NAME} returned no final images${
`${route.name} returned no final images${
finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})`
}; inspect body for prompt feedback and candidate details`,
)
@@ -223,7 +202,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
: [
{
type: "filtered" as const,
message: `${NAME} reported prompt feedback`,
message: `${route.name} reported prompt feedback`,
providerMetadata: { google: { promptFeedback: decoded.promptFeedback } },
},
]),
@@ -233,7 +212,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
: [
{
type: "filtered" as const,
message: `${NAME} candidate ${candidate.index ?? index} finished with ${candidate.finishReason}${
message: `${route.name} candidate ${candidate.index ?? index} finished with ${candidate.finishReason}${
candidate.finishMessage === undefined ? "" : `: ${candidate.finishMessage}`
}`,
providerMetadata: {
@@ -283,9 +262,7 @@ const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
unsupported: ["mask", "size", "format"],
body: { from: fromRequest },
response: { decode: decodeResponse },
@@ -294,8 +271,6 @@ export const protocol = MediaProtocol.inline<Request, ImageResponse>({
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<GoogleImageOptions>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => `/models/${request.model.id}:generateContent`,
+153
View File
@@ -0,0 +1,153 @@
import { Effect, Schema } from "effect"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
import { SpeechStream } from "./utils/speech-stream.js"
const route = MediaProtocol.identity({ id: "google-speech", name: "Google Speech", provider: "google" })
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
const DEFAULT_SAMPLE_RATE = 24000
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
/** Style is directed in the text itself, and `speechConfig.multiSpeakerVoiceConfig` excludes `voice`. */
export type GoogleSpeechOptions = {
readonly temperature?: number
readonly seed?: number
readonly speechConfig?: {
readonly multiSpeakerVoiceConfig?: {
readonly speakerVoiceConfigs: ReadonlyArray<{
readonly speaker: string
readonly voiceConfig: { readonly prebuiltVoiceConfig: { readonly voiceName: string } }
}>
}
}
} & Record<string, unknown>
export type Request = SpeechRequestFor<GoogleSpeechOptions>
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
const GenerateContentChunk = GeminiGenerateContent.chunk(
Schema.Struct({
text: Schema.optional(Schema.String),
inlineData: Schema.optional(Schema.Struct({ mimeType: Schema.String, data: Schema.Uint8ArrayFromBase64 })),
}),
)
const decodeChunk = route.decodeFrame(GenerateContentChunk)
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
interface State extends SpeechStream.Audio, GeminiGenerateContent.Metadata {
readonly mimeType?: string
}
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
// Not in `unsupported`: that list would also reject `timestamps: false`, which asks for nothing.
if (request.timestamps === true)
return yield* route.unsupported("media.timestamps", `${route.name} does not return timestamps`)
if (request.format === "pcm" && request.mode === "generate" && /^gemini-3\.8-.*-tts(?:-|$)/.test(request.model.id))
return yield* route.unsupported(
"media.format",
`${route.name} returns WAV by default for Gemini 3.8 TTS unary requests; omit the format to accept it`,
)
if (request.format !== undefined && request.format !== "pcm")
return yield* route.unsupported(
"media.format",
`${route.name} only accepts raw PCM as an explicit format; omit it to accept the provider's default output`,
)
const voiceName = SpeechStream.voiceID(request.voice)
return MediaProtocol.json(
mergeJsonRecords(
{
contents: [{ role: "user", parts: [{ text: request.text }] }],
generationConfig: mergeJsonRecords(
{
responseModalities: ["AUDIO"],
speechConfig: {
voiceConfig: voiceName === undefined ? undefined : { prebuiltVoiceConfig: { voiceName } },
languageCode: request.language,
},
},
request.providerOptions,
),
},
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const step = Effect.fn("GoogleSpeech.step")(function* (state: State, frame: string) {
const chunk = yield* decodeChunk(frame)
const blocked = GeminiGenerateContent.blocked(route.name, chunk, frame)
if (blocked !== undefined) return yield* blocked
const audio = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) =>
part.inlineData === undefined ? [] : [part.inlineData],
)
const next: State = { ...GeminiGenerateContent.track(state, chunk), mimeType: state.mimeType ?? audio[0]?.mimeType }
return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])] as const
})
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE
const output =
state.mimeType?.split(";")[0]?.toLowerCase() === "audio/wav"
? SpeechStream.container("wav", sampleRate)
: SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`)
if (context.request.format === "pcm" && output.info.format !== "pcm")
return Effect.fail(
route.frameError(`Google Speech returned ${output.info.format} instead of the requested raw PCM`),
)
return SpeechStream.finish(route, state, {
...output,
usage: GeminiGenerateContent.usage(state.usage),
providerMetadata: GeminiGenerateContent.providerMetadata(state),
detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`,
})
}
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string, State>(route, {
unsupported: ["instructions", "speed"],
body: { from: fromRequest },
frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode),
initial: () => ({ chunks: [] }),
step,
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<GoogleSpeechOptions, string, State>(
{
protocol,
baseURL: DEFAULT_BASE_URL,
// Only `gemini-3.1-flash-tts-preview` and later stream; earlier TTS models reject `streamGenerateContent`.
path: ({ request }) => GeminiGenerateContent.path(request.model.id, request.mode),
},
input,
)
export const GoogleSpeech = {
protocol,
model,
} as const
@@ -0,0 +1,203 @@
import { Effect, Schema, SchemaGetter } from "effect"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import {
TranscriptionFinishEvent,
TranscriptionModel,
TranscriptionSegmentEvent,
TranscriptionTextDeltaEvent,
type TranscriptionRequestFor,
type TranscriptionSegment,
type TranscriptionWord,
type TranscriptionEvent,
} from "../transcription.js"
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
const route = MediaProtocol.identity({ id: "google-transcription", name: "Google Transcription", provider: "google" })
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
/**
* Merged into `generationConfig`. The API rejects `customVocabulary` and `mode: "SMART"` alongside diarization or word
* timestamps.
*/
export type GoogleTranscriptionOptions = {
readonly audioTranscriptionConfig?: {
readonly mode?: OpenString<"VERBATIM" | "SMART">
readonly customVocabulary?: ReadonlyArray<string>
readonly languageCodes?: ReadonlyArray<string>
}
} & Record<string, unknown>
export type Request = TranscriptionRequestFor<GoogleTranscriptionOptions>
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
const Seconds = Schema.String.check(Schema.isPattern(/^\d+(\.\d+)?s$/)).pipe(
Schema.decodeTo(Schema.Number, {
decode: SchemaGetter.transform((value) => Number.parseFloat(value)),
encode: SchemaGetter.transform((value) => `${value}s`),
}),
)
const AudioTranscription = Schema.Struct({
text: Schema.String,
speakerLabel: Schema.optional(Schema.String),
words: Schema.optional(
Schema.Array(
Schema.Struct({
word: Schema.String,
startOffset: Seconds,
endOffset: Seconds,
speakerLabel: Schema.optional(Schema.String),
}),
),
),
})
const decodeChunk = route.decodeFrame(
GeminiGenerateContent.chunk(Schema.Struct({ audioTranscription: Schema.optional(AudioTranscription) })),
)
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
interface State extends GeminiGenerateContent.Metadata {
readonly text: string
readonly segments: Array<TranscriptionSegment>
readonly words: Array<TranscriptionWord>
}
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const fromRequest = Effect.fn("GoogleTranscription.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
// General Gemini models ignore `audioTranscriptionConfig` and answer the audio conversationally.
if (!request.model.id.includes("transcribe"))
return yield* route.unsupported(
"transcription.model",
`${request.model.id} is not a transcription model; use a transcribe model such as gemini-3.5-transcribe`,
)
return MediaProtocol.json(
mergeJsonRecords(
{
contents: [{ role: "user", parts: [yield* GeminiGenerateContent.mediaPart(route.id, request.audio)] }],
generationConfig: mergeJsonRecords(
{
audioTranscriptionConfig: {
languageCodes: request.language === undefined ? undefined : [request.language],
// Parts carry no offsets of their own, so segment times come from word offsets.
wordTimestamp:
request.diarize === true || request.timestamps === "word" || request.timestamps === "segment"
? true
: undefined,
diarization: request.diarize === true ? true : undefined,
},
},
request.providerOptions,
),
},
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const turn = (part: Schema.Schema.Type<typeof AudioTranscription>) => {
const words = (part.words ?? []).map((word) => ({
text: word.word,
startSeconds: word.startOffset,
endSeconds: word.endOffset,
speaker: word.speakerLabel ?? part.speakerLabel,
}))
const first = words[0]
const last = words.at(-1)
return {
text: part.text,
words,
segment:
first === undefined || last === undefined
? undefined
: {
text: part.text,
startSeconds: first.startSeconds,
endSeconds: last.endSeconds,
speaker: part.speakerLabel,
},
}
}
const step = Effect.fn("GoogleTranscription.step")(function* (state: State, frame: string) {
const chunk = yield* decodeChunk(frame)
const blocked = GeminiGenerateContent.blocked(route.name, chunk, frame)
if (blocked !== undefined) return yield* blocked
const turns = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) =>
part.audioTranscription === undefined ? [] : [turn(part.audioTranscription)],
)
const segments = turns.flatMap((item) => (item.segment === undefined ? [] : [item.segment]))
state.words.push(...turns.flatMap((item) => item.words))
state.segments.push(...segments)
// Each part is one whole speaker turn without surrounding whitespace, so turns join with a space.
const text = turns
.map((item) => item.text)
.filter((item) => item.length > 0)
.join(" ")
const delta = text.length === 0 || state.text.length === 0 ? text : ` ${text}`
const events: ReadonlyArray<TranscriptionEvent> = [
...(delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta })]),
...segments.map((segment) => TranscriptionSegmentEvent.make({ segment })),
]
return [{ ...GeminiGenerateContent.track(state, chunk), text: state.text + delta }, events] as const
})
const finish = (state: State) => {
if (state.finishReason === undefined) return Effect.fail(route.incomplete())
return Effect.succeed([
TranscriptionFinishEvent.make({
text: state.text,
segments: state.segments.length === 0 ? undefined : state.segments,
words: state.words.length === 0 ? undefined : state.words,
usage: GeminiGenerateContent.usage(state.usage),
providerMetadata: GeminiGenerateContent.providerMetadata(state),
}),
])
}
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, string, State>(route, {
unsupported: ["prompt", "speakers"],
body: { from: fromRequest },
frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode),
initial: () => ({ text: "", segments: [], words: [] }),
step,
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
TranscriptionModel.fromRoute<GoogleTranscriptionOptions, string, State>(
{
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => GeminiGenerateContent.path(request.model.id, request.mode),
},
input,
)
export const GoogleTranscription = {
protocol,
model,
} as const
+220
View File
@@ -0,0 +1,220 @@
import { Duration, Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
import { ProviderShared, optionalArray } from "./shared.js"
const route = MediaProtocol.identity({ id: "google-video", name: "Google Veo", provider: "google" })
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
/** Veo keeps generated files for two days; the asset carries that deadline so callers materialize in time. */
const FILE_RETENTION = Duration.days(2)
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
/** Provider-native `parameters`. Common fields (`aspectRatio`, `resolution`, `durationSeconds`, `seed`) live on the request. */
export type GoogleVideoOptions = {
readonly personGeneration?: OpenString<"allow_all" | "allow_adult" | "dont_allow">
} & Record<string, unknown>
export type Request = VideoRequestFor<GoogleVideoOptions>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
/** The long-running operation name, e.g. `models/veo-3.1-generate-preview/operations/abc123`. */
export const Token = Schema.Struct({ operation: Schema.String })
export type Token = Schema.Schema.Type<typeof Token>
const StartResponse = Schema.Struct({ name: Schema.String })
const Operation = Schema.Struct({
done: Schema.optional(Schema.Boolean),
error: Schema.optional(Schema.Struct({ message: Schema.optional(Schema.String) })),
response: Schema.optional(
Schema.Struct({
generateVideoResponse: Schema.optional(
Schema.Struct({
generatedSamples: optionalArray(
Schema.Struct({
video: Schema.optional(
Schema.Struct({
uri: Schema.optional(Schema.String),
mimeType: Schema.optional(Schema.String),
}),
),
}),
),
raiMediaFilteredCount: Schema.optional(Schema.Number),
raiMediaFilteredReasons: optionalArray(Schema.String),
}),
),
}),
),
metadata: Schema.optional(Schema.Unknown),
})
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
// Veo takes inline media only; a prior Veo output is `Media.url` with transient auth, so materialize it first.
const inlineMedia = (asset: Media.Asset) =>
ProviderShared.requireInlineMedia(route.name, asset).pipe(
Effect.map((inline) => ({ inlineData: { mimeType: inline.mime, data: inline.base64 } })),
)
const fromRequest = Effect.fn("GoogleVideo.fromRequest")(function* (request: Request) {
if (request.n !== undefined && request.n > 1)
return yield* route.unsupported(
"video.n",
`${route.name} generates one video per request; call it once per video instead of n=${request.n}`,
)
if (request.audio === false)
return yield* route.unsupported(
"video.audio",
`${route.name} always generates audio; audio: false cannot be honored`,
)
if (request.frames?.last !== undefined && request.frames.first === undefined)
return yield* ProviderShared.invalidRequest(`${route.name} requires frames.first when frames.last is set`)
const image = request.frames?.first === undefined ? undefined : yield* inlineMedia(request.frames.first)
const lastFrame = request.frames?.last === undefined ? undefined : yield* inlineMedia(request.frames.last)
const video = request.video === undefined ? undefined : yield* inlineMedia(request.video)
const referenceImages = yield* Effect.forEach(request.references ?? [], (asset) =>
inlineMedia(asset).pipe(Effect.map((image) => ({ image, referenceType: "asset" }))),
)
return MediaProtocol.json(
mergeJsonRecords(
{
instances: [
{
prompt: request.prompt,
image,
lastFrame,
referenceImages: referenceImages.length === 0 ? undefined : referenceImages,
video,
},
],
parameters: mergeJsonRecords(
{
aspectRatio: request.aspectRatio,
resolution: request.resolution,
durationSeconds: request.durationSeconds,
negativePrompt: request.negativePrompt,
seed: request.seed,
},
request.providerOptions,
),
},
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
token: { operation: value.name },
snapshot: { id: value.name, status: "running" },
}))
// Operations carry no status string: not done is running, done with `error` failed, otherwise completed.
const statusOf = (operation: typeof Operation.Type): Status => {
if (operation.done !== true) return "running"
return operation.error === undefined ? "completed" : "failed"
}
const decodeOperation = route.decodeJson(Operation)
const decodeStatus = Effect.fn("GoogleVideo.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeOperation(response)
return { id: context.token.operation, status: statusOf(output.value) }
})
const decodeResult = Effect.fn("GoogleVideo.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeOperation(response)
const operation = output.value
const status = statusOf(operation)
if (status === "running") return yield* output.pending(context.token.operation)
if (status === "failed")
return yield* output.ended(
"failed",
`${route.name} operation failed${operation.error?.message === undefined ? "" : `: ${operation.error.message}`}`,
)
const generated = operation.response?.generateVideoResponse
// Downloads require the same API key as the poll; the asset carries it transiently and follows the redirect.
const videos = yield* Effect.forEach(
(generated?.generatedSamples ?? []).flatMap((sample) =>
sample.video?.uri === undefined ? [] : [{ uri: sample.video.uri, mimeType: sample.video.mimeType }],
),
(video) =>
MediaProtocol.expiringUrl(video.uri, FILE_RETENTION, {
mediaType: video.mimeType ?? "video/mp4",
headers: context.auth,
}),
)
const reasons = generated?.raiMediaFilteredReasons ?? []
const notices = reasons.map((reason) => ({
type: "filtered" as const,
message: `${route.name} filtered media: ${reason}`,
providerMetadata: { google: { raiMediaFilteredReason: reason } },
}))
if (videos.length === 0 && (reasons.length > 0 || (generated?.raiMediaFilteredCount ?? 0) > 0))
return yield* output.contentPolicy(
`${route.name} filtered every video${reasons.length === 0 ? "" : `: ${reasons.join("; ")}`}`,
)
if (videos.length === 0) return yield* output.invalid(`${route.name} operation completed without any video`)
return new VideoResponse({
videos,
notices: notices.length === 0 ? undefined : notices,
providerMetadata: {
google: {
operation: context.token.operation,
raiMediaFilteredCount: generated?.raiMediaFilteredCount,
metadata: operation.metadata,
},
},
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
const operationPath = (token: Token) => `/${token.operation}`
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>(route, {
token: Token,
start: { body: { from: fromRequest }, decode: decodeStart },
status: { path: operationPath, decode: decodeStatus },
result: { path: operationPath, decode: decodeResult },
})
export const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<GoogleVideoOptions, Token>(
{
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => `/models/${request.model.id}:predictLongRunning`,
},
input,
)
export const GoogleVideo = {
protocol,
model,
} as const
+18 -29
View File
@@ -4,20 +4,17 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { JsonObject, ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const ADAPTER = "meta-images"
const NAME = "Meta Images"
const PROVIDER = ProviderID.make("meta")
const route = MediaProtocol.identity({ id: "meta-images", name: "Meta Images", provider: "meta" })
export const DEFAULT_BASE_URL = "https://api.meta.ai/v1"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
type OpenString<Known extends string> = Known | (string & {})
/** Provider-native options. Common fields (`n`, `size`, `format`, `images`) live on the request. */
export type ImageOptions = {
readonly responseFormat?: OpenString<"b64_json" | "url">
@@ -70,13 +67,9 @@ const Response = Schema.Struct({
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ image_url: inline.dataUrl })
const url = ProviderShared.mediaUrl(asset)
if (url) return Effect.succeed({ image_url: url })
return Effect.fail(ProviderShared.invalidRequest(`${NAME} accepts image bytes and URLs`))
}
// Meta has no file handles: refs are rejected even when they name this provider.
const reference = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, undefined, route.name).pipe(Effect.map((item) => ({ image_url: item.value })))
const fromRequest = Effect.fn("MetaImages.fromRequest")(function* (request: Request) {
const images = yield* Effect.forEach(request.images ?? [], reference)
@@ -105,24 +98,23 @@ const fromRequest = Effect.fn("MetaImages.fromRequest")(function* (request: Requ
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeDocument = route.decodeJson(Response)
const decodeResponse = Effect.fn("MetaImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.DecodeContext<Request>,
) {
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, Response)(response)
const output = yield* decodeDocument(response)
const decoded = output.value
const requested = context.body.type === "json" ? context.body.value.output_format : undefined
const format = decoded.output_format ?? (typeof requested === "string" ? requested : "webp")
const mediaType = `image/${format}`
const images = yield* Effect.forEach(decoded.data, (item, index) => {
if (item.b64_json)
return MediaInput.decodedAsset(output.invalid, `${NAME} result ${index}`, item.b64_json, mediaType, {
info: { format },
})
if (item.url) return Effect.succeed(Media.url(item.url, { mediaType, info: { format } }))
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
})
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
const images = yield* Effect.forEach(decoded.data, (item, index) =>
MediaInput.imageOutput(output.invalid, `${route.name} result ${index}`, item, mediaType, {
info: { format },
}),
)
if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`)
return new ImageResponse({
images,
usage:
@@ -143,20 +135,17 @@ const decodeResponse = Effect.fn("MetaImages.decodeResponse")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
unsupported: ["mask", "aspectRatio", "seed"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput & { readonly baseURL: string }) =>
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<ImageOptions>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => `/images/${isEdit(request) ? "edits" : "generations"}`,
},
input,
+7 -9
View File
@@ -5,8 +5,7 @@ import { LLMEvent, LLMRequest, Message, ToolResultPart } from "../schema/index.j
import { OpenResponses } from "./open-responses.js"
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
import { ResponsesHostedTools } from "./utils/responses-hosted-tools.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
import { MetaImage } from "./utils/meta-image.js"
import { detectMediaType } from "../utils/media-type.js"
const ADAPTER = "meta-responses"
const NAME = "Meta Responses"
@@ -103,12 +102,7 @@ const fromRequest = Effect.fn("MetaResponses.fromRequest")(function* (request: L
? undefined
: yield* Effect.forEach(projected.tools, (tool) =>
Effect.gen(function* () {
if (tool.native === undefined)
return yield* OpenResponses.lowerTool(
NAME,
tool,
ToolSchemaProjection.modelCompatibility(tool.inputSchema, request.model.compatibility?.toolSchema),
)
if (tool.native === undefined) return yield* OpenResponses.lowerTool(NAME, tool)
return yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(NativeTool))(tool.native.meta)
}),
),
@@ -151,7 +145,11 @@ const HOSTED_TOOLS = {
),
),
)
const mime = MetaImage.mediaType(data, item.output_format)
// Responses image items can omit output_format, including when PNG/JPEG was requested.
const mime =
item.output_format === undefined
? (detectMediaType(data) ?? "application/octet-stream")
: `image/${item.output_format}`
return {
type: "content" as const,
value: [{ type: "file" as const, uri: `data:${mime};base64,${item.result}`, mime }],
+8 -21
View File
@@ -8,7 +8,6 @@ import {
ProviderInternalError,
Usage,
type FinishReason,
type JsonSchema,
type LLMRequest,
type MediaPart,
type ProviderMetadata,
@@ -24,7 +23,6 @@ import { classifyProviderFailure } from "../provider-error.js"
import { effortUpdate } from "../effort-updates.js"
import { OpenResponsesOptions } from "./utils/open-responses-options.js"
import { Lifecycle } from "./utils/lifecycle.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
import { ToolStream } from "./utils/tool-stream.js"
const ADAPTER = "open-responses"
@@ -443,23 +441,24 @@ interface ReasoningStreamItem {
// =============================================================================
// Request Lowering
// =============================================================================
export const lowerTool = Effect.fn("OpenResponses.lowerTool")(function* (
protocolName: string,
tool: ToolDefinition,
inputSchema: JsonSchema,
) {
export const lowerTool = Effect.fn("OpenResponses.lowerTool")(function* (protocolName: string, tool: ToolDefinition) {
if (tool.native !== undefined)
return yield* ProviderShared.invalidRequest(`${protocolName} does not support provider-native tool ${tool.name}`)
return {
type: "function" as const,
name: tool.name,
description: tool.description,
parameters: inputSchema,
parameters: tool.inputSchema,
// The common tool definition does not currently express Responses strict-schema policy.
strict: false,
}
})
export const lowerTools = (tools: ReadonlyArray<ToolDefinition>, adapter: ProviderAdapter) =>
Effect.forEach(tools, (tool) =>
tool.native !== undefined && adapter.nativeTool ? adapter.nativeTool(tool.native) : lowerTool(adapter.name, tool),
)
export const lowerToolChoice = (protocolName: string, toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
ProviderShared.matchToolChoice(protocolName, toolChoice, {
auto: () => "auto" as const,
@@ -818,22 +817,10 @@ export const fromRequestWithAdapter = Effect.fn("OpenResponses.fromRequestWithAd
adapter: ProviderAdapter,
) {
const projected = ProviderShared.flattenToolRequest(request)
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
return {
...(yield* lowerConversation(projected.request, adapter)),
...lowerGeneration(request),
tools:
projected.tools.length === 0
? undefined
: yield* Effect.forEach(projected.tools, (tool) =>
tool.native !== undefined && adapter.nativeTool
? adapter.nativeTool(tool.native)
: lowerTool(
adapter.name,
tool,
ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility),
),
),
tools: projected.tools.length === 0 ? undefined : yield* lowerTools(projected.tools, adapter),
tool_choice:
allowedToolChoice(request) ??
(request.toolChoice ? yield* lowerToolChoice(adapter.name, request.toolChoice) : undefined),
+148 -59
View File
@@ -1,4 +1,4 @@
import { Effect, Schema } from "effect"
import { Effect, Option, Schema } from "effect"
import { Tool } from "@opencode/schema/tool"
import { Route } from "../route/client.js"
import { Auth } from "../route/auth.js"
@@ -17,7 +17,6 @@ import {
type FinishReason,
type FinishReasonDetails,
type CacheHint,
type JsonSchema,
type LLMRequest,
type MediaPart,
type ReasoningPart,
@@ -29,7 +28,6 @@ import { classifyProviderFailure } from "../provider-error.js"
import { isRecord, JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
import { OpenAIOptions } from "./utils/openai-options.js"
import { Lifecycle } from "./utils/lifecycle.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
import { ToolStream } from "./utils/tool-stream.js"
const ADAPTER = "openai-chat"
@@ -76,6 +74,44 @@ const OpenAIChatAssistantToolCall = Schema.Struct({
})
type OpenAIChatAssistantToolCall = Schema.Schema.Type<typeof OpenAIChatAssistantToolCall>
// `reasoning_details` carries two dialects. OpenRouter's `reasoning.*` entries
// must be replayed unmodified (`index` included), so they keep every field they
// arrived with. Kimi's OpenAI-compatible surface streams preserved thinking as
// bare `summary` / `encrypted` entries keyed by a stream-only `index`; Kimi does
// not document this publicly, so the handling follows Kimi Code (Kimi's own
// client): merge summary deltas by `index`, replay without `index`, and always
// send `reasoning_content` alongside. Anything else is dropped at the boundary.
const OpenRouterDetailFields = {
id: Schema.optional(Schema.NullOr(Schema.String)),
format: Schema.optional(Schema.String),
index: Schema.optional(Schema.Number),
signature: Schema.optional(Schema.NullOr(Schema.String)),
}
const ReasoningDetail = Schema.Union([
Schema.StructWithRest(
Schema.Struct({ type: Schema.Literal("reasoning.text"), text: Schema.optional(Schema.String), ...OpenRouterDetailFields }),
[Schema.Record(Schema.String, Schema.Unknown)],
),
Schema.StructWithRest(
Schema.Struct({
type: Schema.Literal("reasoning.summary"),
summary: Schema.optional(Schema.String),
...OpenRouterDetailFields,
}),
[Schema.Record(Schema.String, Schema.Unknown)],
),
Schema.StructWithRest(
Schema.Struct({ type: Schema.Literal("reasoning.encrypted"), data: Schema.String, ...OpenRouterDetailFields }),
[Schema.Record(Schema.String, Schema.Unknown)],
),
Schema.Struct({ type: Schema.Literal("summary"), summary: Schema.String, index: Schema.optional(Schema.Number) }),
Schema.Struct({ type: Schema.Literal("encrypted"), encrypted: Schema.String, index: Schema.optional(Schema.Number) }),
])
type ReasoningDetail = Schema.Schema.Type<typeof ReasoningDetail>
const decodeReasoningDetail = Schema.decodeUnknownOption(ReasoningDetail)
const knownReasoningDetails = (details: ReadonlyArray<unknown>) =>
details.flatMap((detail) => Option.toArray(decodeReasoningDetail(detail)))
// Intentionally omit Gemini's provider-specific `extra_content.google.thought_signature`
// extension until direct Google OpenAI-compatible routing is supported here:
// https://github.com/vercel/ai/issues/11590
@@ -92,6 +128,10 @@ const OpenAIChatUserContent = Schema.Union([
type: Schema.Literal("image_url"),
image_url: Schema.Struct({ url: Schema.String }),
}),
Schema.Struct({
type: Schema.Literal("file"),
file: Schema.Struct({ filename: Schema.String, file_data: Schema.String }),
}),
])
const OpenAIChatMessage = Schema.Union([
@@ -265,7 +305,9 @@ export interface ParserState {
readonly finishReason?: FinishReasonDetails
readonly lifecycle: Lifecycle.State
readonly reasoningField?: string
readonly reasoningDetails: Array<unknown>
/** A scalar reasoning field (`reasoning_content`, ...) has carried text in this stream. */
readonly reasoningTextObserved: boolean
readonly reasoningDetails: Array<ReasoningDetail>
readonly reasoningDetailsObserved: boolean
readonly reasoningEmitted: boolean
readonly latestToolIndex?: number
@@ -286,17 +328,12 @@ interface LoweringOptions {
readonly toolCallID?: (id: string) => string
}
const lowerTool = (
tool: ToolDefinition,
inputSchema: JsonSchema,
options: LoweringOptions,
supportsStrictMode: boolean,
): OpenAIChatTool => ({
const lowerTool = (tool: ToolDefinition, options: LoweringOptions, supportsStrictMode: boolean): OpenAIChatTool => ({
type: "function",
function: {
name: tool.name,
description: tool.description,
parameters: inputSchema,
parameters: tool.inputSchema,
...(supportsStrictMode ? { strict: false } : {}),
},
cache_control: options.cacheControl?.(tool.cache),
@@ -320,6 +357,15 @@ const lowerToolCall = (part: ToolCallPart, options: LoweringOptions): OpenAIChat
})
const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart) {
// Chat Completions accepts PDFs, and no other documents, as inline `file` parts; file URLs are not supported.
if (part.media.mediaType.toLowerCase() === "application/pdf")
return {
type: "file" as const,
file: {
filename: part.filename ?? "document.pdf",
file_data: (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl,
},
}
if (part.media.kind !== "image")
return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.media.mediaType}`)
const url =
@@ -341,10 +387,21 @@ const reasoningDetails = (parts: ReadonlyArray<ReasoningPart>, native: unknown,
return Array.isArray(details) ? details : []
})
if (parts.some((part) => Array.isArray(part.providerMetadata?.[providerMetadataKey]?.reasoningDetails)))
return observed
if (isRecord(native) && Array.isArray(native.reasoning_details)) return native.reasoning_details
return knownReasoningDetails(observed).map(lowerReasoningDetail)
if (isRecord(native) && Array.isArray(native.reasoning_details))
return knownReasoningDetails(native.reasoning_details).map(lowerReasoningDetail)
}
// Kimi rejects its stream-only `index` on requests
// ("the reasoning_details ... must not contain streaming index").
const lowerReasoningDetail = (detail: ReasoningDetail) => {
if (detail.type === "summary") return { type: detail.type, summary: detail.summary }
if (detail.type === "encrypted") return { type: detail.type, encrypted: detail.encrypted }
return detail
}
const isKimiDetail = (detail: { readonly type: string }) => detail.type === "summary" || detail.type === "encrypted"
const lowerUserMessage = Effect.fn("OpenAIChat.lowerUserMessage")(function* (
message: OpenAIChatRequestMessage,
options: LoweringOptions,
@@ -410,6 +467,9 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func
if (observedField !== undefined) return observedField
if (nativeReasoning !== undefined) return "reasoning_content"
if (!fullyStructured || requireReasoning) return "reasoning_content"
// Kimi always expects `reasoning_content` on replayed assistant messages,
// even when thinking arrived only through structured details.
if (details?.some(isKimiDetail)) return "reasoning_content"
})()
const reasoningText = (() => {
if (configuredField !== undefined)
@@ -435,7 +495,7 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (
options: LoweringOptions,
) {
const messages: OpenAIChatMessage[] = []
const images: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
const attachments: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
for (const part of message.content) {
if (!ProviderShared.supportsContent(part, ["tool-result"]))
return yield* ProviderShared.unsupportedContent("OpenAI Chat", "tool", ["tool-result"])
@@ -457,9 +517,9 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (
cache_control: options.cacheControl?.(part.cache),
})
const files = content.filter((item) => item.type === "file")
images.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item)))))
attachments.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item)))))
}
return { messages, images }
return { messages, attachments }
})
const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (
@@ -520,21 +580,21 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request:
if (requireAssistantAfterTool && messages.at(-1)?.role === "tool")
messages.push({ role: "assistant", content: "Done." })
}
const pendingImages: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
const flushImages = () => {
if (pendingImages.length === 0) return
const pendingAttachments: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
const flushAttachments = () => {
if (pendingAttachments.length === 0) return
bridgeTools()
messages.push({ role: "user", content: pendingImages.splice(0) })
messages.push({ role: "user", content: pendingAttachments.splice(0) })
}
for (const message of request.messages) {
if (message.role === "user") bridgeTools()
if (message.role === "system") {
const part = yield* ProviderShared.wrappedSystemUpdate("OpenAI Chat", message)
if (pendingImages.length > 0) {
if (pendingAttachments.length > 0) {
messages.push({
role: "user",
content: [
...pendingImages.splice(0),
...pendingAttachments.splice(0),
{ type: "text", text: part.text, cache_control: options.cacheControl?.(part.cache) },
],
})
@@ -578,13 +638,13 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request:
if (message.role === "tool") {
const lowered = yield* lowerToolMessages(message, lowering)
messages.push(...lowered.messages)
pendingImages.push(...lowered.images)
pendingAttachments.push(...lowered.attachments)
continue
}
flushImages()
flushAttachments()
messages.push(...(yield* lowerMessage(message, reasoningField, requireReasoning, lowering)))
}
flushImages()
flushAttachments()
return messages
})
@@ -736,7 +796,6 @@ export const fromRequest = Effect.fn("OpenAIChat.fromRequest")(function* (
`OpenAI Chat reasoning field conflicts with reserved field ${reasoningField}`,
)
const generation = request.generation
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
const flattened = ProviderShared.flattenToolRequest(request)
const provider = String(request.model.provider)
const baseURL = request.model.route.endpoint.baseURL
@@ -759,14 +818,7 @@ export const fromRequest = Effect.fn("OpenAIChat.fromRequest")(function* (
? hasHistory
? []
: undefined
: flattened.tools.map((tool) =>
lowerTool(
tool,
ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility),
options,
supportsStrictMode,
),
),
: flattened.tools.map((tool) => lowerTool(tool, options, supportsStrictMode)),
tool_choice: hasActiveTools && request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined,
stream: true as const,
...(supportsUsageInStreaming ? { stream_options: { include_usage: true } } : {}),
@@ -881,44 +933,74 @@ const reasoningDelta = (
return undefined
}
const detailText = (details: ReadonlyArray<unknown>) => {
const detailText = (details: ReadonlyArray<ReasoningDetail>, hideKimiSummary: boolean) => {
const text = details.flatMap((detail) => {
if (!isRecord(detail)) return []
if (detail.type === "reasoning.text" && typeof detail.text === "string" && detail.text) return [detail.text]
if (detail.type === "reasoning.summary" && typeof detail.summary === "string" && detail.summary)
return [detail.summary]
if (detail.type === "reasoning.text") return detail.text ? [detail.text] : []
if (detail.type === "reasoning.summary") return detail.summary ? [detail.summary] : []
// Kimi streams the full thinking through `reasoning_content` and a separate
// summary through details; show the summary only when nothing else does.
if (detail.type === "summary") return detail.summary && !hideKimiSummary ? [detail.summary] : []
return []
})
if (text.length > 0) return text.join("")
}
const appendReasoningDetails = (result: Array<unknown>, details: ReadonlyArray<unknown>) => {
const appendReasoningDetails = (result: Array<ReasoningDetail>, details: ReadonlyArray<ReasoningDetail>) => {
for (const detail of details) {
const previous = result.at(-1)
if (
!isRecord(previous) ||
previous.type !== "reasoning.text" ||
!isRecord(detail) ||
detail.type !== "reasoning.text" ||
conflictingReasoningTextDetails(previous, detail)
) {
const merged = previous === undefined ? undefined : mergeReasoningDetails(previous, detail)
if (merged === undefined) {
result.push(detail)
continue
}
result[result.length - 1] = {
...previous,
...Object.fromEntries(Object.entries(detail).filter((entry) => entry[1] !== undefined)),
text: `${typeof previous.text === "string" ? previous.text : ""}${typeof detail.text === "string" ? detail.text : ""}`,
signature: mergeDetailValue(previous.signature, detail.signature),
format: mergeDetailValue(previous.format, detail.format),
}
result[result.length - 1] = merged
}
}
const mergeDetailValue = (previous: unknown, current: unknown) =>
// Consecutive text or summary deltas of the same kind accumulate into one
// entry; encrypted entries are opaque and never merge.
const mergeReasoningDetails = (previous: ReasoningDetail, detail: ReasoningDetail): ReasoningDetail | undefined => {
if (conflictingReasoningDetails(previous, detail)) return undefined
if (previous.type === "reasoning.text" && detail.type === "reasoning.text")
return {
...previous,
...detail,
text: `${previous.text ?? ""}${detail.text ?? ""}`,
...mergeDetailIdentity(previous, detail),
}
if (previous.type === "reasoning.summary" && detail.type === "reasoning.summary")
return {
...previous,
...detail,
summary: `${previous.summary ?? ""}${detail.summary ?? ""}`,
...mergeDetailIdentity(previous, detail),
}
if (previous.type === "summary" && detail.type === "summary")
return { ...previous, ...detail, summary: previous.summary + detail.summary }
}
type DetailIdentity = {
readonly id?: string | null
readonly index?: number
readonly format?: string
readonly signature?: string | null
}
// The first non-empty signature and format win; a later delta may carry the
// signature for text that streamed earlier.
const mergeDetailIdentity = (previous: DetailIdentity, current: DetailIdentity) => {
const signature = mergeDetailValue(previous.signature, current.signature)
const format = mergeDetailValue(previous.format, current.format)
return {
...(signature === undefined ? {} : { signature }),
...(format === undefined ? {} : { format }),
}
}
const mergeDetailValue = <T>(previous: T | undefined, current: T | undefined) =>
previous || current || (previous !== undefined ? previous : current)
const conflictingReasoningTextDetails = (previous: Record<string, unknown>, current: Record<string, unknown>) =>
const conflictingReasoningDetails = (previous: DetailIdentity, current: DetailIdentity) =>
conflictingDetailValue(previous.id, current.id) ||
conflictingDetailValue(previous.index, current.index) ||
conflictingDetailValue(previous.format, current.format) ||
@@ -930,7 +1012,7 @@ const conflictingDetailValue = (previous: unknown, current: unknown) =>
const reasoningMetadata = (
providerMetadataKey: string,
field: ParserState["reasoningField"],
details?: ReadonlyArray<unknown>,
details?: ReadonlyArray<ReasoningDetail>,
) => ({
[providerMetadataKey]: {
...(field ? { reasoningField: field } : {}),
@@ -993,11 +1075,16 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
}
const reasoningField = state.reasoningField ?? reasoning?.field
const detailDelta = Array.isArray(delta?.reasoning_details) ? delta.reasoning_details : undefined
const reasoningTextObserved = state.reasoningTextObserved || reasoning !== undefined
const detailDelta = Array.isArray(delta?.reasoning_details)
? knownReasoningDetails(delta.reasoning_details)
: undefined
if (detailDelta !== undefined) appendReasoningDetails(state.reasoningDetails, detailDelta)
const reasoningDetailsObserved = state.reasoningDetailsObserved || detailDelta !== undefined
const deltaMetadata = reasoningMetadata(state.providerMetadataKey, reasoningField)
const text = detailDelta?.length ? (detailText(detailDelta) ?? reasoning?.text) : reasoning?.text
const text = detailDelta?.length
? (detailText(detailDelta, reasoningTextObserved) ?? reasoning?.text)
: reasoning?.text
if (text !== undefined) lifecycle = Lifecycle.reasoningDelta(lifecycle, events, "reasoning-0", text, deltaMetadata)
else if (
reasoningDetailsObserved &&
@@ -1093,6 +1180,7 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
finishReason,
lifecycle,
reasoningField,
reasoningTextObserved,
reasoningDetails: state.reasoningDetails,
reasoningDetailsObserved,
reasoningEmitted,
@@ -1173,6 +1261,7 @@ export const protocol = Protocol.make({
toolCallEvents: [],
lifecycle: Lifecycle.initial(),
reasoningField: request.model.compatibility?.reasoningField,
reasoningTextObserved: false,
reasoningDetails: [],
reasoningDetailsObserved: false,
reasoningEmitted: false,
+199 -105
View File
@@ -1,16 +1,21 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Effect, Schema, Stream } from "effect"
import {
ImageFinishEvent,
ImageModel,
ImageOutputEvent,
ImagePartialEvent,
type ImageEvent,
type ImageRequestFor,
} from "../image.js"
import { Media } from "../media.js"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { mergeJsonRecords, type MediaUsage, type OpenString } from "../schema/index.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const ADAPTER = "openai-images"
const NAME = "OpenAI Images"
const PROVIDER = ProviderID.make("openai")
const route = MediaProtocol.identity({ id: "openai-images", name: "OpenAI Images", provider: "openai" })
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/images/generations"
export const EDIT_PATH = "/images/edits"
@@ -19,14 +24,14 @@ export const EDIT_PATH = "/images/edits"
// 1. Public model input
// ---------------------------------------------------------------------------
export type OpenAIImageString<Known extends string> = Known | (string & {})
/** Provider-native options. Common fields (`n`, `size`, `format`, `images`, `mask`) live on the request. */
export type OpenAIImageOptions = {
readonly quality?: OpenAIImageString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
readonly background?: OpenAIImageString<"auto" | "opaque" | "transparent">
readonly moderation?: OpenAIImageString<"auto" | "low">
readonly quality?: OpenString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
readonly background?: OpenString<"auto" | "opaque" | "transparent">
readonly moderation?: OpenString<"auto" | "low">
readonly outputCompression?: number
/** Previews sent before the final image when streaming (default 2); ignored by `Image.generate`. */
readonly partialImages?: number
} & Record<string, unknown>
export type Request = ImageRequestFor<OpenAIImageOptions>
@@ -35,26 +40,68 @@ export type Request = ImageRequestFor<OpenAIImageOptions>
// 2. Response schema
// ---------------------------------------------------------------------------
const OpenAIImageResponse = Schema.Struct({
data: Schema.Array(
Schema.Struct({
b64_json: Schema.optional(Schema.String),
url: Schema.optional(Schema.String),
revised_prompt: Schema.optional(Schema.String),
}),
),
output_format: Schema.optional(Schema.String),
usage: Schema.optional(
Schema.Struct({
input_tokens: Schema.optional(Schema.Number),
output_tokens: Schema.optional(Schema.Number),
total_tokens: Schema.optional(Schema.Number),
input_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
output_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}),
),
const Usage = Schema.Struct({
input_tokens: Schema.optional(Schema.Number),
output_tokens: Schema.optional(Schema.Number),
total_tokens: Schema.optional(Schema.Number),
input_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
output_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
})
/** What the provider actually rendered; it can differ from the request when `auto` or a default applied. */
const Settings = {
output_format: Schema.optional(Schema.String),
size: Schema.optional(Schema.String),
quality: Schema.optional(Schema.String),
background: Schema.optional(Schema.String),
}
const OpenAIImageResponse = Schema.Struct({
data: Schema.Array(Schema.Struct({ b64_json: Schema.String })),
...Settings,
usage: Schema.optional(Usage),
})
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
const StreamEvent = Schema.Union([
Schema.Struct({
type: Schema.Literals(["image_generation.partial_image", "image_edit.partial_image"]),
b64_json: Schema.String,
partial_image_index: Schema.Number,
...Settings,
output_format: Schema.String,
}),
Schema.Struct({
type: Schema.Literals(["image_generation.completed", "image_edit.completed"]),
b64_json: Schema.String,
...Settings,
output_format: Schema.String,
usage: Schema.optional(Usage),
}),
])
const decodeEvent = route.decodeFrame(StreamEvent)
const decodeDocument = Schema.decodeUnknownEffect(Schema.fromJsonString(OpenAIImageResponse))
/** `generate` reads the whole JSON response as one frame, with the requested format for responses that omit it. */
type Frame = string | { readonly document: string; readonly requested: string | undefined }
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
interface State {
readonly completed: number
readonly format?: string
readonly size?: string
readonly quality?: string
readonly background?: string
readonly usage?: MediaUsage
}
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
@@ -64,39 +111,35 @@ const RESERVED_FORM_FIELDS = new Set(["model", "prompt", "image", "image[]", "im
const nativeOptions = (options: OpenAIImageOptions | undefined) => {
if (!options) return undefined
const { outputCompression, ...native } = options
const { outputCompression, partialImages: _, ...native } = options
return { output_compression: outputCompression, ...native }
}
const streamOptions = (request: MediaProtocol.Addressed<Request>) => {
if (request.mode !== "stream") return Effect.succeed(undefined)
if (request.n !== undefined && request.n > 1)
return Effect.fail(
route.unsupported("media.n", `${route.name} streams one image; use Image.generate for n=${request.n}`),
)
return Effect.succeed({ stream: true, partial_images: request.providerOptions?.partialImages ?? 2 })
}
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
const isInline = (asset: Media.Asset) => asset.inline() !== undefined
const isInline = (asset: Media.Asset) => asset.source.type === "bytes" || asset.source.type === "base64"
const blob = (data: Uint8Array, mediaType: string) => {
const buffer = new ArrayBuffer(data.byteLength)
new Uint8Array(buffer).set(data)
return new Blob([buffer], { type: mediaType })
}
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ image_url: inline.dataUrl })
const url = ProviderShared.mediaUrl(asset)
if (url) return Effect.succeed({ image_url: url })
const id = MediaInput.refID(asset, PROVIDER)
if (id) return Effect.succeed({ file_id: id })
return Effect.fail(
ProviderShared.invalidRequest("OpenAI Images accepts image URLs, data URLs, bytes, and OpenAI file IDs"),
const reference = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(
Effect.map((item) => (item.type === "ref" ? { file_id: item.value } : { image_url: item.value })),
)
}
const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Request) {
const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
const images = request.images ?? []
const mask = request.mask
if (mask !== undefined && images.length === 0)
return yield* ProviderShared.invalidRequest("An OpenAI image mask requires at least one input image")
const fields = mergeJsonRecords(
{ n: request.n, size: request.size, output_format: request.format },
{ n: request.n, size: request.size, output_format: request.format, ...(yield* streamOptions(request)) },
nativeOptions(request.providerOptions),
request.http?.body,
)
@@ -104,16 +147,17 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Re
// Owned bytes go through multipart edits; remote URLs and file IDs use the JSON edits body instead.
if (images.length > 0 && images.every(isInline) && (mask === undefined || isInline(mask))) {
const form = new FormData()
form.append("model", request.model.id)
form.append("prompt", request.prompt)
Object.entries(fields ?? {}).forEach(([key, value]) => {
if (RESERVED_FORM_FIELDS.has(key)) return
form.append(key, typeof value === "string" ? value : ProviderShared.encodeJson(value))
})
const uploads = yield* Effect.forEach(images, (image) => MediaInput.inlineBytes(ADAPTER, image))
uploads.forEach((data, index) => form.append("image[]", blob(data, images[index].mediaType), `image-${index}`))
MediaInput.appendFields(
form,
{ model: request.model.id, prompt: request.prompt },
{ overlay: fields, reserved: RESERVED_FORM_FIELDS },
)
const uploads = yield* Effect.forEach(images, (image) => MediaInput.inlineBytes(route.id, image))
uploads.forEach((data, index) =>
form.append("image[]", MediaInput.blob(data, images[index].mediaType), `image-${index}`),
)
if (mask !== undefined)
form.append("mask", blob(yield* MediaInput.inlineBytes(ADAPTER, mask), mask.mediaType), "mask")
form.append("mask", MediaInput.blob(yield* MediaInput.inlineBytes(route.id, mask), mask.mediaType), "mask")
return MediaProtocol.multipart(form)
}
@@ -133,71 +177,121 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Re
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// 6. Stream parsing
// ---------------------------------------------------------------------------
const requestedFormat = (body: MediaProtocol.Body) => {
if (body.type === "binary") return undefined
const value = body.type === "json" ? body.value.output_format : body.value.get("output_format")
return typeof value === "string" ? value : undefined
}
const decodeResponse = Effect.fn("OpenAIImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.DecodeContext<Request>,
) {
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, OpenAIImageResponse)(response)
const decoded = output.value
const format = decoded.output_format ?? requestedFormat(context.body) ?? "png"
const mediaType = `image/${format}`
const images = yield* Effect.forEach(decoded.data, (item, index) => {
const providerMetadata =
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } }
if (item.b64_json)
return MediaInput.decodedAsset(output.invalid, `${NAME} result ${index}`, item.b64_json, mediaType, {
info: { format },
providerMetadata,
})
if (item.url) return Effect.succeed(Media.url(item.url, { mediaType, info: { format }, providerMetadata }))
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
})
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
return new ImageResponse({
images,
usage:
decoded.usage === undefined
? undefined
: {
type: "tokens",
input: decoded.usage.input_tokens,
output: decoded.usage.output_tokens,
total: decoded.usage.total_tokens,
details: { openai: decoded.usage },
},
providerMetadata: { openai: { outputFormat: format } },
const usage = (value: Schema.Schema.Type<typeof Usage> | undefined): MediaUsage | undefined =>
value === undefined
? undefined
: {
type: "tokens",
input: value.input_tokens,
output: value.output_tokens,
total: value.total_tokens,
details: { openai: value },
}
/** `size` echoes the rendered `WIDTHxHEIGHT`; `auto` or any other value leaves the dimensions unknown. */
const info = (format: string, size: string | undefined): Media.Info => {
const match = size?.match(/^(\d+)x(\d+)$/)
return match ? { format, width: Number(match[1]), height: Number(match[2]) } : { format }
}
const eventImage = (frame: string, label: string, data: string, format: string, size: string | undefined) =>
MediaInput.decodedAsset((message, cause) => route.frameError(message, frame, cause), label, data, `image/${format}`, {
info: info(format, size),
})
const onEvent = Effect.fn("OpenAIImages.onEvent")(function* (state: State, frame: string) {
const event = yield* decodeEvent(frame)
const format = event.output_format
if ("partial_image_index" in event) {
const image = yield* eventImage(frame, `${route.name} partial image`, event.b64_json, format, event.size)
return [state, [ImagePartialEvent.make({ index: event.partial_image_index, image })]] as const
}
const image = yield* eventImage(frame, `${route.name} result ${state.completed}`, event.b64_json, format, event.size)
return [
{
completed: state.completed + 1,
format,
size: event.size,
quality: event.quality,
background: event.background,
usage: usage(event.usage),
},
[ImageOutputEvent.make({ index: state.completed, image })],
] as const
})
const onDocument = Effect.fn("OpenAIImages.onDocument")(function* (frame: Exclude<Frame, string>) {
const invalid = (message: string, cause?: unknown) => route.frameError(message, frame.document, cause)
const decoded = yield* decodeDocument(frame.document).pipe(
Effect.mapError((cause) => invalid(`${route.name} returned an invalid response`, cause)),
)
const format = decoded.output_format ?? frame.requested ?? "png"
const images = yield* Effect.forEach(decoded.data, (item, index) =>
MediaInput.decodedAsset(invalid, `${route.name} result ${index}`, item.b64_json, `image/${format}`, {
info: info(format, decoded.size),
}),
)
if (images.length === 0) return yield* invalid(`${route.name} returned no images`)
const state: State = {
completed: images.length,
format,
size: decoded.size,
quality: decoded.quality,
background: decoded.background,
usage: usage(decoded.usage),
}
return [state, images.map((image, index) => ImageOutputEvent.make({ index, image }))] as const
})
const step = (state: State, frame: Frame) => (typeof frame === "string" ? onEvent(state, frame) : onDocument(frame))
const finish = (state: State) => {
if (state.completed === 0) return Effect.fail(route.incomplete())
return Effect.succeed([
ImageFinishEvent.make({
usage: state.usage,
providerMetadata: {
openai: {
outputFormat: state.format,
size: state.size,
quality: state.quality,
background: state.background,
},
},
}),
])
}
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
export const protocol = MediaProtocol.stream<Request, ImageEvent, Frame, State>(route, {
unsupported: ["aspectRatio", "seed"],
body: { from: fromRequest },
response: { decode: decodeResponse },
frames: (bytes, context) =>
context.request.mode === "stream"
? Framing.sse.frame(bytes)
: Framing.document
.frame(bytes)
.pipe(Stream.map((document) => ({ document, requested: requestedFormat(context.body) }))),
initial: () => ({ completed: 0 }),
step,
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<OpenAIImageOptions>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
},
ImageModel.fromRoute<OpenAIImageOptions, Frame, State>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH) },
input,
)
+34 -39
View File
@@ -5,14 +5,12 @@ import { Auth } from "../route/auth.js"
import { Endpoint } from "../route/endpoint.js"
import { Protocol } from "../route/protocol.js"
import { HttpTransport } from "../route/transport/index.js"
import { LLMRequest, mergeJsonRecords, type JsonSchema, type ToolDefinition, type ToolEntry } from "../schema/index.js"
import { LLMRequest, type ToolDefinition, type ToolEntry } from "../schema/index.js"
import { resolveEffortUpdates } from "../effort-updates.js"
import { OpenResponses } from "./open-responses.js"
import { OpenResponsesOptions } from "./utils/open-responses-options.js"
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
import { OpenAIImage } from "./utils/openai-image.js"
import { ResponsesHostedTools } from "./utils/responses-hosted-tools.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
import { OpenResponsesChannel } from "./open-responses-channel.js"
import { ResponsesCompaction } from "./utils/responses-compaction.js"
import { ResponsesCheckpoint } from "./utils/responses-checkpoint.js"
@@ -41,7 +39,16 @@ const OpenAIResponsesImageGenerationTool = Schema.Struct({
output_format: Schema.optional(Schema.Literals(["png", "jpeg", "webp"])),
partial_images: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))),
quality: Schema.optional(Schema.Literals(["auto", "low", "medium", "high"])),
size: Schema.optional(OpenAIImage.Size),
size: Schema.optional(
Schema.String.check(
Schema.makeFilter((value) => {
if (value === "auto") return undefined
const match = /^(\d+)x(\d+)$/.exec(value)
if (!match) return "image size must be `auto` or `{width}x{height}`"
return Number(match[1]) > 0 && Number(match[2]) > 0 ? undefined : "image dimensions must be positive integers"
}),
),
),
})
const OpenAIResponsesHostedToolItem = Schema.Union([
@@ -128,11 +135,6 @@ export const CompactionTrigger = Schema.Struct({ type: Schema.Literal("compactio
const CheckpointBody = Schema.Struct({
...OpenAIResponsesBody.fields,
input: Schema.Array(Schema.Union([OpenAIResponsesInputItem, CompactionTrigger])),
store: Schema.Literal(false),
prompt_cache_retention: optionalNull(Schema.String),
prompt_cache_options: optionalNull(
Schema.Struct({ mode: Schema.optional(Schema.String), ttl: Schema.optional(Schema.String) }),
),
})
const adapter = {
@@ -141,12 +143,14 @@ const adapter = {
restoreHostedToolItem: (item: unknown) => (Schema.is(OpenAIResponsesHostedToolItem)(item) ? item : undefined),
} satisfies OpenResponses.ProviderAdapter
// Only GPT-6 Astra accepts `configuration_update`, and never alongside automatic `context_management` compaction.
// GPT-6 Astra, Sol, and Luna accept `configuration_update` only in standard mode (not `reasoning.mode: "pro"` or
// `-pro` slugs), and never alongside automatic `context_management` compaction.
const supportsEffortUpdates = (request: LLMRequest) => {
if (request.providerOptions?.contextManagement !== undefined) return false
if (Schema.is(Schema.Struct({ mode: Schema.Literal("pro") }))(request.http?.body?.reasoning)) return false
const override = request.model.compatibility?.supportsEffortUpdates
if (override !== undefined) return override
return /(?:^|\/)gpt-6-astra$/i.test(request.model.id)
return /(?:^|\/)gpt-6-(?:astra|sol|luna)$/i.test(request.model.id)
}
const nativeImageToolInput = (tool: ToolDefinition) => {
@@ -159,23 +163,19 @@ const nativeImageTool = (tool: ToolDefinition) => {
return Schema.is(OpenAIResponsesImageGenerationTool)(native) ? native : undefined
}
const lowerTool = Effect.fn("OpenAIResponses.lowerTool")(function* (tool: ToolDefinition, inputSchema: JsonSchema) {
const lowerTool = Effect.fn("OpenAIResponses.lowerTool")(function* (tool: ToolDefinition) {
const native = nativeImageToolInput(tool)
if (native !== undefined) {
if (Schema.is(OpenAIResponsesImageGenerationTool)(native)) return native
return yield* ProviderShared.invalidRequest("OpenAI Responses image generation tool options are invalid")
}
return yield* OpenResponses.lowerTool(NAME, tool, inputSchema)
return yield* OpenResponses.lowerTool(NAME, tool)
})
// Native namespaces hold only function tools, so deeper levels flatten into
// the leaf names the same way non-native protocols flatten the whole tree.
const lowerToolEntry = Effect.fn("OpenAIResponses.lowerToolEntry")(function* (
tool: ToolEntry,
compatibility: Parameters<typeof ToolSchemaProjection.modelCompatibility>[1],
) {
if (tool.type === "tool")
return yield* lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, compatibility))
const lowerToolEntry = Effect.fn("OpenAIResponses.lowerToolEntry")(function* (tool: ToolEntry) {
if (tool.type === "tool") return yield* lowerTool(tool)
// OpenAI requires a namespace description; fall back to a generic one so a
// missing description never blocks the request.
return {
@@ -183,11 +183,13 @@ const lowerToolEntry = Effect.fn("OpenAIResponses.lowerToolEntry")(function* (
name: tool.name,
description: tool.description ?? `Tools in the ${tool.name} namespace.`,
tools: yield* Effect.forEach(ProviderShared.flattenTools(tool.tools), (leaf) =>
OpenResponses.lowerTool(NAME, leaf, ToolSchemaProjection.modelCompatibility(leaf.inputSchema, compatibility)),
OpenResponses.lowerTool(NAME, leaf),
),
}
})
const lowerTools = (request: LLMRequest) => Effect.forEach(request.tools, lowerToolEntry)
const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>, tools: ReadonlyArray<ToolEntry>) =>
ProviderShared.matchToolChoice(NAME, toolChoice, {
auto: () => "auto" as const,
@@ -207,15 +209,11 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request:
)(request.providerOptions?.contextManagement)
const options = OpenResponsesOptions.resolve(request)
const updates = resolveEffortUpdates(request, options.reasoningEffort)
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
return yield* decodeBody({
...(yield* OpenResponses.lowerConversation(updates.request, adapter)),
...OpenResponses.lowerGeneration(request, { ...options, reasoningEffort: updates.effort }),
context_management: management?.map((edit) => ({ type: edit.type, compact_threshold: edit.compactThreshold })),
tools:
request.tools.length === 0
? undefined
: yield* Effect.forEach(request.tools, (tool) => lowerToolEntry(tool, toolSchemaCompatibility)),
tools: request.tools.length === 0 ? undefined : yield* lowerTools(request),
tool_choice:
request.tools.length === 0
? undefined
@@ -227,7 +225,6 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request:
const checkpointBody = {
schema: CheckpointBody,
from: Effect.fn("OpenAIResponses.checkpointBody")(function* (request: LLMRequest) {
const native = yield* fromRequest(LLMRequest.update(request, { toolChoice: undefined }))
const overlay = request.http?.body
// Complete history is required for stateless replay and SSE recovery. Raw input overrides bypass that contract.
if (
@@ -238,18 +235,13 @@ const checkpointBody = {
return yield* ProviderShared.invalidRequest(
"Trigger compaction requires complete canonical history, not an input or continuation override",
)
return yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(CheckpointBody))({
...mergeJsonRecords(native, overlay),
input: [...native.input, { type: "compaction_trigger" }],
stream: true,
store: false,
parallel_tool_calls: true,
tool_choice: undefined,
context_management: undefined,
text: undefined,
max_output_tokens: undefined,
max_tool_calls: undefined,
})
if (overlay?.stream !== undefined && overlay.stream !== true)
return yield* ProviderShared.invalidRequest("Trigger compaction requires a streamed response")
const native = yield* fromRequest(request)
return {
...native,
input: [...native.input, { type: "compaction_trigger" as const }],
}
}),
}
@@ -331,7 +323,10 @@ export const transport = channelTransport({
})
export const route = Route.make({
compact: { endpoint: ResponsesCompaction.make(adapter), trigger: ResponsesCheckpoint.make(checkpointBody) },
compact: {
endpoint: ResponsesCompaction.make(adapter, lowerTools),
trigger: ResponsesCheckpoint.make(checkpointBody),
},
id: ADAPTER,
provider: "openai",
providerMetadataKey: "openai",
+144
View File
@@ -0,0 +1,144 @@
import { Effect, Schema } from "effect"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type MediaUsage } from "../schema/index.js"
import { SpeechModel, type SpeechEvent, type SpeechRequestFor } from "../speech.js"
import { SpeechStream } from "./utils/speech-stream.js"
const route = MediaProtocol.identity({ id: "openai-speech", name: "OpenAI Speech", provider: "openai" })
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/audio/speech"
/** `pcm` is raw 24 kHz, 16-bit signed little-endian mono samples without a header. */
const PCM_SAMPLE_RATE = 24000
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
/** `voice`, `instructions`, `speed`, and `format` are common request fields; other native body fields pass through. */
export type OpenAISpeechOptions = {
/** Defaults to `"sse"` in `stream` mode on models that support it; the merged value selects the response framing. */
readonly stream_format?: "sse" | "audio"
} & Record<string, unknown>
export type Request = SpeechRequestFor<OpenAISpeechOptions>
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
const SpeechStreamEvent = Schema.Union([
Schema.Struct({ type: Schema.Literal("speech.audio.delta"), audio: Schema.Uint8ArrayFromBase64 }),
Schema.Struct({
type: Schema.Literal("speech.audio.done"),
usage: Schema.optional(
Schema.Struct({
input_tokens: Schema.optional(Schema.Number),
output_tokens: Schema.optional(Schema.Number),
total_tokens: Schema.optional(Schema.Number),
}),
),
}),
])
const decodeEvent = route.decodeFrame(SpeechStreamEvent)
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
interface State extends SpeechStream.Audio {
readonly done: boolean
readonly usage?: MediaUsage
}
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
// `sse` is not supported for `tts-1` or `tts-1-hd`; those models stream the raw audio body instead.
const supportsSse = (model: string) => !/^tts-1(-hd)?(-|$)/.test(model)
const fromRequest = Effect.fn("OpenAISpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
// Not in `unsupported`: that list would also reject `timestamps: false`, which asks for nothing.
if (request.timestamps === true)
return yield* route.unsupported("media.timestamps", `${route.name} does not return timestamps`)
return MediaProtocol.json(
mergeJsonRecords(
{
model: request.model.id,
input: request.text,
voice: request.voice,
instructions: request.instructions,
response_format: request.format,
speed: request.speed,
stream_format: request.mode === "stream" && supportsSse(request.model.id) ? "sse" : undefined,
},
request.providerOptions,
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const isSse = (body: MediaProtocol.Body) => body.type === "json" && body.value.stream_format === "sse"
const onEvent = Effect.fn("OpenAISpeech.onEvent")(function* (state: State, frame: string) {
const event = yield* decodeEvent(frame)
if (event.type === "speech.audio.delta") return SpeechStream.delta(state, event.audio)
const usage = event.usage
return [
{
...state,
done: true,
usage:
usage === undefined
? undefined
: {
type: "tokens" as const,
input: usage.input_tokens,
output: usage.output_tokens,
total: usage.total_tokens,
details: { openai: usage },
},
},
[],
] as const
})
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
if (isSse(context.body) && !state.done) return Effect.fail(route.incomplete())
const format = context.request.format ?? "mp3"
return SpeechStream.finish(route, state, {
...(format === "pcm" ? SpeechStream.pcm("pcm_s16le", PCM_SAMPLE_RATE) : SpeechStream.container(format)),
usage: state.usage,
})
}
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, SpeechEvent, string | Uint8Array, State>(route, {
unsupported: ["language"],
body: { from: fromRequest },
frames: (bytes, context) => (isSse(context.body) ? Framing.sse.frame(bytes) : bytes),
initial: () => ({ chunks: [], done: false }),
step: SpeechStream.step(onEvent),
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
SpeechModel.fromRoute<OpenAISpeechOptions, string | Uint8Array, State>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
export const OpenAISpeech = {
protocol,
model,
} as const
@@ -0,0 +1,289 @@
import { Effect, Schema, Stream } from "effect"
import { classifyProviderFailure } from "../provider-error.js"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { AIError, mergeJsonRecords, type MediaUsage } from "../schema/index.js"
import {
TranscriptionFinishEvent,
TranscriptionModel,
TranscriptionSegmentEvent,
TranscriptionTextDeltaEvent,
type TranscriptionEvent,
type TranscriptionRequestFor,
type TranscriptionSegment,
} from "../transcription.js"
import { mediaTypeExtension } from "../utils/media-type.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "openai-transcription", name: "OpenAI Transcription", provider: "openai" })
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/audio/transcriptions"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type OpenAITranscriptionOptions = {
readonly chunking_strategy?:
| "auto"
| {
readonly type: "server_vad"
readonly prefix_padding_ms?: number
readonly silence_duration_ms?: number
readonly threshold?: number
}
readonly include?: ReadonlyArray<"logprobs">
readonly keywords?: ReadonlyArray<string>
readonly known_speaker_names?: ReadonlyArray<string>
readonly known_speaker_references?: ReadonlyArray<string>
readonly temperature?: number
} & Record<string, unknown>
export type Request = TranscriptionRequestFor<OpenAITranscriptionOptions>
// ---------------------------------------------------------------------------
// 3. Streaming event schema
// ---------------------------------------------------------------------------
const Segment = Schema.Struct({
text: Schema.String,
start: Schema.Number,
end: Schema.Number,
speaker: Schema.optional(Schema.String),
})
const Usage = Schema.Union([
Schema.Struct({
type: Schema.Literal("tokens"),
input_tokens: Schema.optional(Schema.Number),
output_tokens: Schema.optional(Schema.Number),
total_tokens: Schema.optional(Schema.Number),
input_token_details: Schema.optional(
Schema.Struct({ audio_tokens: Schema.optional(Schema.Number), text_tokens: Schema.optional(Schema.Number) }),
),
}),
Schema.Struct({ type: Schema.Literal("duration"), seconds: Schema.Number }),
])
const transcriptFields = {
text: Schema.String,
language: Schema.optional(Schema.String),
languages: Schema.optional(Schema.Array(Schema.Struct({ code: Schema.String }))),
duration: Schema.optional(Schema.Number),
segments: Schema.optional(Schema.Array(Segment)),
words: Schema.optional(
Schema.Array(Schema.Struct({ word: Schema.String, start: Schema.Number, end: Schema.Number })),
),
usage: Schema.optional(Usage),
}
/** OpenAI may add stream event types; frames outside `EVENT_TYPES` are ignored. */
const EventType = Schema.Struct({ type: Schema.String })
const Event = Schema.Union([
Schema.Struct({ type: Schema.Literal("transcript.text.delta"), delta: Schema.String }),
Schema.Struct({ type: Schema.Literal("transcript.text.segment"), ...Segment.fields }),
Schema.Struct({ type: Schema.Literal("transcript.text.done"), ...transcriptFields }),
Schema.Struct({
type: Schema.Literal("error"),
message: Schema.optional(Schema.String),
error: Schema.optional(Schema.Struct({ message: Schema.optional(Schema.String) })),
}),
])
const EVENT_TYPES = new Set(["transcript.text.delta", "transcript.text.segment", "transcript.text.done", "error"])
const Transcript = Schema.Struct(transcriptFields)
type Transcript = Schema.Schema.Type<typeof Transcript>
const decodeEventType = route.decodeFrame(EventType)
const decodeEvent = route.decodeFrame(Event)
const decodeTranscript = route.decodeFrame(Transcript)
type Frame = string | { readonly document: string }
// ---------------------------------------------------------------------------
// 4. Parser state
// ---------------------------------------------------------------------------
interface State {
readonly segments: Array<TranscriptionSegment>
readonly transcript?: Transcript
}
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
interface Capabilities {
readonly stream: boolean
readonly timestamps: ReadonlyArray<"segment" | "word">
readonly diarize: boolean
readonly languageField: "language" | "languages"
}
const TRANSCRIBE: Capabilities = { stream: true, timestamps: [], diarize: false, languageField: "language" }
const capabilities = (model: string): Capabilities => {
if (model.startsWith("whisper")) return { ...TRANSCRIBE, stream: false, timestamps: ["segment", "word"] }
if (model.includes("diarize")) return { ...TRANSCRIBE, timestamps: ["segment"], diarize: true }
// `gpt-transcribe` replaces `language` with `languages[]` and rejects both together.
if (model.startsWith("gpt-transcribe")) return { ...TRANSCRIBE, languageField: "languages" }
return TRANSCRIBE
}
/** whisper-1 ignores `stream`, so its `stream` mode sends a plain request and emits only `finish`. */
const streamsEvents = (request: MediaProtocol.Addressed<Request>) =>
request.mode === "stream" && capabilities(request.model.id).stream
const validate = (request: MediaProtocol.Addressed<Request>, model: Capabilities) => {
const id = request.model.id
if (request.diarize === true && !model.diarize)
return Effect.fail(route.unsupported("media.diarize", `${id} does not diarize; use gpt-4o-transcribe-diarize`))
if (request.prompt !== undefined && model.diarize)
return Effect.fail(route.unsupported("media.prompt", `${id} does not accept a prompt`))
if (
request.timestamps === undefined ||
request.timestamps === "none" ||
model.timestamps.includes(request.timestamps)
)
return Effect.void
return Effect.fail(route.unsupported("media.timestamps", `${id} does not return ${request.timestamps} timestamps`))
}
const RESERVED_FORM_FIELDS = new Set([
"file",
"model",
"prompt",
"language",
"response_format",
"timestamp_granularities",
"stream",
])
const fromRequest = Effect.fn("OpenAITranscription.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
const model = capabilities(request.model.id)
yield* validate(request, model)
// The API detects the audio format from the upload's filename extension.
const extension = mediaTypeExtension(request.audio.mediaType)
if (extension === undefined)
return yield* ProviderShared.invalidRequest(
`${route.name} cannot name a ${request.audio.mediaType} upload; send mp3, mp4, m4a, wav, webm, ogg, or flac audio`,
)
const audio = yield* MediaInput.inlineBytes(route.id, request.audio)
const responseFormat = model.diarize
? "diarized_json"
: request.timestamps === undefined || request.timestamps === "none"
? undefined
: "verbose_json"
const form = new FormData()
form.append("file", MediaInput.blob(audio, request.audio.mediaType), `audio.${extension}`)
MediaInput.appendFields(
form,
{
model: request.model.id,
language: model.languageField === "language" ? request.language : undefined,
languages: model.languageField === "languages" && request.language !== undefined ? [request.language] : undefined,
prompt: request.prompt,
response_format: responseFormat,
timestamp_granularities: responseFormat === "verbose_json" ? [request.timestamps] : undefined,
// Diarizing audio longer than 30 seconds requires a chunking strategy.
chunking_strategy: model.diarize ? "auto" : undefined,
stream: streamsEvents(request) ? true : undefined,
},
{
overlay: mergeJsonRecords(request.providerOptions, request.http?.body),
reserved: RESERVED_FORM_FIELDS,
repeatArrays: true,
},
)
return MediaProtocol.multipart(form)
})
// ---------------------------------------------------------------------------
// 6. Stream parsing
// ---------------------------------------------------------------------------
const segment = (value: Schema.Schema.Type<typeof Segment>): TranscriptionSegment => ({
text: value.text.trim(),
startSeconds: value.start,
endSeconds: value.end,
speaker: value.speaker,
})
const onEvent = Effect.fn("OpenAITranscription.onEvent")(function* (state: State, frame: string) {
if (!EVENT_TYPES.has((yield* decodeEventType(frame)).type)) return [state, []] as const
const event = yield* decodeEvent(frame)
if (event.type === "error")
return yield* new AIError({
reason: classifyProviderFailure({
message: `${route.name} stream failed: ${event.message ?? event.error?.message ?? "unknown error"}`,
rawBody: frame,
}),
})
if (event.type === "transcript.text.done") return [{ ...state, transcript: event }, []] as const
if (event.type === "transcript.text.delta")
return [state, event.delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta: event.delta })]] as const
const next = segment(event)
state.segments.push(next)
return [state, [TranscriptionSegmentEvent.make({ segment: next })]] as const
})
const step = (state: State, frame: Frame) =>
typeof frame === "string"
? onEvent(state, frame)
: decodeTranscript(frame.document).pipe(Effect.map((transcript) => [{ ...state, transcript }, []] as const))
const usage = (value: Transcript["usage"]): MediaUsage | undefined => {
if (value === undefined) return undefined
if (value.type === "duration") return { type: "seconds", seconds: value.seconds }
return {
type: "tokens",
input: value.input_tokens,
output: value.output_tokens,
total: value.total_tokens,
details: { openai: value },
}
}
const finish = (state: State) => {
const transcript = state.transcript
if (transcript === undefined) return Effect.fail(route.incomplete())
const segments = transcript.segments?.map(segment) ?? state.segments
return Effect.succeed([
TranscriptionFinishEvent.make({
text: transcript.text,
segments: segments.length === 0 ? undefined : segments,
words: transcript.words?.map((word) => ({ text: word.word, startSeconds: word.start, endSeconds: word.end })),
language: (transcript.language ?? transcript.languages?.[0]?.code)?.toLowerCase(),
durationSeconds: transcript.duration,
usage: usage(transcript.usage),
}),
])
}
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, Frame, State>(route, {
unsupported: ["speakers"],
body: { from: fromRequest },
frames: (bytes, context) =>
streamsEvents(context.request)
? Framing.sse.frame(bytes)
: Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))),
initial: () => ({ segments: [] }),
step,
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
TranscriptionModel.fromRoute<OpenAITranscriptionOptions, Frame, State>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
export const OpenAITranscription = {
protocol,
model,
} as const
@@ -0,0 +1,180 @@
import { Duration, Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import type { Status } from "../generation.js"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type AIError } from "../schema/index.js"
import { ProviderShared, optionalNull } from "./shared.js"
const route = MediaProtocol.identity({ id: "replicate-images", name: "Replicate", provider: "replicate" })
export const DEFAULT_BASE_URL = "https://api.replicate.com"
const OUTPUT_RETENTION = Duration.hours(1)
const MAX_DATA_URL_BYTES = 256 * 1024
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
/** Model-defined `input` fields under the model's own names; `Media.Asset` values lower to URLs or data URLs. */
export type ReplicateImageOptions = Record<string, unknown>
export type Request = ImageRequestFor<ReplicateImageOptions>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
export const Token = Schema.Struct({ id: Schema.String, getURL: Schema.String, cancelURL: Schema.String })
export type Token = Schema.Schema.Type<typeof Token>
const Prediction = Schema.Struct({
id: Schema.String,
status: Schema.String,
output: optionalNull(Schema.Unknown),
error: optionalNull(Schema.Unknown),
data_removed: optionalNull(Schema.Boolean),
completed_at: optionalNull(Schema.String),
metrics: optionalNull(Schema.Struct({ predict_time: optionalNull(Schema.Number) })),
urls: Schema.Struct({ get: Schema.String, cancel: Schema.String }),
})
const Output = Schema.Union([Schema.String, Schema.Array(Schema.String)])
const isOutput = Schema.is(Output)
const STATUS = {
starting: "queued",
processing: "running",
succeeded: "completed",
failed: "failed",
canceled: "cancelled",
// The prediction hit its `Cancel-After` deadline before it started running.
aborted: "expired",
} as const satisfies Record<string, Status>
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
// Official models run by `owner/name`; anything else (`owner/name:version` or a bare version id) is a pinned version.
const isOfficial = (model: string) => /^[^/:]+\/[^/:]+$/.test(model)
const inlineSize = (source: Media.Source) => {
if (source.type === "bytes") return source.data.byteLength
if (source.type === "base64") return source.data.length * 0.75
return 0
}
const fileInput = (asset: Media.Asset) => {
if (inlineSize(asset.source) > MAX_DATA_URL_BYTES)
return Effect.fail(
ProviderShared.invalidRequest(
`${route.name} data URL inputs are limited to 256 KB; pass a larger file by https URL`,
),
)
return ProviderShared.mediaReference(asset, undefined, route.name).pipe(Effect.map((reference) => reference.value))
}
const inputValue = (value: unknown): Effect.Effect<unknown, AIError> => {
if (value instanceof Media.Asset) return fileInput(value)
if (Array.isArray(value) && value.some((item) => item instanceof Media.Asset))
return Effect.forEach(value, (item) => (item instanceof Media.Asset ? fileInput(item) : Effect.succeed(item)))
return Effect.succeed(value)
}
const fromRequest = Effect.fn("ReplicateImages.fromRequest")(function* (request: Request) {
const native = yield* Effect.forEach(Object.entries(request.providerOptions ?? {}), ([key, value]) =>
inputValue(value).pipe(Effect.map((lowered) => [key, lowered] as const)),
)
const input = mergeJsonRecords({ prompt: request.prompt }, Object.fromEntries(native))
const model = request.model.id
return MediaProtocol.json(
mergeJsonRecords(isOfficial(model) ? { input } : { version: model, input }, request.http?.body) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodePrediction = route.decodeJson(Prediction)
const decodeStart = Effect.fn("ReplicateImages.decodeStart")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* decodePrediction(response)
const prediction = output.value
return {
token: { id: prediction.id, getURL: prediction.urls.get, cancelURL: prediction.urls.cancel },
// `Prefer: wait` can return an already-finished prediction, so `await` skips straight to the result.
snapshot: { id: prediction.id, status: yield* MediaProtocol.status(STATUS, prediction.status, output) },
}
})
const decodeStatus = Effect.fn("ReplicateImages.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodePrediction(response)
return { id: context.token.id, status: yield* MediaProtocol.status(STATUS, output.value.status, output) }
})
const decodeResult = Effect.fn("ReplicateImages.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodePrediction(response)
const prediction = output.value
const status = yield* MediaProtocol.status(STATUS, prediction.status, output)
if (status === "failed" || status === "cancelled" || status === "expired")
return yield* output.ended(
status,
`${route.name} prediction ${context.token.id} ${prediction.status}${typeof prediction.error === "string" ? `: ${prediction.error}` : ""}`,
)
if (status !== "completed") return yield* output.pending(context.token.id)
if (prediction.data_removed === true)
return yield* output.ended("expired", `${route.name} removed the output of prediction ${context.token.id}`)
if (!isOutput(prediction.output))
return yield* output.invalid(`${route.name} prediction ${context.token.id} returned output that is not image URLs`)
const urls = typeof prediction.output === "string" ? [prediction.output] : prediction.output
if (urls.length === 0) return yield* output.invalid(`${route.name} prediction ${context.token.id} returned no images`)
const predictTime = prediction.metrics?.predict_time ?? undefined
const completedAt = prediction.completed_at ?? undefined
const expiresAt =
completedAt === undefined ? undefined : Date.parse(completedAt) + Duration.toMillis(OUTPUT_RETENTION)
return new ImageResponse({
images: urls.map((url) => Media.url(url, { expiresAt })),
usage: predictTime === undefined ? undefined : { type: "compute", seconds: predictTime },
providerMetadata: { replicate: { predictionId: prediction.id } },
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.queued<Request, ImageResponse, Token>(route, {
token: Token,
unsupported: ["images", "mask", "n", "size", "aspectRatio", "seed", "format"],
start: { body: { from: fromRequest }, decode: decodeStart },
status: { path: (token) => token.getURL, decode: decodeStatus },
result: { path: (token) => token.getURL, decode: decodeResult },
cancel: { method: "POST", path: (token) => token.cancelURL },
})
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<ReplicateImageOptions, Token>(
{
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) =>
isOfficial(request.model.id) ? `/v1/models/${request.model.id}/predictions` : "/v1/predictions",
},
input,
)
export const ReplicateImages = {
protocol,
model,
} as const
+196
View File
@@ -0,0 +1,196 @@
import { Duration, Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
import { ProviderShared, optionalArray, optionalNull } from "./shared.js"
const route = MediaProtocol.identity({ id: "runway-video", name: "Runway", provider: "runway" })
export const DEFAULT_BASE_URL = "https://api.dev.runwayml.com/v1"
/** Every Runway request must pin the API version. */
export const API_VERSION = "2024-11-06"
export const TEXT_TO_VIDEO_PATH = "/text_to_video"
export const IMAGE_TO_VIDEO_PATH = "/image_to_video"
export const VIDEO_TO_VIDEO_PATH = "/video_to_video"
export const TASKS_PATH = "/tasks"
/** Output URLs are valid for 24–48 hours; the asset carries the conservative bound. */
const OUTPUT_RETENTION = Duration.hours(24)
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
/**
* Provider-native options. Common fields lower to Runway's names: `aspectRatio` → `ratio` (Runway expects pixel
* ratios such as `1280:720` for most models), `durationSeconds` → `duration`, `audio`, `negativePrompt`,
* `resolution`, `references`, and `frames` → `promptImage`.
*/
export type RunwayVideoOptions = {
readonly contentModeration?: { readonly publicFigureThreshold?: OpenString<"auto" | "low"> }
readonly outputFormat?: OpenString<"mp4" | "prores" | "png_sequence">
} & Record<string, unknown>
export type Request = VideoRequestFor<RunwayVideoOptions>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
export const Token = Schema.Struct({ taskID: Schema.String })
export type Token = Schema.Schema.Type<typeof Token>
const Cost = Schema.Struct({ credits: Schema.Number })
const StartResponse = Schema.Struct({ id: Schema.String })
const Task = Schema.Struct({
status: Schema.String,
progress: optionalNull(Schema.Number),
output: optionalArray(Schema.String),
failure: optionalNull(Schema.String),
failureCode: optionalNull(Schema.String),
cost: Schema.optional(Cost),
estimatedCost: Schema.optional(Cost),
})
const STATUS = {
PENDING: "queued",
THROTTLED: "queued",
RUNNING: "running",
SUCCEEDED: "completed",
FAILED: "failed",
CANCELLED: "cancelled",
} as const satisfies Record<string, Status>
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
// Runway accepts HTTPS URLs, `runway://` upload URIs, and data URIs, all as one string.
const mediaUri = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(Effect.map((reference) => reference.value))
const fromRequest = Effect.fn("RunwayVideo.fromRequest")(function* (request: Request) {
const first = request.frames?.first === undefined ? undefined : yield* mediaUri(request.frames.first)
const last = request.frames?.last === undefined ? undefined : yield* mediaUri(request.frames.last)
const promptImage = [
...(first === undefined ? [] : [{ uri: first, position: "first" }]),
...(last === undefined ? [] : [{ uri: last, position: "last" }]),
]
const videoUri = request.video === undefined ? undefined : yield* mediaUri(request.video)
const references = yield* Effect.forEach(request.references ?? [], (asset) =>
mediaUri(asset).pipe(Effect.map((uri) => ({ uri }))),
)
return MediaProtocol.json(
mergeJsonRecords(
{
model: request.model.id,
promptText: request.prompt,
promptImage: promptImage.length === 0 ? undefined : promptImage,
videoUri,
references: references.length === 0 ? undefined : references,
ratio: request.aspectRatio,
duration: request.durationSeconds,
resolution: request.resolution,
audio: request.audio,
negativePrompt: request.negativePrompt,
seed: request.seed,
},
request.providerOptions,
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
token: { taskID: value.id },
snapshot: { id: value.id, status: "queued" },
}))
const decodeTask = route.decodeJson(Task)
const decodeStatus = Effect.fn("RunwayVideo.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeTask(response)
const status = yield* MediaProtocol.status(STATUS, output.value.status, output)
return { id: context.token.taskID, status, progress: output.value.progress ?? undefined }
})
const decodeResult = Effect.fn("RunwayVideo.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeTask(response)
const task = output.value
const status = yield* MediaProtocol.status(STATUS, task.status, output)
if (status === "failed") {
const code = task.failureCode ?? undefined
const message = `${route.name} task failed${code === undefined ? "" : ` (${code})`}${task.failure ? `: ${task.failure}` : ""}`
// Runway failure codes are dotted paths; every moderation outcome carries a SAFETY segment.
if (code !== undefined && /(^|\.)SAFETY(\.|$)/.test(code)) return yield* output.contentPolicy(message)
return yield* output.ended("failed", message)
}
if (status === "cancelled")
return yield* output.ended("cancelled", `${route.name} task ${context.token.taskID} was cancelled`)
if (status !== "completed") return yield* output.pending(context.token.taskID)
const urls = task.output ?? []
if (urls.length === 0) return yield* output.invalid(`${route.name} task succeeded without any output`)
return new VideoResponse({
videos: yield* Effect.forEach(urls, (url) =>
MediaProtocol.expiringUrl(url, OUTPUT_RETENTION, { mediaType: "video/mp4" }),
),
usage: task.cost === undefined ? undefined : { type: "credits", credits: task.cost.credits },
providerMetadata: {
runway: {
taskId: context.token.taskID,
estimatedCredits: task.estimatedCost?.credits,
},
},
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
const taskPath = (token: Token) => `${TASKS_PATH}/${token.taskID}`
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>(route, {
token: Token,
unsupported: ["n"],
start: { body: { from: fromRequest }, decode: decodeStart },
status: { path: taskPath, decode: decodeStatus },
result: { path: taskPath, decode: decodeResult },
cancel: { method: "DELETE", path: taskPath, activeOnly: true },
})
const startPath = (request: Request) => {
if (request.video !== undefined) return VIDEO_TO_VIDEO_PATH
if (request.frames?.first !== undefined || request.frames?.last !== undefined) return IMAGE_TO_VIDEO_PATH
return TEXT_TO_VIDEO_PATH
}
export const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<RunwayVideoOptions, Token>(
{
protocol,
baseURL: DEFAULT_BASE_URL,
headers: { "X-Runway-Version": API_VERSION },
path: ({ request }) => startPath(request),
},
input,
)
export const RunwayVideo = {
protocol,
model,
} as const
+36 -50
View File
@@ -1,6 +1,5 @@
import { Tool } from "@opencode/schema/tool"
import { Effect, Option, Schema, Stream } from "effect"
import * as Sse from "effect/unstable/encoding/Sse"
import { Effect, Option, Schema } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { Media } from "../media.js"
import {
@@ -13,17 +12,16 @@ import {
ToolDefinition,
type ContentPart,
type MediaPart,
type OpenString,
type ProviderID,
type TextPart,
type ToolEntry,
type ToolResultPart,
} from "../schema/index.js"
import { Json, decodeJson, encodeJson } from "../utils/json.js"
import { isRecord } from "../utils/record.js"
export { isRecord }
export { Json, decodeJson, encodeJson, isRecord }
export const Json = Schema.fromJsonString(Schema.Unknown)
export const decodeJson = Schema.decodeUnknownSync(Json)
export const encodeJson = Schema.encodeSync(Json)
const isJson = Schema.is(Schema.Json)
export const JsonObject = Schema.Record(Schema.String, Schema.Unknown)
export const optionalArray = <const S extends Schema.Top>(schema: S) => Schema.optional(Schema.Array(schema))
@@ -35,7 +33,7 @@ export const lenient = <const S extends Schema.Top>(schema: S) =>
)
/** Provider-defined string enum: known values for autocomplete, any string accepted at runtime. */
export const knownString = <Known extends string>() =>
Schema.declare<Known | (string & {})>((value): value is Known | (string & {}) => typeof value === "string", {
Schema.declare<OpenString<Known>>((value): value is OpenString<Known> => typeof value === "string", {
expected: "string",
})
@@ -112,6 +110,14 @@ export const sumTokens = (...values: ReadonlyArray<number | undefined>): number
return values.reduce((acc: number, value) => acc + (value ?? 0), 0)
}
/**
* Caps an explicit thinking budget at half the output limit. Thinking counts against the output limit, so a budget
* near it leaves the answer, a tool call, or a summary without room. Smaller budgets, special values such as `-1` and
* `0`, and requests without an output limit pass through unchanged.
*/
export const fitThinkingBudget = (budget: number, maxTokens: number | undefined, minimum = 1) =>
maxTokens === undefined || budget <= maxTokens / 2 ? budget : Math.max(minimum, Math.floor(maxTokens / 2))
export const eventError = (route: string, message: string, body?: string, cause?: unknown) =>
new AIError({
reason: new InvalidProviderOutputError({ route, message, body, cause }),
@@ -192,6 +198,29 @@ export const inlineRequired = (route: string, asset: Media.Asset) =>
/** The remote URL of a `url` asset, for protocols that accept `http(s)` references natively. */
export const mediaUrl = (asset: Media.Asset) => (asset.source.type === "url" ? asset.source.url : undefined)
export type MediaReference = { readonly type: "dataUrl" | "url" | "ref"; readonly value: string }
/**
* The one string a provider can address an asset by: inline payloads as a data URL, `url` sources as their URL, and
* this provider's own `ref` as its id. Other providers' refs are never forwarded and fail typed; omit `provider` for
* APIs with no file handles at all.
*/
export const mediaReference = (
asset: Media.Asset,
provider: ProviderID | undefined,
label: string,
): Effect.Effect<MediaReference, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ type: "dataUrl", value: inline.dataUrl })
const url = mediaUrl(asset)
if (url) return Effect.succeed({ type: "url", value: url })
if (provider !== undefined && asset.source.type === "ref" && asset.source.provider === provider)
return Effect.succeed({ type: "ref", value: asset.source.id })
const accepted = provider === undefined ? "" : `, and ${provider} references`
const got = asset.source.type === "ref" ? `; got ${asset.source.provider}:${asset.source.id}` : ""
return Effect.fail(invalidRequest(`${label} accepts inline bytes, data URLs, http(s) URLs${accepted}${got}`))
}
/**
* Lift a tool-result file into a `MediaPart`. Tool files carry either a data URL, an `http(s)` URL, or raw base64 in
* `uri`; the declared `mime` wins over any data-URL prefix so tool authors control the type the model sees.
@@ -206,8 +235,6 @@ export const toolFileMedia = (item: Tool.FileContent): MediaPart => {
return Message.media(asset, { filename: item.name })
}
export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
export const toolResultText = (part: ToolResultPart) => {
if (part.result.type === "text") return String(part.result.value)
if (part.result.type === "error") {
@@ -229,47 +256,6 @@ export const errorText = (error: unknown) => {
return "Unknown stream error"
}
/**
* `framing` step for Server-Sent Events. Decodes UTF-8, runs the SSE channel
* decoder, optionally filters named events, and drops empty events. `[DONE]`
* is dropped by default or retained for protocols that use it as their stream
* boundary. Retry control events are ignored without interrupting the stream.
* Decoder failures become provider output errors so the public error channel
* stays `AIError`.
*/
export const sseFraming = (
bytes: Stream.Stream<Uint8Array, AIError>,
events?: ReadonlySet<string>,
includeDone = false,
): Stream.Stream<string, AIError> =>
bytes.pipe(
Stream.decodeText(),
Stream.mapAccumEffect(
() => {
const output: Sse.Event[] = []
return {
output,
parser: Sse.makeParser((event) => {
if (event._tag === "Event") output.push(event)
}),
}
},
(state, chunk) =>
Effect.gen(function* () {
const error = state.parser.feed(chunk)
if (error) return yield* eventError("sse", error.message, chunk, error)
return [state, state.output.splice(0)] as const
}),
),
Stream.filter(
(event) =>
(events === undefined || events.has(event.event)) &&
event.data.length > 0 &&
(event.data !== "[DONE]" || includeDone || (events !== undefined && event.event !== "message")),
),
Stream.map((event) => event.data),
)
/**
* Canonical invalid-request constructor shared by protocol lowering.
*/
@@ -0,0 +1,229 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const route = MediaProtocol.identity({ id: "stability-images", name: "Stability AI", provider: "stability" })
const upscaleRoute = MediaProtocol.identity({ id: "stability-upscale", name: "Stability AI", provider: "stability" })
export const DEFAULT_BASE_URL = "https://api.stability.ai"
const RESULTS_PATH = "/v2beta/results"
const UPSCALE_MODEL = "creative"
/** Base64 JSON instead of raw bytes, so the seed and finish reason arrive with the image. */
const HEADERS = { accept: "application/json" }
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type StabilityStylePreset = OpenString<
| "enhance"
| "anime"
| "photographic"
| "digital-art"
| "comic-book"
| "fantasy-art"
| "line-art"
| "analog-film"
| "neon-punk"
| "isometric"
| "low-poly"
| "origami"
| "modeling-compound"
| "cinematic"
| "3d-model"
| "pixel-art"
| "tile-texture"
>
export type StabilityImageOptions = {
readonly negative_prompt?: string
readonly style_preset?: StabilityStylePreset
readonly strength?: number
readonly cfg_scale?: number
} & Record<string, unknown>
/** Creative upscale options; the one `images` source is the image to upscale. */
export type StabilityUpscaleOptions = {
readonly negative_prompt?: string
readonly style_preset?: StabilityStylePreset
readonly creativity?: number
} & Record<string, unknown>
export type Request = ImageRequestFor<StabilityImageOptions>
export type UpscaleRequest = ImageRequestFor<StabilityUpscaleOptions>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
export const Token = Schema.Struct({ id: Schema.String })
export type Token = Schema.Schema.Type<typeof Token>
// Generate names the base64 image `image`; `/v2beta/results/{id}` names it `result`.
const ImageDocument = Schema.Struct({
image: Schema.optional(Schema.String),
result: Schema.optional(Schema.String),
seed: Schema.optional(Schema.Number),
finish_reason: Schema.optional(Schema.String),
})
const Started = Schema.Struct({ id: Schema.String })
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
/** `sd3.5-*` ids share the `sd3` endpoint and select the model with a form field. */
const endpoint = (model: string) => (model.startsWith("sd3") ? "sd3" : model)
const RESERVED_FORM_FIELDS = new Set(["image", "prompt", "mode", "model"])
const form = Effect.fn("StabilityImages.form")(function* (
identity: MediaProtocol.Identity,
fields: Record<string, unknown>,
native: Record<string, unknown> | undefined,
source: Request["images"],
) {
if ((source?.length ?? 0) > 1)
return yield* identity.unsupported("media.images", `${identity.name} takes one source image`)
const body = new FormData()
MediaInput.appendFields(body, fields, { overlay: native, reserved: RESERVED_FORM_FIELDS })
const image = source?.[0]
if (image !== undefined)
body.append("image", MediaInput.blob(yield* MediaInput.inlineBytes(identity.id, image), image.mediaType), "image")
return MediaProtocol.multipart(body)
})
const fromRequest = Effect.fn("StabilityImages.fromRequest")(function* (request: Request) {
if (request.n !== undefined && request.n > 1)
return yield* route.unsupported("media.n", `${route.name} generates one image per request; call it once per image`)
const target = endpoint(request.model.id)
const edit = (request.images?.length ?? 0) > 0
if (edit && target === "core")
return yield* route.unsupported("media.images", `${route.name} core is text-to-image only; use ultra or sd3.5-*`)
return yield* form(
route,
{
prompt: request.prompt,
aspect_ratio: request.aspectRatio,
seed: request.seed,
output_format: request.format,
model: target === "sd3" ? request.model.id : undefined,
mode: target === "sd3" && edit ? "image-to-image" : undefined,
},
mergeJsonRecords(request.providerOptions, request.http?.body),
request.images,
)
})
const fromUpscaleRequest = Effect.fn("StabilityImages.fromUpscaleRequest")(function* (request: UpscaleRequest) {
if ((request.images?.length ?? 0) === 0)
return yield* ProviderShared.invalidRequest(`${upscaleRoute.name} upscale requires the source image in images`)
return yield* form(
upscaleRoute,
{ prompt: request.prompt, seed: request.seed, output_format: request.format },
mergeJsonRecords(request.providerOptions, request.http?.body),
request.images,
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeImageDocument = (identity: MediaProtocol.Identity) => {
const decode = identity.decodeJson(ImageDocument)
return Effect.fn("StabilityImages.decodeImage")(function* (response: HttpClientResponse.HttpClientResponse) {
const output = yield* decode(response)
const document = output.value
const data = document.image ?? document.result
if (data === undefined) return yield* output.invalid(`${identity.name} returned no image`)
const image = yield* MediaInput.decodedAsset(output.invalid, `${identity.name} result`, data, undefined)
return new ImageResponse({
images: [image],
notices:
document.finish_reason === "CONTENT_FILTERED"
? [{ type: "moderated", message: `${identity.name} blurred the image for violating its content policy` }]
: undefined,
providerMetadata: { stability: { seed: document.seed, finishReason: document.finish_reason } },
})
})
}
const decodeResponse = decodeImageDocument(route)
const decodeUpscaleImage = decodeImageDocument(upscaleRoute)
const decodeStart = upscaleRoute.decodeStarted(Started, (value) => ({
token: { id: value.id },
snapshot: { id: value.id, status: "queued" },
}))
// `/v2beta/results/{id}` answers 202 while in progress and 200 with the finished image document.
const decodeStatus = (response: HttpClientResponse.HttpClientResponse, context: MediaProtocol.PollContext<Token>) =>
Effect.succeed({
id: context.token.id,
status: response.status === 202 ? ("running" as const) : ("completed" as const),
})
const decodeUpscaleResult = Effect.fn("StabilityImages.decodeUpscaleResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
if (response.status === 202) {
const output = yield* upscaleRoute.text(response)
return yield* output.pending(context.token.id)
}
return yield* decodeUpscaleImage(response)
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
unsupported: ["size", "mask"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const upscaleProtocol = MediaProtocol.queued<UpscaleRequest, ImageResponse, Token>(upscaleRoute, {
token: Token,
unsupported: ["n", "size", "aspectRatio", "mask"],
start: { body: { from: fromUpscaleRequest }, decode: decodeStart },
status: { path: (token) => `${RESULTS_PATH}/${token.id}`, decode: decodeStatus },
result: { path: (token) => `${RESULTS_PATH}/${token.id}`, decode: decodeUpscaleResult },
})
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<StabilityImageOptions>(
{
protocol,
baseURL: DEFAULT_BASE_URL,
headers: HEADERS,
path: ({ request }) => `/v2beta/stable-image/generate/${endpoint(request.model.id)}`,
},
input,
)
export const upscaleModel = (input: Omit<MediaRoute.ModelInput, "id">) =>
ImageModel.fromRoute<StabilityUpscaleOptions, Token>(
{
protocol: upscaleProtocol,
baseURL: DEFAULT_BASE_URL,
headers: HEADERS,
// Only the creative upscaler is asynchronous; fast and conservative answer inline.
path: `/v2beta/stable-image/upscale/${UPSCALE_MODEL}`,
},
{ ...input, id: UPSCALE_MODEL },
)
export const StabilityImages = {
protocol,
upscaleProtocol,
model,
upscaleModel,
} as const
@@ -0,0 +1,91 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import type { Status } from "../../generation.js"
import type { Media } from "../../media.js"
import { MediaProtocol } from "../../route/media-protocol.js"
import type { AIError } from "../../schema/index.js"
import { ProviderShared, optionalNull } from "../shared.js"
export const DEFAULT_BASE_URL = "https://queue.fal.run"
/** fal hands back absolute follow-up URLs on submit; they are authoritative for status, result, and cancel. */
export const Token = Schema.Struct({
requestID: Schema.String,
statusURL: Schema.String,
responseURL: Schema.String,
cancelURL: Schema.String,
})
export type Token = Schema.Schema.Type<typeof Token>
const StartResponse = Schema.Struct({
request_id: Schema.String,
status_url: Schema.String,
response_url: Schema.String,
cancel_url: Schema.String,
queue_position: optionalNull(Schema.Number),
})
const QueueStatus = Schema.Struct({
status: Schema.String,
queue_position: optionalNull(Schema.Number),
error: optionalNull(Schema.Unknown),
})
const STATUS = {
IN_QUEUE: "queued",
IN_PROGRESS: "running",
COMPLETED: "completed",
} as const satisfies Record<string, Status>
// fal accepts public URLs and data URIs; there is no provider file handle to forward.
export const mediaUrl = (asset: Media.Asset, name: string) =>
ProviderShared.mediaReference(asset, undefined, name).pipe(Effect.map((reference) => reference.value))
export const protocol = <Request, Response>(
route: MediaProtocol.Identity,
input: {
readonly unsupported?: ReadonlyArray<keyof Request & string>
readonly from: (request: Request) => Effect.Effect<MediaProtocol.Body, AIError>
readonly decodeResult: (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) => Effect.Effect<Response, AIError>
},
) => {
const decodeQueueStatus = route.decodeJson(QueueStatus)
return MediaProtocol.queued<Request, Response, Token>(route, {
token: Token,
unsupported: input.unsupported,
start: {
body: { from: input.from },
decode: route.decodeStarted(StartResponse, (value) => ({
token: {
requestID: value.request_id,
statusURL: value.status_url,
responseURL: value.response_url,
cancelURL: value.cancel_url,
},
snapshot: { id: value.request_id, status: "queued", position: value.queue_position ?? undefined },
})),
},
status: {
path: (token) => token.statusURL,
decode: Effect.fn("FalQueue.decodeStatus")(function* (response, context) {
const output = yield* decodeQueueStatus(response)
const decoded = output.value
const status = yield* MediaProtocol.status(STATUS, decoded.status, output)
// fal reports request failures as COMPLETED with an `error`; the response endpoint carries the details.
const failed = status === "completed" && decoded.error !== undefined && decoded.error !== null
return {
id: context.token.requestID,
status: failed ? "failed" : status,
position: status === "queued" ? (decoded.queue_position ?? undefined) : undefined,
}
}),
},
result: { path: (token) => token.responseURL, decode: input.decodeResult },
cancel: { method: "PUT", path: (token) => token.cancelURL },
})
}
export * as FalQueue from "./fal-queue.js"
@@ -0,0 +1,113 @@
import { Effect, Schema, type Stream } from "effect"
import type { Media } from "../../media.js"
import { Framing } from "../../route/framing.js"
import type { MediaProtocol } from "../../route/media-protocol.js"
import { AIError, ContentPolicyError, ProviderID, type MediaUsage, type ProviderMetadata } from "../../schema/index.js"
import { ProviderShared } from "../shared.js"
import { MediaInput } from "./media-input.js"
const PROVIDER = ProviderID.make("google")
const UsageMetadata = Schema.Struct({
promptTokenCount: Schema.optional(Schema.Number),
candidatesTokenCount: Schema.optional(Schema.Number),
totalTokenCount: Schema.optional(Schema.Number),
})
type UsageMetadata = Schema.Schema.Type<typeof UsageMetadata>
export const chunk = <const Part extends Schema.Top>(part: Part) =>
Schema.Struct({
candidates: Schema.optional(
Schema.Array(
Schema.Struct({
content: Schema.optional(Schema.Struct({ parts: Schema.optional(Schema.Array(part)) })),
finishReason: Schema.optional(Schema.String),
}),
),
),
promptFeedback: Schema.optional(
Schema.Struct({
blockReason: Schema.optional(Schema.String),
blockReasonMessage: Schema.optional(Schema.String),
}),
),
usageMetadata: Schema.optional(UsageMetadata),
modelVersion: Schema.optional(Schema.String),
responseId: Schema.optional(Schema.String),
})
interface Chunk {
readonly candidates?: ReadonlyArray<{ readonly finishReason?: string }>
readonly promptFeedback?: { readonly blockReason?: string; readonly blockReasonMessage?: string }
readonly usageMetadata?: UsageMetadata
readonly modelVersion?: string
readonly responseId?: string
}
export interface Metadata {
readonly usage?: UsageMetadata
readonly finishReason?: string
readonly modelVersion?: string
readonly responseId?: string
}
export const track = <State extends Metadata>(state: State, chunk: Chunk): State => ({
...state,
usage: chunk.usageMetadata ?? state.usage,
finishReason: chunk.candidates?.[0]?.finishReason ?? state.finishReason,
modelVersion: chunk.modelVersion ?? state.modelVersion,
responseId: chunk.responseId ?? state.responseId,
})
export const blocked = (name: string, chunk: Chunk, frame: string) => {
const feedback = chunk.promptFeedback
if (feedback?.blockReason === undefined) return undefined
return new AIError({
reason: new ContentPolicyError({
message: `${name} blocked the request (${feedback.blockReason})${
feedback.blockReasonMessage === undefined ? "" : `: ${feedback.blockReasonMessage}`
}`,
body: frame,
}),
})
}
export const usage = (usage: UsageMetadata | undefined): MediaUsage | undefined =>
usage === undefined
? undefined
: {
type: "tokens",
input: usage.promptTokenCount,
output: usage.candidatesTokenCount,
total: ProviderShared.totalTokens(usage.promptTokenCount, usage.candidatesTokenCount, usage.totalTokenCount),
details: { google: usage },
}
export const providerMetadata = (state: Metadata): ProviderMetadata => ({
google: { finishReason: state.finishReason, modelVersion: state.modelVersion, responseId: state.responseId },
})
export const path = (model: string, mode: MediaProtocol.Mode) =>
mode === "stream" ? `/models/${model}:streamGenerateContent?alt=sse` : `/models/${model}:generateContent`
// `generateContent` answers with one document shaped exactly like a streamed chunk, so it is a single frame.
export const frames = (bytes: Stream.Stream<Uint8Array, AIError>, mode: MediaProtocol.Mode) =>
mode === "stream" ? Framing.sse.frame(bytes) : Framing.document.frame(bytes)
// Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs.
export const mediaPart = (
route: string,
asset: Media.Asset,
): Effect.Effect<
| { readonly fileData: { readonly mimeType: string; readonly fileUri: string } }
| { readonly inlineData: { readonly mimeType: string; readonly data: string } },
AIError
> => {
const fileUri = MediaInput.refID(asset, PROVIDER)
if (fileUri !== undefined) return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri } })
return ProviderShared.requireInlineMedia(route, asset).pipe(
Effect.map((media) => ({ inlineData: { mimeType: media.mime, data: media.base64 } })),
)
}
export * as GeminiGenerateContent from "./gemini-generate-content.js"
@@ -0,0 +1,77 @@
import type { JsonSchema } from "../../schema/index.js"
import { isRecord } from "../../utils/record.js"
// Gemini's `parametersJsonSchema` accepts standard JSON Schema, but rejects a few shapes that
// published tool schemas commonly contain. Rewrite only those and send everything else unchanged.
const SCHEMA_MAPS = new Set([
"properties",
"patternProperties",
"$defs",
"definitions",
"dependentSchemas",
"dependencies",
])
const VALUES = new Set(["const", "default", "enum", "examples", "dependentRequired"])
const mapValues = (record: Record<string, unknown>, map: (value: unknown, key: string) => unknown) =>
Object.fromEntries(Object.entries(record).map(([key, value]) => [key, map(value, key)]))
const normalizeNode = (schema: unknown): unknown => {
if (Array.isArray(schema)) return schema.map(normalizeNode)
if (!isRecord(schema)) return schema
const properties = isRecord(schema.properties) ? schema.properties : undefined
return Object.fromEntries(
Object.entries(schema).flatMap(([key, value]) => {
if (VALUES.has(key)) return [[key, value]]
if (SCHEMA_MAPS.has(key) && isRecord(value)) return [[key, mapValues(value, normalizeNode)]]
// `required` may only name declared properties.
if (key === "required" && properties && Array.isArray(value))
return [[key, value.filter((name) => typeof name === "string" && Object.hasOwn(properties, name))]]
// Draft-04 boolean exclusive bounds become the numeric form.
if (key === "exclusiveMinimum" && typeof value === "boolean")
return value && typeof schema.minimum === "number" ? [[key, schema.minimum]] : []
if (key === "exclusiveMaximum" && typeof value === "boolean")
return value && typeof schema.maximum === "number" ? [[key, schema.maximum]] : []
if (key === "minimum" && schema.exclusiveMinimum === true) return []
if (key === "maximum" && schema.exclusiveMaximum === true) return []
// Draft-07 tuples (`items` array plus `additionalItems`) are `prefixItems` plus `items` in 2020-12.
if (key === "items" && Array.isArray(value)) return [["prefixItems", value.map(normalizeNode)]]
if (key === "additionalItems" && Array.isArray(schema.items)) return [["items", normalizeNode(value)]]
return [[key, normalizeNode(value)]]
}),
)
}
// Gemini accepts a recursive `$ref` only when the loop passes through an optional property or
// potentially empty array `items`. Replace other self-references with an unconstrained schema.
const cutLoops = (schema: unknown, target: string, safe: boolean): unknown => {
if (Array.isArray(schema)) return schema.map((item) => cutLoops(item, target, safe))
if (!isRecord(schema)) return schema
if (schema.$ref === target && !safe) return {}
const required = Array.isArray(schema.required) ? schema.required : []
return mapValues(schema, (value, key) => {
if (VALUES.has(key)) return value
if (key === "items") return cutLoops(value, target, safe || !(Number(schema.minItems) > 0))
if (key === "properties" && isRecord(value))
return mapValues(value, (child, name) => cutLoops(child, target, safe || !required.includes(name)))
if (SCHEMA_MAPS.has(key) && isRecord(value)) return mapValues(value, (child) => cutLoops(child, target, safe))
return cutLoops(value, target, safe)
})
}
export const normalize = (schema: JsonSchema): JsonSchema => {
const normalized = normalizeNode(schema)
if (!isRecord(normalized)) return {}
const result = cutLoops(
mapValues(normalized, (value, key) =>
(key === "$defs" || key === "definitions") && isRecord(value)
? mapValues(value, (def, name) => cutLoops(def, `#/${key}/${name}`, false))
: value,
),
"#",
false,
)
return isRecord(result) ? result : {}
}
export * as GeminiJsonSchema from "./gemini-json-schema.js"
@@ -1,119 +0,0 @@
import { isRecord } from "../../utils/record.js"
// Gemini accepts a JSON Schema-like dialect for tool parameters, but rejects a
// handful of common JSON Schema shapes. Keep this projection isolated so the
// Gemini protocol file still reads like the other protocol modules.
const SCHEMA_INTENT_KEYS = [
"type",
"properties",
"items",
"prefixItems",
"enum",
"const",
"$ref",
"additionalProperties",
"patternProperties",
"required",
"not",
"if",
"then",
"else",
]
const hasCombiner = (schema: unknown) =>
isRecord(schema) && (Array.isArray(schema.anyOf) || Array.isArray(schema.oneOf) || Array.isArray(schema.allOf))
const hasSchemaIntent = (schema: unknown) =>
isRecord(schema) && (hasCombiner(schema) || SCHEMA_INTENT_KEYS.some((key) => key in schema))
const sanitizeNode = (schema: unknown): unknown => {
if (!isRecord(schema)) return Array.isArray(schema) ? schema.map(sanitizeNode) : schema
const result: Record<string, unknown> = Object.fromEntries(
Object.entries(schema).map(([key, value]) => [
key,
key === "enum" && Array.isArray(value) ? value.map(String) : sanitizeNode(value),
]),
)
if (Array.isArray(result.enum) && (result.type === "integer" || result.type === "number")) result.type = "string"
const properties = result.properties
if (result.type === "object" && isRecord(properties) && Array.isArray(result.required)) {
result.required = result.required.filter((field) => typeof field === "string" && field in properties)
}
if (result.type === "array" && !hasCombiner(result)) {
result.items = result.items ?? {}
if (isRecord(result.items) && !hasSchemaIntent(result.items)) result.items = { ...result.items, type: "string" }
}
if (typeof result.type === "string" && result.type !== "object" && !hasCombiner(result)) {
delete result.properties
delete result.required
}
return result
}
const emptyObjectSchema = (schema: Record<string, unknown>) =>
schema.type === "object" &&
(!isRecord(schema.properties) || Object.keys(schema.properties).length === 0) &&
!schema.additionalProperties
const projectNode = (schema: unknown, nested = false): Record<string, unknown> | undefined => {
if (!isRecord(schema)) return undefined
if (!nested && emptyObjectSchema(schema)) return undefined
const types = Array.isArray(schema.type) ? schema.type.filter((type) => type !== "null") : undefined
const anyOf = Array.isArray(schema.anyOf) ? schema.anyOf : undefined
const hasNullAnyOf = anyOf?.some((item) => isRecord(item) && item.type === "null") ?? false
const anyOfTypes = hasNullAnyOf ? anyOf?.filter((item) => !isRecord(item) || item.type !== "null") : anyOf
const flattenedAnyOf = hasNullAnyOf && anyOfTypes?.length === 1 ? projectNode(anyOfTypes[0], true) : undefined
const result = Object.fromEntries(
[
["description", schema.description],
["required", schema.required],
["format", schema.format],
["type", types ? (types.length === 0 ? "null" : undefined) : schema.type],
[
"nullable",
(Array.isArray(schema.type) && schema.type.includes("null") && types && types.length > 0) || hasNullAnyOf
? true
: undefined,
],
["enum", schema.const !== undefined ? [schema.const] : schema.enum],
[
"properties",
isRecord(schema.properties)
? Object.fromEntries(Object.entries(schema.properties).map(([key, value]) => [key, projectNode(value, true)]))
: undefined,
],
[
"items",
Array.isArray(schema.items)
? schema.items.map((item) => projectNode(item, true))
: schema.items === undefined
? undefined
: projectNode(schema.items, true),
],
["allOf", Array.isArray(schema.allOf) ? schema.allOf.map((item) => projectNode(item, true)) : undefined],
[
"anyOf",
anyOfTypes
? hasNullAnyOf && anyOfTypes.length === 1
? undefined
: anyOfTypes.map((item) => projectNode(item, true))
: types && types.length > 0
? types.map((type) => ({ type }))
: undefined,
],
["oneOf", Array.isArray(schema.oneOf) ? schema.oneOf.map((item) => projectNode(item, true)) : undefined],
["minLength", schema.minLength],
].filter((entry) => entry[1] !== undefined),
)
return flattenedAnyOf ? { ...result, ...flattenedAnyOf } : result
}
export const convert = (schema: unknown) => projectNode(sanitizeNode(schema))
export * as GeminiToolSchema from "./gemini-tool-schema.js"
+62 -1
View File
@@ -1,6 +1,8 @@
import { Effect, Encoding } from "effect"
import { Media } from "../../media.js"
import type { AIError, ProviderID } from "../../schema/index.js"
import type { MediaProtocol } from "../../route/media-protocol.js"
import { mergeJsonRecords, type AIError, type ProviderID } from "../../schema/index.js"
import { encodeJson } from "../../utils/json.js"
import { ProviderShared } from "../shared.js"
/** Owned bytes for multipart uploads; decodes `base64` sources and rejects remote sources. */
@@ -13,6 +15,31 @@ export const inlineBytes = (route: string, asset: Media.Asset): Effect.Effect<Ui
)
}
/** Copied because `BlobPart` requires a plain `ArrayBuffer`. */
export const blob = (data: Uint8Array, mediaType: string) => {
const buffer = new ArrayBuffer(data.byteLength)
new Uint8Array(buffer).set(data)
return new Blob([buffer], { type: mediaType })
}
const isScalar = (value: unknown): value is string | number | boolean =>
typeof value === "string" || typeof value === "number" || typeof value === "boolean"
export const query = (route: string, values: Record<string, unknown>): Effect.Effect<MediaProtocol.Query, AIError> => {
const entries = Object.entries(values).filter(([, value]) => value !== undefined)
const invalid = entries.find(([, value]) => !isScalar(value) && !(Array.isArray(value) && value.every(isScalar)))
if (invalid !== undefined)
return Effect.fail(ProviderShared.invalidRequest(`${route} cannot send "${invalid[0]}" as a query parameter`))
return Effect.succeed(
Object.fromEntries(entries.map(([key, value]) => [key, Array.isArray(value) ? value.map(String) : String(value)])),
)
}
export const dimensions = (size: string) => {
const [width, height] = size.split("x").map(Number)
return { width, height }
}
/** Provider file handle when the ref belongs to this provider; refs from other providers are never forwarded. */
export const refID = (asset: Media.Asset, provider: ProviderID) =>
asset.source.type === "ref" && asset.source.provider === provider ? asset.source.id : undefined
@@ -30,4 +57,38 @@ export const decodedAsset = (
Effect.map((bytes) => Media.bytes(bytes, mediaType, options)),
)
/** One image of an OpenAI-shaped `data` array, which carries either `b64_json` or a `url`. */
export const imageOutput = (
invalid: (message: string, cause?: unknown) => AIError,
label: string,
item: { readonly b64_json?: string | null; readonly url?: string | null },
mediaType: string | undefined,
options?: Media.AssetOptions,
) => {
if (item.b64_json) return decodedAsset(invalid, label, item.b64_json, mediaType, options)
if (item.url) return Effect.succeed(Media.url(item.url, { ...options, mediaType }))
return Effect.fail(invalid(`${label} has neither image data nor a URL`))
}
/**
* Append multipart text fields: strings as-is, other values as JSON, or arrays as repeated `key[]` parts with
* `repeatArrays`. `overlay` keys in `reserved` are dropped so `http.body` cannot replace route-owned fields.
*/
export const appendFields = (
form: FormData,
fields: Record<string, unknown>,
options: {
readonly overlay?: Record<string, unknown>
readonly reserved: ReadonlySet<string>
readonly repeatArrays?: true
},
) => {
const overlay = Object.entries(options.overlay ?? {}).filter(([key]) => !options.reserved.has(key))
Object.entries(mergeJsonRecords(fields, Object.fromEntries(overlay)) ?? {}).forEach(([key, value]) => {
if (Array.isArray(value) && options.repeatArrays)
return value.forEach((item) => form.append(`${key}[]`, String(item)))
form.append(key, typeof value === "string" ? value : encodeJson(value))
})
}
export * as MediaInput from "./media-input.js"
@@ -1,11 +0,0 @@
// Responses image items can omit output_format, including when PNG/JPEG was requested.
export const mediaType = (data: Uint8Array, format?: string) => {
if (format !== undefined) return `image/${format}`
if (data[0] === 137 && data[1] === 80 && data[2] === 78 && data[3] === 71) return "image/png"
if (data[0] === 255 && data[1] === 216 && data[2] === 255) return "image/jpeg"
if (new TextDecoder().decode(data.slice(0, 4)) === "RIFF" && new TextDecoder().decode(data.slice(8, 12)) === "WEBP")
return "image/webp"
return "application/octet-stream"
}
export * as MetaImage from "./meta-image.js"
@@ -1,20 +0,0 @@
import { Schema } from "effect"
const dimensions = (value: string) => {
const match = /^(\d+)x(\d+)$/.exec(value)
if (!match) return undefined
return { width: Number(match[1]), height: Number(match[2]) }
}
export const Size = Schema.String.check(
Schema.makeFilter((value) => {
if (value === "auto") return undefined
const parsed = dimensions(value)
if (!parsed) return "image size must be `auto` or `{width}x{height}`"
return parsed.width > 0 && parsed.height > 0 ? undefined : "image dimensions must be positive integers"
}),
)
export const OpenAIImage = {
Size,
} as const
@@ -1,7 +1,7 @@
import { Effect, Schema, Stream } from "effect"
import { Route, type RouteBody, type TriggerCompactOperation } from "../../route/client.js"
import { Protocol } from "../../route/protocol.js"
import { CompactionCheckpointResponse, HttpOptions, LLMEvent, LLMRequest } from "../../schema/index.js"
import { CompactionCheckpointResponse, LLMEvent, LLMRequest } from "../../schema/index.js"
import { OpenResponses } from "../open-responses.js"
import { ProviderShared } from "../shared.js"
@@ -109,12 +109,8 @@ export const make = <Body>(body: RouteBody<Body>): TriggerCompactOperation =>
transport: source.transport,
})
const native = yield* body.from(request)
// The body builder already applied and validated overlays. Do not let transport reapply them.
const preparedRequest = LLMRequest.update(request, {
http: request.http === undefined ? undefined : new HttpOptions({ ...request.http, body: undefined }),
})
const prepared = yield* route.prepareTransport(native, preparedRequest, options)
yield* route.streamPrepared(prepared, preparedRequest, { http: executor }, options).pipe(Stream.runDrain)
const prepared = yield* route.prepareTransport(native, request, options)
yield* route.streamPrepared(prepared, request, { http: executor }, options).pipe(Stream.runDrain)
if (!result) return yield* ProviderShared.eventError(source.id, "Compaction response ended without a checkpoint")
return result
})
@@ -19,12 +19,18 @@ import { OpenResponses } from "../open-responses.js"
import { JsonObject, optionalNull, ProviderShared } from "../shared.js"
import { Media } from "../../media.js"
// /compact has a smaller wire contract than /responses; keep the request controls it accepts.
const Body = Schema.Struct({
model: Schema.String,
input: Schema.Array(Schema.Unknown),
instructions: optionalNull(Schema.String),
previous_response_id: optionalNull(Schema.String),
service_tier: optionalNull(Schema.String),
reasoning: Schema.optional(JsonObject),
text: Schema.optional(JsonObject),
include: OpenResponses.coreFields.include,
parallel_tool_calls: OpenResponses.coreFields.parallel_tool_calls,
tools: Schema.optional(Schema.Array(JsonObject)),
prompt_cache_key: optionalNull(Schema.String),
prompt_cache_retention: optionalNull(Schema.String),
prompt_cache_options: optionalNull(
@@ -74,17 +80,27 @@ const Response = Schema.Struct({
usage: Schema.optional(Schema.StructWithRest(OpenResponses.OpenResponsesUsage, [JsonObject])),
})
export const make = (adapter: OpenResponses.ProviderAdapter): CompactOperation =>
export const make = (
adapter: OpenResponses.ProviderAdapter,
lowerTools: (request: LLMRequest) => Effect.Effect<ReadonlyArray<Record<string, unknown>>, AIError>,
): CompactOperation =>
Effect.fn("ResponsesCompaction.execute")(function* (request, executor, options) {
const route = request.model.route
// The standalone compaction endpoint rejects histories containing configuration updates.
const native = yield* OpenResponses.lowerConversation(stripEffortUpdates(request), adapter)
const generation = OpenResponses.lowerGeneration(request)
const tools = request.tools.length === 0 ? undefined : yield* lowerTools(request)
const body = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(Body))(
mergeJsonRecords(
{
...native,
service_tier: request.providerOptions?.serviceTier,
prompt_cache_key: ProviderShared.promptCacheKey(request),
service_tier: generation.service_tier,
reasoning: generation.reasoning,
text: generation.text,
include: generation.include,
parallel_tool_calls: generation.parallel_tool_calls,
tools,
prompt_cache_key: generation.prompt_cache_key,
},
request.http?.body,
),
@@ -0,0 +1,111 @@
import { Effect } from "effect"
import { Media } from "../../media.js"
import type { MediaProtocol } from "../../route/media-protocol.js"
import type { AIError, MediaUsage, ProviderMetadata } from "../../schema/index.js"
import {
SpeechAudioDeltaEvent,
SpeechFinishEvent,
SpeechTimestampsEvent,
type SpeechEvent,
type SpeechVoice,
} from "../../speech.js"
import { concatBytes } from "../../utils/bytes.js"
export interface Audio {
/** Appended in place: the route creates fresh state for each response through `initial`. */
readonly chunks: Array<Uint8Array>
}
export type StepResult<State> = readonly [State, ReadonlyArray<SpeechEvent>]
/** Empty chunks (keep-alive records) emit nothing. */
export const delta = <State extends Audio>(state: State, chunk: Uint8Array): StepResult<State> => {
if (chunk.length === 0) return [state, []]
state.chunks.push(chunk)
return [state, [SpeechAudioDeltaEvent.make({ chunk })]]
}
export const step =
<State extends Audio>(onRecord: (state: State, frame: string) => Effect.Effect<StepResult<State>, AIError>) =>
(state: State, frame: string | Uint8Array) =>
typeof frame === "string" ? onRecord(state, frame) : Effect.succeed(delta(state, frame))
export const timestamps = (
texts: ReadonlyArray<string>,
starts: ReadonlyArray<number>,
ends: ReadonlyArray<number>,
): ReadonlyArray<SpeechEvent> =>
texts.length === 0
? []
: [
SpeechTimestampsEvent.make({
items: texts.map((text, index) => ({ text, startSeconds: starts[index] ?? 0, endSeconds: ends[index] ?? 0 })),
}),
]
export const voiceID = (voice: SpeechVoice | undefined) => (typeof voice === "object" ? voice.id : voice)
const CONTAINER_MEDIA_TYPES: Readonly<Record<string, string>> = {
mp3: "audio/mpeg",
wav: "audio/wav",
opus: "audio/ogg",
aac: "audio/aac",
flac: "audio/flac",
}
export const container = (format: string, sampleRate?: number) => ({
mediaType: CONTAINER_MEDIA_TYPES[format],
info: { format, sampleRate },
})
const PCM_MEDIA_TYPES = {
pcm_s16le: "audio/pcm",
pcm_f32le: "audio/pcm",
pcm_mulaw: "audio/mulaw",
pcm_alaw: "audio/alaw",
} as const
export type PcmEncoding = keyof typeof PCM_MEDIA_TYPES
export const pcm = (encoding: PcmEncoding, sampleRate: number | undefined, mediaType?: string) => ({
mediaType: mediaType ?? PCM_MEDIA_TYPES[encoding],
info: { format: "pcm", encoding, sampleRate, channels: 1 },
})
export const sampleRate = (mediaType: string | undefined) => {
const rate = /rate=(\d+)/i.exec(mediaType ?? "")?.[1]
return rate === undefined ? undefined : Number(rate)
}
/** A declared `mediaType` wins over sniffing: headerless PCM can start with bytes that look like an MPEG frame sync. */
export const finish = (
route: MediaProtocol.Identity,
state: Audio,
output: {
readonly mediaType: string | undefined
readonly info?: Media.Info
readonly usage?: MediaUsage
readonly providerMetadata?: ProviderMetadata
readonly detail?: string
},
): Effect.Effect<ReadonlyArray<SpeechEvent>, AIError> => {
if (state.chunks.length === 0)
return Effect.fail(
route.frameError(`The provider returned no audio${output.detail === undefined ? "" : ` (${output.detail})`}`),
)
return Effect.succeed([
SpeechFinishEvent.make({
audio: Media.bytes(concatBytes(state.chunks), output.mediaType, { info: output.info }),
usage: output.usage,
providerMetadata: output.providerMetadata,
}),
])
}
export const headerUsage = (type: "characters" | "credits", value: string | undefined): MediaUsage | undefined => {
const amount = Number(value)
if (!Number.isFinite(amount)) return undefined
return type === "credits" ? { type, credits: amount } : { type, characters: amount }
}
export * as SpeechStream from "./speech-stream.js"
+61 -15
View File
@@ -1,6 +1,6 @@
import type { JsonSchema, LanguageModelToolSchemaCompatibility } from "../../schema/index.js"
import { ToolDefinition, type JsonSchema, type LanguageModel, type LLMRequest } from "../../schema/index.js"
import { isRecord } from "../../utils/record.js"
import { GeminiToolSchema } from "./gemini-tool-schema.js"
import { GeminiJsonSchema } from "./gemini-json-schema.js"
const tupleItemsSchema = (items: ReadonlyArray<unknown>) => {
const projected = items.map(moonshotNode)
@@ -9,12 +9,24 @@ const tupleItemsSchema = (items: ReadonlyArray<unknown>) => {
return { anyOf: projected }
}
// Moonshot rejects an `enum` without a sibling `type`. Its `type` may be one type, or one type plus "null".
const enumType = (values: ReadonlyArray<unknown>) => {
const types = [
...new Set(values.map((value) => (value === null ? "null" : Array.isArray(value) ? "array" : typeof value))),
]
if (types.length === 1) return types[0]
if (types.length === 2 && types.includes("null")) return [...types.filter((type) => type !== "null"), "null"]
return undefined
}
const moonshotNode = (schema: unknown): unknown => {
if (Array.isArray(schema)) return schema.map(moonshotNode)
if (!isRecord(schema)) return schema
if (typeof schema.$ref === "string") return { $ref: schema.$ref }
return Object.fromEntries(
Object.entries(schema).flatMap(([key, value]) => {
const type = schema.type === undefined && Array.isArray(schema.enum) ? enumType(schema.enum) : undefined
return Object.fromEntries([
...(type === undefined ? [] : [["type", type]]),
...Object.entries(schema).flatMap(([key, value]) => {
if (key === "items" && Array.isArray(value)) return [[key, tupleItemsSchema(value)]]
if (key === "prefixItems") {
if ("items" in schema) return []
@@ -23,7 +35,7 @@ const moonshotNode = (schema: unknown): unknown => {
if (key === "unevaluatedItems") return []
return [[key, moonshotNode(value)]]
}),
)
])
}
const moonshot = (schema: JsonSchema): JsonSchema => {
@@ -34,25 +46,59 @@ const moonshot = (schema: JsonSchema): JsonSchema => {
const openAI = (schema: JsonSchema): JsonSchema => schema
const responses = openAI
const gemini = (schema: JsonSchema): JsonSchema => GeminiToolSchema.convert(schema) ?? {}
const gemini = GeminiJsonSchema.normalize
const modelCompatibility = (
schema: JsonSchema,
compatibility: LanguageModelToolSchemaCompatibility | undefined,
): JsonSchema => {
if (compatibility === undefined) return schema
switch (compatibility) {
const MODEL_NAMES = [
[/gemini/i, "gemini"],
[/kimi/i, "moonshot"],
] as const
// Tool arguments are always a JSON object, and most providers reject a tool schema whose root does not
// declare `type: "object"`, such as `{}` or a bare `properties` map. Effect encodes an empty struct as
// `anyOf` object or array; every object matches its bare object branch, so that `anyOf` is dropped.
const objectRoot = (schema: JsonSchema): JsonSchema => {
if (schema.type !== undefined) return schema
if (
Array.isArray(schema.anyOf) &&
schema.anyOf.some((branch) => isRecord(branch) && branch.type === "object" && Object.keys(branch).length === 1)
)
return { type: "object", ...Object.fromEntries(Object.entries(schema).filter(([key]) => key !== "anyOf")) }
return { type: "object", ...schema }
}
// Every tool schema gets an object root. Then an explicit `sanitizer` wins, and `none` opts out.
// Otherwise the protocol's own default applies (the Gemini API always uses Gemini's rules), then the
// model name selects the family's rules so models reached through gateways and OpenAI-compatible
// endpoints get the same handling.
const modelCompatibility = (schema: JsonSchema, model: LanguageModel): JsonSchema => {
const root = objectRoot(schema)
switch (
model.compatibility?.sanitizer ??
model.route.sanitizer ??
MODEL_NAMES.find(([name]) => name.test(model.id))?.[1]
) {
case "gemini":
return gemini(schema)
return gemini(root)
case "moonshot":
return moonshot(schema)
return moonshot(root)
case "none":
case undefined:
return root
}
}
// Applied once to every request before any protocol builds its body, including tools in namespaces.
const tools = (entries: LLMRequest["tools"], model: LanguageModel): LLMRequest["tools"] =>
entries.map((tool) =>
tool.type === "tool"
? new ToolDefinition({ ...tool, inputSchema: modelCompatibility(tool.inputSchema, model) })
: { ...tool, tools: tools(tool.tools, model) },
)
export const ToolSchemaProjection = {
gemini,
modelCompatibility,
moonshot,
openAI,
responses,
tools,
} as const
+27 -50
View File
@@ -4,13 +4,11 @@ import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords, type AIError } from "../schema/index.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
import { ProviderShared, optionalNull } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
const ADAPTER = "xai-images"
const NAME = "xAI Images"
const PROVIDER = ProviderID.make("xai")
const route = MediaProtocol.identity({ id: "xai-images", name: "xAI Images", provider: "xai" })
export const DEFAULT_BASE_URL = "https://api.x.ai/v1"
export const PATH = "/images/generations"
export const EDIT_PATH = "/images/edits"
@@ -19,13 +17,11 @@ export const EDIT_PATH = "/images/edits"
// 1. Public model input
// ---------------------------------------------------------------------------
export type XAIImageString<Known extends string> = Known | (string & {})
/** Provider-native options. Common fields (`n`, `aspectRatio`, `images`) live on the request. */
export type XAIImageOptions = {
readonly resolution?: XAIImageString<"1k" | "2k">
readonly responseFormat?: XAIImageString<"url" | "b64_json">
readonly response_format?: XAIImageString<"url" | "b64_json">
readonly resolution?: OpenString<"1k" | "2k">
readonly responseFormat?: OpenString<"url" | "b64_json">
readonly response_format?: OpenString<"url" | "b64_json">
} & Record<string, unknown>
export type Request = ImageRequestFor<XAIImageOptions>
@@ -58,15 +54,12 @@ const nativeOptions = (options: XAIImageOptions | undefined) => {
const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
const reference = (asset: Media.Asset): Effect.Effect<Record<string, unknown>, AIError> => {
const inline = asset.inline()
if (inline) return Effect.succeed({ url: inline.dataUrl, type: "image_url" as const })
const url = ProviderShared.mediaUrl(asset)
if (url) return Effect.succeed({ url, type: "image_url" as const })
const id = MediaInput.refID(asset, PROVIDER)
if (id) return Effect.succeed({ file_id: id })
return Effect.fail(ProviderShared.invalidRequest(`${NAME} accepts image URLs, data URLs, bytes, and xAI file IDs`))
}
const reference = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(
Effect.map((item) =>
item.type === "ref" ? { file_id: item.value } : { url: item.value, type: "image_url" as const },
),
)
const fromRequest = Effect.fn("XAIImages.fromRequest")(function* (request: Request) {
const references = yield* Effect.forEach(request.images ?? [], reference)
@@ -91,33 +84,25 @@ const fromRequest = Effect.fn("XAIImages.fromRequest")(function* (request: Reque
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeDocument = route.decodeJson(XAIImageResponse)
const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, XAIImageResponse)(response)
const output = yield* decodeDocument(response)
const decoded = output.value
const images = yield* Effect.forEach(decoded.data, (item, index) => {
const providerMetadata =
item.revised_prompt === undefined || item.revised_prompt === null
? undefined
: { xai: { revisedPrompt: item.revised_prompt } }
if (item.b64_json)
return MediaInput.decodedAsset(
output.invalid,
`${NAME} result ${index}`,
item.b64_json,
item.mime_type ?? undefined,
{
providerMetadata,
},
)
if (item.url)
return Effect.succeed(Media.url(item.url, { mediaType: item.mime_type ?? undefined, providerMetadata }))
return Effect.fail(output.invalid(`${NAME} result ${index} has neither image data nor a URL`))
})
if (images.length === 0) return yield* output.invalid(`${NAME} returned no images`)
const images = yield* Effect.forEach(decoded.data, (item, index) =>
MediaInput.imageOutput(output.invalid, `${route.name} result ${index}`, item, item.mime_type ?? undefined, {
providerMetadata:
item.revised_prompt === undefined || item.revised_prompt === null
? undefined
: { xai: { revisedPrompt: item.revised_prompt } },
}),
)
if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`)
const usage = ProviderShared.isRecord(decoded.usage) ? decoded.usage : undefined
// xAI reports image counts rather than tokens, seconds, or credits; the raw record stays in provider metadata.
// xAI reports a USD cost (`cost_in_usd_ticks`) rather than tokens, seconds, or credits; the raw record stays in
// provider metadata.
return new ImageResponse({
images,
providerMetadata: usage === undefined ? undefined : { xai: { usage } },
@@ -128,9 +113,7 @@ const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
unsupported: ["mask", "size", "seed", "format"],
body: { from: fromRequest },
response: { decode: decodeResponse },
@@ -138,13 +121,7 @@ export const protocol = MediaProtocol.inline<Request, ImageResponse>({
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<XAIImageOptions>(
{
id: ADAPTER,
provider: PROVIDER,
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
},
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH) },
input,
)
+5 -2
View File
@@ -50,7 +50,8 @@ const fromRequest = Effect.fn("XAIResponses.fromRequest")(function* (request: LL
operation: "in-band-compaction",
provider: request.model.provider,
route: request.model.route.id,
message: "xAI requires explicit compaction through LLMClient.compact; automatic context management is not supported",
message:
"xAI requires explicit compaction through LLMClient.compact; automatic context management is not supported",
})
return yield* decodeBody(yield* OpenResponses.fromRequestWithAdapter(request, adapter))
})
@@ -93,6 +94,8 @@ export const protocol = Protocol.make({
},
})
export const compact = ResponsesCompaction.make(adapter)
export const compact = ResponsesCompaction.make(adapter, (request) =>
OpenResponses.lowerTools(ProviderShared.flattenTools(request.tools), adapter),
)
export * as XAIResponses from "./xai-responses.js"
+205
View File
@@ -0,0 +1,205 @@
import { Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords } from "../schema/index.js"
import { VideoModel, VideoResponse, type VideoRequestFor } from "../video.js"
import { ProviderShared, optionalNull } from "./shared.js"
const route = MediaProtocol.identity({ id: "xai-video", name: "xAI Video", provider: "xai" })
export const DEFAULT_BASE_URL = "https://api.x.ai/v1"
export const PATH = "/videos/generations"
export const EDIT_PATH = "/videos/edits"
export const EXTEND_PATH = "/videos/extensions"
export const STATUS_PATH = "/videos"
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
/**
* Provider-native options. Common fields (`frames`, `references`, `video`, `durationSeconds`, `aspectRatio`,
* `resolution`, `audio`) live on the request. `mode` selects the endpoint a `video` input is sent to.
*/
export type XAIVideoOptions = {
readonly mode?: "edit" | "extend"
readonly reference_audios?: ReadonlyArray<{ readonly voice_id: string }>
} & Record<string, unknown>
export type Request = VideoRequestFor<XAIVideoOptions>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
export const Token = Schema.Struct({ requestID: Schema.String })
export type Token = Schema.Schema.Type<typeof Token>
const StartResponse = Schema.Struct({ request_id: Schema.String })
const VideoStatus = Schema.Struct({
status: Schema.String,
progress: optionalNull(Schema.Number),
video: optionalNull(
Schema.Struct({
url: optionalNull(Schema.String),
duration: optionalNull(Schema.Number),
respect_moderation: optionalNull(Schema.Boolean),
}),
),
error: optionalNull(
Schema.Struct({
code: optionalNull(Schema.String),
message: optionalNull(Schema.String),
}),
),
model: optionalNull(Schema.String),
})
const STATUS = {
pending: "running",
done: "completed",
failed: "failed",
expired: "expired",
} as const satisfies Record<string, Status>
// ---------------------------------------------------------------------------
// 5. Request body construction
// ---------------------------------------------------------------------------
const mediaInput = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(
Effect.map((reference) => (reference.type === "ref" ? { file_id: reference.value } : { url: reference.value })),
)
const nativeOptions = (options: XAIVideoOptions | undefined) => {
if (!options) return undefined
const { mode: _mode, ...native } = options
return native
}
const fromRequest = Effect.fn("XAIVideo.fromRequest")(function* (request: Request) {
const image = request.frames?.first === undefined ? undefined : yield* mediaInput(request.frames.first)
const lastFrame = request.frames?.last === undefined ? undefined : yield* mediaInput(request.frames.last)
const video = request.video === undefined ? undefined : yield* mediaInput(request.video)
const references = yield* Effect.forEach(request.references ?? [], mediaInput)
return MediaProtocol.json(
mergeJsonRecords(
{
model: request.model.id,
prompt: request.prompt,
image,
last_frame: lastFrame,
reference_images: references.length === 0 ? undefined : references,
video,
duration: request.durationSeconds,
aspect_ratio: request.aspectRatio,
resolution: request.resolution,
generate_audio: request.audio,
},
nativeOptions(request.providerOptions),
request.http?.body,
) ?? {},
)
})
// ---------------------------------------------------------------------------
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
token: { requestID: value.request_id },
snapshot: { id: value.request_id, status: "running" },
}))
// `progress` is undocumented but observed live as a 0..100 percentage (recorded cassette: 1 → 10 → 37 → 100).
const fraction = (progress: number | null | undefined) =>
progress !== undefined && progress !== null && progress >= 0 && progress <= 100 ? progress / 100 : undefined
const decodeVideoStatus = route.decodeJson(VideoStatus)
const decodeStatus = Effect.fn("XAIVideo.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeVideoStatus(response)
const status = yield* MediaProtocol.status(STATUS, output.value.status, output)
return { id: context.token.requestID, status, progress: fraction(output.value.progress) }
})
const decodeResult = Effect.fn("XAIVideo.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeVideoStatus(response)
const decoded = output.value
const status = yield* MediaProtocol.status(STATUS, decoded.status, output)
if (status === "running") return yield* output.pending(context.token.requestID)
if (status === "failed") {
const code = decoded.error?.code ?? undefined
const message = decoded.error?.message ?? undefined
return yield* output.ended(
"failed",
`${route.name} generation failed${code === undefined ? "" : ` (${code})`}${message === undefined ? "" : `: ${message}`}`,
)
}
if (status !== "completed")
return yield* output.ended("expired", `${route.name} request ${context.token.requestID} expired`)
// `respect_moderation: false` marks a filtered result; a URL may still be present, so report it as a notice.
const notices =
decoded.video?.respect_moderation === false
? [{ type: "moderated" as const, message: `${route.name} flagged the generated video for moderation` }]
: undefined
const url = decoded.video?.url ?? undefined
if (url === undefined && notices !== undefined)
return yield* output.contentPolicy(`${route.name} withheld the video for moderation`)
if (url === undefined) return yield* output.invalid(`${route.name} completed without a video URL`)
const duration = decoded.video?.duration ?? undefined
return new VideoResponse({
videos: [
Media.url(url, {
mediaType: "video/mp4",
info: duration === undefined ? undefined : { durationSeconds: duration },
}),
],
notices,
providerMetadata: { xai: { requestId: context.token.requestID, model: decoded.model ?? undefined } },
})
})
// ---------------------------------------------------------------------------
// 7. Protocol and route
// ---------------------------------------------------------------------------
const statusPath = (token: Token) => `${STATUS_PATH}/${token.requestID}`
export const protocol = MediaProtocol.queued<Request, VideoResponse, Token>(route, {
token: Token,
unsupported: ["n", "seed", "negativePrompt"],
start: { body: { from: fromRequest }, decode: decodeStart },
status: { path: statusPath, decode: decodeStatus },
result: { path: statusPath, decode: decodeResult },
})
// A source video goes to `/videos/edits` unless `providerOptions.mode` asks for an extension.
const startPath = (request: Request) => {
if (request.video === undefined) return PATH
return request.providerOptions?.mode === "extend" ? EXTEND_PATH : EDIT_PATH
}
export const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<XAIVideoOptions, Token>(
{
protocol,
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => startPath(request),
},
input,
)
export const XAIVideo = {
protocol,
model,
} as const
+13 -20
View File
@@ -1,26 +1,22 @@
import { Effect, Schema } from "effect"
import { Duration, Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/unstable/http"
import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, mergeJsonRecords } from "../schema/index.js"
import { mergeJsonRecords, type OpenString } from "../schema/index.js"
const ADAPTER = "zai-images"
const NAME = "Z.ai Images"
const PROVIDER = ProviderID.make("zai")
const route = MediaProtocol.identity({ id: "zai-images", name: "Z.ai Images", provider: "zai" })
export const DEFAULT_BASE_URL = "https://api.z.ai/api/paas/v4"
export const PATH = "/images/generations"
const OUTPUT_RETENTION = Duration.days(30)
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
export type ZAIImageString<Known extends string> = Known | (string & {})
/** Provider-native options. The common `size` field lives on the request. */
export type ZAIImageOptions = {
readonly quality?: ZAIImageString<"hd" | "standard">
readonly quality?: OpenString<"hd" | "standard">
readonly userID?: string
} & Record<string, unknown>
@@ -69,23 +65,25 @@ const fromRequest = Effect.fn("ZAIImages.fromRequest")(function* (request: Reque
// 6. Response decoding
// ---------------------------------------------------------------------------
const decodeDocument = route.decodeJson(ZAIImageResponse)
const decodeResponse = Effect.fn("ZAIImages.decodeResponse")(function* (
response: HttpClientResponse.HttpClientResponse,
) {
const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, ZAIImageResponse)(response)
const output = yield* decodeDocument(response)
const decoded = output.value
if (decoded.data.length === 0) return yield* output.invalid(`${NAME} returned no images`)
if (decoded.data.length === 0) return yield* output.invalid(`${route.name} returned no images`)
const filters = decoded.content_filter ?? []
return new ImageResponse({
// Z.ai returns only URLs and no content type; the media type resolves when the asset is materialized.
images: decoded.data.map((item) => Media.url(item.url)),
images: yield* Effect.forEach(decoded.data, (item) => MediaProtocol.expiringUrl(item.url, OUTPUT_RETENTION)),
// Z.ai reports applied content filters alongside a successful result; surface them instead of dropping them.
notices:
filters.length === 0
? undefined
: filters.map((filter) => ({
type: "moderated" as const,
message: `${NAME} applied a content filter${filter.role === undefined ? "" : ` for ${filter.role}`}${
message: `${route.name} applied a content filter${filter.role === undefined ? "" : ` for ${filter.role}`}${
filter.level === undefined ? "" : ` at level ${filter.level}`
}`,
providerMetadata: { zai: filter },
@@ -105,19 +103,14 @@ const decodeResponse = Effect.fn("ZAIImages.decodeResponse")(function* (
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.inline<Request, ImageResponse>({
id: ADAPTER,
name: NAME,
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
unsupported: ["images", "mask", "n", "aspectRatio", "seed", "format"],
body: { from: fromRequest },
response: { decode: decodeResponse },
})
export const model = (input: MediaRoute.ModelInput) =>
ImageModel.fromRoute<ZAIImageOptions>(
{ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH },
input,
)
ImageModel.fromRoute<ZAIImageOptions>({ protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, input)
export const ZAIImages = {
protocol,
+10 -1
View File
@@ -80,12 +80,19 @@ const SERVER_CODES = new Set([
"slow_down",
"serviceunavailableexception",
])
const INVALID_REQUEST_CODES = new Set(["invalid_prompt", "invalid_request_error", "validationexception"])
// `invalid_request` is the Vercel AI Gateway's code for an upstream request rejection.
const INVALID_REQUEST_CODES = new Set([
"invalid_prompt",
"invalid_request",
"invalid_request_error",
"validationexception",
])
// Azure OpenAI reports `content_filter` with `innererror.code` ResponsibleAIPolicyViolation.
// OpenRouter tags provider failures with a typed `error_type`; its Responses skin also
// emits `image_content_policy_violation` as the native code.
const CONTENT_POLICY_CODES = new Set([
"content_filter",
"content_moderation",
"responsibleaipolicyviolation",
"content_policy_violation",
"image_content_policy_violation",
@@ -204,6 +211,8 @@ function providerCodes(value: unknown) {
const exception = isRecord(decoded.exception) ? decoded.exception : undefined
return [
decoded.code,
// Stability's `{ id, name, errors }` bodies carry the code in `name`.
Array.isArray(decoded.errors) ? decoded.name : undefined,
decoded.error_type,
error?.code,
error?.type,
@@ -31,6 +31,7 @@ export interface Settings extends ProviderPackage.Settings {
readonly profile?: string
readonly region?: string
readonly topP?: number
readonly thinking?: BedrockConverse.OptionsInput["thinking"]
}
export const routes = [BedrockConverse.route]
@@ -71,6 +72,7 @@ export const model: ProviderPackage.Definition<Settings>["model"] = (modelID, se
generation: settings.topP === undefined ? undefined : { topP: settings.topP },
headers: settings.headers === undefined ? undefined : { ...settings.headers },
http: settings.body === undefined ? undefined : { body: { ...settings.body } },
providerOptions: settings.thinking === undefined ? undefined : { thinking: settings.thinking },
profile: settings.profile,
region: settings.region,
}).model(modelID)
+37
View File
@@ -0,0 +1,37 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { AssemblyAITranscription } from "../protocols/assemblyai-transcription.js"
export type { AssemblyAITranscriptionOptions } from "../protocols/assemblyai-transcription.js"
export const id = ProviderID.make("assemblyai")
export type Config = ProviderAuthOption<"optional"> & {
/** `https://api.eu.assemblyai.com` for the EU region. */
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
// The key is the whole `authorization` value, without a scheme.
const auth = (options: ProviderAuthOption<"optional">) => {
if ("auth" in options && options.auth) return options.auth
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
.orElse(Auth.config("ASSEMBLYAI_API_KEY"))
.pipe(Auth.header("authorization"))
}
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const transcription = (modelID: string | ModelID) => AssemblyAITranscription.model({ ...media, id: modelID })
return {
id,
transcription,
configure,
}
}
export const provider = configure()
export const transcription = provider.transcription
+2 -2
View File
@@ -1,11 +1,11 @@
import { Auth } from "../route/auth.js"
import { type AtLeastOne, type ProviderAuthOption } from "../route/auth-options.js"
import type { Route, RouteDefaultsInput, CompactionOperations } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import type { ProviderPackage } from "../provider-package.js"
import { ProviderConfigurationError, ProviderID, type ModelID } from "../schema/index.js"
import * as OpenAIChat from "../protocols/openai-chat.js"
import * as OpenAIResponses from "../protocols/openai-responses.js"
import { ProviderShared } from "../protocols/shared.js"
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options.js"
export const id = ProviderID.make("azure")
@@ -108,7 +108,7 @@ const configuredRoute = <Body, Prepared, Compact extends CompactionOperations |
})
function endpoint(input: Config, modelID: string | ModelID) {
const baseURL = ProviderShared.trimBaseUrl(input.baseURL ?? resourceBaseURL(input.resourceName!))
const baseURL = Endpoint.trimBaseUrl(input.baseURL ?? resourceBaseURL(input.resourceName!))
const query = { "api-version": input.apiVersion ?? "v1", ...input.queryParams }
if (input.useDeploymentBasedUrls) return { baseURL: `${baseURL}/deployments/${modelID}`, query }
@@ -0,0 +1,36 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { BlackForestLabsImages } from "../protocols/bfl-images.js"
export type { BlackForestLabsImageOptions } from "../protocols/bfl-images.js"
export const id = ProviderID.make("black-forest-labs")
export type Config = ProviderAuthOption<"optional"> & {
/** `https://api.eu.bfl.ai` or `https://api.us.bfl.ai` pin inference to one region. */
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
const auth = (options: ProviderAuthOption<"optional">) => {
if ("auth" in options && options.auth) return options.auth
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
.orElse(Auth.config("BFL_API_KEY"))
.pipe(Auth.header("x-key"))
}
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const image = (modelID: string | ModelID) => BlackForestLabsImages.model({ ...media, id: modelID })
return {
id,
image,
configure,
}
}
export const provider = configure()
export const image = provider.image
+29
View File
@@ -0,0 +1,29 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { CartesiaSpeech } from "../protocols/cartesia-speech.js"
export type { CartesiaEncoding, CartesiaSpeechOptions } from "../protocols/cartesia-speech.js"
export const id = ProviderID.make("cartesia")
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "CARTESIA_API_KEY")
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const speech = (modelID: string | ModelID) => CartesiaSpeech.model({ ...media, id: modelID })
return {
id,
speech,
configure,
}
}
export const provider = configure()
export const speech = provider.speech
+38
View File
@@ -0,0 +1,38 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { DeepgramSpeech } from "../protocols/deepgram-speech.js"
import { DeepgramTranscription } from "../protocols/deepgram-transcription.js"
export type { DeepgramEncoding, DeepgramSpeechOptions } from "../protocols/deepgram-speech.js"
export type { DeepgramTranscriptionOptions } from "../protocols/deepgram-transcription.js"
export const id = ProviderID.make("deepgram")
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
const auth = (options: ProviderAuthOption<"optional">) => {
if ("auth" in options && options.auth) return options.auth
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
.orElse(Auth.config("DEEPGRAM_API_KEY"))
.pipe(Auth.scheme("Token"))
}
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
return {
id,
speech: (modelID: string | ModelID) => DeepgramSpeech.model({ ...media, id: modelID }),
transcription: (modelID: string | ModelID) => DeepgramTranscription.model({ ...media, id: modelID }),
configure,
}
}
export const provider = configure()
export const speech = provider.speech
export const transcription = provider.transcription
+35
View File
@@ -0,0 +1,35 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { ElevenLabsSpeech } from "../protocols/elevenlabs-speech.js"
export type { ElevenLabsOutputFormat, ElevenLabsSpeechOptions } from "../protocols/elevenlabs-speech.js"
export const id = ProviderID.make("elevenlabs")
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
const auth = (options: ProviderAuthOption<"optional">) => {
if ("auth" in options && options.auth) return options.auth
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
.orElse(Auth.config("ELEVENLABS_API_KEY"))
.pipe(Auth.header("xi-api-key"))
}
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const speech = (modelID: string | ModelID) => ElevenLabsSpeech.model({ ...media, id: modelID })
return {
id,
speech,
configure,
}
}
export const provider = configure()
export const speech = provider.speech
+39
View File
@@ -0,0 +1,39 @@
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { FalImages } from "../protocols/fal-images.js"
import { FalVideo } from "../protocols/fal-video.js"
export type { FalImageOptions } from "../protocols/fal-images.js"
export type { FalVideoOptions } from "../protocols/fal-video.js"
export const id = ProviderID.make("fal")
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
// fal authenticates with `Authorization: Key <FAL_KEY>` rather than a bearer token.
const auth = (options: ProviderAuthOption<"optional">) => {
if ("auth" in options && options.auth) return options.auth
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
.orElse(Auth.config("FAL_KEY"))
.pipe(Auth.scheme("Key"))
}
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
return {
id,
image: (modelID: string | ModelID) => FalImages.model({ ...media, id: modelID }),
video: (modelID: string | ModelID) => FalVideo.model({ ...media, id: modelID }),
configure,
}
}
export const provider = configure()
export const image = provider.image
export const video = provider.video
+16 -10
View File
@@ -1,12 +1,19 @@
import type { RouteDefaultsInput } from "../route/client.js"
import { Auth } from "../route/auth.js"
import type { ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import type { ProviderPackage } from "../provider-package.js"
import { HttpOptions, ProviderID, mergeHttpOptions, type ModelID } from "../schema/index.js"
import { ProviderID, type ModelID } from "../schema/index.js"
import { Gemini } from "../protocols/gemini.js"
import { GoogleImages } from "../protocols/google-images.js"
import { GoogleSpeech } from "../protocols/google-speech.js"
import { GoogleTranscription } from "../protocols/google-transcription.js"
import { GoogleVideo } from "../protocols/google-video.js"
export type { GoogleImageOptions } from "../protocols/google-images.js"
export type { GoogleSpeechOptions } from "../protocols/google-speech.js"
export type { GoogleTranscriptionOptions } from "../protocols/google-transcription.js"
export type { GoogleVideoOptions } from "../protocols/google-video.js"
export type GeminiOptionsInput = Gemini.OptionsInput
export type GeminiProviderOptionsInput = Gemini.ProviderOptionsInput
@@ -40,18 +47,14 @@ const configuredRoute = (input: Config) => {
export const configure = (input: Config = {}) => {
const route = configuredRoute(input)
const image = (modelID: string | ModelID) =>
GoogleImages.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL,
headers: input.headers,
http: mergeHttpOptions(input.http === undefined ? undefined : HttpOptions.make(input.http)),
})
const media = MediaRoute.deployment(input, auth(input))
return {
id,
model: (modelID: string | ModelID) => route.model<Gemini.ProviderOptionsInput>({ id: modelID }),
image,
image: (modelID: string | ModelID) => GoogleImages.model({ ...media, id: modelID }),
video: (modelID: string | ModelID) => GoogleVideo.model({ ...media, id: modelID }),
speech: (modelID: string | ModelID) => GoogleSpeech.model({ ...media, id: modelID }),
transcription: (modelID: string | ModelID) => GoogleTranscription.model({ ...media, id: modelID }),
configure,
}
}
@@ -70,3 +73,6 @@ export const model: ProviderPackage.Definition<Settings, Gemini.ProviderOptionsI
}).model(modelID)
export const image = provider.image
export const video = provider.video
export const speech = provider.speech
export const transcription = provider.transcription
+9
View File
@@ -3,13 +3,19 @@ export * as Anthropic from "./anthropic.js"
export * as AnthropicCompatible from "./anthropic-compatible.js"
export * as AmazonBedrock from "./amazon-bedrock.js"
export * as AmazonBedrockMantle from "./amazon-bedrock-mantle.js"
export * as AssemblyAI from "./assemblyai.js"
export * as Azure from "./azure.js"
export * as Baseten from "./baseten.js"
export * as BlackForestLabs from "./black-forest-labs.js"
export * as Cartesia from "./cartesia.js"
export * as Cerebras from "./cerebras.js"
export * as CloudflareAIGateway from "./cloudflare-ai-gateway.js"
export * as CloudflareWorkersAI from "./cloudflare-workers-ai.js"
export * as DeepInfra from "./deepinfra.js"
export * as Deepgram from "./deepgram.js"
export * as DeepSeek from "./deepseek.js"
export * as ElevenLabs from "./elevenlabs.js"
export * as Fal from "./fal.js"
export * as Fireworks from "./fireworks.js"
export * as Google from "./google.js"
export * as GoogleVertex from "./google-vertex.js"
@@ -26,6 +32,9 @@ export * as OpenAICompatible from "./openai-compatible.js"
export * as OpenAICompatibleResponses from "./openai-compatible-responses.js"
export * as OpenCodeZen from "./opencode-zen.js"
export * as OpenRouter from "./openrouter.js"
export * as Replicate from "./replicate.js"
export * as Runway from "./runway.js"
export * as Stability from "./stability.js"
export * as TogetherAI from "./togetherai.js"
export * as TypeSafeAI from "./typesafe-ai.js"
export * as VercelAIGateway from "./vercel-ai-gateway.js"
+6 -11
View File
@@ -7,7 +7,8 @@ import { MetaImages } from "../protocols/meta-images.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { HttpOptions, ProviderID, ToolDefinition, type ModelID } from "../schema/index.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, ToolDefinition, type ModelID, type OpenString } from "../schema/index.js"
import type { OpenResponsesProviderOptionsInput } from "./open-responses-options.js"
export const id = ProviderID.make("meta")
@@ -48,8 +49,8 @@ export const webSearch = (options: WebSearchOptions = {}) =>
export interface ImageGenerationOptions {
readonly size?: string
readonly outputFormat?: "webp" | "png" | "jpeg" | (string & {})
readonly reasoningStrength?: "low" | "high" | (string & {})
readonly outputFormat?: OpenString<"webp" | "png" | "jpeg">
readonly reasoningStrength?: OpenString<"low" | "high">
readonly enableImageSearch?: boolean
readonly enableWebSearch?: boolean
readonly enableShell?: boolean
@@ -139,14 +140,8 @@ export const configure = (input: LanguageModelOptions = {}) => {
id: modelID,
compatibility: { requireSignature: false },
})
const image = (modelID: string | ModelID) =>
MetaImages.model({
id: modelID,
baseURL: endpoint ?? baseURL,
auth: options.auth,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
const media = MediaRoute.deployment(input, options.auth)
const image = (modelID: string | ModelID) => MetaImages.model({ ...media, id: modelID })
return { id, model: responses, responses, chat, messages, image, configure }
}
+3 -3
View File
@@ -113,19 +113,19 @@ export const configure = (input: Config = {}) => {
supportsStore: false,
supportsStrictMode: false,
supportsPromptCacheKey: true,
toolSchema: "moonshot",
sanitizer: "moonshot",
reasoningField: "reasoning_content",
},
})
const messages = (modelID: string | ModelID) =>
messagesRoute.with(defaults).model<MessagesOptionsInput>({
id: modelID,
compatibility: { requireSignature: false, toolSchema: "moonshot" },
compatibility: { requireSignature: false, sanitizer: "moonshot" },
})
const responses = (modelID: string | ModelID) =>
responsesRoute
.with(defaults)
.model<ResponsesOptionsInput>({ id: modelID, compatibility: { toolSchema: "moonshot" } })
.model<ResponsesOptionsInput>({ id: modelID, compatibility: { sanitizer: "moonshot" } })
return { id, model: chat, chat, messages, responses, configure }
}
+4 -15
View File
@@ -32,10 +32,7 @@ const openAIProviderOptions = (options: OpenAIOptionsInput | undefined): Provide
return result
}
export const gpt5DefaultOptions = (
modelID: string,
options: { readonly textVerbosity?: boolean } = {},
): ProviderOptions | undefined => {
export const gpt5DefaultOptions = (modelID: string): ProviderOptions | undefined => {
const id = modelID.toLowerCase()
if (!id.includes("gpt-5") || id.includes("gpt-5-chat") || id.includes("gpt-5-pro")) return undefined
return openAIProviderOptions({
@@ -47,27 +44,19 @@ export const gpt5DefaultOptions = (
// this, callers using the default model facade get reasoning summaries
// they cannot replay statelessly.
include: ["reasoning.encrypted_content"],
textVerbosity:
options.textVerbosity === true && id.includes("gpt-5.") && !id.includes("codex") && !id.includes("-chat")
? "low"
: undefined,
})
}
export const openAIDefaultOptions = (
modelID: string,
options: { readonly textVerbosity?: boolean } = {},
): ProviderOptions | undefined =>
mergeProviderOptions(openAIProviderOptions({ store: false }), gpt5DefaultOptions(modelID, options))
export const openAIDefaultOptions = (modelID: string): ProviderOptions | undefined =>
mergeProviderOptions(openAIProviderOptions({ store: false }), gpt5DefaultOptions(modelID))
export const withOpenAIOptions = <Options extends { readonly providerOptions?: OpenAIProviderOptionsInput }>(
modelID: string,
options: Options,
defaults: { readonly textVerbosity?: boolean } = {},
): Omit<Options, "providerOptions"> & { readonly providerOptions?: ProviderOptions } => {
return {
...options,
providerOptions: mergeProviderOptions(openAIDefaultOptions(modelID, defaults), options.providerOptions),
providerOptions: mergeProviderOptions(openAIDefaultOptions(modelID), options.providerOptions),
}
}
+36 -20
View File
@@ -1,14 +1,26 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import type { Route, RouteDefaultsInput, CompactionOperations } from "../route/client.js"
import { MediaRoute } from "../route/media.js"
import type { ProviderPackage } from "../provider-package.js"
import { HttpOptions, ProviderID, ToolDefinition, mergeHttpOptions, type ModelID } from "../schema/index.js"
import {
HttpOptions,
ProviderID,
ToolDefinition,
mergeHttpOptions,
type ModelID,
type OpenString,
} from "../schema/index.js"
import * as OpenAIChat from "../protocols/openai-chat.js"
import * as OpenAIResponses from "../protocols/openai-responses.js"
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options.js"
import { OpenAIImages, type OpenAIImageString } from "../protocols/openai-images.js"
import { OpenAIImages } from "../protocols/openai-images.js"
import { OpenAISpeech } from "../protocols/openai-speech.js"
import { OpenAITranscription } from "../protocols/openai-transcription.js"
export type { OpenAIOptionsInput, OpenAIResponseIncludable } from "./openai-options.js"
export type { OpenAIImageOptions } from "../protocols/openai-images.js"
export type { OpenAISpeechOptions } from "../protocols/openai-speech.js"
export type { OpenAITranscriptionOptions } from "../protocols/openai-transcription.js"
export const id = ProviderID.make("openai")
@@ -25,14 +37,14 @@ export type Config = RouteDefaultsInput &
}
export interface ImageGenerationOptions {
readonly action?: OpenAIImageString<"auto" | "generate" | "edit">
readonly background?: OpenAIImageString<"auto" | "opaque" | "transparent">
readonly inputFidelity?: OpenAIImageString<"low" | "high">
readonly action?: OpenString<"auto" | "generate" | "edit">
readonly background?: OpenString<"auto" | "opaque" | "transparent">
readonly inputFidelity?: OpenString<"low" | "high">
readonly outputCompression?: number
readonly outputFormat?: OpenAIImageString<"png" | "jpeg" | "webp">
readonly outputFormat?: OpenString<"png" | "jpeg" | "webp">
readonly partialImages?: number
readonly quality?: OpenAIImageString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
readonly size?: OpenAIImageString<
readonly quality?: OpenString<"auto" | "low" | "medium" | "high" | "standard" | "hd">
readonly size?: OpenString<
"auto" | "256x256" | "512x512" | "1024x1024" | "1536x1024" | "1024x1536" | "1792x1024" | "1024x1792"
>
}
@@ -88,24 +100,24 @@ export const configure = (input: Config = {}) => {
const modelDefaults = defaults(input)
const responses = (id: string | ModelID) =>
responsesRoute
.with(withOpenAIOptions(id, modelDefaults, { textVerbosity: true }))
.with(withOpenAIOptions(id, modelDefaults))
.model<OpenAIProviderOptionsInput>({ id })
const chat = (id: string | ModelID) =>
chatRoute.with(withOpenAIOptions(id, modelDefaults)).model<OpenAIProviderOptionsInput>({
id,
compatibility: { supportsPromptCacheKey: true },
})
const image = (modelID: string | ModelID) =>
OpenAIImages.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL,
headers: input.headers,
http: mergeHttpOptions(
input.http === undefined ? undefined : HttpOptions.make(input.http),
input.queryParams === undefined ? undefined : new HttpOptions({ query: input.queryParams }),
),
})
const deployment = MediaRoute.deployment(input, auth(input))
const media = {
...deployment,
http: mergeHttpOptions(
deployment.http,
input.queryParams === undefined ? undefined : new HttpOptions({ query: input.queryParams }),
),
}
const image = (modelID: string | ModelID) => OpenAIImages.model({ ...media, id: modelID })
const speech = (modelID: string | ModelID) => OpenAISpeech.model({ ...media, id: modelID })
const transcription = (modelID: string | ModelID) => OpenAITranscription.model({ ...media, id: modelID })
return {
id,
@@ -113,6 +125,8 @@ export const configure = (input: Config = {}) => {
responses,
chat,
image,
speech,
transcription,
configure,
}
}
@@ -159,3 +173,5 @@ export const chatModel: ProviderPackage.Definition<Settings, OpenAIProviderOptio
export const responses = provider.responses
export const chat = provider.chat
export const image = provider.image
export const speech = provider.speech
export const transcription = provider.transcription
+1 -1
View File
@@ -20,7 +20,7 @@ export const configure = (input: Options = {}) => {
auth: AuthOptions.bearer(input, "OPENCODE_API_KEY"),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
http: HttpOptions.make(input.http),
})
return { id, experimental: { evaluation }, configure }
}
+18 -13
View File
@@ -3,29 +3,27 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { Protocol } from "../route/protocol.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { HttpOptions, ProviderID, type CacheHint, type ModelID } from "../schema/index.js"
import { HttpOptions, ProviderID, type CacheHint, type ModelID, type OpenString } from "../schema/index.js"
import type { ProviderPackage } from "../provider-package.js"
import { SystemOne } from "../experimental/system-one.js"
import { OpenAIChat } from "../protocols/openai-chat.js"
import { newBreakpoints, ttlBucket } from "../protocols/utils/cache.js"
import { isRecord } from "../protocols/shared.js"
import { isRecord, ProviderShared } from "../protocols/shared.js"
export const id = ProviderID.make("openrouter")
const baseURL = "https://openrouter.ai/api/v1"
const ADAPTER = "openrouter"
type OpenRouterString<Known extends string> = Known | (string & {})
export interface OpenRouterProviderRouting {
readonly [key: string]: unknown
readonly order?: ReadonlyArray<string>
readonly allow_fallbacks?: boolean
readonly require_parameters?: boolean
readonly data_collection?: OpenRouterString<"allow" | "deny">
readonly data_collection?: OpenString<"allow" | "deny">
readonly only?: ReadonlyArray<string>
readonly ignore?: ReadonlyArray<string>
readonly quantizations?: ReadonlyArray<string>
readonly sort?: OpenRouterString<"price" | "throughput" | "latency">
readonly sort?: OpenString<"price" | "throughput" | "latency">
readonly max_price?: Readonly<{
prompt?: number | string
completion?: number | string
@@ -41,7 +39,7 @@ export type OpenRouterPlugin =
id: "web"
max_results?: number
search_prompt?: string
engine?: OpenRouterString<"native" | "exa">
engine?: OpenString<"native" | "exa">
}>
| Readonly<{ id: "file-parser"; max_files?: number; pdf?: { engine?: string } }>
| Readonly<{ id: "moderation" }>
@@ -58,7 +56,7 @@ export interface OpenRouterOptions {
readonly reasoning?: Readonly<{
enabled?: boolean
exclude?: boolean
effort?: OpenRouterString<"none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max">
effort?: OpenString<"none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max">
max_tokens?: number
}>
readonly usage?: boolean | Readonly<{ include: boolean }>
@@ -66,7 +64,7 @@ export interface OpenRouterOptions {
readonly web_search_options?: Readonly<{
max_results?: number
search_prompt?: string
engine?: OpenRouterString<"native" | "exa">
engine?: OpenString<"native" | "exa">
}>
}
@@ -125,7 +123,7 @@ export const protocol = Protocol.make({
return {
...body,
messages,
...bodyOptions(request.providerOptions),
...bodyOptions(request.providerOptions, request.generation?.maxTokens),
} as OpenRouterBody
}),
),
@@ -145,7 +143,14 @@ const cacheControl = () => {
}
}
const bodyOptions = (input: unknown) => {
// OpenRouter forwards `reasoning.max_tokens` as the upstream thinking budget. Upstreams such as Anthropic and Alibaba
// reject one that is not below the output limit; 1,024 is Anthropic's minimum budget.
const fitReasoning = (reasoning: Record<string, unknown>, maxTokens: number | undefined) =>
typeof reasoning.max_tokens === "number"
? { ...reasoning, max_tokens: ProviderShared.fitThinkingBudget(reasoning.max_tokens, maxTokens, 1_024) }
: reasoning
const bodyOptions = (input: unknown, maxTokens: number | undefined) => {
const openrouter = isRecord(input) ? input : {}
const { usage, models, provider, plugins, web_search_options, debug, user, reasoning, promptCacheKey, ...options } =
openrouter
@@ -164,7 +169,7 @@ const bodyOptions = (input: unknown) => {
...(isRecord(web_search_options) ? { web_search_options } : {}),
...(isRecord(debug) ? { debug } : {}),
...(typeof user === "string" ? { user } : {}),
...(isRecord(reasoning) ? { reasoning } : {}),
...(isRecord(reasoning) ? { reasoning: fitReasoning(reasoning, maxTokens) } : {}),
}
}
@@ -198,7 +203,7 @@ export const configure = (input: LanguageModelOptions = {}) => {
auth: AuthOptions.bearer(input, "OPENROUTER_API_KEY"),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
http: HttpOptions.make(input.http),
})
return {
id,
+30
View File
@@ -0,0 +1,30 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { ReplicateImages } from "../protocols/replicate-images.js"
export type { ReplicateImageOptions } from "../protocols/replicate-images.js"
export const id = ProviderID.make("replicate")
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
/** `{ Prefer: "wait=60" }` holds the submission open until the prediction finishes (up to 60 seconds). */
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "REPLICATE_API_TOKEN")
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const image = (modelID: string | ModelID) => ReplicateImages.model({ ...media, id: modelID })
return {
id,
image,
configure,
}
}
export const provider = configure()
export const image = provider.image
+29
View File
@@ -0,0 +1,29 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { RunwayVideo } from "../protocols/runway-video.js"
export type { RunwayVideoOptions } from "../protocols/runway-video.js"
export const id = ProviderID.make("runway")
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "RUNWAYML_API_SECRET")
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
const video = (modelID: string | ModelID) => RunwayVideo.model({ ...media, id: modelID })
return {
id,
video,
configure,
}
}
export const provider = configure()
export const video = provider.video
+30
View File
@@ -0,0 +1,30 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { StabilityImages } from "../protocols/stability-images.js"
export type { StabilityImageOptions, StabilityUpscaleOptions } from "../protocols/stability-images.js"
export const id = ProviderID.make("stability")
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions.Input
}
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "STABILITY_API_KEY")
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
return {
id,
image: (modelID: string | ModelID) => StabilityImages.model({ ...media, id: modelID }),
upscale: () => StabilityImages.upscaleModel(media),
configure,
}
}
export const provider = configure()
export const image = provider.image
export const upscale = provider.upscale
+1 -1
View File
@@ -20,7 +20,7 @@ export const configure = (input: Options = {}) => {
auth: AuthOptions.bearer(input, "TYPESAFE_API_KEY"),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
http: HttpOptions.make(input.http),
})
return { id, experimental: { evaluation }, configure }
}
@@ -67,7 +67,7 @@ export const configure = (input: Options = {}) => {
EvaluationModel.make<EvaluationOptions>({
id: modelID,
provider: id,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
http: HttpOptions.make(input.http),
route: {
id: "vercel-evaluation",
evaluate: (req, send) =>
+8 -10
View File
@@ -1,11 +1,13 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, type ModelID } from "../schema/index.js"
import { OpenAIChat } from "../protocols/openai-chat.js"
import { OpenResponsesChannel } from "../protocols/open-responses-channel.js"
import { XAIResponses } from "../protocols/xai-responses.js"
import { XAIImages } from "../protocols/xai-images.js"
import { XAIVideo } from "../protocols/xai-video.js"
import type { OpenAIOptionsInput } from "./openai-options.js"
import type { ProviderPackage } from "../provider-package.js"
@@ -27,6 +29,7 @@ export type Settings = ProviderPackage.Settings &
}
export type { XAIImageOptions } from "../protocols/xai-images.js"
export type { XAIVideoOptions } from "../protocols/xai-video.js"
const RESPONSES_WEBSOCKET_ROTATE_AFTER_MS = 24 * 60 * 1000
@@ -87,20 +90,14 @@ export const configure = (input: LanguageModelOptions = {}) => {
const chatRoute = configuredChatRoute(input)
const responses = (modelID: string | ModelID) => responsesRoute.model<XAIProviderOptionsInput>({ id: modelID })
const chat = (modelID: string | ModelID) => chatRoute.model<XAIProviderOptionsInput>({ id: modelID })
const image = (modelID: string | ModelID) =>
XAIImages.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL ?? baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
const media = MediaRoute.deployment(input, auth(input))
return {
id,
model: responses,
responses,
chat,
image,
image: (modelID: string | ModelID) => XAIImages.model({ ...media, id: modelID }),
video: (modelID: string | ModelID) => XAIVideo.model({ ...media, id: modelID }),
configure,
}
}
@@ -121,3 +118,4 @@ export const model: ProviderPackage.Definition<
export const responses = provider.responses
export const chat = provider.chat
export const image = provider.image
export const video = provider.video
+4 -9
View File
@@ -5,7 +5,8 @@ import { OpenAIChat } from "../protocols/openai-chat.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import { MediaRoute } from "../route/media.js"
import { ProviderID, type ModelID } from "../schema/index.js"
export const id = ProviderID.make("zai")
@@ -48,14 +49,8 @@ export const configure = (input: Config = {}) => {
auth: auth(input),
})
.model<ChatOptionsInput>({ id: modelID, compatibility: ZAIChat.compatibility })
const image = (modelID: string | ModelID) =>
ZAIImages.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL,
headers: input.headers,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
const media = MediaRoute.deployment(input, auth(input))
const image = (modelID: string | ModelID) => ZAIImages.model({ ...media, id: modelID })
return {
id,
+30 -15
View File
@@ -1,12 +1,15 @@
import { Config, Effect, Redacted } from "effect"
import { Config, Effect, Option, Redacted } from "effect"
import { Headers } from "effect/unstable/http"
import { AuthenticationError, InvalidRequestError, AIError, type HttpOptions } from "../schema/index.js"
import { AuthenticationError, AIError, type HttpOptions } from "../schema/index.js"
export class MissingCredentialError extends Error {
readonly _tag = "MissingCredentialError"
constructor(readonly source: string) {
super(`Missing auth credential: ${source}`)
constructor(
readonly source: string,
message = `Missing auth credential: ${source}`,
) {
super(message)
}
}
@@ -16,7 +19,7 @@ type Secret = string | Redacted.Redacted | Config.Config<string | Redacted.Redac
export interface AuthInput {
readonly request: { readonly http?: HttpOptions }
readonly method: "POST" | "GET"
readonly method: "POST" | "GET" | "PUT" | "DELETE"
readonly url: string
readonly body: string
readonly headers: Headers.Headers
@@ -89,7 +92,14 @@ export const optional = (secret: Secret | undefined, source = "optional value")
? credential(Effect.fail(new MissingCredentialError(source)))
: credentialFromSecret(secret, source)
export const config = (name: string) => credentialFromSecret(Config.redacted(name), name)
export const config = (name: string) =>
credential(
Effect.gen(function* () {
const secret = yield* Config.option(Config.redacted(name))
if (Option.isSome(secret) && Redacted.value(secret.value) !== "") return secret.value
return yield* Effect.fail(new MissingCredentialError(name, `${name} is not set`))
}),
)
export const effect = (load: Effect.Effect<Redacted.Redacted, CredentialError>) => credential(load)
@@ -134,16 +144,21 @@ export function bearerHeader(name: string, source?: Secret | Credential) {
return render(source)
}
/** `Authorization: <scheme> <secret>` for providers whose scheme is not `Bearer`, such as fal's `Key`. */
export function scheme(name: string): (source: Secret | Credential) => Definition
export function scheme(name: string, source: Secret | Credential): Definition
export function scheme(name: string, source?: Secret | Credential) {
const render = (input: Secret | Credential) =>
fromCredential(credentialInput(input), (secret) => ({ authorization: `${name} ${secret}` }))
if (source === undefined) return render
return render(source)
}
const toAIError = (error: AuthError): AIError => {
if (error instanceof MissingCredentialError || error instanceof Config.ConfigError) {
return new AIError({
reason:
error instanceof MissingCredentialError
? new AuthenticationError({ message: error.message, cause: error })
: new InvalidRequestError({ message: `Failed to resolve auth config: ${error.message}`, cause: error }),
})
}
return error
if (error instanceof AIError) return error
const message =
error instanceof MissingCredentialError ? error.message : `Failed to resolve auth config: ${error.message}`
return new AIError({ reason: new AuthenticationError({ message, cause: error }) })
}
export const toEffect =
+11 -9
View File
@@ -11,7 +11,8 @@ import { applyEffortUpdates } from "../effort-updates.js"
import { normalizeToolHistory } from "../tool-history.js"
import { sanitizeSurrogates } from "../utils/sanitize.js"
import * as ProviderShared from "../protocols/shared.js"
import type { ProtocolID, ProviderOptions } from "../schema/index.js"
import { ToolSchemaProjection } from "../protocols/utils/tool-schema.js"
import type { LanguageModelSanitizerCompatibility, ProtocolID, ProviderOptions } from "../schema/index.js"
import {
AIError,
CompactionResponse,
@@ -57,6 +58,7 @@ export interface Route<
readonly defaults: RouteDefaults
readonly body: RouteBody<Body>
readonly supportsEffortUpdates?: (request: LLMRequest) => boolean
readonly sanitizer?: LanguageModelSanitizerCompatibility
readonly with: {
<Next extends CompactionOperations | undefined>(
patch: RoutePatch<Body, Prepared> & { readonly compact: Next },
@@ -152,7 +154,7 @@ const mergeRouteDefaults = (base: RouteDefaults | undefined, patch: RouteDefault
providerOptions: mergeProviderOptions(base?.providerOptions, patch.providerOptions),
http: mergeHttpOptions(
base?.http,
httpOptions(patch.http),
HttpOptions.make(patch.http),
headers === undefined ? undefined : new HttpOptions({ headers }),
),
}
@@ -172,11 +174,6 @@ const mergeHeaders = (...items: ReadonlyArray<Record<string, string> | undefined
export const generationOptions = (input: GenerationOptions.Input | undefined) =>
input === undefined ? undefined : GenerationOptions.make(input)
export const httpOptions = (input: HttpOptionsInput | undefined) => {
if (input === undefined) return input
return HttpOptions.make(input)
}
export interface Interface {
readonly compact: CompactMethod
readonly stream: StreamMethod
@@ -261,7 +258,9 @@ const unsupportedCompaction = (request: LLMRequest, mechanism: string | undefine
})
}
export class Service extends Context.Service<Service, Interface>()("@opencode/LLMClient") {}
export class LLMClientService extends Context.Service<LLMClientService, Interface>()("@opencode/LLMClient") {}
export const Service = LLMClientService
export type Service = LLMClientService
const resolveRequestOptions = (request: LLMRequest) => {
const messages = normalizeToolHistory(request.messages)
@@ -391,6 +390,7 @@ function makeFromTransport<Body, Prepared, Frame, Event, State>(
defaults: routeInput.defaults ?? {},
body: protocol.body,
supportsEffortUpdates: protocol.supportsEffortUpdates,
sanitizer: protocol.sanitizer,
with: (patch: RoutePatch<Body, Prepared>) => {
const { compact, id, provider, providerMetadataKey, auth, transport, endpoint, ...defaults } = patch
return build({
@@ -562,7 +562,9 @@ const prepareRequest = (request: LLMRequest) => {
tool.type === "tool" ? tool : { ...tool, tools: dedupe(tool.tools) },
)
const resolved = applyCachePolicy(
applyEffortUpdates(LLMRequest.update(sanitized, { tools: dedupe(sanitized.tools) })),
applyEffortUpdates(
LLMRequest.update(sanitized, { tools: ToolSchemaProjection.tools(dedupe(sanitized.tools), sanitized.model) }),
),
)
const headers = resolved.model.route.headers?.({ request: resolved })
return headers === undefined
+3 -2
View File
@@ -1,5 +1,4 @@
import type { LLMRequest } from "../schema/index.js"
import * as ProviderShared from "../protocols/shared.js"
export interface EndpointInput<Body, Request = LLMRequest> {
readonly request: Request
@@ -47,6 +46,8 @@ export const merge = <Body, Request = LLMRequest>(
query: patch.query === undefined ? base.query : { ...base.query, ...patch.query },
})
export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
const renderPart = <Body, Request>(part: EndpointPart<Body, Request>, input: EndpointInput<Body, Request>) =>
typeof part === "function" ? part(input) : part
@@ -54,7 +55,7 @@ export const render = <Body, Request = LLMRequest>(
endpoint: Definition<Body, Request>,
input: EndpointInput<Body, Request>,
) => {
const url = new URL(`${ProviderShared.trimBaseUrl(endpoint.baseURL ?? "")}${renderPart(endpoint.path, input)}`)
const url = new URL(`${trimBaseUrl(endpoint.baseURL ?? "")}${renderPart(endpoint.path, input)}`)
for (const [key, value] of Object.entries(endpoint.query ?? {})) url.searchParams.set(key, value)
return url
}
+5 -1
View File
@@ -19,4 +19,8 @@ export type HttpMiddleware = (
handler: HttpHandler,
) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error>
export class Service extends Context.Service<Service, Interface>()("@opencode/AI/RequestExecutor") {}
export class RequestExecutorService extends Context.Service<RequestExecutorService, Interface>()(
"@opencode/AI/RequestExecutor",
) {}
export const Service = RequestExecutorService
export type Service = RequestExecutorService
+16
View File
@@ -255,4 +255,20 @@ export const layer: Layer.Layer<Service, never, HttpClient.HttpClient> = Layer.e
export const fetchLayer = layer.pipe(Layer.provide(FetchHttpClient.layer))
/** Run `fn` on every request: it sees the raw response before status classification, inside middleware already on `executor`, and outside per-call middleware. */
export const middleware = (fn: HttpMiddleware, executor: Layer.Layer<Service> = fetchLayer): Layer.Layer<Service> =>
Layer.effect(
Service,
Effect.gen(function* () {
const inner = yield* Service
return Service.of({
execute: (request, next) =>
inner.execute(
request,
next === undefined ? fn : (input, handler) => fn(input, (forwarded) => next(forwarded, handler)),
),
})
}),
).pipe(Layer.provide(executor))
export * as RequestExecutor from "./executor.js"
+79 -6
View File
@@ -1,6 +1,6 @@
import type { Stream } from "effect"
import * as ProviderShared from "../protocols/shared.js"
import type { AIError } from "../schema/index.js"
import { Effect, Stream } from "effect"
import { makeParser, type Event } from "effect/unstable/encoding/Sse"
import { AIError, InvalidProviderOutputError } from "../schema/index.js"
/**
* Decode a streaming HTTP response body into provider-protocol frames.
@@ -12,6 +12,8 @@ import type { AIError } from "../schema/index.js"
* `[DONE]`; protocols that use it as a terminal select `sseWithDone`.
* - AWS event stream — length-prefixed binary frames with CRC checksums.
* Each emitted frame is one parsed binary event record.
* - Media streams — newline-delimited JSON (`lines`) or the whole body as one
* frame (`document`); chunked binary bodies need no framing.
*
* The frame type is opaque to this layer; the protocol's event schema decodes
* each frame before its state machine handles it.
@@ -23,19 +25,90 @@ export interface Definition<Frame> {
readonly body?: (frame: Frame) => string | undefined
}
/**
* `framing` step for Server-Sent Events. Decodes UTF-8, runs the SSE channel
* decoder, optionally filters named events, and drops empty events and known
* keepalives that proxies send as data. `[DONE]` is dropped by default or
* retained for protocols that use it as their stream boundary. Retry control events are ignored without
* interrupting the stream. Decoder failures become provider output errors so
* the public error channel stays `AIError`.
*/
export const sseFraming = (
bytes: Stream.Stream<Uint8Array, AIError>,
events?: ReadonlySet<string>,
includeDone = false,
): Stream.Stream<string, AIError> =>
bytes.pipe(
Stream.decodeText(),
Stream.mapAccumEffect(
() => {
const output: Event[] = []
return {
output,
parser: makeParser((event) => {
if (event._tag === "Event") output.push(event)
}),
}
},
(state, chunk) =>
Effect.gen(function* () {
const error = state.parser.feed(chunk)
if (error)
return yield* new AIError({
reason: new InvalidProviderOutputError({
route: "sse",
message: error.message,
body: chunk,
cause: error,
}),
})
return [state, state.output.splice(0)] as const
}),
),
Stream.filter(
(event) =>
(events === undefined || events.has(event.event)) &&
event.data.length > 0 &&
// Some OpenAI-compatible proxies serialize an empty flush as a bare
// `data: null`, between events or after `[DONE]`. No protocol has a
// null event, so it carries nothing and must not abort the stream.
event.data !== "null" &&
// Vertex AI partner models (e.g. `xai/grok-4.6`) send their SSE
// keepalive comment as `data: : keepalive` while reasoning.
event.data !== ": keepalive" &&
(event.data !== "[DONE]" || includeDone || (events !== undefined && event.event !== "message")),
),
Stream.map((event) => event.data),
)
/** Server-Sent Events framing. Used by every JSON-streaming HTTP provider. */
export const sse: Definition<string> = { id: "sse", frame: ProviderShared.sseFraming }
export const sse: Definition<string> = { id: "sse", frame: sseFraming }
/** Server-Sent Events framing that retains the conventional `[DONE]` sentinel. */
export const sseWithDone: Definition<string> = {
id: "sse",
frame: (bytes) => ProviderShared.sseFraming(bytes, undefined, true),
frame: (bytes) => sseFraming(bytes, undefined, true),
}
/** SSE framing restricted to protocol-recognized event names. */
export const sseEvents = (events: ReadonlySet<string>): Definition<string> => ({
id: "sse",
frame: (bytes) => ProviderShared.sseFraming(bytes, events),
frame: (bytes) => sseFraming(bytes, events),
})
export const lines: Definition<string> = {
id: "lines",
frame: (bytes) =>
bytes.pipe(
Stream.decodeText(),
Stream.splitLines,
Stream.filter((line) => line.trim().length > 0),
),
}
export const document: Definition<string> = {
id: "document",
frame: (bytes) => Stream.fromEffect(Stream.mkString(bytes.pipe(Stream.decodeText()))),
}
export * as Framing from "./framing.js"
+1 -1
View File
@@ -7,7 +7,7 @@ export type {
RouteDefaultsInput,
AnyRoute,
Interface as LLMClientShape,
Service as LLMClientService,
LLMClientService,
StreamOptions,
CompactMethod,
CompactionOperations,
+262 -45
View File
@@ -1,17 +1,54 @@
import { Effect, Schema } from "effect"
import { Clock, Duration, Effect, Schema, type Stream } from "effect"
import { HttpClientResponse } from "effect/unstable/http"
import { AIError, HttpContext, InvalidProviderOutputError } from "../schema/index.js"
import type { Snapshot, Status } from "../generation.js"
import { Media } from "../media.js"
import type { AuthInput } from "./auth.js"
import {
AIError,
ContentPolicyError,
HttpContext,
InvalidProviderOutputError,
InvalidRequestError,
ProviderID,
ProviderInternalError,
UnsupportedOperationError,
} from "../schema/index.js"
// ---------------------------------------------------------------------------
// Bodies
// ---------------------------------------------------------------------------
export type Body =
| { readonly type: "json"; readonly value: Record<string, unknown> }
| { readonly type: "multipart"; readonly value: FormData }
/** Array values become repeated parameters (`keyterm=a&keyterm=b`). */
export type Query = Readonly<Record<string, string | ReadonlyArray<string>>>
export const json = (value: Record<string, unknown>): Body => ({ type: "json", value })
/** `query` is appended to the endpoint URL before the route and caller `http.query` overlays. */
export type Body =
| { readonly type: "json"; readonly value: Record<string, unknown>; readonly query?: Query }
| { readonly type: "multipart"; readonly value: FormData }
| {
readonly type: "binary"
readonly value: Uint8Array
readonly contentType: string
readonly query?: Query
}
export const json = (value: Record<string, unknown>, query?: Query): Body => ({
type: "json",
value,
query,
})
export const multipart = (value: FormData): Body => ({ type: "multipart", value })
export const binary = (value: Uint8Array, contentType: string, query?: Query): Body => ({
type: "binary",
value,
contentType,
query,
})
export type Send = (path: string, body: Body) => Effect.Effect<HttpClientResponse.HttpClientResponse, AIError>
/** Runs after unsupported-field rejection and before `body.from`, for providers that need an upload first. */
export type Prepare<Request> = (request: Request, send: Send) => Effect.Effect<Request, AIError>
// ---------------------------------------------------------------------------
// Protocol kinds
@@ -26,7 +63,7 @@ export interface DecodeContext<Request> {
export interface Inline<Request, Response> {
readonly kind: "inline"
readonly id: string
readonly name: string
readonly provider: ProviderID
/** Common request fields this protocol cannot lower; the route rejects them before `body.from` runs. */
readonly unsupported?: ReadonlyArray<keyof Request & string>
readonly body: { readonly from: (request: Request) => Effect.Effect<Body, AIError> }
@@ -39,11 +76,113 @@ export interface Inline<Request, Response> {
}
export const inline = <Request, Response>(
input: Omit<Inline<Request, Response>, "kind">,
): Inline<Request, Response> => ({
kind: "inline",
...input,
})
route: Identity,
input: Omit<Inline<Request, Response>, "kind" | "id" | "provider">,
): Inline<Request, Response> => ({ kind: "inline", id: route.id, provider: route.provider, ...input })
/** What `start` learned from the submission response: the route-owned handle plus the first observation. */
export interface Started<Token> {
readonly token: Token
readonly snapshot: Snapshot
}
/**
* A follow-up call's inputs: the decoded token and the auth headers the route sent, so a protocol can attach them
* to output URLs that require the same credentials to download (Veo). `materialize` downloads an output through the
* route's executor, for URLs that expire too soon to hand back (BFL).
*/
export interface PollContext<Token> {
readonly token: Token
readonly auth: Record<string, string>
readonly materialize: (asset: Media.Asset) => Effect.Effect<Media.Asset, AIError>
}
/**
* Submit, then poll. `start` posts the body to the route endpoint; `status`, `result`, and `cancel` are follow-up
* calls addressed by the token. Paths are relative to the route base URL unless the provider hands back absolute
* URLs (fal `status_url`), in which case they are used verbatim. `result` is always its own GET: providers that
* return the output inside the status body (Veo, xAI, Runway) point `result.path` at the status path and decode the
* same document, so `Generation.await` and `Video.resume(...).await()` behave identically everywhere.
*/
export interface Queued<Request, Response, Token> {
readonly kind: "queued"
readonly id: string
readonly provider: ProviderID
/** Common request fields this protocol cannot lower; the route rejects them before `start.body.from` runs. */
readonly unsupported?: ReadonlyArray<keyof Request & string>
/** Serializable handle. `Generation.token` carries the encoded form so it can be persisted and resumed elsewhere. */
readonly token: Schema.Codec<Token, unknown>
readonly start: {
readonly prepare?: Prepare<Request>
readonly body: { readonly from: (request: Request) => Effect.Effect<Body, AIError> }
readonly decode: (
response: HttpClientResponse.HttpClientResponse,
context: DecodeContext<Request>,
) => Effect.Effect<Started<Token>, AIError>
}
readonly status: {
readonly path: (token: Token) => string
readonly decode: (
response: HttpClientResponse.HttpClientResponse,
context: PollContext<Token>,
) => Effect.Effect<Snapshot, AIError>
}
readonly result: {
readonly path: (token: Token) => string
readonly decode: (
response: HttpClientResponse.HttpClientResponse,
context: PollContext<Token>,
) => Effect.Effect<Response, AIError>
}
readonly cancel?: {
readonly method: AuthInput["method"]
readonly path: (token: Token) => string
/**
* Fetch a fresh status first and skip the call for terminal generations, for providers whose cancel endpoint
* destroys finished work (Runway's `DELETE /v1/tasks/{id}` deletes completed tasks and their outputs).
*/
readonly activeOnly?: boolean
}
}
export const queued = <Request, Response, Token>(
route: Identity,
input: Omit<Queued<Request, Response, Token>, "kind" | "id" | "provider">,
): Queued<Request, Response, Token> => ({ kind: "queued", id: route.id, provider: route.provider, ...input })
export type Mode = "generate" | "stream"
export type Addressed<Request> = Request & { readonly mode: Mode }
export interface ResponseContext<Request> extends DecodeContext<Addressed<Request>> {
readonly http: HttpContext
}
/**
* One request whose body is parsed incrementally, like LLM protocols: `frames` → `step`* → `finish`. `generate` and
* `stream` share this state machine; `request.mode` lets a protocol pick a different body, path, or framing.
*/
export interface Streamed<Request, Event, Frame, State> {
readonly kind: "stream"
readonly id: string
readonly provider: ProviderID
/** Common request fields this protocol cannot lower; the route rejects them before `body.from` runs. */
readonly unsupported?: ReadonlyArray<keyof Request & string>
readonly body: { readonly from: (request: Addressed<Request>) => Effect.Effect<Body, AIError> }
readonly frames: (
bytes: Stream.Stream<Uint8Array, AIError>,
context: DecodeContext<Addressed<Request>>,
) => Stream.Stream<Frame, AIError>
readonly initial: () => State
readonly step: (state: State, frame: Frame) => Effect.Effect<readonly [State, ReadonlyArray<Event>], AIError>
/** Emit exactly one terminal event, or fail when the provider stopped before completing. */
readonly finish: (state: State, context: ResponseContext<Request>) => Effect.Effect<ReadonlyArray<Event>, AIError>
}
export const stream = <Request, Event, Frame, State>(
route: Identity,
input: Omit<Streamed<Request, Event, Frame, State>, "kind" | "id" | "provider">,
): Streamed<Request, Event, Frame, State> => ({ kind: "stream", id: route.id, provider: route.provider, ...input })
// ---------------------------------------------------------------------------
// Response helpers
@@ -52,44 +191,122 @@ export const inline = <Request, Response>(
const context = (response: HttpClientResponse.HttpClientResponse) =>
new HttpContext({ url: response.request.url, status: response.status, headers: response.headers })
/** Read a text body while retaining the original payload and HTTP context on every downstream error. */
export const text = Effect.fn("MediaProtocol.text")(function* (
route: string,
name: string,
response: HttpClientResponse.HttpClientResponse,
) {
const http = context(response)
const body = yield* response.text.pipe(
Effect.mapError(
(cause) =>
/** One protocol's route id, display name, and provider, with the decoders and errors that carry them. */
export const identity = (input: { readonly id: string; readonly name: string; readonly provider: string }) => {
const provider = ProviderID.make(input.provider)
const frameError = (message: string, body?: string, cause?: unknown) =>
new AIError({ reason: new InvalidProviderOutputError({ route: input.id, message, body, cause }) })
/**
* Read a text body while retaining the original payload and HTTP context on every downstream error. `invalid` is a
* malformed provider document; `ended` is a generation that reached a terminal status without output (`failed` is
* provider-side, `cancelled`/`expired` mean the result will never exist); `pending` is a `result()` read before the
* generation finished, which is caller misuse; `contentPolicy` is a moderated result.
*/
const text = Effect.fn("MediaProtocol.text")(function* (response: HttpClientResponse.HttpClientResponse) {
const http = context(response)
const body = yield* response.text.pipe(
Effect.mapError(
(cause) =>
new AIError({
reason: new InvalidProviderOutputError({
route: input.id,
message: `Failed to read the ${input.name} response`,
http,
cause,
}),
}),
),
)
return {
body,
http,
invalid: (message: string, cause?: unknown) =>
new AIError({ reason: new InvalidProviderOutputError({ route: input.id, message, body, http, cause }) }),
ended: (status: Exclude<Status, "queued" | "running" | "completed">, message: string) =>
new AIError({
reason: new InvalidProviderOutputError({
route,
message: `Failed to read the ${name} response`,
reason:
status === "failed"
? new ProviderInternalError({ message, body, http })
: new InvalidRequestError({ message, body, http }),
}),
pending: (id: string) =>
new AIError({
reason: new InvalidRequestError({
message: `${input.name} generation ${id} has not finished; await it before reading the result`,
body,
http,
cause,
}),
}),
),
)
return {
body,
http,
invalid: (message: string, cause?: unknown) =>
new AIError({ reason: new InvalidProviderOutputError({ route, message, body, http, cause }) }),
}
})
/** Read and Schema-decode a JSON body. Decode failures keep the raw body as `reason.body`. */
export const decodeJson = <A>(route: string, name: string, schema: Schema.Codec<A, unknown>) => {
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
return Effect.fn("MediaProtocol.decodeJson")(function* (response: HttpClientResponse.HttpClientResponse) {
const output = yield* text(route, name, response)
const value = yield* decode(output.body).pipe(
Effect.mapError((cause) => output.invalid(`${name} returned an invalid response`, cause)),
)
return { ...output, value }
contentPolicy: (message: string) => new AIError({ reason: new ContentPolicyError({ message, body, http }) }),
}
})
/** Read and Schema-decode a JSON body. Decode failures keep the raw body as `reason.body`. */
const decodeJson = <A>(schema: Schema.Codec<A, unknown>) => {
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
return Effect.fn("MediaProtocol.decodeJson")(function* (response: HttpClientResponse.HttpClientResponse) {
const output = yield* text(response)
const value = yield* decode(output.body).pipe(
Effect.mapError((cause) => output.invalid(`${input.name} returned an invalid response`, cause)),
)
return { ...output, value }
})
}
return {
id: input.id,
name: input.name,
provider,
text,
decodeJson,
/** Decode a submission response into the token and first snapshot. */
decodeStarted: <A, Token>(schema: Schema.Codec<A, unknown>, started: (value: A) => Started<Token>) => {
const decode = decodeJson(schema)
return (response: HttpClientResponse.HttpClientResponse) =>
decode(response).pipe(Effect.map((output) => started(output.value)))
},
/** Schema-decode one JSON stream frame. Decode failures keep the frame as `reason.body`. */
decodeFrame: <A>(schema: Schema.Codec<A, unknown>) => {
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(schema))
return (frame: string) =>
decode(frame).pipe(
Effect.mapError((cause) => frameError(`${input.name} sent an invalid stream event`, frame, cause)),
)
},
/** A stream-time failure; the frame stays on `reason.body`. */
frameError,
incomplete: () =>
new AIError({
reason: new InvalidProviderOutputError({
route: input.id,
message: "The provider response ended unexpectedly.",
classification: "incomplete-stream",
}),
}),
unsupported: (operation: string, message: string) =>
new AIError({ reason: new UnsupportedOperationError({ operation, provider, route: input.id, message }) }),
}
}
export type Identity = ReturnType<typeof identity>
export type Output = Effect.Success<ReturnType<Identity["text"]>>
/** Map a provider status string through the protocol's table; unknown values are an invalid provider document. */
export const status = <Table extends Record<string, Status>>(
table: Table,
raw: string,
output: Output,
): Effect.Effect<Status, AIError> => {
if (!Object.hasOwn(table, raw)) return Effect.fail(output.invalid(`Unknown generation status "${raw}"`))
return Effect.succeed(table[raw])
}
/** A `url` asset whose provider-declared retention window starts now. */
export const expiringUrl = (url: string, retention: Duration.Duration, options?: Parameters<typeof Media.url>[1]) =>
Clock.currentTimeMillis.pipe(
Effect.map((now) => Media.url(url, { ...options, expiresAt: now + Duration.toMillis(retention) })),
)
export * as MediaProtocol from "./media-protocol.js"
+351 -51
View File
@@ -1,11 +1,22 @@
import { Effect } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { Auth } from "./auth.js"
import { Effect, Schema, Stream } from "effect"
import { Headers, HttpClientRequest, type HttpClientResponse } from "effect/unstable/http"
import { Auth, type AuthInput } from "./auth.js"
import { Endpoint } from "./endpoint.js"
import type { Interface } from "./executor-service.js"
import { RequestExecutorService, type Interface } from "./executor-service.js"
import { RequestExecutor } from "./executor.js"
import { MediaProtocol } from "./media-protocol.js"
import { ProviderShared } from "../protocols/shared.js"
import { AIError, HttpOptions, ProviderID, mergeHttpOptions } from "../schema/index.js"
import { Generation, isTerminal } from "../generation.js"
import type { Media } from "../media.js"
import {
AIError,
AIErrorReason,
HttpOptions,
InvalidRequestError,
ProviderID,
UnsupportedOperationError,
mergeHttpOptions,
} from "../schema/index.js"
import { encodeJson } from "../utils/json.js"
import { sanitizeSurrogates } from "../utils/sanitize.js"
export type Execute = Interface["execute"]
@@ -25,73 +36,360 @@ export interface ModelInput {
readonly http?: HttpOptions
}
/** A provider facade's `configure(...)` input as the `ModelInput` every media selector shares, minus the model id. */
export const deployment = (
input: { readonly baseURL?: string; readonly headers?: Record<string, string>; readonly http?: HttpOptions.Input },
auth: Auth.Definition,
): Omit<ModelInput, "id"> => ({
auth,
baseURL: input.baseURL,
headers: input.headers,
http: HttpOptions.make(input.http),
})
// ---------------------------------------------------------------------------
// Route
// Routes
// ---------------------------------------------------------------------------
export interface Route<Request extends MediaRequest, Response> {
/** One request, one response. */
export interface InlineRoute<Request extends MediaRequest, Response> {
readonly kind: "inline"
readonly id: string
readonly provider: ProviderID
readonly protocol: string
readonly generate: (request: Request, execute: Execute) => Effect.Effect<Response, AIError>
}
export interface MakeInput<Request extends MediaRequest, Response> {
/** Submit, then poll through the returned `Generation`. */
export interface QueuedRoute<Request extends MediaRequest, Response> {
readonly kind: "queued"
readonly id: string
readonly provider: string | ProviderID
readonly protocol: MediaProtocol.Inline<Request, Response>
readonly provider: ProviderID
readonly protocol: string
readonly start: (request: Request, execute: Execute) => Effect.Effect<Generation<Response>, AIError>
/** Rebuild a handle from a persisted `Generation.token`; fails typed when the token is not this route's. */
readonly resume: (
model: MediaRequest["model"],
token: unknown,
execute: Execute,
) => Effect.Effect<Generation<Response>, AIError>
}
/** One request whose response parses into events; `generate` runs the same stream and collects it. */
export interface StreamRoute<Request extends MediaRequest, Event, Response> {
readonly kind: "stream"
readonly id: string
readonly provider: ProviderID
readonly protocol: string
readonly stream: (request: Request, execute: Execute) => Stream.Stream<Event, AIError>
readonly generate: (request: Request, execute: Execute) => Effect.Effect<Response, AIError>
}
export type AnyRoute<Request extends MediaRequest, Event, Response> =
| InlineRoute<Request, Response>
| StreamRoute<Request, Event, Response>
| QueuedRoute<Request, Response>
export interface Composition<Request extends MediaRequest> {
readonly endpoint: Endpoint.Definition<MediaProtocol.Body, Request>
readonly auth: Auth.Definition
/** Deployment headers applied before transport authentication. */
readonly headers?: Record<string, string>
}
export interface InlineInput<Request extends MediaRequest, Response> extends Composition<Request> {
readonly protocol: MediaProtocol.Inline<Request, Response>
}
export interface QueuedInput<Request extends MediaRequest, Response, Token> extends Composition<Request> {
readonly protocol: MediaProtocol.Queued<Request, Response, Token>
}
export interface StreamInput<Request extends MediaRequest, Event, Response, Frame, State>
extends Composition<MediaProtocol.Addressed<Request>> {
readonly protocol: MediaProtocol.Streamed<Request, Event, Frame, State>
readonly collect: (events: ReadonlyArray<Event>) => Effect.Effect<Response, AIError>
}
/**
* Compose an inline media protocol with an endpoint and auth into a runnable route. The route owns the transport
* plumbing every media protocol would otherwise duplicate: option merging, surrogate sanitizing, unsupported-field rejection, URL and query
* rendering, auth headers, JSON vs multipart encoding, and handing the response back to the protocol for decoding.
* plumbing every media protocol would otherwise duplicate: option merging, surrogate sanitizing, unsupported-field
* rejection, URL and query rendering, auth headers, JSON, multipart, or binary encoding, and handing responses back
* to the protocol.
*/
export const make = <Request extends MediaRequest, Response>(
input: MakeInput<Request, Response>,
): Route<Request, Response> => {
const provider = ProviderID.make(input.provider)
const routeHttp = input.headers === undefined ? undefined : new HttpOptions({ headers: input.headers })
const authorize = Auth.toEffect(input.auth)
export const inline = <Request extends MediaRequest, Response>(
input: InlineInput<Request, Response>,
): InlineRoute<Request, Response> => {
const transport = makeTransport(input)
return {
id: input.id,
provider,
kind: "inline",
id: input.protocol.id,
provider: input.protocol.provider,
protocol: input.protocol.id,
generate: Effect.fn(`MediaRoute.generate`)(function* (request: Request, execute: Execute) {
yield* rejectUnsupported(input.id, provider, request, input.protocol.unsupported)
const http = mergeHttpOptions(routeHttp, request.model.http, request.http)
// Sanitize after merging so model-level overlays are covered; the model value is restored, not sanitized.
const resolved: Request = { ...sanitizeSurrogates({ ...request, http }), model: request.model }
const body = yield* input.protocol.body.from(resolved)
const url = Endpoint.render(input.endpoint, { request: resolved, body })
for (const [key, value] of Object.entries(http?.query ?? {})) url.searchParams.set(key, value)
const encoded = body.type === "json" ? ProviderShared.encodeJson(body.value) : "[multipart/form-data]"
const baseHeaders = Headers.fromInput(http?.headers)
const headers = yield* authorize({
request: resolved,
method: "POST",
url: url.toString(),
body: encoded,
// The HTTP client sets the multipart boundary; a caller-supplied content-type would corrupt it.
headers: body.type === "multipart" ? Headers.remove(baseHeaders, "content-type") : baseHeaders,
})
const transport = HttpClientRequest.post(url.toString()).pipe(
HttpClientRequest.setHeaders(headers),
body.type === "json"
? HttpClientRequest.bodyText(encoded, "application/json")
: HttpClientRequest.bodyFormData(body.value),
const submitted = yield* transport.submit(
request,
{ unsupported: input.protocol.unsupported, from: input.protocol.body.from },
execute,
)
const response = yield* execute(transport)
return yield* input.protocol.response.decode(response, { request: resolved, body })
return yield* input.protocol.response.decode(submitted.response, submitted.context)
}),
}
}
/** Common fields are never silently dropped: a present field the protocol declared unsupported fails typed. */
/**
* Compose a queued media protocol the same way, adding `start`/`resume` handles whose polls reuse the route's auth,
* deployment headers, and (for `start`) the request's `http` overlay. The token is decoded once at the boundary and
* closed over by the resulting `Generation.Route`.
*/
export const queued = <Request extends MediaRequest, Response, Token>(
input: QueuedInput<Request, Response, Token>,
): QueuedRoute<Request, Response> => {
const transport = makeTransport(input)
const protocol = input.protocol
const decodeToken = Schema.decodeUnknownEffect(protocol.token)
// A protocol producing a token its own codec rejects is a programmer defect, not a provider error.
const encodeToken = Schema.encodeSync(protocol.token)
const generationRoute = (token: Token, http: HttpOptions | undefined, execute: Execute) => {
const materialize = (asset: Media.Asset) =>
asset.materialize().pipe(Effect.provideService(RequestExecutorService, { execute }))
const poll = <A>(operation: {
readonly path: (token: Token) => string
readonly decode: (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) => Effect.Effect<A, AIError>
}) =>
transport
.call("GET", operation.path(token), http, execute)
.pipe(Effect.flatMap((sent) => operation.decode(sent.response, { token, auth: sent.auth, materialize })))
const status = poll(protocol.status)
const cancel = protocol.cancel
const send =
cancel === undefined
? undefined
: transport.call(cancel.method, cancel.path(token), http, execute).pipe(Effect.asVoid)
return {
status,
result: poll(protocol.result),
cancel:
send !== undefined && cancel?.activeOnly
? status.pipe(Effect.flatMap((snapshot) => (isTerminal(snapshot.status) ? Effect.void : send)))
: send,
}
}
const start = Effect.fn("MediaRoute.start")(function* (request: Request, execute: Execute) {
const submitted = yield* transport.submit(
request,
{ unsupported: protocol.unsupported, prepare: protocol.start.prepare, from: protocol.start.body.from },
execute,
)
const started = yield* protocol.start.decode(submitted.response, submitted.context)
const route = generationRoute(started.token, submitted.context.request.http, execute)
return new Generation(route, encodeToken(started.token), started.snapshot)
})
const resume = Effect.fn("MediaRoute.resume")(function* (
model: MediaRequest["model"],
raw: unknown,
execute: Execute,
) {
const token = yield* decodeToken(raw).pipe(
Effect.mapError(
(cause) =>
new AIError({
reason: new InvalidRequestError({
message: `${protocol.id} cannot resume a generation from this token`,
cause,
}),
}),
),
)
const route = generationRoute(token, transport.http(model), execute)
return new Generation(route, encodeToken(token), yield* route.status)
})
return { kind: "queued", id: protocol.id, provider: protocol.provider, protocol: protocol.id, start, resume }
}
/** Compose a streaming media protocol; `generate` runs the same stream in `generate` mode and folds it with `collect`. */
export const stream = <Request extends MediaRequest, Event, Response, Frame, State>(
input: StreamInput<Request, Event, Response, Frame, State>,
): StreamRoute<Request, Event, Response> => {
const transport = makeTransport(input)
const protocol = input.protocol
const events = (request: Request, execute: Execute, mode: MediaProtocol.Mode) =>
Stream.unwrap(
Effect.gen(function* () {
const submitted = yield* transport.submit(
{ ...request, mode },
{ unsupported: protocol.unsupported, from: protocol.body.from },
execute,
)
const http = RequestExecutor.responseHttp(submitted.response)
return Stream.suspend(() => {
// Parser state is local to one response, exactly like `Route.make`'s LLM stream loop.
let state = protocol.initial()
return protocol.frames(RequestExecutor.responseStream(submitted.response), submitted.context).pipe(
Stream.mapEffect((frame) =>
protocol.step(state, frame).pipe(
Effect.map(([next, output]) => {
state = next
return output
}),
),
),
Stream.flattenIterable,
Stream.concat(
Stream.suspend(() => Stream.fromIterableEffect(protocol.finish(state, { ...submitted.context, http }))),
),
Stream.mapError((error) =>
error.reason.http !== undefined
? error
: new AIError({
reason: AIErrorReason.make({
...error.reason,
message: error.reason.message,
cause: error.reason.cause,
http,
}),
}),
),
)
})
}),
)
return {
kind: "stream",
id: protocol.id,
provider: protocol.provider,
protocol: protocol.id,
stream: (request, execute) => events(request, execute, "stream"),
generate: (request, execute) =>
events(request, execute, "generate").pipe(Stream.runCollect, Effect.flatMap(input.collect)),
}
}
// ---------------------------------------------------------------------------
// Transport plumbing shared by every kind
// ---------------------------------------------------------------------------
const makeTransport = <Request extends MediaRequest>(
input: Composition<Request> & { readonly protocol: { readonly id: string; readonly provider: ProviderID } },
) => {
const routeHttp = input.headers === undefined ? undefined : new HttpOptions({ headers: input.headers })
const authorize = Auth.toEffect(input.auth)
const baseURL = (path: string) => new URL(`${Endpoint.trimBaseUrl(input.endpoint.baseURL ?? "")}${path}`)
/** `auth` is only what `Auth` added or changed, never untouched deployment headers. */
const send = Effect.fn("MediaRoute.send")(function* (
call: {
readonly method: AuthInput["method"]
readonly url: URL
readonly headers: Headers.Headers
readonly request: AuthInput["request"]
readonly body?: MediaProtocol.Body
},
execute: Execute,
) {
const encoded = encode(call.body, call.headers)
const url = call.url.toString()
const headers = yield* authorize({
request: call.request,
method: call.method,
url,
body: encoded.text,
headers: encoded.headers,
})
const response = yield* execute(
encoded.apply(HttpClientRequest.make(call.method)(url).pipe(HttpClientRequest.setHeaders(headers))),
)
return {
response,
auth: Object.fromEntries(Object.entries(headers).filter(([key, value]) => encoded.headers[key] !== value)),
}
})
return {
/** Route and model overlays; `start` additionally merges the request's own `http`. */
http: (model: MediaRequest["model"]) => mergeHttpOptions(routeHttp, model.http),
/** POST the protocol body to the route endpoint. */
submit: Effect.fn("MediaRoute.submit")(function* (
request: Request,
protocol: {
readonly unsupported?: ReadonlyArray<keyof Request & string>
readonly prepare?: MediaProtocol.Prepare<Request>
readonly from: (request: Request) => Effect.Effect<MediaProtocol.Body, AIError>
},
execute: Execute,
) {
yield* rejectUnsupported(input.protocol.id, input.protocol.provider, request, protocol.unsupported)
const http = mergeHttpOptions(routeHttp, request.model.http, request.http)
const headers = Headers.fromInput(http?.headers)
const prepared =
protocol.prepare === undefined
? request
: yield* protocol.prepare(request, (path, body) =>
send({ method: "POST", url: baseURL(path), headers, request, body }, execute).pipe(
Effect.map((sent) => sent.response),
),
)
// Sanitize after merging so model-level overlays are covered; the model value is restored, not sanitized.
const resolved: Request = { ...sanitizeSurrogates({ ...prepared, http }), model: request.model }
const body = yield* protocol.from(resolved)
const url = withQuery(
withQuery(
Endpoint.render(input.endpoint, { request: resolved, body }),
body.type === "multipart" ? undefined : body.query,
),
http?.query,
)
const sent = yield* send({ method: "POST", url, headers, request: resolved, body }, execute)
return { response: sent.response, context: { request: resolved, body } }
}),
/** Bodiless follow-up call (status, result, cancel) with the same auth and headers as `submit`. */
call: (method: AuthInput["method"], path: string, http: HttpOptions | undefined, execute: Execute) => {
// Provider-issued absolute URLs (fal `status_url`) are used as-is; everything else resolves against the base.
const url = withQuery(/^https?:\/\//.test(path) ? new URL(path) : baseURL(path), http?.query)
for (const [key, value] of Object.entries(input.endpoint.query ?? {})) url.searchParams.set(key, value)
return send({ method, url, headers: Headers.fromInput(http?.headers), request: { http } }, execute)
},
}
}
const withQuery = (url: URL, query: MediaProtocol.Query | undefined) => {
for (const [key, value] of Object.entries(query ?? {})) {
url.searchParams.delete(key)
for (const item of typeof value === "string" ? [value] : value) url.searchParams.append(key, item)
}
return url
}
const encode = (body: MediaProtocol.Body | undefined, headers: Headers.Headers) => {
if (body === undefined) return { text: "", headers, apply: (request: HttpClientRequest.HttpClientRequest) => request }
if (body.type === "json") {
const text = encodeJson(body.value)
return { text, headers, apply: HttpClientRequest.bodyText(text, "application/json") }
}
if (body.type === "binary")
return {
text: `[${body.contentType}]`,
headers,
apply: HttpClientRequest.bodyUint8Array(body.value, body.contentType),
}
return {
text: "[multipart/form-data]",
// The HTTP client sets the multipart boundary; a caller-supplied content-type would corrupt it.
headers: Headers.remove(headers, "content-type"),
apply: HttpClientRequest.bodyFormData(body.value),
}
}
/**
* Common fields are never silently dropped: a present field the protocol declared unsupported fails typed. `false`
* counts as present because some booleans mean something when false (video `audio`); protocols reject opt-in
* booleans such as speech `timestamps` with `=== true` in `body.from` instead of listing them.
*/
const rejectUnsupported = <Request extends object>(
route: string,
provider: ProviderID,
@@ -104,11 +402,13 @@ const rejectUnsupported = <Request extends object>(
})
if (present.length === 0) return Effect.void
return Effect.fail(
ProviderShared.unsupportedOperation({
operation: `media.${present[0]}`,
provider,
route,
message: `${provider}/${route} does not support ${present.join(", ")}`,
new AIError({
reason: new UnsupportedOperationError({
operation: `media.${present[0]}`,
provider,
route,
message: `${provider}/${route} does not support ${present.join(", ")}`,
}),
}),
)
}

Some files were not shown because too many files have changed in this diff Show More