mirror of
https://github.com/anomalyco/opencode.git
synced 2026-09-01 14:36:20 +00:00
Compare commits
302
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
be6ddabcc2 | ||
|
|
cd25e872e9 | ||
|
|
77deb25801 | ||
|
|
c9ddafa46c | ||
|
|
d0396012a1 | ||
|
|
64746bf124 | ||
|
|
e07ae57caa | ||
|
|
5c2891c2fd | ||
|
|
dbe2998fce | ||
|
|
cd7447e834 | ||
|
|
fa9976a41a | ||
|
|
0d045152fc | ||
|
|
ffb9728068 | ||
|
|
eb083cce63 | ||
|
|
1afb7c614e | ||
|
|
56e773831c | ||
|
|
6b1ed3918a | ||
|
|
1b3eb1138e | ||
|
|
711a0a2da2 | ||
|
|
ac77cc46b8 | ||
|
|
7197fdfb4e | ||
|
|
5a4914c670 | ||
|
|
3a797bf6e4 | ||
|
|
33536da231 | ||
|
|
e56ceed32b | ||
|
|
b2c7246134 | ||
|
|
9c39e75ce2 | ||
|
|
0dad76e618 | ||
|
|
90fb6562ce | ||
|
|
174d263890 | ||
|
|
5ec29e7a87 | ||
|
|
3c6b85acf7 | ||
|
|
50e77f66fd | ||
|
|
d484f070d1 | ||
|
|
8890294bf0 | ||
|
|
a1925de0c1 | ||
|
|
24e826d06b | ||
|
|
19625400c1 | ||
|
|
52cbe7b3ee | ||
|
|
6a2c3e91c7 | ||
|
|
4afd8e81be | ||
|
|
485bdc9c4e | ||
|
|
1a3aee39de | ||
|
|
2ddf257c6f | ||
|
|
ef50e0b6d8 | ||
|
|
583a1a2b6f | ||
|
|
68c1207b52 | ||
|
|
f77647ad12 | ||
|
|
c746ea3210 | ||
|
|
d323b34826 | ||
|
|
bf50194f99 | ||
|
|
4a977b2b31 | ||
|
|
b1e3a7b222 | ||
|
|
b12d43698e | ||
|
|
5cd2c22b27 | ||
|
|
c2ef80a287 | ||
|
|
4df3029536 | ||
|
|
e70d667a9f | ||
|
|
8ba434b597 | ||
|
|
171947787c | ||
|
|
106629aa11 | ||
|
|
3ee2e482ce | ||
|
|
849824efd2 | ||
|
|
cf2c3a536d | ||
|
|
7852cecd72 | ||
|
|
6cfffeb031 | ||
|
|
6e954f75ee | ||
|
|
0116a98371 | ||
|
|
a38cbd42aa | ||
|
|
4ab31867c4 | ||
|
|
51a082cea3 | ||
|
|
e3bda5e2d0 | ||
|
|
ad9117b107 | ||
|
|
6b7a1d419c | ||
|
|
c0d0f5f4bc | ||
|
|
095ed63ea0 | ||
|
|
fa5ccac707 | ||
|
|
6809be2d0a | ||
|
|
a5f8869b35 | ||
|
|
1c9c5305a6 | ||
|
|
cf014bf2c1 | ||
|
|
8e7190f795 | ||
|
|
82b6e0e316 | ||
|
|
5b39f5184f | ||
|
|
1da25727b2 | ||
|
|
51d53f45c1 | ||
|
|
f779b2748a | ||
|
|
f2fb191f53 | ||
|
|
0b32bdf1e5 | ||
|
|
2751813454 | ||
|
|
f61858e683 | ||
|
|
87525e00b9 | ||
|
|
803b7718b8 | ||
|
|
8a24a01bff | ||
|
|
9e39a4fbdf | ||
|
|
d0baff184b | ||
|
|
d82a0b28a9 | ||
|
|
bd379e13cb | ||
|
|
6e1f783aec | ||
|
|
edef6a4b15 | ||
|
|
5990679ebd | ||
|
|
b0c8a8c827 | ||
|
|
d837ffe70f | ||
|
|
b1d7dd82fc | ||
|
|
e409567428 | ||
|
|
9538c2171f | ||
|
|
0a718be0d9 | ||
|
|
67845091ba | ||
|
|
fe788b7842 | ||
|
|
ee42eb3ca3 | ||
|
|
80323a4deb | ||
|
|
a35f96f427 | ||
|
|
d354c3d640 | ||
|
|
ce005ce002 | ||
|
|
964245bc2a | ||
|
|
aea3e7c1d2 | ||
|
|
3625942952 | ||
|
|
0593a6b8eb | ||
|
|
426e5c6389 | ||
|
|
ebdfcf4866 | ||
|
|
000d0882c3 | ||
|
|
6062e30cb9 | ||
|
|
3badee1a3c | ||
|
|
4a0256d374 | ||
|
|
52ec62bef0 | ||
|
|
31af9858fd | ||
|
|
3151660fbb | ||
|
|
0362ef48ff | ||
|
|
facd7ff452 | ||
|
|
134cdda333 | ||
|
|
5634ef1bb6 | ||
|
|
2379ab3d51 | ||
|
|
5c908ebba5 | ||
|
|
ba0755d933 | ||
|
|
f7d6b00c1e | ||
|
|
0bb6cf37be | ||
|
|
da57b27277 | ||
|
|
42a3fec594 | ||
|
|
d15034264b | ||
|
|
1ce3c7e580 | ||
|
|
07f27c4eca | ||
|
|
196893cfeb | ||
|
|
f6992059be | ||
|
|
732f949a65 | ||
|
|
cd3b12c579 | ||
|
|
0d6232ffef | ||
|
|
6da20f0efe | ||
|
|
8e25e83e5a | ||
|
|
fe188f8722 | ||
|
|
e4bc8b765b | ||
|
|
d28b6e9ac2 | ||
|
|
ac3cd1b183 | ||
|
|
c601d3b021 | ||
|
|
9bc2165e5c | ||
|
|
374d317412 | ||
|
|
e50c89834e | ||
|
|
f367c202d9 | ||
|
|
9d33d83bb4 | ||
|
|
8f1eff50aa | ||
|
|
e7918e25fd | ||
|
|
1eff84615c | ||
|
|
f1fd6ba3c0 | ||
|
|
39daec9bec | ||
|
|
484f5faf8d | ||
|
|
9a227d186f | ||
|
|
fc27061838 | ||
|
|
d71cc3be77 | ||
|
|
503e680672 | ||
|
|
1f77f4a4ed | ||
|
|
e8fa7daed5 | ||
|
|
936c73b54d | ||
|
|
b9cb4fc36a | ||
|
|
a808a02f05 | ||
|
|
e142a783f7 | ||
|
|
201536f265 | ||
|
|
0d42e76006 | ||
|
|
ca47949475 | ||
|
|
1c8e557eb4 | ||
|
|
11ca603490 | ||
|
|
85d8b07f09 | ||
|
|
d2ee536c16 | ||
|
|
e12e04f482 | ||
|
|
5743537945 | ||
|
|
18e22cd82e | ||
|
|
bdf019a9ac | ||
|
|
5cbafc57c0 | ||
|
|
55674b858b | ||
|
|
bb390f435c | ||
|
|
92b9eebab2 | ||
|
|
074413a96d | ||
|
|
4685ba8d3e | ||
|
|
d2c307ab30 | ||
|
|
a5193634d0 | ||
|
|
95725d121a | ||
|
|
56f2559798 | ||
|
|
38bffc9db1 | ||
|
|
1d3308d904 | ||
|
|
3c9e5adefa | ||
|
|
a065ad4ba7 | ||
|
|
048aec32e7 | ||
|
|
a258b17fac | ||
|
|
07facd2086 | ||
|
|
6bf342a861 | ||
|
|
2cca1ae860 | ||
|
|
becf5bf482 | ||
|
|
afe9e579d6 | ||
|
|
0b82fe60ea | ||
|
|
6fad330efc | ||
|
|
87f21a301d | ||
|
|
0123fed65d | ||
|
|
64ef85159d | ||
|
|
e63779d9e1 | ||
|
|
ac55c05c30 | ||
|
|
8fa672ed71 | ||
|
|
d84d1421ae | ||
|
|
dd2674e9c4 | ||
|
|
ade8175ad8 | ||
|
|
b0db52da6f | ||
|
|
9d673d7fb3 | ||
|
|
9cab8b45b2 | ||
|
|
17aa942541 | ||
|
|
cbc3e63869 | ||
|
|
2ce126e2df | ||
|
|
1faba03cfe | ||
|
|
09b0edd662 | ||
|
|
0c75b7b3c7 | ||
|
|
fd77a721ac | ||
|
|
11aab207f0 | ||
|
|
04c9e01dad | ||
|
|
762291b2a8 | ||
|
|
e0bad10065 | ||
|
|
e6d656dec1 | ||
|
|
d34dd6c14a | ||
|
|
43c8690c14 | ||
|
|
35d681afab | ||
|
|
8ab08f7fc9 | ||
|
|
c5113ee74a | ||
|
|
88988b8c12 | ||
|
|
2389111bf6 | ||
|
|
b92b84f33d | ||
|
|
078522e534 | ||
|
|
7111e71528 | ||
|
|
0ce3214844 | ||
|
|
132ec56fda | ||
|
|
216af25ecc | ||
|
|
6881cffc41 | ||
|
|
aecea54e95 | ||
|
|
c0fa9aff72 | ||
|
|
d5be4cf888 | ||
|
|
7b6b0f2717 | ||
|
|
a03a1dd854 | ||
|
|
8f350df392 | ||
|
|
4cecfe8e79 | ||
|
|
89ca9fc641 | ||
|
|
a2dcf2dff0 | ||
|
|
7477e21b02 | ||
|
|
59e7b32c92 | ||
|
|
4681c734b2 | ||
|
|
84fa689686 | ||
|
|
62d9aa9838 | ||
|
|
02550b36c7 | ||
|
|
35231b408e | ||
|
|
74f5317489 | ||
|
|
81e099a441 | ||
|
|
498436f225 | ||
|
|
b20748f821 | ||
|
|
f0d5b5e073 | ||
|
|
4f6c4ddf1c | ||
|
|
792f84b59d | ||
|
|
b4bd14864a | ||
|
|
6bbf2e34e7 | ||
|
|
d432823515 | ||
|
|
bdf2e84812 | ||
|
|
a438d34fcd | ||
|
|
fa7a1ca39e | ||
|
|
38e4c97aa9 | ||
|
|
569a39c4ba | ||
|
|
bfb00cbb75 | ||
|
|
a067c45cb6 | ||
|
|
0a7baaaed5 | ||
|
|
454ed81556 | ||
|
|
7fe8bf5b24 | ||
|
|
991a0d05b5 | ||
|
|
3288a12528 | ||
|
|
091396742e | ||
|
|
ba85531931 | ||
|
|
0440bd96be | ||
|
|
ace4249711 | ||
|
|
7d007670a0 | ||
|
|
98e3fa9d0d | ||
|
|
5d4ce14dc4 | ||
|
|
8381153418 | ||
|
|
96d84626f8 | ||
|
|
f607ca4c72 | ||
|
|
39416a0d95 | ||
|
|
84a012a0e9 | ||
|
|
16a0996bd4 | ||
|
|
60d5f83ffd | ||
|
|
1455995ac7 | ||
|
|
52c04508a2 | ||
|
|
5a67fcc17e | ||
|
|
73b575468e |
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Preserve prompt cache prefixes when sessions move between locations with unchanged instructions.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Fix OpenCode Console device authorization URLs when the server returns an origin-rooted verification path.
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Correct directory page headings when the read offset is zero.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Propagate fatal Git ignore-check errors during index refresh so best-effort snapshot capture returns unavailable instead of treating the check as having no matches.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Prompt and synthetic inbox ID reuse is now idempotent: reusing an ID within the same Session succeeds and returns the first admission, ignoring the retried payload, metadata, and delivery mode. Previously reuse with a differing payload failed with a conflict. Cross-Session and cross-type reuse still fail, and control items keep their operation-specific conflict behavior.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Rename the write, patch, and question tool formatting helpers from `toModelOutput` to `toModelContent` to match the result field they populate. Direct imports of these helpers must use the new name; generated content and declared machine output are unchanged.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": minor
|
||||
---
|
||||
|
||||
Rename ModelResolver.supported to ModelResolver.hasPackage. Consumers of the old export must update the name; the predicate remains Boolean(model.package), checking only whether a catalog model declares a provider package, not whether it can be loaded. Default-model selection behavior is unchanged.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Nested AGENTS.md instructions are re-injected after compaction. Previously the in-memory dedup claim outlived the synthetic message that compaction dropped from model-visible history, so nested instructions were silently lost for the rest of the process lifetime. The claim now only guards in-flight loads; the synthetic message metadata in durable history is the sole lasting ledger, so any history truncation (compaction, revert) self-heals on the next read in that subtree.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/plugin": patch
|
||||
---
|
||||
|
||||
Derive Promise plugin API request and response conversion from the canonical protocol schemas.
|
||||
@@ -1,8 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": minor
|
||||
"@opencode-ai/schema": minor
|
||||
"@opencode-ai/protocol": minor
|
||||
"@opencode-ai/client": minor
|
||||
---
|
||||
|
||||
Remove the unused question request API and use session forms for question tool interactions.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Include both paths of renamed files in new snapshot change lists so undo restores the original file instead of only deleting the renamed file.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Apply shared Session model-request preparation to transient generation.
|
||||
@@ -1,9 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": minor
|
||||
"@opencode-ai/schema": minor
|
||||
"@opencode-ai/protocol": minor
|
||||
"@opencode-ai/client": minor
|
||||
---
|
||||
|
||||
Add an opt-in portable shell permission scanner. Opaque commands use normal shell authorization without inferring
|
||||
external directories, while the default tree-sitter path remains unchanged.
|
||||
@@ -1,10 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Make the experimental portable shell scanner authoritative, with no Tree-sitter
|
||||
fallback. Scan common Bash and PowerShell control flow, heredocs, functions,
|
||||
expressions, quoting, and substitutions natively. Preserve existing redirect and
|
||||
declaration permission matching, and make PowerShell saved approvals cover the
|
||||
original command spelling. Parser failures remain visible without changing the
|
||||
permission engine. The default Tree-sitter path is unchanged.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Use parent directory names for SKILL.md IDs so individually downloaded skills do not overwrite each other.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Simplify interrupt continuation: the steer-scoped resume decision now lives in SessionExecution as a post-cleanup inbox check, and the run coordinator drops its continuation state machine. Wakes arriving during cancellation cleanup now restart a normal full drain, and interrupting an idle session with continue now resumes pending steering input. Recovery-applied moves now end with the same full wake as inbox-admitted moves, retrying any stranded inbox work at the new location. Interrupting with continue now also resumes a next-in-line control item: between-turn manual compaction and moves run under any drain scope, while queued prompts remain parked.
|
||||
@@ -1,6 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/client": patch
|
||||
"@opencode-ai/plugin": patch
|
||||
---
|
||||
|
||||
Add form reply and cancellation operations that reconcile terminal forms in the local TUI projection.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Title generation and compaction summaries now build their model requests through the shared session request boundary, gaining unsupported-media filtering and image bounds while explicitly opting out of session context hooks: plugins that shape the agent conversation do not observe title or compaction requests. Title requests gain the fork-aware session prompt cache key, and compaction summaries in forked sessions reuse the fork root's prompt cache key instead of the fork's own.
|
||||
@@ -1,6 +0,0 @@
|
||||
---
|
||||
"@opencode-ai/plugin": patch
|
||||
"@opencode-ai/core": patch
|
||||
---
|
||||
|
||||
Add transport-neutral Session model request hooks and provider-scoped hook registration so eligible OpenAI Responses requests can prefer WebSocket without bypassing HTTP-only middleware.
|
||||
@@ -2,3 +2,4 @@ packages/core/migration/**/snapshot.json linguist-generated
|
||||
packages/core/src/database/migration.gen.ts linguist-generated
|
||||
packages/core/src/models-dev/snapshot.txt linguist-generated
|
||||
packages/core/src/**/*.txt text eol=lf
|
||||
packages/httpapi-codegen/test/generated/*.ts text eol=lf
|
||||
|
||||
@@ -5,6 +5,7 @@ on:
|
||||
branches:
|
||||
- dev
|
||||
- production
|
||||
- beta
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -15,7 +16,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: github.repository == 'anomalyco/opencode' && (github.ref_name == 'dev' || github.ref_name == 'production')
|
||||
if: github.repository == 'anomalyco/opencode' && (github.ref_name == 'dev' || github.ref_name == 'production' || github.ref_name == 'beta')
|
||||
runs-on: ubuntu-latest
|
||||
environment: ${{ github.ref_name }}
|
||||
steps:
|
||||
@@ -28,6 +29,7 @@ jobs:
|
||||
node-version: "24"
|
||||
|
||||
- uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
|
||||
if: github.ref_name != 'beta'
|
||||
with:
|
||||
role-to-assume: ${{ vars.AWS_DEPLOY_ROLE_ARN }}
|
||||
role-session-name: opencode-${{ github.run_id }}
|
||||
|
||||
@@ -41,11 +41,5 @@
|
||||
// Warn when spreading non-plain objects (Headers, class instances, etc.)
|
||||
"typescript/no-misused-spread": "warn"
|
||||
},
|
||||
"options": {
|
||||
"typeAware": true
|
||||
},
|
||||
"options": {
|
||||
"typeAware": true
|
||||
},
|
||||
"ignorePatterns": ["**/node_modules", "**/dist", "**/.build", "**/.sst", "**/*.d.ts", "**/sdk.gen.ts"]
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
- After changing the public Protocol or Server `HttpApi`, run `bun run generate` from `packages/client`. Do not edit generated client files directly.
|
||||
- Keep runtime dependencies directed from Schema to Core and Protocol, then from Core and Protocol to Server. Client runtime code may depend on Schema and Protocol but never Core or Server; `sdk` composes Client, Core, and Server.
|
||||
- Current implementation changes belong in `packages/core`, `packages/cli`, `packages/server`, `packages/protocol`, `packages/schema`, and related generated client surfaces when required.
|
||||
- This repository does not use Changesets. Do not add `.changeset` files; follow the existing release workflow instead.
|
||||
- The default branch in this repo is `v2`.
|
||||
- Base all new branches and worktrees on `v2`, or `origin/v2` when the local `v2` ref is unavailable. Do not base them on `dev`.
|
||||
- Local `main` ref may not exist; use `v2` or `origin/v2` for diffs.
|
||||
@@ -45,6 +46,7 @@ Examples: `fix(tui): simplify thinking toggle styling`, `docs: update contributi
|
||||
### General Principles
|
||||
|
||||
- Keep things in one function unless composable or reusable
|
||||
- Validate unknown values once at the boundary that owns them. Pass typed values inward instead of repeating `typeof value === "object"` and property-existence checks. Do not defensively revalidate values already guaranteed by a schema, constructor, or internal type.
|
||||
- Do not extract single-use helpers preemptively. Inline the logic at the call site unless the helper is reused, hides a genuinely complex boundary, or has a clear independent name that improves the caller.
|
||||
- Before adding complexity for a speculative or vanishingly unlikely race or security edge case, explain the concrete failure mode, likelihood, and complexity cost to the user and get their buy-in. Do not silently expand scope for theoretical robustness.
|
||||
- Avoid `try`/`catch` where possible
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
exact = true
|
||||
# Only install newly resolved package versions published at least 3 days ago.
|
||||
minimumReleaseAge = 259200
|
||||
minimumReleaseAgeExcludes = ["@ai-sdk/amazon-bedrock", "@ai-sdk/anthropic", "@brendonovich/vite-plugin-opencode", "@opencode-ai/sdk", "@opencode-ai/pty", "@opencode-ai/pty-darwin-arm64", "@opencode-ai/pty-darwin-x64", "@opencode-ai/pty-linux-arm64-gnu", "@opencode-ai/pty-linux-arm64-musl", "@opencode-ai/pty-linux-x64-gnu", "@opencode-ai/pty-linux-x64-musl", "@opentui/core", "@opentui/core-darwin-arm64", "@opentui/core-darwin-x64", "@opentui/core-linux-arm64", "@opentui/core-linux-arm64-musl", "@opentui/core-linux-x64", "@opentui/core-linux-x64-musl", "@opentui/core-win32-arm64", "@opentui/core-win32-x64", "@opentui/keymap", "@opentui/solid", "opentui-spinner", "gitlab-ai-provider", "opencode-gitlab-auth", "@ff-labs/fff-node", "@ff-labs/fff-bun", "@ff-labs/fff-bin-darwin-arm64", "@ff-labs/fff-bin-darwin-x64", "@ff-labs/fff-bin-linux-arm64-gnu", "@ff-labs/fff-bin-linux-arm64-musl", "@ff-labs/fff-bin-linux-x64-gnu", "@ff-labs/fff-bin-linux-x64-musl", "@ff-labs/fff-bin-win32-arm64", "@ff-labs/fff-bin-win32-x64", "@pierre/diffs", "@pierre/theming", "app-builder-lib", "dmg-builder", "electron-builder", "electron-publish", "blume"]
|
||||
minimumReleaseAgeExcludes = ["@ai-sdk/amazon-bedrock", "@ai-sdk/anthropic", "@brendonovich/vite-plugin-opencode", "@opencode-ai/sdk", "@opencode-ai/pty", "@opencode-ai/pty-darwin-arm64", "@opencode-ai/pty-darwin-x64", "@opencode-ai/pty-linux-arm64-gnu", "@opencode-ai/pty-linux-arm64-musl", "@opencode-ai/pty-linux-x64-gnu", "@opencode-ai/pty-linux-x64-musl", "@opentui/core", "@opentui/core-darwin-arm64", "@opentui/core-darwin-x64", "@opentui/core-linux-arm64", "@opentui/core-linux-arm64-musl", "@opentui/core-linux-x64", "@opentui/core-linux-x64-musl", "@opentui/core-win32-arm64", "@opentui/core-win32-x64", "@opentui/keymap", "@opentui/solid", "opentui-spinner", "gitlab-ai-provider", "opencode-gitlab-auth", "@ff-labs/fff-node", "@ff-labs/fff-bun", "@ff-labs/fff-bin-darwin-arm64", "@ff-labs/fff-bin-darwin-x64", "@ff-labs/fff-bin-linux-arm64-gnu", "@ff-labs/fff-bin-linux-arm64-musl", "@ff-labs/fff-bin-linux-x64-gnu", "@ff-labs/fff-bin-linux-x64-musl", "@ff-labs/fff-bin-win32-arm64", "@ff-labs/fff-bin-win32-x64", "@pierre/diffs", "@pierre/theming", "app-builder-lib", "dmg-builder", "electron", "electron-builder", "electron-publish", "blume", "mermaid"]
|
||||
|
||||
[test]
|
||||
root = "./do-not-run-tests-from-root"
|
||||
|
||||
+2
-8
@@ -1,4 +1,5 @@
|
||||
import { domain } from "./stage"
|
||||
import { createWebApp } from "./webapp"
|
||||
|
||||
const GITHUB_APP_ID = new sst.Secret("GITHUB_APP_ID")
|
||||
const GITHUB_APP_PRIVATE_KEY = new sst.Secret("GITHUB_APP_PRIVATE_KEY")
|
||||
@@ -59,11 +60,4 @@ new sst.cloudflare.x.Astro("Web", {
|
||||
},
|
||||
})
|
||||
|
||||
new sst.cloudflare.StaticSite("WebApp", {
|
||||
domain: "app." + domain,
|
||||
path: "packages/app",
|
||||
build: {
|
||||
command: "bun turbo build",
|
||||
output: "./dist",
|
||||
},
|
||||
})
|
||||
createWebApp("app." + domain)
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
export function createWebApp(domain: string) {
|
||||
return new sst.cloudflare.StaticSite("WebApp", {
|
||||
domain,
|
||||
path: "packages/app",
|
||||
environment:
|
||||
$app.stage === "beta"
|
||||
? {
|
||||
OPENCODE_CHANNEL: "beta",
|
||||
VITE_SENTRY_ENVIRONMENT: "beta",
|
||||
}
|
||||
: undefined,
|
||||
build: {
|
||||
// Preserve Sentry credentials and run source-map uploads on every deployment.
|
||||
command: "bun run build",
|
||||
output: "./dist",
|
||||
},
|
||||
})
|
||||
}
|
||||
+7
-9
@@ -4,7 +4,7 @@
|
||||
bun,
|
||||
nodejs,
|
||||
darwin,
|
||||
electron_41,
|
||||
callPackage,
|
||||
makeWrapper,
|
||||
writableTmpDirAsHomeHook,
|
||||
autoPatchelfHook,
|
||||
@@ -13,7 +13,7 @@
|
||||
opencode,
|
||||
}:
|
||||
let
|
||||
electron = electron_41;
|
||||
electron = callPackage ./electron.nix { };
|
||||
in
|
||||
stdenv.mkDerivation (finalAttrs: {
|
||||
pname = "opencode-desktop";
|
||||
@@ -48,7 +48,7 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
desktopName = "OpenCode";
|
||||
exec = "opencode-desktop %U";
|
||||
icon = "ai.opencode.desktop";
|
||||
# Electron 41 derives X11 WM_CLASS from app.name.
|
||||
# Electron derives X11 WM_CLASS from app.name.
|
||||
startupWMClass = "OpenCode";
|
||||
categories = [ "Development" ];
|
||||
});
|
||||
@@ -67,12 +67,10 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
# https://github.com/electron/electron/issues/31121
|
||||
# mac builds use a .app bundle which doesnt have this issue
|
||||
+ lib.optionalString stdenv.isLinux ''
|
||||
BASE_PATH=packages/desktop
|
||||
FILES=(src/main/windows.ts)
|
||||
for file in "''${FILES[@]}"; do
|
||||
substituteInPlace $BASE_PATH/$file \
|
||||
--replace-fail "process.resourcesPath" "'$out/opt/opencode-desktop/resources'"
|
||||
done
|
||||
substituteInPlace \
|
||||
packages/desktop/src/main/windows/appearance.ts \
|
||||
packages/desktop/src/main/service/desktop-cli.ts \
|
||||
--replace-fail "process.resourcesPath" "'$out/opt/opencode-desktop/resources'"
|
||||
'';
|
||||
|
||||
preBuild = ''
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
{ callPackage, path }:
|
||||
let
|
||||
version = (builtins.fromJSON (builtins.readFile ../packages/desktop/package.json)).devDependencies.electron;
|
||||
in
|
||||
(callPackage (path + "/pkgs/development/tools/electron/binary/generic.nix") { }) version {
|
||||
# Electron 42.10.1 SHASUMS256.txt; update with the desktop package version.
|
||||
aarch64-linux = "20e68d6c4e47f3ebf59de7c6b1f8b8bec6a6ebda6a451132f9b465f3f13ce467";
|
||||
x86_64-linux = "2452b27112d92387471fa2488aafac85d79ea3f2ee1216c0abd5150d6c12362b";
|
||||
aarch64-darwin = "ac7194a3dfd81930ba35355c01620262c1254752859b42dcb8f4b9e4d174a871";
|
||||
x86_64-darwin = "4489aba55477a0082266cb690db1c829503ba3338048599d8fd243953df37dab";
|
||||
# fetchzip hashes the unpacked headers, not the release tarball.
|
||||
headers = "sha256-4eUy3BZVvxTl7KUOsxio7769lL6ag/ecbeK+qLURWMI=";
|
||||
}
|
||||
+4
-4
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"nodeModules": {
|
||||
"x86_64-linux": "sha256-agxF1Ab8P84tijHvw7jaG+9UZm/dJZhHw8CGNlQQqYc=",
|
||||
"aarch64-linux": "sha256-Kg6CkhoEEsb+ZOGPN/rnNoirLdGSyQl6HU/Fy8t9Mqs=",
|
||||
"aarch64-darwin": "sha256-SnxBpmcfPL2p3OYmSuG5Y/aFW8PeEOLhfQrxAs/53fs=",
|
||||
"x86_64-darwin": "sha256-Dnxl1EcfiE7/rlHmPkko0BKc6hHVk6rQ/2h/PWuLfNo="
|
||||
"x86_64-linux": "sha256-JStMvgtXBA5GrhyBJ5FtdqD8LWkcaPA9NXef+c2xUzw=",
|
||||
"aarch64-linux": "sha256-WQxF+yS0ZImW0KW620XnZUBsKAJDoJ1jLDOMBaXI2/E=",
|
||||
"aarch64-darwin": "sha256-km7G6s45dfFdW3Z6lFrs4NohD+vmwN1vR8PmEL0WCto=",
|
||||
"x86_64-darwin": "sha256-YFbkcHpuspgTp+B+th3IJ3cnu5C2gEUSiy3NDXpD8UA="
|
||||
}
|
||||
}
|
||||
|
||||
+27
-23
@@ -39,24 +39,24 @@
|
||||
"packages/stats/*"
|
||||
],
|
||||
"catalog": {
|
||||
"@effect/opentelemetry": "4.0.0-rc.111",
|
||||
"@effect/platform-node": "4.0.0-rc.111",
|
||||
"@effect/platform-node-shared": "4.0.0-rc.111",
|
||||
"@effect/sql-sqlite-bun": "4.0.0-rc.111",
|
||||
"@effect/opentelemetry": "4.0.0-rc.112",
|
||||
"@effect/platform-node": "4.0.0-rc.112",
|
||||
"@effect/platform-node-shared": "4.0.0-rc.112",
|
||||
"@effect/sql-sqlite-bun": "4.0.0-rc.112",
|
||||
"@npmcli/arborist": "9.4.0",
|
||||
"@types/bun": "1.3.13",
|
||||
"@types/cross-spawn": "6.0.6",
|
||||
"@octokit/rest": "22.0.0",
|
||||
"@hono/standard-validator": "0.2.0",
|
||||
"@hono/zod-validator": "0.4.2",
|
||||
"@opentui/core": "0.5.8",
|
||||
"@opentui/keymap": "0.5.8",
|
||||
"@opentui/solid": "0.5.8",
|
||||
"@opentui/core": "0.5.9",
|
||||
"@opentui/keymap": "0.5.9",
|
||||
"@opentui/solid": "0.5.9",
|
||||
"@tanstack/solid-virtual": "3.13.37",
|
||||
"@shikijs/stream": "4.2.0",
|
||||
"@shikijs/stream": "4.4.3",
|
||||
"@standard-schema/spec": "1.1.0",
|
||||
"ulid": "3.0.1",
|
||||
"@kobalte/core": "0.13.11",
|
||||
"@kobalte/core": "0.13.13",
|
||||
"@corvu/drawer": "0.2.4",
|
||||
"@types/luxon": "3.7.1",
|
||||
"@types/node": "24.12.2",
|
||||
@@ -67,13 +67,16 @@
|
||||
"@openauthjs/openauth": "0.0.0-20250322224806",
|
||||
"@pierre/diffs": "1.2.10",
|
||||
"opentui-spinner": "0.0.7",
|
||||
"@solid-primitives/storage": "4.3.3",
|
||||
"@solid-primitives/event-listener": "2.4.6",
|
||||
"@solid-primitives/media": "2.3.6",
|
||||
"@solid-primitives/resize-observer": "2.2.0",
|
||||
"@solid-primitives/storage": "4.4.0",
|
||||
"@tailwindcss/vite": "4.1.11",
|
||||
"diff": "8.0.2",
|
||||
"dompurify": "3.3.1",
|
||||
"diff": "8.0.4",
|
||||
"dompurify": "3.4.14",
|
||||
"drizzle-kit": "1.0.0-rc.5-ab785fc",
|
||||
"drizzle-orm": "1.0.0-rc.5-169397b",
|
||||
"effect": "4.0.0-rc.111",
|
||||
"effect": "4.0.0-rc.112",
|
||||
"ai": "6.0.168",
|
||||
"cross-spawn": "7.0.6",
|
||||
"hono": "4.10.7",
|
||||
@@ -81,9 +84,9 @@
|
||||
"fuzzysort": "3.1.0",
|
||||
"get-east-asian-width": "1.6.0",
|
||||
"luxon": "3.6.1",
|
||||
"marked": "18.0.7",
|
||||
"marked": "18.0.11",
|
||||
"marked-shiki": "1.2.1",
|
||||
"remend": "1.3.0",
|
||||
"remend": "1.3.1",
|
||||
"@playwright/test": "1.59.1",
|
||||
"semver": "7.7.4",
|
||||
"typescript": "5.8.2",
|
||||
@@ -92,17 +95,17 @@
|
||||
"remeda": "2.26.0",
|
||||
"resolve.exports": "2.0.3",
|
||||
"sst": "4.13.1",
|
||||
"shiki": "4.2.0",
|
||||
"shiki": "4.4.3",
|
||||
"solid-list": "0.3.0",
|
||||
"string-width": "7.2.0",
|
||||
"tailwindcss": "4.1.11",
|
||||
"vite": "7.1.4",
|
||||
"vite": "7.3.6",
|
||||
"@solidjs/meta": "0.29.4",
|
||||
"@solidjs/router": "0.15.4",
|
||||
"@solidjs/router": "1.0.0",
|
||||
"@solidjs/start": "https://pkg.pr.new/@solidjs/start@dfb2020",
|
||||
"@sentry/solid": "10.36.0",
|
||||
"@sentry/vite-plugin": "4.6.0",
|
||||
"solid-js": "1.9.10",
|
||||
"@sentry/solid": "10.71.0",
|
||||
"@sentry/vite-plugin": "5.4.0",
|
||||
"solid-js": "1.9.15",
|
||||
"solid-sonner": "0.3.1",
|
||||
"vite-plugin-solid": "2.11.10",
|
||||
"@lydell/node-pty": "1.2.0-beta.12"
|
||||
@@ -158,7 +161,8 @@
|
||||
"@effect/platform-node-shared": "catalog:",
|
||||
"@types/bun": "catalog:",
|
||||
"@types/node": "catalog:",
|
||||
"effect": "catalog:"
|
||||
"effect": "catalog:",
|
||||
"solid-js": "catalog:"
|
||||
},
|
||||
"patchedDependencies": {
|
||||
"@ai-sdk/openai-compatible@2.0.41": "patches/@ai-sdk%2Fopenai-compatible@2.0.41.patch",
|
||||
@@ -166,7 +170,7 @@
|
||||
"@npmcli/agent@4.0.2": "patches/@npmcli%2Fagent@4.0.2.patch",
|
||||
"@silvia-odwyer/photon-node@0.3.4": "patches/@silvia-odwyer%2Fphoton-node@0.3.4.patch",
|
||||
"@standard-community/standard-openapi@0.2.9": "patches/@standard-community%2Fstandard-openapi@0.2.9.patch",
|
||||
"solid-js@1.9.10": "patches/solid-js@1.9.10.patch",
|
||||
"solid-js@1.9.15": "patches/solid-js@1.9.15.patch",
|
||||
"@ai-sdk/mistral@3.0.51": "patches/@ai-sdk%2Fmistral@3.0.51.patch",
|
||||
"gcp-metadata@8.1.2": "patches/gcp-metadata@8.1.2.patch",
|
||||
"pacote@21.5.0": "patches/pacote@21.5.0.patch",
|
||||
|
||||
+136
-11
@@ -214,22 +214,147 @@ the requests sent by code under test:
|
||||
import { Effect } from "effect"
|
||||
import { TestLLM } from "@opencode-ai/ai/testing"
|
||||
|
||||
const testLLM = TestLLM.layer({
|
||||
fallback: TestLLM.text("Hello from the test model", "text-1"),
|
||||
})
|
||||
|
||||
// TestLLM.clientLayer provides LLMClient.Service and consumes TestLLM.Service.
|
||||
const programWithTestClient = Effect.gen(function* () {
|
||||
const test = yield* TestLLM.Test
|
||||
yield* test.push(TestLLM.text("Hello from the test model", "text-1"))
|
||||
const result = yield* program
|
||||
const test = yield* TestLLM.Service
|
||||
console.log(test.requests)
|
||||
console.log(yield* test.requests())
|
||||
return result
|
||||
}).pipe(Effect.provide(TestLLM.clientLayer), Effect.provide(testLLM))
|
||||
}).pipe(Effect.provide(TestLLM.testLayer()))
|
||||
```
|
||||
|
||||
`TestLLM.push(...)` scripts one-shot responses, `TestLLM.always(...)` changes the fallback, and
|
||||
`TestLLM.wait(...)` lets concurrent tests wait until a request has arrived. Every received canonical request is
|
||||
available on the yielded `TestLLM.Service`.
|
||||
`testLayer()` provides the same object under `LLMClient.Service` and `TestLLM.Test`. Production consumes the
|
||||
normal client; tests use the additional controls. Each layer build has fresh state.
|
||||
|
||||
- `test.push(...)` queues one-shot responses in execution order. Each argument is one response.
|
||||
- `test.always(response)` installs a repeatable fallback. The layer's `fallback` option sets its initial value.
|
||||
- `test.serve(request => response)` installs a request-dependent fallback. `always` and `serve` replace each
|
||||
other without changing queued replies; queued replies take precedence.
|
||||
- `test.requests()` returns an array snapshot. `transformRequest` changes only the recorded observation;
|
||||
`serve` receives the original canonical request.
|
||||
- `test.wait(count)` waits for request arrivals, not output or completion, and supports concurrent waiters.
|
||||
- `test.gate()` returns a scoped gate with countable `started` notifications and a `release` Effect. Release
|
||||
unblocks all requests captured by that gate; closing its scope also releases it. Effect-aware test runners
|
||||
already provide Scope.
|
||||
|
||||
Constructing `stream()` or `generate()` does not record a request, invoke a responder, or consume a script.
|
||||
Each execution does. An exhausted queue without a fallback defects immediately rather than waiting for a
|
||||
future reply.
|
||||
|
||||
Generation responses remain canonical event arrays or arbitrary `Stream<LLMEvent, AIError>` values. The client consumes
|
||||
supplied streams directly, preserving failure identity, finalizers, incomplete output, and post-finish tails;
|
||||
it does not repair or truncate them.
|
||||
|
||||
For explicit compaction, script a `CompactionResponse` through `push`, `always`, or `serve`. The client returns that replacement window directly, including retained user messages and usage, with the same lazy request recording and gates. Generation and compaction reject fixtures for the wrong operation instead of converting between response shapes.
|
||||
|
||||
The published legacy `Service`, `layer`, `clientLayer`, and module-level controls remain available as adapters
|
||||
over the same implementation, including the legacy live `requests` array. New tests should use `Test` and
|
||||
`testLayer`.
|
||||
|
||||
## Provider compaction
|
||||
|
||||
Compaction is opt-in. The package supports automatic compaction in OpenAI/Azure Responses and Anthropic Messages (including Claude on Vertex and Bedrock Messages), and explicit compaction calls in OpenAI/Azure/xAI Responses. Model and deployment support still depends on the provider.
|
||||
|
||||
This is different from prompt caching, server-side history storage, or truncation. Compaction returns provider-owned context that must be replayed to continue the conversation.
|
||||
|
||||
### Automatic compaction
|
||||
|
||||
Inside an `Effect.gen`, enable OpenAI compaction with typed provider options:
|
||||
|
||||
```ts
|
||||
import { LLM, LLMClient, LLMRequest, Message } from "@opencode-ai/ai"
|
||||
import { OpenAI } from "@opencode-ai/ai/providers"
|
||||
|
||||
const request = LLM.request({
|
||||
model: OpenAI.configure({ apiKey }).responses("gpt-5.3-codex"),
|
||||
messages,
|
||||
providerOptions: {
|
||||
contextManagement: [{ type: "compaction", compactThreshold: 200_000 }],
|
||||
},
|
||||
})
|
||||
const response = yield * LLMClient.generate(request)
|
||||
const next = LLMRequest.update(request, {
|
||||
messages: [...request.messages, response.message, Message.user("Continue")],
|
||||
})
|
||||
```
|
||||
|
||||
`store: false` remains the default. Keep the entire `response.message`, not just `response.text`. Compaction events become ordered `CompactionPart`s alongside text and reasoning. The conversation contains everything needed to continue; there is no separate replay object or hidden provider transcript.
|
||||
|
||||
A compaction part has `provider` and exactly one representation: `encrypted` for Responses, or `text` for Anthropic. Responses also preserves the optional checkpoint `id`. These fields survive message serialization without becoming visible assistant text. Sending a checkpoint to another provider or an incompatible API fails rather than silently losing context.
|
||||
|
||||
```ts
|
||||
import { CompactionPart, ProviderID } from "@opencode-ai/ai"
|
||||
|
||||
CompactionPart.make({ provider: ProviderID.make("openai"), id: "cmp_123", encrypted: "..." })
|
||||
CompactionPart.make({ provider: ProviderID.make("anthropic"), text: "Summary of the conversation..." })
|
||||
```
|
||||
|
||||
For Anthropic, use:
|
||||
|
||||
```ts
|
||||
providerOptions: {
|
||||
contextManagement: {
|
||||
edits: [{
|
||||
type: "compact_20260112",
|
||||
trigger: { type: "input_tokens", value: 150_000 },
|
||||
pauseAfterCompaction: true,
|
||||
instructions: "Summarize the task and decisions. Do not call tools while summarizing.",
|
||||
}],
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
- The trigger is optional (provider default: 150,000 tokens), with a minimum of 50,000.
|
||||
- Custom instructions replace Anthropic's default summarization instructions.
|
||||
- The route adds `compact-2026-01-12` to existing beta headers, including when replaying a checkpoint without enabling new compactions.
|
||||
- A pause is exposed as `response.finishReason.raw === "compaction"`. The caller explicitly issues the next request; the package never automatically resumes.
|
||||
- Anthropic can return a compaction block with `content: null` when summarization fails. This becomes a compaction part with `text: null`, which is **not** a successful replacement for prior history. The package never prunes history automatically.
|
||||
- `Usage` totals include all reported Anthropic `usage.iterations`, including compaction. `contextTokens` separately reports the final message iteration's inclusive input size, when available. A compaction-only pause does not report a post-compaction context size. Raw iteration usage remains in `providerMetadata`.
|
||||
|
||||
Bedrock's Converse API does not support this feature. Select the native Claude Messages route explicitly; the default `.model(...)` remains Converse:
|
||||
|
||||
```ts
|
||||
import { AmazonBedrock } from "@opencode-ai/ai/providers"
|
||||
|
||||
const model = AmazonBedrock.configure({ region: "us-east-1", credentials }).messages("us.anthropic.claude-opus-4-6-v1")
|
||||
```
|
||||
|
||||
The corresponding package entrypoint is `@opencode-ai/ai/providers/amazon-bedrock/messages`. It uses InvokeModelWithResponseStream, AWS event-stream framing, bearer or SigV4 auth, and `anthropic_beta` in the request body.
|
||||
|
||||
### Explicit compaction
|
||||
|
||||
`LLMClient.compact(request)` performs exactly one HTTP call to `/responses/compact`, using the selected route's endpoint, credentials, query, and HTTP middleware. It returns a `CompactionResponse` containing replacement `messages` and usage, not a normal generation response.
|
||||
|
||||
The selected model carries explicit-compaction capability through request construction and updates. Calls using unsupported routes fail type checking. When the model is selected dynamically, narrow the request with `LLMClient.canCompact(request)` before calling `LLMClient.compact`; a model or route switch does not inherit the old capability. Runtime validation still rejects unsupported calls from untyped consumers. Capability describes the route's API, not whether every model or custom deployment supports the operation.
|
||||
|
||||
```ts
|
||||
const compacted = yield * LLMClient.compact(request)
|
||||
const next = LLMRequest.update(request, {
|
||||
messages: [...compacted.messages, Message.user("Continue")],
|
||||
})
|
||||
const response = yield * LLMClient.generate(next)
|
||||
```
|
||||
|
||||
Replace the prior window with `compacted.messages`. Do not append it to the original transcript or extract only the encrypted item: the provider may retain additional messages in its output. Retained user and assistant messages remain ordinary messages with typed text, media, or reasoning parts, in their original order. Provider-specific message IDs, status, and phase use `providerMetadata`, not a raw output array hidden in an assistant message. Unsupported returned item types fail explicitly. Generation-only body overlays such as `stream` and `store` are not sent to the compact endpoint.
|
||||
|
||||
Supported compact controls such as service tier and prompt-cache settings preserve request defaults and HTTP-overlay precedence. Retained image and file detail settings survive serialization and replay.
|
||||
|
||||
The input must still fit the model's context window. Explicit compaction is not an overflow-recovery operation. xAI supports this explicit path, not the automatic OpenAI option. Unsupported routes, including Bedrock Mantle, do not inherit an explicit compact endpoint simply because they use a Responses protocol.
|
||||
|
||||
### Ownership and verification
|
||||
|
||||
The AI package transports options and typed conversation parts. It does not schedule compaction, persist Session checkpoints, select history, switch providers, or replace Core's existing local compaction policy. Native compaction is not enabled for OpenCode Sessions by this feature; Session integration must persist these parts before enabling it. The AI SDK bridge rejects native compaction parts rather than dropping them. Provider-executed tool APIs and persistence changes are a separate follow-up.
|
||||
|
||||
Tests cover serialized round trips, real local HTTP plus a tool loop, AWS binary frames and signing, provider errors, malformed blocks, and usage accounting. Live provider tests are gated by `RECORD=true` and the relevant API keys:
|
||||
|
||||
```sh
|
||||
# Run from packages/ai. Only records the selected new cassette group.
|
||||
RECORD=true RECORDED_PREFIX=openai-compaction bun test test/provider/compaction.recorded.test.ts
|
||||
RECORD=true RECORDED_PREFIX=xai-compaction bun test test/provider/compaction.recorded.test.ts
|
||||
RECORD=true RECORDED_PREFIX=anthropic-compaction bun test test/provider/compaction.recorded.test.ts
|
||||
```
|
||||
|
||||
Provider references: [OpenAI](https://developers.openai.com/api/docs/guides/compaction), [Azure](https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/responses#server-side-compaction), [Anthropic](https://platform.claude.com/docs/en/build-with-claude/compaction), [Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/claude-messages-compaction.html), [xAI](https://docs.x.ai/developers/advanced-api-usage/context-compaction).
|
||||
|
||||
## Caching
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ const RESPECTS_INLINE_HINTS = new Set([
|
||||
"anthropic-messages",
|
||||
"google-vertex-messages",
|
||||
"bedrock-converse",
|
||||
"bedrock-messages",
|
||||
"openrouter",
|
||||
])
|
||||
|
||||
|
||||
@@ -6,8 +6,12 @@ import { Auth } from "../route/auth.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { Headers } from "effect/unstable/http"
|
||||
import { HttpTransport } from "../route/transport/index.js"
|
||||
import {
|
||||
AIError,
|
||||
HttpOptions,
|
||||
LLMRequest,
|
||||
LLMEvent,
|
||||
mergeJsonRecords,
|
||||
Usage,
|
||||
@@ -15,7 +19,6 @@ import {
|
||||
type FinishReasonDetails,
|
||||
type FinishReason,
|
||||
type JsonSchema,
|
||||
type LLMRequest,
|
||||
type MediaPart,
|
||||
type ProviderMetadata,
|
||||
type ToolCallPart,
|
||||
@@ -61,6 +64,7 @@ export type ThinkingInput =
|
||||
))
|
||||
|
||||
export interface OptionsInput {
|
||||
readonly contextManagement?: ContextManagement
|
||||
readonly [key: string]: unknown
|
||||
readonly thinking?: ThinkingInput
|
||||
readonly effort?: string
|
||||
@@ -89,6 +93,23 @@ export interface OptionsInput {
|
||||
|
||||
export type ProviderOptionsInput = OptionsInput
|
||||
|
||||
export const ContextManagement = Schema.Struct({
|
||||
edits: Schema.Array(
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("compact_20260112"),
|
||||
trigger: Schema.optional(
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("input_tokens"),
|
||||
value: Schema.Int.check(Schema.isGreaterThanOrEqualTo(50000)),
|
||||
}),
|
||||
),
|
||||
pauseAfterCompaction: Schema.optional(Schema.Boolean),
|
||||
instructions: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
})
|
||||
export type ContextManagement = typeof ContextManagement.Type
|
||||
|
||||
// =============================================================================
|
||||
// Request Body Schema
|
||||
// =============================================================================
|
||||
@@ -236,7 +257,12 @@ const AnthropicUserBlock = Schema.Union([
|
||||
AnthropicToolResultBlock,
|
||||
])
|
||||
type AnthropicUserBlock = Schema.Schema.Type<typeof AnthropicUserBlock>
|
||||
const AnthropicCompactionBlock = Schema.Struct({
|
||||
type: Schema.Literal("compaction"),
|
||||
content: Schema.NullOr(Schema.String),
|
||||
})
|
||||
const AnthropicAssistantBlock = Schema.Union([
|
||||
AnthropicCompactionBlock,
|
||||
AnthropicTextBlock,
|
||||
AnthropicThinkingBlock,
|
||||
AnthropicRedactedThinkingBlock,
|
||||
@@ -312,6 +338,18 @@ const AnthropicContainer = Schema.Union([
|
||||
])
|
||||
|
||||
const AnthropicBodyFields = {
|
||||
context_management: Schema.optional(
|
||||
Schema.Struct({
|
||||
edits: Schema.Array(
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("compact_20260112"),
|
||||
trigger: ContextManagement.fields.edits.value.fields.trigger,
|
||||
pause_after_compaction: Schema.optional(Schema.Boolean),
|
||||
instructions: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
),
|
||||
model: Schema.String,
|
||||
system: optionalArray(AnthropicTextBlock),
|
||||
messages: Schema.Array(AnthropicMessage),
|
||||
@@ -335,7 +373,7 @@ const AnthropicBodyFields = {
|
||||
export const AnthropicMessagesBody = Schema.Struct(AnthropicBodyFields)
|
||||
export type AnthropicMessagesBody = Schema.Schema.Type<typeof AnthropicMessagesBody>
|
||||
|
||||
const AnthropicUsage = Schema.StructWithRest(
|
||||
const AnthropicIterationUsage = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
input_tokens: optionalNull(Schema.Number),
|
||||
output_tokens: Schema.optional(Schema.Number),
|
||||
@@ -354,6 +392,13 @@ const AnthropicUsage = Schema.StructWithRest(
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
)
|
||||
const AnthropicUsage = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
...AnthropicIterationUsage.schema.fields,
|
||||
iterations: Schema.optional(Schema.Array(AnthropicIterationUsage)),
|
||||
}),
|
||||
[JsonObject],
|
||||
)
|
||||
type AnthropicUsage = Schema.Schema.Type<typeof AnthropicUsage>
|
||||
|
||||
const AnthropicStreamBlock = Schema.Struct({
|
||||
@@ -377,6 +422,7 @@ type AnthropicStreamBlock = Schema.Schema.Type<typeof AnthropicStreamBlock>
|
||||
const decodeAnthropicStreamBlock = Schema.decodeUnknownOption(AnthropicStreamBlock)
|
||||
|
||||
const AnthropicStreamDelta = Schema.Struct({
|
||||
content: optionalNull(Schema.String),
|
||||
type: Schema.optional(Schema.String),
|
||||
text: Schema.optional(Schema.String),
|
||||
thinking: Schema.optional(Schema.String),
|
||||
@@ -406,6 +452,8 @@ const AnthropicEvent = Schema.Struct({
|
||||
type AnthropicEvent = Schema.Schema.Type<typeof AnthropicEvent>
|
||||
|
||||
interface ParserState {
|
||||
readonly provider: LLMRequest["model"]["provider"]
|
||||
readonly compactions: Readonly<Record<number, string | null>>
|
||||
readonly providerMetadataKey: string
|
||||
readonly tools: ToolStream.State<number>
|
||||
readonly reasoningSignatures: Readonly<Record<number, string>>
|
||||
@@ -831,6 +879,7 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* (
|
||||
const content: AnthropicUserBlock[] = []
|
||||
for (const part of message.content) {
|
||||
if (part.type === "text") {
|
||||
if (part.text.trim().length === 0) continue
|
||||
content.push({ type: "text", text: part.text, cache_control: cacheControl(breakpoints, part.cache) })
|
||||
continue
|
||||
}
|
||||
@@ -840,14 +889,21 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* (
|
||||
}
|
||||
return yield* ProviderShared.unsupportedContent("Anthropic Messages", "user", ["text", "media"])
|
||||
}
|
||||
messages.push({ role: "user", content })
|
||||
if (content.length > 0) messages.push({ role: "user", content })
|
||||
continue
|
||||
}
|
||||
|
||||
if (message.role === "assistant") {
|
||||
const content: AnthropicAssistantBlock[] = []
|
||||
for (const part of message.content) {
|
||||
if (part.type === "compaction") {
|
||||
if (part.provider !== request.model.provider || part.text === undefined)
|
||||
return yield* invalid("Compaction state must be replayed to its originating provider and API")
|
||||
content.push({ type: "compaction", content: part.text })
|
||||
continue
|
||||
}
|
||||
if (part.type === "text") {
|
||||
if (part.text.trim().length === 0) continue
|
||||
content.push({ type: "text", text: part.text, cache_control: cacheControl(breakpoints, part.cache) })
|
||||
continue
|
||||
}
|
||||
@@ -891,7 +947,7 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* (
|
||||
`Anthropic Messages assistant messages only support text, reasoning, and tool-call content for now`,
|
||||
)
|
||||
}
|
||||
messages.push({ role: "assistant", content })
|
||||
if (content.length > 0) messages.push({ role: "assistant", content })
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1001,6 +1057,9 @@ const resolveThinking = Effect.fn("AnthropicMessages.resolveThinking")(function*
|
||||
})
|
||||
|
||||
const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (request: LLMRequest) {
|
||||
const management = yield* ProviderShared.validateWith(
|
||||
Schema.decodeUnknownEffect(Schema.UndefinedOr(ContextManagement)),
|
||||
)(request.providerOptions?.contextManagement)
|
||||
const generation = request.generation
|
||||
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
|
||||
// Allocate the 4-breakpoint budget in invalidation order: tools → system →
|
||||
@@ -1019,10 +1078,11 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
|
||||
)
|
||||
// Anthropic rejects tool_choice when tools are absent; "none" is only meaningful with tools present.
|
||||
const toolChoice = tools === undefined || !request.toolChoice ? undefined : yield* lowerToolChoice(request.toolChoice)
|
||||
const systemParts = request.system.filter((part) => part.text.length > 0)
|
||||
const system =
|
||||
request.system.length === 0
|
||||
systemParts.length === 0
|
||||
? undefined
|
||||
: request.system.map((part) => ({
|
||||
: systemParts.map((part) => ({
|
||||
type: "text" as const,
|
||||
text: part.text,
|
||||
cache_control: cacheControl(breakpoints, part.cache),
|
||||
@@ -1034,7 +1094,7 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
|
||||
)
|
||||
}
|
||||
const options = yield* resolveOptions(request)
|
||||
return {
|
||||
const body = {
|
||||
model: request.model.id,
|
||||
system,
|
||||
messages,
|
||||
@@ -1055,6 +1115,18 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
|
||||
metadata: options.metadata,
|
||||
service_tier: options.service_tier,
|
||||
}
|
||||
if (!management) return body
|
||||
return {
|
||||
...body,
|
||||
context_management: {
|
||||
edits: management.edits.map((edit) => ({
|
||||
type: edit.type,
|
||||
trigger: edit.trigger,
|
||||
pause_after_compaction: edit.pauseAfterCompaction,
|
||||
instructions: edit.instructions,
|
||||
})),
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
// =============================================================================
|
||||
@@ -1076,18 +1148,31 @@ const mapFinishReason = (reason: string | null | undefined): FinishReason => {
|
||||
// expose that subset through `output_tokens_details.thinking_tokens`.
|
||||
const mapUsage = (usage: AnthropicUsage | undefined, providerMetadataKey: string): Usage | undefined => {
|
||||
if (!usage) return undefined
|
||||
const nonCached = usage.input_tokens ?? undefined
|
||||
const cacheRead = usage.cache_read_input_tokens ?? undefined
|
||||
const cacheWrite = usage.cache_creation_input_tokens ?? undefined
|
||||
const iterations = usage.iterations?.length ? usage.iterations : [usage]
|
||||
const last = usage.iterations?.at(-1)
|
||||
const nonCached = ProviderShared.sumTokens(...iterations.map((item) => item.input_tokens ?? undefined))
|
||||
const cacheRead = ProviderShared.sumTokens(...iterations.map((item) => item.cache_read_input_tokens ?? undefined))
|
||||
const cacheWrite = ProviderShared.sumTokens(
|
||||
...iterations.map((item) => item.cache_creation_input_tokens ?? undefined),
|
||||
)
|
||||
const inputTokens = ProviderShared.sumTokens(nonCached, cacheRead, cacheWrite)
|
||||
const outputTokens = ProviderShared.sumTokens(...iterations.map((item) => item.output_tokens))
|
||||
return new Usage({
|
||||
inputTokens,
|
||||
outputTokens: usage.output_tokens,
|
||||
outputTokens,
|
||||
contextTokens:
|
||||
last?.type === "message"
|
||||
? ProviderShared.sumTokens(
|
||||
last.input_tokens ?? undefined,
|
||||
last.cache_read_input_tokens ?? undefined,
|
||||
last.cache_creation_input_tokens ?? undefined,
|
||||
)
|
||||
: undefined,
|
||||
nonCachedInputTokens: nonCached,
|
||||
cacheReadInputTokens: cacheRead,
|
||||
cacheWriteInputTokens: cacheWrite,
|
||||
reasoningTokens: usage.output_tokens_details?.thinking_tokens,
|
||||
totalTokens: ProviderShared.totalTokens(inputTokens, usage.output_tokens, undefined),
|
||||
reasoningTokens: ProviderShared.sumTokens(...iterations.map((item) => item.output_tokens_details?.thinking_tokens)),
|
||||
totalTokens: ProviderShared.totalTokens(inputTokens, outputTokens, undefined),
|
||||
providerMetadata: { [providerMetadataKey]: usage },
|
||||
})
|
||||
}
|
||||
@@ -1109,6 +1194,7 @@ const mergeUsage = (left: Usage | undefined, right: Usage | undefined, providerM
|
||||
return new Usage({
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
contextTokens: right.contextTokens ?? left.contextTokens,
|
||||
nonCachedInputTokens,
|
||||
cacheReadInputTokens,
|
||||
cacheWriteInputTokens,
|
||||
@@ -1167,7 +1253,6 @@ const onContentBlockStart = (
|
||||
event: AnthropicEvent & { readonly content_block: AnthropicStreamBlock },
|
||||
): StepResult => {
|
||||
const block = event.content_block
|
||||
if (!block) return [state, NO_EVENTS]
|
||||
|
||||
if (block.type === "tool_use" || block.type === "server_tool_use") {
|
||||
if (event.index === undefined || !block.id) return [state, NO_EVENTS]
|
||||
@@ -1262,7 +1347,16 @@ const onContentBlockDelta = Effect.fn("AnthropicMessages.onContentBlockDelta")(f
|
||||
) {
|
||||
const delta = event.delta
|
||||
|
||||
if (delta?.type === "text_delta" && delta.text) {
|
||||
if (delta.type === "compaction_delta") {
|
||||
if (event.index === undefined || !(event.index in state.compactions) || delta.content === undefined)
|
||||
return yield* ProviderShared.eventError(ADAPTER, "Compaction delta is missing its block or content")
|
||||
return [
|
||||
{ ...state, compactions: { ...state.compactions, [event.index]: delta.content } },
|
||||
NO_EVENTS,
|
||||
] satisfies StepResult
|
||||
}
|
||||
|
||||
if (delta.type === "text_delta" && delta.text) {
|
||||
if (!state.lifecycle.text.has(`text-${event.index ?? 0}`)) return [state, NO_EVENTS] satisfies StepResult
|
||||
const events: LLMEvent[] = []
|
||||
return [
|
||||
@@ -1271,7 +1365,7 @@ const onContentBlockDelta = Effect.fn("AnthropicMessages.onContentBlockDelta")(f
|
||||
] satisfies StepResult
|
||||
}
|
||||
|
||||
if (delta?.type === "thinking_delta" && delta.thinking) {
|
||||
if (delta.type === "thinking_delta" && delta.thinking) {
|
||||
if (!state.lifecycle.reasoning.has(`reasoning-${event.index ?? 0}`)) return [state, NO_EVENTS] satisfies StepResult
|
||||
const events: LLMEvent[] = []
|
||||
return [
|
||||
@@ -1283,7 +1377,7 @@ const onContentBlockDelta = Effect.fn("AnthropicMessages.onContentBlockDelta")(f
|
||||
] satisfies StepResult
|
||||
}
|
||||
|
||||
if (delta?.type === "signature_delta" && delta.signature) {
|
||||
if (delta.type === "signature_delta" && delta.signature) {
|
||||
const index = event.index ?? 0
|
||||
if (!state.lifecycle.reasoning.has(`reasoning-${index}`)) return [state, NO_EVENTS] satisfies StepResult
|
||||
return [
|
||||
@@ -1295,7 +1389,7 @@ const onContentBlockDelta = Effect.fn("AnthropicMessages.onContentBlockDelta")(f
|
||||
] satisfies StepResult
|
||||
}
|
||||
|
||||
if (delta?.type === "input_json_delta" && event.index !== undefined) {
|
||||
if (delta.type === "input_json_delta" && event.index !== undefined) {
|
||||
if (!delta.partial_json) return [state, NO_EVENTS] satisfies StepResult
|
||||
if (!state.tools[event.index]) return [state, NO_EVENTS] satisfies StepResult
|
||||
const result = ToolStream.appendExisting(
|
||||
@@ -1320,6 +1414,18 @@ const onContentBlockStop = Effect.fn("AnthropicMessages.onContentBlockStop")(fun
|
||||
event: AnthropicEvent,
|
||||
) {
|
||||
if (event.index === undefined) return [state, NO_EVENTS] satisfies StepResult
|
||||
if (event.index in state.compactions) {
|
||||
const { [event.index]: content, ...compactions } = state.compactions
|
||||
const events: LLMEvent[] = []
|
||||
const lifecycle = Lifecycle.stepStart(state.lifecycle, events)
|
||||
events.push(
|
||||
LLMEvent.compaction({
|
||||
provider: state.provider,
|
||||
text: content,
|
||||
}),
|
||||
)
|
||||
return [{ ...state, compactions, lifecycle }, events] satisfies StepResult
|
||||
}
|
||||
const result = yield* ToolStream.finish(ADAPTER, state.tools, event.index)
|
||||
const events: LLMEvent[] = []
|
||||
const resultEvents = result.events ?? []
|
||||
@@ -1343,31 +1449,51 @@ const onMessageDelta = (
|
||||
event: AnthropicEvent & { readonly delta?: AnthropicStreamDelta },
|
||||
): StepResult => {
|
||||
const usage = mergeUsage(state.usage, mapUsage(event.usage, state.providerMetadataKey), state.providerMetadataKey)
|
||||
const pendingFinish = (() => {
|
||||
const stopReason = event.delta?.stop_reason
|
||||
if (stopReason === null || stopReason === undefined) return state.pendingFinish
|
||||
|
||||
const stopSequence = event.delta?.stop_sequence
|
||||
const finishMetadata =
|
||||
stopSequence === null || stopSequence === undefined
|
||||
? state.pendingFinish?.providerMetadata
|
||||
: providerMetadata(state.providerMetadataKey, { stopSequence })
|
||||
return {
|
||||
reason: {
|
||||
normalized: mapFinishReason(stopReason),
|
||||
raw: stopReason,
|
||||
},
|
||||
providerMetadata: finishMetadata,
|
||||
}
|
||||
})()
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
usage,
|
||||
pendingFinish: {
|
||||
reason: {
|
||||
normalized: mapFinishReason(event.delta?.stop_reason),
|
||||
raw: event.delta?.stop_reason ?? undefined,
|
||||
},
|
||||
providerMetadata:
|
||||
event.delta?.stop_sequence === null || event.delta?.stop_sequence === undefined
|
||||
? undefined
|
||||
: providerMetadata(state.providerMetadataKey, { stopSequence: event.delta.stop_sequence }),
|
||||
},
|
||||
pendingFinish,
|
||||
},
|
||||
NO_EVENTS,
|
||||
]
|
||||
}
|
||||
|
||||
const onMessageStop = Effect.fn("AnthropicMessages.onMessageStop")(function* (state: ParserState) {
|
||||
if (Object.keys(state.compactions).length)
|
||||
return yield* ProviderShared.eventError(ADAPTER, "Response ended with an incomplete compaction block")
|
||||
const result = yield* ToolStream.finishAll(ADAPTER, state.tools)
|
||||
const events: LLMEvent[] = []
|
||||
const lifecycle = result.events.length ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle
|
||||
events.push(...result.events)
|
||||
const finished = Lifecycle.finish(lifecycle, events, {
|
||||
const closed = Object.entries(state.reasoningSignatures).reduce(
|
||||
(current, [index, signature]) =>
|
||||
Lifecycle.reasoningEnd(
|
||||
current,
|
||||
events,
|
||||
`reasoning-${index}`,
|
||||
providerMetadata(state.providerMetadataKey, { signature }),
|
||||
),
|
||||
lifecycle,
|
||||
)
|
||||
const finished = Lifecycle.finish(closed, events, {
|
||||
reason: state.pendingFinish?.reason ?? {
|
||||
normalized: "unknown",
|
||||
raw: undefined,
|
||||
@@ -1397,16 +1523,21 @@ const onError = (event: AnthropicEvent) => {
|
||||
)
|
||||
}
|
||||
|
||||
const isKnownStreamBlockType = (type: string) =>
|
||||
type === "text" ||
|
||||
type === "thinking" ||
|
||||
type === "redacted_thinking" ||
|
||||
type === "tool_use" ||
|
||||
type === "server_tool_use" ||
|
||||
isServerToolResultType(type)
|
||||
|
||||
const isKnownStreamDeltaType = (type: string) =>
|
||||
type === "text_delta" || type === "thinking_delta" || type === "signature_delta" || type === "input_json_delta"
|
||||
const STREAM_BLOCK_TYPES = new Set([
|
||||
"compaction",
|
||||
"text",
|
||||
"thinking",
|
||||
"redacted_thinking",
|
||||
"tool_use",
|
||||
"server_tool_use",
|
||||
])
|
||||
const STREAM_DELTA_TYPES = new Set([
|
||||
"compaction_delta",
|
||||
"text_delta",
|
||||
"thinking_delta",
|
||||
"signature_delta",
|
||||
"input_json_delta",
|
||||
])
|
||||
|
||||
const invalidStreamEvent = (event: AnthropicEvent) =>
|
||||
Effect.fail(
|
||||
@@ -1435,7 +1566,16 @@ const step = (state: ParserState, event: AnthropicEvent) => {
|
||||
if (event.type === "content_block_start") {
|
||||
if (!ProviderShared.isRecord(event.content_block) || typeof event.content_block.type !== "string")
|
||||
return invalidStreamEvent(event)
|
||||
if (!isKnownStreamBlockType(event.content_block.type)) return Effect.succeed<StepResult>([state, NO_EVENTS])
|
||||
if (event.content_block.type === "compaction") {
|
||||
const decoded = Schema.decodeUnknownOption(AnthropicCompactionBlock)(event.content_block)
|
||||
if (event.index === undefined || Option.isNone(decoded)) return invalidStreamEvent(event)
|
||||
return Effect.succeed<StepResult>([
|
||||
{ ...state, compactions: { ...state.compactions, [event.index]: decoded.value.content } },
|
||||
NO_EVENTS,
|
||||
])
|
||||
}
|
||||
if (!STREAM_BLOCK_TYPES.has(event.content_block.type) && !isServerToolResultType(event.content_block.type))
|
||||
return Effect.succeed<StepResult>([state, NO_EVENTS])
|
||||
const decoded = decodeAnthropicStreamBlock(event.content_block)
|
||||
if (Option.isNone(decoded)) return invalidStreamEvent(event)
|
||||
const block = decoded.value
|
||||
@@ -1449,7 +1589,7 @@ const step = (state: ParserState, event: AnthropicEvent) => {
|
||||
}
|
||||
if (event.type === "content_block_delta") {
|
||||
if (!ProviderShared.isRecord(event.delta)) return invalidStreamEvent(event)
|
||||
if (typeof event.delta.type === "string" && !isKnownStreamDeltaType(event.delta.type))
|
||||
if (typeof event.delta.type === "string" && !STREAM_DELTA_TYPES.has(event.delta.type))
|
||||
return Effect.succeed<StepResult>([state, NO_EVENTS])
|
||||
const decoded = decodeAnthropicStreamDelta(event.delta)
|
||||
if (Option.isNone(decoded)) return invalidStreamEvent(event)
|
||||
@@ -1483,6 +1623,8 @@ export const protocol = Protocol.make({
|
||||
stream: {
|
||||
event: Protocol.jsonEvent(AnthropicEvent),
|
||||
initial: (request) => ({
|
||||
provider: request.model.provider,
|
||||
compactions: {},
|
||||
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
|
||||
tools: ToolStream.empty<number>(),
|
||||
reasoningSignatures: {},
|
||||
@@ -1492,6 +1634,37 @@ export const protocol = Protocol.make({
|
||||
},
|
||||
})
|
||||
|
||||
export const transport = <Body extends Pick<AnthropicMessagesBody, "messages" | "context_management">>() => {
|
||||
const http = HttpTransport.httpJson<Body, string>({ framing })
|
||||
return {
|
||||
...http,
|
||||
prepare: (input: Parameters<typeof http.prepare>[0]) => {
|
||||
if (
|
||||
!input.body.context_management?.edits.length &&
|
||||
!input.body.messages.some((message) => message.content.some((block) => block.type === "compaction"))
|
||||
)
|
||||
return http.prepare(input)
|
||||
const headers = Headers.fromInput(input.request.http?.headers)
|
||||
const betas = new Set(
|
||||
(headers["anthropic-beta"] ?? "")
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean),
|
||||
)
|
||||
betas.add("compact-2026-01-12")
|
||||
return http.prepare({
|
||||
...input,
|
||||
request: LLMRequest.update(input.request, {
|
||||
http: new HttpOptions({
|
||||
...input.request.http,
|
||||
headers: { ...headers, "anthropic-beta": [...betas].join(",") },
|
||||
}),
|
||||
}),
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export const route = Route.make({
|
||||
id: ADAPTER,
|
||||
provider: "anthropic",
|
||||
@@ -1501,7 +1674,7 @@ export const route = Route.make({
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
}),
|
||||
auth: Auth.none,
|
||||
framing,
|
||||
transport: transport<AnthropicMessagesBody>(),
|
||||
headers: () => ({ "anthropic-version": "2023-06-01" }),
|
||||
})
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { Effect, Encoding, Schema } from "effect"
|
||||
import { Route } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
@@ -262,24 +262,28 @@ const providerMetadata = (key: string, metadata: Record<string, unknown>): Provi
|
||||
|
||||
const reasoningSignature = (part: ReasoningPart, providerMetadataKey: string) => {
|
||||
const metadata = part.providerMetadata?.[providerMetadataKey]
|
||||
return (
|
||||
part.encrypted ??
|
||||
(ProviderShared.isRecord(metadata) && typeof metadata.signature === "string" ? metadata.signature : undefined)
|
||||
)
|
||||
if (part.encrypted !== undefined) return part.encrypted
|
||||
if (ProviderShared.isRecord(metadata) && typeof metadata.signature === "string") return metadata.signature
|
||||
}
|
||||
|
||||
const reasoningRedactedData = (part: ReasoningPart, providerMetadataKey: string) => {
|
||||
const metadata = part.providerMetadata?.[providerMetadataKey]
|
||||
return ProviderShared.isRecord(metadata) && typeof metadata.redactedData === "string"
|
||||
? metadata.redactedData
|
||||
: undefined
|
||||
if (ProviderShared.isRecord(metadata) && typeof metadata.redactedData === "string") return metadata.redactedData
|
||||
}
|
||||
|
||||
const removeEmptyToolInputKeys = (input: unknown): unknown => {
|
||||
if (Array.isArray(input)) return input.map(removeEmptyToolInputKeys)
|
||||
if (!ProviderShared.isRecord(input)) return input
|
||||
return Object.fromEntries(
|
||||
Object.entries(input).flatMap(([key, value]) => (key === "" ? [] : [[key, removeEmptyToolInputKeys(value)]])),
|
||||
)
|
||||
}
|
||||
|
||||
const lowerToolCall = (part: ToolCallPart): BedrockToolUseBlock => ({
|
||||
toolUse: {
|
||||
toolUseId: part.id,
|
||||
name: part.name,
|
||||
input: part.input,
|
||||
input: removeEmptyToolInputKeys(part.input),
|
||||
},
|
||||
})
|
||||
|
||||
@@ -374,6 +378,12 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (
|
||||
content.push({ reasoningContent: { redactedContent: redactedData } })
|
||||
continue
|
||||
}
|
||||
if (signature === undefined || signature.trim().length === 0) {
|
||||
// Interrupted streams and model switches can leave unsigned reasoning.
|
||||
// Preserve readable history as text rather than replay invalid reasoningContent.
|
||||
if (part.text.trim().length > 0) content.push(...textWithCache(breakpoints, part.text, part.cache))
|
||||
continue
|
||||
}
|
||||
content.push({ reasoningContent: { reasoningText: { text: part.text, signature } } })
|
||||
continue
|
||||
}
|
||||
@@ -382,7 +392,7 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (
|
||||
continue
|
||||
}
|
||||
}
|
||||
messages.push({ role: "assistant", content })
|
||||
if (content.length > 0) messages.push({ role: "assistant", content })
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -408,7 +418,12 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (
|
||||
const lowerSystem = (
|
||||
breakpoints: BedrockCache.Breakpoints,
|
||||
system: ReadonlyArray<LLMRequest["system"][number]>,
|
||||
): BedrockSystemBlock[] => system.flatMap((part) => textWithCache(breakpoints, part.text, part.cache))
|
||||
) => {
|
||||
const content = system
|
||||
.filter((part) => part.text.length > 0)
|
||||
.flatMap((part) => textWithCache(breakpoints, part.text, part.cache))
|
||||
return content.length === 0 ? undefined : content
|
||||
}
|
||||
|
||||
const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request: LLMRequest) {
|
||||
const toolChoice = request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined
|
||||
@@ -416,38 +431,42 @@ const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request:
|
||||
// Bedrock-Claude shares Anthropic's 4-breakpoint cap. Spend the budget in
|
||||
// tools → system → messages order to favour the highest-impact prefixes.
|
||||
const breakpoints = BedrockCache.breakpoints()
|
||||
const toolConfig =
|
||||
request.tools.length > 0
|
||||
? {
|
||||
tools: lowerTools(request.model.compatibility?.toolSchema, breakpoints, request.tools),
|
||||
// Converse has no native "none". Keep definitions stable for prompt
|
||||
// caching and omit only the unsupported choice.
|
||||
toolChoice,
|
||||
}
|
||||
: undefined
|
||||
const system = request.system.length === 0 ? undefined : lowerSystem(breakpoints, request.system)
|
||||
const toolConfig = (() => {
|
||||
if (request.tools.length === 0) return undefined
|
||||
return {
|
||||
tools: lowerTools(request.model.compatibility?.toolSchema, breakpoints, request.tools),
|
||||
// Converse has no native "none". Keep definitions stable for prompt
|
||||
// caching and omit only the unsupported choice.
|
||||
toolChoice,
|
||||
}
|
||||
})()
|
||||
const system = lowerSystem(breakpoints, request.system)
|
||||
const messages = yield* lowerMessages(request, breakpoints)
|
||||
if (breakpoints.dropped > 0) {
|
||||
yield* Effect.logWarning(
|
||||
`Bedrock Converse: dropped ${breakpoints.dropped} cache breakpoint(s); the API allows at most ${BedrockCache.BEDROCK_BREAKPOINT_CAP} per request.`,
|
||||
)
|
||||
}
|
||||
return {
|
||||
modelId: request.model.id,
|
||||
messages,
|
||||
system,
|
||||
inferenceConfig:
|
||||
const inferenceConfig = (() => {
|
||||
if (
|
||||
generation?.maxTokens === undefined &&
|
||||
generation?.temperature === undefined &&
|
||||
generation?.topP === undefined &&
|
||||
(generation?.stop === undefined || generation.stop.length === 0)
|
||||
? undefined
|
||||
: {
|
||||
maxTokens: generation?.maxTokens,
|
||||
temperature: generation?.temperature,
|
||||
topP: generation?.topP,
|
||||
stopSequences: generation?.stop,
|
||||
},
|
||||
)
|
||||
return undefined
|
||||
return {
|
||||
maxTokens: generation?.maxTokens,
|
||||
temperature: generation?.temperature,
|
||||
topP: generation?.topP,
|
||||
stopSequences: generation?.stop,
|
||||
}
|
||||
})()
|
||||
return {
|
||||
modelId: request.model.id,
|
||||
messages,
|
||||
system,
|
||||
inferenceConfig,
|
||||
toolConfig,
|
||||
// Converse's base inferenceConfig has no topK; Anthropic/Nova accept it
|
||||
// as a model-specific field, so it goes through additionalModelRequestFields.
|
||||
@@ -463,7 +482,6 @@ const mapFinishReason = (reason: string): FinishReason => {
|
||||
if (reason === "max_tokens" || reason === "model_context_window_exceeded") return "length"
|
||||
if (reason === "tool_use") return "tool-calls"
|
||||
if (reason === "content_filtered" || reason === "guardrail_intervened") return "content-filter"
|
||||
if (reason === "malformed_model_output" || reason === "malformed_tool_use") return "error"
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
@@ -490,6 +508,7 @@ const mapUsage = (usage: BedrockUsageSchema | undefined, providerMetadataKey: st
|
||||
interface ParserState {
|
||||
readonly providerMetadataKey: string
|
||||
readonly tools: ToolStream.State<number>
|
||||
readonly finishedTools: ReadonlySet<number>
|
||||
// Bedrock splits the finish into `messageStop` (carries `stopReason`) and
|
||||
// `metadata` (carries usage). Hold the terminal event in state so `onHalt`
|
||||
// can emit exactly one finish after both chunks have had a chance to arrive.
|
||||
@@ -497,6 +516,16 @@ interface ParserState {
|
||||
readonly hasToolCalls: boolean
|
||||
readonly lifecycle: Lifecycle.State
|
||||
readonly reasoningSignatures: Readonly<Record<number, string>>
|
||||
readonly reasoningRedactedContent: Readonly<Record<number, ReadonlyArray<Uint8Array>>>
|
||||
}
|
||||
|
||||
const encodeRedactedContent = (chunks: ReadonlyArray<Uint8Array>) => {
|
||||
const bytes = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.length, 0))
|
||||
chunks.reduce((offset, chunk) => {
|
||||
bytes.set(chunk, offset)
|
||||
return offset + chunk.length
|
||||
}, 0)
|
||||
return Encoding.encodeBase64(bytes)
|
||||
}
|
||||
|
||||
const step = (state: ParserState, event: BedrockEvent) =>
|
||||
@@ -544,23 +573,46 @@ const step = (state: ParserState, event: BedrockEvent) =>
|
||||
const index = event.contentBlockDelta.contentBlockIndex
|
||||
const reasoning = event.contentBlockDelta.delta.reasoningContent
|
||||
const events: LLMEvent[] = []
|
||||
const redactedData = reasoning.redactedContent ?? reasoning.data
|
||||
const metadata = reasoning.signature
|
||||
? providerMetadata(state.providerMetadataKey, { signature: reasoning.signature })
|
||||
: redactedData !== undefined
|
||||
? providerMetadata(state.providerMetadataKey, { redactedData })
|
||||
: undefined
|
||||
const lifecycle =
|
||||
reasoning.text !== undefined || metadata !== undefined
|
||||
? Lifecycle.reasoningDelta(state.lifecycle, events, `reasoning-${index}`, reasoning.text ?? "", metadata)
|
||||
: state.lifecycle
|
||||
const redactedChunks = yield* (() => {
|
||||
if (reasoning.redactedContent === undefined) return Effect.succeed(undefined)
|
||||
return Effect.fromResult(Encoding.decodeBase64(reasoning.redactedContent)).pipe(
|
||||
Effect.map((chunk) => [...(state.reasoningRedactedContent[index] ?? []), chunk]),
|
||||
Effect.mapError((cause) =>
|
||||
ProviderShared.eventError(
|
||||
ADAPTER,
|
||||
"Bedrock Converse reasoningContent.redactedContent contains invalid base64 data",
|
||||
undefined,
|
||||
cause,
|
||||
),
|
||||
),
|
||||
)
|
||||
})()
|
||||
const redactedData = redactedChunks === undefined ? reasoning.data : encodeRedactedContent(redactedChunks)
|
||||
const metadata = (() => {
|
||||
if (reasoning.signature) return providerMetadata(state.providerMetadataKey, { signature: reasoning.signature })
|
||||
if (redactedData !== undefined) return providerMetadata(state.providerMetadataKey, { redactedData })
|
||||
})()
|
||||
const lifecycle = (() => {
|
||||
if (reasoning.text === undefined && metadata === undefined) return state.lifecycle
|
||||
return Lifecycle.reasoningDelta(state.lifecycle, events, `reasoning-${index}`, reasoning.text ?? "", metadata)
|
||||
})()
|
||||
const reasoningRedactedContent = (() => {
|
||||
if (redactedChunks !== undefined) return { ...state.reasoningRedactedContent, [index]: redactedChunks }
|
||||
if (reasoning.data === undefined) return state.reasoningRedactedContent
|
||||
return Object.fromEntries(
|
||||
Object.entries(state.reasoningRedactedContent).filter(([key]) => key !== String(index)),
|
||||
)
|
||||
})()
|
||||
const reasoningSignatures = (() => {
|
||||
if (!reasoning.signature) return state.reasoningSignatures
|
||||
return { ...state.reasoningSignatures, [index]: reasoning.signature }
|
||||
})()
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
lifecycle,
|
||||
reasoningSignatures: reasoning.signature
|
||||
? { ...state.reasoningSignatures, [index]: reasoning.signature }
|
||||
: state.reasoningSignatures,
|
||||
reasoningSignatures,
|
||||
reasoningRedactedContent,
|
||||
},
|
||||
events,
|
||||
] as const
|
||||
@@ -568,6 +620,7 @@ const step = (state: ParserState, event: BedrockEvent) =>
|
||||
|
||||
if (event.contentBlockDelta?.delta?.toolUse) {
|
||||
const index = event.contentBlockDelta.contentBlockIndex
|
||||
if (state.finishedTools.has(index)) return [state, []] as const
|
||||
const result = ToolStream.appendExisting(
|
||||
ADAPTER,
|
||||
state.tools,
|
||||
@@ -587,16 +640,24 @@ const step = (state: ParserState, event: BedrockEvent) =>
|
||||
const result = yield* ToolStream.finish(ADAPTER, state.tools, index)
|
||||
const events: LLMEvent[] = []
|
||||
const resultEvents = result.events ?? []
|
||||
const lifecycle = resultEvents.length
|
||||
? Lifecycle.stepStart(state.lifecycle, events)
|
||||
: Lifecycle.reasoningEnd(
|
||||
Lifecycle.textEnd(state.lifecycle, events, `text-${index}`),
|
||||
events,
|
||||
`reasoning-${index}`,
|
||||
state.reasoningSignatures[index]
|
||||
? providerMetadata(state.providerMetadataKey, { signature: state.reasoningSignatures[index] })
|
||||
: undefined,
|
||||
)
|
||||
const lifecycle = (() => {
|
||||
if (resultEvents.length) return Lifecycle.stepStart(state.lifecycle, events)
|
||||
const metadata = (() => {
|
||||
const signature = state.reasoningSignatures[index]
|
||||
if (signature) return providerMetadata(state.providerMetadataKey, { signature })
|
||||
const redactedContent = state.reasoningRedactedContent[index]
|
||||
if (redactedContent)
|
||||
return providerMetadata(state.providerMetadataKey, {
|
||||
redactedData: encodeRedactedContent(redactedContent),
|
||||
})
|
||||
})()
|
||||
return Lifecycle.reasoningEnd(
|
||||
Lifecycle.textEnd(state.lifecycle, events, `text-${index}`),
|
||||
events,
|
||||
`reasoning-${index}`,
|
||||
metadata,
|
||||
)
|
||||
})()
|
||||
events.push(...resultEvents)
|
||||
return [
|
||||
{
|
||||
@@ -606,15 +667,28 @@ const step = (state: ParserState, event: BedrockEvent) =>
|
||||
state.hasToolCalls,
|
||||
lifecycle,
|
||||
tools: result.tools,
|
||||
finishedTools: resultEvents.length > 0 ? new Set([...state.finishedTools, index]) : state.finishedTools,
|
||||
reasoningSignatures: Object.fromEntries(
|
||||
Object.entries(state.reasoningSignatures).filter(([key]) => key !== String(index)),
|
||||
),
|
||||
reasoningRedactedContent: Object.fromEntries(
|
||||
Object.entries(state.reasoningRedactedContent).filter(([key]) => key !== String(index)),
|
||||
),
|
||||
},
|
||||
events,
|
||||
] as const
|
||||
}
|
||||
|
||||
if (event.messageStop) {
|
||||
if (
|
||||
event.messageStop.stopReason === "malformed_model_output" ||
|
||||
event.messageStop.stopReason === "malformed_tool_use"
|
||||
)
|
||||
return yield* ProviderShared.eventError(
|
||||
ADAPTER,
|
||||
`Bedrock Converse stopped with ${event.messageStop.stopReason}`,
|
||||
ProviderShared.encodeJson(event),
|
||||
)
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
@@ -661,23 +735,22 @@ const step = (state: ParserState, event: BedrockEvent) =>
|
||||
|
||||
const framing = BedrockEventStream.framing(ADAPTER)
|
||||
|
||||
const onHalt = (state: ParserState): ReadonlyArray<LLMEvent> =>
|
||||
state.pendingFinish
|
||||
? (() => {
|
||||
const events: LLMEvent[] = []
|
||||
Lifecycle.finish(state.lifecycle, events, {
|
||||
reason: {
|
||||
...state.pendingFinish.reason,
|
||||
normalized:
|
||||
state.pendingFinish.reason.normalized === "stop" && state.hasToolCalls
|
||||
? "tool-calls"
|
||||
: state.pendingFinish.reason.normalized,
|
||||
},
|
||||
usage: state.pendingFinish.usage,
|
||||
})
|
||||
return events
|
||||
})()
|
||||
: []
|
||||
const onHalt = (state: ParserState): ReadonlyArray<LLMEvent> => {
|
||||
if (!state.pendingFinish) return []
|
||||
const normalized = (() => {
|
||||
if (state.pendingFinish.reason.normalized === "stop" && state.hasToolCalls) return "tool-calls"
|
||||
return state.pendingFinish.reason.normalized
|
||||
})()
|
||||
const events: LLMEvent[] = []
|
||||
Lifecycle.finish(state.lifecycle, events, {
|
||||
reason: {
|
||||
...state.pendingFinish.reason,
|
||||
normalized,
|
||||
},
|
||||
usage: state.pendingFinish.usage,
|
||||
})
|
||||
return events
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Protocol And Bedrock Route
|
||||
@@ -697,10 +770,12 @@ export const protocol = Protocol.make({
|
||||
initial: (request) => ({
|
||||
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
|
||||
tools: ToolStream.empty<number>(),
|
||||
finishedTools: new Set<number>(),
|
||||
pendingFinish: undefined,
|
||||
hasToolCalls: false,
|
||||
lifecycle: Lifecycle.initial(),
|
||||
reasoningSignatures: {},
|
||||
reasoningRedactedContent: {},
|
||||
}),
|
||||
step,
|
||||
onHalt: (state) => Effect.succeed(onHalt(state)),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { EventStreamCodec } from "@smithy/eventstream-codec"
|
||||
import { fromUtf8, toUtf8 } from "@smithy/util-utf8"
|
||||
import { Effect, Encoding, Stream } from "effect"
|
||||
import { AIError, AIErrorReason } from "../schema/index.js"
|
||||
import { AIError, AIErrorReason, InvalidProviderOutputError } from "../schema/index.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { ProviderShared } from "./shared.js"
|
||||
|
||||
@@ -22,6 +22,10 @@ interface FrameBufferState {
|
||||
|
||||
const initialFrameBuffer: FrameBufferState = { buffer: new Uint8Array(0), offset: 0 }
|
||||
|
||||
type FrameInput = { readonly _tag: "Chunk"; readonly bytes: Uint8Array } | { readonly _tag: "End" }
|
||||
|
||||
const endOfStream: FrameInput = { _tag: "End" }
|
||||
|
||||
const appendChunk = (state: FrameBufferState, chunk: Uint8Array): FrameBufferState => {
|
||||
const remaining = state.buffer.length - state.offset
|
||||
// Compact: drop the consumed prefix and append the new chunk in one alloc.
|
||||
@@ -33,9 +37,23 @@ const appendChunk = (state: FrameBufferState, chunk: Uint8Array): FrameBufferSta
|
||||
return { buffer: next, offset: 0 }
|
||||
}
|
||||
|
||||
const consumeFrames = (route: string) => (state: FrameBufferState, chunk: Uint8Array) =>
|
||||
const consumeFrames = (route: string) => (state: FrameBufferState, input: FrameInput) =>
|
||||
Effect.gen(function* () {
|
||||
let cursor = appendChunk(state, chunk)
|
||||
if (input._tag === "End") {
|
||||
const remaining = state.buffer.subarray(state.offset)
|
||||
if (remaining.length > 0)
|
||||
return yield* new AIError({
|
||||
reason: new InvalidProviderOutputError({
|
||||
route,
|
||||
classification: "incomplete-stream",
|
||||
message: `Incomplete Bedrock Converse event-stream frame: ${remaining.length} buffered bytes remain at end of stream`,
|
||||
body: Encoding.encodeBase64(remaining),
|
||||
}),
|
||||
})
|
||||
return [state, []] as const
|
||||
}
|
||||
|
||||
let cursor = appendChunk(state, input.bytes)
|
||||
const out: object[] = []
|
||||
while (cursor.buffer.length - cursor.offset >= 4) {
|
||||
const view = cursor.buffer.subarray(cursor.offset)
|
||||
@@ -113,7 +131,12 @@ const consumeFrames = (route: string) => (state: FrameBufferState, chunk: Uint8A
|
||||
export const framing = (route: string): Framing.Definition<object> => ({
|
||||
id: "aws-event-stream",
|
||||
body: (frame) => ("rawBody" in frame && typeof frame.rawBody === "string" ? frame.rawBody : undefined),
|
||||
frame: (bytes) => bytes.pipe(Stream.mapAccumEffect(() => initialFrameBuffer, consumeFrames(route))),
|
||||
frame: (bytes) =>
|
||||
bytes.pipe(
|
||||
Stream.map((bytes): FrameInput => ({ _tag: "Chunk", bytes })),
|
||||
Stream.concat(Stream.succeed(endOfStream)),
|
||||
Stream.mapAccumEffect(() => initialFrameBuffer, consumeFrames(route)),
|
||||
),
|
||||
})
|
||||
|
||||
export * as BedrockEventStream from "./bedrock-event-stream.js"
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
import { Effect, Encoding, Schema, Struct } from "effect"
|
||||
import { Headers } from "effect/unstable/http"
|
||||
import { AIError } from "../schema/index.js"
|
||||
import { Route } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { classifyProviderFailure } from "../provider-error.js"
|
||||
import { AnthropicMessages } from "./anthropic-messages.js"
|
||||
import { BedrockEventStream } from "./bedrock-event-stream.js"
|
||||
import { BedrockAuth } from "./utils/bedrock-auth.js"
|
||||
import { JsonObject, ProviderShared } from "./shared.js"
|
||||
|
||||
const ID = "bedrock-messages"
|
||||
const VERSION = "bedrock-2023-05-31"
|
||||
const Body = Schema.Struct({
|
||||
...Struct.omit(AnthropicMessages.AnthropicMessagesBody.fields, ["model", "stream"]),
|
||||
anthropic_version: Schema.Literal(VERSION),
|
||||
anthropic_beta: Schema.optional(Schema.Array(Schema.String)),
|
||||
}).check(
|
||||
Schema.makeFilter(
|
||||
(body) =>
|
||||
body.messages.flatMap((message) => message.content.map(mediaIssue)).find((issue) => issue !== undefined) ?? true,
|
||||
),
|
||||
)
|
||||
const Event = Schema.Struct({
|
||||
chunk: Schema.optional(Schema.Struct({ bytes: Schema.String })),
|
||||
exception: Schema.optional(
|
||||
Schema.Struct({
|
||||
type: Schema.String,
|
||||
details: Schema.StructWithRest(
|
||||
Schema.Struct({ message: Schema.optional(Schema.String), originalMessage: Schema.optional(Schema.String) }),
|
||||
[JsonObject],
|
||||
),
|
||||
}),
|
||||
),
|
||||
})
|
||||
|
||||
export const protocol = Protocol.make({
|
||||
id: ID,
|
||||
body: {
|
||||
schema: Body,
|
||||
from: Effect.fn("BedrockMessages.fromRequest")(function* (request) {
|
||||
const body = yield* AnthropicMessages.protocol.body.from(request)
|
||||
const headers = Headers.fromInput(request.http?.headers)
|
||||
const betas = new Set(
|
||||
(headers["anthropic-beta"] ?? "")
|
||||
.split(",")
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean),
|
||||
)
|
||||
if (
|
||||
body.context_management?.edits.length ||
|
||||
body.messages.some((message) => message.content.some((block) => block.type === "compaction"))
|
||||
)
|
||||
betas.add("compact-2026-01-12")
|
||||
return {
|
||||
...Struct.omit(body, ["model", "stream"]),
|
||||
anthropic_version: VERSION,
|
||||
anthropic_beta: betas.size ? [...betas] : undefined,
|
||||
} satisfies typeof Body.Type
|
||||
}),
|
||||
},
|
||||
stream: {
|
||||
event: Event,
|
||||
initial: AnthropicMessages.protocol.stream.initial,
|
||||
step: Effect.fn("BedrockMessages.step")(function* (state, event) {
|
||||
if (event.exception)
|
||||
return yield* new AIError({
|
||||
reason: classifyProviderFailure({
|
||||
message: event.exception.details.message ?? event.exception.details.originalMessage ?? event.exception.type,
|
||||
rawBody: ProviderShared.encodeJson(event),
|
||||
}),
|
||||
})
|
||||
if (!event.chunk) return yield* ProviderShared.eventError(ID, "Bedrock Messages event is missing its chunk")
|
||||
const text = yield* Effect.fromResult(Encoding.decodeBase64String(event.chunk.bytes)).pipe(
|
||||
Effect.mapError((cause) =>
|
||||
ProviderShared.eventError(ID, "Invalid Bedrock Messages chunk encoding", undefined, cause),
|
||||
),
|
||||
)
|
||||
const decoded = yield* Schema.decodeUnknownEffect(AnthropicMessages.protocol.stream.event)(text).pipe(
|
||||
Effect.mapError((cause) => ProviderShared.eventError(ID, "Invalid Bedrock Messages event", undefined, cause)),
|
||||
)
|
||||
return yield* AnthropicMessages.protocol.stream.step(state, decoded)
|
||||
}),
|
||||
},
|
||||
})
|
||||
|
||||
function mediaIssue(
|
||||
block: AnthropicMessages.AnthropicMessagesBody["messages"][number]["content"][number],
|
||||
): string | undefined {
|
||||
if (block.type === "tool_result")
|
||||
return typeof block.content === "string"
|
||||
? undefined
|
||||
: block.content.map(mediaIssue).find((issue) => issue !== undefined)
|
||||
if (block.type !== "image" && block.type !== "document") return undefined
|
||||
if (block.source.type === "url" || block.source.type === "file")
|
||||
return "Bedrock Messages does not support URL or file-ID media sources"
|
||||
if (
|
||||
block.type === "image" &&
|
||||
!["image/jpeg", "image/png", "image/webp", "image/gif"].includes(block.source.media_type)
|
||||
)
|
||||
return "Bedrock Messages requires a JPEG, PNG, WebP, or GIF image"
|
||||
if (block.source.type === "base64" && Encoding.decodeBase64(block.source.data)._tag === "Failure")
|
||||
return "Bedrock Messages media data must be valid base64"
|
||||
return undefined
|
||||
}
|
||||
|
||||
export const route = Route.make({
|
||||
id: ID,
|
||||
provider: "amazon-bedrock",
|
||||
providerMetadataKey: "anthropic",
|
||||
protocol,
|
||||
endpoint: Endpoint.path(
|
||||
({ request }) => `/model/${encodeURIComponent(request.model.id)}/invoke-with-response-stream`,
|
||||
{ baseURL: "https://bedrock-runtime.us-east-1.amazonaws.com" },
|
||||
),
|
||||
auth: BedrockAuth.auth,
|
||||
framing: BedrockEventStream.framing(ID),
|
||||
})
|
||||
|
||||
export * as BedrockMessages from "./bedrock-messages.js"
|
||||
@@ -240,6 +240,10 @@ interface ParserState {
|
||||
readonly lifecycle: Lifecycle.State
|
||||
readonly reasoningSignature?: string
|
||||
readonly textSignature?: string
|
||||
readonly reasoningId?: string
|
||||
readonly textId?: string
|
||||
readonly nextReasoningId: number
|
||||
readonly nextTextId: number
|
||||
readonly seenCallIds?: ReadonlySet<string>
|
||||
}
|
||||
|
||||
@@ -571,19 +575,23 @@ const finish = (state: ParserState): ReadonlyArray<LLMEvent> => {
|
||||
|
||||
const events: LLMEvent[] = []
|
||||
let lifecycle = state.lifecycle
|
||||
if (state.reasoningSignature !== undefined)
|
||||
if (state.reasoningId !== undefined)
|
||||
lifecycle = Lifecycle.reasoningEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
"reasoning-0",
|
||||
providerMetadata(state.providerMetadataKey, { thoughtSignature: state.reasoningSignature }),
|
||||
state.reasoningId,
|
||||
state.reasoningSignature === undefined
|
||||
? undefined
|
||||
: providerMetadata(state.providerMetadataKey, { thoughtSignature: state.reasoningSignature }),
|
||||
)
|
||||
if (state.textSignature !== undefined)
|
||||
if (state.textId !== undefined)
|
||||
lifecycle = Lifecycle.textEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
"text-0",
|
||||
providerMetadata(state.providerMetadataKey, { thoughtSignature: state.textSignature }),
|
||||
state.textId,
|
||||
state.textSignature === undefined
|
||||
? undefined
|
||||
: providerMetadata(state.providerMetadataKey, { thoughtSignature: state.textSignature }),
|
||||
)
|
||||
Lifecycle.finish(lifecycle, events, {
|
||||
reason: {
|
||||
@@ -632,6 +640,10 @@ const step = (state: ParserState, event: GeminiEvent) => {
|
||||
let lifecycle = nextState.lifecycle
|
||||
let reasoningSignature = nextState.reasoningSignature
|
||||
let textSignature = nextState.textSignature
|
||||
let reasoningId = nextState.reasoningId
|
||||
let textId = nextState.textId
|
||||
let nextReasoningId = nextState.nextReasoningId
|
||||
let nextTextId = nextState.nextTextId
|
||||
// Supplier ids must be tracked across chunks of the same response, not just within one event's parts.
|
||||
const seenCallIds = new Set(nextState.seenCallIds)
|
||||
|
||||
@@ -657,27 +669,51 @@ const step = (state: ParserState, event: GeminiEvent) => {
|
||||
else if (signature !== undefined && "text" in part) textSignature = signature
|
||||
if ("text" in part && part.text.length > 0) {
|
||||
if (part.thought) {
|
||||
if (textId !== undefined) {
|
||||
lifecycle = Lifecycle.textEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
textId,
|
||||
textSignature
|
||||
? providerMetadata(state.providerMetadataKey, { thoughtSignature: textSignature })
|
||||
: undefined,
|
||||
)
|
||||
textId = undefined
|
||||
textSignature = undefined
|
||||
}
|
||||
if (reasoningId === undefined) {
|
||||
reasoningId = `reasoning-${nextReasoningId}`
|
||||
nextReasoningId += 1
|
||||
}
|
||||
lifecycle = Lifecycle.reasoningDelta(
|
||||
lifecycle,
|
||||
events,
|
||||
"reasoning-0",
|
||||
reasoningId,
|
||||
part.text,
|
||||
signature ? providerMetadata(state.providerMetadataKey, { thoughtSignature: signature }) : undefined,
|
||||
)
|
||||
continue
|
||||
}
|
||||
lifecycle = Lifecycle.reasoningEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
"reasoning-0",
|
||||
reasoningSignature
|
||||
? providerMetadata(state.providerMetadataKey, { thoughtSignature: reasoningSignature })
|
||||
: undefined,
|
||||
)
|
||||
if (reasoningId !== undefined) {
|
||||
lifecycle = Lifecycle.reasoningEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
reasoningId,
|
||||
reasoningSignature
|
||||
? providerMetadata(state.providerMetadataKey, { thoughtSignature: reasoningSignature })
|
||||
: undefined,
|
||||
)
|
||||
reasoningId = undefined
|
||||
reasoningSignature = undefined
|
||||
}
|
||||
if (textId === undefined) {
|
||||
textId = `text-${nextTextId}`
|
||||
nextTextId += 1
|
||||
}
|
||||
lifecycle = Lifecycle.textDelta(
|
||||
lifecycle,
|
||||
events,
|
||||
"text-0",
|
||||
textId,
|
||||
part.text,
|
||||
textSignature ? providerMetadata(state.providerMetadataKey, { thoughtSignature: textSignature }) : undefined,
|
||||
)
|
||||
@@ -695,14 +731,28 @@ const step = (state: ParserState, event: GeminiEvent) => {
|
||||
const duplicate = supplied !== undefined && seenCallIds.has(supplied)
|
||||
if (supplied !== undefined) seenCallIds.add(supplied)
|
||||
const id = supplied !== undefined && !duplicate ? supplied : `tool_${crypto.randomUUID().replaceAll("-", "")}`
|
||||
lifecycle = Lifecycle.reasoningEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
"reasoning-0",
|
||||
reasoningSignature
|
||||
? providerMetadata(state.providerMetadataKey, { thoughtSignature: reasoningSignature })
|
||||
: undefined,
|
||||
)
|
||||
if (reasoningId !== undefined) {
|
||||
lifecycle = Lifecycle.reasoningEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
reasoningId,
|
||||
reasoningSignature
|
||||
? providerMetadata(state.providerMetadataKey, { thoughtSignature: reasoningSignature })
|
||||
: undefined,
|
||||
)
|
||||
reasoningId = undefined
|
||||
reasoningSignature = undefined
|
||||
}
|
||||
if (textId !== undefined) {
|
||||
lifecycle = Lifecycle.textEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
textId,
|
||||
textSignature ? providerMetadata(state.providerMetadataKey, { thoughtSignature: textSignature }) : undefined,
|
||||
)
|
||||
textId = undefined
|
||||
textSignature = undefined
|
||||
}
|
||||
lifecycle = Lifecycle.stepStart(lifecycle, events)
|
||||
events.push(
|
||||
LLMEvent.toolCall({
|
||||
@@ -725,6 +775,10 @@ const step = (state: ParserState, event: GeminiEvent) => {
|
||||
lifecycle,
|
||||
reasoningSignature,
|
||||
textSignature,
|
||||
reasoningId,
|
||||
textId,
|
||||
nextReasoningId,
|
||||
nextTextId,
|
||||
seenCallIds,
|
||||
finishReason: candidate.finishReason ?? nextState.finishReason,
|
||||
},
|
||||
@@ -752,6 +806,8 @@ export const protocol = Protocol.make({
|
||||
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
|
||||
hasToolCalls: false,
|
||||
lifecycle: Lifecycle.initial(),
|
||||
nextReasoningId: 0,
|
||||
nextTextId: 0,
|
||||
}),
|
||||
step,
|
||||
onHalt: (state) => Effect.succeed(finish(state)),
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
export * as AnthropicMessages from "./anthropic-messages.js"
|
||||
export * as BedrockConverse from "./bedrock-converse.js"
|
||||
export { BedrockMessages } from "./bedrock-messages.js"
|
||||
export * as Gemini from "./gemini.js"
|
||||
export * as MistralChat from "./mistral-chat.js"
|
||||
export * as OpenAIChat from "./openai-chat.js"
|
||||
export * as OpenAIImages from "./openai-images.js"
|
||||
export * as OpenAICompatibleChat from "./openai-compatible-chat.js"
|
||||
|
||||
@@ -0,0 +1,780 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { Auth } from "../route/auth.js"
|
||||
import { Route } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { HttpTransport } from "../route/transport/index.js"
|
||||
import {
|
||||
AIError,
|
||||
InvalidProviderOutputError,
|
||||
LLMEvent,
|
||||
Usage,
|
||||
type FinishReasonDetails,
|
||||
type LLMRequest,
|
||||
type MediaPart,
|
||||
type ToolCallPart,
|
||||
type ToolDefinition,
|
||||
} from "../schema/index.js"
|
||||
import { classifyProviderFailure } from "../provider-error.js"
|
||||
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
|
||||
import { Lifecycle } from "./utils/lifecycle.js"
|
||||
import { ToolStream } from "./utils/tool-stream.js"
|
||||
|
||||
const ADAPTER = "mistral-chat"
|
||||
const DONE = "[DONE]" as const
|
||||
const TOOL_ID = /^[A-Za-z0-9]{9}$/
|
||||
export const DEFAULT_BASE_URL = "https://api.mistral.ai/v1"
|
||||
export const PATH = "/chat/completions"
|
||||
|
||||
const MistralTextContent = Schema.Struct({
|
||||
type: Schema.Literal("text"),
|
||||
text: Schema.String,
|
||||
})
|
||||
|
||||
const MistralThinkingUnit = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
type: Schema.optional(Schema.String),
|
||||
text: Schema.optional(Schema.String),
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
)
|
||||
type MistralThinkingUnit = Schema.Schema.Type<typeof MistralThinkingUnit>
|
||||
|
||||
const MistralThinkingContent = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("thinking"),
|
||||
thinking: Schema.Array(MistralThinkingUnit),
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
)
|
||||
type MistralThinkingContent = Schema.Schema.Type<typeof MistralThinkingContent>
|
||||
const isMistralThinkingContent = Schema.is(MistralThinkingContent)
|
||||
|
||||
const MistralUserContent = Schema.Union([
|
||||
MistralTextContent,
|
||||
Schema.Struct({ type: Schema.Literal("image_url"), image_url: Schema.String }),
|
||||
Schema.Struct({ type: Schema.Literal("document_url"), document_url: Schema.String }),
|
||||
])
|
||||
type MistralUserContent = Schema.Schema.Type<typeof MistralUserContent>
|
||||
|
||||
const MistralAssistantToolCall = Schema.Struct({
|
||||
id: Schema.String,
|
||||
type: Schema.Literal("function"),
|
||||
function: Schema.Struct({ name: Schema.String, arguments: Schema.String }),
|
||||
})
|
||||
type MistralAssistantToolCall = Schema.Schema.Type<typeof MistralAssistantToolCall>
|
||||
|
||||
const MistralMessage = Schema.Union([
|
||||
Schema.Struct({ role: Schema.Literal("system"), content: Schema.String }),
|
||||
Schema.Struct({
|
||||
role: Schema.Literal("user"),
|
||||
content: Schema.Union([Schema.String, Schema.Array(MistralUserContent)]),
|
||||
}),
|
||||
Schema.Struct({
|
||||
role: Schema.Literal("assistant"),
|
||||
content: Schema.Union([Schema.String, Schema.Array(Schema.Union([MistralTextContent, MistralThinkingContent]))]),
|
||||
tool_calls: optionalArray(MistralAssistantToolCall),
|
||||
prefix: Schema.optional(Schema.Literal(true)),
|
||||
}),
|
||||
Schema.Struct({
|
||||
role: Schema.Literal("tool"),
|
||||
tool_call_id: Schema.String,
|
||||
name: Schema.String,
|
||||
content: Schema.Union([Schema.String, Schema.Array(MistralUserContent)]),
|
||||
}),
|
||||
]).pipe(Schema.toTaggedUnion("role"))
|
||||
type MistralMessage = Schema.Schema.Type<typeof MistralMessage>
|
||||
|
||||
const MistralTool = Schema.Struct({
|
||||
type: Schema.Literal("function"),
|
||||
function: Schema.Struct({
|
||||
name: Schema.String,
|
||||
description: Schema.String,
|
||||
parameters: JsonObject,
|
||||
strict: Schema.Literal(false),
|
||||
}),
|
||||
})
|
||||
type MistralTool = Schema.Schema.Type<typeof MistralTool>
|
||||
|
||||
const MistralOptions = Schema.Struct({
|
||||
safePrompt: Schema.optional(Schema.Boolean),
|
||||
documentImageLimit: Schema.optional(Schema.Number),
|
||||
documentPageLimit: Schema.optional(Schema.Number),
|
||||
parallelToolCalls: Schema.optional(Schema.Boolean),
|
||||
reasoningEffort: Schema.optional(Schema.String),
|
||||
promptMode: Schema.optional(Schema.Literal("reasoning")),
|
||||
promptCacheKey: Schema.optional(Schema.String),
|
||||
})
|
||||
|
||||
export type ReasoningEffort = "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | (string & {})
|
||||
|
||||
export type ProviderOptionsInput = {
|
||||
readonly safePrompt?: boolean
|
||||
readonly documentImageLimit?: number
|
||||
readonly documentPageLimit?: number
|
||||
readonly parallelToolCalls?: boolean
|
||||
readonly reasoningEffort?: ReasoningEffort
|
||||
readonly promptMode?: "reasoning"
|
||||
readonly promptCacheKey?: string
|
||||
readonly [key: string]: unknown
|
||||
}
|
||||
|
||||
const MistralBody = Schema.Struct({
|
||||
model: Schema.String,
|
||||
messages: Schema.Array(MistralMessage),
|
||||
tools: optionalArray(MistralTool),
|
||||
tool_choice: Schema.optional(
|
||||
Schema.Union([
|
||||
Schema.Literals(["auto", "none", "any"]),
|
||||
Schema.Struct({ type: Schema.Literal("function"), function: Schema.Struct({ name: Schema.String }) }),
|
||||
]),
|
||||
),
|
||||
stream: Schema.Literal(true),
|
||||
max_tokens: Schema.optional(Schema.Number),
|
||||
random_seed: Schema.optional(Schema.Number),
|
||||
temperature: Schema.optional(Schema.Number),
|
||||
top_p: Schema.optional(Schema.Number),
|
||||
frequency_penalty: Schema.optional(Schema.Number),
|
||||
presence_penalty: Schema.optional(Schema.Number),
|
||||
stop: optionalArray(Schema.String),
|
||||
prompt_cache_key: Schema.optional(Schema.String),
|
||||
safe_prompt: Schema.optional(Schema.Boolean),
|
||||
document_image_limit: Schema.optional(Schema.Number),
|
||||
document_page_limit: Schema.optional(Schema.Number),
|
||||
parallel_tool_calls: Schema.optional(Schema.Boolean),
|
||||
reasoning_effort: Schema.optional(Schema.String),
|
||||
prompt_mode: Schema.optional(Schema.Literal("reasoning")),
|
||||
})
|
||||
export type MistralBody = Schema.Schema.Type<typeof MistralBody>
|
||||
|
||||
const MistralUsageDetails = Schema.StructWithRest(Schema.Struct({ cached_tokens: optionalNull(Schema.Number) }), [
|
||||
Schema.Record(Schema.String, Schema.Unknown),
|
||||
])
|
||||
|
||||
const MistralUsage = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
prompt_tokens: optionalNull(Schema.Number),
|
||||
completion_tokens: optionalNull(Schema.Number),
|
||||
total_tokens: optionalNull(Schema.Number),
|
||||
num_cached_tokens: optionalNull(Schema.Number),
|
||||
prompt_token_details: optionalNull(MistralUsageDetails),
|
||||
prompt_tokens_details: optionalNull(MistralUsageDetails),
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
)
|
||||
|
||||
const MistralOutputContent = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
type: Schema.String,
|
||||
text: optionalNull(Schema.String),
|
||||
thinking: optionalNull(Schema.Unknown),
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
)
|
||||
type MistralOutputContent = Schema.Schema.Type<typeof MistralOutputContent>
|
||||
|
||||
const MistralToolDelta = Schema.Struct({
|
||||
index: optionalNull(Schema.Number),
|
||||
id: optionalNull(Schema.String),
|
||||
function: optionalNull(
|
||||
Schema.Struct({
|
||||
name: optionalNull(Schema.String),
|
||||
arguments: optionalNull(Schema.Union([Schema.String, JsonObject])),
|
||||
}),
|
||||
),
|
||||
})
|
||||
type MistralToolDelta = Schema.Schema.Type<typeof MistralToolDelta>
|
||||
|
||||
const MistralChoice = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
delta: optionalNull(
|
||||
Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
content: optionalNull(Schema.Union([Schema.String, Schema.Array(MistralOutputContent)])),
|
||||
tool_calls: optionalNull(Schema.Array(MistralToolDelta)),
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
),
|
||||
),
|
||||
finish_reason: optionalNull(Schema.String),
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
)
|
||||
|
||||
const MistralError = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
message: Schema.String,
|
||||
code: optionalNull(Schema.Union([Schema.String, Schema.Number])),
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
)
|
||||
|
||||
const MistralEvent = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
choices: optionalNull(Schema.Array(MistralChoice)),
|
||||
usage: optionalNull(MistralUsage),
|
||||
error: optionalNull(MistralError),
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
)
|
||||
type MistralEvent = Schema.Schema.Type<typeof MistralEvent>
|
||||
const MistralStreamEvent = Schema.Union([Schema.Literal(DONE), Protocol.jsonEvent(MistralEvent)])
|
||||
|
||||
const hashID = (value: string) => {
|
||||
const hash = (seed: number) => {
|
||||
let result = seed
|
||||
for (const char of value) result = Math.imul(result ^ char.charCodeAt(0), 16777619)
|
||||
return (result >>> 0).toString(36)
|
||||
}
|
||||
return `${hash(2166136261).padStart(7, "0")}${hash(2246822519).padStart(7, "0")}`.slice(-9)
|
||||
}
|
||||
|
||||
const toolIDNormalizer = (request: LLMRequest) => {
|
||||
const ids = request.messages.flatMap((message) =>
|
||||
message.content.flatMap((part) => (part.type === "tool-call" || part.type === "tool-result" ? [part.id] : [])),
|
||||
)
|
||||
const used = new Set(ids.filter((id) => TOOL_ID.test(id)))
|
||||
const normalized = new Map<string, string>()
|
||||
return (id: string) => {
|
||||
if (TOOL_ID.test(id)) return id
|
||||
const previous = normalized.get(id)
|
||||
if (previous) return previous
|
||||
let attempt = 0
|
||||
let candidate = hashID(id)
|
||||
while (used.has(candidate)) candidate = hashID(`${id}:${++attempt}`)
|
||||
used.add(candidate)
|
||||
normalized.set(id, candidate)
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
|
||||
const lowerMedia = Effect.fn("MistralChat.lowerMedia")(function* (part: MediaPart) {
|
||||
const media = ProviderShared.normalizeMedia(part)
|
||||
const url = typeof part.data === "string" && /^(?:https?:|data:)/.test(part.data) ? part.data : media.dataUrl
|
||||
if (media.mime.startsWith("image/")) return { type: "image_url" as const, image_url: url }
|
||||
if (media.mime === "application/pdf") return { type: "document_url" as const, document_url: url }
|
||||
return yield* ProviderShared.invalidRequest(`Mistral Chat does not support media type ${part.mediaType}`)
|
||||
})
|
||||
|
||||
const lowerUser = Effect.fn("MistralChat.lowerUser")(function* (message: LLMRequest["messages"][number]) {
|
||||
const content: MistralUserContent[] = []
|
||||
for (const part of message.content) {
|
||||
if (part.type === "text") {
|
||||
content.push({ type: "text", text: part.text })
|
||||
continue
|
||||
}
|
||||
if (part.type === "media") {
|
||||
content.push(yield* lowerMedia(part))
|
||||
continue
|
||||
}
|
||||
return yield* ProviderShared.unsupportedContent("Mistral Chat", "user", ["text", "media"])
|
||||
}
|
||||
if (content.every((part) => part.type === "text"))
|
||||
return { role: "user" as const, content: content.map((part) => part.text).join("") }
|
||||
return { role: "user" as const, content }
|
||||
})
|
||||
|
||||
const lowerToolCall = (part: ToolCallPart, normalizeID: (id: string) => string): MistralAssistantToolCall => ({
|
||||
id: normalizeID(part.id),
|
||||
type: "function",
|
||||
function: { name: part.name, arguments: ProviderShared.encodeJson(part.input) },
|
||||
})
|
||||
|
||||
const lowerAssistant = Effect.fn("MistralChat.lowerAssistant")(function* (
|
||||
message: LLMRequest["messages"][number],
|
||||
normalizeID: (id: string) => string,
|
||||
prefix: boolean,
|
||||
) {
|
||||
const structured = message.content.some(
|
||||
(part) => part.type === "reasoning" && isMistralThinkingContent(part.providerMetadata?.mistral?.thinking),
|
||||
)
|
||||
const content: Array<Schema.Schema.Type<typeof MistralTextContent> | MistralThinkingContent> = []
|
||||
const text: string[] = []
|
||||
const toolCalls: MistralAssistantToolCall[] = []
|
||||
for (const part of message.content) {
|
||||
if (part.type === "text") {
|
||||
if (structured) content.push({ type: "text", text: part.text })
|
||||
else text.push(part.text)
|
||||
continue
|
||||
}
|
||||
if (part.type === "reasoning") {
|
||||
const native = part.providerMetadata?.mistral?.thinking
|
||||
if (structured && isMistralThinkingContent(native)) content.push(native)
|
||||
else if (structured) content.push({ type: "text", text: part.text })
|
||||
else text.push(part.text)
|
||||
continue
|
||||
}
|
||||
if (part.type === "tool-call") {
|
||||
toolCalls.push(lowerToolCall(part, normalizeID))
|
||||
continue
|
||||
}
|
||||
return yield* ProviderShared.unsupportedContent("Mistral Chat", "assistant", ["text", "reasoning", "tool-call"])
|
||||
}
|
||||
return {
|
||||
role: "assistant" as const,
|
||||
content: structured ? content : text.join(""),
|
||||
...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),
|
||||
...(prefix ? { prefix: true as const } : {}),
|
||||
}
|
||||
})
|
||||
|
||||
const lowerToolResults = Effect.fn("MistralChat.lowerToolResults")(function* (
|
||||
message: LLMRequest["messages"][number],
|
||||
normalizeID: (id: string) => string,
|
||||
) {
|
||||
const output: MistralMessage[] = []
|
||||
for (const part of message.content) {
|
||||
if (part.type !== "tool-result")
|
||||
return yield* ProviderShared.unsupportedContent("Mistral Chat", "tool", ["tool-result"])
|
||||
if (part.result.type !== "content") {
|
||||
output.push({
|
||||
role: "tool",
|
||||
tool_call_id: normalizeID(part.id),
|
||||
name: part.name,
|
||||
content: ProviderShared.toolResultText(part),
|
||||
})
|
||||
continue
|
||||
}
|
||||
const content: MistralUserContent[] = []
|
||||
for (const item of part.result.value) {
|
||||
if (item.type === "text") {
|
||||
content.push({ type: "text", text: item.text })
|
||||
continue
|
||||
}
|
||||
content.push(yield* lowerMedia({ type: "media", mediaType: item.mime, data: item.uri, filename: item.name }))
|
||||
}
|
||||
output.push({
|
||||
role: "tool",
|
||||
tool_call_id: normalizeID(part.id),
|
||||
name: part.name,
|
||||
content: content.some((item) => item.type !== "text")
|
||||
? content
|
||||
: content.map((item) => (item.type === "text" ? item.text : "")).join(""),
|
||||
})
|
||||
}
|
||||
return output
|
||||
})
|
||||
|
||||
const lowerMessages = Effect.fn("MistralChat.lowerMessages")(function* (request: LLMRequest) {
|
||||
const normalizeID = toolIDNormalizer(request)
|
||||
const messages: MistralMessage[] =
|
||||
request.system.length === 0 ? [] : [{ role: "system", content: ProviderShared.joinText(request.system) }]
|
||||
for (const message of request.messages) {
|
||||
if (message.role === "system") {
|
||||
const update = yield* ProviderShared.wrappedSystemUpdate("Mistral Chat", message)
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: update.text,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (message.role === "user") {
|
||||
messages.push(yield* lowerUser(message))
|
||||
continue
|
||||
}
|
||||
if (message.role === "assistant") {
|
||||
const hasToolCalls = message.content.some((part) => part.type === "tool-call")
|
||||
const hasNativeThinking = message.content.some(
|
||||
(part) => part.type === "reasoning" && isMistralThinkingContent(part.providerMetadata?.mistral?.thinking),
|
||||
)
|
||||
const text = message.content
|
||||
.flatMap((part) => (part.type === "text" || part.type === "reasoning" ? [part.text] : []))
|
||||
.join("")
|
||||
if (!hasToolCalls && !hasNativeThinking && text.trim() === "") continue
|
||||
messages.push(yield* lowerAssistant(message, normalizeID, !hasToolCalls && message === request.messages.at(-1)))
|
||||
continue
|
||||
}
|
||||
messages.push(...(yield* lowerToolResults(message, normalizeID)))
|
||||
}
|
||||
return messages
|
||||
})
|
||||
|
||||
const lowerTool = (tool: ToolDefinition): MistralTool => ({
|
||||
type: "function",
|
||||
function: { name: tool.name, description: tool.description, parameters: tool.inputSchema, strict: false },
|
||||
})
|
||||
|
||||
export const fromRequest = Effect.fn("MistralChat.fromRequest")(function* (request: LLMRequest) {
|
||||
const options = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(MistralOptions))(
|
||||
request.providerOptions ?? {},
|
||||
)
|
||||
const selected = request.toolChoice?.type === "tool" ? request.toolChoice.name : undefined
|
||||
if (request.toolChoice?.type === "tool" && !selected)
|
||||
return yield* ProviderShared.invalidRequest("Mistral Chat tool choice requires a tool name")
|
||||
if (options.reasoningEffort !== undefined && options.promptMode !== undefined)
|
||||
return yield* ProviderShared.invalidRequest(
|
||||
"Mistral Chat reasoningEffort and promptMode provider options are mutually exclusive",
|
||||
)
|
||||
const toolChoice = request.toolChoice
|
||||
? yield* ProviderShared.matchToolChoice("Mistral Chat", request.toolChoice, {
|
||||
auto: () => "auto" as const,
|
||||
none: () => "none" as const,
|
||||
required: () => "any" as const,
|
||||
tool: (name) => ({ type: "function" as const, function: { name } }),
|
||||
})
|
||||
: undefined
|
||||
return {
|
||||
model: request.model.id,
|
||||
messages: yield* lowerMessages(request),
|
||||
tools: request.tools.length > 0 ? request.tools.map(lowerTool) : undefined,
|
||||
tool_choice: toolChoice,
|
||||
stream: true as const,
|
||||
max_tokens: request.generation?.maxTokens,
|
||||
random_seed: request.generation?.seed,
|
||||
temperature: request.generation?.temperature,
|
||||
top_p: request.generation?.topP,
|
||||
frequency_penalty: request.generation?.frequencyPenalty,
|
||||
presence_penalty: request.generation?.presencePenalty,
|
||||
stop: request.generation?.stop,
|
||||
prompt_cache_key: request.cache === "none" ? undefined : (options.promptCacheKey ?? request.promptCacheKey),
|
||||
safe_prompt: options.safePrompt,
|
||||
document_image_limit: options.documentImageLimit,
|
||||
document_page_limit: options.documentPageLimit,
|
||||
parallel_tool_calls:
|
||||
options.parallelToolCalls ?? (request.toolChoice?.disableParallelToolUse === true ? false : undefined),
|
||||
reasoning_effort: options.reasoningEffort,
|
||||
prompt_mode: options.promptMode,
|
||||
}
|
||||
})
|
||||
|
||||
type ToolKey = string | number
|
||||
interface PendingTool {
|
||||
readonly id: string
|
||||
readonly name?: string
|
||||
readonly input: string
|
||||
}
|
||||
|
||||
interface ActiveContent {
|
||||
readonly type: "text" | "reasoning"
|
||||
readonly id: string
|
||||
readonly thinking?: MistralThinkingContent
|
||||
}
|
||||
|
||||
export interface ParserState {
|
||||
readonly tools: ToolStream.State<ToolKey>
|
||||
readonly pendingTools: Partial<Record<ToolKey, PendingTool>>
|
||||
readonly toolIDs: ReadonlyMap<string, string>
|
||||
readonly usedToolIDs: ReadonlySet<string>
|
||||
readonly completedTools: ReadonlyArray<LLMEvent>
|
||||
readonly latestToolKey?: ToolKey
|
||||
readonly generatedTools: number
|
||||
readonly lifecycle: Lifecycle.State
|
||||
readonly active?: ActiveContent
|
||||
readonly nextContent: number
|
||||
readonly usage?: Usage
|
||||
readonly finishReason?: FinishReasonDetails
|
||||
}
|
||||
|
||||
const mapUsage = (usage: MistralEvent["usage"]): Usage | undefined => {
|
||||
if (!usage) return undefined
|
||||
const input = usage.prompt_tokens ?? undefined
|
||||
const reported =
|
||||
usage.num_cached_tokens ??
|
||||
usage.prompt_tokens_details?.cached_tokens ??
|
||||
usage.prompt_token_details?.cached_tokens ??
|
||||
undefined
|
||||
const cached = input === undefined || reported === undefined ? undefined : Math.max(0, Math.min(input, reported))
|
||||
const output = usage.completion_tokens ?? undefined
|
||||
return new Usage({
|
||||
inputTokens: input,
|
||||
outputTokens: output,
|
||||
nonCachedInputTokens: ProviderShared.subtractTokens(input, cached),
|
||||
cacheReadInputTokens: cached,
|
||||
totalTokens: ProviderShared.totalTokens(input, output, usage.total_tokens ?? undefined),
|
||||
providerMetadata: { mistral: usage },
|
||||
})
|
||||
}
|
||||
|
||||
const mapFinishReason = (reason: string) => {
|
||||
switch (reason) {
|
||||
case "stop":
|
||||
return "stop" as const
|
||||
case "length":
|
||||
case "model_length":
|
||||
return "length" as const
|
||||
case "tool_calls":
|
||||
return "tool-calls" as const
|
||||
case "content_filter":
|
||||
return "content-filter" as const
|
||||
case "error":
|
||||
case "network_error":
|
||||
return "error" as const
|
||||
default:
|
||||
return "unknown" as const
|
||||
}
|
||||
}
|
||||
|
||||
const thinkingUnits = (value: unknown): ReadonlyArray<MistralThinkingUnit> => {
|
||||
if (typeof value === "string") return [{ type: "text", text: value }]
|
||||
if (!Array.isArray(value)) return []
|
||||
return value.filter(Schema.is(MistralThinkingUnit))
|
||||
}
|
||||
|
||||
const thinkingText = (thinking: ReadonlyArray<MistralThinkingUnit>) =>
|
||||
thinking.flatMap((unit) => (typeof unit.text === "string" ? [unit.text] : [])).join("")
|
||||
|
||||
const thinkingMetadata = (thinking: MistralThinkingContent) => ({ mistral: { thinking } })
|
||||
|
||||
const closeActive = (state: ParserState, events: LLMEvent[]) => {
|
||||
if (!state.active) return state
|
||||
const lifecycle =
|
||||
state.active.type === "text"
|
||||
? Lifecycle.textEnd(state.lifecycle, events, state.active.id)
|
||||
: Lifecycle.reasoningEnd(
|
||||
state.lifecycle,
|
||||
events,
|
||||
state.active.id,
|
||||
thinkingMetadata(state.active.thinking ?? { type: "thinking", thinking: [] }),
|
||||
thinkingText(state.active.thinking?.thinking ?? []),
|
||||
)
|
||||
return { ...state, lifecycle, active: undefined }
|
||||
}
|
||||
|
||||
const appendText = (state: ParserState, events: LLMEvent[], text: string) => {
|
||||
if (text.length === 0) return state
|
||||
const current = state.active?.type === "text" ? state : closeActive(state, events)
|
||||
const active = current.active ?? { type: "text" as const, id: `text-${current.nextContent}` }
|
||||
return {
|
||||
...current,
|
||||
lifecycle: Lifecycle.textDelta(current.lifecycle, events, active.id, text),
|
||||
active,
|
||||
nextContent: current.active ? current.nextContent : current.nextContent + 1,
|
||||
}
|
||||
}
|
||||
|
||||
const appendThinking = (state: ParserState, events: LLMEvent[], part: MistralOutputContent) => {
|
||||
const current = state.active?.type === "reasoning" ? state : closeActive(state, events)
|
||||
const units = thinkingUnits(part.thinking)
|
||||
const active = current.active ?? { type: "reasoning" as const, id: `reasoning-${current.nextContent}` }
|
||||
const thinking = {
|
||||
...active.thinking,
|
||||
...part,
|
||||
type: "thinking" as const,
|
||||
thinking: [...(active.thinking?.thinking ?? []), ...units],
|
||||
}
|
||||
const text = thinkingText(units)
|
||||
return {
|
||||
...current,
|
||||
lifecycle:
|
||||
text.length > 0
|
||||
? Lifecycle.reasoningDelta(current.lifecycle, events, active.id, text, thinkingMetadata(thinking))
|
||||
: Lifecycle.reasoningStart(current.lifecycle, events, active.id, thinkingMetadata(thinking)),
|
||||
active: { ...active, thinking },
|
||||
nextContent: current.active ? current.nextContent : current.nextContent + 1,
|
||||
}
|
||||
}
|
||||
|
||||
const appendContent = (
|
||||
state: ParserState,
|
||||
events: LLMEvent[],
|
||||
content: string | ReadonlyArray<MistralOutputContent>,
|
||||
) => {
|
||||
if (typeof content === "string") return appendText(state, events, content)
|
||||
return content.reduce((current, part) => {
|
||||
if (part.type === "text") return appendText(current, events, part.text ?? "")
|
||||
if (part.type === "thinking") return appendThinking(current, events, part)
|
||||
return closeActive(current, events)
|
||||
}, state)
|
||||
}
|
||||
|
||||
const normalizeStreamToolID = (state: ParserState, source: string) => {
|
||||
if (TOOL_ID.test(source))
|
||||
return { id: source, state: { ...state, usedToolIDs: new Set([...state.usedToolIDs, source]) } }
|
||||
const previous = state.toolIDs.get(source)
|
||||
if (previous) return { id: previous, state }
|
||||
let attempt = 0
|
||||
let id = hashID(source)
|
||||
while (state.usedToolIDs.has(id)) id = hashID(`${source}:${++attempt}`)
|
||||
return {
|
||||
id,
|
||||
state: {
|
||||
...state,
|
||||
toolIDs: new Map([...state.toolIDs, [source, id]]),
|
||||
usedToolIDs: new Set([...state.usedToolIDs, id]),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const toolText = (tool: MistralToolDelta) => {
|
||||
const value = tool.function?.arguments
|
||||
if (typeof value === "string") return value
|
||||
return value === null || value === undefined ? "" : ProviderShared.encodeJson(value)
|
||||
}
|
||||
|
||||
const appendTools = Effect.fn("MistralChat.appendTools")(function* (
|
||||
initial: ParserState,
|
||||
events: LLMEvent[],
|
||||
deltas: ReadonlyArray<MistralToolDelta>,
|
||||
) {
|
||||
if (deltas.length === 0) return initial
|
||||
let state = closeActive(initial, events)
|
||||
for (const [position, delta] of deltas.entries()) {
|
||||
const wireID = delta.id?.trim() || undefined
|
||||
const providedID = wireID === "null" ? undefined : wireID
|
||||
const key =
|
||||
delta.index ??
|
||||
(providedID
|
||||
? `id:${providedID}`
|
||||
: deltas.length > 1
|
||||
? `position:${position}`
|
||||
: (state.latestToolKey ?? `missing:${state.generatedTools}`))
|
||||
const existing = state.tools[key]
|
||||
const pending = state.pendingTools[key]
|
||||
const source = providedID ?? `generated:${String(key)}`
|
||||
const normalized =
|
||||
existing || pending ? { id: existing?.id ?? pending?.id ?? "", state } : normalizeStreamToolID(state, source)
|
||||
state = normalized.state
|
||||
const name = existing?.name ?? pending?.name ?? (delta.function?.name?.trim() || undefined)
|
||||
const text = `${pending?.input ?? ""}${toolText(delta)}`
|
||||
if (!name) {
|
||||
state = {
|
||||
...state,
|
||||
pendingTools: { ...state.pendingTools, [key]: { id: normalized.id, input: text } },
|
||||
latestToolKey: key,
|
||||
generatedTools: state.generatedTools + (!providedID && !pending ? 1 : 0),
|
||||
}
|
||||
continue
|
||||
}
|
||||
const result = ToolStream.appendOrStart(
|
||||
ADAPTER,
|
||||
state.tools,
|
||||
key,
|
||||
{ id: normalized.id, name, text },
|
||||
"Mistral Chat tool call delta is missing a name",
|
||||
)
|
||||
if (ToolStream.isError(result)) return yield* result
|
||||
if (result.events.length > 0) state = { ...state, lifecycle: Lifecycle.stepStart(state.lifecycle, events) }
|
||||
events.push(...result.events)
|
||||
const pendingTools = { ...state.pendingTools }
|
||||
delete pendingTools[key]
|
||||
state = {
|
||||
...state,
|
||||
tools: result.tools,
|
||||
pendingTools,
|
||||
latestToolKey: key,
|
||||
generatedTools: state.generatedTools + (!providedID && !existing && !pending ? 1 : 0),
|
||||
}
|
||||
}
|
||||
return state
|
||||
})
|
||||
|
||||
const hasLateContent = (event: MistralEvent) => {
|
||||
const delta = event.choices?.[0]?.delta
|
||||
if (typeof delta?.content === "string" && delta.content.length > 0) return true
|
||||
if (Array.isArray(delta?.content) && delta.content.length > 0) return true
|
||||
return (delta?.tool_calls ?? []).some(
|
||||
(tool) => Boolean(tool.id) || Boolean(tool.function?.name) || tool.function?.arguments !== undefined,
|
||||
)
|
||||
}
|
||||
|
||||
const step = Effect.fn("MistralChat.step")(function* (state: ParserState, event: MistralEvent) {
|
||||
if (event.error) {
|
||||
const body = ProviderShared.encodeJson(event)
|
||||
return yield* new AIError({
|
||||
reason: classifyProviderFailure({
|
||||
message: event.error.message,
|
||||
status: typeof event.error.code === "number" ? event.error.code : undefined,
|
||||
rawBody: body,
|
||||
}),
|
||||
})
|
||||
}
|
||||
const events: LLMEvent[] = []
|
||||
const usage = mapUsage(event.usage) ?? state.usage
|
||||
if (state.finishReason) {
|
||||
if (hasLateContent(event))
|
||||
return yield* ProviderShared.eventError(
|
||||
ADAPTER,
|
||||
"Mistral Chat received content after the finish reason",
|
||||
ProviderShared.encodeJson(event),
|
||||
)
|
||||
return [{ ...state, usage }, events] as const
|
||||
}
|
||||
const choice = event.choices?.[0]
|
||||
const withContent = choice?.delta?.content == null ? state : appendContent(state, events, choice.delta.content)
|
||||
const withTools = yield* appendTools(withContent, events, choice?.delta?.tool_calls ?? [])
|
||||
if (!choice?.finish_reason) return [{ ...withTools, usage }, events] as const
|
||||
|
||||
const finishReason = {
|
||||
normalized: mapFinishReason(choice.finish_reason),
|
||||
raw: choice.finish_reason,
|
||||
}
|
||||
const incomplete = finishReason.normalized === "length" || finishReason.normalized === "content-filter"
|
||||
if (!incomplete && Object.keys(withTools.pendingTools).length > 0)
|
||||
return yield* ProviderShared.eventError(
|
||||
ADAPTER,
|
||||
"Mistral Chat tool call delta is missing a name",
|
||||
ProviderShared.encodeJson(event),
|
||||
)
|
||||
const finished =
|
||||
!incomplete && Object.keys(withTools.tools).length > 0
|
||||
? yield* ToolStream.finishAll(ADAPTER, withTools.tools)
|
||||
: undefined
|
||||
return [
|
||||
{
|
||||
...withTools,
|
||||
tools: finished?.tools ?? withTools.tools,
|
||||
completedTools: finished?.events ?? withTools.completedTools,
|
||||
usage,
|
||||
finishReason,
|
||||
},
|
||||
events,
|
||||
] as const
|
||||
})
|
||||
|
||||
const finishEvents = Effect.fn("MistralChat.finishEvents")(function* (state: ParserState) {
|
||||
if (!state.finishReason)
|
||||
return yield* new AIError({
|
||||
reason: new InvalidProviderOutputError({
|
||||
message: "Mistral Chat stream ended without finish_reason",
|
||||
classification: "incomplete-stream",
|
||||
route: ADAPTER,
|
||||
}),
|
||||
})
|
||||
const events: LLMEvent[] = []
|
||||
const closed = closeActive(state, events)
|
||||
const lifecycle = closed.completedTools.length > 0 ? Lifecycle.stepStart(closed.lifecycle, events) : closed.lifecycle
|
||||
events.push(...closed.completedTools)
|
||||
const reason =
|
||||
state.finishReason.normalized === "stop" && closed.completedTools.some(LLMEvent.is.toolCall)
|
||||
? { ...state.finishReason, normalized: "tool-calls" as const }
|
||||
: state.finishReason
|
||||
Lifecycle.finish(lifecycle, events, { reason, usage: closed.usage })
|
||||
return events
|
||||
})
|
||||
|
||||
export const protocol = Protocol.make({
|
||||
id: ADAPTER,
|
||||
body: { schema: MistralBody, from: fromRequest },
|
||||
stream: {
|
||||
event: MistralStreamEvent,
|
||||
initial: (): ParserState => ({
|
||||
tools: ToolStream.empty<ToolKey>(),
|
||||
pendingTools: {},
|
||||
toolIDs: new Map(),
|
||||
usedToolIDs: new Set(),
|
||||
completedTools: [],
|
||||
generatedTools: 0,
|
||||
lifecycle: Lifecycle.initial(),
|
||||
nextContent: 0,
|
||||
}),
|
||||
step: (state: ParserState, event) => (event === DONE ? Effect.succeed([state, []] as const) : step(state, event)),
|
||||
terminal: (event) => event === DONE,
|
||||
onHalt: finishEvents,
|
||||
},
|
||||
})
|
||||
|
||||
export const framing = Framing.sseWithDone
|
||||
export const httpTransport = HttpTransport.sseJson.with<MistralBody>().with({ framing })
|
||||
|
||||
export const route = Route.make({
|
||||
id: ADAPTER,
|
||||
provider: "mistral",
|
||||
providerMetadataKey: "mistral",
|
||||
protocol,
|
||||
endpoint: Endpoint.path(PATH, { baseURL: DEFAULT_BASE_URL }),
|
||||
auth: Auth.none,
|
||||
transport: httpTransport,
|
||||
})
|
||||
|
||||
export * as MistralChat from "./mistral-chat.js"
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { Effect, Option, Schema } from "effect"
|
||||
import type { Content } from "@opencode-ai/schema/tool"
|
||||
import { HttpTransport } from "../route/transport/index.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
@@ -32,17 +32,19 @@ export const PATH = "/responses"
|
||||
// =============================================================================
|
||||
// Request Body Schema
|
||||
// =============================================================================
|
||||
const OpenResponsesInputText = Schema.Struct({
|
||||
export const OpenResponsesInputText = Schema.Struct({
|
||||
type: Schema.tag("input_text"),
|
||||
text: Schema.String,
|
||||
})
|
||||
const OpenResponsesInputImage = Schema.Struct({
|
||||
export const OpenResponsesInputImage = Schema.Struct({
|
||||
type: Schema.tag("input_image"),
|
||||
image_url: Schema.String,
|
||||
detail: Schema.optional(Schema.String),
|
||||
})
|
||||
const OpenResponsesInputFile = Schema.Struct({
|
||||
export const OpenResponsesInputFile = Schema.Struct({
|
||||
type: Schema.tag("input_file"),
|
||||
filename: Schema.String,
|
||||
detail: Schema.optional(Schema.String),
|
||||
file_data: Schema.optional(Schema.String),
|
||||
file_url: Schema.optional(Schema.String),
|
||||
})
|
||||
@@ -54,7 +56,7 @@ const MediaInput = Schema.Union([OpenResponsesInputImage, OpenResponsesInputFile
|
||||
export type MediaInput = Schema.Schema.Type<typeof MediaInput>
|
||||
const OpenResponsesInputContent = Schema.Union([OpenResponsesInputText, MediaInput])
|
||||
|
||||
const OpenResponsesOutputText = Schema.Struct({
|
||||
export const OpenResponsesOutputText = Schema.Struct({
|
||||
type: Schema.tag("output_text"),
|
||||
text: Schema.String,
|
||||
})
|
||||
@@ -62,6 +64,13 @@ const OpenResponsesOutputText = Schema.Struct({
|
||||
export const MessagePhase = Schema.NullOr(Schema.Literals(["commentary", "final_answer"]))
|
||||
type MessagePhase = Schema.Schema.Type<typeof MessagePhase>
|
||||
|
||||
export const MessageMetadata = Schema.Struct({
|
||||
itemId: Schema.optional(Schema.String),
|
||||
type: Schema.optional(Schema.Literal("message")),
|
||||
status: Schema.optional(Schema.String),
|
||||
phase: Schema.optional(MessagePhase),
|
||||
})
|
||||
|
||||
const messagePhase = (value: unknown): MessagePhase | undefined => {
|
||||
if (value === null || value === "commentary" || value === "final_answer") return value
|
||||
return undefined
|
||||
@@ -72,7 +81,7 @@ const OpenResponsesReasoningSummaryText = Schema.Struct({
|
||||
text: Schema.String,
|
||||
})
|
||||
|
||||
const OpenResponsesReasoningItem = Schema.Struct({
|
||||
export const OpenResponsesReasoningItem = Schema.Struct({
|
||||
type: Schema.tag("reasoning"),
|
||||
id: Schema.optionalKey(Schema.String),
|
||||
summary: Schema.Array(OpenResponsesReasoningSummaryText),
|
||||
@@ -149,16 +158,30 @@ const OpenResponsesFunctionCallOutput = Schema.Union([
|
||||
Schema.Array(OpenResponsesFunctionCallOutputContent),
|
||||
])
|
||||
|
||||
export const CompactionItem = Schema.Struct({
|
||||
type: Schema.Literal("compaction"),
|
||||
id: optionalNull(Schema.String),
|
||||
encrypted_content: Schema.String,
|
||||
})
|
||||
|
||||
export const InputItem = Schema.Union([
|
||||
CompactionItem,
|
||||
Schema.Struct({ role: Schema.tag("system"), content: Schema.String }),
|
||||
Schema.Struct({ role: Schema.tag("developer"), content: Schema.String }),
|
||||
Schema.Struct({ role: Schema.tag("user"), content: Schema.Array(OpenResponsesInputContent) }),
|
||||
Schema.Struct({
|
||||
role: Schema.tag("user"),
|
||||
content: Schema.Array(OpenResponsesInputContent),
|
||||
type: Schema.optional(Schema.Literal("message")),
|
||||
id: Schema.optional(Schema.String),
|
||||
status: Schema.optional(Schema.String),
|
||||
}),
|
||||
Schema.Struct({
|
||||
type: Schema.tag("message"),
|
||||
id: Schema.optionalKey(Schema.String),
|
||||
role: Schema.tag("assistant"),
|
||||
content: Schema.Array(OpenResponsesOutputText),
|
||||
phase: Schema.optionalKey(MessagePhase),
|
||||
status: Schema.optional(Schema.String),
|
||||
}),
|
||||
OpenResponsesReasoningItem,
|
||||
Schema.Struct({
|
||||
@@ -267,7 +290,7 @@ const OpenResponsesBody = Schema.Struct({
|
||||
})
|
||||
export type OpenResponsesBody = Schema.Schema.Type<typeof OpenResponsesBody>
|
||||
|
||||
const OpenResponsesUsage = Schema.Struct({
|
||||
export const OpenResponsesUsage = Schema.Struct({
|
||||
input_tokens: Schema.optional(Schema.Number),
|
||||
input_tokens_details: optionalNull(
|
||||
Schema.Struct({
|
||||
@@ -387,21 +410,25 @@ export interface Extension {
|
||||
const BASE: Extension = { id: ADAPTER, name: NAME }
|
||||
|
||||
export interface ParserState {
|
||||
readonly provider: LLMRequest["model"]["provider"]
|
||||
readonly completedCompactions: ReadonlySet<string>
|
||||
readonly id: string
|
||||
readonly name: string
|
||||
readonly providerMetadataKey: string
|
||||
readonly tools: ToolStream.State<string>
|
||||
// Call ids stay independent of item ids, which may be omitted or reused.
|
||||
readonly completedTools: ReadonlySet<string>
|
||||
readonly hasFunctionCall: boolean
|
||||
readonly lifecycle: Lifecycle.State
|
||||
readonly outputItems: Readonly<Record<number, string>>
|
||||
readonly messageItems: ReadonlySet<string>
|
||||
readonly messagePhases: Readonly<Record<string, MessagePhase | null>>
|
||||
readonly message: { readonly id: string; readonly phase: MessagePhase | null | undefined } | undefined
|
||||
readonly reasoningItems: Readonly<Record<string, ReasoningStreamItem>>
|
||||
}
|
||||
|
||||
type ReasoningSummaryStatus = "active" | "can-conclude" | "concluded"
|
||||
|
||||
interface ReasoningStreamItem {
|
||||
readonly open: boolean
|
||||
readonly encryptedContent: string | null | undefined
|
||||
// Keyed by the wire protocol's numeric `summary_index`. JS object keys coerce to
|
||||
// strings, but typing the map as `Record<number, ...>` documents intent
|
||||
@@ -486,6 +513,9 @@ const lowerMedia = Effect.fn("OpenResponses.lowerMedia")(function* (
|
||||
const media = ProviderShared.normalizeMedia(part)
|
||||
const extended = extension.lowerMedia?.({ part, media, request })
|
||||
if (extended) return extended
|
||||
const detail = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(OpenResponsesInputImage.fields.detail))(
|
||||
part.providerMetadata?.[metadataKey(request.model)]?.detail,
|
||||
)
|
||||
const url =
|
||||
typeof part.data === "string" && (part.data.startsWith("https://") || part.data.startsWith("http://"))
|
||||
? part.data
|
||||
@@ -496,10 +526,15 @@ const lowerMedia = Effect.fn("OpenResponses.lowerMedia")(function* (
|
||||
return {
|
||||
type: "input_file" as const,
|
||||
filename: part.filename ?? (media.mime === "application/pdf" ? "document.pdf" : "file"),
|
||||
detail,
|
||||
...(url ? { file_url: url } : { file_data: media.dataUrl }),
|
||||
}
|
||||
}
|
||||
return { type: "input_image" as const, image_url: url ?? media.dataUrl }
|
||||
return {
|
||||
type: "input_image" as const,
|
||||
image_url: url ?? media.dataUrl,
|
||||
detail,
|
||||
}
|
||||
})
|
||||
|
||||
const lowerUserContent = Effect.fnUntraced(function* (
|
||||
@@ -563,9 +598,12 @@ const lowerToolResultOutput = Effect.fnUntraced(function* (
|
||||
|
||||
const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (request: LLMRequest, extension: Extension) {
|
||||
const input: LoweredInputItem[] = []
|
||||
const providerMetadataKey = request.model.route.providerMetadataKey ?? "openresponses"
|
||||
const providerMetadataKey = metadataKey(request.model)
|
||||
|
||||
for (const message of request.messages) {
|
||||
const metadata = yield* ProviderShared.validateWith(
|
||||
Schema.decodeUnknownEffect(Schema.UndefinedOr(MessageMetadata)),
|
||||
)(message.providerMetadata?.[providerMetadataKey])
|
||||
if (message.role === "system") {
|
||||
input.push({
|
||||
role: "developer",
|
||||
@@ -575,10 +613,9 @@ const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (reques
|
||||
}
|
||||
|
||||
if (message.role === "user") {
|
||||
input.push({
|
||||
role: "user",
|
||||
content: yield* Effect.forEach(message.content, (part) => lowerUserContent(part, request, extension)),
|
||||
})
|
||||
const content = yield* Effect.forEach(message.content, (part) => lowerUserContent(part, request, extension))
|
||||
if (content.length > 0)
|
||||
input.push({ role: "user", content, type: metadata?.type, id: metadata?.itemId, status: metadata?.status })
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -591,9 +628,10 @@ const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (reques
|
||||
const groups = content.reduce<
|
||||
Array<{ id: string | undefined; phase: MessagePhase | null | undefined; parts: TextPart[] }>
|
||||
>((groups, part) => {
|
||||
const metadata = part.providerMetadata?.[providerMetadataKey]
|
||||
const id = itemID(part.providerMetadata, providerMetadataKey)
|
||||
const phase = ProviderShared.isRecord(metadata) ? messagePhase(metadata.phase) : undefined
|
||||
const partMetadata = part.providerMetadata?.[providerMetadataKey]
|
||||
const id = itemID(part.providerMetadata, providerMetadataKey) ?? metadata?.itemId
|
||||
const partPhase = messagePhase(partMetadata?.phase)
|
||||
const phase = partPhase === undefined ? metadata?.phase : partPhase
|
||||
const group = groups.at(-1)
|
||||
if (group && group.id === id && group.phase === phase) group.parts.push(part)
|
||||
else groups.push({ id, phase, parts: [part] })
|
||||
@@ -604,6 +642,7 @@ const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (reques
|
||||
type: "message" as const,
|
||||
...(group.id === undefined ? {} : { id: group.id }),
|
||||
role: "assistant" as const,
|
||||
status: metadata?.status,
|
||||
content: group.parts.map((part) => ({ type: "output_text" as const, text: part.text })),
|
||||
...(group.phase === undefined ? {} : { phase: group.phase }),
|
||||
})),
|
||||
@@ -611,6 +650,15 @@ const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (reques
|
||||
content.splice(0, content.length)
|
||||
}
|
||||
for (const part of message.content) {
|
||||
if (part.type === "compaction") {
|
||||
flushText()
|
||||
if (part.provider !== request.model.provider || part.encrypted === undefined)
|
||||
return yield* ProviderShared.invalidRequest(
|
||||
"Compaction state must be replayed to its originating provider and API",
|
||||
)
|
||||
input.push({ type: "compaction", id: part.id, encrypted_content: part.encrypted })
|
||||
continue
|
||||
}
|
||||
if (part.type === "text") {
|
||||
content.push(part)
|
||||
continue
|
||||
@@ -689,13 +737,30 @@ const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (reques
|
||||
return input
|
||||
})
|
||||
|
||||
const lowerOptions = (request: LLMRequest) => {
|
||||
const options = OpenResponsesOptions.resolve(request)
|
||||
export const lowerConversation = Effect.fn("OpenResponses.lowerConversation")(function* (
|
||||
request: LLMRequest,
|
||||
extension: Extension,
|
||||
) {
|
||||
const instructions = ProviderShared.joinText(request.system)
|
||||
return {
|
||||
model: request.model.id,
|
||||
input: yield* lowerMessages(request, extension),
|
||||
...(instructions ? { instructions } : {}),
|
||||
}
|
||||
})
|
||||
|
||||
export const lowerGeneration = (request: LLMRequest) => {
|
||||
const options = OpenResponsesOptions.resolve(request)
|
||||
const generation = request.generation
|
||||
const cacheKey = ProviderShared.promptCacheKey(request)
|
||||
const parallelToolCalls = resolveParallelToolCalls(request)
|
||||
return {
|
||||
...(instructions ? { instructions } : {}),
|
||||
stream: true as const,
|
||||
max_output_tokens: generation?.maxTokens,
|
||||
temperature: generation?.temperature,
|
||||
top_p: generation?.topP,
|
||||
presence_penalty: generation?.presencePenalty,
|
||||
frequency_penalty: generation?.frequencyPenalty,
|
||||
...(options.store !== undefined ? { store: options.store } : {}),
|
||||
...(options.metadata ? { metadata: options.metadata } : {}),
|
||||
...(options.safetyIdentifier ? { safety_identifier: options.safetyIdentifier } : {}),
|
||||
@@ -723,7 +788,7 @@ export const resolveParallelToolCalls = (request: LLMRequest) => {
|
||||
return disabled === undefined ? undefined : !disabled
|
||||
}
|
||||
|
||||
const allowedToolChoice = (request: LLMRequest) => {
|
||||
export const allowedToolChoice = (request: LLMRequest) => {
|
||||
const allowed = OpenResponsesOptions.resolve(request).allowedTools
|
||||
if (!allowed) return undefined
|
||||
return {
|
||||
@@ -737,11 +802,10 @@ export const fromRequestWithExtension = Effect.fn("OpenResponses.fromRequestWith
|
||||
request: LLMRequest,
|
||||
extension: Extension,
|
||||
) {
|
||||
const generation = request.generation
|
||||
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
|
||||
return {
|
||||
model: request.model.id,
|
||||
input: yield* lowerMessages(request, extension),
|
||||
...(yield* lowerConversation(request, extension)),
|
||||
...lowerGeneration(request),
|
||||
tools:
|
||||
request.tools.length === 0
|
||||
? undefined
|
||||
@@ -755,13 +819,6 @@ export const fromRequestWithExtension = Effect.fn("OpenResponses.fromRequestWith
|
||||
tool_choice:
|
||||
allowedToolChoice(request) ??
|
||||
(request.toolChoice ? yield* lowerToolChoice(extension.name, request.toolChoice) : undefined),
|
||||
stream: true as const,
|
||||
max_output_tokens: generation?.maxTokens,
|
||||
temperature: generation?.temperature,
|
||||
top_p: generation?.topP,
|
||||
presence_penalty: generation?.presencePenalty,
|
||||
frequency_penalty: generation?.frequencyPenalty,
|
||||
...lowerOptions(request),
|
||||
}
|
||||
})
|
||||
|
||||
@@ -778,7 +835,7 @@ export const fromRequest = Effect.fn("OpenResponses.fromRequest")(function* (req
|
||||
// cached-read and cache-write subsets, and `output_tokens` (inclusive total)
|
||||
// with a `reasoning_tokens` subset. Pass the totals through and derive the
|
||||
// non-cached breakdown.
|
||||
const mapUsage = (usage: OpenResponsesUsage | null | undefined, providerMetadataKey: string) => {
|
||||
export const mapUsage = (usage: OpenResponsesUsage | null | undefined, providerMetadataKey: string) => {
|
||||
if (!usage) return undefined
|
||||
const cached = usage.input_tokens_details?.cached_tokens
|
||||
const cacheWrite = usage.input_tokens_details?.cache_write_tokens
|
||||
@@ -808,6 +865,8 @@ const mapFinishReason = (event: Event, hasFunctionCall: boolean): FinishReason =
|
||||
return hasFunctionCall ? "tool-calls" : "unknown"
|
||||
}
|
||||
|
||||
export const metadataKey = (model: LLMRequest["model"]) => model.route.providerMetadataKey ?? "openresponses"
|
||||
|
||||
export const providerMetadata = (state: ParserState, metadata: Record<string, unknown>): ProviderMetadata => ({
|
||||
[state.providerMetadataKey]: metadata,
|
||||
})
|
||||
@@ -826,16 +885,16 @@ const TERMINAL_TYPES = new Set(["error", "response.completed", "response.incompl
|
||||
export const terminal = (event: Event) => TERMINAL_TYPES.has(event.type)
|
||||
|
||||
const onOutputTextDelta = (state: ParserState, event: Event, id: string): StepResult => {
|
||||
if (!event.delta || !state.messageItems.has(id)) return [state, NO_EVENTS]
|
||||
if (!event.delta || state.message?.id !== id) return [state, NO_EVENTS]
|
||||
const events: LLMEvent[] = []
|
||||
const phase = state.messagePhases[id]
|
||||
const phase = state.message.phase
|
||||
const metadata = providerMetadata(state, { itemId: id, ...(phase === undefined ? {} : { phase }) })
|
||||
const lifecycle = Lifecycle.textStart(state.lifecycle, events, id, metadata)
|
||||
return [{ ...state, lifecycle: Lifecycle.textDelta(lifecycle, events, id, event.delta) }, events]
|
||||
}
|
||||
|
||||
const onOutputTextDone = (state: ParserState, event: Event, id: string): StepResult => {
|
||||
if (state.messageItems.has(id)) {
|
||||
if (state.message?.id === id) {
|
||||
if (state.lifecycle.text.has(id) || event.text === undefined) return [state, NO_EVENTS]
|
||||
return onOutputTextDelta(state, { ...event, delta: event.text }, id)
|
||||
}
|
||||
@@ -843,21 +902,80 @@ const onOutputTextDone = (state: ParserState, event: Event, id: string): StepRes
|
||||
return [{ ...state, lifecycle: Lifecycle.textEnd(state.lifecycle, events, id) }, events]
|
||||
}
|
||||
|
||||
const decodeMessagePart = Schema.decodeUnknownOption(
|
||||
Schema.Union([OpenResponsesOutputText, Schema.Struct({ type: Schema.tag("refusal"), refusal: Schema.String })]),
|
||||
)
|
||||
|
||||
const decodeSummaryPart = Schema.decodeUnknownOption(OpenResponsesReasoningSummaryText)
|
||||
|
||||
const decodeReasoningPart = Schema.decodeUnknownOption(
|
||||
Schema.Struct({ type: Schema.tag("reasoning_text"), text: Schema.String }),
|
||||
)
|
||||
|
||||
const joinReasoningText = (parts: ReadonlyArray<string | undefined>) => {
|
||||
if (!parts.some((part) => part !== undefined && part.length > 0)) return undefined
|
||||
return parts.filter((part) => part !== undefined).join("\n\n")
|
||||
}
|
||||
|
||||
export const outputItemID = (state: ParserState, event: Event) =>
|
||||
event.output_index === undefined ? event.item_id : (state.outputItems[event.output_index] ?? event.item_id)
|
||||
|
||||
export const onReasoningDelta = (state: ParserState, event: Event, itemID: string): StepResult => {
|
||||
const startReasoningSummaryPart = (state: ParserState, itemID: string, index: number): StepResult => {
|
||||
const item = state.reasoningItems[itemID]
|
||||
if (!event.delta || !item) return [state, NO_EVENTS]
|
||||
const index = event.summary_index ?? 0
|
||||
if (!item?.open || index === 0 || item.summaryParts[index] !== undefined) return [state, NO_EVENTS]
|
||||
|
||||
const events: LLMEvent[] = []
|
||||
const lifecycle = Object.entries(item.summaryParts)
|
||||
.filter((entry) => entry[1] !== "concluded")
|
||||
.reduce(
|
||||
(lifecycle, entry) =>
|
||||
Lifecycle.reasoningEnd(lifecycle, events, `${itemID}:${entry[0]}`, providerMetadata(state, { itemId: itemID })),
|
||||
state.lifecycle,
|
||||
)
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
lifecycle: Lifecycle.reasoningDelta(state.lifecycle, events, `${itemID}:${index}`, event.delta),
|
||||
lifecycle: Lifecycle.reasoningStart(
|
||||
lifecycle,
|
||||
events,
|
||||
`${itemID}:${index}`,
|
||||
providerMetadata(state, { itemId: itemID, reasoningEncryptedContent: item.encryptedContent ?? null }),
|
||||
),
|
||||
reasoningItems: {
|
||||
...state.reasoningItems,
|
||||
[itemID]: { ...item, deltaIndexes: new Set([...item.deltaIndexes, index]) },
|
||||
[itemID]: {
|
||||
...item,
|
||||
summaryParts: {
|
||||
...Object.fromEntries(
|
||||
Object.entries(item.summaryParts).map((entry) =>
|
||||
entry[1] === "concluded" ? entry : [entry[0], "concluded" as const],
|
||||
),
|
||||
),
|
||||
[index]: "active",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
events,
|
||||
]
|
||||
}
|
||||
|
||||
export const onReasoningDelta = (state: ParserState, event: Event, itemID: string): StepResult => {
|
||||
const item = state.reasoningItems[itemID]
|
||||
if (!event.delta || !item?.open) return [state, NO_EVENTS]
|
||||
const index = event.summary_index ?? 0
|
||||
if (item.summaryParts[index] === "concluded") return [state, NO_EVENTS]
|
||||
const [started, emitted] = startReasoningSummaryPart(state, itemID, index)
|
||||
const current = started.reasoningItems[itemID]
|
||||
if (!current) return [started, emitted]
|
||||
const events: LLMEvent[] = [...emitted]
|
||||
return [
|
||||
{
|
||||
...started,
|
||||
lifecycle: Lifecycle.reasoningDelta(started.lifecycle, events, `${itemID}:${index}`, event.delta),
|
||||
reasoningItems: {
|
||||
...started.reasoningItems,
|
||||
[itemID]: { ...current, deltaIndexes: new Set([...current.deltaIndexes, index]) },
|
||||
},
|
||||
},
|
||||
events,
|
||||
@@ -869,7 +987,7 @@ export const onReasoningDelta = (state: ParserState, event: Event, itemID: strin
|
||||
// as a single delta unless that summary index already streamed one.
|
||||
export const onReasoningDone = (state: ParserState, event: Event, itemID: string): StepResult => {
|
||||
const item = state.reasoningItems[itemID]
|
||||
if (!item || typeof event.text !== "string") return [state, NO_EVENTS]
|
||||
if (!item?.open || typeof event.text !== "string") return [state, NO_EVENTS]
|
||||
const index = event.summary_index ?? 0
|
||||
if (item.deltaIndexes.has(index)) return [state, NO_EVENTS]
|
||||
return onReasoningDelta(state, { ...event, delta: event.text }, itemID)
|
||||
@@ -878,32 +996,48 @@ export const onReasoningDone = (state: ParserState, event: Event, itemID: string
|
||||
const reasoningMetadata = (state: ParserState, item: StreamItem & { id: string }) =>
|
||||
providerMetadata(state, { itemId: item.id, reasoningEncryptedContent: item.encrypted_content ?? null })
|
||||
|
||||
// Responses APIs stream reasoning items in a stable order:
|
||||
// Responses APIs normally stream reasoning items in this order:
|
||||
// `output_item.added` (reasoning) →
|
||||
// `reasoning_summary_part.added` (index=0) →
|
||||
// `reasoning_summary_text.delta` →
|
||||
// `reasoning_summary_part.done` (index=0) →
|
||||
// (repeat for index>0) →
|
||||
// `output_item.done` (reasoning).
|
||||
// The handlers below rely on this ordering: `onOutputItemAdded` seeds the
|
||||
// per-item entry, `onReasoningSummaryPartAdded` for `summary_index === 0`
|
||||
// short-circuits when the entry already exists, and higher-index handlers
|
||||
// fold against the same entry. Behaviour for out-of-order events is
|
||||
// best-effort, not guaranteed.
|
||||
// `onOutputItemAdded` seeds the per-item entry, while each later part start is
|
||||
// also an implicit boundary for the previous part. This keeps the common event
|
||||
// lifecycle ordered when a compatible provider omits or delays a part-done event.
|
||||
const onOutputItemAdded = (state: ParserState, event: Event): StepResult => {
|
||||
const item = event.item
|
||||
if (item?.type === "message" && item.id !== undefined) {
|
||||
const itemID = item.id
|
||||
const phase = messagePhase(item.phase)
|
||||
// A new message closes earlier messages, including ones that never streamed.
|
||||
const events: LLMEvent[] = []
|
||||
const lifecycle = [...state.lifecycle.text]
|
||||
.filter((id) => id !== itemID)
|
||||
.reduce((lifecycle, id) => {
|
||||
const openPhase = state.message?.id === id ? state.message.phase : undefined
|
||||
return Lifecycle.textEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
id,
|
||||
providerMetadata(state, { itemId: id, ...(openPhase === undefined ? {} : { phase: openPhase }) }),
|
||||
)
|
||||
}, state.lifecycle)
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
messageItems: new Set([...state.messageItems, item.id]),
|
||||
messagePhases: phase === undefined ? state.messagePhases : { ...state.messagePhases, [item.id]: phase },
|
||||
lifecycle,
|
||||
message: {
|
||||
id: itemID,
|
||||
phase: phase === undefined && state.message?.id === itemID ? state.message.phase : phase,
|
||||
},
|
||||
},
|
||||
NO_EVENTS,
|
||||
events,
|
||||
]
|
||||
}
|
||||
if (item && isReasoningItem(item)) {
|
||||
if (state.reasoningItems[item.id] !== undefined) return [state, NO_EVENTS]
|
||||
const events: LLMEvent[] = []
|
||||
return [
|
||||
{
|
||||
@@ -912,6 +1046,7 @@ const onOutputItemAdded = (state: ParserState, event: Event): StepResult => {
|
||||
reasoningItems: {
|
||||
...state.reasoningItems,
|
||||
[item.id]: {
|
||||
open: true,
|
||||
encryptedContent: item.encrypted_content,
|
||||
summaryParts: { 0: "active" },
|
||||
deltaIndexes: new Set(),
|
||||
@@ -923,6 +1058,8 @@ const onOutputItemAdded = (state: ParserState, event: Event): StepResult => {
|
||||
}
|
||||
if (item?.type !== "function_call" || !item.call_id) return [state, NO_EVENTS]
|
||||
const id = item.id ?? item.call_id
|
||||
if (Object.values(state.tools).some((tool) => tool?.id === item.call_id) || state.completedTools.has(item.call_id))
|
||||
return [state, NO_EVENTS]
|
||||
const metadata = item.id !== undefined ? providerMetadata(state, { itemId: item.id }) : undefined
|
||||
const events: LLMEvent[] = []
|
||||
const lifecycle = Lifecycle.stepStart(state.lifecycle, events)
|
||||
@@ -943,55 +1080,14 @@ const onOutputItemAdded = (state: ParserState, event: Event): StepResult => {
|
||||
|
||||
const onReasoningSummaryPartAdded = (state: ParserState, event: Event): StepResult => {
|
||||
if (event.item_id === undefined || event.summary_index === undefined) return [state, NO_EVENTS]
|
||||
const item = state.reasoningItems[event.item_id]
|
||||
if (!item) return [state, NO_EVENTS]
|
||||
if (event.summary_index === 0) return [state, NO_EVENTS]
|
||||
|
||||
const events: LLMEvent[] = []
|
||||
const closed = Object.entries(item.summaryParts)
|
||||
.filter((entry) => entry[1] === "can-conclude")
|
||||
.reduce(
|
||||
(lifecycle, entry) =>
|
||||
Lifecycle.reasoningEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
`${event.item_id}:${entry[0]}`,
|
||||
providerMetadata(state, { itemId: event.item_id }),
|
||||
),
|
||||
state.lifecycle,
|
||||
)
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
lifecycle: Lifecycle.reasoningStart(
|
||||
closed,
|
||||
events,
|
||||
`${event.item_id}:${event.summary_index}`,
|
||||
providerMetadata(state, { itemId: event.item_id, reasoningEncryptedContent: item.encryptedContent ?? null }),
|
||||
),
|
||||
reasoningItems: {
|
||||
...state.reasoningItems,
|
||||
[event.item_id]: {
|
||||
...item,
|
||||
summaryParts: {
|
||||
...Object.fromEntries(
|
||||
Object.entries(item.summaryParts).map((entry) =>
|
||||
entry[1] === "can-conclude" ? [entry[0], "concluded" as const] : entry,
|
||||
),
|
||||
),
|
||||
[event.summary_index]: "active",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
events,
|
||||
]
|
||||
return startReasoningSummaryPart(state, event.item_id, event.summary_index)
|
||||
}
|
||||
|
||||
const onReasoningSummaryPartDone = (state: ParserState, event: Event): StepResult => {
|
||||
if (event.item_id === undefined || event.summary_index === undefined) return [state, NO_EVENTS]
|
||||
const item = state.reasoningItems[event.item_id]
|
||||
if (!item) return [state, NO_EVENTS]
|
||||
if (!item?.open) return [state, NO_EVENTS]
|
||||
if (item.summaryParts[event.summary_index] !== "active") return [state, NO_EVENTS]
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
@@ -1041,28 +1137,52 @@ const onFunctionCallArgumentsDelta = Effect.fn("OpenResponses.onFunctionCallArgu
|
||||
return [{ ...state, lifecycle, tools: result.tools }, events] satisfies StepResult
|
||||
})
|
||||
|
||||
const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* (state: ParserState, event: Event) {
|
||||
const item = event.item
|
||||
const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* (
|
||||
state: ParserState,
|
||||
item: Event["item"],
|
||||
) {
|
||||
if (!item) return [state, NO_EVENTS] satisfies StepResult
|
||||
|
||||
if (item.type === "message" && item.id !== undefined) {
|
||||
const itemPhase = messagePhase(item.phase)
|
||||
const phase = itemPhase === undefined ? state.messagePhases[item.id] : itemPhase
|
||||
if (item.type === "compaction") {
|
||||
if (!item.id || typeof item.encrypted_content !== "string")
|
||||
return yield* ProviderShared.eventError(state.id, "Compaction output is missing its id or encrypted content")
|
||||
if (state.completedCompactions.has(item.id)) return [state, NO_EVENTS] satisfies StepResult
|
||||
const events: LLMEvent[] = []
|
||||
const messageItems = new Set(state.messageItems)
|
||||
messageItems.delete(item.id)
|
||||
const { [item.id]: _phase, ...messagePhases } = state.messagePhases
|
||||
const lifecycle = Lifecycle.stepStart(state.lifecycle, events)
|
||||
events.push(
|
||||
LLMEvent.compaction({
|
||||
provider: state.provider,
|
||||
id: item.id,
|
||||
encrypted: item.encrypted_content,
|
||||
}),
|
||||
)
|
||||
return [
|
||||
{ ...state, lifecycle, completedCompactions: new Set([...state.completedCompactions, item.id]) },
|
||||
events,
|
||||
] satisfies StepResult
|
||||
}
|
||||
|
||||
if (item.type === "message" && item.id !== undefined) {
|
||||
const message = state.message?.id === item.id ? state.message : undefined
|
||||
const itemPhase = messagePhase(item.phase)
|
||||
const phase = itemPhase === undefined ? message?.phase : itemPhase
|
||||
const parts: ReadonlyArray<unknown> = Array.isArray(item.content) ? item.content : []
|
||||
const content: string[] = []
|
||||
for (const part of parts) {
|
||||
const decoded = Option.getOrUndefined(decodeMessagePart(part))
|
||||
if (!decoded) continue
|
||||
content.push(decoded.type === "output_text" ? decoded.text : decoded.refusal)
|
||||
}
|
||||
const text = content.length > 0 ? content.join("") : undefined
|
||||
const metadata = providerMetadata(state, { itemId: item.id, ...(phase === undefined ? {} : { phase }) })
|
||||
const events: LLMEvent[] = []
|
||||
const lifecycle =
|
||||
message && text ? Lifecycle.textStart(state.lifecycle, events, item.id, metadata) : state.lifecycle
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
lifecycle: Lifecycle.textEnd(
|
||||
state.lifecycle,
|
||||
events,
|
||||
item.id,
|
||||
providerMetadata(state, { itemId: item.id, ...(phase === undefined ? {} : { phase }) }),
|
||||
),
|
||||
messageItems,
|
||||
messagePhases,
|
||||
lifecycle: Lifecycle.textEnd(lifecycle, events, item.id, metadata, text),
|
||||
message: message ? undefined : state.message,
|
||||
},
|
||||
events,
|
||||
] satisfies StepResult
|
||||
@@ -1070,20 +1190,36 @@ const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* (
|
||||
|
||||
if (item.type === "function_call") {
|
||||
if (!item.call_id || !item.name) return [state, NO_EVENTS] satisfies StepResult
|
||||
const id = item.id ?? item.call_id
|
||||
const tools = state.tools[id]
|
||||
? state.tools
|
||||
: ToolStream.start(state.tools, id, {
|
||||
id: item.call_id,
|
||||
name: item.name,
|
||||
providerMetadata: item.id !== undefined ? providerMetadata(state, { itemId: item.id }) : undefined,
|
||||
})
|
||||
const callID = item.call_id
|
||||
if (state.completedTools.has(callID)) return [state, NO_EVENTS] satisfies StepResult
|
||||
const metadata = item.id !== undefined ? providerMetadata(state, { itemId: item.id }) : undefined
|
||||
const fallback = item.id ?? callID
|
||||
// Match the pending tool by call id so item events that disagree on
|
||||
// whether `item.id` is present still resolve the same call.
|
||||
const registered =
|
||||
state.tools[fallback] !== undefined
|
||||
? fallback
|
||||
: Object.keys(state.tools).find((key) => state.tools[key]?.id === callID)
|
||||
const id = registered ?? fallback
|
||||
const tools =
|
||||
registered !== undefined
|
||||
? state.tools
|
||||
: ToolStream.start(state.tools, id, {
|
||||
id: callID,
|
||||
name: item.name,
|
||||
providerMetadata: metadata,
|
||||
})
|
||||
const result =
|
||||
item.arguments === undefined
|
||||
? yield* ToolStream.finish(state.id, tools, id)
|
||||
: yield* ToolStream.finishWithInput(state.id, tools, id, item.arguments)
|
||||
const events: LLMEvent[] = []
|
||||
const resultEvents = result.events ?? []
|
||||
const finished = result.events ?? []
|
||||
// A done-only call never streamed a start event, so open its lifecycle here.
|
||||
const resultEvents =
|
||||
registered !== undefined || finished.length === 0
|
||||
? finished
|
||||
: [LLMEvent.toolInputStart({ id: callID, name: item.name, providerMetadata: metadata }), ...finished]
|
||||
const lifecycle = resultEvents.length ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle
|
||||
events.push(...resultEvents)
|
||||
return [
|
||||
@@ -1094,30 +1230,82 @@ const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* (
|
||||
resultEvents.some((event) => LLMEvent.is.toolCall(event) || LLMEvent.is.toolInputError(event)) ||
|
||||
state.hasFunctionCall,
|
||||
tools: result.tools,
|
||||
completedTools: new Set([...state.completedTools, callID]),
|
||||
},
|
||||
events,
|
||||
] satisfies StepResult
|
||||
}
|
||||
|
||||
if (isReasoningItem(item)) {
|
||||
const events: LLMEvent[] = []
|
||||
if (state.reasoningItems[item.id]?.open === false) return [state, NO_EVENTS] satisfies StepResult
|
||||
const metadata = reasoningMetadata(state, item)
|
||||
const summaryParts: ReadonlyArray<unknown> = Array.isArray(item.summary) ? item.summary : []
|
||||
const summary: Array<string | undefined> = []
|
||||
for (const part of summaryParts) {
|
||||
const decoded = Option.getOrUndefined(decodeSummaryPart(part))
|
||||
// Keep missing entries so the array still matches the provider's summary indexes.
|
||||
summary.push(decoded?.text)
|
||||
}
|
||||
const reasoningParts: ReadonlyArray<unknown> = Array.isArray(item.content) ? item.content : []
|
||||
const content: string[] = []
|
||||
for (const part of reasoningParts) {
|
||||
const decoded = Option.getOrUndefined(decodeReasoningPart(part))
|
||||
if (decoded) content.push(decoded.text)
|
||||
}
|
||||
const itemText = joinReasoningText(summary) ?? joinReasoningText(content)
|
||||
const events: LLMEvent[] = []
|
||||
const reasoningItem = state.reasoningItems[item.id]
|
||||
if (reasoningItem) {
|
||||
const lifecycle = Object.entries(reasoningItem.summaryParts)
|
||||
.filter((entry) => entry[1] === "active" || entry[1] === "can-conclude")
|
||||
.reduce(
|
||||
(lifecycle, entry) => Lifecycle.reasoningEnd(lifecycle, events, `${item.id}:${entry[0]}`, metadata),
|
||||
state.lifecycle,
|
||||
)
|
||||
const { [item.id]: _removed, ...reasoningItems } = state.reasoningItems
|
||||
return [{ ...state, lifecycle, reasoningItems }, events] satisfies StepResult
|
||||
const fragments = Object.entries(reasoningItem.summaryParts)
|
||||
let lifecycle = state.lifecycle
|
||||
for (const [index, status] of fragments) {
|
||||
if (status === "concluded") continue
|
||||
// Do not repeat earlier summaries that were already emitted as separate fragments.
|
||||
const finalText = fragments.length === 1 ? itemText : summary[Number(index)]
|
||||
lifecycle = Lifecycle.reasoningEnd(lifecycle, events, `${item.id}:${index}`, metadata, finalText || undefined)
|
||||
}
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
lifecycle,
|
||||
reasoningItems: {
|
||||
...state.reasoningItems,
|
||||
[item.id]: {
|
||||
...reasoningItem,
|
||||
open: false,
|
||||
encryptedContent: item.encrypted_content ?? reasoningItem.encryptedContent,
|
||||
},
|
||||
},
|
||||
},
|
||||
events,
|
||||
] satisfies StepResult
|
||||
}
|
||||
if (!state.lifecycle.reasoning.has(item.id)) {
|
||||
const lifecycle = Lifecycle.stepStart(state.lifecycle, events)
|
||||
events.push(LLMEvent.reasoningStart({ id: item.id, providerMetadata: metadata }))
|
||||
events.push(LLMEvent.reasoningEnd({ id: item.id, providerMetadata: metadata }))
|
||||
return [{ ...state, lifecycle }, events] satisfies StepResult
|
||||
events.push(
|
||||
LLMEvent.reasoningEnd({
|
||||
id: item.id,
|
||||
providerMetadata: metadata,
|
||||
text: itemText,
|
||||
}),
|
||||
)
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
lifecycle,
|
||||
reasoningItems: {
|
||||
...state.reasoningItems,
|
||||
[item.id]: {
|
||||
open: false,
|
||||
encryptedContent: item.encrypted_content,
|
||||
summaryParts: { 0: "concluded" },
|
||||
deltaIndexes: new Set(),
|
||||
},
|
||||
},
|
||||
},
|
||||
events,
|
||||
] satisfies StepResult
|
||||
}
|
||||
return [
|
||||
{ ...state, lifecycle: Lifecycle.reasoningEnd(state.lifecycle, events, item.id, metadata) },
|
||||
@@ -1129,38 +1317,38 @@ const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* (
|
||||
})
|
||||
|
||||
const onResponseFinish = Effect.fn("OpenResponses.onResponseFinish")(function* (state: ParserState, event: Event) {
|
||||
const reconciled =
|
||||
event.type === "response.completed"
|
||||
? yield* Effect.reduce(
|
||||
event.response?.output ?? [],
|
||||
() => [state, NO_EVENTS] satisfies StepResult,
|
||||
([current, events], item) => {
|
||||
const id = item.id ?? (item.type === "function_call" ? item.call_id : undefined)
|
||||
if (
|
||||
id === undefined ||
|
||||
((item.type !== "function_call" || !current.tools[id]) &&
|
||||
(item.type !== "reasoning" || !current.reasoningItems[id]))
|
||||
)
|
||||
return Effect.succeed([current, events] satisfies StepResult)
|
||||
return onOutputItemDone(current, { type: "response.output_item.done", item }).pipe(
|
||||
Effect.map(([next, emitted]) => [next, [...events, ...emitted]] satisfies StepResult),
|
||||
)
|
||||
},
|
||||
)
|
||||
: ([state, NO_EVENTS] satisfies StepResult)
|
||||
const current = reconciled[0]
|
||||
// Some compatible providers omit output_item.done even after completing the response.
|
||||
const pending =
|
||||
event.type === "response.completed"
|
||||
? yield* ToolStream.finishAll(current.id, current.tools)
|
||||
: { tools: current.tools, events: NO_EVENTS }
|
||||
const events: LLMEvent[] = [...reconciled[1], ...pending.events]
|
||||
const hasFunctionCall =
|
||||
pending.events.some((event) => LLMEvent.is.toolCall(event) || LLMEvent.is.toolInputError(event)) ||
|
||||
current.hasFunctionCall
|
||||
let current = state
|
||||
const events: LLMEvent[] = []
|
||||
if (event.type === "response.completed") {
|
||||
for (const item of event.response?.output ?? []) {
|
||||
if (item.type !== "compaction" && item.type !== "function_call") continue
|
||||
if (item.type === "compaction") {
|
||||
// Terminal recovery cannot insert a checkpoint before already-emitted content.
|
||||
if (state.lifecycle.stepStarted && !state.completedCompactions.has(item.id ?? ""))
|
||||
return yield* ProviderShared.eventError(
|
||||
state.id,
|
||||
"Cannot recover a compaction checkpoint after output has been emitted",
|
||||
)
|
||||
}
|
||||
if (item.type === "function_call" && !current.tools[item.id ?? item.call_id ?? ""]) continue
|
||||
const [next, emitted] = yield* onOutputItemDone(current, item)
|
||||
current = next
|
||||
events.push(...emitted)
|
||||
}
|
||||
// Some compatible providers omit output_item.done even after completing the response.
|
||||
const pending = yield* ToolStream.finishAll(current.id, current.tools)
|
||||
current = {
|
||||
...current,
|
||||
tools: pending.tools,
|
||||
hasFunctionCall:
|
||||
current.hasFunctionCall ||
|
||||
pending.events.some((event) => LLMEvent.is.toolCall(event) || LLMEvent.is.toolInputError(event)),
|
||||
}
|
||||
events.push(...pending.events)
|
||||
}
|
||||
const lifecycle = Lifecycle.finish(current.lifecycle, events, {
|
||||
reason: {
|
||||
normalized: mapFinishReason(event, hasFunctionCall),
|
||||
normalized: mapFinishReason(event, current.hasFunctionCall),
|
||||
raw: event.response?.incomplete_details?.reason,
|
||||
},
|
||||
usage: mapUsage(event.response?.usage, current.providerMetadataKey),
|
||||
@@ -1172,7 +1360,7 @@ const onResponseFinish = Effect.fn("OpenResponses.onResponseFinish")(function* (
|
||||
})
|
||||
: undefined,
|
||||
})
|
||||
return [{ ...current, lifecycle, hasFunctionCall, tools: pending.tools }, events] satisfies StepResult
|
||||
return [{ ...current, lifecycle }, events] satisfies StepResult
|
||||
})
|
||||
|
||||
// Build the prettiest summary available from whatever the provider supplied.
|
||||
@@ -1256,6 +1444,13 @@ export const step = (state: ParserState, input: Event) => {
|
||||
if (event.type === "response.output_item.added") {
|
||||
if (event.item?.type === "message" && event.item.id === undefined)
|
||||
return ProviderShared.eventError(state.id, `${event.type} message is missing id`)
|
||||
if (
|
||||
event.item &&
|
||||
isReasoningItem(event.item) &&
|
||||
state.reasoningItems[event.item.id] === undefined &&
|
||||
state.lifecycle.reasoning.size > 0
|
||||
)
|
||||
return ProviderShared.eventError(state.id, `${event.type} started reasoning before the previous item ended`)
|
||||
const id = event.item?.id ?? (event.item?.type === "function_call" ? event.item.call_id : undefined)
|
||||
return Effect.succeed(
|
||||
onOutputItemAdded(
|
||||
@@ -1273,7 +1468,7 @@ export const step = (state: ParserState, input: Event) => {
|
||||
if (event.type === "response.output_item.done") {
|
||||
if (event.item?.type === "message" && event.item.id === undefined)
|
||||
return ProviderShared.eventError(state.id, `${event.type} message is missing id`)
|
||||
return onOutputItemDone(state, event)
|
||||
return onOutputItemDone(state, event.item)
|
||||
}
|
||||
if (event.type === "response.completed" || event.type === "response.incomplete") return onResponseFinish(state, event)
|
||||
if (event.type === "response.failed") return providerFailure(event, `${state.name} response failed`)
|
||||
@@ -1300,15 +1495,17 @@ export const step = (state: ParserState, input: Event) => {
|
||||
* implementations compose this baseline with their own tools and event variants.
|
||||
*/
|
||||
export const initial = (request: LLMRequest, extension: Extension = BASE): ParserState => ({
|
||||
provider: request.model.provider,
|
||||
completedCompactions: new Set<string>(),
|
||||
id: extension.id,
|
||||
name: extension.name,
|
||||
providerMetadataKey: request.model.route.providerMetadataKey ?? "openresponses",
|
||||
providerMetadataKey: metadataKey(request.model),
|
||||
hasFunctionCall: false,
|
||||
tools: ToolStream.empty<string>(),
|
||||
completedTools: new Set<string>(),
|
||||
lifecycle: Lifecycle.initial(),
|
||||
outputItems: {},
|
||||
messageItems: new Set<string>(),
|
||||
messagePhases: {},
|
||||
message: undefined,
|
||||
reasoningItems: {},
|
||||
})
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import { Tool } from "@opencode-ai/schema/tool"
|
||||
import { Route } from "../route/client.js"
|
||||
import { Auth } from "../route/auth.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { HttpTransport } from "../route/transport/index.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import {
|
||||
@@ -245,6 +246,8 @@ export const OpenAIChatEvent = Schema.StructWithRest(
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
)
|
||||
export type OpenAIChatEvent = Schema.Schema.Type<typeof OpenAIChatEvent>
|
||||
const DONE = "[DONE]" as const
|
||||
const OpenAIChatStreamEvent = Schema.Union([Schema.Literal(DONE), Protocol.jsonEvent(OpenAIChatEvent)])
|
||||
type OpenAIChatRequestMessage = LLMRequest["messages"][number]
|
||||
|
||||
interface PendingToolDelta {
|
||||
@@ -812,7 +815,12 @@ const mapFinishReason = Effect.fn("OpenAIChat.mapFinishReason")(function* (event
|
||||
case "tool_calls":
|
||||
return "tool-calls" as const
|
||||
default:
|
||||
return "unknown" as const
|
||||
return yield* new AIError({
|
||||
reason: new UnknownProviderError({
|
||||
message: `Provider finish_reason: ${reason}`,
|
||||
body: ProviderShared.encodeJson(event),
|
||||
}),
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
@@ -997,33 +1005,12 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
|
||||
lifecycle = Lifecycle.reasoningStart(lifecycle, events, "reasoning-0", deltaMetadata)
|
||||
const reasoningEmitted = state.reasoningEmitted || lifecycle.reasoning.has("reasoning-0")
|
||||
|
||||
if (delta?.content) {
|
||||
lifecycle = Lifecycle.reasoningEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
"reasoning-0",
|
||||
reasoningMetadata(
|
||||
state.providerMetadataKey,
|
||||
reasoningField,
|
||||
reasoningDetailsObserved ? state.reasoningDetails : undefined,
|
||||
),
|
||||
)
|
||||
lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.content)
|
||||
}
|
||||
// Reasoning is one response-wide channel: it stays open alongside text and
|
||||
// refusal output so late reasoning deltas and details join the same block,
|
||||
// and `finishEvents` closes it once with the complete metadata.
|
||||
if (delta?.content) lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.content)
|
||||
|
||||
if (delta?.refusal) {
|
||||
lifecycle = Lifecycle.reasoningEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
"reasoning-0",
|
||||
reasoningMetadata(
|
||||
state.providerMetadataKey,
|
||||
reasoningField,
|
||||
reasoningDetailsObserved ? state.reasoningDetails : undefined,
|
||||
),
|
||||
)
|
||||
lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.refusal)
|
||||
}
|
||||
if (delta?.refusal) lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.refusal)
|
||||
|
||||
// Compatible providers may omit indexes. Prefer durable identity, then use
|
||||
// batch position for parallel deltas or the latest call for sparse chunks.
|
||||
@@ -1072,17 +1059,25 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
|
||||
events.push(...result.events)
|
||||
}
|
||||
|
||||
if (finishReason !== undefined && state.finishReason === undefined && Object.keys(pendingTools).length > 0)
|
||||
const incompleteTools = finishReason?.normalized === "content-filter" || finishReason?.normalized === "length"
|
||||
if (
|
||||
finishReason !== undefined &&
|
||||
!incompleteTools &&
|
||||
state.finishReason === undefined &&
|
||||
Object.keys(pendingTools).length
|
||||
)
|
||||
return yield* ProviderShared.eventError(
|
||||
ADAPTER,
|
||||
"OpenAI Chat tool call delta is missing id or name",
|
||||
ProviderShared.encodeJson(event),
|
||||
)
|
||||
|
||||
// Finalize accumulated tool inputs eagerly when finish_reason arrives so
|
||||
// valid calls and malformed local calls settle independently.
|
||||
// Filtering or truncation terminates the response without confirming pending tool calls.
|
||||
const finished =
|
||||
finishReason !== undefined && state.finishReason === undefined && Object.keys(tools).length > 0
|
||||
finishReason !== undefined &&
|
||||
!incompleteTools &&
|
||||
state.finishReason === undefined &&
|
||||
Object.keys(tools).length > 0
|
||||
? yield* ToolStream.finishAll(ADAPTER, tools)
|
||||
: undefined
|
||||
|
||||
@@ -1129,10 +1124,12 @@ const finishEvents = Effect.fn("OpenAIChat.finishEvents")(function* (state: Pars
|
||||
state.finishReason.normalized === "stop" && hasToolCalls ? "tool-calls" : state.finishReason.normalized,
|
||||
}
|
||||
: { normalized: hasToolCalls ? ("tool-calls" as const) : ("stop" as const) }
|
||||
// Snapshot details at publish time so the emitted event never observes later
|
||||
// mutation of the accumulated `reasoningDetails` array.
|
||||
const metadata = reasoningMetadata(
|
||||
state.providerMetadataKey,
|
||||
state.reasoningField,
|
||||
state.reasoningDetailsObserved ? state.reasoningDetails : undefined,
|
||||
state.reasoningDetailsObserved ? [...state.reasoningDetails] : undefined,
|
||||
)
|
||||
const started =
|
||||
state.reasoningDetailsObserved && !state.reasoningEmitted
|
||||
@@ -1166,7 +1163,7 @@ export const protocol = Protocol.make({
|
||||
from: fromRequest,
|
||||
},
|
||||
stream: {
|
||||
event: Protocol.jsonEvent(OpenAIChatEvent),
|
||||
event: OpenAIChatStreamEvent,
|
||||
initial: (request) => ({
|
||||
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
|
||||
tools: ToolStream.empty<number>(),
|
||||
@@ -1180,12 +1177,14 @@ export const protocol = Protocol.make({
|
||||
nextToolIndex: 0,
|
||||
requireFinishReason: request.model.compatibility?.requireFinishReason ?? true,
|
||||
}),
|
||||
step,
|
||||
step: (state: ParserState, event) => (event === DONE ? Effect.succeed([state, []] as const) : step(state, event)),
|
||||
terminal: (event) => event === DONE,
|
||||
onHalt: finishEvents,
|
||||
},
|
||||
})
|
||||
|
||||
export const httpTransport = HttpTransport.sseJson.with<OpenAIChatBody>()
|
||||
export const framing = Framing.sseWithDone
|
||||
export const httpTransport = HttpTransport.sseJson.with<OpenAIChatBody>().with({ framing })
|
||||
|
||||
export const route = Route.make({
|
||||
id: ADAPTER,
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { Route, type RouteRoutedLanguageModelInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import * as OpenAIChat from "./openai-chat.js"
|
||||
|
||||
const ADAPTER = "openai-compatible-chat"
|
||||
@@ -19,7 +18,7 @@ export const route = Route.make({
|
||||
providerMetadataKey: "openai",
|
||||
protocol: OpenAIChat.protocol,
|
||||
endpoint: Endpoint.path("/chat/completions"),
|
||||
framing: Framing.sse,
|
||||
framing: OpenAIChat.framing,
|
||||
})
|
||||
|
||||
export * as OpenAICompatibleChat from "./openai-compatible-chat.js"
|
||||
|
||||
@@ -5,13 +5,14 @@ import { Auth } from "../route/auth.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { HttpTransport } from "../route/transport/index.js"
|
||||
import { LLMRequest, type JsonSchema, type ToolDefinition } from "../schema/index.js"
|
||||
import type { LLMRequest, JsonSchema, ToolDefinition } from "../schema/index.js"
|
||||
import { OpenResponses } from "./open-responses.js"
|
||||
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
|
||||
import { OpenAIImage } from "./utils/openai-image.js"
|
||||
import { ResponsesHostedTools } from "./utils/responses-hosted-tools.js"
|
||||
import { ToolSchemaProjection } from "./utils/tool-schema.js"
|
||||
import { OpenResponsesChannel } from "./open-responses-channel.js"
|
||||
import { ResponsesCompaction } from "./utils/responses-compaction.js"
|
||||
|
||||
const ADAPTER = "openai-responses"
|
||||
const NAME = "OpenAI Responses"
|
||||
@@ -20,6 +21,14 @@ const WEBSOCKET_ROTATE_AFTER_MS = 55 * 60 * 1000
|
||||
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
||||
export const PATH = OpenResponses.PATH
|
||||
|
||||
export const ContextManagement = Schema.Array(
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("compaction"),
|
||||
compactThreshold: Schema.optional(Schema.Int.check(Schema.isGreaterThan(0))),
|
||||
}),
|
||||
)
|
||||
export type ContextManagement = typeof ContextManagement.Type
|
||||
|
||||
const OpenAIResponsesImageGenerationTool = Schema.Struct({
|
||||
type: Schema.tag("image_generation"),
|
||||
action: Schema.optional(Schema.Literals(["auto", "generate", "edit"])),
|
||||
@@ -78,6 +87,14 @@ const OpenAIResponsesCoreFields = {
|
||||
input: Schema.Array(Schema.Union([OpenResponses.InputItem, OpenAIResponsesHostedToolItem])),
|
||||
tools: optionalArray(OpenAIResponsesTools),
|
||||
tool_choice: Schema.optional(OpenAIResponsesToolChoice),
|
||||
context_management: Schema.optional(
|
||||
Schema.Array(
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("compaction"),
|
||||
compact_threshold: Schema.optional(Schema.Int.check(Schema.isGreaterThan(0))),
|
||||
}),
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
const OpenAIResponsesBody = Schema.Struct({
|
||||
@@ -125,15 +142,14 @@ const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>, tool
|
||||
const decodeBody = ProviderShared.validateWith(Schema.decodeUnknownEffect(OpenAIResponsesBody))
|
||||
|
||||
const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request: LLMRequest) {
|
||||
const body = yield* OpenResponses.fromRequestWithExtension(
|
||||
LLMRequest.update(request, { tools: [], toolChoice: undefined }),
|
||||
extension,
|
||||
)
|
||||
const management = yield* ProviderShared.validateWith(
|
||||
Schema.decodeUnknownEffect(Schema.UndefinedOr(ContextManagement)),
|
||||
)(request.providerOptions?.contextManagement)
|
||||
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
|
||||
const parallelToolCalls = OpenResponses.resolveParallelToolCalls(request)
|
||||
return yield* decodeBody({
|
||||
...body,
|
||||
...(parallelToolCalls === undefined ? {} : { parallel_tool_calls: parallelToolCalls }),
|
||||
...(yield* OpenResponses.lowerConversation(request, extension)),
|
||||
...OpenResponses.lowerGeneration(request),
|
||||
context_management: management?.map((edit) => ({ type: edit.type, compact_threshold: edit.compactThreshold })),
|
||||
tools:
|
||||
request.tools.length === 0
|
||||
? undefined
|
||||
@@ -141,7 +157,8 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request:
|
||||
lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility)),
|
||||
),
|
||||
tool_choice:
|
||||
body.tool_choice ?? (request.toolChoice ? yield* lowerToolChoice(request.toolChoice, request.tools) : undefined),
|
||||
OpenResponses.allowedToolChoice(request) ??
|
||||
(request.toolChoice ? yield* lowerToolChoice(request.toolChoice, request.tools) : undefined),
|
||||
})
|
||||
})
|
||||
|
||||
@@ -223,6 +240,7 @@ export const transport = channelTransport({
|
||||
})
|
||||
|
||||
export const route = Route.make({
|
||||
compact: ResponsesCompaction.make(extension),
|
||||
id: ADAPTER,
|
||||
provider: "openai",
|
||||
providerMetadataKey: "openai",
|
||||
|
||||
@@ -207,15 +207,16 @@ export const errorText = (error: unknown) => {
|
||||
|
||||
/**
|
||||
* `framing` step for Server-Sent Events. Decodes UTF-8, runs the SSE channel
|
||||
* decoder, optionally filters named events, and drops empty / `[DONE]`
|
||||
* keep-alive events so the protocol event schema sees one JSON string per
|
||||
* element. Retry control events are ignored without interrupting the stream.
|
||||
* decoder, optionally filters named events, and drops empty events. `[DONE]`
|
||||
* is dropped by default or retained for protocols that use it as their stream
|
||||
* boundary. Retry control events are ignored without interrupting the stream.
|
||||
* Decoder failures become provider output errors so the public error channel
|
||||
* stays `AIError`.
|
||||
*/
|
||||
export const sseFraming = (
|
||||
bytes: Stream.Stream<Uint8Array, AIError>,
|
||||
events?: ReadonlySet<string>,
|
||||
includeDone = false,
|
||||
): Stream.Stream<string, AIError> =>
|
||||
bytes.pipe(
|
||||
Stream.decodeText(),
|
||||
@@ -240,7 +241,7 @@ export const sseFraming = (
|
||||
(event) =>
|
||||
(events === undefined || events.has(event.event)) &&
|
||||
event.data.length > 0 &&
|
||||
(event.data !== "[DONE]" || (events !== undefined && event.event !== "message")),
|
||||
(event.data !== "[DONE]" || includeDone || (events !== undefined && event.event !== "message")),
|
||||
),
|
||||
Stream.map((event) => event.data),
|
||||
)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { Effect, Encoding, Schema } from "effect"
|
||||
import type { MediaPart } from "../../schema/index.js"
|
||||
import { ProviderShared } from "../shared.js"
|
||||
|
||||
@@ -57,6 +57,16 @@ const documentBlock = (name: string, format: DocumentFormat, bytes: string): Doc
|
||||
},
|
||||
})
|
||||
|
||||
const mediaBase64 = Effect.fn("BedrockMedia.mediaBase64")(function* (part: MediaPart) {
|
||||
const media = ProviderShared.normalizeMedia(part)
|
||||
const bytes = yield* Effect.fromResult(Encoding.decodeBase64(media.base64)).pipe(
|
||||
Effect.mapError((cause) =>
|
||||
ProviderShared.invalidRequest("Bedrock Converse media data must be valid base64", cause),
|
||||
),
|
||||
)
|
||||
return Encoding.encodeBase64(bytes)
|
||||
})
|
||||
|
||||
// Route by MIME. Known image/document formats lower into a typed block; anything
|
||||
// else fails with a clear error instead of silently degrading to a malformed
|
||||
// document block. Image MIME types not in `IMAGE_FORMATS` (e.g. `image/svg+xml`)
|
||||
@@ -66,8 +76,7 @@ export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart)
|
||||
const mime = part.mediaType.toLowerCase()
|
||||
const imageFormat = IMAGE_FORMATS[mime as keyof typeof IMAGE_FORMATS]
|
||||
if (imageFormat) {
|
||||
const media = ProviderShared.normalizeMedia(part)
|
||||
return { image: { format: imageFormat, source: { bytes: media.base64 } } } satisfies ImageBlock
|
||||
return { image: { format: imageFormat, source: { bytes: yield* mediaBase64(part) } } } satisfies ImageBlock
|
||||
}
|
||||
if (mime.startsWith("image/"))
|
||||
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`)
|
||||
@@ -75,8 +84,7 @@ export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart)
|
||||
if (documentFormat) {
|
||||
if (!part.filename)
|
||||
return yield* ProviderShared.invalidRequest("Bedrock Converse document media requires a filename")
|
||||
const media = ProviderShared.normalizeMedia(part)
|
||||
return documentBlock(part.filename, documentFormat, media.base64)
|
||||
return documentBlock(part.filename, documentFormat, yield* mediaBase64(part))
|
||||
}
|
||||
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
|
||||
})
|
||||
|
||||
@@ -62,22 +62,31 @@ export const reasoningEnd = (
|
||||
events: LLMEvent[],
|
||||
id: string,
|
||||
providerMetadata?: ProviderMetadata,
|
||||
/** Authoritative complete value; replaces accumulated deltas when present. */
|
||||
text?: string,
|
||||
): State => {
|
||||
if (!state.reasoning.has(id)) return state
|
||||
const stepped = stepStart(state, events)
|
||||
events.push(LLMEvent.reasoningEnd({ id, providerMetadata }))
|
||||
events.push(LLMEvent.reasoningEnd({ id, text, providerMetadata }))
|
||||
const reasoning = new Set(stepped.reasoning)
|
||||
reasoning.delete(id)
|
||||
return { ...stepped, reasoning }
|
||||
}
|
||||
|
||||
export const textEnd = (state: State, events: LLMEvent[], id: string, providerMetadata?: ProviderMetadata): State => {
|
||||
export const textEnd = (
|
||||
state: State,
|
||||
events: LLMEvent[],
|
||||
id: string,
|
||||
providerMetadata?: ProviderMetadata,
|
||||
/** Authoritative complete value; replaces accumulated deltas when present. */
|
||||
text?: string,
|
||||
): State => {
|
||||
if (!state.text.has(id)) return state
|
||||
const stepped = stepStart(state, events)
|
||||
events.push(LLMEvent.textEnd({ id, providerMetadata }))
|
||||
const text = new Set(stepped.text)
|
||||
text.delete(id)
|
||||
return { ...stepped, text }
|
||||
events.push(LLMEvent.textEnd({ id, text, providerMetadata }))
|
||||
const open = new Set(stepped.text)
|
||||
open.delete(id)
|
||||
return { ...stepped, text: open }
|
||||
}
|
||||
|
||||
const closeOpenBlocks = (state: State, events: LLMEvent[]): State => {
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import {
|
||||
AIError,
|
||||
InvalidProviderOutputError,
|
||||
CompactionPart,
|
||||
CompactionResponse,
|
||||
HttpOptions,
|
||||
LLMRequest,
|
||||
Message,
|
||||
type ContentPart,
|
||||
mergeJsonRecords,
|
||||
} from "../../schema/index.js"
|
||||
import type { CompactOperation } from "../../route/client.js"
|
||||
import { Endpoint } from "../../route/endpoint.js"
|
||||
import { RequestExecutor } from "../../route/executor.js"
|
||||
import { HttpTransport } from "../../route/transport/index.js"
|
||||
import { OpenResponses } from "../open-responses.js"
|
||||
import { JsonObject, optionalNull, ProviderShared } from "../shared.js"
|
||||
|
||||
const Body = Schema.Struct({
|
||||
model: Schema.String,
|
||||
input: Schema.Array(Schema.Unknown),
|
||||
instructions: optionalNull(Schema.String),
|
||||
previous_response_id: optionalNull(Schema.String),
|
||||
service_tier: optionalNull(Schema.String),
|
||||
prompt_cache_key: optionalNull(Schema.String),
|
||||
prompt_cache_retention: optionalNull(Schema.String),
|
||||
prompt_cache_options: optionalNull(
|
||||
Schema.Struct({ mode: Schema.optional(Schema.String), ttl: Schema.optional(Schema.String) }),
|
||||
),
|
||||
})
|
||||
|
||||
const Text = Schema.Union([OpenResponses.OpenResponsesInputText, OpenResponses.OpenResponsesOutputText])
|
||||
const File = Schema.Union([
|
||||
Schema.Struct({
|
||||
...OpenResponses.OpenResponsesInputFile.fields,
|
||||
file_url: Schema.String,
|
||||
file_data: Schema.optional(Schema.Never),
|
||||
}),
|
||||
Schema.Struct({
|
||||
...OpenResponses.OpenResponsesInputFile.fields,
|
||||
file_data: Schema.String,
|
||||
file_url: Schema.optional(Schema.Never),
|
||||
}),
|
||||
])
|
||||
const MessageFields = {
|
||||
type: Schema.Literal("message"),
|
||||
id: Schema.optional(Schema.String),
|
||||
status: Schema.optional(Schema.String),
|
||||
phase: Schema.optional(OpenResponses.MessagePhase),
|
||||
}
|
||||
const Response = Schema.Struct({
|
||||
object: Schema.Literal("response.compaction"),
|
||||
output: Schema.Array(
|
||||
Schema.Union([
|
||||
OpenResponses.CompactionItem,
|
||||
OpenResponses.OpenResponsesReasoningItem,
|
||||
Schema.Struct({
|
||||
...MessageFields,
|
||||
role: Schema.Literal("user"),
|
||||
content: Schema.Array(Schema.Union([Text, OpenResponses.OpenResponsesInputImage, File])).check(
|
||||
Schema.isMinLength(1),
|
||||
),
|
||||
}),
|
||||
Schema.Struct({
|
||||
...MessageFields,
|
||||
role: Schema.Literal("assistant"),
|
||||
content: Schema.Array(Text).check(Schema.isMinLength(1)),
|
||||
}),
|
||||
]),
|
||||
),
|
||||
usage: Schema.optional(Schema.StructWithRest(OpenResponses.OpenResponsesUsage, [JsonObject])),
|
||||
})
|
||||
|
||||
export const make = (extension: OpenResponses.Extension): CompactOperation =>
|
||||
Effect.fn("ResponsesCompaction.execute")(function* (request, executor, options) {
|
||||
const route = request.model.route
|
||||
const native = yield* OpenResponses.lowerConversation(request, extension)
|
||||
const body = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(Body))(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
...native,
|
||||
service_tier: request.providerOptions?.serviceTier,
|
||||
prompt_cache_key: ProviderShared.promptCacheKey(request),
|
||||
},
|
||||
request.http?.body,
|
||||
),
|
||||
)
|
||||
const url = Endpoint.render(route.endpoint, { request, body: native })
|
||||
url.pathname = `${url.pathname.replace(/\/$/, "")}/compact`
|
||||
const parts = yield* HttpTransport.jsonRequestParts({
|
||||
request: LLMRequest.update(request, {
|
||||
http: request.http === undefined ? undefined : new HttpOptions({ ...request.http, body: undefined }),
|
||||
}),
|
||||
body,
|
||||
endpoint: Endpoint.path(url.toString()),
|
||||
auth: route.auth,
|
||||
encodeBody: Schema.encodeSync(Schema.fromJsonString(Body)),
|
||||
})
|
||||
const response = yield* executor.execute(
|
||||
ProviderShared.jsonPost({ url: parts.url, body: parts.bodyText, headers: parts.headers }),
|
||||
options?.http,
|
||||
)
|
||||
const text = yield* RequestExecutor.responseStream(response).pipe(
|
||||
Stream.decodeText(),
|
||||
Stream.runFold(
|
||||
() => "",
|
||||
(text, chunk) => text + chunk,
|
||||
),
|
||||
)
|
||||
const invalid = (message: string, cause?: unknown) =>
|
||||
new AIError({
|
||||
reason: new InvalidProviderOutputError({
|
||||
route: route.id,
|
||||
message,
|
||||
body: text,
|
||||
cause,
|
||||
http: RequestExecutor.responseHttp(response),
|
||||
}),
|
||||
})
|
||||
const result = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Response))(text).pipe(
|
||||
Effect.mapError((cause) => invalid("Invalid compaction response", cause)),
|
||||
)
|
||||
if (!result.output.some((item) => item.type === "compaction"))
|
||||
return yield* invalid("Compaction response did not contain a checkpoint")
|
||||
return new CompactionResponse({
|
||||
messages: result.output.map((item) => toMessage(item, request.model)),
|
||||
usage: OpenResponses.mapUsage(result.usage, OpenResponses.metadataKey(request.model)),
|
||||
})
|
||||
})
|
||||
|
||||
function toMessage(item: (typeof Response.Type.output)[number], model: LLMRequest["model"]): Message {
|
||||
if (item.type === "compaction")
|
||||
return Message.assistant(
|
||||
CompactionPart.make({ provider: model.provider, id: item.id ?? undefined, encrypted: item.encrypted_content }),
|
||||
)
|
||||
|
||||
const key = OpenResponses.metadataKey(model)
|
||||
if (item.type === "reasoning") {
|
||||
const summary = item.summary.length ? item.summary : [{ text: "" }]
|
||||
return Message.assistant(
|
||||
summary.map((part) => ({
|
||||
type: "reasoning" as const,
|
||||
text: part.text,
|
||||
providerMetadata: { [key]: { itemId: item.id, reasoningEncryptedContent: item.encrypted_content } },
|
||||
})),
|
||||
)
|
||||
}
|
||||
|
||||
return Message.make({
|
||||
role: item.role,
|
||||
providerMetadata: { [key]: { itemId: item.id, type: item.type, status: item.status, phase: item.phase } },
|
||||
content: item.content.map((part): ContentPart => {
|
||||
if (part.type === "input_text" || part.type === "output_text") return { type: "text", text: part.text }
|
||||
if (part.type === "input_image")
|
||||
return {
|
||||
type: "media",
|
||||
data: part.image_url,
|
||||
mediaType: /^data:([^;,]+)/.exec(part.image_url)?.[1] ?? "image/*",
|
||||
providerMetadata: part.detail === undefined ? undefined : { [key]: { detail: part.detail } },
|
||||
}
|
||||
const data = part.file_url === undefined ? part.file_data : part.file_url
|
||||
return {
|
||||
type: "media",
|
||||
data,
|
||||
filename: part.filename,
|
||||
mediaType: /^data:([^;,]+)/.exec(data)?.[1] ?? "application/octet-stream",
|
||||
providerMetadata: part.detail === undefined ? undefined : { [key]: { detail: part.detail } },
|
||||
}
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
export * as ResponsesCompaction from "./responses-compaction.js"
|
||||
@@ -4,6 +4,7 @@ import type { LLMRequest } from "../schema/index.js"
|
||||
import { OpenResponses } from "./open-responses.js"
|
||||
import { JsonObject, optionalNull, ProviderShared } from "./shared.js"
|
||||
import { ResponsesHostedTools } from "./utils/responses-hosted-tools.js"
|
||||
import { ResponsesCompaction } from "./utils/responses-compaction.js"
|
||||
|
||||
const ADAPTER = "xai-responses"
|
||||
const NAME = "xAI Responses"
|
||||
@@ -44,6 +45,10 @@ const extension = {
|
||||
|
||||
const decodeBody = ProviderShared.validateWith(Schema.decodeUnknownEffect(XAIResponsesBody))
|
||||
const fromRequest = Effect.fn("XAIResponses.fromRequest")(function* (request: LLMRequest) {
|
||||
if (request.providerOptions?.contextManagement !== undefined)
|
||||
return yield* ProviderShared.invalidRequest(
|
||||
"xAI requires explicit compaction through LLMClient.compact; automatic context management is not supported",
|
||||
)
|
||||
return yield* decodeBody(yield* OpenResponses.fromRequestWithExtension(request, extension))
|
||||
})
|
||||
|
||||
@@ -84,4 +89,6 @@ export const protocol = Protocol.make({
|
||||
},
|
||||
})
|
||||
|
||||
export const compact = ResponsesCompaction.make(extension)
|
||||
|
||||
export * as XAIResponses from "./xai-responses.js"
|
||||
|
||||
@@ -37,6 +37,7 @@ const patterns = [
|
||||
/too large for model with \d+ maximum context length/i,
|
||||
/prompt has [\d,]+ tokens?, but the configured context size is [\d,]+ tokens?/i,
|
||||
/model_context_window_exceeded/i,
|
||||
/range of input length should be/i,
|
||||
/too many tokens/i,
|
||||
/token limit exceeded/i,
|
||||
/request_too_large/i,
|
||||
@@ -59,6 +60,7 @@ export const isContextOverflowFailure = (failure: unknown) =>
|
||||
|
||||
const decodeJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown))
|
||||
const QUOTA_CODES = new Set(["insufficient_quota", "usage_not_included", "billing_error"])
|
||||
const AUTH_CODES = new Set(["authentication_error", "permission_error"])
|
||||
const SERVER_CODES = new Set([
|
||||
"api_error",
|
||||
"internal_error",
|
||||
@@ -74,7 +76,8 @@ const INVALID_REQUEST_CODES = new Set(["invalid_prompt", "invalid_request_error"
|
||||
const RATE_LIMIT_TEXT = /rate increased too quickly|rate[-_\s]?limit|too[_\s]?many[_\s]?requests/i
|
||||
const QUOTA_TEXT = /insufficient[-_\s]?quota|quota[-_\s]?exceeded/i
|
||||
const CONTENT_POLICY_TEXT = /content[-_\s]?policy|content_filter|safety/i
|
||||
const NETWORK_ERROR_TEXT = /network[-_\s]error/i
|
||||
const SERVER_ERROR_TEXT =
|
||||
/\b(?:try again|(?:please |you can )?retry (?:the |this |your )?request|try (?:the |this |your )?request again|(?:currently |temporarily )?at capacity|overloaded|temporarily unavailable|service[-_\s]?unavailable|(?:server|internal)[-_\s]?error|server (?:is )?busy|provider returned (?:an )?error|resource[-_\s]?exhausted|upstream (?:connect|connection|request)|request buffer limit while retrying upstream)\b/i
|
||||
|
||||
export interface ProviderFailure {
|
||||
readonly message: string
|
||||
@@ -90,8 +93,12 @@ export interface ProviderFailure {
|
||||
readonly rateLimit?: HttpRateLimitDetails | undefined
|
||||
}
|
||||
|
||||
// Keep HTTP failures and provider-reported stream failures on one typed path so
|
||||
// session retry policy never needs provider-specific string matching.
|
||||
// Classification records affirmative evidence about a failure. Deterministic
|
||||
// failures need positive identification (a 4xx status, quota/auth/policy
|
||||
// signals); anything unrecognized stays UnknownProvider, which the session
|
||||
// retry policy treats as retry-eligible because transient failures arrive in
|
||||
// unpredictable shapes while deterministic rejections almost always carry a
|
||||
// status or known code.
|
||||
export function classifyProviderFailure(input: ProviderFailure): AIError["reason"] {
|
||||
const details = { message: input.message, body: input.rawBody, http: input.http, cause: input.cause }
|
||||
const body = input.rawBody ?? ""
|
||||
@@ -116,46 +123,36 @@ export function classifyProviderFailure(input: ProviderFailure): AIError["reason
|
||||
if (CONTENT_POLICY_TEXT.test(text)) return new ContentPolicyError(details)
|
||||
if (codes.some((code) => QUOTA_CODES.has(code)) || (input.status === 429 && QUOTA_TEXT.test(text)))
|
||||
return new QuotaExceededError(details)
|
||||
if (input.status === 401) return new AuthenticationError({ ...details, kind: "invalid" })
|
||||
if (input.status === 403) return new AuthenticationError({ ...details, kind: "insufficient-permissions" })
|
||||
if (codes.includes("authentication_error")) return new AuthenticationError({ ...details, kind: "invalid" })
|
||||
if (codes.includes("permission_error"))
|
||||
return new AuthenticationError({ ...details, kind: "insufficient-permissions" })
|
||||
if (input.status === 401 || input.status === 403 || codes.some((code) => AUTH_CODES.has(code)))
|
||||
return new AuthenticationError(details)
|
||||
if (
|
||||
codes.some((code) => code.includes("rate_limit") || code === "too_many_requests" || code === "throttlingexception")
|
||||
input.status === 429 ||
|
||||
codes.some(
|
||||
(code) => code.includes("rate_limit") || code === "too_many_requests" || code === "throttlingexception",
|
||||
) ||
|
||||
RATE_LIMIT_TEXT.test(text)
|
||||
)
|
||||
return new RateLimitError({
|
||||
...details,
|
||||
retryAfterMs: input.retryAfterMs,
|
||||
rateLimit: input.rateLimit,
|
||||
})
|
||||
if (RATE_LIMIT_TEXT.test(text))
|
||||
return new RateLimitError({
|
||||
...details,
|
||||
retryAfterMs: input.retryAfterMs,
|
||||
rateLimit: input.rateLimit,
|
||||
})
|
||||
if (NETWORK_ERROR_TEXT.test(text)) return new ProviderInternalError(details)
|
||||
if (codes.some((code) => SERVER_CODES.has(code) || code.includes("exhausted") || code.includes("unavailable")))
|
||||
return new ProviderInternalError({
|
||||
...details,
|
||||
retryAfterMs: input.retryAfterMs,
|
||||
})
|
||||
if (input.status === 429) {
|
||||
return new RateLimitError({
|
||||
...details,
|
||||
retryAfterMs: input.retryAfterMs,
|
||||
rateLimit: input.rateLimit,
|
||||
})
|
||||
}
|
||||
if (input.status === 408 || input.status === 409 || (input.status !== undefined && input.status >= 500))
|
||||
if (
|
||||
input.status === 408 ||
|
||||
input.status === 409 ||
|
||||
(input.status !== undefined && input.status >= 500) ||
|
||||
((input.status === undefined || input.status < 400) &&
|
||||
!codes.some((code) => INVALID_REQUEST_CODES.has(code)) &&
|
||||
SERVER_ERROR_TEXT.test(text)) ||
|
||||
codes.some((code) => SERVER_CODES.has(code) || code.includes("exhausted") || code.includes("unavailable"))
|
||||
)
|
||||
return new ProviderInternalError({
|
||||
...details,
|
||||
retryAfterMs: input.retryAfterMs,
|
||||
})
|
||||
if (codes.some((code) => INVALID_REQUEST_CODES.has(code))) return new InvalidRequestError(details)
|
||||
if (input.status === 400 || input.status === 404 || input.status === 413 || input.status === 422)
|
||||
return new InvalidRequestError(details)
|
||||
// Any remaining 4xx is a deterministic rejection of this request.
|
||||
if (input.status !== undefined && input.status >= 400 && input.status < 500) return new InvalidRequestError(details)
|
||||
return new UnknownProviderError(details)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { LanguageModel, ProviderOptions } from "./schema/index.js"
|
||||
import type { CompactOperation } from "./route/client.js"
|
||||
|
||||
export interface Settings extends Readonly<Record<string, unknown>> {
|
||||
readonly baseURL?: string
|
||||
@@ -9,8 +10,9 @@ export interface Settings extends Readonly<Record<string, unknown>> {
|
||||
export interface Definition<
|
||||
ProviderSettings extends Settings = Settings,
|
||||
Options extends ProviderOptions = ProviderOptions,
|
||||
Compact extends CompactOperation | undefined = CompactOperation | undefined,
|
||||
> {
|
||||
readonly model: (modelID: string, settings: ProviderSettings) => LanguageModel<Options>
|
||||
readonly model: (modelID: string, settings: ProviderSettings) => LanguageModel<Options, Compact>
|
||||
}
|
||||
|
||||
export * as ProviderPackage from "./provider-package.js"
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import type { RouteDefaultsInput } from "../route/client.js"
|
||||
import type { Route, RouteDefaultsInput } from "../route/client.js"
|
||||
import { Auth } from "../route/auth.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import * as BedrockConverse from "../protocols/bedrock-converse.js"
|
||||
import type { BedrockCredentials } from "../protocols/bedrock-converse.js"
|
||||
import { BedrockMessages } from "../protocols/bedrock-messages.js"
|
||||
import type { AnthropicMessages } from "../protocols/anthropic-messages.js"
|
||||
|
||||
export const id = ProviderID.make("amazon-bedrock")
|
||||
|
||||
@@ -25,38 +27,40 @@ export interface Settings extends ProviderPackage.Settings {
|
||||
readonly region?: string
|
||||
readonly topP?: number
|
||||
}
|
||||
export const routes = [BedrockConverse.route]
|
||||
export const routes = [BedrockConverse.route, BedrockMessages.route]
|
||||
|
||||
const bedrockBaseURL = (region: string) => `https://bedrock-runtime.${region}.amazonaws.com`
|
||||
|
||||
const configuredRoute = (input: Config) => {
|
||||
const configuredRoute = <Body, Prepared>(route: Route<Body, Prepared>, input: Config) => {
|
||||
const { apiKey, credentials, region, baseURL, ...rest } = input
|
||||
const resolvedRegion = region ?? credentials?.region ?? "us-east-1"
|
||||
return BedrockConverse.route.with({
|
||||
return route.with({
|
||||
...rest,
|
||||
provider: id,
|
||||
providerMetadataKey: "bedrock",
|
||||
providerMetadataKey: route.providerMetadataKey,
|
||||
endpoint: { baseURL: baseURL ?? bedrockBaseURL(resolvedRegion) },
|
||||
auth: apiKey === undefined ? BedrockConverse.sigV4Auth(credentials) : Auth.bearer(apiKey),
|
||||
})
|
||||
}
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const route = configuredRoute(input)
|
||||
const route = configuredRoute(BedrockConverse.route, input)
|
||||
const messages = configuredRoute(BedrockMessages.route, input)
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => route.model({ id: modelID }),
|
||||
messages: (modelID: string | ModelID) => messages.model<AnthropicMessages.ProviderOptionsInput>({ id: modelID }),
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
export const model: ProviderPackage.Definition<Settings>["model"] = (modelID, settings) => {
|
||||
const config = (settings: Settings): Config => {
|
||||
if (settings.auth === "bearer" && settings.apiKey === undefined)
|
||||
throw new Error("Amazon Bedrock bearer auth requires apiKey")
|
||||
if (settings.auth === "sigv4" && settings.apiKey !== undefined)
|
||||
throw new Error("Amazon Bedrock SigV4 auth does not accept apiKey")
|
||||
return configure({
|
||||
return {
|
||||
apiKey: settings.auth === "sigv4" ? undefined : settings.apiKey,
|
||||
baseURL: settings.baseURL,
|
||||
credentials: settings.credentials,
|
||||
@@ -64,5 +68,13 @@ export const model: ProviderPackage.Definition<Settings>["model"] = (modelID, se
|
||||
headers: settings.headers === undefined ? undefined : { ...settings.headers },
|
||||
http: settings.body === undefined ? undefined : { body: { ...settings.body } },
|
||||
region: settings.region,
|
||||
}).model(modelID)
|
||||
}
|
||||
}
|
||||
|
||||
export const model: ProviderPackage.Definition<Settings>["model"] = (modelID, settings) =>
|
||||
configure(config(settings)).model(modelID)
|
||||
export const messagesModel: ProviderPackage.Definition<
|
||||
Settings & { readonly providerOptions?: AnthropicMessages.ProviderOptionsInput },
|
||||
AnthropicMessages.ProviderOptionsInput
|
||||
>["model"] = (modelID, settings) =>
|
||||
configure({ ...config(settings), providerOptions: settings.providerOptions }).messages(modelID)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export { messagesModel as model } from "../amazon-bedrock.js"
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Headers } from "effect/unstable/http"
|
||||
import { Auth } from "../route/auth.js"
|
||||
import { type AtLeastOne, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import type { Route as RouteDef, RouteDefaultsInput } from "../route/client.js"
|
||||
import type { Route, RouteDefaultsInput, CompactOperation } from "../route/client.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import * as OpenAIChat from "../protocols/openai-chat.js"
|
||||
@@ -102,7 +102,11 @@ const auth = (input: Config) => {
|
||||
)
|
||||
}
|
||||
|
||||
const configuredRoute = <Body, Prepared>(route: RouteDef<Body, Prepared>, input: Config, modelID: string | ModelID) =>
|
||||
const configuredRoute = <Body, Prepared, Compact extends CompactOperation | undefined>(
|
||||
route: Route<Body, Prepared, Compact>,
|
||||
input: Config,
|
||||
modelID: string | ModelID,
|
||||
) =>
|
||||
route.with({
|
||||
auth: auth(input),
|
||||
endpoint: endpoint(input, modelID),
|
||||
@@ -161,10 +165,11 @@ const config = (settings: Settings): Config => {
|
||||
throw new Error("Azure requires resourceName or baseURL")
|
||||
}
|
||||
|
||||
export const responsesModel: ProviderPackage.Definition<Settings, OpenAIProviderOptionsInput>["model"] = (
|
||||
modelID,
|
||||
settings,
|
||||
) => configure(config(settings)).responses(modelID)
|
||||
export const responsesModel: ProviderPackage.Definition<
|
||||
Settings,
|
||||
OpenAIProviderOptionsInput,
|
||||
CompactOperation
|
||||
>["model"] = (modelID, settings) => configure(config(settings)).responses(modelID)
|
||||
export const chatModel: ProviderPackage.Definition<Settings, OpenAIProviderOptionsInput>["model"] = (
|
||||
modelID,
|
||||
settings,
|
||||
|
||||
@@ -57,7 +57,9 @@ const route = Route.make({
|
||||
}),
|
||||
endpoint: Endpoint.path(({ request }) => `/${request.model.id}:streamRawPredict`),
|
||||
auth: Auth.none,
|
||||
framing: AnthropicMessages.framing,
|
||||
transport: AnthropicMessages.transport<
|
||||
Omit<AnthropicMessages.AnthropicMessagesBody, "model"> & { readonly anthropic_version: typeof VERSION }
|
||||
>(),
|
||||
headers: () => ({ "anthropic-version": HEADER_VERSION }),
|
||||
})
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ import { ProviderShared } from "../protocols/shared.js"
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { ProviderID, type ModelID, type LLMRequest } from "../schema/index.js"
|
||||
import { profiles } from "./openai-compatible-profile.js"
|
||||
@@ -75,7 +74,7 @@ export const route = Route.make({
|
||||
providerMetadataKey: "openai",
|
||||
protocol,
|
||||
endpoint: Endpoint.path("/chat/completions", { baseURL: profiles.groq.baseURL }),
|
||||
framing: Framing.sse,
|
||||
framing: OpenAIChat.framing,
|
||||
})
|
||||
|
||||
export const configure = (input: LanguageModelOptions = {}) => {
|
||||
|
||||
@@ -13,6 +13,7 @@ export * as GoogleVertexChat from "./google-vertex-chat.js"
|
||||
export * as GoogleVertexMessages from "./google-vertex-messages.js"
|
||||
export * as GoogleVertexResponses from "./google-vertex-responses.js"
|
||||
export * as Groq from "./groq.js"
|
||||
export * as Mistral from "./mistral.js"
|
||||
export * as OpenAI from "./openai.js"
|
||||
export * as OpenAICompatible from "./openai-compatible.js"
|
||||
export * as OpenAICompatibleResponses from "./openai-compatible-responses.js"
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { MistralChat } from "../protocols/mistral-chat.js"
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import type { RouteDefaultsInput } from "../route/client.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
|
||||
export const id = ProviderID.make("mistral")
|
||||
|
||||
export type ProviderOptions = MistralChat.ProviderOptionsInput
|
||||
|
||||
export type LanguageModelOptions = Omit<RouteDefaultsInput, "providerOptions"> &
|
||||
ProviderAuthOption<"optional"> & {
|
||||
readonly baseURL?: string
|
||||
readonly providerOptions?: ProviderOptions
|
||||
}
|
||||
|
||||
export interface Settings extends ProviderPackage.Settings {
|
||||
readonly apiKey?: string
|
||||
readonly baseURL?: string
|
||||
readonly providerOptions?: ProviderOptions
|
||||
}
|
||||
|
||||
export const route = MistralChat.route
|
||||
export const routes = [route]
|
||||
|
||||
export const configure = (input: LanguageModelOptions = {}) => {
|
||||
const { apiKey: _apiKey, auth: _auth, baseURL, ...defaults } = input
|
||||
const configured = route.with({
|
||||
...defaults,
|
||||
endpoint: { baseURL: baseURL ?? MistralChat.DEFAULT_BASE_URL },
|
||||
auth: AuthOptions.bearer(input, "MISTRAL_API_KEY"),
|
||||
})
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => configured.model<ProviderOptions>({ id: modelID }),
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
export const model: ProviderPackage.Definition<Settings, ProviderOptions>["model"] = (modelID, settings) =>
|
||||
configure({
|
||||
apiKey: settings.apiKey,
|
||||
baseURL: settings.baseURL,
|
||||
headers: settings.headers === undefined ? undefined : { ...settings.headers },
|
||||
http: settings.body === undefined ? undefined : { body: { ...settings.body } },
|
||||
providerOptions: settings.providerOptions,
|
||||
}).model(modelID)
|
||||
|
||||
export * as Mistral from "./mistral.js"
|
||||
@@ -1,10 +1,12 @@
|
||||
import { mergeProviderOptions, type ProviderOptions } from "../schema/index.js"
|
||||
import type { OpenAIServiceTier } from "../protocols/utils/openai-options.js"
|
||||
import type { Options } from "../protocols/utils/open-responses-options.js"
|
||||
import type { ContextManagement } from "../protocols/openai-responses.js"
|
||||
|
||||
export type { OpenAIResponseIncludable, OpenAIServiceTier } from "../protocols/utils/openai-options.js"
|
||||
|
||||
export type OpenAIOptionsInput = Omit<Options, "serviceTier"> & {
|
||||
readonly contextManagement?: ContextManagement
|
||||
readonly serviceTier?: OpenAIServiceTier
|
||||
readonly [key: string]: unknown
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import type { Route, RouteDefaultsInput } from "../route/client.js"
|
||||
import type { Route, RouteDefaultsInput, CompactOperation } from "../route/client.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { HttpOptions, ProviderID, ToolDefinition, mergeHttpOptions, type ModelID } from "../schema/index.js"
|
||||
import * as OpenAIChat from "../protocols/openai-chat.js"
|
||||
@@ -73,7 +73,10 @@ const defaults = (input: Config) => {
|
||||
return rest
|
||||
}
|
||||
|
||||
const configuredRoute = <Body, Prepared>(route: Route<Body, Prepared>, input: Config) =>
|
||||
const configuredRoute = <Body, Prepared, Compact extends CompactOperation | undefined>(
|
||||
route: Route<Body, Prepared, Compact>,
|
||||
input: Config,
|
||||
) =>
|
||||
route.with({
|
||||
auth: auth(input),
|
||||
endpoint: { baseURL: input.baseURL, query: input.queryParams },
|
||||
@@ -129,7 +132,10 @@ const config = (settings: Settings): Config => {
|
||||
}
|
||||
}
|
||||
|
||||
export const model: ProviderPackage.Definition<Settings, OpenAIProviderOptionsInput>["model"] = (modelID, settings) => {
|
||||
export const model: ProviderPackage.Definition<Settings, OpenAIProviderOptionsInput, CompactOperation>["model"] = (
|
||||
modelID,
|
||||
settings,
|
||||
) => {
|
||||
return configure(config(settings)).responses(modelID)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { ProviderID, type CacheHint, type ModelID } from "../schema/index.js"
|
||||
@@ -167,7 +166,7 @@ export const route = Route.make({
|
||||
providerMetadataKey: "openrouter",
|
||||
protocol,
|
||||
endpoint: Endpoint.path("/chat/completions", { baseURL: profile.baseURL }),
|
||||
framing: Framing.sse,
|
||||
framing: OpenAIChat.framing,
|
||||
})
|
||||
|
||||
export const routes = [route]
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Route, type RouteDefaultsInput, type CompactOperation } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
|
||||
import * as OpenAICompatibleProfiles from "./openai-compatible-profile.js"
|
||||
@@ -13,7 +13,7 @@ import type { ProviderPackage } from "../provider-package.js"
|
||||
|
||||
export const id = ProviderID.make("xai")
|
||||
|
||||
export type XAIProviderOptionsInput = OpenAIOptionsInput
|
||||
export type XAIProviderOptionsInput = OpenAIOptionsInput & { readonly contextManagement?: never }
|
||||
|
||||
export type LanguageModelOptions = Omit<RouteDefaultsInput, "providerOptions"> &
|
||||
ProviderAuthOption<"optional"> & {
|
||||
@@ -32,6 +32,7 @@ export type { XAIImageOptions } from "../protocols/xai-images.js"
|
||||
const RESPONSES_WEBSOCKET_ROTATE_AFTER_MS = 24 * 60 * 1000
|
||||
|
||||
const responsesRoute = Route.make({
|
||||
compact: XAIResponses.compact,
|
||||
id: "openai-responses",
|
||||
provider: id,
|
||||
providerMetadataKey: "xai",
|
||||
@@ -102,7 +103,10 @@ export const configure = (input: LanguageModelOptions = {}) => {
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
export const model: ProviderPackage.Definition<Settings, XAIProviderOptionsInput>["model"] = (modelID, settings) =>
|
||||
export const model: ProviderPackage.Definition<Settings, XAIProviderOptionsInput, CompactOperation>["model"] = (
|
||||
modelID,
|
||||
settings,
|
||||
) =>
|
||||
configure({
|
||||
apiKey: settings.apiKey,
|
||||
baseURL: settings.baseURL,
|
||||
|
||||
@@ -139,7 +139,7 @@ const toAIError = (error: AuthError): AIError => {
|
||||
return new AIError({
|
||||
reason:
|
||||
error instanceof MissingCredentialError
|
||||
? new AuthenticationError({ message: error.message, cause: error, kind: "missing" })
|
||||
? new AuthenticationError({ message: error.message, cause: error })
|
||||
: new InvalidRequestError({ message: `Failed to resolve auth config: ${error.message}`, cause: error }),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7,11 +7,13 @@ import { HttpTransport } from "./transport/index.js"
|
||||
import type { HttpMiddleware, Transport, TransportRuntime, WebSocketChannelExecutor } from "./transport/index.js"
|
||||
import type { Protocol } from "./protocol.js"
|
||||
import { applyCachePolicy } from "../cache-policy.js"
|
||||
import { normalizeToolHistory } from "../tool-history.js"
|
||||
import { sanitizeSurrogates } from "../utils/sanitize.js"
|
||||
import * as ProviderShared from "../protocols/shared.js"
|
||||
import type { ProtocolID, ProviderOptions } from "../schema/index.js"
|
||||
import {
|
||||
AIError,
|
||||
CompactionResponse,
|
||||
AIErrorReason,
|
||||
GenerationOptions,
|
||||
HttpOptions,
|
||||
@@ -33,7 +35,12 @@ export interface RouteBody<Body> {
|
||||
readonly from: (request: LLMRequest) => Effect.Effect<Body, AIError>
|
||||
}
|
||||
|
||||
export interface Route<Body, Prepared = unknown> {
|
||||
export interface Route<
|
||||
Body,
|
||||
Prepared = unknown,
|
||||
Compact extends CompactOperation | undefined = CompactOperation | undefined,
|
||||
> {
|
||||
readonly compact: Compact
|
||||
readonly id: string
|
||||
readonly provider?: ProviderID
|
||||
/** ProviderMetadata namespace emitted and consumed by this route. */
|
||||
@@ -41,13 +48,15 @@ export interface Route<Body, Prepared = unknown> {
|
||||
readonly protocol: ProtocolID
|
||||
readonly endpoint: Endpoint.Definition<Body>
|
||||
readonly auth: Auth.Definition
|
||||
/** Deployment headers resolved once for every operation, before transport authentication. */
|
||||
readonly headers?: (input: { readonly request: LLMRequest }) => Record<string, string>
|
||||
readonly transport: Transport<Body, Prepared, unknown>
|
||||
readonly defaults: RouteDefaults
|
||||
readonly body: RouteBody<Body>
|
||||
readonly with: (patch: RoutePatch<Body, Prepared>) => Route<Body, Prepared>
|
||||
readonly with: (patch: RoutePatch<Body, Prepared>) => Route<Body, Prepared, Compact>
|
||||
readonly model: <Options extends ProviderOptions = ProviderOptions>(
|
||||
input: RouteMappedLanguageModelInput,
|
||||
) => LanguageModel<Options>
|
||||
) => LanguageModel<Options, Compact>
|
||||
readonly prepareTransport: (
|
||||
body: Body,
|
||||
request: LLMRequest,
|
||||
@@ -65,7 +74,11 @@ export interface Route<Body, Prepared = unknown> {
|
||||
// Normal call sites use `OpenAIChat.route`; callers only need body types
|
||||
// when preparing a request with a protocol-specific type assertion.
|
||||
// oxlint-disable-next-line typescript-eslint/no-explicit-any
|
||||
export type AnyRoute = Route<any, any>
|
||||
export type AnyRoute<Compact extends CompactOperation | undefined = CompactOperation | undefined> = Route<
|
||||
any,
|
||||
any,
|
||||
Compact
|
||||
>
|
||||
|
||||
export type HttpOptionsInput = HttpOptions.Input
|
||||
|
||||
@@ -98,15 +111,15 @@ export interface RoutePatch<Body, Prepared> extends RouteDefaultsInput {
|
||||
|
||||
type RouteMappedLanguageModelInput = RouteLanguageModelInput | RouteRoutedLanguageModelInput
|
||||
|
||||
const makeRouteLanguageModel = <Options extends ProviderOptions = ProviderOptions>(
|
||||
route: AnyRoute,
|
||||
const makeRouteLanguageModel = <Options extends ProviderOptions, Compact extends CompactOperation | undefined>(
|
||||
route: AnyRoute<Compact>,
|
||||
mapped: RouteMappedLanguageModelInput,
|
||||
) => {
|
||||
const provider = route.provider ?? ("provider" in mapped ? mapped.provider : undefined)
|
||||
if (!provider) throw new Error(`Route.model(${route.id}) requires a provider`)
|
||||
if (!endpointBaseURL(route.endpoint))
|
||||
throw new Error(`Route.model(${route.id}) requires an endpoint baseURL — configure it on the route first`)
|
||||
return LanguageModel.make<Options>({
|
||||
return LanguageModel.make<Options, Compact>({
|
||||
...mapped,
|
||||
provider,
|
||||
route,
|
||||
@@ -149,6 +162,10 @@ export const httpOptions = (input: HttpOptionsInput | undefined) => {
|
||||
}
|
||||
|
||||
export interface Interface {
|
||||
readonly compact: (
|
||||
request: CompactionRequest,
|
||||
options?: Pick<StreamOptions, "http">,
|
||||
) => Effect.Effect<CompactionResponse, AIError>
|
||||
readonly stream: StreamMethod
|
||||
readonly generate: GenerateMethod
|
||||
}
|
||||
@@ -166,24 +183,38 @@ export interface GenerateMethod {
|
||||
(request: LLMRequest, options?: StreamOptions): Effect.Effect<LLMResponse, AIError>
|
||||
}
|
||||
|
||||
export type CompactOperation = (
|
||||
request: LLMRequest,
|
||||
executor: RequestExecutor.Interface,
|
||||
options?: Pick<StreamOptions, "http">,
|
||||
) => Effect.Effect<CompactionResponse, AIError>
|
||||
|
||||
export type CompactionRequest = LLMRequest<LanguageModel<ProviderOptions, CompactOperation>>
|
||||
|
||||
export const canCompact = (request: LLMRequest): request is CompactionRequest =>
|
||||
request.model.route.compact !== undefined
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/LLMClient") {}
|
||||
|
||||
const resolveRequestOptions = (request: LLMRequest) => {
|
||||
const routeDefaults = request.model.route.defaults
|
||||
const modelDefaults = request.model.defaults
|
||||
const generation = mergeGenerationOptions(routeDefaults.generation, modelDefaults?.generation, request.generation)
|
||||
return LLMRequest.update(request, {
|
||||
const messages = normalizeToolHistory(request.messages)
|
||||
const normalized = messages === request.messages ? request : LLMRequest.update(request, { messages })
|
||||
const routeDefaults = normalized.model.route.defaults
|
||||
const modelDefaults = normalized.model.defaults
|
||||
const generation = mergeGenerationOptions(routeDefaults.generation, modelDefaults?.generation, normalized.generation)
|
||||
return LLMRequest.update(normalized, {
|
||||
generation: generation ?? new GenerationOptions({}),
|
||||
providerOptions: mergeProviderOptions(
|
||||
routeDefaults.providerOptions,
|
||||
modelDefaults?.providerOptions,
|
||||
request.providerOptions,
|
||||
normalized.providerOptions,
|
||||
),
|
||||
http: mergeHttpOptions(routeDefaults.http, modelDefaults?.http, request.http),
|
||||
http: mergeHttpOptions(routeDefaults.http, modelDefaults?.http, normalized.http),
|
||||
})
|
||||
}
|
||||
|
||||
export interface MakeInput<Body, Frame, Event, State> {
|
||||
readonly compact?: CompactOperation
|
||||
/** Route id used in diagnostics and prepared request metadata. */
|
||||
readonly id: string
|
||||
/** Provider identity for route-owned model construction. */
|
||||
@@ -205,6 +236,7 @@ export interface MakeInput<Body, Frame, Event, State> {
|
||||
}
|
||||
|
||||
export interface MakeTransportInput<Body, Prepared, Frame, Event, State> {
|
||||
readonly compact?: CompactOperation
|
||||
/** Route id used in diagnostics and prepared request metadata. */
|
||||
readonly id: string
|
||||
/** Provider identity for route-owned model construction. */
|
||||
@@ -280,12 +312,14 @@ function makeFromTransport<Body, Prepared, Frame, Event, State>(
|
||||
|
||||
const build = (routeInput: BuiltRouteInput): Route<Body, Prepared> => {
|
||||
const route: Route<Body, Prepared> = {
|
||||
compact: routeInput.compact,
|
||||
id: routeInput.id,
|
||||
provider: routeInput.provider === undefined ? undefined : ProviderID.make(routeInput.provider),
|
||||
providerMetadataKey: routeInput.providerMetadataKey,
|
||||
protocol: protocol.id,
|
||||
endpoint: routeInput.endpoint,
|
||||
auth: routeInput.auth ?? Auth.none,
|
||||
headers: routeInput.headers,
|
||||
transport: routeInput.transport,
|
||||
defaults: routeInput.defaults ?? {},
|
||||
body: protocol.body,
|
||||
@@ -307,7 +341,7 @@ function makeFromTransport<Body, Prepared, Frame, Event, State>(
|
||||
})
|
||||
},
|
||||
model: <Options extends ProviderOptions = ProviderOptions>(input: RouteMappedLanguageModelInput) =>
|
||||
makeRouteLanguageModel<Options>(route, input),
|
||||
makeRouteLanguageModel<Options, CompactOperation | undefined>(route, input),
|
||||
prepareTransport: (body, request, options) =>
|
||||
routeInput.transport.prepare({
|
||||
body,
|
||||
@@ -315,7 +349,6 @@ function makeFromTransport<Body, Prepared, Frame, Event, State>(
|
||||
endpoint: routeInput.endpoint,
|
||||
auth: routeInput.auth ?? Auth.none,
|
||||
encodeBody,
|
||||
headers: routeInput.headers,
|
||||
middleware: options?.http,
|
||||
webSocket: options?.webSocket,
|
||||
}),
|
||||
@@ -405,6 +438,12 @@ function makeFromTransport<Body, Prepared, Frame, Event, State>(
|
||||
return build({ ...input, defaults: mergeRouteDefaults(undefined, input.defaults ?? {}) })
|
||||
}
|
||||
|
||||
export function make<Body, Prepared, Frame, Event, State>(
|
||||
input: MakeTransportInput<Body, Prepared, Frame, Event, State> & { readonly compact: CompactOperation },
|
||||
): Route<Body, Prepared, CompactOperation>
|
||||
export function make<Body, Frame, Event, State>(
|
||||
input: MakeInput<Body, Frame, Event, State> & { readonly compact: CompactOperation },
|
||||
): Route<Body, HttpTransport.HttpPrepared<Frame>, CompactOperation>
|
||||
export function make<Body, Prepared, Frame, Event, State>(
|
||||
input: MakeTransportInput<Body, Prepared, Frame, Event, State>,
|
||||
): Route<Body, Prepared>
|
||||
@@ -432,6 +471,7 @@ export function make<Body, Prepared, Frame, Event, State>(
|
||||
if ("transport" in input) return makeFromTransport(input)
|
||||
const protocol = input.protocol
|
||||
return makeFromTransport({
|
||||
compact: input.compact,
|
||||
id: input.id,
|
||||
provider: input.provider,
|
||||
providerMetadataKey: input.providerMetadataKey,
|
||||
@@ -444,9 +484,19 @@ export function make<Body, Prepared, Frame, Event, State>(
|
||||
})
|
||||
}
|
||||
|
||||
const compile = Effect.fn("LLM.compile")(function* (request: LLMRequest, options?: StreamOptions) {
|
||||
const prepareRequest = (request: LLMRequest) => {
|
||||
const original = applyCachePolicy(resolveRequestOptions(request))
|
||||
const resolved = LLMRequest.update(original, sanitizeSurrogates({ ...LLMRequest.input(original), model: undefined }))
|
||||
const sanitized = LLMRequest.update(original, sanitizeSurrogates({ ...LLMRequest.input(original), model: undefined }))
|
||||
const tools = [...new Map(sanitized.tools.map((tool) => [tool.name, tool])).values()]
|
||||
const resolved = tools.length === sanitized.tools.length ? sanitized : LLMRequest.update(sanitized, { tools })
|
||||
const headers = resolved.model.route.headers?.({ request: resolved })
|
||||
return headers === undefined
|
||||
? resolved
|
||||
: LLMRequest.update(resolved, { http: mergeHttpOptions(new HttpOptions({ headers }), resolved.http) })
|
||||
}
|
||||
|
||||
const compile = Effect.fn("LLM.compile")(function* (request: LLMRequest, options?: StreamOptions) {
|
||||
const resolved = prepareRequest(request)
|
||||
const route = resolved.model.route
|
||||
|
||||
const body = yield* route.body
|
||||
@@ -505,6 +555,15 @@ export function generate(request: LLMRequest, options?: StreamOptions): Effect.E
|
||||
})
|
||||
}
|
||||
|
||||
export const compact = (
|
||||
request: CompactionRequest,
|
||||
options?: Pick<StreamOptions, "http">,
|
||||
): Effect.Effect<CompactionResponse, AIError, Service> =>
|
||||
Effect.gen(function* () {
|
||||
const client = yield* Service
|
||||
return yield* client.compact(request, options)
|
||||
})
|
||||
|
||||
export const streamRequest = (request: LLMRequest, options?: StreamOptions) =>
|
||||
Stream.unwrap(
|
||||
Effect.gen(function* () {
|
||||
@@ -515,16 +574,29 @@ export const streamRequest = (request: LLMRequest, options?: StreamOptions) =>
|
||||
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const stream = streamRequestWith({
|
||||
http: yield* RequestExecutor.Service,
|
||||
const executor = yield* RequestExecutor.Service
|
||||
const stream = streamRequestWith({ http: executor })
|
||||
return Service.of({
|
||||
stream,
|
||||
generate: generateWith(stream),
|
||||
compact: (request, options) =>
|
||||
Effect.suspend(() => {
|
||||
const operation = request.model.route.compact
|
||||
if (!operation)
|
||||
return ProviderShared.invalidRequest(
|
||||
`${request.model.provider}/${request.model.route.id} does not support explicit compaction`,
|
||||
)
|
||||
return operation(prepareRequest(request), executor, options)
|
||||
}),
|
||||
})
|
||||
return Service.of({ stream, generate: generateWith(stream) })
|
||||
}),
|
||||
)
|
||||
|
||||
export const Route = { make } as const
|
||||
|
||||
export const LLMClient = {
|
||||
canCompact,
|
||||
compact,
|
||||
Service,
|
||||
layer,
|
||||
stream,
|
||||
|
||||
@@ -8,8 +8,8 @@ import type { AIError } from "../schema/index.js"
|
||||
* `Framing` is the byte-stream-shaped seam between transport and protocol:
|
||||
*
|
||||
* - SSE (`Framing.sse`) — UTF-8 decode the body, run the SSE channel decoder,
|
||||
* drop empty / `[DONE]` keep-alives. Each emitted frame is the JSON `data:`
|
||||
* payload of one event.
|
||||
* and emit the `data:` payload of each non-empty event. The default drops
|
||||
* `[DONE]`; protocols that use it as a terminal select `sseWithDone`.
|
||||
* - AWS event stream — length-prefixed binary frames with CRC checksums.
|
||||
* Each emitted frame is one parsed binary event record.
|
||||
*
|
||||
@@ -26,6 +26,12 @@ export interface Definition<Frame> {
|
||||
/** Server-Sent Events framing. Used by every JSON-streaming HTTP provider. */
|
||||
export const sse: Definition<string> = { id: "sse", frame: ProviderShared.sseFraming }
|
||||
|
||||
/** Server-Sent Events framing that retains the conventional `[DONE]` sentinel. */
|
||||
export const sseWithDone: Definition<string> = {
|
||||
id: "sse",
|
||||
frame: (bytes) => ProviderShared.sseFraming(bytes, undefined, true),
|
||||
}
|
||||
|
||||
/** SSE framing restricted to protocol-recognized event names. */
|
||||
export const sseEvents = (events: ReadonlySet<string>): Definition<string> => ({
|
||||
id: "sse",
|
||||
|
||||
@@ -44,10 +44,7 @@ export class NoRouteError extends Schema.TaggedError<NoRouteError>("AI.Error.NoR
|
||||
|
||||
export class AuthenticationError extends Schema.TaggedError<AuthenticationError>("AI.Error.Authentication")(
|
||||
"Authentication",
|
||||
{
|
||||
...ReasonFields,
|
||||
kind: Schema.Literals(["missing", "invalid", "expired", "insufficient-permissions", "unknown"]),
|
||||
},
|
||||
ReasonFields,
|
||||
) {}
|
||||
|
||||
export class RateLimitError extends Schema.TaggedError<RateLimitError>("AI.Error.RateLimit")("RateLimit", {
|
||||
|
||||
@@ -3,6 +3,7 @@ import { LLM } from "@opencode-ai/schema/llm"
|
||||
import { ContentBlockID, ToolCallID } from "./ids.js"
|
||||
import {
|
||||
Message,
|
||||
CompactionPart,
|
||||
ProviderMetadata,
|
||||
ToolCallPart,
|
||||
ToolOutput,
|
||||
@@ -62,6 +63,8 @@ export { ProviderMetadata } from "./messages.js"
|
||||
* Matches the same escape-hatch field on `LLMEvent`.
|
||||
*/
|
||||
export class Usage extends Schema.Class<Usage>("AI.Usage")({
|
||||
/** Effective input size of the final message iteration, when reported; not billed totals. */
|
||||
contextTokens: Schema.optional(Schema.Number),
|
||||
inputTokens: Schema.optional(Schema.Number),
|
||||
outputTokens: Schema.optional(Schema.Number),
|
||||
nonCachedInputTokens: Schema.optional(Schema.Number),
|
||||
@@ -72,7 +75,7 @@ export class Usage extends Schema.Class<Usage>("AI.Usage")({
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}) {
|
||||
/**
|
||||
* Visible output tokens — `outputTokens` minus `reasoningTokens`, clamped
|
||||
* Non-reasoning output tokens (including compaction summaries) — `outputTokens` minus `reasoningTokens`, clamped
|
||||
* to zero. The one place subtraction happens in this contract; the clamp
|
||||
* means a provider reporting `reasoningTokens > outputTokens` produces a
|
||||
* harmless zero rather than a negative that crashes downstream schemas.
|
||||
@@ -88,6 +91,12 @@ export class Usage extends Schema.Class<Usage>("AI.Usage")({
|
||||
|
||||
export type UsageInput = Usage | ConstructorParameters<typeof Usage>[0]
|
||||
|
||||
/** A replacement context window. Replace prior history with these messages. */
|
||||
export class CompactionResponse extends Schema.Class<CompactionResponse>("LLM.CompactionResponse")({
|
||||
messages: Schema.Array(Message),
|
||||
usage: Schema.optional(Usage),
|
||||
}) {}
|
||||
|
||||
export const StepStart = Schema.Struct({
|
||||
type: Schema.tag("step-start"),
|
||||
index: Schema.Number,
|
||||
@@ -112,6 +121,8 @@ export type TextDelta = Schema.Schema.Type<typeof TextDelta>
|
||||
export const TextEnd = Schema.Struct({
|
||||
type: Schema.tag("text-end"),
|
||||
id: ContentBlockID,
|
||||
/** Authoritative complete value; replaces accumulated deltas when present. */
|
||||
text: Schema.optional(Schema.String),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}).annotate({ identifier: "LLM.Event.TextEnd" })
|
||||
export type TextEnd = Schema.Schema.Type<typeof TextEnd>
|
||||
@@ -134,6 +145,8 @@ export type ReasoningDelta = Schema.Schema.Type<typeof ReasoningDelta>
|
||||
export const ReasoningEnd = Schema.Struct({
|
||||
type: Schema.tag("reasoning-end"),
|
||||
id: ContentBlockID,
|
||||
/** Authoritative complete value; replaces accumulated deltas when present. */
|
||||
text: Schema.optional(Schema.String),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}).annotate({ identifier: "LLM.Event.ReasoningEnd" })
|
||||
export type ReasoningEnd = Schema.Schema.Type<typeof ReasoningEnd>
|
||||
@@ -237,6 +250,7 @@ export const ProviderErrorEvent = Schema.Struct({
|
||||
export type ProviderErrorEvent = Schema.Schema.Type<typeof ProviderErrorEvent>
|
||||
|
||||
const llmEventTagged = Schema.Union([
|
||||
CompactionPart,
|
||||
StepStart,
|
||||
TextStart,
|
||||
TextDelta,
|
||||
@@ -270,6 +284,7 @@ const toolCallID = (value: ToolCallID | string) => ToolCallID.make(value)
|
||||
* `events.filter(LLMEvent.guards["tool-call"])`.
|
||||
*/
|
||||
export const LLMEvent = Object.assign(llmEventTagged, {
|
||||
compaction: CompactionPart.make,
|
||||
stepStart: StepStart.make,
|
||||
textStart: (input: WithID<TextStart, ContentBlockID>) => TextStart.make({ ...input, id: contentBlockID(input.id) }),
|
||||
textDelta: (input: WithID<TextDelta, ContentBlockID>) => TextDelta.make({ ...input, id: contentBlockID(input.id) }),
|
||||
@@ -307,6 +322,7 @@ export const LLMEvent = Object.assign(llmEventTagged, {
|
||||
}),
|
||||
providerError: ProviderErrorEvent.make,
|
||||
is: {
|
||||
compaction: llmEventTagged.guards.compaction,
|
||||
stepStart: llmEventTagged.guards["step-start"],
|
||||
textStart: llmEventTagged.guards["text-start"],
|
||||
textDelta: llmEventTagged.guards["text-delta"],
|
||||
@@ -328,17 +344,32 @@ export const LLMEvent = Object.assign(llmEventTagged, {
|
||||
})
|
||||
export type LLMEvent = Schema.Schema.Type<typeof llmEventTagged>
|
||||
|
||||
/** Joins deltas per fragment, letting an authoritative end value replace that fragment's accumulated deltas. */
|
||||
const joinFragments = (
|
||||
events: ReadonlyArray<LLMEvent>,
|
||||
isDelta: (event: LLMEvent) => event is LLMEvent & { id: string; text: string },
|
||||
isEnd: (event: LLMEvent) => event is LLMEvent & { id: string; text?: string },
|
||||
) => {
|
||||
const order: string[] = []
|
||||
const parts = new Map<string, string>()
|
||||
for (const event of events) {
|
||||
if (isDelta(event)) {
|
||||
if (!parts.has(event.id)) order.push(event.id)
|
||||
parts.set(event.id, (parts.get(event.id) ?? "") + event.text)
|
||||
}
|
||||
if (isEnd(event) && event.text !== undefined) {
|
||||
if (!parts.has(event.id)) order.push(event.id)
|
||||
parts.set(event.id, event.text)
|
||||
}
|
||||
}
|
||||
return order.map((id) => parts.get(id)).join("")
|
||||
}
|
||||
|
||||
const responseText = (events: ReadonlyArray<LLMEvent>) =>
|
||||
events
|
||||
.filter(LLMEvent.is.textDelta)
|
||||
.map((event) => event.text)
|
||||
.join("")
|
||||
joinFragments(events, LLMEvent.is.textDelta, LLMEvent.is.textEnd)
|
||||
|
||||
const responseReasoning = (events: ReadonlyArray<LLMEvent>) =>
|
||||
events
|
||||
.filter(LLMEvent.is.reasoningDelta)
|
||||
.map((event) => event.text)
|
||||
.join("")
|
||||
joinFragments(events, LLMEvent.is.reasoningDelta, LLMEvent.is.reasoningEnd)
|
||||
|
||||
const responseUsage = (events: ReadonlyArray<LLMEvent>) =>
|
||||
events.reduce<Usage | undefined>(
|
||||
@@ -445,10 +476,11 @@ const reduceTextDelta = (state: ResponseState, event: TextDelta): ResponseState
|
||||
const reduceTextEnd = (state: ResponseState, event: TextEnd): ResponseState => {
|
||||
const current = state.textParts[event.id]
|
||||
if (!current) return state
|
||||
const text = event.text ?? current.text
|
||||
const providerMetadata = event.providerMetadata ?? current.providerMetadata
|
||||
return {
|
||||
...replaceContent(state, current.contentIndex, textContent(current.text, providerMetadata)),
|
||||
textParts: { ...state.textParts, [event.id]: { ...current, providerMetadata } },
|
||||
...replaceContent(state, current.contentIndex, textContent(text, providerMetadata)),
|
||||
textParts: { ...state.textParts, [event.id]: { ...current, text, providerMetadata } },
|
||||
}
|
||||
}
|
||||
|
||||
@@ -478,10 +510,11 @@ const reduceReasoningDelta = (state: ResponseState, event: ReasoningDelta): Resp
|
||||
const reduceReasoningEnd = (state: ResponseState, event: ReasoningEnd): ResponseState => {
|
||||
const current = state.reasoningParts[event.id]
|
||||
if (!current) return state
|
||||
const text = event.text ?? current.text
|
||||
const providerMetadata = event.providerMetadata ?? current.providerMetadata
|
||||
return {
|
||||
...replaceContent(state, current.contentIndex, reasoningContent(current.text, providerMetadata)),
|
||||
reasoningParts: { ...state.reasoningParts, [event.id]: { ...current, providerMetadata } },
|
||||
...replaceContent(state, current.contentIndex, reasoningContent(text, providerMetadata)),
|
||||
reasoningParts: { ...state.reasoningParts, [event.id]: { ...current, text, providerMetadata } },
|
||||
}
|
||||
}
|
||||
|
||||
@@ -542,6 +575,8 @@ const reduceToolCall = (state: ResponseState, event: ToolCall): ResponseState =>
|
||||
const reduceResponseState = (state: ResponseState, event: LLMEvent): ResponseState => {
|
||||
const next = appendEvent(state, event)
|
||||
switch (event.type) {
|
||||
case "compaction":
|
||||
return appendContent(next, event)
|
||||
case "text-start":
|
||||
return ensureText(next, event.id, event.providerMetadata)
|
||||
case "text-delta":
|
||||
@@ -579,12 +614,12 @@ export class LLMResponse extends Schema.Class<LLMResponse>("LLM.Response")({
|
||||
usage: Schema.optional(Usage),
|
||||
finishReason: FinishReasonDetails,
|
||||
}) {
|
||||
/** Concatenated assistant text assembled from streamed `text-delta` events. */
|
||||
/** Concatenated assistant text; each fragment's `text-end` value replaces its accumulated deltas when present. */
|
||||
get text() {
|
||||
return responseText(this.events)
|
||||
}
|
||||
|
||||
/** Concatenated reasoning text assembled from streamed `reasoning-delta` events. */
|
||||
/** Concatenated reasoning text; each fragment's `reasoning-end` value replaces its accumulated deltas when present. */
|
||||
get reasoning() {
|
||||
return responseReasoning(this.events)
|
||||
}
|
||||
|
||||
@@ -7,9 +7,10 @@ import {
|
||||
HttpOptions,
|
||||
JsonSchema,
|
||||
LanguageModelSchema,
|
||||
type LanguageModel,
|
||||
ProviderOptions,
|
||||
} from "./options.js"
|
||||
import { isRecord } from "../utils/record.js"
|
||||
import { ProviderID } from "./ids.js"
|
||||
|
||||
export const MessageRole = Schema.Literals(["system", "user", "assistant", "tool"])
|
||||
export type MessageRole = Schema.Schema.Type<typeof MessageRole>
|
||||
@@ -53,14 +54,10 @@ export const MediaPart = Schema.Struct({
|
||||
filename: Schema.optional(Schema.String),
|
||||
cache: Schema.optional(CacheHint),
|
||||
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}).annotate({ identifier: "LLM.Content.Media" })
|
||||
export type MediaPart = Schema.Schema.Type<typeof MediaPart>
|
||||
|
||||
const isToolResultValue = (value: unknown): value is ToolResultValue =>
|
||||
isRecord(value) &&
|
||||
(value.type === "text" || value.type === "json" || value.type === "error" || value.type === "content") &&
|
||||
"value" in value
|
||||
|
||||
const toolResultValueSchema = Schema.Union([
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("json"),
|
||||
@@ -80,6 +77,7 @@ const toolResultValueSchema = Schema.Union([
|
||||
}),
|
||||
]).annotate({ identifier: "LLM.ToolResult" })
|
||||
export type ToolResultValue = Schema.Schema.Type<typeof toolResultValueSchema>
|
||||
const isToolResultValue = Schema.is(toolResultValueSchema)
|
||||
|
||||
export const ToolResultValue = Object.assign(toolResultValueSchema, {
|
||||
is: isToolResultValue,
|
||||
@@ -190,9 +188,40 @@ export const ReasoningPart = Schema.Struct({
|
||||
}).annotate({ identifier: "LLM.Content.Reasoning" })
|
||||
export type ReasoningPart = Schema.Schema.Type<typeof ReasoningPart>
|
||||
|
||||
export const ContentPart = Schema.Union([TextPart, MediaPart, ToolCallPart, ToolResultPart, ReasoningPart]).pipe(
|
||||
Schema.toTaggedUnion("type"),
|
||||
)
|
||||
/** A provider-generated context checkpoint, distinct from visible assistant text. */
|
||||
type CompactionContent =
|
||||
| { readonly encrypted: string; readonly text?: never }
|
||||
| { readonly text: string | null; readonly encrypted?: never }
|
||||
|
||||
const compactionPartSchema = Schema.Struct({
|
||||
type: Schema.Literal("compaction"),
|
||||
provider: ProviderID,
|
||||
id: Schema.optional(Schema.String),
|
||||
encrypted: Schema.optional(Schema.String),
|
||||
/** Null means the provider failed to produce a summary; prior history must be retained. */
|
||||
text: Schema.optional(Schema.NullOr(Schema.String)),
|
||||
})
|
||||
.pipe(
|
||||
Schema.refine(
|
||||
(part): part is typeof part & CompactionContent => (part.encrypted !== undefined) !== (part.text !== undefined),
|
||||
{ message: "Compaction requires either encrypted content or a summary" },
|
||||
),
|
||||
)
|
||||
.annotate({ identifier: "LLM.Content.Compaction" })
|
||||
export type CompactionPart = typeof compactionPartSchema.Type
|
||||
export const CompactionPart = Object.assign(compactionPartSchema, {
|
||||
make: (input: Omit<CompactionPart, "type" | "encrypted" | "text"> & CompactionContent): CompactionPart =>
|
||||
Schema.decodeUnknownSync(compactionPartSchema)({ type: "compaction", ...input }),
|
||||
})
|
||||
|
||||
export const ContentPart = Schema.Union([
|
||||
TextPart,
|
||||
MediaPart,
|
||||
ToolCallPart,
|
||||
ToolResultPart,
|
||||
ReasoningPart,
|
||||
CompactionPart,
|
||||
]).pipe(Schema.toTaggedUnion("type"))
|
||||
export type ContentPart = Schema.Schema.Type<typeof ContentPart>
|
||||
|
||||
export class Message extends Schema.Class<Message>("LLM.Message")({
|
||||
@@ -200,6 +229,7 @@ export class Message extends Schema.Class<Message>("LLM.Message")({
|
||||
role: MessageRole,
|
||||
content: Schema.Array(ContentPart),
|
||||
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
native: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
}) {}
|
||||
|
||||
@@ -277,7 +307,7 @@ export namespace ToolChoice {
|
||||
}
|
||||
}
|
||||
|
||||
export class LLMRequest extends Schema.Class<LLMRequest>("LLM.Request")({
|
||||
const requestSchema = Schema.Struct({
|
||||
id: Schema.optional(Schema.String),
|
||||
model: LanguageModelSchema,
|
||||
system: Schema.Array(SystemPart),
|
||||
@@ -291,12 +321,26 @@ export class LLMRequest extends Schema.Class<LLMRequest>("LLM.Request")({
|
||||
// Stable cache affinity for protocols that support provider-managed prompt caching.
|
||||
promptCacheKey: Schema.optional(Schema.String),
|
||||
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
}) {}
|
||||
})
|
||||
|
||||
export class LLMRequest<Model extends LanguageModel = LanguageModel> extends Schema.Class<LLMRequest>("LLM.Request")(
|
||||
requestSchema.fields,
|
||||
) {
|
||||
declare readonly model: Model
|
||||
|
||||
// Preserve model inference instead of inheriting the schema's erased constructor signature.
|
||||
// oxlint-disable-next-line no-useless-constructor
|
||||
constructor(input: LLMRequest.Input<Model>) {
|
||||
super(input)
|
||||
}
|
||||
}
|
||||
|
||||
export namespace LLMRequest {
|
||||
export type Input = ConstructorParameters<typeof LLMRequest>[0]
|
||||
export type Input<Model extends LanguageModel = LanguageModel> = Omit<typeof requestSchema.Type, "model"> & {
|
||||
readonly model: Model
|
||||
}
|
||||
|
||||
export const input = (request: LLMRequest): Input => ({
|
||||
export const input = <Model extends LanguageModel>(request: LLMRequest<Model>): Input<Model> => ({
|
||||
id: request.id,
|
||||
model: request.model,
|
||||
system: request.system,
|
||||
@@ -311,7 +355,16 @@ export namespace LLMRequest {
|
||||
metadata: request.metadata,
|
||||
})
|
||||
|
||||
export const update = (request: LLMRequest, patch: Partial<Input>) => {
|
||||
export function update<Model extends LanguageModel>(
|
||||
request: LLMRequest,
|
||||
patch: Partial<Input<Model>> & { readonly model: Model },
|
||||
): LLMRequest<Model>
|
||||
export function update<Model extends LanguageModel>(
|
||||
request: LLMRequest<Model>,
|
||||
patch: Partial<Omit<Input, "model">> & { readonly model?: undefined },
|
||||
): LLMRequest<Model>
|
||||
export function update(request: LLMRequest, patch: Partial<Input>): LLMRequest
|
||||
export function update(request: LLMRequest, patch: Partial<Input>) {
|
||||
if (Object.keys(patch).length === 0) return request
|
||||
return new LLMRequest({
|
||||
...input(request),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Schema } from "effect"
|
||||
import { ModelID, ProviderID } from "./ids.js"
|
||||
import type { AnyRoute } from "../route/client.js"
|
||||
import type { AnyRoute, CompactOperation } from "../route/client.js"
|
||||
import { isRecord } from "../utils/record.js"
|
||||
|
||||
export const JsonSchema = Schema.Record(Schema.String, Schema.Unknown)
|
||||
@@ -173,15 +173,18 @@ export namespace LanguageModelCompatibility {
|
||||
input instanceof LanguageModelCompatibility ? input : new LanguageModelCompatibility(input)
|
||||
}
|
||||
|
||||
export class LanguageModel<Options extends ProviderOptions = ProviderOptions> {
|
||||
export class LanguageModel<
|
||||
Options extends ProviderOptions = ProviderOptions,
|
||||
Compact extends CompactOperation | undefined = CompactOperation | undefined,
|
||||
> {
|
||||
declare protected readonly _ProviderOptions: Options
|
||||
readonly id: ModelID
|
||||
readonly provider: ProviderID
|
||||
readonly route: AnyRoute
|
||||
readonly route: AnyRoute<Compact>
|
||||
readonly defaults?: LanguageModelDefaults
|
||||
readonly compatibility?: LanguageModelCompatibility
|
||||
|
||||
constructor(input: LanguageModel.ConstructorInput) {
|
||||
constructor(input: LanguageModel.ConstructorInput<Compact>) {
|
||||
this.id = input.id
|
||||
this.provider = input.provider
|
||||
this.route = input.route
|
||||
@@ -189,8 +192,11 @@ export class LanguageModel<Options extends ProviderOptions = ProviderOptions> {
|
||||
this.compatibility = input.compatibility
|
||||
}
|
||||
|
||||
static make<Options extends ProviderOptions = ProviderOptions>(input: LanguageModel.Input) {
|
||||
return new LanguageModel<Options>({
|
||||
static make<
|
||||
Options extends ProviderOptions = ProviderOptions,
|
||||
Compact extends CompactOperation | undefined = CompactOperation | undefined,
|
||||
>(input: LanguageModel.Input<Compact>) {
|
||||
return new LanguageModel<Options, Compact>({
|
||||
id: ModelID.make(input.id),
|
||||
provider: ProviderID.make(input.provider),
|
||||
route: input.route,
|
||||
@@ -200,7 +206,9 @@ export class LanguageModel<Options extends ProviderOptions = ProviderOptions> {
|
||||
})
|
||||
}
|
||||
|
||||
static input<Options extends ProviderOptions>(model: LanguageModel<Options>): LanguageModel.ConstructorInput {
|
||||
static input<Options extends ProviderOptions, Compact extends CompactOperation | undefined>(
|
||||
model: LanguageModel<Options, Compact>,
|
||||
): LanguageModel.ConstructorInput<Compact> {
|
||||
return {
|
||||
id: model.id,
|
||||
provider: model.provider,
|
||||
@@ -210,25 +218,41 @@ export class LanguageModel<Options extends ProviderOptions = ProviderOptions> {
|
||||
}
|
||||
}
|
||||
|
||||
static update<Options extends ProviderOptions, Compact extends CompactOperation | undefined>(
|
||||
model: LanguageModel<Options>,
|
||||
patch: Partial<LanguageModel.Input<Compact>> & { readonly route: AnyRoute<Compact> },
|
||||
): LanguageModel<Options, Compact>
|
||||
static update<Options extends ProviderOptions, Compact extends CompactOperation | undefined>(
|
||||
model: LanguageModel<Options, Compact>,
|
||||
patch: Partial<Omit<LanguageModel.Input, "route">> & { readonly route?: undefined },
|
||||
): LanguageModel<Options, Compact>
|
||||
static update<Options extends ProviderOptions>(
|
||||
model: LanguageModel<Options>,
|
||||
patch: Partial<LanguageModel.Input>,
|
||||
): LanguageModel<Options>
|
||||
static update<Options extends ProviderOptions>(model: LanguageModel<Options>, patch: Partial<LanguageModel.Input>) {
|
||||
if (Object.keys(patch).length === 0) return model
|
||||
return LanguageModel.make<Options>({
|
||||
...LanguageModel.input(model),
|
||||
...patch,
|
||||
route: patch.route ?? model.route,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export namespace LanguageModel {
|
||||
export type ConstructorInput = {
|
||||
export type ConstructorInput<Compact extends CompactOperation | undefined = CompactOperation | undefined> = {
|
||||
readonly id: ModelID
|
||||
readonly provider: ProviderID
|
||||
readonly route: AnyRoute
|
||||
readonly route: AnyRoute<Compact>
|
||||
readonly defaults?: LanguageModelDefaults
|
||||
readonly compatibility?: LanguageModelCompatibility
|
||||
}
|
||||
|
||||
export type Input = Omit<ConstructorInput, "id" | "provider" | "defaults" | "compatibility"> & {
|
||||
export type Input<Compact extends CompactOperation | undefined = CompactOperation | undefined> = Omit<
|
||||
ConstructorInput<Compact>,
|
||||
"id" | "provider" | "defaults" | "compatibility"
|
||||
> & {
|
||||
readonly id: string | ModelID
|
||||
readonly provider: string | ProviderID
|
||||
readonly defaults?: LanguageModelDefaults.Input
|
||||
@@ -267,7 +291,7 @@ export const CachePolicyObject = Schema.Struct({
|
||||
Schema.Union([
|
||||
Schema.Literal("latest-user-message"),
|
||||
Schema.Literal("latest-assistant"),
|
||||
Schema.Struct({ tail: Schema.Number }),
|
||||
Schema.Struct({ tail: Schema.Natural }),
|
||||
]),
|
||||
),
|
||||
ttlSeconds: Schema.optional(Schema.Number),
|
||||
|
||||
+119
-54
@@ -1,9 +1,10 @@
|
||||
export * as TestLLM from "./testing.js"
|
||||
|
||||
import { LLMClient, type Interface as LLMClientShape } from "./route/client.js"
|
||||
import { LLMClient } from "./route/client.js"
|
||||
import {
|
||||
LLMEvent,
|
||||
LLMResponse,
|
||||
CompactionResponse,
|
||||
type FinishReasonDetails,
|
||||
type AIError,
|
||||
type LLMRequest,
|
||||
@@ -12,17 +13,37 @@ import {
|
||||
} from "./schema/index.js"
|
||||
import { Context, Deferred, Effect, Latch, Layer, Queue, Scope, Stream } from "effect"
|
||||
|
||||
export type Response = readonly LLMEvent[] | Stream.Stream<LLMEvent, AIError>
|
||||
export type Response = readonly LLMEvent[] | Stream.Stream<LLMEvent, AIError> | CompactionResponse
|
||||
|
||||
export type Gate = Readonly<{ started: Effect.Effect<void>; release: Effect.Effect<void> }>
|
||||
|
||||
type ClientInterface = Context.Service.Shape<typeof LLMClient.Service>
|
||||
|
||||
export type Responder = (request: LLMRequest) => Response
|
||||
|
||||
export interface TestInterface extends ClientInterface {
|
||||
/** Returns a snapshot of requests observed at execution time. */
|
||||
readonly requests: () => Effect.Effect<readonly LLMRequest[]>
|
||||
readonly push: (...responses: readonly Response[]) => Effect.Effect<void>
|
||||
/** Replaces the fallback without changing queued responses. */
|
||||
readonly always: (response: Response) => Effect.Effect<void>
|
||||
/** Answers requests after the one-shot queue is exhausted; receives the original request. */
|
||||
readonly serve: (responder: Responder) => Effect.Effect<void>
|
||||
/** Waits for request arrivals, not output or completion. */
|
||||
readonly wait: (count: number) => Effect.Effect<void>
|
||||
readonly gate: () => Effect.Effect<Gate, never, Scope.Scope>
|
||||
}
|
||||
|
||||
export class Test extends Context.Service<Test, TestInterface>()("@opencode/ai/TestLLM/Test") {}
|
||||
|
||||
/** @deprecated Use TestInterface through Test and testLayer. */
|
||||
export interface Interface {
|
||||
readonly requests: LLMRequest[]
|
||||
readonly push: (...responses: readonly Response[]) => Effect.Effect<void>
|
||||
readonly always: (response: Response) => Effect.Effect<void>
|
||||
readonly wait: (count: number) => Effect.Effect<void>
|
||||
readonly gate: Effect.Effect<Gate, never, Scope.Scope>
|
||||
readonly client: LLMClientShape
|
||||
readonly client: ClientInterface
|
||||
}
|
||||
|
||||
export interface LayerOptions {
|
||||
@@ -31,6 +52,7 @@ export interface LayerOptions {
|
||||
readonly fallback?: Response
|
||||
}
|
||||
|
||||
/** @deprecated Use Test and testLayer for normal client methods and test controls. */
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/ai/TestLLM") {}
|
||||
|
||||
export const complete = (
|
||||
@@ -78,61 +100,79 @@ export const failAfter = (error: AIError, ...events: readonly LLMEvent[]) =>
|
||||
|
||||
export const hangAfter = (...events: readonly LLMEvent[]) => Stream.concat(Stream.fromIterable(events), Stream.never)
|
||||
|
||||
const toStream = (response: Response) => (Stream.isStream(response) ? response : Stream.fromIterable(response))
|
||||
const make = (options: LayerOptions) =>
|
||||
Effect.sync(() => {
|
||||
const requests: LLMRequest[] = []
|
||||
const responses: Response[] = []
|
||||
let started = Deferred.makeUnsafe<void>()
|
||||
let fallback: Response | Responder | undefined = options.fallback
|
||||
let activeGate: { readonly started: Queue.Queue<void>; readonly release: Latch.Latch } | undefined
|
||||
const wait = (count: number): Effect.Effect<void> =>
|
||||
Effect.suspend(() =>
|
||||
requests.length >= count ? Effect.void : Deferred.await(started).pipe(Effect.andThen(wait(count))),
|
||||
)
|
||||
|
||||
export const layer = (options: LayerOptions = {}) =>
|
||||
Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const requests: LLMRequest[] = []
|
||||
const responses: Response[] = []
|
||||
let started = Deferred.makeUnsafe<void>()
|
||||
let fallback = options.fallback
|
||||
let activeGate: { readonly started: Queue.Queue<void>; readonly release: Latch.Latch } | undefined
|
||||
const wait = (count: number): Effect.Effect<void> =>
|
||||
Effect.suspend(() =>
|
||||
requests.length >= count ? Effect.void : Deferred.await(started).pipe(Effect.andThen(wait(count))),
|
||||
)
|
||||
|
||||
const stream = ((request: LLMRequest) => {
|
||||
requests.push(options.transformRequest?.(request) ?? request)
|
||||
const take = (request: LLMRequest) =>
|
||||
Effect.suspend(() => {
|
||||
const count = requests.push(options.transformRequest?.(request) ?? request)
|
||||
const waiting = started
|
||||
started = Deferred.makeUnsafe()
|
||||
Deferred.doneUnsafe(waiting, Effect.void)
|
||||
const response = responses.shift() ?? fallback
|
||||
if (!response) return Stream.die(new Error(`TestLLM has no response for request ${requests.length}`))
|
||||
const streamed = toStream(response)
|
||||
const gate = activeGate
|
||||
if (!gate) return streamed
|
||||
return Stream.unwrap(
|
||||
Queue.offer(gate.started, undefined).pipe(Effect.andThen(gate.release.await), Effect.as(streamed)),
|
||||
)
|
||||
}) as LLMClientShape["stream"]
|
||||
const client = LLMClient.Service.of({
|
||||
stream,
|
||||
generate: (request) =>
|
||||
stream(request).pipe(
|
||||
Stream.runFold(LLMResponse.empty, LLMResponse.reduce),
|
||||
Effect.flatMap((state) => {
|
||||
const response = LLMResponse.complete(state)
|
||||
if (response) return Effect.succeed(response)
|
||||
return Effect.die("TestLLM response ended without a terminal finish event")
|
||||
}),
|
||||
),
|
||||
try {
|
||||
const response = responses.shift() ?? (typeof fallback === "function" ? fallback(request) : fallback)
|
||||
if (!response) return Effect.die(new Error(`TestLLM has no response for request ${count}`))
|
||||
if (!gate) return Effect.succeed(response)
|
||||
return Queue.offer(gate.started, undefined).pipe(Effect.andThen(gate.release.await), Effect.as(response))
|
||||
} finally {
|
||||
// Waiters can resume synchronously; assign the reply and gate before notifying them.
|
||||
Deferred.doneUnsafe(waiting, Effect.void)
|
||||
}
|
||||
})
|
||||
|
||||
return Service.of({
|
||||
requests,
|
||||
push: (...input) =>
|
||||
Effect.sync(() => {
|
||||
responses.push(...input)
|
||||
const stream: ClientInterface["stream"] = (request) =>
|
||||
Stream.unwrap(
|
||||
take(request).pipe(
|
||||
Effect.map((response) => {
|
||||
if (response instanceof CompactionResponse)
|
||||
return Stream.die("TestLLM generation requires an event response")
|
||||
return Stream.isStream(response) ? response : Stream.fromIterable(response)
|
||||
}),
|
||||
always: (response) =>
|
||||
Effect.sync(() => {
|
||||
fallback = response
|
||||
),
|
||||
)
|
||||
const test = Test.of({
|
||||
compact: (request) =>
|
||||
take(request).pipe(
|
||||
Effect.flatMap((response) =>
|
||||
response instanceof CompactionResponse
|
||||
? Effect.succeed(response)
|
||||
: Effect.die("TestLLM compaction requires a CompactionResponse"),
|
||||
),
|
||||
),
|
||||
stream,
|
||||
generate: (request) =>
|
||||
stream(request).pipe(
|
||||
Stream.runFold(LLMResponse.empty, LLMResponse.reduce),
|
||||
Effect.flatMap((state) => {
|
||||
const response = LLMResponse.complete(state)
|
||||
if (response) return Effect.succeed(response)
|
||||
return Effect.die("TestLLM response ended without a terminal finish event")
|
||||
}),
|
||||
wait,
|
||||
gate: Effect.gen(function* () {
|
||||
),
|
||||
requests: () => Effect.sync(() => [...requests]),
|
||||
push: (...input) =>
|
||||
Effect.sync(() => {
|
||||
responses.push(...input)
|
||||
}),
|
||||
always: (response) =>
|
||||
Effect.sync(() => {
|
||||
fallback = response
|
||||
}),
|
||||
serve: (responder) =>
|
||||
Effect.sync(() => {
|
||||
fallback = responder
|
||||
}),
|
||||
wait,
|
||||
gate: () =>
|
||||
Effect.gen(function* () {
|
||||
const gate = {
|
||||
started: yield* Effect.acquireRelease(Queue.unbounded<void>(), Queue.shutdown),
|
||||
release: yield* Latch.make(),
|
||||
@@ -147,11 +187,36 @@ export const layer = (options: LayerOptions = {}) =>
|
||||
release,
|
||||
}
|
||||
}),
|
||||
client,
|
||||
})
|
||||
}),
|
||||
})
|
||||
|
||||
return { test, requests }
|
||||
})
|
||||
|
||||
/** Provides one shared implementation under the normal client and test-control tags. */
|
||||
export const testLayer = (options: LayerOptions = {}) =>
|
||||
Layer.effectContext(
|
||||
Effect.map(make(options), (implementation) =>
|
||||
Context.make(LLMClient.Service, implementation.test).pipe(Context.add(Test, implementation.test)),
|
||||
),
|
||||
)
|
||||
|
||||
/** @deprecated Use testLayer; retained for published callers of the legacy control interface. */
|
||||
export const layer = (options: LayerOptions = {}) =>
|
||||
Layer.effect(
|
||||
Service,
|
||||
Effect.map(make(options), (implementation) =>
|
||||
Service.of({
|
||||
requests: implementation.requests,
|
||||
push: implementation.test.push,
|
||||
always: implementation.test.always,
|
||||
wait: implementation.test.wait,
|
||||
gate: implementation.test.gate(),
|
||||
client: implementation.test,
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
/** @deprecated testLayer provides LLMClient.Service directly. */
|
||||
export const clientLayer = Layer.effect(
|
||||
LLMClient.Service,
|
||||
Effect.map(Service, (service) => service.client),
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { Message, ToolResultPart, type ToolCallPart } from "./schema/messages.js"
|
||||
|
||||
const EMPTY_TOOL_OUTPUT = "(no tool output)"
|
||||
const MISSING_TOOL_RESULT = "Tool result missing"
|
||||
|
||||
export function normalizeToolHistory(messages: ReadonlyArray<Message>) {
|
||||
const normalized: Message[] = []
|
||||
const pending = new Map<string, ToolCallPart>()
|
||||
const appendMissingResults = () => {
|
||||
if (pending.size === 0) return
|
||||
normalized.push(missingToolResults(pending.values()))
|
||||
pending.clear()
|
||||
}
|
||||
|
||||
for (const message of messages) {
|
||||
if (message.role === "user" || message.role === "assistant") appendMissingResults()
|
||||
|
||||
if (message.role === "tool") {
|
||||
const tool = normalizeToolMessage(message, pending)
|
||||
if (tool) normalized.push(tool)
|
||||
continue
|
||||
}
|
||||
|
||||
normalized.push(message)
|
||||
if (message.role !== "assistant") continue
|
||||
for (const part of message.content) {
|
||||
if (part.type === "tool-call" && part.providerExecuted !== true) pending.set(part.id, part)
|
||||
}
|
||||
}
|
||||
|
||||
return normalized.length === messages.length && normalized.every((message, index) => message === messages[index])
|
||||
? messages
|
||||
: normalized
|
||||
}
|
||||
|
||||
function missingToolResults(calls: Iterable<ToolCallPart>) {
|
||||
return new Message({
|
||||
role: "tool",
|
||||
content: [...calls].map((call) =>
|
||||
ToolResultPart.make({ id: call.id, name: call.name, result: MISSING_TOOL_RESULT, resultType: "error" }),
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
function normalizeToolMessage(message: Message, pending: Map<string, ToolCallPart>): Message | undefined {
|
||||
const content = message.content.map((part) => {
|
||||
if (part.type !== "tool-result" || part.providerExecuted === true) return part
|
||||
const call = pending.get(part.id)
|
||||
if (call) pending.delete(part.id)
|
||||
return normalizeToolResult(part, call?.name ?? part.name)
|
||||
})
|
||||
if (content.length === 0) return undefined
|
||||
if (content.every((part, index) => part === message.content[index])) return message
|
||||
return new Message({
|
||||
id: message.id,
|
||||
role: message.role,
|
||||
content,
|
||||
metadata: message.metadata,
|
||||
providerMetadata: message.providerMetadata,
|
||||
native: message.native,
|
||||
})
|
||||
}
|
||||
|
||||
function normalizeToolResult(part: ToolResultPart, name: string): ToolResultPart {
|
||||
const named = part.name === name ? part : { ...part, name }
|
||||
if (named.result.type === "text" && named.result.value === "")
|
||||
return { ...named, result: { type: "text", value: EMPTY_TOOL_OUTPUT } }
|
||||
if (named.result.type === "error" && named.result.value === "")
|
||||
return { ...named, result: { type: "error", value: EMPTY_TOOL_OUTPUT } }
|
||||
if (named.result.type !== "content") return named
|
||||
const value = named.result.value.filter((item) => item.type !== "text" || item.text !== "")
|
||||
if (value.length === 0) return { ...named, result: { type: "text", value: EMPTY_TOOL_OUTPUT } }
|
||||
if (value.length === named.result.value.length) return named
|
||||
return { ...named, result: { type: "content", value } }
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import { LLM, LLMRequest, LLMResponse } from "../src/index.js"
|
||||
import { Route, Endpoint, LLMClient, Protocol, type FramingDef } from "../src/route.js"
|
||||
@@ -148,15 +148,13 @@ describe("llm route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("builds models from configured routes", () =>
|
||||
Effect.gen(function* () {
|
||||
const configured = fake.with({ provider: "fake-provider", endpoint: { baseURL: "https://fake.local" } })
|
||||
test("builds models from configured routes", () => {
|
||||
const configured = fake.with({ provider: "fake-provider", endpoint: { baseURL: "https://fake.local" } })
|
||||
|
||||
expect(configured.model({ id: "fake-model" })).toMatchObject({
|
||||
provider: "fake-provider",
|
||||
})
|
||||
}),
|
||||
)
|
||||
expect(configured.model({ id: "fake-model" })).toMatchObject({
|
||||
provider: "fake-provider",
|
||||
})
|
||||
})
|
||||
|
||||
it.effect("does not register duplicate route ids globally", () =>
|
||||
Effect.gen(function* () {
|
||||
|
||||
@@ -126,6 +126,38 @@ describe("applyCachePolicy", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
for (const fixture of [
|
||||
{ name: "default", cache: undefined, control: { type: "ephemeral" } },
|
||||
{ name: "auto", cache: "auto", control: { type: "ephemeral" } },
|
||||
{
|
||||
name: "explicit one-hour",
|
||||
cache: { tools: true, system: true, messages: { tail: 1 }, ttlSeconds: 3600 },
|
||||
control: { type: "ephemeral", ttl: "1h" },
|
||||
},
|
||||
{ name: "disabled", cache: "none", control: undefined },
|
||||
] as const) {
|
||||
it.effect(`Bedrock Messages respects ${fixture.name} caching`, () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model: AmazonBedrock.configure({ apiKey: "test" }).messages("anthropic.claude-opus-4-6-v1"),
|
||||
system: "Stable instructions",
|
||||
tools: [
|
||||
{ name: "lookup", description: "Look up a value", inputSchema: { type: "object", properties: {} } },
|
||||
],
|
||||
prompt: "hello",
|
||||
cache: fixture.cache,
|
||||
}),
|
||||
)
|
||||
expect(prepared.body).toMatchObject({
|
||||
tools: [{ name: "lookup", cache_control: fixture.control }],
|
||||
system: [{ type: "text", text: "Stable instructions", cache_control: fixture.control }],
|
||||
messages: [{ role: "user", content: [{ type: "text", text: "hello", cache_control: fixture.control }] }],
|
||||
})
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
it.effect("'auto' is a no-op on Gemini (out-of-band caching protocol)", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import { expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { FetchHttpClient } from "effect/unstable/http"
|
||||
import { LLM, LLMRequest, Message } from "../src/index.js"
|
||||
import { LLMClient } from "../src/route/client.js"
|
||||
import { OpenAI } from "../src/providers.js"
|
||||
import { testEffect } from "./lib/effect.js"
|
||||
import { runtimeLayer } from "./lib/http.js"
|
||||
import { sseEvents } from "./lib/sse.js"
|
||||
|
||||
testEffect(runtimeLayer(FetchHttpClient.layer)).live("compaction and a tool loop work end to end over HTTP", () =>
|
||||
Effect.gen(function* () {
|
||||
const checkpoint = { type: "compaction", id: "cmp_local", encrypted_content: "opaque-local-state" }
|
||||
const calls: string[] = []
|
||||
const server = yield* Effect.acquireRelease(
|
||||
Effect.sync(() =>
|
||||
Bun.serve({
|
||||
hostname: "127.0.0.1",
|
||||
port: 0,
|
||||
async fetch(request) {
|
||||
const path = new URL(request.url).pathname
|
||||
calls.push(path)
|
||||
const body = await request.json()
|
||||
expect(request.headers.get("authorization")).toBe("Bearer fixture")
|
||||
if (path === "/v1/responses/compact") {
|
||||
expect(body.stream).toBeUndefined()
|
||||
return Response.json({
|
||||
object: "response.compaction",
|
||||
output: [checkpoint],
|
||||
usage: { input_tokens: 100, output_tokens: 10, total_tokens: 110 },
|
||||
})
|
||||
}
|
||||
expect(body.input[0]).toEqual(checkpoint)
|
||||
expect(body.stream).toBe(true)
|
||||
if (calls.length === 2)
|
||||
return new Response(
|
||||
sseEvents(
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: { type: "function_call", id: "fc_1", call_id: "call_1", name: "lookup", arguments: "{}" },
|
||||
},
|
||||
{ type: "response.completed", response: { id: "resp_1" } },
|
||||
),
|
||||
{ headers: { "content-type": "text/event-stream" } },
|
||||
)
|
||||
expect(body.input.at(-2)).toMatchObject({ type: "function_call", call_id: "call_1" })
|
||||
expect(body.input.at(-1)).toEqual({ type: "function_call_output", call_id: "call_1", output: "42" })
|
||||
const output = sseEvents(
|
||||
{ type: "response.output_item.added", item: { type: "message", id: "msg_1" } },
|
||||
{ type: "response.output_text.delta", item_id: "msg_1", delta: "The answer is 42." },
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: { type: "message", id: "msg_1", content: [{ type: "output_text", text: "The answer is 42." }] },
|
||||
},
|
||||
{ type: "response.completed", response: { id: "resp_2" } },
|
||||
)
|
||||
return new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode(output.slice(0, 37)))
|
||||
controller.enqueue(new TextEncoder().encode(output.slice(37)))
|
||||
controller.close()
|
||||
},
|
||||
}),
|
||||
{ headers: { "content-type": "text/event-stream" } },
|
||||
)
|
||||
},
|
||||
}),
|
||||
),
|
||||
(server) => Effect.sync(() => server.stop(true)),
|
||||
)
|
||||
const model = OpenAI.configure({ apiKey: "fixture", baseURL: `http://127.0.0.1:${server.port}/v1` }).responses(
|
||||
"fixture",
|
||||
)
|
||||
const request = LLM.request({
|
||||
model,
|
||||
prompt: "original",
|
||||
tools: [{ name: "lookup", description: "Lookup a number", inputSchema: { type: "object", properties: {} } }],
|
||||
})
|
||||
const compacted = yield* LLMClient.compact(request)
|
||||
const messages = [...compacted.messages, Message.user("Look up the answer")]
|
||||
const first = yield* LLMClient.generate(LLMRequest.update(request, { messages }))
|
||||
expect(first.toolCalls).toHaveLength(1)
|
||||
const call = first.toolCalls[0]!
|
||||
const last = yield* LLMClient.generate(
|
||||
LLMRequest.update(request, {
|
||||
messages: [
|
||||
...messages,
|
||||
first.message,
|
||||
Message.tool({ id: call.id, name: call.name, result: "42", resultType: "text" }),
|
||||
],
|
||||
}),
|
||||
)
|
||||
expect(last.text).toBe("The answer is 42.")
|
||||
expect(calls).toEqual(["/v1/responses/compact", "/v1/responses", "/v1/responses"])
|
||||
}),
|
||||
)
|
||||
@@ -0,0 +1,64 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import { Schema } from "effect"
|
||||
import { CompactionPart, LLMEvent, LLMResponse, Message, ProviderID } from "../src/schema/index.js"
|
||||
import { LLM, LLMClient, LLMRequest, LanguageModel } from "../src/index.js"
|
||||
import { OpenAI, Anthropic } from "../src/providers.js"
|
||||
|
||||
test("runtime capability checks follow model and route updates", () => {
|
||||
const supported = OpenAI.configure({ apiKey: "test" }).responses("fixture")
|
||||
const unsupported = Anthropic.configure({ apiKey: "test" }).model("fixture")
|
||||
const request = LLM.request({ model: supported, prompt: "hello" })
|
||||
expect(LLMClient.canCompact(request)).toBe(true)
|
||||
expect(LLMClient.canCompact(LLMRequest.update(request, { messages: [] }))).toBe(true)
|
||||
expect(LLMClient.canCompact(LLMRequest.update(request, { model: unsupported }))).toBe(false)
|
||||
expect(
|
||||
LLMClient.canCompact(LLM.request({ model: LanguageModel.update(supported, { route: unsupported.route }) })),
|
||||
).toBe(false)
|
||||
expect(LLMClient.canCompact(LLM.request({ model: LanguageModel.update(supported, { route: undefined }) }))).toBe(true)
|
||||
})
|
||||
|
||||
test("compaction survives event assembly and message serialization without becoming text", () => {
|
||||
const part = CompactionPart.make({
|
||||
provider: ProviderID.make("openai"),
|
||||
id: "cmp_1",
|
||||
encrypted: "opaque",
|
||||
})
|
||||
const response = LLMResponse.fromEvents([
|
||||
LLMEvent.textStart({ id: "before" }),
|
||||
LLMEvent.textDelta({ id: "before", text: "Before" }),
|
||||
LLMEvent.textEnd({ id: "before" }),
|
||||
part,
|
||||
LLMEvent.textStart({ id: "after" }),
|
||||
LLMEvent.textDelta({ id: "after", text: "After" }),
|
||||
LLMEvent.textEnd({ id: "after" }),
|
||||
LLMEvent.finish({ reason: { normalized: "stop" } }),
|
||||
])!
|
||||
expect(response.message.content.map((part) => part.type)).toEqual(["text", "compaction", "text"])
|
||||
expect(response.text).toBe("BeforeAfter")
|
||||
expect(response.reasoning).toBe("")
|
||||
expect(response.events.filter(LLMEvent.is.compaction)).toEqual([part])
|
||||
const codec = Schema.fromJsonString(Message)
|
||||
expect(Schema.decodeSync(codec)(Schema.encodeSync(codec)(response.message))).toEqual(response.message)
|
||||
})
|
||||
|
||||
test("compaction requires exactly one typed representation", () => {
|
||||
const provider = ProviderID.make("anthropic")
|
||||
expect(CompactionPart.make({ provider, text: null })).toEqual({ type: "compaction", provider, text: null })
|
||||
const decode = Schema.decodeUnknownSync(CompactionPart)
|
||||
expect(() => decode({ type: "compaction", provider })).toThrow()
|
||||
expect(() => decode({ type: "compaction", provider, text: "summary", encrypted: "opaque" })).toThrow()
|
||||
})
|
||||
|
||||
test("tagged content and event guards accept both checkpoint representations", () => {
|
||||
for (const part of [
|
||||
CompactionPart.make({ provider: ProviderID.make("openai"), encrypted: "opaque" }),
|
||||
CompactionPart.make({ provider: ProviderID.make("anthropic"), text: "summary" }),
|
||||
CompactionPart.make({ provider: ProviderID.make("anthropic"), text: null }),
|
||||
]) {
|
||||
expect(LLMEvent.is.compaction(part)).toBe(true)
|
||||
expect(LLMEvent.guards.compaction(part)).toBe(true)
|
||||
const codec = Schema.fromJsonString(Message)
|
||||
const message = Message.assistant(part)
|
||||
expect(Schema.decodeSync(codec)(Schema.encodeSync(codec)(message))).toEqual(message)
|
||||
}
|
||||
})
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { Effect, Ref, Schema } from "effect"
|
||||
import { HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import { LLM, Message, ToolCallPart, mergeProviderOptions } from "../src/index.js"
|
||||
import { LLM, LLMRequest, Message, ToolCallPart, ToolDefinition, mergeProviderOptions } from "../src/index.js"
|
||||
import { AnthropicMessages, OpenAIChat } from "../src/protocols.js"
|
||||
import { Auth, LLMClient } from "../src/route.js"
|
||||
import { compileRequest } from "../src/route/client.js"
|
||||
@@ -77,6 +77,55 @@ describe("request option precedence", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps the last tool definition for duplicate names", () =>
|
||||
Effect.gen(function* () {
|
||||
const request = LLM.request({
|
||||
model: OpenAIChat.route.model({ id: "gpt-4o-mini" }),
|
||||
prompt: "Use a tool.",
|
||||
})
|
||||
const prepared = yield* compileRequest(
|
||||
LLMRequest.update(request, {
|
||||
tools: [
|
||||
ToolDefinition.make({ name: "lookup", description: "old", inputSchema: { type: "object" } }),
|
||||
ToolDefinition.make({ name: "search", description: "search", inputSchema: { type: "object" } }),
|
||||
ToolDefinition.make({ name: "lookup", description: "new", inputSchema: { type: "object" } }),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.tools).toEqual([
|
||||
{
|
||||
type: "function",
|
||||
function: { name: "lookup", description: "new", parameters: { type: "object" }, strict: false },
|
||||
},
|
||||
{
|
||||
type: "function",
|
||||
function: { name: "search", description: "search", parameters: { type: "object" }, strict: false },
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("normalizes tool history before protocol lowering", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model: OpenAIChat.route.model({ id: "gpt-4o-mini" }),
|
||||
messages: [
|
||||
Message.assistant(ToolCallPart.make({ id: "call_1", name: "lookup", input: {} })),
|
||||
Message.user("Continue."),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.messages).toMatchObject([
|
||||
{ role: "assistant", tool_calls: [{ id: "call_1", function: { name: "lookup" } }] },
|
||||
{ role: "tool", tool_call_id: "call_1", content: "Tool result missing" },
|
||||
{ role: "user", content: "Continue." },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("applies model HTTP defaults before request HTTP overlays", () =>
|
||||
LLMClient.generate(
|
||||
LLM.request({
|
||||
|
||||
+115
-167
@@ -1,10 +1,10 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Deferred, Effect, Fiber, Layer, Ref, Stream } from "effect"
|
||||
import { Headers, HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import { Deferred, Effect, Fiber, Ref, Stream } from "effect"
|
||||
import { Headers, HttpClientError, HttpClientRequest } from "effect/unstable/http"
|
||||
import { LLM, AIError, HttpContext, InvalidProviderOutputError, TransportError } from "../src/index.js"
|
||||
import { LLMClient, RequestExecutor, WebSocketTransport, type WebSocketChannelExecutor } from "../src/route.js"
|
||||
import * as OpenAIChat from "../src/protocols/openai-chat.js"
|
||||
import * as OpenAI from "../src/providers/openai.js"
|
||||
import { route } from "../src/protocols/openai-chat.js"
|
||||
import { configure } from "../src/providers/openai.js"
|
||||
import { dynamicResponse, fixedResponse, systemError } from "./lib/http.js"
|
||||
import { deltaChunk } from "./lib/openai-chunks.js"
|
||||
import { sseEvents, sseRaw } from "./lib/sse.js"
|
||||
@@ -18,47 +18,6 @@ const secretRequest = HttpClientRequest.post("https://provider.test/v1/chat?api_
|
||||
HttpClientRequest.setHeaders(Headers.fromInput({ authorization: "Bearer header-secret-456" })),
|
||||
)
|
||||
|
||||
const responsesLayer = (responses: ReadonlyArray<Response>) =>
|
||||
RequestExecutor.layer.pipe(
|
||||
Layer.provide(
|
||||
Layer.unwrap(
|
||||
Effect.gen(function* () {
|
||||
const cursor = yield* Ref.make(0)
|
||||
return Layer.succeed(
|
||||
HttpClient.HttpClient,
|
||||
HttpClient.make((request) =>
|
||||
Effect.gen(function* () {
|
||||
const index = yield* Ref.getAndUpdate(cursor, (value) => value + 1)
|
||||
return HttpClientResponse.fromWeb(request, responses[index] ?? responses[responses.length - 1])
|
||||
}),
|
||||
),
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
const countedResponsesLayer = (attempts: Ref.Ref<number>, responses: ReadonlyArray<Response>) =>
|
||||
RequestExecutor.layer.pipe(
|
||||
Layer.provide(
|
||||
Layer.unwrap(
|
||||
Effect.gen(function* () {
|
||||
const cursor = yield* Ref.make(0)
|
||||
return Layer.succeed(
|
||||
HttpClient.HttpClient,
|
||||
HttpClient.make((request) =>
|
||||
Effect.gen(function* () {
|
||||
yield* Ref.update(attempts, (value) => value + 1)
|
||||
const index = yield* Ref.getAndUpdate(cursor, (value) => value + 1)
|
||||
return HttpClientResponse.fromWeb(request, responses[index] ?? responses[responses.length - 1])
|
||||
}),
|
||||
),
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
const expectAIError = (error: unknown) => {
|
||||
expect(error).toBeInstanceOf(AIError)
|
||||
if (!(error instanceof AIError)) throw new Error("expected AIError")
|
||||
@@ -107,19 +66,17 @@ describe("RequestExecutor", () => {
|
||||
return yield* executor.execute(request).pipe(Effect.flip)
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.error(cause)
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 503,
|
||||
headers: { "x-request-id": "req_failed_body" },
|
||||
fixedResponse(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.error(cause)
|
||||
},
|
||||
),
|
||||
]),
|
||||
}),
|
||||
{
|
||||
status: 503,
|
||||
headers: { "x-request-id": "req_failed_body" },
|
||||
},
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -148,15 +105,14 @@ describe("RequestExecutor", () => {
|
||||
})
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.error(systemError("ECONNRESET", "disconnected query-secret-123 header-secret-456"))
|
||||
},
|
||||
}),
|
||||
),
|
||||
]),
|
||||
fixedResponse(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.error(systemError("ECONNRESET", "disconnected query-secret-123 header-secret-456"))
|
||||
},
|
||||
}),
|
||||
{},
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -176,15 +132,14 @@ describe("RequestExecutor", () => {
|
||||
})
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
pull(controller) {
|
||||
controller.error(new TypeError("fetch failed", { cause: systemError("ECONNRESET", "socket closed") }))
|
||||
},
|
||||
}),
|
||||
),
|
||||
]),
|
||||
fixedResponse(
|
||||
new ReadableStream({
|
||||
pull(controller) {
|
||||
controller.error(new TypeError("fetch failed", { cause: systemError("ECONNRESET", "socket closed") }))
|
||||
},
|
||||
}),
|
||||
{},
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -200,7 +155,7 @@ describe("RequestExecutor", () => {
|
||||
expect(error.message).toBe("plugin rejected request")
|
||||
expect(error.reason.cause).toBeInstanceOf(Error)
|
||||
expect(error.reason.http).toBeUndefined()
|
||||
}).pipe(Effect.provide(responsesLayer([]))),
|
||||
}).pipe(Effect.provide(dynamicResponse(() => Effect.die(new Error("unexpected HTTP request"))))),
|
||||
)
|
||||
|
||||
it.effect("reports the request sent by middleware", () =>
|
||||
@@ -249,11 +204,9 @@ describe("RequestExecutor", () => {
|
||||
expect(error.reason).toMatchObject({ _tag: "InvalidRequest", classification: "context-overflow" })
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response('{"error":{"code":"context_length_exceeded","message":"prompt too long"}}', {
|
||||
status: 400,
|
||||
}),
|
||||
]),
|
||||
fixedResponse('{"error":{"code":"context_length_exceeded","message":"prompt too long"}}', {
|
||||
status: 400,
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -269,7 +222,7 @@ describe("RequestExecutor", () => {
|
||||
classification: "payload-too-large",
|
||||
})
|
||||
expect(error.reason.http?.status).toBe(413)
|
||||
}).pipe(Effect.provide(responsesLayer([new Response("request too large", { status: 413 })]))),
|
||||
}).pipe(Effect.provide(fixedResponse("request too large", { status: 413 }))),
|
||||
)
|
||||
|
||||
it.effect("classifies Anthropic request_too_large as context overflow", () =>
|
||||
@@ -285,11 +238,9 @@ describe("RequestExecutor", () => {
|
||||
expect(error.reason.http?.status).toBe(413)
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response('{"error":{"type":"request_too_large","message":"Request exceeds the maximum size"}}', {
|
||||
status: 413,
|
||||
}),
|
||||
]),
|
||||
fixedResponse('{"error":{"type":"request_too_large","message":"Request exceeds the maximum size"}}', {
|
||||
status: 413,
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -303,7 +254,7 @@ describe("RequestExecutor", () => {
|
||||
expect(error.reason).toMatchObject({ _tag: "InvalidRequest" })
|
||||
expect("classification" in error.reason ? error.reason.classification : undefined).toBeUndefined()
|
||||
expect(error.message).toBe("Provider request failed with HTTP 400")
|
||||
}).pipe(Effect.provide(responsesLayer([new Response("invalid parameter", { status: 400 })]))),
|
||||
}).pipe(Effect.provide(fixedResponse("invalid parameter", { status: 400 }))),
|
||||
)
|
||||
|
||||
it.effect("preserves structured provider messages from large error bodies", () =>
|
||||
@@ -317,15 +268,13 @@ describe("RequestExecutor", () => {
|
||||
expect(error.reason.body).toContain(largeProviderMessage)
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
model: "gpt-5.6-sol",
|
||||
error: { type: "invalid_request", message: largeProviderMessage },
|
||||
}),
|
||||
{ status: 400 },
|
||||
),
|
||||
]),
|
||||
fixedResponse(
|
||||
JSON.stringify({
|
||||
model: "test-model",
|
||||
error: { type: "invalid_request", message: largeProviderMessage },
|
||||
}),
|
||||
{ status: 400 },
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -340,7 +289,7 @@ describe("RequestExecutor", () => {
|
||||
_tag: "InvalidRequest",
|
||||
})
|
||||
expect(error.message).toBe("Provider request failed with HTTP 400")
|
||||
}).pipe(Effect.provide(responsesLayer([new Response('{"error":{"message":" "}}', { status: 400 })]))),
|
||||
}).pipe(Effect.provide(fixedResponse('{"error":{"message":" "}}', { status: 400 }))),
|
||||
)
|
||||
|
||||
it.effect("classifies provider rate limits hidden behind HTTP 400", () =>
|
||||
@@ -352,7 +301,7 @@ describe("RequestExecutor", () => {
|
||||
|
||||
expectAIError(error)
|
||||
expect(error.reason).toMatchObject({ _tag: "RateLimit" })
|
||||
}).pipe(Effect.provide(responsesLayer([new Response(body, { status: 400 })])))
|
||||
}).pipe(Effect.provide(fixedResponse(body, { status: 400 })))
|
||||
|
||||
yield* classify("Request rate increased too quickly")
|
||||
yield* classify('{"type":"error","error":{"type":"too_many_requests"}}')
|
||||
@@ -369,7 +318,7 @@ describe("RequestExecutor", () => {
|
||||
|
||||
expectAIError(error)
|
||||
expect(error.reason).toMatchObject({ _tag: "ProviderInternal" })
|
||||
}).pipe(Effect.provide(responsesLayer([new Response(body, { status: 400 })])))
|
||||
}).pipe(Effect.provide(fixedResponse(body, { status: 400 })))
|
||||
|
||||
yield* classify('{"code":"resource_exhausted"}')
|
||||
yield* classify('{"code":"service_unavailable"}')
|
||||
@@ -399,12 +348,10 @@ describe("RequestExecutor", () => {
|
||||
expect(error.reason.body).toBe("rate limited")
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response("rate limited", {
|
||||
status: 429,
|
||||
headers: { "retry-after-ms": "0", "x-request-id": "req_123", "x-api-key": "secret" },
|
||||
}),
|
||||
]),
|
||||
fixedResponse("rate limited", {
|
||||
status: 429,
|
||||
headers: { "retry-after-ms": "0", "x-request-id": "req_123", "x-api-key": "secret" },
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -417,7 +364,7 @@ describe("RequestExecutor", () => {
|
||||
expectAIError(error)
|
||||
expect(error.reason.http?.headers["x-safe"]).toBe("response-secret")
|
||||
}).pipe(
|
||||
Effect.provide(responsesLayer([new Response("bad", { status: 400, headers: { "x-safe": "response-secret" } })])),
|
||||
Effect.provide(fixedResponse("bad", { status: 400, headers: { "x-safe": "response-secret" } })),
|
||||
Effect.provideService(Headers.CurrentRedactedNames, ["x-safe"]),
|
||||
),
|
||||
)
|
||||
@@ -437,20 +384,18 @@ describe("RequestExecutor", () => {
|
||||
})
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response("rate limited", {
|
||||
status: 429,
|
||||
headers: {
|
||||
"retry-after-ms": "0",
|
||||
"x-ratelimit-limit-requests": "500",
|
||||
"x-ratelimit-limit-tokens": "30000",
|
||||
"x-ratelimit-remaining-requests": "499",
|
||||
"x-ratelimit-remaining-tokens": "29900",
|
||||
"x-ratelimit-reset-requests": "1s",
|
||||
"x-ratelimit-reset-tokens": "10s",
|
||||
},
|
||||
}),
|
||||
]),
|
||||
fixedResponse("rate limited", {
|
||||
status: 429,
|
||||
headers: {
|
||||
"retry-after-ms": "0",
|
||||
"x-ratelimit-limit-requests": "500",
|
||||
"x-ratelimit-limit-tokens": "30000",
|
||||
"x-ratelimit-remaining-requests": "499",
|
||||
"x-ratelimit-remaining-tokens": "29900",
|
||||
"x-ratelimit-reset-requests": "1s",
|
||||
"x-ratelimit-reset-tokens": "10s",
|
||||
},
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -470,20 +415,18 @@ describe("RequestExecutor", () => {
|
||||
})
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response("rate limited", {
|
||||
status: 429,
|
||||
headers: {
|
||||
"retry-after-ms": "0",
|
||||
"anthropic-ratelimit-requests-limit": "100",
|
||||
"anthropic-ratelimit-requests-remaining": "12",
|
||||
"anthropic-ratelimit-requests-reset": "2026-05-06T12:00:00Z",
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "9000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-05-06T12:00:10Z",
|
||||
},
|
||||
}),
|
||||
]),
|
||||
fixedResponse("rate limited", {
|
||||
status: 429,
|
||||
headers: {
|
||||
"retry-after-ms": "0",
|
||||
"anthropic-ratelimit-requests-limit": "100",
|
||||
"anthropic-ratelimit-requests-remaining": "12",
|
||||
"anthropic-ratelimit-requests-reset": "2026-05-06T12:00:00Z",
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "9000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-05-06T12:00:10Z",
|
||||
},
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -496,10 +439,14 @@ describe("RequestExecutor", () => {
|
||||
return yield* executor.execute(request).pipe(Effect.flip)
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
countedResponsesLayer(attempts, [
|
||||
new Response("busy", { status: 503, headers: { "retry-after-ms": "0" } }),
|
||||
new Response("ok", { status: 200 }),
|
||||
]),
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const attempt = yield* Ref.getAndUpdate(attempts, (value) => value + 1)
|
||||
return attempt === 0
|
||||
? input.respond("busy", { status: 503, headers: { "retry-after-ms": "0" } })
|
||||
: input.respond("ok", { status: 200 })
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -522,12 +469,10 @@ describe("RequestExecutor", () => {
|
||||
expect(error.reason.http?.status).toBe(status)
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response("provider failure", {
|
||||
status,
|
||||
headers: { "retry-after-ms": "0" },
|
||||
}),
|
||||
]),
|
||||
fixedResponse("provider failure", {
|
||||
status,
|
||||
headers: { "retry-after-ms": "0" },
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -538,20 +483,28 @@ describe("RequestExecutor", () => {
|
||||
|
||||
it.effect("preserves large authentication error bodies", () =>
|
||||
Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
const error = yield* executor.execute(request).pipe(Effect.flip)
|
||||
const attempts = yield* Ref.make(0)
|
||||
const error = yield* Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
return yield* executor.execute(request).pipe(Effect.flip)
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const attempt = yield* Ref.getAndUpdate(attempts, (value) => value + 1)
|
||||
return attempt === 0
|
||||
? input.respond("x".repeat(20_000), { status: 401 })
|
||||
: input.respond("should not retry", { status: 200 })
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expectAIError(error)
|
||||
expect(error.reason).toMatchObject({ _tag: "Authentication" })
|
||||
expect(error.reason.body).toHaveLength(20_000)
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response("x".repeat(20_000), { status: 401 }),
|
||||
new Response("should not retry", { status: 200 }),
|
||||
]),
|
||||
),
|
||||
),
|
||||
expect(yield* Ref.get(attempts)).toBe(1)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves response body fields", () =>
|
||||
@@ -563,11 +516,9 @@ describe("RequestExecutor", () => {
|
||||
expect(error.reason.body).toBe('{"error":{"message":"bad","key":"body-secret","detail":"api_key=query-secret"}}')
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response('{"error":{"message":"bad","key":"body-secret","detail":"api_key=query-secret"}}', {
|
||||
status: 400,
|
||||
}),
|
||||
]),
|
||||
fixedResponse('{"error":{"message":"bad","key":"body-secret","detail":"api_key=query-secret"}}', {
|
||||
status: 400,
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -581,9 +532,7 @@ describe("RequestExecutor", () => {
|
||||
expect(error.reason.body).toBe("provider echoed query-secret-123 and authorization header-secret-456")
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
responsesLayer([
|
||||
new Response("provider echoed query-secret-123 and authorization header-secret-456", { status: 400 }),
|
||||
]),
|
||||
fixedResponse("provider echoed query-secret-123 and authorization header-secret-456", { status: 400 }),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -591,9 +540,7 @@ describe("RequestExecutor", () => {
|
||||
it.effect("does not re-execute after a successful response reaches stream parsing", () =>
|
||||
Effect.gen(function* () {
|
||||
const attempts = yield* Ref.make(0)
|
||||
const model = OpenAIChat.route
|
||||
.with({ endpoint: { baseURL: "https://api.openai.test/v1" } })
|
||||
.model({ id: "gpt-4o-mini" })
|
||||
const model = route.with({ endpoint: { baseURL: "https://api.openai.test/v1" } }).model({ id: "gpt-4o-mini" })
|
||||
const error = yield* LLMClient.generate(LLM.request({ model, prompt: "Say hello." })).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
@@ -624,9 +571,10 @@ describe("RequestExecutor", () => {
|
||||
})
|
||||
|
||||
describe("WebSocket channel execution", () => {
|
||||
const model = OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).responses("gpt-4.1-mini")
|
||||
const model = configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).responses("gpt-4.1-mini")
|
||||
const request = LLM.request({ model, prompt: "Say hello." })
|
||||
const frames = [
|
||||
JSON.stringify({ type: "response.output_item.added", item: { type: "message", id: "msg_1" } }),
|
||||
JSON.stringify({ type: "response.output_text.delta", item_id: "msg_1", delta: "Hi" }),
|
||||
JSON.stringify({ type: "response.completed", response: { id: "resp_1" } }),
|
||||
]
|
||||
|
||||
@@ -32,6 +32,8 @@ describe("public exports", () => {
|
||||
expect(Provider.make).toBeFunction()
|
||||
expect(ProviderSubpath.make).toBe(Provider.make)
|
||||
expect(TestLLM.layer).toBeFunction()
|
||||
expect(TestLLM.testLayer).toBeFunction()
|
||||
expect(TestLLM.Test.of).toBeFunction()
|
||||
})
|
||||
|
||||
test("route barrel exposes route-authoring APIs", () => {
|
||||
|
||||
+2
-2
@@ -10,7 +10,7 @@
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.anthropic.com/v1/messages",
|
||||
"url": "https://api.anthropic.com/v1/messages?beta=true",
|
||||
"headers": {
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json"
|
||||
@@ -29,7 +29,7 @@
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.anthropic.com/v1/messages",
|
||||
"url": "https://api.anthropic.com/v1/messages?beta=true",
|
||||
"headers": {
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json"
|
||||
|
||||
+2
-2
@@ -10,7 +10,7 @@
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.anthropic.com/v1/messages",
|
||||
"url": "https://api.anthropic.com/v1/messages?beta=true",
|
||||
"headers": {
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json"
|
||||
@@ -29,7 +29,7 @@
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.anthropic.com/v1/messages",
|
||||
"url": "https://api.anthropic.com/v1/messages?beta=true",
|
||||
"headers": {
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json"
|
||||
|
||||
Vendored
+1
-1
@@ -24,7 +24,7 @@
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.anthropic.com/v1/messages",
|
||||
"url": "https://api.anthropic.com/v1/messages?beta=true",
|
||||
"headers": {
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json"
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"model": "zai-glm-5-2",
|
||||
"tags": [
|
||||
"prefix:mistral-chat-glm",
|
||||
"provider:mistral",
|
||||
"protocol:mistral-chat",
|
||||
"hosted-model",
|
||||
"tool",
|
||||
"tool-call"
|
||||
],
|
||||
"name": "mistral-chat-glm/streams-an-indexed-tool-call",
|
||||
"recordedAt": "2026-08-30T17:38:02.921Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.mistral.ai/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"zai-glm-5-2\",\"messages\":[{\"role\":\"system\",\"content\":\"Call lookup_weather exactly once with Paris.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"lookup_weather\",\"description\":\"Look up the current weather for a city\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\",\"enum\":[\"Paris\"]}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}}],\"tool_choice\":{\"type\":\"function\",\"function\":{\"name\":\"lookup_weather\"}},\"stream\":true,\"max_tokens\":256,\"temperature\":0}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream; charset=utf-8"
|
||||
},
|
||||
"body": "data: {\"id\":\"f139bf0e4b984e51aabf6a83c237674d\",\"object\":\"chat.completion.chunk\",\"created\":1788111482,\"model\":\"zai-glm-5-2\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"f139bf0e4b984e51aabf6a83c237674d\",\"object\":\"chat.completion.chunk\",\"created\":1788111482,\"model\":\"zai-glm-5-2\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"index\":0,\"content\":\"\"},\"finish_reason\":null,\"logprobs\":null}]}\n\ndata: {\"id\":\"f139bf0e4b984e51aabf6a83c237674d\",\"object\":\"chat.completion.chunk\",\"created\":1788111482,\"model\":\"zai-glm-5-2\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"id\":\"chatcmpl-tool-8cc4d8f9f07b298a\",\"type\":\"function\",\"function\":{\"name\":\"lookup_weather\",\"arguments\":\"{\\\"city\\\": \\\"\"},\"index\":0}],\"index\":0,\"content\":\"\"},\"finish_reason\":null,\"logprobs\":null}]}\n\ndata: {\"id\":\"f139bf0e4b984e51aabf6a83c237674d\",\"object\":\"chat.completion.chunk\",\"created\":1788111482,\"model\":\"zai-glm-5-2\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"type\":\"function\",\"function\":{\"name\":\"\",\"arguments\":\"Paris\\\"}\"},\"index\":0}],\"index\":0,\"content\":\"\"},\"finish_reason\":null,\"logprobs\":null}]}\n\ndata: {\"id\":\"f139bf0e4b984e51aabf6a83c237674d\",\"object\":\"chat.completion.chunk\",\"created\":1788111482,\"model\":\"zai-glm-5-2\",\"choices\":[{\"index\":0,\"delta\":{\"index\":0,\"content\":\"\"},\"finish_reason\":\"stop\",\"logprobs\":null}],\"usage\":{\"prompt_tokens\":171,\"total_tokens\":182,\"completion_tokens\":11,\"prompt_tokens_details\":{\"cached_tokens\":0}}}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"model": "mistral-small-latest",
|
||||
"tags": ["prefix:mistral-chat", "provider:mistral", "protocol:mistral-chat", "tool", "tool-loop", "usage"],
|
||||
"name": "mistral-chat/drives-a-tool-loop",
|
||||
"recordedAt": "2026-08-30T17:18:49.552Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.mistral.ai/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"mistral-small-latest\",\"messages\":[{\"role\":\"system\",\"content\":\"Call lookup_weather exactly once with Paris.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"lookup_weather\",\"description\":\"Look up the current weather for a city\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\",\"enum\":[\"Paris\"]}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}}],\"tool_choice\":{\"type\":\"function\",\"function\":{\"name\":\"lookup_weather\"}},\"stream\":true,\"max_tokens\":160,\"temperature\":0,\"reasoning_effort\":\"none\"}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream; charset=utf-8"
|
||||
},
|
||||
"body": "data: {\"id\":\"07491e37a5ed48f9987f1583753a466b\",\"object\":\"chat.completion.chunk\",\"created\":1788110328,\"model\":\"mistral-small-latest\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"07491e37a5ed48f9987f1583753a466b\",\"object\":\"chat.completion.chunk\",\"created\":1788110328,\"model\":\"mistral-small-latest\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"id\":\"ffJovBNqY\",\"type\":\"function\",\"function\":{\"name\":\"lookup_weather\",\"arguments\":\"{\\\"city\\\": \\\"Paris\\\"}\"},\"index\":0}]},\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":110,\"total_tokens\":122,\"completion_tokens\":12,\"prompt_tokens_details\":{\"cached_tokens\":0},\"service_tier\":\"standard\"},\"p\":\"abcdefghijklm\"}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.mistral.ai/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"mistral-small-latest\",\"messages\":[{\"role\":\"system\",\"content\":\"Call lookup_weather exactly once with Paris.\"},{\"role\":\"user\",\"content\":\"What is the weather?\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"ffJovBNqY\",\"type\":\"function\",\"function\":{\"name\":\"lookup_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"ffJovBNqY\",\"name\":\"lookup_weather\",\"content\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"temperature\\\":\\\"18C\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"lookup_weather\",\"description\":\"Look up the current weather for a city\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\",\"enum\":[\"Paris\"]}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}}],\"tool_choice\":\"none\",\"stream\":true,\"max_tokens\":160,\"temperature\":0,\"reasoning_effort\":\"none\"}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream; charset=utf-8"
|
||||
},
|
||||
"body": "data: {\"id\":\"8fcd293093b849139fc0893a48bbc7ce\",\"object\":\"chat.completion.chunk\",\"created\":1788110328,\"model\":\"mistral-small-latest\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"8fcd293093b849139fc0893a48bbc7ce\",\"object\":\"chat.completion.chunk\",\"created\":1788110328,\"model\":\"mistral-small-latest\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"The\"},\"finish_reason\":null}],\"p\":\"abcdefghijklmn\"}\n\ndata: {\"id\":\"8fcd293093b849139fc0893a48bbc7ce\",\"object\":\"chat.completion.chunk\",\"created\":1788110328,\"model\":\"mistral-small-latest\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" weather in Paris is\"},\"finish_reason\":null}],\"p\":\"abcdefghijklmn\"}\n\ndata: {\"id\":\"8fcd293093b849139fc0893a48bbc7ce\",\"object\":\"chat.completion.chunk\",\"created\":1788110328,\"model\":\"mistral-small-latest\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" currently sunny with\"},\"finish_reason\":null}],\"p\":\"abcdefghijklmnopqrstu\"}\n\ndata: {\"id\":\"8fcd293093b849139fc0893a48bbc7ce\",\"object\":\"chat.completion.chunk\",\"created\":1788110328,\"model\":\"mistral-small-latest\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" a temperature of \"},\"finish_reason\":null}],\"p\":\"abcdef\"}\n\ndata: {\"id\":\"8fcd293093b849139fc0893a48bbc7ce\",\"object\":\"chat.completion.chunk\",\"created\":1788110328,\"model\":\"mistral-small-latest\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"18°C\"},\"finish_reason\":null}],\"p\":\"abcdefghijklmnopqr\"}\n\ndata: {\"id\":\"8fcd293093b849139fc0893a48bbc7ce\",\"object\":\"chat.completion.chunk\",\"created\":1788110328,\"model\":\"mistral-small-latest\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":57,\"total_tokens\":74,\"completion_tokens\":17,\"prompt_tokens_details\":{\"cached_tokens\":0},\"service_tier\":\"standard\"},\"p\":\"abcdefghijklmnopqrstuvwxyz\"}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+47
File diff suppressed because one or more lines are too long
+29
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"model": "mistral-small-latest",
|
||||
"tags": ["prefix:mistral-chat", "provider:mistral", "protocol:mistral-chat", "text", "usage"],
|
||||
"name": "mistral-chat/streams-text-with-usage",
|
||||
"recordedAt": "2026-08-30T17:18:45.432Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.mistral.ai/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"mistral-small-latest\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: hello\"}],\"stream\":true,\"max_tokens\":40,\"temperature\":0,\"reasoning_effort\":\"none\"}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream; charset=utf-8"
|
||||
},
|
||||
"body": "data: {\"id\":\"9a4d16bdddb74e5e89c2cf9e9b91e065\",\"object\":\"chat.completion.chunk\",\"created\":1788110325,\"model\":\"mistral-small-latest\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"9a4d16bdddb74e5e89c2cf9e9b91e065\",\"object\":\"chat.completion.chunk\",\"created\":1788110325,\"model\":\"mistral-small-latest\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hi\"},\"finish_reason\":null}],\"p\":\"abcdefghijklmnopqrs\"}\n\ndata: {\"id\":\"9a4d16bdddb74e5e89c2cf9e9b91e065\",\"object\":\"chat.completion.chunk\",\"created\":1788110325,\"model\":\"mistral-small-latest\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":22,\"total_tokens\":24,\"completion_tokens\":2,\"prompt_tokens_details\":{\"cached_tokens\":0},\"service_tier\":\"standard\"},\"p\":\"abcdefghijklmnopqrstuvwxyz0\"}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -23,7 +23,7 @@
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":1536,\"reasoning\":{\"max_tokens\":1024}}"
|
||||
"body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"reasoning\":{\"max_tokens\":1024},\"max_completion_tokens\":1536,\"store\":false,\"usage\":{\"include\":true}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
@@ -41,7 +41,7 @@
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"Sure! Let me check the weather in Paris for you right now!\",\"tool_calls\":[{\"id\":\"toolu_01PaChhcyw3yu2P2bDS2bgAA\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}],\"reasoning\":\"The user wants to know the weather in Paris. I'll use the get_weather tool.\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"The user wants to know the weather in Paris. I'll use the get_weather tool.\",\"format\":\"anthropic-claude-v1\",\"index\":0,\"signature\":\"ErkCCosBCA8YAipAjKnRKpxkZ4eHrMPJ63IWEOYPSzb+XSHyG+vLK+2ks2O9T4N9M37Xn2kausQSH1rfsrdmKxgUlBg6yUFRgMVR7DIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDRjMGYwNDZmLTI1ZmQtNDVmYi1iZmIzLWEwOGE4ZTI0OWNhNxIMNFb5O6pb4nX0HojdGgyL5h+CAIpsxpdM1QgiMGm/i3ST6F5mAhxB+Uez0Cm95ra9yvQkrzHaA/AmWoXpdmPlczSn1S1RDk2IqeA57Spbf7JT44jygtLQt6yZmGzoTBHn3VkwaNZsuuAtbdo4B5QJXooa/AoKKs54QZ2kfS640vsv5flQVCg7CoQCFuLKjIeLMO7MnxVyuskXJr1DgesTa7I0ScF53U9JGhgB\"}]},{\"role\":\"tool\",\"tool_call_id\":\"toolu_01PaChhcyw3yu2P2bDS2bgAA\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":1536,\"reasoning\":{\"max_tokens\":1024}}"
|
||||
"body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"Sure! Let me check the weather in Paris for you right now!\",\"tool_calls\":[{\"id\":\"toolu_01PaChhcyw3yu2P2bDS2bgAA\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}],\"reasoning\":\"The user wants to know the weather in Paris. I'll use the get_weather tool.\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"The user wants to know the weather in Paris. I'll use the get_weather tool.\",\"format\":\"anthropic-claude-v1\",\"index\":0,\"signature\":\"ErkCCosBCA8YAipAjKnRKpxkZ4eHrMPJ63IWEOYPSzb+XSHyG+vLK+2ks2O9T4N9M37Xn2kausQSH1rfsrdmKxgUlBg6yUFRgMVR7DIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDRjMGYwNDZmLTI1ZmQtNDVmYi1iZmIzLWEwOGE4ZTI0OWNhNxIMNFb5O6pb4nX0HojdGgyL5h+CAIpsxpdM1QgiMGm/i3ST6F5mAhxB+Uez0Cm95ra9yvQkrzHaA/AmWoXpdmPlczSn1S1RDk2IqeA57Spbf7JT44jygtLQt6yZmGzoTBHn3VkwaNZsuuAtbdo4B5QJXooa/AoKKs54QZ2kfS640vsv5flQVCg7CoQCFuLKjIeLMO7MnxVyuskXJr1DgesTa7I0ScF53U9JGhgB\"}]},{\"role\":\"tool\",\"tool_call_id\":\"toolu_01PaChhcyw3yu2P2bDS2bgAA\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"reasoning\":{\"max_tokens\":1024},\"max_completion_tokens\":1536,\"store\":false,\"usage\":{\"include\":true}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Think through the arithmetic, then reply with only the final integer.\"},{\"role\":\"user\",\"content\":\"What is 173 multiplied by 219?\"}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":1536,\"temperature\":0,\"reasoning\":{\"max_tokens\":1024}}"
|
||||
"body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Think through the arithmetic, then reply with only the final integer.\"},{\"role\":\"user\",\"content\":\"What is 173 multiplied by 219?\"}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"temperature\":0,\"reasoning\":{\"max_tokens\":1024},\"max_completion_tokens\":1536,\"store\":false,\"usage\":{\"include\":true}}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
|
||||
@@ -10,11 +10,11 @@
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.anthropic.com/v1/messages",
|
||||
"url": "https://api.anthropic.com/v1/messages?beta=true",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"claude-haiku-4-5-20251001\",\"system\":[{\"type\":\"text\",\"text\":\"Read the PDF returned by the tool and follow the user's response format exactly.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Return only the verification code from the PDF.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"call_pdf_1\",\"name\":\"read_pdf\",\"input\":{}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"call_pdf_1\",\"content\":[{\"type\":\"text\",\"text\":\"PDF read successfully\"},{\"type\":\"document\",\"source\":{\"type\":\"base64\",\"media_type\":\"application/pdf\",\"data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}}]}]}],\"tools\":[{\"name\":\"read_pdf\",\"description\":\"Read the attached PDF.\",\"input_schema\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false}}],\"stream\":true,\"max_tokens\":40,\"temperature\":0}"
|
||||
"body": "{\"model\":\"claude-haiku-4-5-20251001\",\"system\":[{\"type\":\"text\",\"text\":\"Read the PDF returned by the tool and follow the user's response format exactly.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Return only the verification code from the PDF.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"call_pdf_1\",\"name\":\"read_pdf\",\"input\":{}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"call_pdf_1\",\"content\":[{\"type\":\"text\",\"text\":\"PDF read successfully\"},{\"type\":\"document\",\"source\":{\"type\":\"base64\",\"media_type\":\"application/pdf\",\"data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"},\"title\":\"verification.pdf\"}]}]}],\"tools\":[{\"name\":\"read_pdf\",\"description\":\"Read the attached PDF.\",\"input_schema\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false}}],\"stream\":true,\"max_tokens\":40,\"temperature\":0}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
|
||||
@@ -10,11 +10,11 @@
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://api.anthropic.com/v1/messages",
|
||||
"url": "https://api.anthropic.com/v1/messages?beta=true",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"claude-haiku-4-5-20251001\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"document\",\"source\":{\"type\":\"base64\",\"media_type\":\"application/pdf\",\"data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}},{\"type\":\"text\",\"text\":\"Return only the verification code from the PDF.\"}]}],\"stream\":true,\"max_tokens\":40,\"temperature\":0}"
|
||||
"body": "{\"model\":\"claude-haiku-4-5-20251001\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"document\",\"source\":{\"type\":\"base64\",\"media_type\":\"application/pdf\",\"data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"},\"title\":\"verification.pdf\"},{\"type\":\"text\",\"text\":\"Return only the verification code from the PDF.\"}]}],\"stream\":true,\"max_tokens\":40,\"temperature\":0}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
|
||||
@@ -44,21 +44,23 @@ describe("Tool.make (dynamic JSON Schema)", () => {
|
||||
expect(definition?.inputSchema).toEqual(jsonSchema)
|
||||
})
|
||||
|
||||
test("execute receives the raw input untouched", async () => {
|
||||
const seen: unknown[] = []
|
||||
const tool = Tool.make({
|
||||
description: "echo",
|
||||
jsonSchema: { type: "object" },
|
||||
execute: (params) =>
|
||||
Effect.sync(() => {
|
||||
seen.push(params)
|
||||
return { ok: true }
|
||||
}),
|
||||
})
|
||||
const result = await Effect.runPromise(tool.execute({ hello: "world" }))
|
||||
expect(seen).toEqual([{ hello: "world" }])
|
||||
expect(result).toEqual({ ok: true })
|
||||
})
|
||||
it.effect("execute receives the raw input untouched", () =>
|
||||
Effect.gen(function* () {
|
||||
const seen: unknown[] = []
|
||||
const tool = Tool.make({
|
||||
description: "echo",
|
||||
jsonSchema: { type: "object" },
|
||||
execute: (params) =>
|
||||
Effect.sync(() => {
|
||||
seen.push(params)
|
||||
return { ok: true }
|
||||
}),
|
||||
})
|
||||
const result = yield* tool.execute({ hello: "world" })
|
||||
expect(seen).toEqual([{ hello: "world" }])
|
||||
expect(result).toEqual({ ok: true })
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
describe("LLM.generateObject", () => {
|
||||
|
||||
@@ -11,6 +11,7 @@ describe("provider error classification", () => {
|
||||
"Input length 131393 exceeds the maximum allowed input length of 131040 tokens.",
|
||||
"The input (516368 tokens) is longer than the model's context length (262144 tokens).",
|
||||
"Prompt has 5,958,968 tokens, but the configured context size is 256,000 tokens",
|
||||
"Range of input length should be [1, 129024]",
|
||||
"Too many tokens",
|
||||
"Token limit exceeded",
|
||||
]
|
||||
@@ -80,6 +81,46 @@ describe("provider error classification", () => {
|
||||
).toEqual(["ProviderInternal", "ProviderInternal", "ProviderInternal"])
|
||||
})
|
||||
|
||||
test("classifies retryable server messages as provider internal", () => {
|
||||
const message =
|
||||
"The model is currently at capacity due to high demand. Please try again in a few minutes, or use a higher service tier for priority processing."
|
||||
|
||||
expect(
|
||||
[
|
||||
message,
|
||||
"Try again",
|
||||
"Please retry your request shortly.",
|
||||
"You can retry the request.",
|
||||
"Try your request again.",
|
||||
"The service is temporarily at capacity.",
|
||||
"The model is overloaded.",
|
||||
"Service unavailable",
|
||||
"Internal server error",
|
||||
"The server is busy.",
|
||||
"Provider returned error",
|
||||
"Provider returned an error",
|
||||
"ResourceExhausted",
|
||||
"Upstream connection failed",
|
||||
"Exceeded request buffer limit while retrying upstream",
|
||||
].map((message) => classifyProviderFailure({ message })._tag),
|
||||
).toEqual(Array(15).fill("ProviderInternal"))
|
||||
expect(
|
||||
classifyProviderFailure({ message: "Provider request failed", rawBody: "Please try again later." })._tag,
|
||||
).toBe("ProviderInternal")
|
||||
})
|
||||
|
||||
test("prioritizes specific failures over retryable server text", () => {
|
||||
expect(
|
||||
[
|
||||
classifyProviderFailure({ message: "Invalid credentials, try again", status: 401 }),
|
||||
classifyProviderFailure({ message: "Quota exceeded, try again", status: 429 }),
|
||||
classifyProviderFailure({ message: "Rate limit exceeded, try again" }),
|
||||
classifyProviderFailure({ message: "Upstream request failed: validation failed", status: 400 }),
|
||||
classifyProviderFailure({ message: "Try again", status: 200 }),
|
||||
].map((failure) => failure._tag),
|
||||
).toEqual(["Authentication", "QuotaExceeded", "RateLimit", "InvalidRequest", "ProviderInternal"])
|
||||
})
|
||||
|
||||
test("classifies transient client statuses as provider internal", () => {
|
||||
expect([408, 409].map((status) => classifyProviderFailure({ message: `HTTP ${status}`, status })._tag)).toEqual([
|
||||
"ProviderInternal",
|
||||
@@ -87,10 +128,12 @@ describe("provider error classification", () => {
|
||||
])
|
||||
})
|
||||
|
||||
test("classifies network error text as provider internal", () => {
|
||||
test("classifies any remaining 4xx status as an invalid request", () => {
|
||||
expect(
|
||||
["network error", "network-error", "network_error"].map((message) => classifyProviderFailure({ message })._tag),
|
||||
).toEqual(["ProviderInternal", "ProviderInternal", "ProviderInternal"])
|
||||
[400, 402, 404, 418, 422, 451].map(
|
||||
(status) => classifyProviderFailure({ message: `HTTP ${status}`, status })._tag,
|
||||
),
|
||||
).toEqual(Array(6).fill("InvalidRequest"))
|
||||
})
|
||||
|
||||
test("classifies nested provider codes when a top-level code is also present", () => {
|
||||
@@ -103,10 +146,11 @@ describe("provider error classification", () => {
|
||||
).toEqual(["QuotaExceeded", "ProviderInternal", "InvalidRequest"])
|
||||
})
|
||||
|
||||
test("keeps unknown and malformed provider payloads non-retryable", () => {
|
||||
test("leaves unrecognized failures unclassified for the retry default", () => {
|
||||
expect(classifyProviderFailure({ message: '{"error":{"message":"no_kv_space"}}' })._tag).toBe("UnknownProvider")
|
||||
expect(classifyProviderFailure({ message: '{"type":"error","error":{"code":123}}' })._tag).toBe("UnknownProvider")
|
||||
expect(classifyProviderFailure({ message: "not-json" })._tag).toBe("UnknownProvider")
|
||||
expect(classifyProviderFailure({ message: "network error" })._tag).toBe("UnknownProvider")
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import { Effect } from "effect"
|
||||
import {
|
||||
CompactionPart,
|
||||
LanguageModel,
|
||||
LLM,
|
||||
LLMClient,
|
||||
LLMEvent,
|
||||
LLMRequest,
|
||||
Message,
|
||||
ProviderID,
|
||||
} from "../../src/index.js"
|
||||
import {
|
||||
OpenAI,
|
||||
Azure,
|
||||
XAI,
|
||||
Anthropic,
|
||||
AmazonBedrock,
|
||||
AmazonBedrockMantle,
|
||||
OpenAICompatibleResponses,
|
||||
} from "../../src/providers.js"
|
||||
|
||||
const openai = OpenAI.configure({
|
||||
apiKey: "test",
|
||||
providerOptions: { contextManagement: [{ type: "compaction", compactThreshold: 100000 }] },
|
||||
}).responses("gpt-5.3-codex")
|
||||
LLMClient.compact(LLM.request({ model: openai, prompt: "hello" }))
|
||||
|
||||
for (const model of [
|
||||
OpenAI.configure().responses("fixture"),
|
||||
Azure.configure({ resourceName: "test" }).responses("fixture"),
|
||||
XAI.configure().responses("fixture"),
|
||||
OpenAI.model("fixture", {}),
|
||||
Azure.responsesModel("fixture", { resourceName: "test" }),
|
||||
XAI.model("fixture", {}),
|
||||
openai.route.with({ headers: { "x-test": "test" } }).model({ id: "fixture" }),
|
||||
LanguageModel.update(openai, { defaults: { generation: { maxTokens: 100 } } }),
|
||||
LanguageModel.make(LanguageModel.input(openai)),
|
||||
]) {
|
||||
LLMClient.compact(LLM.request({ model, prompt: "hello" }))
|
||||
}
|
||||
|
||||
const unsupported = {
|
||||
anthropic: LLM.request({ model: Anthropic.configure().model("fixture") }),
|
||||
openaiChat: LLM.request({ model: OpenAI.configure().chat("fixture") }),
|
||||
azureChat: LLM.request({ model: Azure.configure({ resourceName: "test" }).chat("fixture") }),
|
||||
xaiChat: LLM.request({ model: XAI.configure().chat("fixture") }),
|
||||
converse: LLM.request({ model: AmazonBedrock.configure().model("fixture") }),
|
||||
bedrockMessages: LLM.request({ model: AmazonBedrock.configure().messages("fixture") }),
|
||||
mantle: LLM.request({ model: AmazonBedrockMantle.configure().responses("fixture") }),
|
||||
compatible: LLM.request({
|
||||
model: OpenAICompatibleResponses.configure({ baseURL: "https://example.com" }).model("fixture"),
|
||||
}),
|
||||
}
|
||||
// @ts-expect-error Anthropic has no standalone compact endpoint.
|
||||
LLMClient.compact(unsupported.anthropic)
|
||||
// @ts-expect-error Chat does not expose Responses compaction.
|
||||
LLMClient.compact(unsupported.openaiChat)
|
||||
// @ts-expect-error Azure Chat does not expose Responses compaction.
|
||||
LLMClient.compact(unsupported.azureChat)
|
||||
// @ts-expect-error xAI Chat does not expose Responses compaction.
|
||||
LLMClient.compact(unsupported.xaiChat)
|
||||
// @ts-expect-error Converse has no standalone compact endpoint.
|
||||
LLMClient.compact(unsupported.converse)
|
||||
// @ts-expect-error Bedrock Messages has no standalone compact endpoint.
|
||||
LLMClient.compact(unsupported.bedrockMessages)
|
||||
// @ts-expect-error Mantle does not inherit the OpenAI compact endpoint.
|
||||
LLMClient.compact(unsupported.mantle)
|
||||
// @ts-expect-error Protocol compatibility does not guarantee endpoint support.
|
||||
LLMClient.compact(unsupported.compatible)
|
||||
LLMClient.Service.use((client) => {
|
||||
// @ts-expect-error The service enforces the same capability as the convenience function.
|
||||
return client.compact(unsupported.anthropic)
|
||||
})
|
||||
|
||||
const request = LLM.request({ model: openai, prompt: "hello" })
|
||||
LLMClient.compact(LLMRequest.update(request, { messages: [Message.user("continue")] }))
|
||||
LLMClient.compact(new LLMRequest(LLMRequest.input(request)))
|
||||
const switched = LLMRequest.update(request, { model: Anthropic.configure().model("fixture") })
|
||||
// @ts-expect-error Switching models replaces, rather than inherits, the capability.
|
||||
LLMClient.compact(switched)
|
||||
LLMClient.compact(LLMRequest.update(switched, { model: openai }))
|
||||
LLMClient.compact(
|
||||
// @ts-expect-error Replacing the route also replaces compaction capability.
|
||||
LLM.request({ model: LanguageModel.update(openai, { route: Anthropic.configure().model("fixture").route }) }),
|
||||
)
|
||||
|
||||
declare const dynamicModel: LanguageModel
|
||||
declare const dynamicPatch: Partial<LLMRequest.Input>
|
||||
const dynamicRequest = LLM.request({ model: dynamicModel, prompt: "hello" })
|
||||
// @ts-expect-error A dynamically selected model must be narrowed first.
|
||||
LLMClient.compact(dynamicRequest)
|
||||
if (LLMClient.canCompact(dynamicRequest)) LLMClient.compact(dynamicRequest)
|
||||
// @ts-expect-error An optional model override cannot retain the old capability statically.
|
||||
LLMClient.compact(LLMRequest.update(request, dynamicPatch))
|
||||
|
||||
const checkpoint = CompactionPart.make({ provider: ProviderID.make("openai"), id: "cmp_1", encrypted: "opaque" })
|
||||
const provider = ProviderID.make("anthropic")
|
||||
CompactionPart.make({ provider, text: "summary" })
|
||||
CompactionPart.make({ provider, text: null })
|
||||
// @ts-expect-error A checkpoint must have a representation.
|
||||
CompactionPart.make({ provider })
|
||||
// @ts-expect-error Encrypted and summary representations are mutually exclusive.
|
||||
CompactionPart.make({ provider, encrypted: "opaque", text: "summary" })
|
||||
// @ts-expect-error A failed summary cannot also carry encrypted content.
|
||||
LLMEvent.compaction({ provider, encrypted: "opaque", text: null })
|
||||
// @ts-expect-error The canonical message type also enforces the invariant.
|
||||
Message.assistant({ type: "compaction", provider })
|
||||
if (checkpoint.encrypted !== undefined) {
|
||||
checkpoint.encrypted satisfies string
|
||||
checkpoint.text satisfies undefined
|
||||
}
|
||||
if (checkpoint.text !== undefined) {
|
||||
checkpoint.text satisfies string | null
|
||||
checkpoint.encrypted satisfies undefined
|
||||
}
|
||||
checkpoint.encrypted
|
||||
// @ts-expect-error Compaction parts do not contain a generic provider payload.
|
||||
checkpoint.value
|
||||
LLMClient.compact(LLM.request({ model: openai, prompt: "hello" })).pipe(
|
||||
Effect.map((result) => {
|
||||
result.messages
|
||||
// @ts-expect-error Compaction returns replacement history, not a synthetic assistant message.
|
||||
result.message
|
||||
}),
|
||||
)
|
||||
LLM.request({
|
||||
model: openai,
|
||||
providerOptions: {
|
||||
// @ts-expect-error A token threshold is numeric.
|
||||
contextManagement: [{ type: "compaction", compactThreshold: "100000" }],
|
||||
},
|
||||
})
|
||||
for (const model of [
|
||||
Anthropic.configure().model("claude-opus-4-6"),
|
||||
AmazonBedrock.configure().messages("anthropic.claude-opus-4-6-v1"),
|
||||
]) {
|
||||
LLM.request({
|
||||
model,
|
||||
providerOptions: {
|
||||
contextManagement: {
|
||||
edits: [
|
||||
{ type: "compact_20260112", pauseAfterCompaction: true, instructions: "Summarize without using tools" },
|
||||
],
|
||||
},
|
||||
},
|
||||
})
|
||||
LLM.request({
|
||||
model,
|
||||
providerOptions: {
|
||||
// @ts-expect-error A pause setting is boolean.
|
||||
contextManagement: { edits: [{ type: "compact_20260112", pauseAfterCompaction: "yes" }] },
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { LLM } from "../../src/index.js"
|
||||
import { Mistral } from "../../src/providers.js"
|
||||
|
||||
const selected = Mistral.provider.model("mistral-small-latest")
|
||||
|
||||
LLM.request({ model: selected, prompt: "Hello", providerOptions: { reasoningEffort: "high" } })
|
||||
LLM.request({ model: selected, prompt: "Hello", providerOptions: { reasoningEffort: "future-effort" } })
|
||||
LLM.request({ model: selected, prompt: "Hello", providerOptions: { promptMode: "reasoning" } })
|
||||
LLM.request({ model: selected, prompt: "Hello", providerOptions: { parallelToolCalls: false } })
|
||||
LLM.request({ model: selected, prompt: "Hello", providerOptions: { promptCacheKey: "session-1" } })
|
||||
|
||||
LLM.request({
|
||||
model: selected,
|
||||
prompt: "Hello",
|
||||
// @ts-expect-error Mistral reasoning effort must be a string.
|
||||
providerOptions: { reasoningEffort: 1 },
|
||||
})
|
||||
|
||||
LLM.request({
|
||||
model: selected,
|
||||
prompt: "Hello",
|
||||
// @ts-expect-error Mistral prompt mode only supports reasoning.
|
||||
providerOptions: { promptMode: "standard" },
|
||||
})
|
||||
@@ -1,5 +1,7 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { model } from "@opencode-ai/ai/providers/openai"
|
||||
import { LLM } from "../src/index.js"
|
||||
import { Endpoint } from "../src/route/endpoint.js"
|
||||
|
||||
describe("provider package entrypoints", () => {
|
||||
test("semantic API aliases expose the same contract", async () => {
|
||||
@@ -36,7 +38,8 @@ describe("provider package entrypoints", () => {
|
||||
expect(modules[0].model).toBe(modules[1].model)
|
||||
expect(modules[8].model).toBe(modules[9].model)
|
||||
expect(modules[12].model).toBe(modules[13].model)
|
||||
expect(modules[19].model).toBe(modules[20].model)
|
||||
expect(modules[19].model).toBe(modules[21].model)
|
||||
expect(modules[19].model).not.toBe(modules[20].model)
|
||||
})
|
||||
|
||||
test("maps DeepInfra package settings onto its native executable model", async () => {
|
||||
@@ -139,8 +142,10 @@ describe("provider package entrypoints", () => {
|
||||
expect(selected.route.id).toBe("anthropic-messages")
|
||||
expect(selected.route.endpoint).toMatchObject({
|
||||
baseURL: "https://messages.example.test/v1",
|
||||
path: "/messages",
|
||||
})
|
||||
expect(
|
||||
Endpoint.render(selected.route.endpoint, { request: LLM.request({ model: selected }), body: {} }).toString(),
|
||||
).toBe("https://messages.example.test/v1/messages")
|
||||
expect(selected.route.defaults.headers).toEqual({ "x-application": "opencode" })
|
||||
expect(selected.route.defaults.http?.body).toEqual({ metadata: { user_id: "user_1" } })
|
||||
expect(selected.route.defaults.providerOptions).toEqual({ effort: "low" })
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
import { expect } from "bun:test"
|
||||
import { Effect, Schema } from "effect"
|
||||
import { LLM, LLMRequest, Message } from "../../src/index.js"
|
||||
import { LLMClient } from "../../src/route/client.js"
|
||||
import { Anthropic, GoogleVertexMessages } from "../../src/providers/index.js"
|
||||
import { testEffect } from "../lib/effect.js"
|
||||
import { dynamicResponse, fixedResponse } from "../lib/http.js"
|
||||
import { sseEvents } from "../lib/sse.js"
|
||||
|
||||
for (const fixture of [
|
||||
{
|
||||
name: "empty iterations fall back to top-level usage",
|
||||
usage: { input_tokens: 2, output_tokens: 3, cache_read_input_tokens: null, iterations: [] },
|
||||
expected: { inputTokens: 2, outputTokens: 3, totalTokens: 5, contextTokens: undefined },
|
||||
},
|
||||
{
|
||||
name: "compaction-only usage has no post-compaction context size",
|
||||
usage: {
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
iterations: [{ type: "compaction", input_tokens: 7, cache_read_input_tokens: 3, output_tokens: 2 }],
|
||||
},
|
||||
expected: { inputTokens: 10, outputTokens: 2, totalTokens: 12, contextTokens: undefined },
|
||||
},
|
||||
{
|
||||
name: "partially reported iterations preserve known totals",
|
||||
usage: {
|
||||
iterations: [
|
||||
{ type: "compaction", input_tokens: 7, cache_creation_input_tokens: 2 },
|
||||
{ type: "message", output_tokens: 3 },
|
||||
],
|
||||
},
|
||||
expected: { inputTokens: 9, outputTokens: 3, totalTokens: 12, contextTokens: undefined },
|
||||
},
|
||||
{
|
||||
name: "missing counters remain unknown rather than zero",
|
||||
usage: { iterations: [{ type: "message" }] },
|
||||
expected: { inputTokens: undefined, outputTokens: undefined, totalTokens: undefined, contextTokens: undefined },
|
||||
},
|
||||
]) {
|
||||
testEffect(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
{ type: "message_start", message: { usage: fixture.usage } },
|
||||
{ type: "message_delta", delta: { stop_reason: "end_turn" } },
|
||||
{ type: "message_stop" },
|
||||
),
|
||||
),
|
||||
).effect(fixture.name, () =>
|
||||
Effect.gen(function* () {
|
||||
const result = yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
model: Anthropic.configure({ apiKey: "test" }).model("claude-opus-4-6"),
|
||||
prompt: "hello",
|
||||
}),
|
||||
)
|
||||
expect(result.usage).toMatchObject(fixture.expected)
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
for (const model of [
|
||||
Anthropic.configure({ apiKey: "test" }).model("claude-opus-4-6"),
|
||||
GoogleVertexMessages.configure({ accessToken: "test", project: "test" }).model("claude-opus-4-6"),
|
||||
]) {
|
||||
for (const summary of ["Summary of the conversation", null]) {
|
||||
const block = { type: "compaction", content: summary }
|
||||
testEffect(
|
||||
dynamicResponse(({ request, text, respond }) =>
|
||||
Effect.sync(() => {
|
||||
const body = JSON.parse(text)
|
||||
expect(request.headers["anthropic-beta"]).toBe("existing-beta,compact-2026-01-12")
|
||||
if (body.messages.length === 1) {
|
||||
expect(body.context_management.edits).toEqual([
|
||||
{
|
||||
type: "compact_20260112",
|
||||
trigger: { type: "input_tokens", value: 50000 },
|
||||
pause_after_compaction: true,
|
||||
},
|
||||
])
|
||||
}
|
||||
if (body.messages.length > 1) {
|
||||
expect(body.messages[1].content).toEqual([block])
|
||||
expect(body.context_management).toBeUndefined()
|
||||
}
|
||||
return respond(
|
||||
sseEvents(
|
||||
{ type: "message_start", message: { usage: { input_tokens: 50000, output_tokens: 0 } } },
|
||||
{ type: "content_block_start", index: 0, content_block: { type: "compaction", content: null } },
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "compaction_delta", content: summary } },
|
||||
{ type: "content_block_stop", index: 0 },
|
||||
{
|
||||
type: "message_delta",
|
||||
delta: { stop_reason: "compaction" },
|
||||
usage: {
|
||||
input_tokens: 1000,
|
||||
output_tokens: 5,
|
||||
iterations: [
|
||||
{ type: "compaction", input_tokens: 50000, output_tokens: 1000, cache_read_input_tokens: 10 },
|
||||
{ type: "message", input_tokens: 1000, output_tokens: 5 },
|
||||
],
|
||||
},
|
||||
},
|
||||
{ type: "message_stop" },
|
||||
),
|
||||
{ headers: { "content-type": "text/event-stream" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
).effect(
|
||||
`${model.provider} replays ${summary === null ? "failed" : "successful"} compaction with billing and context usage`,
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const request = LLM.request({
|
||||
model,
|
||||
prompt: "hello",
|
||||
http: { headers: { "anthropic-beta": "existing-beta" } },
|
||||
providerOptions: {
|
||||
contextManagement: {
|
||||
edits: [
|
||||
{
|
||||
type: "compact_20260112",
|
||||
trigger: { type: "input_tokens", value: 50000 },
|
||||
pauseAfterCompaction: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
})
|
||||
const first = yield* LLMClient.generate(request)
|
||||
expect(first.finishReason.raw).toBe("compaction")
|
||||
expect(first.message.content).toEqual([{ type: "compaction", provider: model.provider, text: summary }])
|
||||
expect(first.text).toBe("")
|
||||
expect(first.usage?.inputTokens).toBe(51010)
|
||||
expect(first.usage?.outputTokens).toBe(1005)
|
||||
expect(first.usage?.totalTokens).toBe(52015)
|
||||
expect(first.usage?.contextTokens).toBe(1000)
|
||||
const codec = Schema.fromJsonString(Message)
|
||||
const message = Schema.decodeSync(codec)(Schema.encodeSync(codec)(first.message))
|
||||
yield* LLMClient.generate(
|
||||
LLMRequest.update(request, {
|
||||
providerOptions: {},
|
||||
messages: [...request.messages, message, Message.user("continue")],
|
||||
}),
|
||||
)
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
for (const events of [
|
||||
[{ type: "content_block_start", index: 0, content_block: { type: "compaction", content: 42 } }],
|
||||
[{ type: "content_block_delta", index: 0, delta: { type: "compaction_delta", content: "no start" } }],
|
||||
[
|
||||
{ type: "content_block_start", index: 0, content_block: { type: "compaction", content: null } },
|
||||
{ type: "message_stop" },
|
||||
],
|
||||
]) {
|
||||
testEffect(fixedResponse(sseEvents(...events))).effect(
|
||||
`rejects malformed compaction lifecycle: ${JSON.stringify(events)}`,
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(
|
||||
LLM.request({ model: Anthropic.configure({ apiKey: "test" }).model("claude-opus-4-6"), prompt: "hello" }),
|
||||
).pipe(Effect.flip)
|
||||
expect(error.reason._tag).toBe("InvalidProviderOutput")
|
||||
expect(error.reason.http?.status).toBe(200)
|
||||
}),
|
||||
)
|
||||
}
|
||||
@@ -74,6 +74,48 @@ describe("Anthropic Messages route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("omits empty system text while preserving whitespace", () =>
|
||||
Effect.gen(function* () {
|
||||
const empty = yield* compileRequest(LLMRequest.update(request, { system: [{ type: "text", text: "" }] }))
|
||||
const whitespace = yield* compileRequest(LLMRequest.update(request, { system: [{ type: "text", text: " " }] }))
|
||||
|
||||
expect(empty.body.system).toBeUndefined()
|
||||
expect(whitespace.body.system).toEqual([{ type: "text", text: " " }])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("filters whitespace-only text and removes empty messages", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
Message.user(" \n\t"),
|
||||
Message.user([]),
|
||||
Message.user([
|
||||
{ type: "text", text: "" },
|
||||
{ type: "text", text: " Keep this spacing. " },
|
||||
{ type: "text", text: " \n\t" },
|
||||
]),
|
||||
Message.assistant(" \n\t"),
|
||||
Message.assistant([]),
|
||||
Message.assistant([{ type: "reasoning", text: "" }]),
|
||||
Message.assistant([
|
||||
{ type: "text", text: "" },
|
||||
{ type: "reasoning", text: "", providerMetadata: { anthropic: { signature: "sig_1" } } },
|
||||
]),
|
||||
],
|
||||
cache: "none",
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.messages).toEqual([
|
||||
{ role: "user", content: [{ type: "text", text: " Keep this spacing. " }] },
|
||||
{ role: "assistant", content: [{ type: "thinking", thinking: "", signature: "sig_1" }] },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers adaptive thinking settings with effort", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
@@ -903,6 +945,54 @@ describe("Anthropic Messages route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves a reasoning signature when message_stop closes the block", () =>
|
||||
Effect.gen(function* () {
|
||||
const compatible = Route.make({
|
||||
id: "custom-anthropic-messages",
|
||||
provider: "custom-anthropic",
|
||||
protocol: AnthropicMessages.protocol,
|
||||
endpoint: Endpoint.path("/messages", { baseURL: "https://compatible.test/v1" }),
|
||||
auth: Auth.header("x-api-key", "test"),
|
||||
framing: AnthropicMessages.framing,
|
||||
}).model({ id: "custom-model" })
|
||||
const body = sseEvents(
|
||||
{ type: "message_start", message: { usage: { input_tokens: 5 } } },
|
||||
{
|
||||
type: "content_block_start",
|
||||
index: 0,
|
||||
content_block: { type: "thinking", thinking: "", signature: "" },
|
||||
},
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "thinking_delta", thinking: "Reasoning." } },
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "signature_delta", signature: "sig_1" } },
|
||||
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 2 } },
|
||||
{ type: "message_stop" },
|
||||
)
|
||||
const response = yield* LLMClient.generate(LLM.request({ model: compatible, prompt: "Think." })).pipe(
|
||||
Effect.provide(fixedResponse(body)),
|
||||
)
|
||||
|
||||
const reasoningEnds = response.events.filter((event) => event.type === "reasoning-end")
|
||||
expect(reasoningEnds).toHaveLength(1)
|
||||
expect(reasoningEnds[0]).toMatchObject({
|
||||
providerMetadata: { "custom-anthropic": { signature: "sig_1" } },
|
||||
})
|
||||
expect(response.message.content).toEqual([
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Reasoning.",
|
||||
providerMetadata: { "custom-anthropic": { signature: "sig_1" } },
|
||||
},
|
||||
])
|
||||
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({ model: compatible, messages: [response.message], cache: "none" }),
|
||||
)
|
||||
expect(prepared.body.messages).toEqual([
|
||||
{ role: "assistant", content: [{ type: "thinking", thinking: "Reasoning.", signature: "sig_1" }] },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("parses text, reasoning, and usage stream fixtures", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
@@ -936,6 +1026,7 @@ describe("Anthropic Messages route", () => {
|
||||
expect(response.events.find((event) => event.type === "reasoning-end")).toMatchObject({
|
||||
providerMetadata: { anthropic: { signature: "sig_1" } },
|
||||
})
|
||||
expect(response.events.filter((event) => event.type === "reasoning-end")).toHaveLength(1)
|
||||
expect(response.events.find((event) => event.type === "reasoning-delta" && event.text === "")).toBeUndefined()
|
||||
expect(response.message.content).toEqual([
|
||||
{ type: "text", text: "Hello!" },
|
||||
@@ -949,6 +1040,41 @@ describe("Anthropic Messages route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves terminal state across usage-only message deltas", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
{ type: "message_start", message: { usage: { input_tokens: 5 } } },
|
||||
{
|
||||
type: "message_delta",
|
||||
delta: { stop_reason: "end_turn", stop_sequence: "X" },
|
||||
usage: { output_tokens: 8 },
|
||||
},
|
||||
{ type: "message_delta", delta: {}, usage: { output_tokens: 10 } },
|
||||
{ type: "message_stop" },
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.usage).toMatchObject({ inputTokens: 5, outputTokens: 10, totalTokens: 15 })
|
||||
expect(response.finishReason).toEqual({ normalized: "stop", raw: "end_turn" })
|
||||
expect(response.events.find((event) => event.type === "step-finish")).toMatchObject({
|
||||
reason: { normalized: "stop", raw: "end_turn" },
|
||||
usage: { inputTokens: 5, outputTokens: 10, totalTokens: 15 },
|
||||
providerMetadata: { anthropic: { stopSequence: "X" } },
|
||||
})
|
||||
expect(response.events.at(-1)).toMatchObject({
|
||||
type: "finish",
|
||||
reason: { normalized: "stop", raw: "end_turn" },
|
||||
usage: { inputTokens: 5, outputTokens: 10, totalTokens: 15 },
|
||||
providerMetadata: { anthropic: { stopSequence: "X" } },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("requires message_stop before completing a streamed message", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(request).pipe(
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { EventStreamCodec } from "@smithy/eventstream-codec"
|
||||
import { fromUtf8, toUtf8 } from "@smithy/util-utf8"
|
||||
import { expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { LLM, LLMRequest, Message } from "../../src/index.js"
|
||||
import { LLMClient } from "../../src/route/client.js"
|
||||
import { AmazonBedrock } from "../../src/providers/index.js"
|
||||
import { testEffect } from "../lib/effect.js"
|
||||
import { dynamicResponse } from "../lib/http.js"
|
||||
|
||||
const codec = new EventStreamCodec(toUtf8, fromUtf8)
|
||||
const frame = (event: object) =>
|
||||
codec.encode({
|
||||
headers: { ":message-type": { type: "string", value: "event" }, ":event-type": { type: "string", value: "chunk" } },
|
||||
body: new TextEncoder().encode(JSON.stringify({ bytes: Buffer.from(JSON.stringify(event)).toString("base64") })),
|
||||
})
|
||||
const response = Buffer.concat(
|
||||
[
|
||||
{ type: "message_start", message: { usage: { input_tokens: 60000 } } },
|
||||
{ type: "content_block_start", index: 0, content_block: { type: "compaction", content: null } },
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "compaction_delta", content: "Summary" } },
|
||||
{ type: "content_block_stop", index: 0 },
|
||||
{ type: "content_block_start", index: 1, content_block: { type: "text", text: "" } },
|
||||
{ type: "content_block_delta", index: 1, delta: { type: "text_delta", text: "Hello" } },
|
||||
{ type: "content_block_stop", index: 1 },
|
||||
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 10 } },
|
||||
{ type: "message_stop" },
|
||||
].map(frame),
|
||||
)
|
||||
|
||||
for (const auth of [
|
||||
{ apiKey: "test" },
|
||||
{ credentials: { accessKeyId: "test", secretAccessKey: "test", region: "us-west-2" } },
|
||||
]) {
|
||||
testEffect(
|
||||
dynamicResponse(({ request, text, respond }) =>
|
||||
Effect.sync(() => {
|
||||
expect(request.url).toBe(
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-opus-4-6-v1%3A0/invoke-with-response-stream",
|
||||
)
|
||||
expect(request.headers.authorization).toStartWith(auth.apiKey ? "Bearer test" : "AWS4-HMAC-SHA256")
|
||||
const body = JSON.parse(text)
|
||||
expect(body.model).toBeUndefined()
|
||||
expect(body.stream).toBeUndefined()
|
||||
expect(body.anthropic_version).toBe("bedrock-2023-05-31")
|
||||
expect(body.anthropic_beta).toEqual(["compact-2026-01-12"])
|
||||
expect(body.context_management.edits).toEqual([{ type: "compact_20260112" }])
|
||||
if (body.messages.length > 1)
|
||||
expect(body.messages[1].content[0]).toEqual({ type: "compaction", content: "Summary" })
|
||||
return respond(response, { headers: { "content-type": "application/vnd.amazon.eventstream" } })
|
||||
}),
|
||||
),
|
||||
).effect(`Bedrock Messages compaction round trip with ${auth.apiKey ? "bearer" : "SigV4"} authentication`, () =>
|
||||
Effect.gen(function* () {
|
||||
const model = AmazonBedrock.configure({ ...auth, region: "us-west-2" }).messages("anthropic.claude-opus-4-6-v1:0")
|
||||
const request = LLM.request({
|
||||
model,
|
||||
prompt: "hello",
|
||||
providerOptions: { contextManagement: { edits: [{ type: "compact_20260112" }] } },
|
||||
})
|
||||
const first = yield* LLMClient.generate(request)
|
||||
expect(first.text).toBe("Hello")
|
||||
expect(first.message.content.map((part) => part.type)).toEqual(["compaction", "text"])
|
||||
yield* LLMClient.generate(
|
||||
LLMRequest.update(request, { messages: [...request.messages, first.message, Message.user("continue")] }),
|
||||
)
|
||||
}),
|
||||
)
|
||||
}
|
||||
@@ -1,11 +1,12 @@
|
||||
import { EventStreamCodec } from "@smithy/eventstream-codec"
|
||||
import { fromUtf8, toUtf8 } from "@smithy/util-utf8"
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { Effect, Encoding, Ref, Stream } from "effect"
|
||||
import {
|
||||
CacheHint,
|
||||
GenerationOptions,
|
||||
LLM,
|
||||
LLMEvent,
|
||||
LLMRequest,
|
||||
Message,
|
||||
ToolCallPart,
|
||||
@@ -83,6 +84,17 @@ const eventStreamBody = (...payloads: ReadonlyArray<readonly [string, object]>)
|
||||
const fixedBytes = (bytes: Uint8Array) =>
|
||||
fixedResponse(bytes.slice().buffer, { headers: { "content-type": "application/vnd.amazon.eventstream" } })
|
||||
|
||||
const fixedByteChunks = (...chunks: ReadonlyArray<Uint8Array>) =>
|
||||
fixedResponse(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
chunks.forEach((chunk) => controller.enqueue(chunk))
|
||||
controller.close()
|
||||
},
|
||||
}),
|
||||
{ headers: { "content-type": "application/vnd.amazon.eventstream" } },
|
||||
)
|
||||
|
||||
const model = AmazonBedrock.configure({
|
||||
baseURL: "https://bedrock-runtime.test",
|
||||
apiKey: "test-bearer",
|
||||
@@ -113,6 +125,50 @@ describe("Bedrock Converse route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("omits empty initial system blocks", () =>
|
||||
Effect.gen(function* () {
|
||||
const empty = yield* compileRequest(LLM.request({ model, system: "", prompt: "hello" }))
|
||||
const cachedEmpty = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
system: [{ type: "text", text: "", cache: new CacheHint({ type: "ephemeral" }) }],
|
||||
prompt: "hello",
|
||||
cache: "none",
|
||||
}),
|
||||
)
|
||||
|
||||
expect(empty.body.system).toBeUndefined()
|
||||
expect(cachedEmpty.body.system).toBeUndefined()
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("omits empty system blocks while preserving order and cache hints", () =>
|
||||
Effect.gen(function* () {
|
||||
const cache = new CacheHint({ type: "ephemeral" })
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
system: [
|
||||
{ type: "text", text: "", cache },
|
||||
{ type: "text", text: "First." },
|
||||
{ type: "text", text: " " },
|
||||
{ type: "text", text: "" },
|
||||
{ type: "text", text: "Second.", cache },
|
||||
],
|
||||
prompt: "hello",
|
||||
cache: "none",
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.system).toEqual([
|
||||
{ text: "First." },
|
||||
{ text: " " },
|
||||
{ text: "Second." },
|
||||
{ cachePoint: { type: "default" } },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("passes topK through additionalModelRequestFields as top_k", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
@@ -255,6 +311,79 @@ describe("Bedrock Converse route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("removes empty keys recursively from outbound tool inputs without mutating history", () =>
|
||||
Effect.gen(function* () {
|
||||
const input = {
|
||||
path: "file.ts",
|
||||
edits: [
|
||||
{ oldText: "a", newText: "b", "": "" },
|
||||
null,
|
||||
true,
|
||||
7,
|
||||
"text",
|
||||
["kept", { "": false, nested: { "": null, value: "ok" } }],
|
||||
],
|
||||
nested: { "": "drop", empty: {}, onlyEmpty: { "": 1 } },
|
||||
" ": "preserve whitespace key",
|
||||
"": "drop",
|
||||
}
|
||||
const original = structuredClone(input)
|
||||
const call = ToolCallPart.make({ id: "tool_1", name: "edit", input })
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({ model, messages: [Message.assistant([call])], cache: "none" }),
|
||||
)
|
||||
|
||||
expect(prepared.body.messages).toEqual([
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
toolUse: {
|
||||
toolUseId: "tool_1",
|
||||
name: "edit",
|
||||
input: {
|
||||
path: "file.ts",
|
||||
edits: [{ oldText: "a", newText: "b" }, null, true, 7, "text", ["kept", { nested: { value: "ok" } }]],
|
||||
nested: { empty: {}, onlyEmpty: {} },
|
||||
" ": "preserve whitespace key",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
])
|
||||
expect(input).toEqual(original)
|
||||
expect(call.input).toBe(input)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps empty tool inputs and empties inputs containing only empty keys", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
Message.assistant([
|
||||
ToolCallPart.make({ id: "tool_empty_key", name: "first", input: { "": { value: true } } }),
|
||||
ToolCallPart.make({ id: "tool_empty_object", name: "second", input: {} }),
|
||||
]),
|
||||
],
|
||||
cache: "none",
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.messages).toEqual([
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ toolUse: { toolUseId: "tool_empty_key", name: "first", input: {} } },
|
||||
{ toolUse: { toolUseId: "tool_empty_object", name: "second", input: {} } },
|
||||
],
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("merges parallel tool results into one user message", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
@@ -385,19 +514,77 @@ describe("Bedrock Converse route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("maps truncation and malformed output stop reasons", () =>
|
||||
it.effect("rejects truncated event-stream frames after message stop", () =>
|
||||
Effect.gen(function* () {
|
||||
const reasons = [
|
||||
["model_context_window_exceeded", "length"],
|
||||
["malformed_model_output", "error"],
|
||||
["malformed_tool_use", "error"],
|
||||
] as const
|
||||
const partialFrames = [
|
||||
eventFrame("metadata", { usage: { inputTokens: 5, outputTokens: 2, totalTokens: 7 } }).subarray(0, 3),
|
||||
exceptionFrame("modelStreamErrorException", { originalMessage: "Upstream model failed" }).subarray(0, -1),
|
||||
]
|
||||
|
||||
for (const [raw, normalized] of reasons) {
|
||||
const response = yield* LLMClient.generate(baseRequest).pipe(
|
||||
Effect.provide(fixedBytes(eventStreamBody(["messageStop", { stopReason: raw }]))),
|
||||
for (const partial of partialFrames) {
|
||||
const error = yield* LLMClient.generate(baseRequest).pipe(
|
||||
Effect.provide(fixedBytes(concat([eventFrame("messageStop", { stopReason: "end_turn" }), partial]))),
|
||||
Effect.flip,
|
||||
)
|
||||
expect(response.finishReason).toEqual({ normalized, raw })
|
||||
|
||||
expect(error).toMatchObject({
|
||||
reason: { _tag: "InvalidProviderOutput", classification: "incomplete-stream" },
|
||||
message: `Incomplete Bedrock Converse event-stream frame: ${partial.length} buffered bytes remain at end of stream`,
|
||||
})
|
||||
expect(error.reason.body).toBe(Encoding.encodeBase64(partial))
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes frames split across transport chunks through exact-boundary EOF", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = eventStreamBody(
|
||||
["messageStart", { role: "assistant" }],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { text: "Hello" } }],
|
||||
["messageStop", { stopReason: "end_turn" }],
|
||||
)
|
||||
const response = yield* LLMClient.generate(baseRequest).pipe(
|
||||
Effect.provide(fixedByteChunks(body.subarray(0, 2), body.subarray(2, 17), body.subarray(17))),
|
||||
)
|
||||
|
||||
expect(response.text).toBe("Hello")
|
||||
expect(response.finishReason).toEqual({ normalized: "stop", raw: "end_turn" })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("maps model context window exhaustion to length", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(baseRequest).pipe(
|
||||
Effect.provide(fixedBytes(eventStreamBody(["messageStop", { stopReason: "model_context_window_exceeded" }]))),
|
||||
)
|
||||
|
||||
expect(response.finishReason).toEqual({
|
||||
normalized: "length",
|
||||
raw: "model_context_window_exceeded",
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails malformed output stop reasons", () =>
|
||||
Effect.gen(function* () {
|
||||
for (const reason of ["malformed_model_output", "malformed_tool_use"] as const) {
|
||||
const events = yield* Ref.make<ReadonlyArray<LLMEvent>>([])
|
||||
const error = yield* LLMClient.stream(baseRequest).pipe(
|
||||
Stream.tap((event) => Ref.update(events, (current) => [...current, event])),
|
||||
Stream.runDrain,
|
||||
Effect.provide(fixedBytes(eventStreamBody(["messageStop", { stopReason: reason }]))),
|
||||
Effect.flip,
|
||||
)
|
||||
|
||||
expect(error).toMatchObject({
|
||||
reason: { _tag: "InvalidProviderOutput" },
|
||||
message: `Bedrock Converse stopped with ${reason}`,
|
||||
})
|
||||
expect(JSON.parse(error.reason.body ?? "")).toMatchObject({
|
||||
headers: { ":event-type": { value: "messageStop" } },
|
||||
body: JSON.stringify({ stopReason: reason }),
|
||||
})
|
||||
expect((yield* Ref.get(events)).some((event) => event.type === "finish")).toBeFalse()
|
||||
}
|
||||
}),
|
||||
)
|
||||
@@ -491,6 +678,59 @@ describe("Bedrock Converse route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("ignores late tool deltas after contentBlockStop", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = eventStreamBody(
|
||||
[
|
||||
"contentBlockStart",
|
||||
{
|
||||
contentBlockIndex: 0,
|
||||
start: { toolUse: { toolUseId: "tool_1", name: "lookup" } },
|
||||
},
|
||||
],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { toolUse: { input: '{"query":"weather"}' } } }],
|
||||
["contentBlockStop", { contentBlockIndex: 0 }],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { toolUse: { input: '{"late":true}' } } }],
|
||||
["messageStop", { stopReason: "tool_use" }],
|
||||
)
|
||||
const response = yield* LLMClient.generate(baseRequest).pipe(Effect.provide(fixedBytes(body)))
|
||||
|
||||
expect(response.toolCalls).toEqual([
|
||||
{ type: "tool-call", id: "tool_1", name: "lookup", input: { query: "weather" } },
|
||||
])
|
||||
expect(response.events.filter((event) => event.type === "tool-input-delta")).toEqual([
|
||||
{
|
||||
type: "tool-input-delta",
|
||||
id: "tool_1",
|
||||
name: "lookup",
|
||||
text: '{"query":"weather"}',
|
||||
input: { query: "weather" },
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects tool deltas without contentBlockStart", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(baseRequest).pipe(
|
||||
Effect.provide(
|
||||
fixedBytes(
|
||||
eventStreamBody(
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { toolUse: { input: "{}" } } }],
|
||||
["messageStop", { stopReason: "tool_use" }],
|
||||
),
|
||||
),
|
||||
),
|
||||
Effect.flip,
|
||||
)
|
||||
|
||||
expect(error).toMatchObject({
|
||||
reason: { _tag: "InvalidProviderOutput" },
|
||||
message: "Bedrock Converse tool delta is missing its tool call",
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("recovers incomplete tool input at finalization", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = eventStreamBody(
|
||||
@@ -531,6 +771,104 @@ describe("Bedrock Converse route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
for (const signature of [undefined, "", " "]) {
|
||||
for (const cache of ["none", "auto"] as const) {
|
||||
it.effect(`demotes unsigned reasoning to text (${JSON.stringify(signature)}, cache: ${cache})`, () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
Message.user("Think"),
|
||||
Message.assistant([
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Partial thought",
|
||||
providerMetadata: signature === undefined ? undefined : { bedrock: { signature } },
|
||||
cache: new CacheHint({ type: "ephemeral" }),
|
||||
},
|
||||
]),
|
||||
Message.user("Continue"),
|
||||
],
|
||||
cache,
|
||||
}),
|
||||
)
|
||||
expect(prepared.body.messages[1]).toEqual({
|
||||
role: "assistant",
|
||||
content: [{ text: "Partial thought" }, { cachePoint: { type: "default" } }],
|
||||
})
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
it.effect("omits empty unsigned reasoning without leaving an empty or cache-only assistant", () =>
|
||||
Effect.gen(function* () {
|
||||
const cache = new CacheHint({ type: "ephemeral" })
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
Message.user("Think"),
|
||||
Message.assistant([
|
||||
{ type: "reasoning", text: "", cache },
|
||||
{ type: "reasoning", text: " ", providerMetadata: { bedrock: { signature: "" } }, cache },
|
||||
]),
|
||||
Message.user([{ type: "text", text: "Continue", cache }]),
|
||||
],
|
||||
cache: "none",
|
||||
}),
|
||||
)
|
||||
expect(prepared.body.messages).toEqual([
|
||||
{
|
||||
role: "user",
|
||||
content: [{ text: "Think" }, { text: "Continue" }, { cachePoint: { type: "default" } }],
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("demotes foreign reasoning while preserving signed, redacted, text, and tool blocks", () =>
|
||||
Effect.gen(function* () {
|
||||
const cache = new CacheHint({ type: "ephemeral" })
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
Message.assistant([
|
||||
{ type: "reasoning", text: "Foreign thought", providerMetadata: { anthropic: { signature: "old" } } },
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Signed thought",
|
||||
providerMetadata: { bedrock: { signature: "sig_1" } },
|
||||
cache,
|
||||
},
|
||||
{ type: "reasoning", text: "", encrypted: "sig_2", cache },
|
||||
{ type: "reasoning", text: "", providerMetadata: { bedrock: { redactedData: "cmVkYWN0ZWQ=" } }, cache },
|
||||
{ type: "text", text: "Checking" },
|
||||
ToolCallPart.make({ id: "call_1", name: "lookup", input: {} }),
|
||||
]),
|
||||
],
|
||||
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
|
||||
cache: "none",
|
||||
}),
|
||||
)
|
||||
expect(prepared.body.messages).toEqual([
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ text: "Foreign thought" },
|
||||
{ reasoningContent: { reasoningText: { text: "Signed thought", signature: "sig_1" } } },
|
||||
{ reasoningContent: { reasoningText: { text: "", signature: "sig_2" } } },
|
||||
{ reasoningContent: { redactedContent: "cmVkYWN0ZWQ=" } },
|
||||
{ text: "Checking" },
|
||||
{ toolUse: { toolUseId: "call_1", name: "lookup", input: {} } },
|
||||
],
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves streamed reasoning signatures for continuation lowering", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = eventStreamBody(
|
||||
@@ -702,7 +1040,7 @@ describe("Bedrock Converse route", () => {
|
||||
Effect.gen(function* () {
|
||||
// Bedrock represents redactedContent blobs as base64 strings on its JSON
|
||||
// wire. The provider owns the payload and requires byte-exact replay.
|
||||
const redactedData = "cmVkYWN0ZWQtdGhpbmtpbmc="
|
||||
const redactedData = "AQID"
|
||||
const response = yield* LLMClient.generate(
|
||||
LLMRequest.update(baseRequest, {
|
||||
tools: [ToolDefinition.make({ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } })],
|
||||
@@ -712,10 +1050,8 @@ describe("Bedrock Converse route", () => {
|
||||
fixedBytes(
|
||||
eventStreamBody(
|
||||
["messageStart", { role: "assistant" }],
|
||||
[
|
||||
"contentBlockDelta",
|
||||
{ contentBlockIndex: 0, delta: { reasoningContent: { redactedContent: redactedData } } },
|
||||
],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { redactedContent: "AQ==" } } }],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { redactedContent: "AgM=" } } }],
|
||||
["contentBlockStop", { contentBlockIndex: 0 }],
|
||||
[
|
||||
"contentBlockStart",
|
||||
@@ -731,12 +1067,17 @@ describe("Bedrock Converse route", () => {
|
||||
),
|
||||
),
|
||||
)
|
||||
expect(response.events.find((event) => event.type === "reasoning-delta" && event.text === "")).toEqual({
|
||||
expect(response.events.filter((event) => event.type === "reasoning-delta" && event.text === "").at(-1)).toEqual({
|
||||
type: "reasoning-delta",
|
||||
id: "reasoning-0",
|
||||
text: "",
|
||||
providerMetadata: { bedrock: { redactedData } },
|
||||
})
|
||||
expect(response.events.find((event) => event.type === "reasoning-end")).toEqual({
|
||||
type: "reasoning-end",
|
||||
id: "reasoning-0",
|
||||
providerMetadata: { bedrock: { redactedData } },
|
||||
})
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
@@ -767,6 +1108,73 @@ describe("Bedrock Converse route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps redacted reasoning accumulation separate by content block index", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(baseRequest).pipe(
|
||||
Effect.provide(
|
||||
fixedBytes(
|
||||
eventStreamBody(
|
||||
["messageStart", { role: "assistant" }],
|
||||
["contentBlockDelta", { contentBlockIndex: 2, delta: { reasoningContent: { redactedContent: "AQ==" } } }],
|
||||
["contentBlockDelta", { contentBlockIndex: 2, delta: { reasoningContent: { redactedContent: "Ag==" } } }],
|
||||
["contentBlockStop", { contentBlockIndex: 2 }],
|
||||
["contentBlockDelta", { contentBlockIndex: 7, delta: { reasoningContent: { redactedContent: "Aw==" } } }],
|
||||
["contentBlockDelta", { contentBlockIndex: 7, delta: { reasoningContent: { redactedContent: "BA==" } } }],
|
||||
["contentBlockStop", { contentBlockIndex: 7 }],
|
||||
["messageStop", { stopReason: "end_turn" }],
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.message.content).toEqual([
|
||||
{ type: "reasoning", text: "", providerMetadata: { bedrock: { redactedData: "AQI=" } } },
|
||||
{ type: "reasoning", text: "", providerMetadata: { bedrock: { redactedData: "AwQ=" } } },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves split redacted reasoning when contentBlockStop is missing", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(baseRequest).pipe(
|
||||
Effect.provide(
|
||||
fixedBytes(
|
||||
eventStreamBody(
|
||||
["messageStart", { role: "assistant" }],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { redactedContent: "AQ==" } } }],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { redactedContent: "AgM=" } } }],
|
||||
["messageStop", { stopReason: "end_turn" }],
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.message.content).toEqual([
|
||||
{ type: "reasoning", text: "", providerMetadata: { bedrock: { redactedData: "AQID" } } },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects invalid redacted reasoning base64 with the triggering event", () =>
|
||||
Effect.gen(function* () {
|
||||
const payload = { contentBlockIndex: 0, delta: { reasoningContent: { redactedContent: "%%==" } } }
|
||||
const error = yield* LLMClient.generate(baseRequest).pipe(
|
||||
Effect.provide(fixedBytes(eventStreamBody(["contentBlockDelta", payload]))),
|
||||
Effect.flip,
|
||||
)
|
||||
|
||||
expect(error).toMatchObject({
|
||||
reason: { _tag: "InvalidProviderOutput" },
|
||||
message: "Bedrock Converse reasoningContent.redactedContent contains invalid base64 data",
|
||||
})
|
||||
expect(JSON.parse(error.reason.body ?? "")).toMatchObject({
|
||||
headers: { ":event-type": { value: "contentBlockDelta" } },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
expect(error.reason.cause).toBeInstanceOf(Error)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("ignores unknown normal stream events", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = concat([
|
||||
@@ -1021,6 +1429,20 @@ describe("Bedrock Converse route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects image media that is not valid base64", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [Message.user({ type: "media", mediaType: "image/png", data: "https://example.test/image.png" })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error).toMatchObject({ reason: { _tag: "InvalidRequest" } })
|
||||
expect(error.message).toContain("Bedrock Converse media data must be valid base64")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers document media into Bedrock document blocks with format and name", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
@@ -1144,6 +1566,37 @@ describe("Bedrock Converse route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects remote media URLs in tool results", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: {} })]),
|
||||
Message.tool({
|
||||
id: "call_1",
|
||||
name: "read",
|
||||
result: {
|
||||
type: "content",
|
||||
value: [
|
||||
{
|
||||
type: "file",
|
||||
uri: "https://example.test/report.pdf",
|
||||
mime: "application/pdf",
|
||||
name: "report.pdf",
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error).toMatchObject({ reason: { _tag: "InvalidRequest" } })
|
||||
expect(error.message).toContain("Bedrock Converse media data must be valid base64")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported image media types", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* compileRequest(
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
import { EventStreamCodec } from "@smithy/eventstream-codec"
|
||||
import { fromUtf8, toUtf8 } from "@smithy/util-utf8"
|
||||
import { expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { LLM, LLMRequest, Message } from "../../src/index.js"
|
||||
import { LLMClient, compileRequest } from "../../src/route/client.js"
|
||||
import { AmazonBedrock } from "../../src/providers/index.js"
|
||||
import { it, testEffect } from "../lib/effect.js"
|
||||
import { dynamicResponse, fixedResponse } from "../lib/http.js"
|
||||
|
||||
const codec = new EventStreamCodec(toUtf8, fromUtf8)
|
||||
const response = Buffer.concat(
|
||||
[
|
||||
{ type: "message_start", message: { usage: { input_tokens: 10 } } },
|
||||
{ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Hello" } },
|
||||
{ type: "content_block_stop", index: 0 },
|
||||
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 2 } },
|
||||
{ type: "message_stop" },
|
||||
].map((event) =>
|
||||
codec.encode({
|
||||
headers: {
|
||||
":message-type": { type: "string", value: "event" },
|
||||
":event-type": { type: "string", value: "chunk" },
|
||||
},
|
||||
body: new TextEncoder().encode(JSON.stringify({ bytes: Buffer.from(JSON.stringify(event)).toString("base64") })),
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
for (const auth of [
|
||||
{ apiKey: "test" },
|
||||
{ credentials: { accessKeyId: "test", secretAccessKey: "test", region: "us-west-2" } },
|
||||
]) {
|
||||
testEffect(
|
||||
dynamicResponse(({ request, text, respond }) =>
|
||||
Effect.sync(() => {
|
||||
expect(request.url).toBe(
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-opus-4-6-v1%3A0/invoke-with-response-stream",
|
||||
)
|
||||
expect(request.headers.authorization).toStartWith(auth.apiKey ? "Bearer test" : "AWS4-HMAC-SHA256")
|
||||
const body = JSON.parse(text)
|
||||
expect(body.model).toBeUndefined()
|
||||
expect(body.stream).toBeUndefined()
|
||||
expect(body.anthropic_version).toBe("bedrock-2023-05-31")
|
||||
expect(body.anthropic_beta).toEqual(["existing-beta"])
|
||||
if (body.messages.length > 1) expect(body.messages[1].content).toEqual([{ type: "text", text: "Hello" }])
|
||||
return respond(response, { headers: { "content-type": "application/vnd.amazon.eventstream" } })
|
||||
}),
|
||||
),
|
||||
).effect(`Bedrock Messages text round trip with ${auth.apiKey ? "bearer" : "SigV4"} authentication`, () =>
|
||||
Effect.gen(function* () {
|
||||
const provider = AmazonBedrock.configure({ ...auth, region: "us-west-2" })
|
||||
expect(provider.model("fixture").route.id).toBe("bedrock-converse")
|
||||
const request = LLM.request({
|
||||
model: provider.messages("anthropic.claude-opus-4-6-v1:0"),
|
||||
prompt: "hello",
|
||||
http: { headers: { "anthropic-beta": "existing-beta, existing-beta" } },
|
||||
})
|
||||
const first = yield* LLMClient.generate(request)
|
||||
expect(first.text).toBe("Hello")
|
||||
expect(first.usage?.totalTokens).toBe(12)
|
||||
yield* LLMClient.generate(
|
||||
LLMRequest.update(request, {
|
||||
messages: [...request.messages, first.message, Message.user("continue")],
|
||||
}),
|
||||
)
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
testEffect(
|
||||
fixedResponse(
|
||||
codec.encode({
|
||||
headers: {
|
||||
":message-type": { type: "string", value: "exception" },
|
||||
":exception-type": { type: "string", value: "throttlingException" },
|
||||
},
|
||||
body: new TextEncoder().encode(JSON.stringify({ message: "Too many requests", trace: "keep-original" })),
|
||||
}),
|
||||
),
|
||||
).effect("Bedrock Messages retains the original exception frame", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
model: AmazonBedrock.configure({ apiKey: "test" }).messages("claude"),
|
||||
prompt: "hello",
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
expect(error.reason.body).toContain("keep-original")
|
||||
expect(error.reason.http?.status).toBe(200)
|
||||
}),
|
||||
)
|
||||
|
||||
for (const mediaType of ["image/png", "application/pdf"]) {
|
||||
for (const data of ["https://example.com/media", "invalid base64!"]) {
|
||||
for (const role of ["user", "tool"] as const) {
|
||||
it.effect(`rejects ${role} ${mediaType} with ${data}`, () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* compileRequest(
|
||||
LLM.request({
|
||||
model: AmazonBedrock.configure({ apiKey: "test" }).messages("claude"),
|
||||
messages:
|
||||
role === "user"
|
||||
? [Message.user({ type: "media", mediaType, data })]
|
||||
: [
|
||||
Message.tool({
|
||||
id: "call_1",
|
||||
name: "read",
|
||||
result: { type: "content", value: [{ type: "file", mime: mediaType, uri: data }] },
|
||||
}),
|
||||
],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
expect(error.reason._tag).toBe("InvalidRequest")
|
||||
expect(error.message).toContain("Bedrock Messages")
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
it.effect("accepts inline image and document sources", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model: AmazonBedrock.configure({ apiKey: "test" }).messages("claude"),
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "media", mediaType: "image/png", data: "data:image/png;base64,AQID" },
|
||||
{ type: "media", mediaType: "application/pdf", data: "data:application/pdf;base64,AQID" },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
)
|
||||
expect(prepared.body.messages[0].content.map((block: { source: { type: string } }) => block.source.type)).toEqual([
|
||||
"base64",
|
||||
"base64",
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects Anthropic file IDs", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* compileRequest(
|
||||
LLM.request({
|
||||
model: AmazonBedrock.configure({ apiKey: "test" }).messages("claude"),
|
||||
messages: [Message.user({ type: "media", mediaType: "image/png", data: "", metadata: { file_id: "file_1" } })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
expect(error.message).toContain("file-ID")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported image formats", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* compileRequest(
|
||||
LLM.request({
|
||||
model: AmazonBedrock.configure({ apiKey: "test" }).messages("claude"),
|
||||
messages: [Message.user({ type: "media", mediaType: "image/svg+xml", data: "AQID" })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
expect(error.reason._tag).toBe("InvalidRequest")
|
||||
expect(error.message).toContain("JPEG, PNG, WebP, or GIF")
|
||||
}),
|
||||
)
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { ConfigProvider, Effect, Schema } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { LLM, LLMEvent } from "../../src/index.js"
|
||||
@@ -136,17 +136,15 @@ describe("Cloudflare", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("defaults AI Gateway id to default when omitted or blank", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(
|
||||
CloudflareAIGateway.configure({
|
||||
accountId: "test-account",
|
||||
gatewayId: "",
|
||||
gatewayApiKey: "test-token",
|
||||
}).model("workers-ai/@cf/meta/llama-3.3-70b-instruct").route.endpoint.baseURL,
|
||||
).toBe("https://gateway.ai.cloudflare.com/v1/test-account/default/compat")
|
||||
}),
|
||||
)
|
||||
test("defaults AI Gateway id to default when omitted or blank", () => {
|
||||
expect(
|
||||
CloudflareAIGateway.configure({
|
||||
accountId: "test-account",
|
||||
gatewayId: "",
|
||||
gatewayApiKey: "test-token",
|
||||
}).model("workers-ai/@cf/meta/llama-3.3-70b-instruct").route.endpoint.baseURL,
|
||||
).toBe("https://gateway.ai.cloudflare.com/v1/test-account/default/compat")
|
||||
})
|
||||
|
||||
it.effect("supports authenticated AI Gateway plus upstream provider auth", () =>
|
||||
Effect.gen(function* () {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user