Compare commits

..
10 Commits
47 changed files with 1102 additions and 265 deletions
+5
View File
@@ -0,0 +1,5 @@
---
"@opencode-ai/ai": patch
---
Report OpenAI prompt cache write tokens in normalized usage.
+5
View File
@@ -0,0 +1,5 @@
---
"@opencode-ai/ai": patch
---
Improve Anthropic and Bedrock prompt reuse with layered cache breakpoints that roll through long tool loops.
+3 -3
View File
@@ -10,7 +10,7 @@
## Conventions
Per-type constructors live on the type, not as top-level re-exports. Use `Message.system(...)`, `Message.user(...)`, `Message.assistant(...)`, `Message.tool(...)`, `Model.make(...)`, `ToolDefinition.make(...)`, `ToolCallPart.make(...)`, `ToolResultPart.make(...)`, `ToolChoice.make(...)`, `ToolChoice.named(...)`, `SystemPart.make(...)`, and `GenerationOptions.make(...)` directly. The top-level `LLM` namespace is reserved for request-shaped call APIs: `LLM.request`, `LLM.generateTurn`, `LLM.streamTurn`, and `LLM.generateObject`. Two ways to construct the same thing is one too many.
Per-type constructors live on the type, not as top-level re-exports. Use `Message.system(...)`, `Message.user(...)`, `Message.assistant(...)`, `Message.tool(...)`, `Model.make(...)`, `ToolDefinition.make(...)`, `ToolCallPart.make(...)`, `ToolResultPart.make(...)`, `ToolChoice.make(...)`, `ToolChoice.named(...)`, `SystemPart.make(...)`, and `GenerationOptions.make(...)` directly. The top-level `LLM` namespace is reserved for request-shaped call APIs: `LLM.request`, `LLM.generate`, `LLM.stream`, `LLM.updateRequest`, and `LLM.generateObject`. Two ways to construct the same thing is one too many.
## Tests
@@ -223,7 +223,7 @@ Routes lower these into provider-native assistant tool-call messages and tool-re
### Tool dispatch
`LLM.streamTurn(request)` and `LLM.generateTurn(request)` each run exactly one provider turn. Add tool schemas to `request.tools` with `Tool.toDefinitions(tools)`. When a caller wants the package's typed one-call execution behavior, pass each canonical local `tool-call` event to `ToolRuntime.dispatch(tools, call)`.
`LLM.stream(request)` and `LLM.generate(request)` each run exactly one provider turn. Add tool schemas to `request.tools` with `Tool.toDefinitions(tools)`. When a caller wants the package's typed one-call execution behavior, pass each canonical local `tool-call` event to `ToolRuntime.dispatch(tools, call)`.
```ts
const get_weather = tool({
@@ -240,7 +240,7 @@ const get_weather = tool({
})
const tools = { get_weather, get_time, ... }
const events = yield* LLM.streamTurn(
const events = yield* LLM.stream(
LLMRequest.update(request, { tools: Tool.toDefinitions(tools) }),
).pipe(Stream.runCollect)
+4 -4
View File
@@ -175,8 +175,8 @@ const request = LLM.request({
prompt: "Say hello.",
})
// Current API: this performs one provider turn.
const response = yield * LLM.generateTurn(request)
// Current API: this performs one provider turn, despite the broad name.
const response = yield * LLM.generate(request)
// Current API: execution also needs LLMClient.layer and RequestExecutor services.
```
@@ -432,7 +432,7 @@ const request = LLM.request({
tools: Tool.toDefinitions(tools),
})
const events = yield * LLM.streamTurn(request).pipe(Stream.runCollect)
const events = yield * LLM.stream(request).pipe(Stream.runCollect)
const call = Array.from(events).find(LLMEvent.is.toolCall)
if (call && !call.providerExecuted) {
@@ -1076,7 +1076,7 @@ The redesign intentionally removes or changes these current concepts:
| Current | Proposed |
| --------------------------------------- | ----------------------------------------------------------- |
| Mandatory `LLM.request({ model, ... })` | Inline calls or model-free portable requests |
| No complete-run API | Add `LLM.generate` / `LLM.stream` |
| `LLM.generate` means one turn | `LLM.generate` means complete run |
| `LLMClient.generate/stream` | `LLM.generateTurn/streamTurn` for one turn |
| `LLMClient.layer` requirement | Standard Effect requirements exposed directly |
| Public `Route` mental model | Hidden behind executable `Model` |
+8 -6
View File
@@ -176,7 +176,7 @@ Conversational image generation remains part of the LLM interaction. OpenAI Resp
```ts
const program = Effect.gen(function* () {
const response = yield* LLM.generateTurn(
const response = yield* LLM.generate(
LLM.request({
model: OpenAI.configure({ apiKey }).responses("gpt-5"),
prompt: "Design a solarpunk rooftop garden, then show me.",
@@ -193,7 +193,7 @@ The hosted result is represented as a provider-executed tool call and tool resul
## Public API
- **`LLM.request({...})`** — build a provider-neutral `LLMRequest`. Accepts ergonomic inputs (`system: string`, `prompt: string`) that normalize into the canonical Schema classes.
- **`LLM.generateTurn` / `LLM.streamTurn`** — execute exactly one provider turn, re-exported from `LLMClient` for one-import use.
- **`LLM.generate` / `LLM.stream`** — re-exported from `LLMClient` for one-import use.
- **`Message.user(...)` / `Message.assistant(...)` / `Message.tool(...)`** — message constructors from the canonical schema model.
- **`Model.make(...)` / `ToolCallPart.make(...)` / `ToolResultPart.make(...)` / `ToolDefinition.make(...)`** — model and tool-related constructors from the canonical schema model.
- **`LLMClient.prepare(request)`** — compile a request through protocol body construction, validation, and HTTP preparation without sending. Useful for inspection and testing.
@@ -207,7 +207,9 @@ Prompt caching is **on by default**. Every `LLMRequest` resolves to `cache: "aut
### Auto placement
`"auto"` places three breakpoints — last tool definition, last system part, latest user message. The last-user-message boundary is the load-bearing detail: in a tool-use loop, a single user turn expands into many assistant/tool round-trips, all sharing that prefix. Caching at that boundary lets every intra-turn API call hit.
`"auto"` places up to four breakpoints — the last tool definition, the first system part, the last system part when distinct, and the final message boundary. These expose successively larger reusable prefixes for tools, the base agent, project instructions, and the active conversation. The rolling final-message boundary is the load-bearing detail in tool loops: it advances on every request so the previous cache entry stays within Anthropic's 20-block lookback.
Tools precede every system and conversation block in the provider prefix, so tool definitions must remain byte-stable and deterministically ordered for downstream breakpoints to remain reusable.
The math justifies the default: Anthropic's 5-minute cache write is 1.25× base, read is 0.1×, so a single reuse within 5 minutes already wins. One-shot completions below the per-model minimum-cacheable-token threshold silently no-op on the wire, so the worst case is harmless.
@@ -235,7 +237,7 @@ cache: {
### Manual hints
Inline `CacheHint` on any text / system / tool / tool-result part overrides automatic placement. The auto policy preserves manual hints; it only fills gaps.
Inline `CacheHint` on any text / system / tool / tool-result part overrides automatic placement. The auto policy preserves manual hints, counts them against Anthropic and Bedrock's four-breakpoint limit, and only fills the remaining slots.
```ts
LLM.request({
@@ -251,8 +253,8 @@ LLM.request({
| Protocol | `cache: "auto"` |
| ----------------------- | ------------------------------------------------------------------------- |
| Anthropic Messages | emits up to 3 `cache_control` markers (4-breakpoint cap enforced) |
| Bedrock Converse | emits up to 3 `cachePoint` blocks (4-breakpoint cap enforced) |
| Anthropic Messages | emits up to 4 `cache_control` markers (4-breakpoint cap enforced) |
| Bedrock Converse | emits up to 4 `cachePoint` blocks (4-breakpoint cap enforced) |
| OpenAI Chat / Responses | no-op (implicit caching above 1024 tokens) |
| Gemini | no-op (implicit caching on 2.5+; explicit `CachedContent` is out-of-band) |
+1 -1
View File
@@ -334,7 +334,7 @@ Final request call site stays boring:
```ts
const response =
yield *
LLM.generateTurn(
LLM.generate(
LLM.request({
model: DeepSeek.model("deepseek-chat"),
prompt: "Hello.",
+3 -3
View File
@@ -65,7 +65,7 @@ const rawOverlayExample = LLM.request({
// 3. `generate` sends the request and collects the event stream into one
// response object. `response.text` is the collected text output.
const generateOnce = Effect.gen(function* () {
const response = yield* LLM.generateTurn(request)
const response = yield* LLM.generate(request)
console.log("\n== generate ==")
console.log("generated text:", response.text)
@@ -74,7 +74,7 @@ const generateOnce = Effect.gen(function* () {
// 4. `stream` exposes provider output as common `LLMEvent`s for UIs that want
// incremental text, reasoning, tool input, usage, or finish events.
const streamText = LLM.streamTurn(request).pipe(
const streamText = LLM.stream(request).pipe(
Stream.tap((event) =>
Effect.sync(() => {
if (event.type === "text-delta") process.stdout.write(`\ntext: ${event.text}`)
@@ -106,7 +106,7 @@ const streamWithTools = Effect.gen(function* () {
generation: { maxTokens: 80, temperature: 0 },
tools: Tool.toDefinitions(tools),
})
const events = Array.from(yield* LLM.streamTurn(request).pipe(Stream.runCollect))
const events = Array.from(yield* LLM.stream(request).pipe(Stream.runCollect))
for (const event of events) {
if (event.type === "tool-call") console.log("tool call", event.name, event.input)
if (event.type === "text-delta") process.stdout.write(event.text)
+61 -25
View File
@@ -2,32 +2,31 @@
// the policy designates. Runs once at compile time, before the per-protocol
// body builder, so the existing inline-hint lowering path handles the rest.
//
// The default `"auto"` shape places one breakpoint at the last tool definition,
// one at the last system part, and one at the latest user message. This
// matches what production agent harnesses (LangChain's caching middleware,
// kern-ai's 10x cost-reduction playbook) converge on for tool-use loops: the
// latest user message stays put while a single turn explodes into many
// assistant/tool round-trips, so caching at that boundary lets every
// intra-turn API call hit the prefix.
// The default `"auto"` shape places breakpoints at the last tool definition,
// the first and last distinct system parts, and the conversation tail. This
// exposes reusable tool, base-agent, project, and session prefixes while
// advancing the tail after each tool result keeps the previous cache entry
// within Anthropic's 20-block lookback during long agent turns.
//
// Manual `cache: CacheHint` placements on individual parts are preserved
// this function only fills gaps the caller left empty.
// Manual `cache: CacheHint` placements on individual parts are preserved and
// count against the four-breakpoint budget; auto only fills remaining slots.
import { CacheHint, type CachePolicy, type CachePolicyObject } from "./schema/options"
import { LLMRequest, Message, ToolDefinition, type ContentPart } from "./schema/messages"
const AUTO: CachePolicyObject = {
tools: true,
system: true,
messages: "latest-user-message",
messages: { tail: 1 },
}
const NONE: CachePolicyObject = {}
const BREAKPOINT_CAP = 4
// Resolution rules:
// - undefined → "auto" — caching is on by default. The math favors it:
// Anthropic 5m-cache write is 1.25x base, read is 0.1x,
// so a single reuse within 5 minutes already wins.
// - "auto" → tools + system + latest user msg.
// - "auto" → tools + first/last system + final message boundary.
// - "none" → no auto placement; manual `CacheHint`s still flow.
// - object form → exactly what the caller asked for.
const resolve = (policy: CachePolicy | undefined): CachePolicyObject => {
@@ -44,18 +43,32 @@ const RESPECTS_INLINE_HINTS = new Set(["anthropic-messages", "bedrock-converse"]
const makeHint = (ttlSeconds: number | undefined): CacheHint =>
ttlSeconds !== undefined ? new CacheHint({ type: "ephemeral", ttlSeconds }) : new CacheHint({ type: "ephemeral" })
const markLastTool = (tools: ReadonlyArray<ToolDefinition>, hint: CacheHint): ReadonlyArray<ToolDefinition> => {
interface Budget {
remaining: number
}
const markLastTool = (
tools: ReadonlyArray<ToolDefinition>,
hint: CacheHint,
budget: Budget,
): ReadonlyArray<ToolDefinition> => {
if (tools.length === 0) return tools
const last = tools.length - 1
if (tools[last]!.cache) return tools
if (tools[last]!.cache || budget.remaining === 0) return tools
budget.remaining -= 1
return tools.map((tool, i) => (i === last ? new ToolDefinition({ ...tool, cache: hint }) : tool))
}
const markLastSystem = (system: LLMRequest["system"], hint: CacheHint): LLMRequest["system"] => {
const markSystemBoundaries = (system: LLMRequest["system"], hint: CacheHint, budget: Budget): LLMRequest["system"] => {
if (system.length === 0) return system
const last = system.length - 1
if (system[last]!.cache) return system
return system.map((part, i) => (i === last ? { ...part, cache: hint } : part))
let changed = false
const next = system.map((part, index) => {
if ((index !== 0 && index !== system.length - 1) || part.cache || budget.remaining === 0) return part
budget.remaining -= 1
changed = true
return { ...part, cache: hint }
})
return changed ? next : system
}
const lastIndexOfRole = (messages: ReadonlyArray<Message>, role: Message["role"]): number =>
@@ -64,14 +77,20 @@ const lastIndexOfRole = (messages: ReadonlyArray<Message>, role: Message["role"]
// Mark the last text part of `messages[index]`. If no text part exists, mark
// the last content part regardless of type — that's the breakpoint position
// in tool-result-only messages too.
const markMessageAt = (messages: ReadonlyArray<Message>, index: number, hint: CacheHint): ReadonlyArray<Message> => {
const markMessageAt = (
messages: ReadonlyArray<Message>,
index: number,
hint: CacheHint,
budget: Budget,
): ReadonlyArray<Message> => {
if (index < 0 || index >= messages.length) return messages
const target = messages[index]!
if (target.content.length === 0) return messages
const lastTextIndex = target.content.findLastIndex((part) => part.type === "text")
const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.length - 1
const existing = target.content[markAt]!
if ("cache" in existing && existing.cache) return messages
if (("cache" in existing && existing.cache) || budget.remaining === 0) return messages
budget.remaining -= 1
const nextContent = target.content.map((part, i) => (i === markAt ? ({ ...part, cache: hint } as ContentPart) : part))
const next = new Message({ ...target, content: nextContent })
// Single pass over `messages`, substituting the one updated entry. Long
@@ -86,25 +105,42 @@ const markMessages = (
messages: ReadonlyArray<Message>,
strategy: NonNullable<CachePolicyObject["messages"]>,
hint: CacheHint,
budget: Budget,
): ReadonlyArray<Message> => {
if (messages.length === 0) return messages
if (strategy === "latest-user-message") return markMessageAt(messages, lastIndexOfRole(messages, "user"), hint)
if (strategy === "latest-assistant") return markMessageAt(messages, lastIndexOfRole(messages, "assistant"), hint)
if (strategy === "latest-user-message")
return markMessageAt(messages, lastIndexOfRole(messages, "user"), hint, budget)
if (strategy === "latest-assistant")
return markMessageAt(messages, lastIndexOfRole(messages, "assistant"), hint, budget)
const start = Math.max(0, messages.length - strategy.tail)
let next = messages
for (let i = start; i < messages.length; i++) next = markMessageAt(next, i, hint)
for (let i = start; i < messages.length; i++) next = markMessageAt(next, i, hint, budget)
return next
}
const countHints = (request: LLMRequest) =>
request.tools.reduce((count, tool) => count + (tool.cache === undefined ? 0 : 1), 0) +
request.system.reduce((count, part) => count + (part.cache === undefined ? 0 : 1), 0) +
request.messages.reduce(
(count, message) =>
count +
message.content.reduce(
(contentCount, part) => contentCount + ("cache" in part && part.cache !== undefined ? 1 : 0),
0,
),
0,
)
export const applyCachePolicy = (request: LLMRequest): LLMRequest => {
if (!RESPECTS_INLINE_HINTS.has(request.model.route.id)) return request
const policy = resolve(request.cache)
if (!policy.tools && !policy.system && !policy.messages) return request
const hint = makeHint(policy.ttlSeconds)
const tools = policy.tools ? markLastTool(request.tools, hint) : request.tools
const system = policy.system ? markLastSystem(request.system, hint) : request.system
const messages = policy.messages ? markMessages(request.messages, policy.messages, hint) : request.messages
const budget = { remaining: Math.max(0, BREAKPOINT_CAP - countHints(request)) }
const tools = policy.tools ? markLastTool(request.tools, hint, budget) : request.tools
const system = policy.system ? markSystemBoundaries(request.system, hint, budget) : request.system
const messages = policy.messages ? markMessages(request.messages, policy.messages, hint, budget) : request.messages
if (tools === request.tools && system === request.system && messages === request.messages) return request
return LLMRequest.update(request, { tools, system, messages })
+2 -2
View File
@@ -31,9 +31,9 @@ export type RequestInput = Omit<
readonly http?: HttpOptions.Input
}
export const generateTurn = LLMClient.generate
export const generate = LLMClient.generate
export const streamTurn = LLMClient.stream
export const stream = LLMClient.stream
export const request = (input: RequestInput) => {
const {
+66 -23
View File
@@ -9,6 +9,7 @@ import {
LLMEvent,
Usage,
type CacheHint,
type FinishReasonDetails,
type FinishReason,
type JsonSchema,
type LLMRequest,
@@ -288,7 +289,12 @@ type AnthropicEvent = Schema.Schema.Type<typeof AnthropicEvent>
interface ParserState {
readonly tools: ToolStream.State<number>
readonly reasoningSignatures: Readonly<Record<number, string>>
readonly usage?: Usage
readonly pendingFinish?: {
readonly reason: FinishReasonDetails
readonly providerMetadata?: ProviderMetadata
}
readonly lifecycle: Lifecycle.State
}
@@ -763,6 +769,10 @@ const onContentBlockStart = (state: ParserState, event: AnthropicEvent): StepRes
tools: ToolStream.start(state.tools, event.index, {
id: block.id ?? String(event.index),
name: block.name ?? "",
input:
block.input !== undefined && (!ProviderShared.isRecord(block.input) || Object.keys(block.input).length > 0)
? ProviderShared.encodeJson(block.input)
: undefined,
providerExecuted: block.type === "server_tool_use",
}),
},
@@ -777,20 +787,31 @@ const onContentBlockStart = (state: ParserState, event: AnthropicEvent): StepRes
]
}
if (block.type === "text" && block.text) {
if (block.type === "text" && block.text !== undefined) {
const events: LLMEvent[] = []
const id = `text-${event.index ?? 0}`
const lifecycle = Lifecycle.textStart(state.lifecycle, events, id)
return [
{ ...state, lifecycle: Lifecycle.textDelta(state.lifecycle, events, `text-${event.index ?? 0}`, block.text) },
{ ...state, lifecycle: block.text ? Lifecycle.textDelta(lifecycle, events, id, block.text) : lifecycle },
events,
]
}
if (block.type === "thinking" && block.thinking) {
if (block.type === "thinking" && block.thinking !== undefined) {
const events: LLMEvent[] = []
const id = `reasoning-${event.index ?? 0}`
const providerMetadata = block.signature === undefined ? undefined : anthropicMetadata({ signature: block.signature })
const lifecycle = Lifecycle.reasoningStart(state.lifecycle, events, id, providerMetadata)
return [
{
...state,
lifecycle: Lifecycle.reasoningDelta(state.lifecycle, events, `reasoning-${event.index ?? 0}`, block.thinking),
lifecycle: block.thinking
? Lifecycle.reasoningDelta(lifecycle, events, id, block.thinking, providerMetadata)
: lifecycle,
reasoningSignatures:
event.index === undefined || block.signature === undefined
? state.reasoningSignatures
: { ...state.reasoningSignatures, [event.index]: block.signature },
},
events,
]
@@ -799,7 +820,7 @@ const onContentBlockStart = (state: ParserState, event: AnthropicEvent): StepRes
// Redacted thinking surfaces as an empty reasoning part carrying the opaque
// payload as `redactedData` metadata (same model as Vercel's
// @ai-sdk/anthropic). The existing content_block_stop closes the part.
if (block.type === "redacted_thinking" && block.data) {
if (block.type === "redacted_thinking" && block.data !== undefined) {
const events: LLMEvent[] = []
return [
{
@@ -847,18 +868,13 @@ const onContentBlockDelta = Effect.fn("AnthropicMessages.onContentBlockDelta")(f
}
if (delta?.type === "signature_delta" && delta.signature) {
const events: LLMEvent[] = []
const index = event.index ?? 0
return [
{
...state,
lifecycle: Lifecycle.reasoningEnd(
state.lifecycle,
events,
`reasoning-${event.index ?? 0}`,
anthropicMetadata({ signature: delta.signature }),
),
reasoningSignatures: { ...state.reasoningSignatures, [index]: delta.signature },
},
events,
NO_EVENTS,
] satisfies StepResult
}
@@ -889,31 +905,53 @@ const onContentBlockStop = Effect.fn("AnthropicMessages.onContentBlockStop")(fun
const result = yield* ToolStream.finish(ADAPTER, state.tools, event.index)
const events: LLMEvent[] = []
const resultEvents = result.events ?? []
const signature = state.reasoningSignatures[event.index]
const lifecycle = resultEvents.length
? Lifecycle.stepStart(state.lifecycle, events)
: Lifecycle.reasoningEnd(
Lifecycle.textEnd(state.lifecycle, events, `text-${event.index}`),
events,
`reasoning-${event.index}`,
signature === undefined ? undefined : anthropicMetadata({ signature }),
)
events.push(...resultEvents)
return [{ ...state, lifecycle, tools: result.tools }, events] satisfies StepResult
const reasoningSignatures = { ...state.reasoningSignatures }
delete reasoningSignatures[event.index]
return [{ ...state, lifecycle, tools: result.tools, reasoningSignatures }, events] satisfies StepResult
})
const onMessageDelta = (state: ParserState, event: AnthropicEvent): StepResult => {
const usage = mergeUsage(state.usage, mapUsage(event.usage))
return [
{
...state,
usage,
pendingFinish: {
reason: {
normalized: mapFinishReason(event.delta?.stop_reason),
raw: event.delta?.stop_reason ?? undefined,
},
providerMetadata:
event.delta?.stop_sequence === null || event.delta?.stop_sequence === undefined
? undefined
: anthropicMetadata({ stopSequence: event.delta.stop_sequence }),
},
},
NO_EVENTS,
]
}
const onMessageStop = (state: ParserState): StepResult => {
const events: LLMEvent[] = []
const lifecycle = Lifecycle.finish(state.lifecycle, events, {
reason: {
normalized: mapFinishReason(event.delta?.stop_reason),
raw: event.delta?.stop_reason ?? undefined,
reason: state.pendingFinish?.reason ?? {
normalized: "unknown",
raw: undefined,
},
usage,
providerMetadata: event.delta?.stop_sequence
? anthropicMetadata({ stopSequence: event.delta.stop_sequence })
: undefined,
usage: state.usage,
providerMetadata: state.pendingFinish?.providerMetadata,
})
return [{ ...state, lifecycle, usage }, events]
return [{ ...state, lifecycle }, events]
}
// Prefix `error.type` so overloads, rate limits, and quota errors are visible
@@ -938,6 +976,7 @@ const step = (state: ParserState, event: AnthropicEvent) => {
if (event.type === "content_block_delta") return onContentBlockDelta(state, event)
if (event.type === "content_block_stop") return onContentBlockStop(state, event)
if (event.type === "message_delta") return Effect.succeed(onMessageDelta(state, event))
if (event.type === "message_stop") return Effect.succeed(onMessageStop(state))
if (event.type === "error") return onError(event)
return Effect.succeed<StepResult>([state, NO_EVENTS])
}
@@ -958,7 +997,11 @@ export const protocol = Protocol.make({
},
stream: {
event: Protocol.jsonEvent(AnthropicEvent),
initial: () => ({ tools: ToolStream.empty<number>(), lifecycle: Lifecycle.initial() }),
initial: () => ({
tools: ToolStream.empty<number>(),
reasoningSignatures: {},
lifecycle: Lifecycle.initial(),
}),
step,
},
})
+11 -4
View File
@@ -167,7 +167,12 @@ export type OpenResponsesBody = Schema.Schema.Type<typeof OpenResponsesBody>
const OpenResponsesUsage = Schema.Struct({
input_tokens: Schema.optional(Schema.Number),
input_tokens_details: optionalNull(Schema.Struct({ cached_tokens: Schema.optional(Schema.Number) })),
input_tokens_details: optionalNull(
Schema.Struct({
cached_tokens: Schema.optional(Schema.Number),
cache_write_tokens: Schema.optional(Schema.Number),
}),
),
output_tokens: Schema.optional(Schema.Number),
output_tokens_details: optionalNull(Schema.Struct({ reasoning_tokens: Schema.optional(Schema.Number) })),
total_tokens: Schema.optional(Schema.Number),
@@ -540,19 +545,21 @@ export const fromRequest = Effect.fn("OpenResponses.fromRequest")(function* (
// Stream Parsing
// =============================================================================
// Responses APIs report `input_tokens` (inclusive total) with a
// `cached_tokens` subset, and `output_tokens` (inclusive total) with a
// `reasoning_tokens` subset. Pass the totals through and derive the
// cached-read and cache-write subsets, and `output_tokens` (inclusive total)
// with a `reasoning_tokens` subset. Pass the totals through and derive the
// non-cached breakdown.
const mapUsage = (usage: OpenResponsesUsage | null | undefined, providerMetadataKey: string) => {
if (!usage) return undefined
const cached = usage.input_tokens_details?.cached_tokens
const cacheWrite = usage.input_tokens_details?.cache_write_tokens
const reasoning = usage.output_tokens_details?.reasoning_tokens
const nonCached = ProviderShared.subtractTokens(usage.input_tokens, cached)
const nonCached = ProviderShared.subtractTokens(usage.input_tokens, ProviderShared.sumTokens(cached, cacheWrite))
return new Usage({
inputTokens: usage.input_tokens,
outputTokens: usage.output_tokens,
nonCachedInputTokens: nonCached,
cacheReadInputTokens: cached,
cacheWriteInputTokens: cacheWrite,
reasoningTokens: reasoning,
totalTokens: ProviderShared.totalTokens(usage.input_tokens, usage.output_tokens, usage.total_tokens),
providerMetadata: { [providerMetadataKey]: usage },
+7 -4
View File
@@ -131,6 +131,7 @@ const OpenAIChatUsage = Schema.Struct({
prompt_tokens_details: optionalNull(
Schema.Struct({
cached_tokens: Schema.optional(Schema.Number),
cache_write_tokens: Schema.optional(Schema.Number),
}),
),
completion_tokens_details: optionalNull(
@@ -453,20 +454,22 @@ const mapFinishReason = (reason: string | null | undefined): FinishReason => {
}
// OpenAI Chat reports `prompt_tokens` (inclusive total) with a
// `cached_tokens` subset, and `completion_tokens` (inclusive total) with
// a `reasoning_tokens` subset. We pass the inclusive totals through and
// derive the non-cached breakdown so the `LLM.Usage` contract is
// cached-read and cache-write subsets, and `completion_tokens` (inclusive
// total) with a `reasoning_tokens` subset. We pass the inclusive totals
// through and derive the non-cached breakdown so the `LLM.Usage` contract is
// satisfied on both sides.
const mapUsage = (usage: OpenAIChatEvent["usage"]): Usage | undefined => {
if (!usage) return undefined
const cached = usage.prompt_tokens_details?.cached_tokens
const cacheWrite = usage.prompt_tokens_details?.cache_write_tokens
const reasoning = usage.completion_tokens_details?.reasoning_tokens
const nonCached = ProviderShared.subtractTokens(usage.prompt_tokens, cached)
const nonCached = ProviderShared.subtractTokens(usage.prompt_tokens, ProviderShared.sumTokens(cached, cacheWrite))
return new Usage({
inputTokens: usage.prompt_tokens,
outputTokens: usage.completion_tokens,
nonCachedInputTokens: nonCached,
cacheReadInputTokens: cached,
cacheWriteInputTokens: cacheWrite,
reasoningTokens: reasoning,
totalTokens: ProviderShared.totalTokens(usage.prompt_tokens, usage.completion_tokens, usage.total_tokens),
providerMetadata: { openai: usage },
+9 -6
View File
@@ -14,16 +14,19 @@ export const stepStart = (state: State, events: LLMEvent[]): State => {
return { ...state, stepStarted: true }
}
export const textDelta = (state: State, events: LLMEvent[], id: string, text: string): State => {
export const textStart = (state: State, events: LLMEvent[], id: string, providerMetadata?: ProviderMetadata): State => {
if (state.text.has(id)) return state
const stepped = stepStart(state, events)
if (stepped.text.has(id)) {
events.push(LLMEvent.textDelta({ id, text }))
return stepped
}
events.push(LLMEvent.textStart({ id }), LLMEvent.textDelta({ id, text }))
events.push(LLMEvent.textStart({ id, providerMetadata }))
return { ...stepped, text: new Set([...stepped.text, id]) }
}
export const textDelta = (state: State, events: LLMEvent[], id: string, text: string): State => {
const started = textStart(state, events, id)
events.push(LLMEvent.textDelta({ id, text }))
return started
}
export const reasoningStart = (
state: State,
events: LLMEvent[],
+1 -1
View File
@@ -412,7 +412,7 @@ const streamRequestWith = (runtime: TransportRuntime) => (request: LLMRequest) =
)
const generateWith = (stream: Interface["stream"]) =>
Effect.fn("LLM.generateTurn")(function* (request: LLMRequest) {
Effect.fn("LLM.generate")(function* (request: LLMRequest) {
const state = yield* stream(request).pipe(Stream.runFold(LLMResponse.empty, LLMResponse.reduce))
const response = LLMResponse.complete(state)
if (response) return response
+5 -5
View File
@@ -251,11 +251,11 @@ export class CacheHint extends Schema.Class<CacheHint>("LLM.CacheHint")({
// Auto-placement policy for prompt caching. The protocol-neutral lowering step
// reads this and injects `CacheHint`s at the configured boundaries; the
// per-protocol body builders then translate those hints into wire markers as
// usual. `"auto"` is the recommended default for agent loops — it places one
// breakpoint at the last tool definition, one at the last system part, and one
// at the latest user message. The combination of provider invalidation
// hierarchy (tools → system → messages) and Anthropic/Bedrock's 20-block
// lookback means three trailing breakpoints reliably cover the static prefix.
// usual. `"auto"` is the recommended default for agent loops — it places
// breakpoints at the last tool definition, the first and last distinct system
// parts, and the conversation tail. The rolling message breakpoint keeps a
// prior cache entry within Anthropic/Bedrock's 20-block lookback during long
// tool loops.
//
// Pass `"none"` to opt out entirely (the legacy behavior). Pass the granular
// object form to override individual choices.
+68 -8
View File
@@ -39,8 +39,8 @@ describe("applyCachePolicy", () => {
}),
)
// No explicit cache field → auto policy fires → last system part + latest
// user message both get cache_control markers.
// A single system block is both the first and last boundary, so the auto
// policy deduplicates it and still marks the conversation tail.
expect(prepared.body).toMatchObject({
system: [{ type: "text", text: "You are concise.", cache_control: { type: "ephemeral" } }],
messages: [{ role: "user", content: [{ type: "text", text: "hi", cache_control: { type: "ephemeral" } }] }],
@@ -48,12 +48,15 @@ describe("applyCachePolicy", () => {
}),
)
it.effect("'auto' marks the last tool, last system part, and latest user message on Anthropic", () =>
it.effect("'auto' marks the last tool, first and last system parts, and final message boundary on Anthropic", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.request({
model: anthropicModel,
system: "Sys A",
system: [
{ type: "text", text: "Base agent" },
{ type: "text", text: "Project instructions" },
],
tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
messages: [
Message.user("first user"),
@@ -66,7 +69,10 @@ describe("applyCachePolicy", () => {
expect(prepared.body).toMatchObject({
tools: [{ name: "t1", cache_control: { type: "ephemeral" } }],
system: [{ type: "text", text: "Sys A", cache_control: { type: "ephemeral" } }],
system: [
{ type: "text", text: "Base agent", cache_control: { type: "ephemeral" } },
{ type: "text", text: "Project instructions", cache_control: { type: "ephemeral" } },
],
messages: [
{ role: "user", content: [{ type: "text", text: "first user" }] },
{ role: "assistant", content: [{ type: "text", text: "assistant reply" }] },
@@ -120,7 +126,10 @@ describe("applyCachePolicy", () => {
const prepared = yield* LLMClient.prepare(
LLM.request({
model: bedrockModel,
system: "Sys",
system: [
{ type: "text", text: "Base agent" },
{ type: "text", text: "Project instructions" },
],
tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
messages: [Message.user("first user"), Message.assistant("reply"), Message.user("latest user")],
cache: "auto",
@@ -131,7 +140,12 @@ describe("applyCachePolicy", () => {
toolConfig: {
tools: [{ toolSpec: { name: "t1" } }, { cachePoint: { type: "default" } }],
},
system: [{ text: "Sys" }, { cachePoint: { type: "default" } }],
system: [
{ text: "Base agent" },
{ cachePoint: { type: "default" } },
{ text: "Project instructions" },
{ cachePoint: { type: "default" } },
],
messages: [
{ role: "user", content: [{ text: "first user" }] },
{ role: "assistant", content: [{ text: "reply" }] },
@@ -193,9 +207,55 @@ describe("applyCachePolicy", () => {
}),
)
const body = prepared.body as { system: Array<{ text: string; cache_control?: unknown }> }
const body = prepared.body as {
system: Array<{ text: string; cache_control?: unknown }>
messages: Array<{ content: Array<{ cache_control?: unknown }> }>
}
expect(body.system[0]?.cache_control).toEqual({ type: "ephemeral", ttl: "1h" })
expect(body.system[1]?.cache_control).toEqual({ type: "ephemeral" })
expect(body.messages[0]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
}),
)
it.effect("auto policy stays within the four-breakpoint cap when preserving manual hints", () =>
Effect.gen(function* () {
const request = LLM.request({
model: anthropicModel,
system: [
{ type: "text", text: "Base agent" },
{
type: "text",
text: "Manual context",
cache: new CacheHint({ type: "ephemeral", ttlSeconds: 3600 }),
},
{ type: "text", text: "Project instructions" },
],
tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
prompt: "hi",
cache: "auto",
})
const applied = applyCachePolicy(request)
expect(applied.tools[0]?.cache).toBeDefined()
expect(applied.system.map((part) => part.cache !== undefined)).toEqual([true, true, true])
const tail = applied.messages[0]!.content[0]!
expect("cache" in tail ? tail.cache : undefined).toBeUndefined()
expect(applyCachePolicy(applied)).toBe(applied)
const prepared = yield* LLMClient.prepare(request)
const body = prepared.body as {
tools: Array<{ cache_control?: unknown }>
system: Array<{ cache_control?: unknown }>
messages: Array<{ content: Array<{ cache_control?: unknown }> }>
}
const marked = [
...body.tools.map((tool) => tool.cache_control),
...body.system.map((part) => part.cache_control),
...body.messages.flatMap((message) => message.content.map((part) => part.cache_control)),
].filter((cache) => cache !== undefined)
expect(marked).toHaveLength(4)
expect(body.system[1]?.cache_control).toEqual({ type: "ephemeral", ttl: "1h" })
expect(body.messages[0]?.content[0]?.cache_control).toBeUndefined()
}),
)
-4
View File
@@ -23,10 +23,6 @@ import * as AnthropicMessages from "@opencode-ai/ai/protocols/anthropic-messages
describe("public exports", () => {
test("root exposes app-facing runtime APIs", () => {
expect(LLM.request).toBeFunction()
expect(LLM.generateTurn).toBeFunction()
expect(LLM.streamTurn).toBeFunction()
expect(LLM).not.toHaveProperty("generate")
expect(LLM).not.toHaveProperty("stream")
expect(LLMClient.Service).toBeFunction()
expect(LLMClient.layer).toBeDefined()
expect(ImageInput.bytes).toBeFunction()
File diff suppressed because one or more lines are too long
@@ -1,6 +1,6 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { CacheHint, LLM } from "../../src"
import { CacheHint, LLM, LLMRequest, Message, ToolCallPart, ToolDefinition } from "../../src"
import { LLMClient } from "../../src/route"
import * as Anthropic from "../../src/providers/anthropic"
import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
@@ -24,6 +24,39 @@ const cacheRequest = LLM.request({
generation: { maxTokens: 16, temperature: 0 },
})
const lookup = ToolDefinition.make({
name: "lookup",
description: "Look up a fixture value.",
inputSchema: {
type: "object",
properties: { index: { type: "number" } },
required: ["index"],
additionalProperties: false,
},
})
const longToolTurn = [
Message.user("Run the fixture lookups."),
...Array.from({ length: 11 }, (_, index) => {
const id = `lookup_${index}`
return [
Message.assistant(ToolCallPart.make({ id, name: lookup.name, input: { index } })),
Message.tool({
id,
name: lookup.name,
result: `Fixture result ${index}. `.repeat(80),
}),
]
}).flat(),
]
const longToolTurnRequest = LLM.request({
id: "recorded_anthropic_cache_long_tool_turn",
model,
system: LARGE_CACHEABLE_SYSTEM,
messages: longToolTurn,
tools: [lookup],
generation: { maxTokens: 16, temperature: 0 },
})
const recorded = recordedTests({
prefix: "anthropic-messages-cache",
provider: "anthropic",
@@ -50,4 +83,28 @@ describe("Anthropic Messages cache recorded", () => {
expect(second.usage?.cacheReadInputTokens ?? 0).toBeGreaterThan(0)
}),
)
recorded.effect.with("keeps a long tool turn inside the cache lookback", { tags: ["cache", "tool"] }, () =>
Effect.gen(function* () {
const first = yield* LLMClient.generate(longToolTurnRequest)
const firstRead = first.usage?.cacheReadInputTokens ?? 0
const firstWrite = first.usage?.cacheWriteInputTokens ?? 0
const firstCached = firstRead + firstWrite
// The prefix may already be warm when recording, so either a read or a
// write establishes that Anthropic recognized the cache boundary.
expect(firstCached).toBeGreaterThan(0)
const second = yield* LLMClient.generate(
LLMRequest.update(longToolTurnRequest, {
messages: [
...longToolTurn,
Message.assistant("The fixture lookups are complete."),
Message.user("Reply exactly: OK"),
],
}),
)
expect(second.usage?.cacheReadInputTokens ?? 0).toBeGreaterThanOrEqual(firstCached)
expect(second.usage?.cacheWriteInputTokens ?? 0).toBeLessThan(firstCached)
}),
)
})
@@ -506,6 +506,7 @@ describe("Anthropic Messages route", () => {
expect(response.events.find((event) => event.type === "reasoning-end")).toMatchObject({
providerMetadata: { anthropic: { signature: "sig_1" } },
})
expect(response.events.find((event) => event.type === "reasoning-delta" && event.text === "")).toBeUndefined()
expect(response.message.content).toEqual([
{ type: "text", text: "Hello!" },
{ type: "reasoning", text: "thinking", providerMetadata: { anthropic: { signature: "sig_1" } } },
@@ -518,6 +519,139 @@ describe("Anthropic Messages route", () => {
}),
)
it.effect("requires message_stop before completing a streamed message", () =>
Effect.gen(function* () {
const error = yield* LLMClient.generate(request).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{ type: "message_start", message: { usage: { input_tokens: 5 } } },
{ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
{ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Hello" } },
{ type: "content_block_stop", index: 0 },
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } },
),
),
),
Effect.flip,
)
expect(error.reason).toMatchObject({
_tag: "InvalidProviderOutput",
message: "Provider stream ended without a terminal finish event",
})
}),
)
it.effect("round-trips omitted thinking carried only by a signature delta", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(request).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{ type: "message_start", message: { usage: { input_tokens: 5 } } },
{
type: "content_block_start",
index: 0,
content_block: { type: "thinking", thinking: "", signature: "" },
},
{ type: "content_block_delta", index: 0, delta: { type: "signature_delta", signature: "sig_1" } },
{ type: "content_block_stop", index: 0 },
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } },
{ type: "message_stop" },
),
),
),
)
expect(response.message.content).toEqual([
{ type: "reasoning", text: "", providerMetadata: { anthropic: { signature: "sig_1" } } },
])
const prepared = yield* LLMClient.prepare<AnthropicMessages.AnthropicMessagesBody>(
LLM.request({ model, messages: [response.message], cache: "none" }),
)
expect(prepared.body.messages).toEqual([
{ role: "assistant", content: [{ type: "thinking", thinking: "", signature: "sig_1" }] },
])
}),
)
it.effect("retains a thinking signature supplied in content_block_start", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(request).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{ type: "message_start", message: { usage: { input_tokens: 5 } } },
{
type: "content_block_start",
index: 0,
content_block: { type: "thinking", thinking: "", signature: "sig_1" },
},
{ type: "content_block_stop", index: 0 },
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } },
{ type: "message_stop" },
),
),
),
)
expect(response.message.content).toEqual([
{ type: "reasoning", text: "", providerMetadata: { anthropic: { signature: "sig_1" } } },
])
expect(response.events.find((event) => event.type === "reasoning-end")).toMatchObject({
providerMetadata: { anthropic: { signature: "sig_1" } },
})
}),
)
it.effect("retains complete tool input from content_block_start", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(request).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{ type: "message_start", message: { usage: { input_tokens: 5 } } },
{
type: "content_block_start",
index: 0,
content_block: { type: "tool_use", id: "call_1", name: "lookup", input: { query: "weather" } },
},
{ type: "content_block_stop", index: 0 },
{ type: "message_delta", delta: { stop_reason: "tool_use" }, usage: { output_tokens: 1 } },
{ type: "message_stop" },
),
),
),
)
expect(response.toolCalls).toMatchObject([
{ id: "call_1", name: "lookup", input: { query: "weather" } },
])
}),
)
it.effect("retains empty text blocks", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(request).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{ type: "message_start", message: { usage: { input_tokens: 5 } } },
{ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
{ type: "content_block_stop", index: 0 },
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } },
{ type: "message_stop" },
),
),
),
)
expect(response.message.content).toEqual([{ type: "text", text: "" }])
}),
)
it.effect("parses redacted thinking into empty reasoning with redactedData metadata", () =>
Effect.gen(function* () {
const body = sseEvents(
@@ -629,6 +763,7 @@ describe("Anthropic Messages route", () => {
delta: { stop_reason: "model_context_window_exceeded" },
usage: { output_tokens: 1 },
},
{ type: "message_stop" },
),
),
),
@@ -646,6 +781,7 @@ describe("Anthropic Messages route", () => {
sseEvents(
{ type: "message_start", message: { usage: { input_tokens: 5 } } },
{ type: "message_delta", delta: { stop_reason: "pause_turn" }, usage: { output_tokens: 1 } },
{ type: "message_stop" },
),
),
),
@@ -664,6 +800,7 @@ describe("Anthropic Messages route", () => {
{ type: "content_block_delta", index: 0, delta: { type: "input_json_delta", partial_json: ':"weather"}' } },
{ type: "content_block_stop", index: 0 },
{ type: "message_delta", delta: { stop_reason: "tool_use" }, usage: { output_tokens: 1 } },
{ type: "message_stop" },
)
const response = yield* LLMClient.generate(
LLMRequest.update(request, {
@@ -849,6 +986,7 @@ describe("Anthropic Messages route", () => {
{ type: "content_block_delta", index: 2, delta: { type: "text_delta", text: "Found it." } },
{ type: "content_block_stop", index: 2 },
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 8 } },
{ type: "message_stop" },
)
const response = yield* LLMClient.generate(
LLMRequest.update(request, {
@@ -912,6 +1050,7 @@ describe("Anthropic Messages route", () => {
},
{ type: "content_block_stop", index: 1 },
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } },
{ type: "message_stop" },
)
const response = yield* LLMClient.generate(
LLMRequest.update(request, {
@@ -939,6 +939,7 @@ describe("Bedrock Converse route", () => {
const prepared = yield* LLMClient.prepare<BedrockConverse.BedrockConverseBody>(
LLM.request({
model,
cache: "none",
messages: [
Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: { path: "report.pdf" } })]),
Message.tool({
+5 -5
View File
@@ -47,7 +47,7 @@ describe("Cloudflare", () => {
it.effect("posts to the derived gateway endpoint with bearer auth", () =>
Effect.gen(function* () {
const response = yield* LLM.generateTurn(
const response = yield* LLM.generate(
LLM.request({
model: CloudflareAIGateway.configure({
accountId: "test-account",
@@ -104,7 +104,7 @@ describe("Cloudflare", () => {
index: 0,
},
]
const response = yield* LLM.generateTurn(LLM.request({ model, prompt: "Say hello." })).pipe(
const response = yield* LLM.generate(LLM.request({ model, prompt: "Say hello." })).pipe(
Effect.provide(
dynamicResponse((input) =>
Effect.succeed(
@@ -150,7 +150,7 @@ describe("Cloudflare", () => {
it.effect("supports authenticated AI Gateway plus upstream provider auth", () =>
Effect.gen(function* () {
yield* LLM.generateTurn(
yield* LLM.generate(
LLM.request({
model: CloudflareAIGateway.configure({
accountId: "test-account",
@@ -221,7 +221,7 @@ describe("Cloudflare", () => {
it.effect("posts direct Workers AI requests to the account endpoint with bearer auth", () =>
Effect.gen(function* () {
const response = yield* LLM.generateTurn(
const response = yield* LLM.generate(
LLM.request({
model: CloudflareWorkersAI.configure({
accountId: "test-account",
@@ -256,7 +256,7 @@ describe("Cloudflare", () => {
it.effect("supports direct Workers AI token aliases through auth config", () =>
Effect.gen(function* () {
yield* LLM.generateTurn(
yield* LLM.generate(
LLM.request({
model: CloudflareWorkersAI.configure({
accountId: "test-account",
@@ -550,7 +550,7 @@ describe("OpenAI Chat route", () => {
prompt_tokens: 5,
completion_tokens: 2,
total_tokens: 7,
prompt_tokens_details: { cached_tokens: 1 },
prompt_tokens_details: { cached_tokens: 1, cache_write_tokens: 2 },
completion_tokens_details: { reasoning_tokens: 0 },
}),
)
@@ -558,8 +558,9 @@ describe("OpenAI Chat route", () => {
const usage = new Usage({
inputTokens: 5,
outputTokens: 2,
nonCachedInputTokens: 4,
nonCachedInputTokens: 2,
cacheReadInputTokens: 1,
cacheWriteInputTokens: 2,
reasoningTokens: 0,
totalTokens: 7,
providerMetadata: {
@@ -567,7 +568,7 @@ describe("OpenAI Chat route", () => {
prompt_tokens: 5,
completion_tokens: 2,
total_tokens: 7,
prompt_tokens_details: { cached_tokens: 1 },
prompt_tokens_details: { cached_tokens: 1, cache_write_tokens: 2 },
completion_tokens_details: { reasoning_tokens: 0 },
},
},
@@ -29,7 +29,7 @@ describe("OpenAI Responses image generation recorded", () => {
partialImages: 0,
}),
]
const response = yield* LLM.generateTurn(
const response = yield* LLM.generate(
LLM.request({
model: openai.responses("gpt-5-mini"),
messages: [initial],
@@ -49,7 +49,7 @@ describe("OpenAI Responses image generation recorded", () => {
expect(result.result.value[0].mime).toBe("image/jpeg")
expect(result.result.value[0].uri.startsWith("data:image/jpeg;base64,")).toBe(true)
const edited = yield* LLM.generateTurn(
const edited = yield* LLM.generate(
LLM.request({
model: openai.responses("gpt-5-mini"),
messages: [initial, response.message, Message.user("Now make the triangle blue.")],
@@ -832,7 +832,7 @@ describe("OpenAI Responses route", () => {
input_tokens: 5,
output_tokens: 2,
total_tokens: 7,
input_tokens_details: { cached_tokens: 1 },
input_tokens_details: { cached_tokens: 1, cache_write_tokens: 2 },
output_tokens_details: { reasoning_tokens: 0 },
},
},
@@ -842,8 +842,9 @@ describe("OpenAI Responses route", () => {
const usage = new Usage({
inputTokens: 5,
outputTokens: 2,
nonCachedInputTokens: 4,
nonCachedInputTokens: 2,
cacheReadInputTokens: 1,
cacheWriteInputTokens: 2,
reasoningTokens: 0,
totalTokens: 7,
providerMetadata: {
@@ -851,7 +852,7 @@ describe("OpenAI Responses route", () => {
input_tokens: 5,
output_tokens: 2,
total_tokens: 7,
input_tokens_details: { cached_tokens: 1 },
input_tokens_details: { cached_tokens: 1, cache_write_tokens: 2 },
output_tokens_details: { reasoning_tokens: 0 },
},
},
+3
View File
@@ -539,6 +539,7 @@ describe("LLMClient tools", () => {
},
{ type: "content_block_stop", index: 1 },
{ type: "message_delta", delta: { stop_reason: "tool_use" }, usage: { output_tokens: 5 } },
{ type: "message_stop" },
)
: sseEvents(
{ type: "message_start", message: { usage: { input_tokens: 5 } } },
@@ -546,6 +547,7 @@ describe("LLMClient tools", () => {
{ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Done." } },
{ type: "content_block_stop", index: 0 },
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } },
{ type: "message_stop" },
),
{ headers: { "content-type": "text/event-stream" } },
)
@@ -801,6 +803,7 @@ describe("LLMClient tools", () => {
{ type: "content_block_delta", index: 2, delta: { type: "text_delta", text: "Done." } },
{ type: "content_block_stop", index: 2 },
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 8 } },
{ type: "message_stop" },
),
{ headers: { "content-type": "text/event-stream" } },
)
+3 -3
View File
@@ -32,9 +32,9 @@ Tool.make({
],
})
LLM.streamTurn(request)
LLM.generateTurn(LLMRequest.update(request, { tools: toDefinitions({ schemaOnly }) }))
LLM.stream(request)
LLM.generate(LLMRequest.update(request, { tools: toDefinitions({ schemaOnly }) }))
ToolRuntime.dispatch({ executable }, { type: "tool-call", id: "call_1", name: "executable", input: { city: "Paris" } })
// @ts-expect-error High-level tool orchestration overloads are intentionally not supported.
LLM.streamTurn({ request, tools: { schemaOnly } })
LLM.stream({ request, tools: { schemaOnly } })
+1 -1
View File
@@ -40,7 +40,7 @@ export type ExecuteOptions<Provided extends Record<string, unknown> = {}> = {
limits?: ExecutionLimits
/** Observes decoded tool input immediately before tool execution. */
onToolCallStart?: (call: ToolRuntime.ToolCallStarted) => Effect.Effect<void, never, Services<Provided>>
/** Observes each admitted tool call as it settles, with outcome and duration. */
/** Observes each admitted tool call as it succeeds, fails, or is interrupted. */
onToolCallEnd?: (call: ToolRuntime.ToolCallEnded) => Effect.Effect<void, never, Services<Provided>>
}
+15 -19
View File
@@ -1,4 +1,4 @@
import { Cause, Effect, Schema } from "effect"
import { Cause, Effect, Exit, Schema } from "effect"
import { ToolError, toolError } from "./tool-error.js"
import {
decodeInput as decodeToolInput,
@@ -52,7 +52,7 @@ export type ToolCallEnded = {
readonly name: string
readonly input: unknown
readonly durationMs: number
readonly outcome: "success" | "failure"
readonly outcome: "success" | "failure" | "interrupted"
readonly message?: string
}
@@ -495,22 +495,19 @@ export const make = <R>(
const root = toolTrie(tools)
const searchTool = makeSearchTool(searchIndex)
// End hooks observe settled success or failure; interruption emits neither outcome.
const observeEnd = <A, E>(effect: Effect.Effect<A, E, R>, call: ToolCallStarted): Effect.Effect<A, E, R> => {
const onEnd = hooks?.onToolCallEnd
if (onEnd === undefined) return effect
const startedAt = Date.now()
return effect.pipe(
Effect.tap(() => onEnd({ ...call, durationMs: Date.now() - startedAt, outcome: "success" })),
Effect.tapError((error) => {
Effect.onExit((exit) => {
const durationMs = Date.now() - startedAt
if (Exit.isSuccess(exit)) return onEnd({ ...call, durationMs, outcome: "success" })
if (Cause.hasInterruptsOnly(exit.cause)) return onEnd({ ...call, durationMs, outcome: "interrupted" })
const error = Cause.squash(exit.cause)
const message =
error instanceof ToolError || error instanceof ToolRuntimeError ? error.message : "Tool execution failed"
return onEnd({
...call,
durationMs: Date.now() - startedAt,
outcome: "failure",
message,
})
return onEnd({ ...call, durationMs, outcome: "failure", message })
}),
)
}
@@ -528,12 +525,6 @@ export const make = <R>(
calls.push(call)
}
const recordAndObserve = (name: string, input: unknown) =>
Effect.sync(() => {
recordCall({ name })
return calls.length - 1
}).pipe(Effect.tap((index) => hooks?.onToolCallStart?.({ index, name, input }) ?? Effect.void))
const executeTool = (name: string, tool: Tool<R>, externalArgs: Array<unknown>) =>
Effect.gen(function* () {
if (externalArgs.length !== 1)
@@ -547,9 +538,14 @@ export const make = <R>(
name === "search" ? [] : ["The signature may have changed. Use search to get the current signature."],
),
})
const index = yield* recordAndObserve(name, input)
const index = yield* Effect.sync(() => {
recordCall({ name })
return calls.length - 1
})
const call = { index, name, input }
return yield* observeEnd(
Effect.gen(function* () {
if (hooks?.onToolCallStart !== undefined) yield* hooks.onToolCallStart(call)
const raw = yield* runHost(Effect.suspend(() => tool.execute(input)))
const result = yield* Effect.try({
try: () => decodeToolOutput(tool, raw),
@@ -557,7 +553,7 @@ export const make = <R>(
})
return yield* decodeOutput(result, name)
}),
{ index, name, input },
call,
)
})
+90 -1
View File
@@ -189,7 +189,12 @@ describe("CodeMode tool-call observation", () => {
description: "Look up a value",
input: Schema.Struct({ query: Schema.String }),
output: Schema.String,
execute: ({ query }) => (query === "boom" ? Effect.fail(toolError("Lookup refused")) : Effect.succeed(query)),
execute: ({ query }) =>
query === "boom"
? Effect.fail(toolError("Lookup refused"))
: query === "defect"
? Effect.die("broken")
: Effect.succeed(query),
})
const runtime = CodeMode.make({
@@ -215,14 +220,98 @@ describe("CodeMode tool-call observation", () => {
expect(success.ok).toBe(true)
const failure = await Effect.runPromise(runtime.execute(`return await tools.context.lookup({ query: "boom" })`))
expect(failure.ok).toBe(false)
const defect = await Effect.runPromise(runtime.execute(`return await tools.context.lookup({ query: "defect" })`))
expect(defect.ok).toBe(false)
expect(events).toStrictEqual([
{ phase: "start", index: 0, name: "context.lookup" },
{ phase: "end", index: 0, name: "context.lookup", outcome: "success" },
{ phase: "start", index: 0, name: "context.lookup" },
{ phase: "end", index: 0, name: "context.lookup", outcome: "failure", message: "Lookup refused" },
{ phase: "start", index: 0, name: "context.lookup" },
{ phase: "end", index: 0, name: "context.lookup", outcome: "failure", message: "Tool execution failed" },
])
})
test("observes interrupted calls", async () => {
const events: Array<string> = []
const call = Tool.make({
description: "Interrupt",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.interrupt,
})
const exit = await Effect.runPromiseExit(
CodeMode.make({
tools: { host: { call } },
onToolCallStart: () => Effect.sync(() => events.push("start")),
onToolCallEnd: (call) => Effect.sync(() => events.push(`end:${call.outcome}`)),
}).execute("return await tools.host.call({})"),
)
expect(exit._tag).toBe("Failure")
expect(events).toEqual(["start", "end:interrupted"])
})
test("observes running calls interrupted during completion", async () => {
const events: Array<string> = []
const call = Tool.make({
description: "Pending",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.never,
})
const result = await Effect.runPromise(
CodeMode.make({
tools: { host: { call } },
onToolCallStart: () => Effect.sync(() => events.push("start")),
onToolCallEnd: (call) => Effect.sync(() => events.push(`end:${call.outcome}`)),
}).execute('tools.host.call({}); return "done"'),
)
expect(result).toMatchObject({ ok: true, value: "done" })
expect(events).toEqual(["start", "end:interrupted"])
})
test("ends calls interrupted during start observation", async () => {
const events: Array<string> = []
const call = Tool.make({
description: "Unused",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.succeed("unused"),
})
const exit = await Effect.runPromiseExit(
CodeMode.make({
tools: { host: { call } },
onToolCallStart: () => Effect.interrupt,
onToolCallEnd: (call) => Effect.sync(() => events.push(call.outcome)),
}).execute("return await tools.host.call({})"),
)
expect(exit._tag).toBe("Failure")
expect(events).toEqual(["interrupted"])
})
test("observes calls interrupted by the execution timeout", async () => {
const outcomes: Array<string> = []
const call = Tool.make({
description: "Pending",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.never,
})
const result = await Effect.runPromise(
CodeMode.make({
tools: { host: { call } },
limits: { timeoutMs: 10 },
onToolCallEnd: (call) => Effect.sync(() => outcomes.push(call.outcome)),
}).execute("return await tools.host.call({})"),
)
expect(result).toMatchObject({ ok: false, error: { kind: "TimeoutExceeded" } })
expect(outcomes).toEqual(["interrupted"])
})
})
describe("CodeMode console capture", () => {
+3 -5
View File
@@ -6,15 +6,13 @@ import { Instructions } from "../instructions/index"
import { CodeModeCatalog } from "./catalog"
// prettier-ignore
const prompt = (hasMoreTools: boolean) => `Run JavaScript to orchestrate tool calls and compose their results. Imports, direct filesystem access, and timers are unavailable. Do not use \`fetch\`; all external access goes through \`tools\`.
const prompt = (hasMoreTools: boolean) => `The Code Mode tool catalog below is ${hasMoreTools ? "partial" : "complete"}.
Prefer an explicit \`return\`; if omitted, the final top-level expression becomes the result. Await tool calls before returning; any calls still pending when execution ends are interrupted. Run independent calls concurrently with \`Promise.all\`.
Do not infer or normalize tool names; use only the exact signatures shown below${hasMoreTools ? " or returned by `search`" : ""}, preserving bracket notation such as \`tools.<namespace>["tool-name"](input)\`.${hasMoreTools ? `
Inside Code Mode, \`tools\` contains only the tools shown below${hasMoreTools ? " or returned by `search`" : ""}; surrounding top-level agent tools are not available and must not be called from the code.${hasMoreTools ? `
## Search
Only some tool signatures are shown. Use \`search\` to discover exact paths and signatures for additional tools:
Use \`search\` to discover exact paths and signatures for additional tools:
- ${searchSignature}` : ""}
+5
View File
@@ -218,6 +218,11 @@ const layer = Layer.effect(
if (result.effect === "allow") return
const item = yield* create(request(input), input.agent)
return yield* restore(Deferred.await(item.deferred)).pipe(
// Deliberate defect tunnel: leaves wrap execution in blanket `mapError`, which
// must not convert a user's decline into model-facing tool output. The decline
// resurfaces as a typed failure at SessionModelRequest.executeTool. A decline
// WITH feedback (CorrectedError) intentionally stays typed so the leaf can turn
// it into ToolFailure and the model continues.
Effect.catchTag("PermissionV2.DeclinedError", (error) => Effect.die(error)),
Effect.ensuring(
Effect.sync(() => {
+28 -4
View File
@@ -2,11 +2,14 @@ export * as SessionModelRequest from "./model-request"
import { LLM, Message, SystemPart, type LLMRequest, type ToolContent } from "@opencode-ai/ai"
import { SessionError } from "@opencode-ai/schema/session-error"
import { Context, Effect, Layer } from "effect"
import { Cause, Context, Effect, Layer, Result } from "effect"
import { makeLocationNode } from "@opencode-ai/util/effect/app-node"
import { App } from "../app"
import { ModelV2 } from "../model"
import { PermissionV2 } from "../permission"
import { PluginHooks } from "../plugin/hooks"
import { QuestionTool } from "../tool/question"
import { ToolOutputStore } from "../tool-output-store"
import { ToolRegistry } from "../tool/registry"
import { SessionContext } from "./context"
import { SessionModelHeaders } from "./model-headers"
@@ -14,13 +17,32 @@ import { MAX_STEPS_PROMPT } from "./runner/max-steps"
import PROMPT_DEFAULT from "./runner/prompt/base.txt"
import { toLLMMessages } from "./runner/to-llm-message"
/** Failures a prepared execution can surface: infrastructure errors plus user declines resurfaced from the defect tunnel. */
export type ExecuteError = ToolOutputStore.Error | PermissionV2.DeclinedError | QuestionTool.CancelledError
// User declines dive under the leaves' blanket `mapError` as defects (the deliberate
// tunnel entered in PermissionV2.assert and the question tool), so a user's "no" can
// never become model-facing tool output. They resurface as typed failures exactly once,
// here at the seam the runner executes through.
const declineDefect = (cause: Cause.Cause<ToolOutputStore.Error>) => {
const decline = cause.reasons.flatMap((reason) =>
Cause.isDieReason(reason) &&
(reason.defect instanceof PermissionV2.DeclinedError || reason.defect instanceof QuestionTool.CancelledError)
? [reason.defect]
: [],
)[0]
return decline ? Result.succeed(decline) : Result.fail(cause)
}
interface Prepared {
readonly request: LLMRequest
/**
* One request-scoped execution operation. Unknown, hook-removed, and
* step-limit-violating calls fail individually through the same seam.
*/
readonly executeTool: ToolRegistry.ToolSet["execute"]
readonly executeTool: (
input: ToolRegistry.ExecuteInput,
) => Effect.Effect<ToolRegistry.ToolOutcome, ExecuteError>
/** True when this request is the final Step; violating calls are rejected and no continuation follows. */
readonly stepLimitReached: boolean
}
@@ -134,7 +156,7 @@ export const layer = Layer.effect(
tools: hookedTools,
toolChoice: stepLimitReached ? "none" : undefined,
})
const executeTool: ToolRegistry.ToolSet["execute"] = (executeInput) => {
const executeTool: Prepared["executeTool"] = (executeInput) => {
if (stepLimitReached)
return Effect.succeed({
status: "error",
@@ -145,7 +167,9 @@ export const layer = Layer.effect(
status: "error",
error: { type: "tool.unknown", message: `Tool is not available for this request: ${executeInput.call.name}` },
})
return toolSet.execute(executeInput)
return toolSet
.execute(executeInput)
.pipe(Effect.catchCauseFilter(declineDefect, (decline) => Effect.fail(decline)))
}
return {
request,
+53 -19
View File
@@ -36,27 +36,44 @@ type CallOutcome = Data.TaggedEnum<{
const CallOutcome = Data.taggedEnum<CallOutcome>()
// Declining an interactive prompt halts the drain instead of becoming model-facing tool output.
const isUserDeclined = (cause: Cause.Cause<unknown>) =>
cause.reasons.some(
(reason) =>
Cause.isDieReason(reason) &&
(reason.defect instanceof PermissionV2.DeclinedError || reason.defect instanceof QuestionTool.CancelledError),
)
const isDecline = (
error: SessionModelRequest.ExecuteError,
): error is PermissionV2.DeclinedError | QuestionTool.CancelledError =>
error._tag === "PermissionV2.DeclinedError" || error._tag === "QuestionTool.CancelledError"
/**
* Classifies how the owned tool fibers ended. Interrupts and interactive declines abort
* the step; a defect from a tool implementation becomes a failed tool call the model can
* read; a typed infrastructure failure must fail the assistant and then the drain.
* Classifies how the owned tool fibers ended. Interrupts abort the step; a user decline
* settles its own call and then aborts the step; a defect from a tool implementation
* becomes a failed tool call the model can read; a typed infrastructure failure must
* fail the assistant and then the drain.
*/
const classifyToolExits = (settled: Exit.Exit<Array<Exit.Exit<void, ToolOutputStore.Error>>, never>) => {
const classifyToolExits = (
settled: Exit.Exit<Array<Exit.Exit<void, SessionModelRequest.ExecuteError>>, never>,
calls: ReadonlyArray<ToolCall>,
) => {
// Exits align with calls by construction: one owned fiber per accepted local call.
const exits = settled._tag === "Success" ? settled.value : []
const declines = exits.flatMap((exit, index) =>
exit._tag === "Failure"
? exit.cause.reasons.flatMap((reason) =>
Cause.isFailReason(reason) && isDecline(reason.error) ? [{ call: calls[index], reason: reason.error }] : [],
)
: [],
)
const causes =
settled._tag === "Failure"
? [settled.cause]
: settled.value.flatMap((exit) => (exit._tag === "Failure" ? [exit.cause] : []))
const failure = causes.find((cause) => !Cause.hasInterrupts(cause) && !isUserDeclined(cause))
settled._tag === "Failure" ? [settled.cause] : exits.flatMap((exit) => (exit._tag === "Failure" ? [exit.cause] : []))
// The first non-interrupt, non-decline failure, rebuilt without decline reasons so the
// drain's error channel never carries a decline.
const failure = causes.flatMap((cause) => {
if (Cause.hasInterrupts(cause)) return []
const reasons = cause.reasons.flatMap((reason): Array<Cause.Reason<ToolOutputStore.Error>> =>
Cause.isFailReason(reason) ? (isDecline(reason.error) ? [] : [Cause.makeFailReason(reason.error)]) : [reason],
)
return reasons.length > 0 ? [Cause.fromReasons(reasons)] : []
})[0]
return {
interrupted: causes.some(Cause.hasInterrupts),
declined: causes.some(isUserDeclined),
declines,
failure,
infraError: failure === undefined ? undefined : Option.getOrUndefined(Cause.findErrorOption(failure)),
}
@@ -204,7 +221,10 @@ const layer = Layer.effect(
step: currentStep,
})
// Every local tool call forked here is owned until it reaches one durable settlement.
const toolRuns: Array<{ readonly call: ToolCall; readonly fiber: Fiber.Fiber<void, ToolOutputStore.Error> }> = []
const toolRuns: Array<{
readonly call: ToolCall
readonly fiber: Fiber.Fiber<void, SessionModelRequest.ExecuteError>
}> = []
const interruptTools = Effect.suspend(() => Fiber.interruptAll(toolRuns.map((run) => run.fiber)))
let needsContinuation = false
const startSnapshot = yield* snapshots.capture()
@@ -348,9 +368,23 @@ const layer = Layer.effect(
Effect.forEach(toolRuns, (run) => Fiber.await(run.fiber), { concurrency: "unbounded" }),
).pipe(Effect.exit)
if (settled._tag === "Failure") yield* interruptTools
const tools = classifyToolExits(settled)
const tools = classifyToolExits(
settled,
toolRuns.map((run) => run.call),
)
if (tools.declined || streamInterrupted || tools.interrupted) {
// A declined call settles durably with its reason before the generic sweeps.
for (const decline of tools.declines)
yield* serialized(
publisher.failTool(decline.call.id, {
type: "aborted",
message:
decline.reason._tag === "QuestionTool.CancelledError"
? decline.reason.message
: "The user declined this tool call",
}),
)
if (tools.declines.length > 0 || streamInterrupted || tools.interrupted) {
yield* serialized(publisher.failUnsettledTools({ type: "aborted", message: "Tool execution interrupted" }))
yield* serialized(publisher.failAssistant({ type: "aborted", message: "Step interrupted" }))
}
@@ -390,7 +424,7 @@ const layer = Layer.effect(
}
if (stream._tag === "Failure") return yield* Effect.failCause(stream.cause)
if (tools.declined) return yield* Effect.interrupt
if (tools.declines.length > 0) return yield* Effect.interrupt
if ((tools.interrupted || tools.infraError !== undefined) && tools.failure)
return yield* Effect.failCause(tools.failure)
if (tools.interrupted && settled._tag === "Failure") return yield* Effect.failCause(settled.cause)
@@ -232,21 +232,27 @@ export const createLLMEventPublisher = (events: Pick<EventV2.Interface, "publish
yield* flushFragments()
})
const failTool = Effect.fnUntraced(function* (callID: string, error: SessionError.Error) {
const tool = tools.get(callID)
if (!tool || tool.settled) return false
tool.settled = true
yield* events.publish(SessionEvent.Tool.Failed, {
sessionID: input.sessionID,
assistantMessageID: tool.assistantMessageID,
callID,
error,
...failureSnapshot(tool),
executed: tool.providerExecuted,
})
return true
})
const failTools = Effect.fnUntraced(function* (error: SessionError.Error, mode: "all" | "hosted" | "uncalled") {
let failed = false
for (const [callID, tool] of tools) {
if (tool.settled || (mode === "hosted" && !tool.providerExecuted) || (mode === "uncalled" && tool.called))
continue
tool.settled = true
failed = true
yield* events.publish(SessionEvent.Tool.Failed, {
sessionID: input.sessionID,
assistantMessageID: tool.assistantMessageID,
callID,
error,
...failureSnapshot(tool),
executed: tool.providerExecuted,
})
failed = (yield* failTool(callID, error)) || failed
}
return failed
})
@@ -525,6 +531,7 @@ export const createLLMEventPublisher = (events: Pick<EventV2.Interface, "publish
toolExecution,
flush,
failAssistant,
failTool,
publishStepFailure,
failUnsettledTools,
hasProviderError: () => providerFailed,
+1 -1
View File
@@ -24,7 +24,7 @@ const source = {
}
```
Leaves own resolution, permission, and side-effect ordering. Translate only expected typed errors into `ToolFailure`; do not use `catchCause`, because interruption and defects must survive.
Leaves own resolution, permission, and side-effect ordering. Translate only expected typed errors into `ToolFailure`; do not use `catchCause`, because interruption and defects must survive. User declines from `PermissionV2.assert` and question dismissals travel as defects beneath leaf `mapError` blankets and resurface as typed failures at `SessionModelRequest.executeTool`; leaves must never catch or convert them. A decline with feedback (`PermissionV2.CorrectedError`) stays typed so the leaf converts it into `ToolFailure` and the model continues.
## Registration
+30 -28
View File
@@ -3,7 +3,7 @@ export type { Registration } from "./tool"
import { CodeMode, Tool, toolError } from "@opencode-ai/codemode"
import type { ToolContent } from "@opencode-ai/ai"
import { Effect, Ref, Schema } from "effect"
import { Effect, Ref, Schema, Semaphore } from "effect"
import { execute, make, toLLMDefinition, type Content, type Metadata, type Registration } from "./tool"
const ExecuteFile = Schema.Struct({
@@ -34,10 +34,11 @@ type CollectedFiles = {
// Invariant model-facing guidance; the changing tool catalog is delivered through Instructions.
const description = [
"Run JavaScript in a confined Code Mode runtime through { code }.",
"Call Code Mode tools through `tools` using the exact paths and signatures from the instructions.",
"Use `search({ query })` to discover exact signatures when needed.",
"Await important calls and use `Promise.all` for independent calls.",
"Run JavaScript to orchestrate tool calls and compose their results through `{ code }` in a confined Code Mode runtime.",
"Imports, direct filesystem access, and timers are unavailable. Do not use `fetch`; all external access goes through `tools`.",
'Call Code Mode tools through `tools` using only exact paths and signatures from the current catalog or `search`. Do not infer or normalize tool names; preserve bracket notation such as `tools.<namespace>["tool-name"](input)`.',
"Prefer an explicit `return`; if omitted, the final top-level expression becomes the result.",
"Await every call whose completion matters; pending calls are interrupted when execution ends. Run independent calls concurrently with `Promise.all`.",
].join("\n")
export const create = (registrations: ReadonlyMap<string, Registration>) => {
@@ -50,12 +51,11 @@ export const create = (registrations: ReadonlyMap<string, Registration>) => {
const callIndex = yield* Ref.make(0)
const files = yield* Ref.make<Array<CollectedFiles>>([])
const calls = yield* Ref.make<Array<ExecuteCall>>([])
// TODO: Publish live call-list updates once V2 has a generic tool progress API.
const finalCalls = Ref.get(calls).pipe(
Effect.map((items) =>
items.map((call) => (call.status === "running" ? { ...call, status: "error" as const } : call)),
),
)
const lock = Semaphore.makeUnsafe(1)
const updateCalls = (update: (items: Array<ExecuteCall>) => Array<ExecuteCall>) =>
lock.withPermit(
Ref.updateAndGet(calls, update).pipe(Effect.flatMap((toolCalls) => context.progress({ toolCalls }))),
)
const result = yield* runtime(
registrations,
(name, registration, input) =>
@@ -66,7 +66,7 @@ export const create = (registrations: ReadonlyMap<string, Registration>) => {
agent: context.agent,
messageID: context.messageID,
callID: context.callID,
progress: context.progress,
progress: () => Effect.void,
}).pipe(Effect.mapError((failure) => toolError(failure.message, failure)))
const outputFileParts = outputFiles(executed.content)
if (outputFileParts.length > 0)
@@ -74,26 +74,28 @@ export const create = (registrations: ReadonlyMap<string, Registration>) => {
return executed.output
}),
{
onToolCallStart: ({ index, name, input }) =>
Effect.gen(function* () {
const shown = displayInput(input)
yield* Ref.update(calls, (items) => {
const next = [...items]
next[index] = { tool: name, status: "running", ...(shown ? { input: shown } : {}) }
return next
})
}),
onToolCallEnd: ({ index, outcome }) =>
Ref.update(calls, (items) => {
const current = items[index]
if (!current) return items
onToolCallStart: ({ index, name, input }) => {
const shown = displayInput(input)
return updateCalls((items) => {
const next = [...items]
next[index] = { ...current, status: outcome === "success" ? "completed" : "error" }
next[index] = { tool: name, status: "running", ...(shown ? { input: shown } : {}) }
return next
}),
})
},
onToolCallEnd: ({ index, name, input, outcome }) => {
const shown = displayInput(input)
return updateCalls((items) => {
const next = [...items]
next[index] = {
...(items[index] ?? { tool: name, ...(shown ? { input: shown } : {}) }),
status: outcome === "success" ? "completed" : "error",
}
return next
})
},
},
).execute(code)
const toolCalls = yield* finalCalls
const toolCalls = yield* Ref.get(calls)
const collected = (yield* Ref.get(files))
.toSorted((left, right) => left.index - right.index)
.flatMap((item) => item.files)
+17 -5
View File
@@ -7,6 +7,7 @@ import path from "path"
import { FileSystem } from "../filesystem"
import { FSUtil } from "@opencode-ai/util/fs-util"
import { Location } from "../location"
import { LocationMutation } from "../location-mutation"
import { Ripgrep } from "../ripgrep"
import { RelativePath } from "../schema"
import { PermissionV2 } from "../permission"
@@ -45,6 +46,7 @@ export const Plugin = {
const fs = yield* FSUtil.Service
const ripgrep = yield* Ripgrep.Service
const location = yield* Location.Service
const mutation = yield* LocationMutation.Service
const permission = yield* PermissionV2.Service
yield* ctx.tool
@@ -58,6 +60,16 @@ export const Plugin = {
output: Output,
execute: (input, context) =>
Effect.gen(function* () {
const source = { type: "tool" as const, messageID: context.messageID, callID: context.callID }
const target = yield* mutation.resolve({ path: input.path ?? ".", kind: "directory" })
const external = target.externalDirectory
if (external)
yield* permission.assert({
...LocationMutation.externalDirectoryPermission(external),
sessionID: context.sessionID,
agent: context.agent,
source,
})
yield* permission.assert({
action: name,
resources: [input.pattern],
@@ -69,20 +81,20 @@ export const Plugin = {
},
sessionID: context.sessionID,
agent: context.agent,
source: { type: "tool", messageID: context.messageID, callID: context.callID },
source,
})
const cwd = path.resolve(location.directory, input.path ?? ".")
yield* fs
.stat(cwd)
.stat(target.canonical)
.pipe(
Effect.catchReason("PlatformError", "NotFound", () =>
Effect.fail(new ToolFailure({ message: `Search path does not exist: ${input.path ?? "."}` })),
),
)
const root = path.resolve(location.directory, input.path ?? ".")
const limit = input.limit ?? FileSystem.DEFAULT_SEARCH_LIMIT
const entries = yield* ripgrep
.glob({
cwd,
cwd: target.canonical,
pattern: input.pattern,
limit: limit + 1,
})
@@ -91,7 +103,7 @@ export const Plugin = {
result.map((entry) =>
FileSystem.Entry.make({
...entry,
path: RelativePath.make(path.relative(location.directory, path.resolve(cwd, entry.path))),
path: RelativePath.make(path.relative(location.directory, path.resolve(root, entry.path))),
}),
),
),
+3
View File
@@ -91,6 +91,9 @@ export const Plugin = {
.pipe(Effect.orDie),
),
Effect.flatMap((state) => {
// Deliberate defect tunnel (see PermissionV2.assert): a dismissal must dodge
// leaf `mapError` blankets so it never becomes model-facing tool output; it
// resurfaces as a typed failure at SessionModelRequest.executeTool.
if (state.status === "cancelled") return Effect.die(new CancelledError())
const output = {
answers: input.questions.map((_, index): QuestionV2.Answer => {
+7 -15
View File
@@ -66,20 +66,10 @@ describe("CodeModeInstructions.render", () => {
expect(instructions).toContain("- orders (1 tool)")
expect(instructions).toContain(` - ${lookup.signature} // Look up an order by ID`)
expect(instructions).not.toContain("## Search")
expect(instructions).toContain("Do not infer or normalize tool names")
expect(instructions).toContain('`tools.<namespace>["tool-name"](input)`')
})
test("describes the runtime and execution lifecycle concisely", () => {
const instructions = render([lookup])
expect(instructions).toContain("Run JavaScript to orchestrate tool calls and compose their results.")
expect(instructions).toContain("Imports, direct filesystem access, and timers are unavailable.")
expect(instructions).toContain("Do not use `fetch`; all external access goes through `tools`.")
expect(instructions).toContain("The Code Mode tool catalog below is complete.")
expect(instructions).toContain(
"Prefer an explicit `return`; if omitted, the final top-level expression becomes the result.",
"`tools` contains only the tools shown below; surrounding top-level agent tools are not available and must not be called from the code.",
)
expect(instructions).toContain("any calls still pending when execution ends are interrupted")
expect(instructions).toContain("Run independent calls concurrently with `Promise.all`.")
})
test("adds search guidance when the catalog exceeds the budget", () => {
@@ -87,10 +77,12 @@ describe("CodeModeInstructions.render", () => {
expect(partial).toContain("## Available tools")
expect(partial).toContain("- orders (1 tool, none shown)")
expect(partial).toContain("## Search")
expect(partial).toContain("Only some tool signatures are shown.")
expect(partial).toContain("The Code Mode tool catalog below is partial.")
expect(partial).toContain(
"`tools` contains only the tools shown below or returned by `search`; surrounding top-level agent tools are not available and must not be called from the code.",
)
expect(partial).toContain("- search(input: {")
expect(partial).toContain(" limit?: number,\n offset?: number,")
expect(partial).toContain("or returned by `search`")
expect(partial).not.toContain("tools.orders.lookup(input:")
})
@@ -169,7 +161,7 @@ describe("CodeModeInstructions.update", () => {
expect(text).toContain("This catalog supersedes the previous Code Mode tool catalog.")
expect(text).toContain("## Available tools")
expect(text).not.toContain("## Search")
expect(text).not.toContain("must not be called")
expect(text).not.toContain("The following tools are no longer available")
})
test("renders namespace-only deltas without persisting hidden tool entries", () => {
@@ -515,7 +515,7 @@ describe("ToolRegistry", () => {
}),
)
it.effect("executes codemode tools advertised in a model request", () =>
it.effect("executes and reports progress for codemode tools advertised in a model request", () =>
Effect.gen(function* () {
const service = yield* ToolRegistry.Service
const executed: string[] = []
@@ -526,8 +526,11 @@ describe("ToolRegistry", () => {
description: "Echo text",
input: Schema.Struct({ text: Schema.String }),
output: Schema.Struct({ text: Schema.String }),
execute: ({ text }) =>
Effect.sync(() => executed.push(`old:${text}`)).pipe(Effect.as({ output: { text } })),
execute: ({ text }, context) =>
Effect.sync(() => executed.push(`old:${text}`)).pipe(
Effect.andThen(context.progress({ stage: "old" })),
Effect.as({ output: { text } }),
),
}),
})
.pipe(Scope.provide(scope))
@@ -546,6 +549,7 @@ describe("ToolRegistry", () => {
}),
})
const progress: ToolRegistry.Progress[] = []
const execution = yield* toolSet.execute({
...call("execute"),
call: {
@@ -554,10 +558,15 @@ describe("ToolRegistry", () => {
name: "execute",
input: { code: 'return await tools.echo({ text: "request" })' },
},
progress: (update) => Effect.sync(() => progress.push(update)),
})
expect(execution).toMatchObject({ status: "completed", content: [{ type: "text" }] })
expect(executed).toEqual(["old:request"])
expect(progress).toEqual([
{ toolCalls: [{ tool: "echo", status: "running", input: { text: "request" } }] },
{ toolCalls: [{ tool: "echo", status: "completed", input: { text: "request" } }] },
])
}),
)
})
+2 -2
View File
@@ -3569,7 +3569,7 @@ describe("SessionRunnerLLM", () => {
{
type: "tool",
id: "call-declined",
state: { status: "error", error: { message: "Tool execution interrupted" } },
state: { status: "error", error: { type: "aborted", message: "The user declined this tool call" } },
},
],
},
@@ -3721,7 +3721,7 @@ describe("SessionRunnerLLM", () => {
{
type: "tool",
id: "call-question",
state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } },
state: { status: "error", error: { type: "aborted", message: "The user dismissed this question" } },
},
],
},
+56 -1
View File
@@ -4,7 +4,7 @@ import { Tool } from "@opencode-ai/core/tool/tool"
import { Agent } from "@opencode-ai/schema/agent"
import { Session } from "@opencode-ai/schema/session"
import { SessionMessage } from "@opencode-ai/schema/session-message"
import { Effect, Schema } from "effect"
import { Deferred, Effect, Fiber, Schema } from "effect"
const context = {
sessionID: Session.ID.make("ses_execute"),
@@ -14,6 +14,18 @@ const context = {
progress: () => Effect.void,
}
test("execute describes invariant Code Mode behavior", () => {
expect(ExecuteTool.create(new Map()).description).toBe(
[
"Run JavaScript to orchestrate tool calls and compose their results through `{ code }` in a confined Code Mode runtime.",
"Imports, direct filesystem access, and timers are unavailable. Do not use `fetch`; all external access goes through `tools`.",
'Call Code Mode tools through `tools` using only exact paths and signatures from the current catalog or `search`. Do not infer or normalize tool names; preserve bracket notation such as `tools.<namespace>["tool-name"](input)`.',
"Prefer an explicit `return`; if omitted, the final top-level expression becomes the result.",
"Await every call whose completion matters; pending calls are interrupted when execution ends. Run independent calls concurrently with `Promise.all`.",
].join("\n"),
)
})
test("canonical execution distinguishes declared, model-only, and raw schema outputs", async () => {
const declared = Tool.make({
description: "Declared",
@@ -131,3 +143,46 @@ test("execute supports callable namespace tools", async () => {
})
expect(result.content).toEqual([{ type: "text", text: '[\n "admin",\n "created"\n]' }])
})
test("execute marks every admitted child call failed when interrupted", async () => {
const child = Tool.make({
description: "Wait forever",
input: Schema.Struct({ id: Schema.Number }),
output: Schema.String,
execute: () => Effect.never,
})
const execute = ExecuteTool.create(new Map([["wait", { tool: child, name: "wait", permission: "wait" }]]))
const updates: Tool.Metadata[] = []
await Effect.runPromise(
Effect.gen(function* () {
const started = yield* Deferred.make<void>()
const fiber = yield* Tool.execute(
execute,
{ code: "return await Promise.all([tools.wait({ id: 1 }), tools.wait({ id: 2 })])" },
{
...context,
progress: (update) =>
Effect.gen(function* () {
updates.push(update)
if (updates.length > 1) return
yield* Deferred.succeed(started, undefined)
yield* Effect.never
}),
},
).pipe(Effect.forkChild)
yield* Deferred.await(started)
yield* Effect.yieldNow
yield* Effect.yieldNow
yield* Fiber.interrupt(fiber)
}),
)
expect(updates[0]).toEqual({ toolCalls: [{ tool: "wait", status: "running", input: { id: 1 } }] })
expect(updates.at(-1)).toEqual({
toolCalls: [
{ tool: "wait", status: "error", input: { id: 1 } },
{ tool: "wait", status: "error", input: { id: 2 } },
],
})
})
+98 -14
View File
@@ -8,6 +8,7 @@ import { LayerNode } from "@opencode-ai/util/effect/layer-node"
import { FileSystem } from "@opencode-ai/core/filesystem"
import { FSUtil } from "@opencode-ai/util/fs-util"
import { Location } from "@opencode-ai/core/location"
import { LocationMutation } from "@opencode-ai/core/location-mutation"
import { PermissionV2 } from "@opencode-ai/core/permission"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { AbsolutePath } from "@opencode-ai/core/schema"
@@ -24,27 +25,27 @@ import { executeTool, registerToolPlugin, toolIdentity } from "./lib/tool"
const globToolNode = makeLocationNode({
name: "test/glob-tool-plugin",
layer: Layer.effectDiscard(registerToolPlugin(GlobTool.Plugin)),
deps: [ToolRegistry.toolsNode, FSUtil.node, Ripgrep.node, Location.node, PermissionV2.node],
deps: [
ToolRegistry.toolsNode,
FSUtil.node,
Ripgrep.node,
Location.node,
LocationMutation.node,
PermissionV2.node,
],
})
const grepToolNode = makeLocationNode({
name: "test/grep-tool-plugin",
layer: Layer.effectDiscard(registerToolPlugin(GrepTool.Plugin)),
deps: [ToolRegistry.toolsNode, FSUtil.node, Ripgrep.node, Location.node, PermissionV2.node],
})
const permission = Layer.succeed(
PermissionV2.Service,
PermissionV2.Service.of({
assert: () => Effect.void,
ask: () => Effect.die("unused"),
reply: () => Effect.die("unused"),
get: () => Effect.die("unused"),
forSession: () => Effect.die("unused"),
list: () => Effect.die("unused"),
}),
)
const sessionID = SessionV2.ID.make("ses_search_tool_test")
const withTools = <A, E, R>(directory: string, body: (registry: ToolRegistry.Interface) => Effect.Effect<A, E, R>) =>
const withTools = <A, E, R>(
directory: string,
body: (registry: ToolRegistry.Interface) => Effect.Effect<A, E, R>,
assertions?: PermissionV2.AssertInput[],
) =>
Effect.gen(function* () {
return yield* body(yield* ToolRegistry.Service)
}).pipe(
@@ -54,7 +55,23 @@ const withTools = <A, E, R>(directory: string, body: (registry: ToolRegistry.Int
Location.node,
Layer.succeed(Location.Service, Location.Service.of(location({ directory: AbsolutePath.make(directory) }))),
],
[PermissionV2.node, permission],
[
PermissionV2.node,
Layer.succeed(
PermissionV2.Service,
PermissionV2.Service.of({
assert: (input) =>
Effect.sync(() => {
assertions?.push(input)
}),
ask: () => Effect.die("unused"),
reply: () => Effect.die("unused"),
get: () => Effect.die("unused"),
forSession: () => Effect.die("unused"),
list: () => Effect.die("unused"),
}),
),
],
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
]),
),
@@ -125,4 +142,71 @@ describe("search tools", () => {
),
)
}
it.live("requires external_directory approval for an explicit external glob path", () =>
Effect.acquireUseRelease(
Effect.promise(() => Promise.all([tmpdir(), tmpdir()])),
([active, outside]) => {
const assertions: PermissionV2.AssertInput[] = []
return Effect.promise(() => fs.writeFile(path.join(outside.path, "outside.txt"), "outside\n")).pipe(
Effect.andThen(
withTools(
active.path,
(registry) => executeTool(registry, call("glob", { path: outside.path, pattern: "*.txt" })),
assertions,
),
),
Effect.tap((result) =>
Effect.sync(() => {
expect(result.status).toBe("completed")
expect(assertions.map((input) => input.action)).toEqual(["external_directory", "glob"])
expect(assertions[0]?.resources).toEqual([
path.join(outside.path, "*").replaceAll("\\", "/"),
])
}),
),
)
},
([active, outside]) =>
Effect.promise(() =>
Promise.all([active[Symbol.asyncDispose](), outside[Symbol.asyncDispose]()]).then(() => undefined),
),
),
)
it.live("globs through an in-location external symlink without external approval", () =>
Effect.acquireUseRelease(
Effect.promise(() => Promise.all([tmpdir(), tmpdir()])),
([active, outside]) => {
if (process.platform === "win32") return Effect.void
const assertions: PermissionV2.AssertInput[] = []
return Effect.promise(async () => {
await fs.writeFile(path.join(outside.path, "outside.txt"), "outside\n")
await fs.symlink(outside.path, path.join(active.path, "linked"))
}).pipe(
Effect.andThen(
withTools(
active.path,
(registry) => executeTool(registry, call("glob", { path: "linked", pattern: "*.txt" })),
assertions,
),
),
Effect.tap((result) =>
Effect.sync(() => {
expect(result.status).toBe("completed")
expect(assertions.map((input) => input.action)).toEqual(["glob"])
expect(result).toMatchObject({
output: [{ path: path.join("linked", "outside.txt"), type: "file" }],
content: [{ type: "text", text: path.join(active.path, "linked", "outside.txt") }],
})
}),
),
)
},
([active, outside]) =>
Effect.promise(() =>
Promise.all([active[Symbol.asyncDispose](), outside[Symbol.asyncDispose]()]).then(() => undefined),
),
),
)
})
+20 -14
View File
@@ -81,7 +81,15 @@ import { PluginSlot } from "../../plugin/context"
import { Keymap, type KeymapCommand } from "../../context/keymap"
import { usePathFormatter } from "../../context/path-format"
import { useLocation } from "../../context/location"
import { createSessionRows, messageBoundaryIDs, resolvePart, type PartRef, type SessionRow } from "./rows"
import {
cacheReuseDrop,
createSessionRows,
messageBoundaryIDs,
resolvePart,
type CacheUsage,
type PartRef,
type SessionRow,
} from "./rows"
import { switchLabel } from "../../util/model"
import { findMessageBoundary, messageNavigationSlack } from "./message-navigation"
import { stringWidth } from "../../util/string-width"
@@ -1079,7 +1087,7 @@ function SessionRowView(props: SessionRowViewProps) {
{(row) => (
<TurnTokenUsage
messageIDs={row().messageIDs}
previousCacheRead={row().previousCacheRead}
previousCache={row().previousCache}
message={props.message}
/>
)}
@@ -1091,13 +1099,13 @@ function SessionRowView(props: SessionRowViewProps) {
function TurnTokenUsage(props: {
messageIDs: string[]
previousCacheRead?: number
previousCache?: CacheUsage
message: (messageID: string) => SessionMessageInfo | undefined
}) {
const config = useConfig()
const { themeV2 } = useTheme()
const steps = createMemo(() => {
let previousCacheRead = props.previousCacheRead
let previousCache = props.previousCache
return props.messageIDs.flatMap((messageID) => {
const message = props.message(messageID)
if (message?.type !== "assistant" || !message.tokens) return []
@@ -1109,18 +1117,16 @@ function TurnTokenUsage(props: {
message.tokens.cache.write
if (total === 0) return []
const newTokens = total - message.tokens.cache.read
const cacheBust =
previousCacheRead !== undefined && message.tokens.cache.read < previousCacheRead
? previousCacheRead - message.tokens.cache.read
: undefined
previousCacheRead = message.tokens.cache.read
const currentCache = { read: message.tokens.cache.read, model: message.model }
const reuseDrop = cacheReuseDrop(previousCache, currentCache)
previousCache = currentCache
return [
{
finish: message.finish === "tool-calls" ? "tool-call" : (message.finish ?? "unknown"),
newTokens,
cached: message.tokens.cache.read,
total,
cacheBust,
reuseDrop,
},
]
})
@@ -1165,9 +1171,9 @@ function TurnTokenUsage(props: {
{" "}
{item.total.toLocaleString().padStart(columns().total)}
</text>
<Show when={item.cacheBust !== undefined}>
<text fg={themeV2.text.feedback.error.default}>
! Cache bust: {item.cacheBust?.toLocaleString()} fewer cached tokens than the previous step
<Show when={item.reuseDrop !== undefined}>
<text fg={themeV2.text.feedback.warning.default}>
! Likely cache bust: {item.reuseDrop?.toLocaleString()} fewer cached tokens than the previous step
</text>
</Show>
</box>
@@ -2866,7 +2872,7 @@ function Execute(props: ToolProps) {
const isLoading = createMemo(() => props.part.state.status === "streaming" || props.part.state.status === "running")
const calls = createMemo(() => executeCalls(props.metadata.toolCalls))
const output = createMemo(() => stripAnsi(props.output?.trim() ?? ""))
const hasRuntimeError = createMemo(() => props.metadata.error === true)
const hasRuntimeError = createMemo(() => props.metadata.error === true || props.part.state.status === "error")
const outputPreview = createMemo(() => collapseToolOutput(output(), 4, 4 * Math.max(20, ctx.width - 6)).output)
const showOutput = createMemo(() => output() && hasRuntimeError())
const content = createMemo(() => {
+25 -6
View File
@@ -10,6 +10,11 @@ export type PartRef = {
partID: string
}
export type CacheUsage = {
read: number
model: SessionMessageAssistant["model"]
}
export type SessionRow =
| { type: "message"; messageID: string }
| { type: "compaction-queued"; inputID: string }
@@ -28,7 +33,7 @@ export type SessionRow =
completed: boolean
}
| { type: "assistant-footer"; messageID: string }
| { type: "turn-usage"; messageIDs: string[]; previousCacheRead?: number }
| { type: "turn-usage"; messageIDs: string[]; previousCache?: CacheUsage }
export function createSessionRows(sessionID: Accessor<string>) {
const data = useData()
@@ -280,7 +285,7 @@ export function reduceSessionRows(
const pendingCompactions = messages.filter((message) => message.type === "compaction" && message.status === "running")
const pending = new Set([...pendingCompactions.map((message) => message.id), ...inputs])
const usage = turnTokens
? { steps: [] as SessionMessageAssistant[], previousTurnCacheRead: undefined as number | undefined }
? { steps: [] as SessionMessageAssistant[], previousTurnCache: undefined as CacheUsage | undefined }
: undefined
return [
...messages.filter((message) => !pending.has(message.id)),
@@ -289,6 +294,8 @@ export function reduceSessionRows(
].reduce<SessionRow[]>((rows, message) => {
if (message.type !== "assistant") {
if (message.type === "synthetic" && !message.description?.trim()) return rows
if (message.type === "compaction" && message.status === "completed" && usage)
usage.previousTurnCache = undefined
if (!pending.has(message.id)) completePrevious(rows)
rows.push({ type: "message", messageID: message.id })
return rows
@@ -312,11 +319,9 @@ export function reduceSessionRows(
rows.push({
type: "turn-usage",
messageIDs: stepsWithUsage.map((step) => step.id),
...(usage.previousTurnCacheRead === undefined
? {}
: { previousCacheRead: usage.previousTurnCacheRead }),
...(usage.previousTurnCache === undefined ? {} : { previousCache: usage.previousTurnCache }),
})
usage.previousTurnCacheRead = last.tokens.cache.read
usage.previousTurnCache = { read: last.tokens.cache.read, model: last.model }
}
usage.steps.length = 0
}
@@ -324,6 +329,20 @@ export function reduceSessionRows(
}, [])
}
export function cacheReuseDrop(previous: CacheUsage | undefined, current: CacheUsage) {
if (previous === undefined) return
if (
previous.model.providerID !== current.model.providerID ||
previous.model.id !== current.model.id ||
previous.model.variant !== current.model.variant
)
return
const drop = previous.read - current.read
// OpenAI cache reads can move between one and two 1,024-token buckets without a material loss of reuse.
if (current.model.providerID === "openai" && drop >= 1_024 && drop <= 2_048) return
return drop > 0 ? drop : undefined
}
function hasTokenUsage(
message: SessionMessageAssistant,
): message is SessionMessageAssistant & { tokens: NonNullable<SessionMessageAssistant["tokens"]> } {
+87 -1
View File
@@ -1,6 +1,92 @@
import { expect, test } from "bun:test"
import type { SessionMessageAssistant, SessionMessageInfo } from "@opencode-ai/client"
import { messageBoundaryIDs, reduceSessionRows } from "../../../src/routes/session/rows"
import { cacheReuseDrop, messageBoundaryIDs, reduceSessionRows } from "../../../src/routes/session/rows"
test("filters OpenAI cache quantization from cache reuse drops", () => {
const openai = { id: "gpt", providerID: "openai" }
expect(cacheReuseDrop(undefined, { read: 10_000, model: openai })).toBeUndefined()
expect(cacheReuseDrop({ read: 10_000, model: openai }, { read: 11_000, model: openai })).toBeUndefined()
expect(cacheReuseDrop({ read: 10_000, model: openai }, { read: 8_977, model: openai })).toBe(1_023)
expect(cacheReuseDrop({ read: 10_000, model: openai }, { read: 8_976, model: openai })).toBeUndefined()
expect(cacheReuseDrop({ read: 10_000, model: openai }, { read: 8_500, model: openai })).toBeUndefined()
expect(cacheReuseDrop({ read: 10_000, model: openai }, { read: 7_952, model: openai })).toBeUndefined()
expect(cacheReuseDrop({ read: 10_000, model: openai }, { read: 7_951, model: openai })).toBe(2_049)
})
test("compares cache reuse only for the same model", () => {
const previous = { read: 10_000, model: { id: "claude", providerID: "anthropic" } }
expect(cacheReuseDrop(previous, { read: 8_976, model: { id: "gpt", providerID: "openai" } })).toBeUndefined()
expect(cacheReuseDrop(previous, { read: 8_976, model: { id: "claude", providerID: "anthropic" } })).toBe(1_024)
expect(
cacheReuseDrop(
{ read: 10_000, model: { id: "gpt", providerID: "openai", variant: "low" } },
{ read: 8_976, model: { id: "gpt", providerID: "openai", variant: "high" } },
),
).toBeUndefined()
})
test("carries model identity with the cross-turn cache baseline", () => {
const first = assistant("assistant-1", [])
first.model = { id: "claude", providerID: "anthropic" }
first.finish = "stop"
first.tokens = { input: 1, output: 0, reasoning: 0, cache: { read: 10_000, write: 0 } }
const second = assistant("assistant-2", [])
second.model = { id: "gpt", providerID: "openai" }
second.finish = "stop"
second.tokens = { input: 1, output: 0, reasoning: 0, cache: { read: 8_976, write: 0 } }
const rows = reduceSessionRows(
[
{ type: "user", id: "user-1", text: "First", time: { created: 0 } },
first,
{ type: "user", id: "user-2", text: "Second", time: { created: 2 } },
second,
],
new Set(),
true,
).filter((row) => row.type === "turn-usage")
expect(rows).toEqual([
{ type: "turn-usage", messageIDs: ["assistant-1"] },
{
type: "turn-usage",
messageIDs: ["assistant-2"],
previousCache: { read: 10_000, model: { id: "claude", providerID: "anthropic" } },
},
])
})
test("resets the cross-turn cache baseline after compaction", () => {
const first = assistant("assistant-1", [])
first.finish = "stop"
first.tokens = { input: 1, output: 0, reasoning: 0, cache: { read: 370_176, write: 0 } }
const second = assistant("assistant-2", [])
second.finish = "stop"
second.tokens = { input: 1, output: 0, reasoning: 0, cache: { read: 13_824, write: 0 } }
const rows = reduceSessionRows(
[
first,
{
type: "compaction",
id: "compaction-1",
status: "completed",
reason: "auto",
summary: "Compacted context",
recent: "",
time: { created: 2 },
},
second,
],
new Set(),
true,
).filter((row) => row.type === "turn-usage")
expect(rows).toEqual([
{ type: "turn-usage", messageIDs: ["assistant-1"] },
{ type: "turn-usage", messageIDs: ["assistant-2"] },
])
})
test("assigns assistant boundaries to the first rendered row instead of the first text row", () => {
const messages: SessionMessageInfo[] = [