diff --git a/packages/ai/src/protocols/anthropic-messages.ts b/packages/ai/src/protocols/anthropic-messages.ts index b2d8fa70dc..26d127eb0e 100644 --- a/packages/ai/src/protocols/anthropic-messages.ts +++ b/packages/ai/src/protocols/anthropic-messages.ts @@ -703,7 +703,14 @@ const onContentBlockStart = (state: ParserState, event: AnthropicEvent): StepRes providerExecuted: block.type === "server_tool_use", }), }, - [...events, LLMEvent.toolInputStart({ id: block.id ?? String(event.index), name: block.name ?? "" })], + [ + ...events, + LLMEvent.toolInputStart({ + id: block.id ?? String(event.index), + name: block.name ?? "", + providerExecuted: block.type === "server_tool_use" ? true : undefined, + }), + ], ] } diff --git a/packages/ai/src/protocols/bedrock-converse.ts b/packages/ai/src/protocols/bedrock-converse.ts index 4ac3eeaf68..b94f23d62f 100644 --- a/packages/ai/src/protocols/bedrock-converse.ts +++ b/packages/ai/src/protocols/bedrock-converse.ts @@ -561,7 +561,9 @@ const step = (state: ParserState, event: BedrockEvent) => return [ { ...state, - hasToolCalls: resultEvents.some(LLMEvent.is.toolCall) ? true : state.hasToolCalls, + hasToolCalls: + resultEvents.some((event) => LLMEvent.is.toolCall(event) || LLMEvent.is.toolInputError(event)) || + state.hasToolCalls, lifecycle, tools: result.tools, reasoningSignatures: Object.fromEntries( diff --git a/packages/ai/src/protocols/openai-chat.ts b/packages/ai/src/protocols/openai-chat.ts index 101ac61cf6..68ab2e3949 100644 --- a/packages/ai/src/protocols/openai-chat.ts +++ b/packages/ai/src/protocols/openai-chat.ts @@ -464,7 +464,7 @@ const step = (state: ParserState, event: OpenAIChatEvent) => } // Finalize accumulated tool inputs eagerly when finish_reason arrives so - // JSON parse failures fail the stream at the boundary rather than at halt. + // valid calls and malformed local calls settle independently. const finished = finishReason !== undefined && state.finishReason === undefined && Object.keys(tools).length > 0 ? yield* ToolStream.finishAll(ADAPTER, tools) diff --git a/packages/ai/src/protocols/openai-responses.ts b/packages/ai/src/protocols/openai-responses.ts index d03cb003ff..65eb7e6c1a 100644 --- a/packages/ai/src/protocols/openai-responses.ts +++ b/packages/ai/src/protocols/openai-responses.ts @@ -835,7 +835,9 @@ const onOutputItemDone = Effect.fn("OpenAIResponses.onOutputItemDone")(function* { ...state, lifecycle, - hasFunctionCall: resultEvents.some(LLMEvent.is.toolCall) ? true : state.hasFunctionCall, + hasFunctionCall: + resultEvents.some((event) => LLMEvent.is.toolCall(event) || LLMEvent.is.toolInputError(event)) || + state.hasFunctionCall, tools: result.tools, }, events, diff --git a/packages/ai/src/protocols/utils/tool-stream.ts b/packages/ai/src/protocols/utils/tool-stream.ts index 8e07a64bfe..667c5d3967 100644 --- a/packages/ai/src/protocols/utils/tool-stream.ts +++ b/packages/ai/src/protocols/utils/tool-stream.ts @@ -1,5 +1,5 @@ import { Effect } from "effect" -import { LLMError, LLMEvent, type ProviderMetadata, type ToolCall } from "../../schema" +import { LLMError, LLMEvent, type ProviderMetadata, type ToolCall, type ToolInputError } from "../../schema" import { eventError, parseToolInput, type ToolAccumulator } from "../shared" type StreamKey = string | number @@ -53,6 +53,7 @@ const inputStart = (tool: PendingTool) => LLMEvent.toolInputStart({ id: tool.id, name: tool.name, + providerExecuted: tool.providerExecuted ? true : undefined, providerMetadata: tool.providerMetadata, }) @@ -63,19 +64,38 @@ const inputDelta = (tool: PendingTool, text: string) => text, }) -const toolCall = (route: string, tool: PendingTool, inputOverride?: string) => - parseToolInput(route, tool.name, inputOverride ?? tool.input).pipe( - Effect.map( - (input): ToolCall => - LLMEvent.toolCall({ - id: tool.id, - name: tool.name, - input, - providerExecuted: tool.providerExecuted ? true : undefined, - providerMetadata: tool.providerMetadata, - }), +const toolCall = (route: string, tool: PendingTool, inputOverride?: string) => { + const raw = inputOverride ?? tool.input + return parseToolInput(route, tool.name, raw).pipe( + Effect.map((input): ToolCall | ToolInputError => + LLMEvent.toolCall({ + id: tool.id, + name: tool.name, + input, + providerExecuted: tool.providerExecuted ? true : undefined, + providerMetadata: tool.providerMetadata, + }), + ), + Effect.catch((error) => + tool.providerExecuted + ? Effect.fail(error) + : Effect.succeed( + LLMEvent.toolInputError({ + id: tool.id, + name: tool.name, + raw, + message: error.reason.message, + providerMetadata: tool.providerMetadata, + }), + ), ), ) +} + +const finishEvents = (tool: PendingTool, event: ToolCall | ToolInputError): ReadonlyArray => + event.type === "tool-input-error" + ? [event] + : [LLMEvent.toolInputEnd({ id: tool.id, name: tool.name, providerMetadata: tool.providerMetadata }), event] /** Store the updated tool and produce the optional public delta event. */ const appendTool = ( @@ -158,8 +178,9 @@ export const appendExisting = ( /** * Finalize one pending tool call: parse the accumulated raw JSON, remove it - * from state, and return the optional public `tool-call` event. Missing keys are - * a no-op because some providers emit stop events for non-tool content blocks. + * from state, and return either a call or a non-executable local input error. + * Missing keys are a no-op because some providers emit stop events for + * non-tool content blocks. */ export const finish = (route: string, tools: State, key: K) => Effect.gen(function* () { @@ -167,10 +188,7 @@ export const finish = (route: string, tools: State, key: if (!tool) return { tools } return { tools: withoutTool(tools, key), - events: [ - LLMEvent.toolInputEnd({ id: tool.id, name: tool.name, providerMetadata: tool.providerMetadata }), - yield* toolCall(route, tool), - ], + events: finishEvents(tool, yield* toolCall(route, tool)), } }) @@ -185,17 +203,14 @@ export const finishWithInput = (route: string, tools: State if (!tool) return { tools } return { tools: withoutTool(tools, key), - events: [ - LLMEvent.toolInputEnd({ id: tool.id, name: tool.name, providerMetadata: tool.providerMetadata }), - yield* toolCall(route, tool, input), - ], + events: finishEvents(tool, yield* toolCall(route, tool, input)), } }) /** * Finalize every pending tool call at once. OpenAI Chat has this shape: it does - * not emit per-tool stop events, so all accumulated calls finish when the choice - * receives a terminal `finish_reason`. + * not emit per-tool stop events, so all accumulated calls finish independently + * when the choice receives a terminal `finish_reason`. */ export const finishAll = (route: string, tools: State) => Effect.gen(function* () { @@ -205,12 +220,7 @@ export const finishAll = (route: string, tools: State) = return { tools: empty(), events: yield* Effect.forEach(pending, (tool) => - toolCall(route, tool).pipe( - Effect.map((call) => [ - LLMEvent.toolInputEnd({ id: tool.id, name: tool.name, providerMetadata: tool.providerMetadata }), - call, - ]), - ), + toolCall(route, tool).pipe(Effect.map((event) => finishEvents(tool, event))), ).pipe(Effect.map((events) => events.flat())), } }) diff --git a/packages/ai/src/schema/events.ts b/packages/ai/src/schema/events.ts index f43251f670..74e2e10ea6 100644 --- a/packages/ai/src/schema/events.ts +++ b/packages/ai/src/schema/events.ts @@ -129,6 +129,7 @@ export const ToolInputStart = Schema.Struct({ type: Schema.tag("tool-input-start"), id: ToolCallID, name: Schema.String, + providerExecuted: Schema.optional(Schema.Boolean), providerMetadata: Schema.optional(ProviderMetadata), }).annotate({ identifier: "LLM.Event.ToolInputStart" }) export type ToolInputStart = Schema.Schema.Type @@ -149,6 +150,17 @@ export const ToolInputEnd = Schema.Struct({ }).annotate({ identifier: "LLM.Event.ToolInputEnd" }) export type ToolInputEnd = Schema.Schema.Type +/** A local tool call that could not be decoded. `raw` is diagnostic-only. */ +export const ToolInputError = Schema.Struct({ + type: Schema.tag("tool-input-error"), + id: ToolCallID, + name: Schema.String, + raw: Schema.String, + message: Schema.String, + providerMetadata: Schema.optional(ProviderMetadata), +}).annotate({ identifier: "LLM.Event.ToolInputError" }) +export type ToolInputError = Schema.Schema.Type + export const ToolCall = Schema.Struct({ type: Schema.tag("tool-call"), id: ToolCallID, @@ -216,6 +228,7 @@ const llmEventTagged = Schema.Union([ ToolInputStart, ToolInputDelta, ToolInputEnd, + ToolInputError, ToolCall, ToolResult, ToolError, @@ -253,6 +266,8 @@ export const LLMEvent = Object.assign(llmEventTagged, { toolInputDelta: (input: WithID) => ToolInputDelta.make({ ...input, id: toolCallID(input.id) }), toolInputEnd: (input: WithID) => ToolInputEnd.make({ ...input, id: toolCallID(input.id) }), + toolInputError: (input: WithID) => + ToolInputError.make({ ...input, id: toolCallID(input.id) }), toolCall: (input: WithID) => ToolCall.make({ ...input, id: toolCallID(input.id) }), toolResult: (input: WithID) => ToolResult.make({ @@ -283,6 +298,7 @@ export const LLMEvent = Object.assign(llmEventTagged, { toolInputStart: llmEventTagged.guards["tool-input-start"], toolInputDelta: llmEventTagged.guards["tool-input-delta"], toolInputEnd: llmEventTagged.guards["tool-input-end"], + toolInputError: llmEventTagged.guards["tool-input-error"], toolCall: llmEventTagged.guards["tool-call"], toolResult: llmEventTagged.guards["tool-result"], toolError: llmEventTagged.guards["tool-error"], @@ -548,6 +564,10 @@ const reduceResponseState = (state: ResponseState, event: LLMEvent): ResponseSta return reduceToolInputDelta(next, event) case "tool-input-end": return reduceToolInputEnd(next, event) + case "tool-input-error": { + const { [event.id]: _finished, ...toolInputs } = next.toolInputs + return { ...next, toolInputs } + } case "tool-call": return reduceToolCall(next, event) case "tool-result": diff --git a/packages/ai/test/provider/anthropic-messages.test.ts b/packages/ai/test/provider/anthropic-messages.test.ts index dcb20aa0ae..69a27e6228 100644 --- a/packages/ai/test/provider/anthropic-messages.test.ts +++ b/packages/ai/test/provider/anthropic-messages.test.ts @@ -484,6 +484,30 @@ describe("Anthropic Messages route", () => { }), ) + it.effect("keeps malformed server tool input terminal", () => + Effect.gen(function* () { + const body = sseEvents( + { type: "message_start", message: { usage: { input_tokens: 5 } } }, + { + type: "content_block_start", + index: 0, + content_block: { type: "server_tool_use", id: "call_1", name: "web_search" }, + }, + { + type: "content_block_delta", + index: 0, + delta: { type: "input_json_delta", partial_json: '{"query":"partial' }, + }, + { type: "content_block_stop", index: 0 }, + ) + + const error = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)), Effect.flip) + + expect(error).toBeInstanceOf(LLMError) + expect(error.message).toContain("Invalid JSON input for anthropic-messages tool call web_search") + }), + ) + it.effect("fails with a typed provider error for stream error frames", () => Effect.gen(function* () { const error = yield* LLMClient.generate(request).pipe( diff --git a/packages/ai/test/provider/bedrock-converse.test.ts b/packages/ai/test/provider/bedrock-converse.test.ts index c7b519d368..a550d1eb90 100644 --- a/packages/ai/test/provider/bedrock-converse.test.ts +++ b/packages/ai/test/provider/bedrock-converse.test.ts @@ -303,6 +303,32 @@ describe("Bedrock Converse route", () => { }), ) + it.effect("emits malformed tool input as an unexecuted tool error", () => + Effect.gen(function* () { + const body = eventStreamBody( + ["messageStart", { role: "assistant" }], + [ + "contentBlockStart", + { + contentBlockIndex: 0, + start: { toolUse: { toolUseId: "tool_1", name: "lookup" } }, + }, + ], + ["contentBlockDelta", { contentBlockIndex: 0, delta: { toolUse: { input: '{"query":"partial' } } }], + ["contentBlockStop", { contentBlockIndex: 0 }], + ["messageStop", { stopReason: "end_turn" }], + ) + const response = yield* LLMClient.generate(baseRequest).pipe(Effect.provide(fixedBytes(body))) + + expect(response.events.find((event) => event.type === "tool-input-error")).toMatchObject({ + id: "tool_1", + name: "lookup", + raw: '{"query":"partial', + }) + expect(response.finishReason).toBe("tool-calls") + }), + ) + it.effect("decodes reasoning deltas", () => Effect.gen(function* () { const body = eventStreamBody( diff --git a/packages/ai/test/provider/openai-responses.test.ts b/packages/ai/test/provider/openai-responses.test.ts index 7548340690..cde2a531dd 100644 --- a/packages/ai/test/provider/openai-responses.test.ts +++ b/packages/ai/test/provider/openai-responses.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { ConfigProvider, Effect, Layer, Stream } from "effect" import { Headers, HttpClientRequest } from "effect/unstable/http" -import { LLM, LLMError, Message, Model, ToolCallPart, Usage } from "../../src" +import { LLM, LLMError, LLMEvent, Message, Model, ToolCallPart, Usage } from "../../src" import { Auth, LLMClient, RequestExecutor, WebSocketExecutor } from "../../src/route" import * as Azure from "../../src/providers/azure" import * as OpenAI from "../../src/providers/openai" @@ -1259,6 +1259,71 @@ describe("OpenAI Responses route", () => { }), ) + it.effect("emits malformed final function arguments as an unexecuted tool error", () => + Effect.gen(function* () { + const body = sseEvents( + { + type: "response.output_item.added", + item: { type: "function_call", id: "item_1", call_id: "call_1", name: "lookup", arguments: "" }, + }, + { type: "response.function_call_arguments.delta", item_id: "item_1", delta: '{"query":"streamed"}' }, + { + type: "response.output_item.done", + item: { + type: "function_call", + id: "item_1", + call_id: "call_1", + name: "lookup", + arguments: '{"query":"partial', + }, + }, + { type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } }, + ) + const response = yield* LLMClient.generate( + LLM.updateRequest(request, { + tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], + }), + ).pipe(Effect.provide(fixedResponse(body))) + + expect(response.events.find(LLMEvent.is.toolInputError)).toEqual({ + type: "tool-input-error", + id: "call_1", + name: "lookup", + raw: '{"query":"partial', + message: "Invalid JSON input for openai-responses tool call lookup", + providerMetadata: { openai: { itemId: "item_1" } }, + }) + expect(response.finishReason).toBe("tool-calls") + expect(response.events.some(LLMEvent.is.toolCall)).toBeFalse() + }), + ) + + it.effect("settles malformed function arguments when output_item.added is absent", () => + Effect.gen(function* () { + const body = sseEvents( + { + type: "response.output_item.done", + item: { + type: "function_call", + id: "item_1", + call_id: "call_1", + name: "lookup", + arguments: '{"query":"partial', + }, + }, + { type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } }, + ) + const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body))) + + expect(response.events.find(LLMEvent.is.toolInputError)).toMatchObject({ + id: "call_1", + name: "lookup", + raw: '{"query":"partial', + }) + expect(response.finishReason).toBe("tool-calls") + }), + ) + it.effect("decodes web_search_call as provider-executed tool-call + tool-result", () => Effect.gen(function* () { const item = { diff --git a/packages/ai/test/response.test.ts b/packages/ai/test/response.test.ts index 5e48e5ef45..f1b3c5db4f 100644 --- a/packages/ai/test/response.test.ts +++ b/packages/ai/test/response.test.ts @@ -95,4 +95,20 @@ describe("LLMResponse reducer", () => { { type: "tool-call", id: "call_1", name: "lookup", input: { query: "weather" } }, ]) }) + + test("clears malformed tool input without appending an executable call", () => { + const state = reduce([ + LLMEvent.toolInputStart({ id: "call_1", name: "lookup" }), + LLMEvent.toolInputDelta({ id: "call_1", name: "lookup", text: '{"query":"partial' }), + LLMEvent.toolInputError({ + id: "call_1", + name: "lookup", + raw: '{"query":"partial', + message: "Invalid JSON input", + }), + ]) + + expect(state.toolInputs).toEqual({}) + expect(state.message.content).toEqual([]) + }) }) diff --git a/packages/ai/test/tool-stream.test.ts b/packages/ai/test/tool-stream.test.ts index b005d2666c..95c1bb7b67 100644 --- a/packages/ai/test/tool-stream.test.ts +++ b/packages/ai/test/tool-stream.test.ts @@ -64,6 +64,75 @@ describe("ToolStream", () => { }), ) + it.effect("finalizes malformed local input as a non-executable tool error", () => + Effect.gen(function* () { + const tools = ToolStream.start(ToolStream.empty(), "item_1", { + id: "call_1", + name: "lookup", + input: '{"query":"partial', + }) + const finished = yield* ToolStream.finish(ADAPTER, tools, "item_1") + + expect(finished).toEqual({ + tools: {}, + events: [ + { + type: "tool-input-error", + id: "call_1", + name: "lookup", + raw: '{"query":"partial', + message: "Invalid JSON input for test-route tool call lookup", + }, + ], + }) + }), + ) + + it.effect("preserves valid siblings when one parallel input is malformed", () => + Effect.gen(function* () { + const valid = ToolStream.start(ToolStream.empty(), 0, { + id: "call_valid", + name: "lookup", + input: '{"query":"weather"}', + }) + const tools = ToolStream.start(valid, 1, { + id: "call_invalid", + name: "lookup", + input: '{"query":"partial', + }) + const finished = yield* ToolStream.finishAll(ADAPTER, tools) + + expect(finished).toEqual({ + tools: {}, + events: [ + { type: "tool-input-end", id: "call_valid", name: "lookup" }, + { type: "tool-call", id: "call_valid", name: "lookup", input: { query: "weather" } }, + { + type: "tool-input-error", + id: "call_invalid", + name: "lookup", + raw: '{"query":"partial', + message: "Invalid JSON input for test-route tool call lookup", + }, + ], + }) + }), + ) + + it.effect("keeps malformed provider-executed input terminal", () => + Effect.gen(function* () { + const tools = ToolStream.start(ToolStream.empty(), "item_1", { + id: "call_1", + name: "web_search", + input: '{"query":"partial', + providerExecuted: true, + }) + const result = yield* Effect.exit(ToolStream.finish(ADAPTER, tools, "item_1")) + + expect(result._tag).toBe("Failure") + }), + ) + it.effect("preserves providerExecuted and clears all tools", () => Effect.gen(function* () { const first: ToolStream.State = ToolStream.start(ToolStream.empty(), 0, { diff --git a/packages/client/src/promise/generated/types.ts b/packages/client/src/promise/generated/types.ts index a22956f74b..7e58e06ef4 100644 --- a/packages/client/src/promise/generated/types.ts +++ b/packages/client/src/promise/generated/types.ts @@ -1276,6 +1276,8 @@ export type SessionStepFailed = { error: SessionStructuredError cost?: MoneyUSD tokens?: TokenUsageInfo + snapshot?: string + files?: Array } } diff --git a/packages/core/src/aisdk.ts b/packages/core/src/aisdk.ts index 4ecda81b9e..2aacaa7939 100644 --- a/packages/core/src/aisdk.ts +++ b/packages/core/src/aisdk.ts @@ -30,6 +30,7 @@ import { type UsageInput, } from "@opencode-ai/ai" import { Auth, Endpoint, type AnyRoute } from "@opencode-ai/ai/route" +import { ProviderShared } from "@opencode-ai/ai/protocols/shared" import { Cause, Context, Effect, Layer, Option, Schema, Scope, Stream } from "effect" import { ModelV2 } from "./model" import { ProviderV2 } from "./provider" @@ -605,6 +606,7 @@ function streamPartEvents( LLMEvent.toolInputStart({ id: event.id, name: event.toolName, + providerExecuted: event.providerExecuted, providerMetadata: providerMetadata(event.providerMetadata), }), ]) @@ -622,15 +624,30 @@ function streamPartEvents( ]) case "tool-call": state.toolNames[event.toolCallId] = event.toolName - return Effect.succeed([ - LLMEvent.toolCall({ - id: event.toolCallId, - name: event.toolName, - input: parseToolInput(event.input), - providerExecuted: event.providerExecuted, - providerMetadata: providerMetadata(event.providerMetadata), - }), - ]) + return ProviderShared.parseToolInput("aisdk", event.toolName, event.input).pipe( + Effect.map((input) => [ + LLMEvent.toolCall({ + id: event.toolCallId, + name: event.toolName, + input, + providerExecuted: event.providerExecuted, + providerMetadata: providerMetadata(event.providerMetadata), + }), + ]), + Effect.catch((error) => + event.providerExecuted + ? Effect.fail(error) + : Effect.succeed([ + LLMEvent.toolInputError({ + id: event.toolCallId, + name: event.toolName, + raw: event.input, + message: error.reason.message, + providerMetadata: providerMetadata(event.providerMetadata), + }), + ]), + ), + ) case "tool-result": delete state.toolNames[event.toolCallId] return Effect.succeed([ @@ -685,14 +702,6 @@ function providerMetadata(value: unknown) { return Schema.is(ProviderMetadata)(value) ? value : undefined } -function parseToolInput(value: string) { - try { - return JSON.parse(value) as unknown - } catch { - return value - } -} - function jsonObject(input: Record) { return Object.fromEntries(Object.entries(input).map(([key, value]) => [key, jsonValue(value)])) } diff --git a/packages/core/src/session/message-updater.ts b/packages/core/src/session/message-updater.ts index e4c895284c..00b0927cf4 100644 --- a/packages/core/src/session/message-updater.ts +++ b/packages/core/src/session/message-updater.ts @@ -297,6 +297,12 @@ export function update(adapter: Adapter, event: SessionEvent.Event) { draft.cost = event.data.cost draft.tokens = castDraft(event.data.tokens) } + if (event.data.snapshot || event.data.files) + draft.snapshot = { + ...draft.snapshot, + end: event.data.snapshot, + files: event.data.files ? Array.from(event.data.files) : undefined, + } }) }, "session.text.started": (event) => { diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 6ed210fde1..a6c7f96749 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -80,6 +80,7 @@ const layer = Layer.effect( sessionID: SessionSchema.ID, promotion: SessionPending.Delivery | undefined, step: number, + recoverMalformedToolInput: boolean, recoverOverflow?: typeof compaction.compact, assistantMessageID?: SessionMessage.ID, ) { @@ -195,25 +196,27 @@ const layer = Layer.effect( tokens: settlement.tokens, }) - // Captures the end snapshot, diffs it against the step's start, and durably ends the - // assistant step. + const captureStepEnd = Effect.fnUntraced(function* () { + const snapshot = yield* snapshots.capture() + const files = + startSnapshot && snapshot + ? yield* snapshots + .files({ from: startSnapshot, to: snapshot }) + .pipe(Effect.catch(() => Effect.succeed(undefined))) + : undefined + return { snapshot, files } + }) + const publishStepEnd = (settlement: NonNullable>) => Effect.gen(function* () { - const endSnapshot = yield* snapshots.capture() - const files = - startSnapshot && endSnapshot - ? yield* snapshots - .files({ from: startSnapshot, to: endSnapshot }) - .pipe(Effect.catch(() => Effect.succeed(undefined))) - : undefined + const end = yield* captureStepEnd() yield* serialized( events.publish(SessionEvent.Step.Ended, { sessionID: session.id, assistantMessageID: yield* publisher.startAssistant(), finish: settlement.finish, ...stepUsage(settlement), - snapshot: endSnapshot, - files, + ...end, }), ) }) @@ -253,7 +256,7 @@ const layer = Layer.effect( step: currentStep, }) } - yield* serialized(publisher.failAssistant(error)) + yield* serialized(publisher.failAssistant(error, true)) } // Provider error events only arrive from the stream, so the flag is final here. const providerFailed = publisher.hasProviderError() @@ -274,7 +277,7 @@ const layer = Layer.effect( if (settled._tag === "Failure") yield* FiberSet.clear(toolFibers) if (userDeclined || streamInterrupted || toolsInterrupted) { yield* serialized(publisher.failUnsettledTools({ type: "aborted", message: "Tool execution interrupted" })) - yield* serialized(publisher.failAssistant({ type: "aborted", message: "Step interrupted" })) + yield* serialized(publisher.failAssistant({ type: "aborted", message: "Step interrupted" }, true)) } // A settled tool fiber failure is one of two things. A defect from a tool // implementation becomes a failed tool call the model can read, and the step still @@ -287,7 +290,7 @@ const layer = Layer.effect( const failure = infraError ?? Cause.squash(settledFailure) const error = toSessionError(failure) yield* serialized(publisher.failUnsettledTools(error)) - if (infraError !== undefined) yield* serialized(publisher.failAssistant(error)) + if (infraError !== undefined) yield* serialized(publisher.failAssistant(error, true)) } // Fail unresolved calls before the terminal step event. Local calls have joined, so @@ -315,27 +318,49 @@ const layer = Layer.effect( : false if (hostedResultMissing && !publisher.stepSettlement()) yield* serialized( - publisher.failAssistant({ - type: "tool.result-missing", - message: "Provider did not return a tool result", - }), + publisher.failAssistant( + { + type: "tool.result-missing", + message: "Provider did not return a tool result", + }, + true, + ), ) const stepFailure = publisher.stepFailure() const stepSettlement = publisher.stepSettlement() if (stepSettlement && !stepFailure) yield* publishStepEnd(stepSettlement) - if (stepFailure) - yield* serialized(publisher.publishStepFailure(stepSettlement ? stepUsage(stepSettlement) : undefined)) + if (stepFailure) { + const end = yield* captureStepEnd() + yield* serialized( + publisher.publishStepFailure({ + ...(stepSettlement ? stepUsage(stepSettlement) : {}), + ...end, + }), + ) + } + + const recoveredMalformedToolInput = + recoverMalformedToolInput && + publisher.hasMalformedToolInput() && + stream._tag === "Success" && + stepSettlement !== undefined && + !providerFailed && + !streamInterrupted && + !userDeclined && + !toolsInterrupted && + infraError === undefined if (stream._tag === "Failure") return yield* Effect.failCause(stream.cause) if (userDeclined) return yield* Effect.interrupt if ((toolsInterrupted || infraError !== undefined) && settledFailure) return yield* Effect.failCause(settledFailure) if (toolsInterrupted && settled._tag === "Failure") return yield* Effect.failCause(settled.cause) - if (stepFailure) return yield* new StepFailedError({ error: stepFailure }) + if (stepFailure && !recoveredMalformedToolInput) return yield* new StepFailedError({ error: stepFailure }) return { _tag: "Completed", - needsContinuation, + needsContinuation: needsContinuation || recoveredMalformedToolInput, + malformedToolInput: recoveredMalformedToolInput, step: currentStep, } as const }), @@ -346,6 +371,7 @@ const layer = Layer.effect( sessionID: SessionSchema.ID, promotion: SessionPending.Delivery | undefined, step: number, + recoverMalformedToolInput: boolean, ) { // Compaction restarts rebuild the request from compacted history without re-promoting. // Overflow recovery is one-shot: a post-compaction attempt must not recover another @@ -356,7 +382,14 @@ const layer = Layer.effect( let assistantMessageID: SessionMessage.ID | undefined while (true) { const attempt = yield* Effect.suspend(() => - attemptStep(sessionID, currentPromotion, currentStep, recoverOverflow, assistantMessageID), + attemptStep( + sessionID, + currentPromotion, + currentStep, + recoverMalformedToolInput, + recoverOverflow, + assistantMessageID, + ), ).pipe( Effect.tapError((error) => error instanceof SessionRunnerRetry.RetryableFailure @@ -378,7 +411,12 @@ const layer = Layer.effect( .pipe(Effect.andThen(Effect.fail(error.cause))) }), ) - if (attempt._tag === "Completed") return { needsContinuation: attempt.needsContinuation, step: attempt.step } + if (attempt._tag === "Completed") + return { + needsContinuation: attempt.needsContinuation, + malformedToolInput: attempt.malformedToolInput, + step: attempt.step, + } if (attempt._tag === "RestartAfterOverflowCompaction") recoverOverflow = undefined yield* Effect.yieldNow currentPromotion = undefined @@ -436,12 +474,18 @@ const layer = Layer.effect( while (shouldRun) { let needsContinuation = true let step = 1 + let canRecoverMalformedToolInput = true // Repeat steps while continuation is needed. A step needs continuation only // when it recorded local tool calls whose results the model has not yet seen; // a provider error suppresses it. Pending steers also continue the loop so // interjections are answered before the session goes idle. while (needsContinuation) { - const result = yield* runStep(input.sessionID, promotion, step) + const result = yield* runStep( + input.sessionID, + promotion, + step, + canRecoverMalformedToolInput, + ) // Steer/queue promotion inside runStep has already made the pending input a visible // user message by this point, so the first-user-message check below is reliable. if (!titleAttempted.has(input.sessionID)) { @@ -449,6 +493,7 @@ const layer = Layer.effect( forkTitle(title.generateForFirstPrompt(yield* getSession(input.sessionID)).pipe(Effect.ignore)) } needsContinuation = result.needsContinuation + if (result.malformedToolInput) canRecoverMalformedToolInput = false step = result.step + 1 if (needsContinuation) { yield* runPendingCompaction(input.sessionID) diff --git a/packages/core/src/session/runner/publish-llm-event.ts b/packages/core/src/session/runner/publish-llm-event.ts index 6a71f5d888..0a07e92a75 100644 --- a/packages/core/src/session/runner/publish-llm-event.ts +++ b/packages/core/src/session/runner/publish-llm-event.ts @@ -9,6 +9,7 @@ import { SessionError } from "@opencode-ai/schema/session-error" import { Money } from "@opencode-ai/schema/money" import { AgentV2 } from "../../agent" import { Snapshot } from "../../snapshot" +import { RelativePath } from "../../schema" import { SessionUsage } from "../usage" type Input = { @@ -60,6 +61,7 @@ export const createLLMEventPublisher = (events: Pick) { + const end = Effect.fnUntraced(function* (id: string, state?: Record, value?: string) { const current = chunks.get(id) if (!current) return yield* Effect.die(new Error(`${name} end before start: ${id}`)) yield* ended( id, - current.values.join(""), + value ?? current.values.join(""), current.ordinal, state === undefined ? current.state : { ...current.state, ...state }, ) @@ -175,7 +177,11 @@ export const createLLMEventPublisher = (events: Pick ${event.name}`)) if (!toolInput.has(event.id)) return yield* Effect.die(new Error(`Duplicate tool input end: ${event.id}`)) - yield* toolInput.end(event.id) + yield* toolInput.end(event.id, undefined, value) + }) + + const failMalformedToolInput = Effect.fnUntraced(function* (event: { + readonly id: string + readonly name: string + readonly raw: string + readonly message: string + }) { + if (!tools.has(event.id)) yield* startToolInput(event) + const tool = tools.get(event.id) + if (!tool || tool.called || tool.settled) + return yield* Effect.die(new Error(`Malformed tool input after call settlement: ${event.id}`)) + if (tool.name !== event.name) + return yield* Effect.die(new Error(`Tool input name changed for ${event.id}: ${tool.name} -> ${event.name}`)) + if (toolInput.has(event.id)) yield* endToolInput(event, event.raw) + tool.settled = true + malformedToolInput = true + yield* events.publish(SessionEvent.Tool.Failed, { + sessionID: input.sessionID, + assistantMessageID: tool.assistantMessageID, + callID: event.id, + error: { + type: "tool.input-json", + message: "Tool call arguments were malformed JSON and were not executed. Retry with valid JSON.", + }, + executed: false, + }) + if (stepFailure === undefined) stepFailure = { type: "provider.invalid-output", message: event.message } }) const flush = Effect.fn("SessionRunner.flush")(function* () { @@ -236,9 +273,11 @@ export const createLLMEventPublisher = (events: Pick + const publishStepFailure = Effect.fnUntraced(function* (details?: { + readonly cost?: Money.USD + readonly tokens?: ReturnType + readonly snapshot?: Snapshot.ID + readonly files?: readonly RelativePath[] }) { if (stepFailed || stepFailure === undefined) return const assistantMessageID = yield* startAssistant() @@ -247,7 +286,7 @@ export const createLLMEventPublisher = (events: Pick providerFailed, + hasMalformedToolInput: () => malformedToolInput, hasRetryEvidence: () => retryEvidence, stepFailure: () => stepFailure, stepSettlement: () => stepSettlement, diff --git a/packages/core/src/session/runner/to-llm-message.ts b/packages/core/src/session/runner/to-llm-message.ts index e91eef2c28..7aa52e53e3 100644 --- a/packages/core/src/session/runner/to-llm-message.ts +++ b/packages/core/src/session/runner/to-llm-message.ts @@ -136,14 +136,20 @@ const assistant = (message: SessionMessage.Assistant, model: ModelV2.Ref, provid : item.text.length > 0 ? [{ type: "text", text: item.text }] : [] + const reuseToolProviderMetadata = + sameModel && + (message.error === undefined || + (item.executed === true && + (item.state.status === "completed" || + (item.state.status === "error" && item.state.result !== undefined)))) const call = toolCall( item, - reuseProviderMetadata ? providerMetadata(providerMetadataKey, item.providerState) : undefined, + reuseToolProviderMetadata ? providerMetadata(providerMetadataKey, item.providerState) : undefined, ) if (item.executed !== true) return [call] const result = toolResult( item, - reuseProviderMetadata + reuseToolProviderMetadata ? providerMetadata(providerMetadataKey, item.providerResultState ?? item.providerState) : undefined, ) diff --git a/packages/core/test/aisdk.test.ts b/packages/core/test/aisdk.test.ts index 59c8f8e8d3..2ab0a977fa 100644 --- a/packages/core/test/aisdk.test.ts +++ b/packages/core/test/aisdk.test.ts @@ -1,11 +1,11 @@ -import type { LanguageModelV3CallOptions } from "@ai-sdk/provider" +import type { LanguageModelV3, LanguageModelV3CallOptions, LanguageModelV3StreamPart } from "@ai-sdk/provider" import { AISDK } from "@opencode-ai/core/aisdk" import { ModelV2 } from "@opencode-ai/core/model" import { ProviderV2 } from "@opencode-ai/core/provider" -import { LLM, Message } from "@opencode-ai/ai" -import { LLMClient } from "@opencode-ai/ai/route" +import { LLM, LLMError, LLMEvent, Message } from "@opencode-ai/ai" +import { LLMClient, RequestExecutor } from "@opencode-ai/ai/route" import { expect } from "bun:test" -import { Effect } from "effect" +import { Effect, Layer } from "effect" import { testEffect } from "./lib/effect" const it = testEffect(AISDK.locationLayer) @@ -19,6 +19,37 @@ const model = (packageName: string, settings: Record = {}) => limit: { context: 100, output: 20 }, }) +const streamModel = (events: ReadonlyArray): LanguageModelV3 => ({ + specificationVersion: "v3", + provider: "test", + modelId: "test", + supportedUrls: {}, + doGenerate: () => Promise.reject(new Error("Unexpected non-streaming request")), + doStream: () => + Promise.resolve({ + stream: new ReadableStream({ + start(controller) { + events.forEach((event) => controller.enqueue(event)) + controller.close() + }, + }), + }), +}) + +const usage = { + inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 1, text: 0, reasoning: 0 }, +} as const + +const client = LLMClient.layer.pipe( + Layer.provide( + Layer.succeed( + RequestExecutor.Service, + RequestExecutor.Service.of({ execute: () => Effect.die("Unexpected HTTP request") }), + ), + ), +) + it.effect("keys language models by package and flattened overlays", () => Effect.gen(function* () { const aisdk = yield* AISDK.Service @@ -238,3 +269,69 @@ it.effect("projects replay metadata onto AI SDK prompt parts", () => ]) }), ) + +it.effect("emits malformed AI SDK tool input without executing it", () => + Effect.gen(function* () { + const aisdk = yield* AISDK.Service + const raw = '{"query":"partial' + yield* aisdk.hook.sdk((event) => { + event.sdk = { + languageModel: () => + streamModel([ + { type: "tool-input-start", id: "call_1", toolName: "lookup" }, + { type: "tool-input-delta", id: "call_1", delta: raw }, + { type: "tool-input-end", id: "call_1" }, + { type: "tool-call", toolCallId: "call_1", toolName: "lookup", input: raw }, + { type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage }, + ]), + } + }) + + const resolved = yield* aisdk.model(model("test-ai-sdk")) + const response = yield* LLMClient.generate(LLM.request({ model: resolved, prompt: "Lookup" })).pipe( + Effect.provide(client), + ) + + expect(response.events.find(LLMEvent.is.toolInputError)).toMatchObject({ + id: "call_1", + name: "lookup", + raw, + message: "Invalid JSON input for aisdk tool call lookup", + }) + expect(response.events.some(LLMEvent.is.toolInputEnd)).toBeTrue() + expect(response.events.some(LLMEvent.is.toolCall)).toBeFalse() + }), +) + +it.effect("keeps malformed provider-executed AI SDK input terminal", () => + Effect.gen(function* () { + const aisdk = yield* AISDK.Service + const raw = '{"query":"partial' + yield* aisdk.hook.sdk((event) => { + event.sdk = { + languageModel: () => + streamModel([ + { type: "tool-input-start", id: "call_1", toolName: "web_search", providerExecuted: true }, + { type: "tool-input-delta", id: "call_1", delta: raw }, + { type: "tool-input-end", id: "call_1" }, + { + type: "tool-call", + toolCallId: "call_1", + toolName: "web_search", + input: raw, + providerExecuted: true, + }, + ]), + } + }) + + const resolved = yield* aisdk.model(model("hosted-test-ai-sdk")) + const error = yield* LLMClient.generate(LLM.request({ model: resolved, prompt: "Search" })).pipe( + Effect.provide(client), + Effect.flip, + ) + + expect(error).toBeInstanceOf(LLMError) + expect(error.message).toContain("Invalid JSON input for aisdk tool call web_search") + }), +) diff --git a/packages/core/test/session-runner-message.test.ts b/packages/core/test/session-runner-message.test.ts index fe70ac5943..a79b351cfa 100644 --- a/packages/core/test/session-runner-message.test.ts +++ b/packages/core/test/session-runner-message.test.ts @@ -565,6 +565,22 @@ Recent work text: "Partial thought", state: { itemId: "rs_failed", reasoningEncryptedContent: null }, }), + SessionMessage.AssistantTool.make({ + type: "tool", + id: "hosted-completed", + name: "web_search", + executed: true, + providerState: { itemId: "call_completed" }, + providerResultState: { itemId: "result_completed" }, + state: SessionMessage.ToolStateCompleted.make({ + status: "completed", + input: { query: "Effect" }, + content: [], + structured: {}, + result: { type: "json", value: { found: true } }, + }), + time: { created, completed: created }, + }), SessionMessage.AssistantTool.make({ type: "tool", id: "hosted-failed", @@ -592,6 +608,22 @@ Recent work expect(messages[0]?.content).toEqual([ { type: "text", text: "Partial thought" }, + { + type: "tool-call", + id: "hosted-completed", + name: "web_search", + input: { query: "Effect" }, + providerExecuted: true, + providerMetadata: { provider: { itemId: "call_completed" } }, + }, + { + type: "tool-result", + id: "hosted-completed", + name: "web_search", + result: { type: "json", value: { found: true } }, + providerExecuted: true, + providerMetadata: { provider: { itemId: "result_completed" } }, + }, { type: "tool-call", id: "hosted-failed", diff --git a/packages/core/test/session-runner-tool-events.test.ts b/packages/core/test/session-runner-tool-events.test.ts index 93600c3058..b6f44dccdb 100644 --- a/packages/core/test/session-runner-tool-events.test.ts +++ b/packages/core/test/session-runner-tool-events.test.ts @@ -9,6 +9,8 @@ import { SessionMessage } from "@opencode-ai/core/session/message" import { SessionV2 } from "@opencode-ai/core/session" import { ModelV2 } from "@opencode-ai/core/model" import { ProviderV2 } from "@opencode-ai/core/provider" +import { RelativePath } from "@opencode-ai/core/schema" +import { Snapshot } from "@opencode-ai/core/snapshot" import { createLLMEventPublisher } from "@opencode-ai/core/session/runner/publish-llm-event" const sessionID = SessionV2.ID.make("ses_tool_event_test") @@ -229,6 +231,8 @@ test("content-filter finish retains failure evidence until step closeout", async publisher.publishStepFailure({ cost: Money.USD.make(1.25), tokens: settlement.tokens, + snapshot: Snapshot.ID.make("tree-end"), + files: [RelativePath.make("src/changed.ts")], }), ) expect(published.map((event) => event.type)).toEqual(["session.step.started.1", "session.step.failed.1"]) @@ -236,6 +240,8 @@ test("content-filter finish retains failure evidence until step closeout", async error: { type: "provider.content-filter", message: "Provider blocked the response" }, cost: 1.25, tokens: { input: 8, output: 2, reasoning: 1 }, + snapshot: "tree-end", + files: ["src/changed.ts"], }) }) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 6bd8b23be1..df8134cf2b 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -4164,6 +4164,299 @@ describe("SessionRunnerLLM", () => { }), ) + it.effect("continues once after malformed local tool input without exposing raw arguments", () => + Effect.gen(function* () { + const session = yield* setup + yield* admit(session, "Recover malformed tool input") + const marker = "raw-malformed-marker" + const raw = `{"text":"${marker}` + responses = [ + [ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolInputStart({ id: "call-malformed", name: "echo" }), + LLMEvent.toolInputDelta({ id: "call-malformed", name: "echo", text: raw }), + LLMEvent.toolInputEnd({ id: "call-malformed", name: "echo" }), + LLMEvent.toolInputError({ + id: "call-malformed", + name: "echo", + raw, + message: "Invalid JSON input for test tool call echo", + }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + LLMEvent.finish({ reason: "tool-calls" }), + ], + reply.stop(), + ] + + yield* session.resume(sessionID) + + expect(requests).toHaveLength(2) + expect(executions).toEqual([]) + expect(JSON.stringify(requests[1])).not.toContain(marker) + expect(requests[1]?.messages).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + role: "assistant", + content: expect.arrayContaining([ + expect.objectContaining({ type: "tool-call", id: "call-malformed", name: "echo", input: {} }), + ]), + }), + expect.objectContaining({ + role: "tool", + content: expect.arrayContaining([ + expect.objectContaining({ + type: "tool-result", + id: "call-malformed", + result: expect.objectContaining({ + type: "error", + value: expect.objectContaining({ + error: expect.objectContaining({ + message: "Tool call arguments were malformed JSON and were not executed. Retry with valid JSON.", + }), + }), + }), + }), + ]), + }), + ]), + ) + const context = yield* session.context(sessionID) + const failed = context.find( + (message): message is SessionMessage.Assistant => + message.type === "assistant" && message.content.some((item) => item.type === "tool"), + ) + expect(failed).toMatchObject({ + error: { type: "provider.invalid-output", message: "Invalid JSON input for test tool call echo" }, + content: [ + { + type: "tool", + id: "call-malformed", + executed: false, + state: { + status: "error", + input: {}, + error: { + type: "tool.input-json", + message: "Tool call arguments were malformed JSON and were not executed. Retry with valid JSON.", + }, + }, + }, + ], + }) + if (!failed) throw new Error("Malformed tool assistant missing") + expect((yield* recordedStepSettlementEvents(sessionID, failed.id)).map((event) => event.type)).toEqual([ + "session.step.started.1", + "session.tool.failed.1", + "session.step.failed.1", + ]) + const database = (yield* Database.Service).db + const durable = yield* database + .select({ type: EventTable.type, data: EventTable.data }) + .from(EventTable) + .where(eq(EventTable.aggregate_id, sessionID)) + .all() + .pipe(Effect.orDie) + expect(durable.find((event) => event.type === "session.tool.input.ended.1")?.data).toMatchObject({ + callID: "call-malformed", + text: raw, + }) + }), + ) + + it.effect("settles a valid sibling before recovering malformed tool input", () => + Effect.gen(function* () { + const session = yield* setup + yield* admit(session, "Run parallel tools") + toolExecutionGate = yield* Deferred.make() + toolExecutionsStarted = yield* Deferred.make() + toolExecutionsReady = 1 + responses = [ + [ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolCall({ id: "call-valid", name: "echo", input: { text: "valid" } }), + LLMEvent.toolInputError({ + id: "call-malformed", + name: "echo", + raw: '{"text":"partial', + message: "Invalid JSON input for test tool call echo", + }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + LLMEvent.finish({ reason: "tool-calls" }), + ], + reply.stop(), + ] + + const run = yield* session.resume(sessionID).pipe(Effect.forkChild) + yield* Deferred.await(toolExecutionsStarted) + expect(requests).toHaveLength(1) + yield* Deferred.succeed(toolExecutionGate, undefined) + yield* Fiber.join(run) + toolExecutionGate = undefined + toolExecutionsStarted = undefined + + expect(requests).toHaveLength(2) + expect(executions).toEqual(["valid"]) + const request = requests[1] + if (!request) throw new Error("Malformed recovery request missing") + expect(request.messages.flatMap((message) => (message.role === "tool" ? message.content : []))).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "call-valid", type: "tool-result" }), + expect.objectContaining({ id: "call-malformed", type: "tool-result" }), + ]), + ) + }), + ) + + it.effect("does not recover malformed input after sibling execution is interrupted", () => + Effect.gen(function* () { + const session = yield* setup + yield* admit(session, "Interrupt malformed recovery") + toolExecutionGate = yield* Deferred.make() + toolExecutionsStarted = yield* Deferred.make() + toolExecutionsReady = 1 + response = [ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolCall({ id: "call-valid", name: "echo", input: { text: "blocked" } }), + LLMEvent.toolInputError({ + id: "call-malformed", + name: "echo", + raw: '{"text":"partial', + message: "Invalid JSON input for test tool call echo", + }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + LLMEvent.finish({ reason: "tool-calls" }), + ] + + const run = yield* session.resume(sessionID).pipe(Effect.forkChild) + yield* Deferred.await(toolExecutionsStarted) + while ( + !(yield* session.context(sessionID)).some( + (message) => + message.type === "assistant" && + message.content.some((item) => item.type === "tool" && item.id === "call-malformed"), + ) + ) + yield* Effect.yieldNow + yield* session.interrupt(sessionID) + toolExecutionGate = undefined + toolExecutionsStarted = undefined + + expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" }) + expect(requests).toHaveLength(1) + expect(yield* session.context(sessionID)).toMatchObject([ + { type: "user", text: "Interrupt malformed recovery" }, + { + type: "assistant", + error: { type: "aborted", message: "Step interrupted" }, + content: [ + { type: "tool", id: "call-valid", state: { status: "error", error: { type: "aborted" } } }, + { type: "tool", id: "call-malformed", state: { status: "error" } }, + ], + }, + ]) + }), + ) + + it.effect("records malformed provider-executed input as executed", () => + Effect.gen(function* () { + const session = yield* setup + yield* admit(session, "Fail malformed hosted input") + const failure = new LLMError({ + module: "test", + method: "stream", + reason: new InvalidProviderOutputReason({ message: "Invalid hosted tool input" }), + }) + responseStream = Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolInputStart({ id: "call-hosted", name: "web_search", providerExecuted: true }), + LLMEvent.toolInputDelta({ id: "call-hosted", name: "web_search", text: '{"query":"partial' }), + ]).pipe(Stream.concat(Stream.fail(failure))) + + expect(yield* session.resume(sessionID).pipe(Effect.flip)).toBe(failure) + expect(requireAssistant(yield* session.context(sessionID))).toMatchObject({ + error: { type: "provider.invalid-output", message: "Invalid hosted tool input" }, + content: [ + { + type: "tool", + id: "call-hosted", + executed: true, + state: { status: "error", error: { type: "provider.invalid-output" } }, + }, + ], + }) + }), + ) + + it.effect("replaces malformed input diagnosis with a later provider failure", () => + Effect.gen(function* () { + const session = yield* setup + yield* admit(session, "Fail after malformed input") + const failure = new LLMError({ + module: "test", + method: "stream", + reason: new InvalidProviderOutputReason({ message: "Provider failed after malformed input" }), + }) + responseStream = Stream.fromIterable([ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolInputError({ + id: "call-malformed", + name: "echo", + raw: '{"text":"partial', + message: "Invalid JSON input for test tool call echo", + }), + ]).pipe(Stream.concat(Stream.fail(failure))) + + expect(yield* session.resume(sessionID).pipe(Effect.flip)).toBe(failure) + expect(requireAssistant(yield* session.context(sessionID))).toMatchObject({ + error: { type: "provider.invalid-output", message: "Provider failed after malformed input" }, + content: [ + { + type: "tool", + id: "call-malformed", + executed: false, + state: { status: "error", error: { type: "tool.input-json" } }, + }, + ], + }) + expect(requests).toHaveLength(1) + }), + ) + + it.effect("does not reset the malformed recovery budget after a valid tool step", () => + Effect.gen(function* () { + const session = yield* setup + yield* admit(session, "Keep producing malformed tools") + const malformed = (id: string) => [ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolInputError({ + id, + name: "echo", + raw: '{"text":"partial', + message: "Invalid JSON input for test tool call echo", + }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + LLMEvent.finish({ reason: "tool-calls" }), + ] + responses = [ + malformed("call-first"), + reply.tool("call-valid-between", "echo", { text: "valid" }), + malformed("call-second"), + reply.stop(), + ] + + expect(yield* session.resume(sessionID).pipe(Effect.flip)).toMatchObject({ + error: { + type: "provider.invalid-output", + message: "Invalid JSON input for test tool call echo", + }, + }) + + expect(requests).toHaveLength(3) + expect(executions).toEqual(["valid"]) + expect((yield* recordedEventTypes(sessionID)).filter((type) => type === "session.step.failed.1")).toHaveLength(2) + }), + ) + it.effect("does not continue automatically after a provider error follows a local tool call", () => Effect.gen(function* () { const session = yield* setup diff --git a/packages/schema/src/session-event.ts b/packages/schema/src/session-event.ts index aaeb6565c3..730b0b1ffb 100644 --- a/packages/schema/src/session-event.ts +++ b/packages/schema/src/session-event.ts @@ -276,6 +276,8 @@ export namespace Step { error: SessionError.Error, cost: Money.USD.pipe(optional), tokens: TokenUsage.Info.pipe(optional), + snapshot: Snapshot.ID.pipe(optional), + files: Schema.Array(RelativePath).pipe(optional), }, }) export type Failed = typeof Failed.Type