Compare commits

..
2 Commits
Author SHA1 Message Date
8352addf6c fix(console): preserve usage reset boundaries (#44729)
Co-authored-by: MrMushrooooom <MrMushrooooom@users.noreply.github.com>
Co-authored-by: Jack <jack@anoma.ly>
2026-09-04 15:12:54 +08:00
Aiden Cline 32f89748af feat(ai): add tool namespaces (#46548) 2026-09-04 01:09:52 -05:00
46 changed files with 825 additions and 246 deletions
+17 -11
View File
@@ -11,7 +11,7 @@
// Manual `cache: CacheHint` placements on individual parts are preserved and
// count against the four-breakpoint budget; auto only fills remaining slots.
import { CacheHint, type CachePolicy, type CachePolicyObject } from "./schema/options.js"
import { LLMRequest, Message, ToolDefinition, type ContentPart } from "./schema/messages.js"
import { LLMRequest, Message, ToolDefinition, type ContentPart, type ToolEntry } from "./schema/messages.js"
const AUTO: CachePolicyObject = {
tools: true,
@@ -50,18 +50,24 @@ interface Budget {
remaining: number
}
const markLastTool = (
tools: ReadonlyArray<ToolDefinition>,
hint: CacheHint,
budget: Budget,
): ReadonlyArray<ToolDefinition> => {
if (tools.length === 0) return tools
const last = tools.length - 1
if (tools[last]!.cache || budget.remaining === 0) return tools
const markLastTool = (tools: ReadonlyArray<ToolEntry>, hint: CacheHint, budget: Budget): ReadonlyArray<ToolEntry> => {
const target = tools.at(-1)
if (target === undefined) return tools
if (target.type === "namespace") {
const nested = markLastTool(target.tools, hint, budget)
return nested === target.tools ? tools : [...tools.slice(0, -1), { ...target, tools: nested }]
}
if (target.cache || budget.remaining === 0) return tools
budget.remaining -= 1
return tools.map((tool, i) => (i === last ? new ToolDefinition({ ...tool, cache: hint }) : tool))
return [...tools.slice(0, -1), new ToolDefinition({ ...target, cache: hint })]
}
const countToolHints = (tools: ReadonlyArray<ToolEntry>): number =>
tools.reduce(
(count, tool) => count + (tool.type === "tool" ? (tool.cache === undefined ? 0 : 1) : countToolHints(tool.tools)),
0,
)
const markSystemBoundaries = (system: LLMRequest["system"], hint: CacheHint, budget: Budget): LLMRequest["system"] => {
if (system.length === 0) return system
let changed = false
@@ -122,7 +128,7 @@ const markMessages = (
}
const countHints = (request: LLMRequest) =>
request.tools.reduce((count, tool) => count + (tool.cache === undefined ? 0 : 1), 0) +
countToolHints(request.tools) +
request.system.reduce((count, part) => count + (part.cache === undefined ? 0 : 1), 0) +
request.messages.reduce(
(count, message) =>
+4 -3
View File
@@ -12,9 +12,10 @@ import {
LanguageModel,
SystemPart,
ToolChoice,
ToolDefinition,
ToolEntry,
type ContentPart,
type LanguageModelProviderOptions,
type ToolEntryInput,
} from "./schema/index.js"
import { make as makeTool, toDefinitions, type ToolSchema } from "./tool.js"
@@ -27,7 +28,7 @@ export type RequestInput<SelectedLanguageModel extends LanguageModel = LanguageM
readonly system?: string | SystemPart | ReadonlyArray<SystemPart>
readonly prompt?: string | ContentPart | ReadonlyArray<ContentPart>
readonly messages?: ReadonlyArray<Message | Message.Input>
readonly tools?: ReadonlyArray<ToolDefinition.Input>
readonly tools?: ReadonlyArray<ToolEntryInput>
readonly toolChoice?: ToolChoice.Input
readonly generation?: GenerationOptions.Input
readonly providerOptions?: NoInfer<LanguageModelProviderOptions<SelectedLanguageModel>>
@@ -56,7 +57,7 @@ export const request = <const SelectedLanguageModel extends LanguageModel>(
...rest,
system: SystemPart.content(requestSystem),
messages: [...(messages?.map(Message.make) ?? []), ...(prompt === undefined ? [] : [Message.user(prompt)])],
tools: tools?.map(ToolDefinition.make) ?? [],
tools: tools?.map(ToolEntry.make) ?? [],
toolChoice: requestToolChoice ? ToolChoice.make(requestToolChoice) : undefined,
generation: requestGeneration === undefined ? undefined : GenerationOptions.make(requestGeneration),
providerOptions: requestProviderOptions,
@@ -1067,10 +1067,11 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
// messages. Tools live highest in the cache hierarchy, so when callers
// over-mark we keep their tool hints and shed the message-tail ones first.
const breakpoints = Cache.newBreakpoints(ANTHROPIC_BREAKPOINT_CAP)
const flattened = ProviderShared.flattenToolRequest(request)
const tools =
request.tools.length === 0
flattened.tools.length === 0
? undefined
: request.tools.map((tool) =>
: flattened.tools.map((tool) =>
lowerTool(
breakpoints,
tool,
@@ -1088,7 +1089,7 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
text: part.text,
cache_control: cacheControl(breakpoints, part.cache),
}))
const messages = yield* lowerMessages(request, breakpoints)
const messages = yield* lowerMessages(flattened.request, breakpoints)
if (breakpoints.dropped > 0) {
yield* Effect.logWarning(
`Anthropic Messages: dropped ${breakpoints.dropped} cache breakpoint(s); the API allows at most ${ANTHROPIC_BREAKPOINT_CAP} per request.`,
@@ -415,7 +415,10 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (
// System prompts share the cache-point convention: emit the text block, then
// optionally a positional `cachePoint` marker.
const lowerSystem = (breakpoints: BedrockCache.Breakpoints, system: ReadonlyArray<LLMRequest["system"][number]>) => {
const lowerSystem = (
breakpoints: BedrockCache.Breakpoints,
system: ReadonlyArray<LLMRequest["system"][number]>,
) => {
const content = system
.filter((part) => part.text.length > 0)
.flatMap((part) => textWithCache(breakpoints, part.text, part.cache))
@@ -424,21 +427,22 @@ const lowerSystem = (breakpoints: BedrockCache.Breakpoints, system: ReadonlyArra
const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request: LLMRequest) {
const toolChoice = request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined
const flattened = ProviderShared.flattenToolRequest(request)
const generation = request.generation
// Bedrock-Claude shares Anthropic's 4-breakpoint cap. Spend the budget in
// tools → system → messages order to favour the highest-impact prefixes.
const breakpoints = BedrockCache.breakpoints()
const toolConfig = (() => {
if (request.tools.length === 0) return undefined
if (flattened.tools.length === 0) return undefined
return {
tools: lowerTools(request.model.compatibility?.toolSchema, breakpoints, request.tools),
tools: lowerTools(request.model.compatibility?.toolSchema, breakpoints, flattened.tools),
// Converse has no native "none". Keep definitions stable for prompt
// caching and omit only the unsupported choice.
toolChoice,
}
})()
const system = lowerSystem(breakpoints, request.system)
const messages = yield* lowerMessages(request, breakpoints)
const messages = yield* lowerMessages(flattened.request, breakpoints)
if (breakpoints.dropped > 0) {
yield* Effect.logWarning(
`Bedrock Converse: dropped ${breakpoints.dropped} cache breakpoint(s); the API allows at most ${BedrockCache.BEDROCK_BREAKPOINT_CAP} per request.`,
+4 -3
View File
@@ -465,7 +465,8 @@ function mapSafetySettings(value: unknown) {
}
const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMRequest) {
const hasTools = request.tools.length > 0
const flattened = ProviderShared.flattenToolRequest(request)
const hasTools = flattened.tools.length > 0
const generation = request.generation
const options = resolveOptions(request)
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
@@ -483,7 +484,7 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
return {
cachedContent: options.cachedContent,
contents: yield* lowerMessages(request),
contents: yield* lowerMessages(flattened.request),
safetySettings: options.safetySettings,
serviceTier: options.serviceTier,
systemInstruction:
@@ -491,7 +492,7 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
tools: hasTools
? [
{
functionDeclarations: request.tools.map((tool) =>
functionDeclarations: flattened.tools.map((tool) =>
lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility)),
),
},
+3 -2
View File
@@ -414,10 +414,11 @@ export const fromRequest = Effect.fn("MistralChat.fromRequest")(function* (reque
tool: (name) => ({ type: "function" as const, function: { name } }),
})
: undefined
const flattened = ProviderShared.flattenToolRequest(request)
return {
model: request.model.id,
messages: yield* lowerMessages(request),
tools: request.tools.length > 0 ? request.tools.map(lowerTool) : undefined,
messages: yield* lowerMessages(flattened.request),
tools: flattened.tools.length > 0 ? flattened.tools.map(lowerTool) : undefined,
tool_choice: toolChoice,
stream: true as const,
max_tokens: request.generation?.maxTokens,
+32 -6
View File
@@ -189,6 +189,7 @@ export const InputItem = Schema.Union([
id: Schema.optionalKey(Schema.String),
call_id: Schema.String,
name: Schema.String,
namespace: Schema.optional(Schema.String),
arguments: Schema.String,
}),
Schema.Struct({
@@ -315,6 +316,7 @@ export const StreamItem = Schema.StructWithRest(
id: Schema.optional(Schema.String),
call_id: Schema.optional(Schema.String),
name: Schema.optional(Schema.String),
namespace: Schema.optional(Schema.String),
arguments: Schema.optional(Schema.String),
encrypted_content: optionalNull(Schema.String),
}),
@@ -488,6 +490,7 @@ const lowerToolCall = (part: ToolCallPart, providerMetadataKey: string): OpenRes
...(id === undefined ? {} : { id }),
call_id: part.id,
name: part.name,
namespace: part.namespace,
arguments: ProviderShared.encodeJson(part.input),
}
}
@@ -807,14 +810,15 @@ export const fromRequestWithAdapter = Effect.fn("OpenResponses.fromRequestWithAd
request: LLMRequest,
adapter: ProviderAdapter,
) {
const projected = ProviderShared.flattenToolRequest(request)
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
return {
...(yield* lowerConversation(request, adapter)),
...(yield* lowerConversation(projected.request, adapter)),
...lowerGeneration(request),
tools:
request.tools.length === 0
projected.tools.length === 0
? undefined
: yield* Effect.forEach(request.tools, (tool) =>
: yield* Effect.forEach(projected.tools, (tool) =>
lowerTool(
adapter.name,
tool,
@@ -1094,11 +1098,20 @@ const onOutputItemAdded = (state: ParserState, event: NormalizedEvent): StepResu
tools: ToolStream.start(state.tools, item.id, {
id: item.call_id,
name: item.name ?? "",
namespace: item.namespace,
input: item.arguments ?? "",
providerMetadata: metadata,
}),
},
[...events, LLMEvent.toolInputStart({ id: item.call_id, name: item.name ?? "", providerMetadata: metadata })],
[
...events,
LLMEvent.toolInputStart({
id: item.call_id,
name: item.name ?? "",
namespace: item.namespace,
providerMetadata: metadata,
}),
],
]
}
@@ -1217,7 +1230,12 @@ const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* (
const registered = state.tools[item.id] !== undefined
const tools = registered
? state.tools
: ToolStream.start(state.tools, item.id, { id: item.call_id, name: item.name, providerMetadata: metadata })
: ToolStream.start(state.tools, item.id, {
id: item.call_id,
name: item.name,
namespace: item.namespace,
providerMetadata: metadata,
})
const result =
item.arguments === undefined
? yield* ToolStream.finish(state.id, tools, item.id)
@@ -1228,7 +1246,15 @@ const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* (
const resultEvents =
registered || finished.length === 0
? finished
: [LLMEvent.toolInputStart({ id: item.call_id, name: item.name, providerMetadata: metadata }), ...finished]
: [
LLMEvent.toolInputStart({
id: item.call_id,
name: item.name,
namespace: item.namespace,
providerMetadata: metadata,
}),
...finished,
]
const lifecycle = resultEvents.length ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle
events.push(...resultEvents)
return [
+5 -4
View File
@@ -736,6 +736,7 @@ export const fromRequest = Effect.fn("OpenAIChat.fromRequest")(function* (
)
const generation = request.generation
const toolSchemaCompatibility = request.model.compatibility?.toolSchema
const flattened = ProviderShared.flattenToolRequest(request)
const provider = String(request.model.provider)
const baseURL = request.model.route.endpoint.baseURL
const detectedMaxTokensField = detectMaxTokensField(provider, baseURL)
@@ -748,16 +749,16 @@ export const fromRequest = Effect.fn("OpenAIChat.fromRequest")(function* (
const zaiToolStream =
request.model.compatibility?.zaiToolStream ?? detectZaiToolStream(provider, baseURL, request.model.id)
const hasHistory = hasToolHistory(request.messages)
const hasActiveTools = request.tools.length > 0
const hasActiveTools = flattened.tools.length > 0
return {
model: request.model.id,
messages: yield* lowerMessages(request, options),
messages: yield* lowerMessages(flattened.request, options),
tools:
request.tools.length === 0
flattened.tools.length === 0
? hasHistory
? []
: undefined
: request.tools.map((tool) =>
: flattened.tools.map((tool) =>
lowerTool(
tool,
ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility),
+36 -7
View File
@@ -5,7 +5,7 @@ import { Auth } from "../route/auth.js"
import { Endpoint } from "../route/endpoint.js"
import { Protocol } from "../route/protocol.js"
import { HttpTransport } from "../route/transport/index.js"
import type { LLMRequest, JsonSchema, ToolDefinition } from "../schema/index.js"
import type { LLMRequest, JsonSchema, ToolDefinition, ToolEntry } from "../schema/index.js"
import { OpenResponses } from "./open-responses.js"
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
import { OpenAIImage } from "./utils/openai-image.js"
@@ -75,7 +75,18 @@ const OpenAIResponsesHostedToolItem = Schema.Union([
),
])
const OpenAIResponsesTools = Schema.Union([OpenResponses.Tool, OpenAIResponsesImageGenerationTool])
const OpenAIResponsesNamespace = Schema.Struct({
type: Schema.tag("namespace"),
name: Schema.String,
description: Schema.String,
tools: Schema.Array(OpenResponses.Tool),
})
const OpenAIResponsesTools = Schema.Union([
OpenResponses.Tool,
OpenAIResponsesNamespace,
OpenAIResponsesImageGenerationTool,
])
const OpenAIResponsesToolChoice = Schema.Union([
OpenResponses.ToolChoice,
@@ -128,13 +139,33 @@ const lowerTool = Effect.fn("OpenAIResponses.lowerTool")(function* (tool: ToolDe
return yield* OpenResponses.lowerTool(NAME, tool, inputSchema)
})
const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>, tools: ReadonlyArray<ToolDefinition>) =>
// Native namespaces hold only function tools, so deeper levels flatten into
// the leaf names the same way non-native protocols flatten the whole tree.
const lowerToolEntry = Effect.fn("OpenAIResponses.lowerToolEntry")(function* (
tool: ToolEntry,
compatibility: Parameters<typeof ToolSchemaProjection.modelCompatibility>[1],
) {
if (tool.type === "tool")
return yield* lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, compatibility))
// OpenAI requires a namespace description; fall back to a generic one so a
// missing description never blocks the request.
return {
type: "namespace" as const,
name: tool.name,
description: tool.description ?? `Tools in the ${tool.name} namespace.`,
tools: yield* Effect.forEach(ProviderShared.flattenTools(tool.tools), (leaf) =>
OpenResponses.lowerTool(NAME, leaf, ToolSchemaProjection.modelCompatibility(leaf.inputSchema, compatibility)),
),
}
})
const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>, tools: ReadonlyArray<ToolEntry>) =>
ProviderShared.matchToolChoice(NAME, toolChoice, {
auto: () => "auto" as const,
none: () => "none" as const,
required: () => "required" as const,
tool: (name) =>
tools.some((tool) => tool.name === name && nativeImageTool(tool) !== undefined)
tools.some((tool) => tool.type === "tool" && tool.name === name && nativeImageTool(tool) !== undefined)
? ({ type: "image_generation" } as const)
: { type: "function" as const, name },
})
@@ -153,9 +184,7 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request:
tools:
request.tools.length === 0
? undefined
: yield* Effect.forEach(request.tools, (tool) =>
lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility)),
),
: yield* Effect.forEach(request.tools, (tool) => lowerToolEntry(tool, toolSchemaCompatibility)),
tool_choice:
OpenResponses.allowedToolChoice(request) ??
(request.toolChoice ? yield* lowerToolChoice(request.toolChoice, request.tools) : undefined),
+37 -1
View File
@@ -9,11 +9,14 @@ import {
UnsupportedOperationError,
AIError,
HttpContext,
LLMRequest,
Message,
ToolDefinition,
type ContentPart,
type LLMRequest,
type MediaPart,
type ProviderID,
type TextPart,
type ToolEntry,
type ToolResultPart,
} from "../schema/index.js"
import { isRecord } from "../utils/record.js"
@@ -46,6 +49,7 @@ export const promptCacheKey = (request: LLMRequest): string | undefined => {
export interface ToolAccumulator {
readonly id: string
readonly name: string
readonly namespace?: string
readonly input: string
}
@@ -279,6 +283,38 @@ export const unsupportedOperation = (input: {
}),
})
/**
* Lower namespaces to flat definitions for protocols without a native
* namespace construct. Leaf names join their namespace path with `_` because
* `.` is not broadly accepted in provider tool names.
*/
export const flattenTools = (tools: ReadonlyArray<ToolEntry>, path: ReadonlyArray<string> = []) => {
const flat = tools.flatMap((tool): ReadonlyArray<ToolDefinition> => {
if (tool.type === "namespace") return flattenTools(tool.tools, [...path, tool.name])
if (path.length === 0) return [tool]
return [new ToolDefinition({ ...tool, name: [...path, tool.name].join("_") })]
})
return [...new Map(flat.map((tool) => [tool.name, tool])).values()]
}
export const flattenToolRequest = (request: LLMRequest) => {
const messages = request.messages.map((message) => {
const content = message.content.map((part) => {
if ((part.type !== "tool-call" && part.type !== "tool-result") || part.namespace === undefined) return part
return { ...part, name: `${part.namespace}_${part.name}`, namespace: undefined }
})
return content.every((part, index) => part === message.content[index])
? message
: new Message({ ...message, content })
})
return {
tools: flattenTools(request.tools),
request: messages.every((message, index) => message === request.messages[index])
? request
: LLMRequest.update(request, { messages }),
}
}
export const imageResponse = Effect.fn("ProviderShared.imageResponse")(function* (
route: string,
name: string,
+10 -1
View File
@@ -55,6 +55,7 @@ const inputStart = (tool: PendingTool) =>
LLMEvent.toolInputStart({
id: tool.id,
name: tool.name,
namespace: tool.namespace,
providerExecuted: tool.providerExecuted ? true : undefined,
providerMetadata: tool.providerMetadata,
})
@@ -63,6 +64,7 @@ const inputDelta = (tool: PendingTool, text: string) =>
LLMEvent.toolInputDelta({
id: tool.id,
name: tool.name,
namespace: tool.namespace,
text,
input: Option.getOrElse(parsePartialInput(tool.input), () => ({})),
})
@@ -85,6 +87,7 @@ const toolCall = (route: string, tool: PendingTool, inputOverride?: string) => {
LLMEvent.toolCall({
id: tool.id,
name: tool.name,
namespace: tool.namespace,
input,
providerExecuted: tool.providerExecuted ? true : undefined,
providerMetadata: tool.providerMetadata,
@@ -94,7 +97,12 @@ const toolCall = (route: string, tool: PendingTool, inputOverride?: string) => {
}
const finishEvents = (tool: PendingTool, event: ToolCall): ReadonlyArray<LLMEvent> => [
LLMEvent.toolInputEnd({ id: tool.id, name: tool.name, providerMetadata: tool.providerMetadata }),
LLMEvent.toolInputEnd({
id: tool.id,
name: tool.name,
namespace: tool.namespace,
providerMetadata: tool.providerMetadata,
}),
event,
]
@@ -150,6 +158,7 @@ export const appendOrStart = <K extends StreamKey>(
const tool = {
id,
name,
namespace: current?.namespace,
input: `${current?.input ?? ""}${delta.text}`,
providerExecuted: current?.providerExecuted,
providerMetadata: current?.providerMetadata,
+7 -3
View File
@@ -487,10 +487,14 @@ export function make<Body, Prepared, Frame, Event, State>(
}
const prepareRequest = (request: LLMRequest) => {
const original = applyCachePolicy(resolveRequestOptions(request))
const original = resolveRequestOptions(request)
const sanitized = LLMRequest.update(original, sanitizeSurrogates({ ...LLMRequest.input(original), model: undefined }))
const tools = [...new Map(sanitized.tools.map((tool) => [tool.name, tool])).values()]
const resolved = tools.length === sanitized.tools.length ? sanitized : LLMRequest.update(sanitized, { tools })
// Deduplicate per sibling level; a tool and a namespace may share a name.
const dedupe = (tools: LLMRequest["tools"]): LLMRequest["tools"] =>
[...new Map(tools.map((tool) => [`${tool.type}:${tool.name}`, tool])).values()].map((tool) =>
tool.type === "tool" ? tool : { ...tool, tools: dedupe(tool.tools) },
)
const resolved = applyCachePolicy(LLMRequest.update(sanitized, { tools: dedupe(sanitized.tools) }))
const headers = resolved.model.route.headers?.({ request: resolved })
return headers === undefined
? resolved
+19 -3
View File
@@ -155,6 +155,7 @@ export const ToolInputStart = Schema.Struct({
type: Schema.tag("tool-input-start"),
id: ToolCallID,
name: Schema.String,
namespace: Schema.optional(Schema.String),
providerExecuted: Schema.optional(Schema.Boolean),
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Event.ToolInputStart" })
@@ -164,6 +165,7 @@ export const ToolInputDelta = Schema.Struct({
type: Schema.tag("tool-input-delta"),
id: ToolCallID,
name: Schema.String,
namespace: Schema.optional(Schema.String),
text: Schema.String,
/** Best-effort parse of all input fragments received through this delta. */
input: Schema.optional(Schema.Unknown),
@@ -174,6 +176,7 @@ export const ToolInputEnd = Schema.Struct({
type: Schema.tag("tool-input-end"),
id: ToolCallID,
name: Schema.String,
namespace: Schema.optional(Schema.String),
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Event.ToolInputEnd" })
export type ToolInputEnd = Schema.Schema.Type<typeof ToolInputEnd>
@@ -183,6 +186,7 @@ export const ToolInputError = Schema.Struct({
type: Schema.tag("tool-input-error"),
id: ToolCallID,
name: Schema.String,
namespace: Schema.optional(Schema.String),
raw: Schema.String,
}).annotate({ identifier: "LLM.Event.ToolInputError" })
export type ToolInputError = Schema.Schema.Type<typeof ToolInputError>
@@ -191,6 +195,7 @@ export const ToolCall = Schema.Struct({
type: Schema.tag("tool-call"),
id: ToolCallID,
name: Schema.String,
namespace: Schema.optional(Schema.String),
input: Schema.Unknown,
providerExecuted: Schema.optional(Schema.Boolean),
providerMetadata: Schema.optional(ProviderMetadata),
@@ -201,6 +206,7 @@ export const ToolResult = Schema.Struct({
type: Schema.tag("tool-result"),
id: ToolCallID,
name: Schema.String,
namespace: Schema.optional(Schema.String),
result: ToolResultValue,
output: Schema.optional(ToolOutput),
providerExecuted: Schema.optional(Schema.Boolean),
@@ -212,6 +218,7 @@ export const ToolError = Schema.Struct({
type: Schema.tag("tool-error"),
id: ToolCallID,
name: Schema.String,
namespace: Schema.optional(Schema.String),
message: Schema.String,
error: Schema.optional(Schema.Defect()),
providerMetadata: Schema.optional(ProviderMetadata),
@@ -385,6 +392,7 @@ interface ContentAssembly {
interface ToolInputAssembly {
readonly name: string
readonly namespace?: string
readonly text: string
readonly providerMetadata?: ProviderMetadata
}
@@ -522,12 +530,17 @@ const reduceToolInputStart = (state: ResponseState, event: ToolInputStart): Resp
...state,
toolInputs: {
...state.toolInputs,
[event.id]: { name: event.name, text: "", providerMetadata: event.providerMetadata },
[event.id]: {
name: event.name,
namespace: event.namespace,
text: "",
providerMetadata: event.providerMetadata,
},
},
})
const reduceToolInputDelta = (state: ResponseState, event: ToolInputDelta): ResponseState => {
const current = state.toolInputs[event.id] ?? { name: event.name, text: "" }
const current = state.toolInputs[event.id] ?? { name: event.name, namespace: event.namespace, text: "" }
return {
...state,
toolInputs: { ...state.toolInputs, [event.id]: { ...current, text: current.text + event.text } },
@@ -535,7 +548,7 @@ const reduceToolInputDelta = (state: ResponseState, event: ToolInputDelta): Resp
}
const reduceToolInputEnd = (state: ResponseState, event: ToolInputEnd): ResponseState => {
const current = state.toolInputs[event.id] ?? { name: event.name, text: "" }
const current = state.toolInputs[event.id] ?? { name: event.name, namespace: event.namespace, text: "" }
return {
...state,
toolInputs: {
@@ -543,6 +556,7 @@ const reduceToolInputEnd = (state: ResponseState, event: ToolInputEnd): Response
[event.id]: {
...current,
name: event.name,
namespace: event.namespace,
providerMetadata: event.providerMetadata ?? current.providerMetadata,
},
},
@@ -553,6 +567,7 @@ const toolCallContent = (event: ToolCall): ContentPart =>
ToolCallPart.make({
id: event.id,
name: event.name,
namespace: event.namespace,
input: event.input,
...(event.providerExecuted === undefined ? {} : { providerExecuted: event.providerExecuted }),
...(event.providerMetadata === undefined ? {} : { providerMetadata: event.providerMetadata }),
@@ -562,6 +577,7 @@ const toolResultContent = (event: ToolResult): ContentPart =>
ToolResultPart.make({
id: event.id,
name: event.name,
namespace: event.namespace,
result: event.result,
...(event.providerExecuted === undefined ? {} : { providerExecuted: event.providerExecuted }),
...(event.providerMetadata === undefined ? {} : { providerMetadata: event.providerMetadata }),
+63 -4
View File
@@ -135,6 +135,7 @@ export const ToolCallPart = Object.assign(
type: Schema.Literal("tool-call"),
id: Schema.String,
name: Schema.String,
namespace: Schema.optional(Schema.String),
input: Schema.Unknown,
providerExecuted: Schema.optional(Schema.Boolean),
cache: Schema.optional(CacheHint),
@@ -152,6 +153,7 @@ export const ToolResultPart = Object.assign(
type: Schema.Literal("tool-result"),
id: Schema.String,
name: Schema.String,
namespace: Schema.optional(Schema.String),
result: ToolResultValue,
providerExecuted: Schema.optional(Schema.Boolean),
cache: Schema.optional(CacheHint),
@@ -168,6 +170,7 @@ export const ToolResultPart = Object.assign(
type: "tool-result",
id: input.id,
name: input.name,
namespace: input.namespace,
result: ToolResultValue.make(input.result, input.resultType),
providerExecuted: input.providerExecuted,
cache: input.cache,
@@ -266,7 +269,7 @@ export namespace Message {
make({ role: "tool", content: ["type" in result ? result : ToolResultPart.make(result)] })
}
export class ToolDefinition extends Schema.Class<ToolDefinition>("LLM.ToolDefinition")({
const toolDefinitionFields = {
name: Schema.String,
description: Schema.String,
inputSchema: JsonSchema,
@@ -274,15 +277,71 @@ export class ToolDefinition extends Schema.Class<ToolDefinition>("LLM.ToolDefini
cache: Schema.optional(CacheHint),
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
native: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}) {}
}
export type ToolDefinitionInput = Schema.Struct.Type<typeof toolDefinitionFields>
export class ToolDefinition extends Schema.Class<ToolDefinition>("LLM.ToolDefinition")({
type: Schema.Literal("tool"),
...toolDefinitionFields,
}) {
constructor(input: ToolDefinitionInput) {
super({ ...input, type: "tool" })
}
}
export namespace ToolDefinition {
export type Input = ToolDefinition | ConstructorParameters<typeof ToolDefinition>[0]
export type Input = ToolDefinition | ToolDefinitionInput
/** Normalize tool definition input into the canonical `ToolDefinition` class. */
export const make = (input: Input) => (input instanceof ToolDefinition ? input : new ToolDefinition(input))
}
export type ToolNamespace = {
readonly type: "namespace"
readonly name: string
readonly description?: string
readonly tools: ReadonlyArray<ToolEntry>
}
export type ToolNamespaceInput = Omit<ToolNamespace, "type" | "tools"> & {
readonly tools: ReadonlyArray<ToolEntryInput>
}
export type ToolNamespaceEntryInput = ToolNamespaceInput & { readonly type: "namespace" }
export const ToolNamespace: Schema.Codec<ToolNamespace> & {
readonly make: (input: ToolNamespace | ToolNamespaceInput) => ToolNamespace
} = Object.assign(
Schema.Struct({
type: Schema.Literal("namespace"),
name: Schema.String,
description: Schema.optional(Schema.UndefinedOr(Schema.String)),
tools: Schema.Array(Schema.suspend((): Schema.Codec<ToolEntry> => ToolEntry)),
}).annotate({ identifier: "LLM.ToolNamespace" }),
{
make: (input: ToolNamespace | ToolNamespaceInput): ToolNamespace => ({
...input,
type: "namespace",
tools: input.tools.map(ToolEntry.make),
}),
},
)
export type ToolEntry = ToolDefinition | ToolNamespace
export type ToolEntryInput = ToolDefinition.Input | ToolNamespaceEntryInput
export const ToolEntry: Schema.Codec<ToolEntry> & {
readonly make: (input: ToolEntryInput) => ToolEntry
} = Object.assign(
Schema.Union([ToolDefinition, ToolNamespace]).pipe(
Schema.toTaggedUnion("type"),
Schema.annotate({ identifier: "LLM.ToolEntry" }),
),
{
make: (input: ToolEntryInput): ToolEntry =>
"type" in input && input.type === "namespace" ? ToolNamespace.make(input) : ToolDefinition.make(input),
},
)
export class ToolChoice extends Schema.Class<ToolChoice>("LLM.ToolChoice")({
type: Schema.Literals(["auto", "none", "required", "tool"]),
name: Schema.optional(Schema.String),
@@ -312,7 +371,7 @@ const requestSchema = Schema.Struct({
model: LanguageModelSchema,
system: Schema.Array(SystemPart),
messages: Schema.Array(Message),
tools: Schema.Array(ToolDefinition),
tools: Schema.Array(ToolEntry),
toolChoice: Schema.optional(ToolChoice),
generation: Schema.optional(GenerationOptions),
providerOptions: Schema.optional(ProviderOptions),
+13 -4
View File
@@ -37,7 +37,13 @@ function missingToolResults(calls: Iterable<ToolCallPart>) {
return new Message({
role: "tool",
content: [...calls].map((call) =>
ToolResultPart.make({ id: call.id, name: call.name, result: MISSING_TOOL_RESULT, resultType: "error" }),
ToolResultPart.make({
id: call.id,
name: call.name,
namespace: call.namespace,
result: MISSING_TOOL_RESULT,
resultType: "error",
}),
),
})
}
@@ -47,7 +53,7 @@ function normalizeToolMessage(message: Message, pending: Map<string, ToolCallPar
if (part.type !== "tool-result" || part.providerExecuted === true) return part
const call = pending.get(part.id)
if (call) pending.delete(part.id)
return normalizeToolResult(part, call?.name ?? part.name)
return normalizeToolResult(part, call)
})
if (content.length === 0) return undefined
if (content.every((part, index) => part === message.content[index])) return message
@@ -61,8 +67,11 @@ function normalizeToolMessage(message: Message, pending: Map<string, ToolCallPar
})
}
function normalizeToolResult(part: ToolResultPart, name: string): ToolResultPart {
const named = part.name === name ? part : { ...part, name }
function normalizeToolResult(part: ToolResultPart, call: ToolCallPart | undefined): ToolResultPart {
const named =
call === undefined || (part.name === call.name && part.namespace === call.namespace)
? part
: { ...part, name: call.name, namespace: call.namespace }
if (named.result.type === "text" && named.result.value === "")
return { ...named, result: { type: "text", value: EMPTY_TOOL_OUTPUT } }
if (named.result.type === "error" && named.result.value === "")
+12 -4
View File
@@ -21,10 +21,11 @@ export interface DispatchResult extends ToolSettlement {
/** Execute one canonical tool call without owning provider IO or continuation. */
export const dispatch = (tools: Tools, call: ToolCallPart): Effect.Effect<DispatchResult> => {
const tool = tools[call.name]
if (!tool) return Effect.succeed(result(call, { type: "error", value: `Unknown tool: ${call.name}` }))
const name = call.namespace === undefined ? call.name : `${call.namespace}.${call.name}`
const tool = tools[name]
if (!tool) return Effect.succeed(result(call, { type: "error", value: `Unknown tool: ${name}` }))
if (!tool.execute)
return Effect.succeed(result(call, { type: "error", value: `Tool has no execute handler: ${call.name}` }))
return Effect.succeed(result(call, { type: "error", value: `Tool has no execute handler: ${name}` }))
return decodeAndExecute(tool, call).pipe(
Effect.map((value) => result(call, value)),
@@ -38,7 +39,11 @@ const decodeAndExecute = (tool: AnyTool, call: ToolCallPart): Effect.Effect<Tool
tool._decode(call.input).pipe(
Effect.mapError((error) => new ToolFailure({ message: `Invalid tool input: ${error.message}` })),
Effect.flatMap((decoded) =>
tool.execute!(decoded, { id: call.id, name: call.name }).pipe(
tool.execute!(decoded, {
id: call.id,
name: call.name,
namespace: call.namespace,
}).pipe(
Effect.flatMap((value) =>
tool._encode(value).pipe(
Effect.mapError(
@@ -71,6 +76,7 @@ const result = (call: ToolCallPart, value: ToolResultValueType | ToolSettlement,
LLMEvent.toolError({
id: call.id,
name: call.name,
namespace: call.namespace,
message: String(settlement.result.value),
error,
providerMetadata: call.providerMetadata,
@@ -78,6 +84,7 @@ const result = (call: ToolCallPart, value: ToolResultValueType | ToolSettlement,
LLMEvent.toolResult({
id: call.id,
name: call.name,
namespace: call.namespace,
result: settlement.result,
providerMetadata: call.providerMetadata,
}),
@@ -86,6 +93,7 @@ const result = (call: ToolCallPart, value: ToolResultValueType | ToolSettlement,
LLMEvent.toolResult({
id: call.id,
name: call.name,
namespace: call.namespace,
result: settlement.result,
output: settlement.output,
providerMetadata: call.providerMetadata,
+1
View File
@@ -16,6 +16,7 @@ export type ToolSchema<T> = Schema.Codec<T, any, never, never>
export interface ToolExecuteContext {
readonly id: ToolCallPart["id"]
readonly name: ToolCallPart["name"]
readonly namespace?: ToolCallPart["namespace"]
}
export type ToolExecute<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>> = (
+53
View File
@@ -215,6 +215,35 @@ describe("applyCachePolicy", () => {
}),
)
it.effect("deduplicates tools before counting cache hints", () =>
Effect.gen(function* () {
const manual = new CacheHint({ type: "ephemeral" })
const duplicate = (description: string) => ({
name: "lookup",
description,
inputSchema: { type: "object" },
cache: manual,
})
const prepared = yield* compileRequest(
LLM.request({
model: anthropicModel,
tools: [
duplicate("first"),
duplicate("second"),
duplicate("third"),
duplicate("fourth"),
{ name: "lookup", description: "final", inputSchema: { type: "object" } },
],
cache: { tools: true },
}),
)
expect(prepared.body.tools).toEqual([
expect.objectContaining({ name: "lookup", description: "final", cache_control: { type: "ephemeral" } }),
])
}),
)
it.effect("auto policy preserves manual CacheHints on other parts", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
@@ -281,6 +310,30 @@ describe("applyCachePolicy", () => {
}),
)
test("marks the final leaf inside a tool namespace", () => {
const request = LLM.request({
model: anthropicModel,
tools: [
{
type: "namespace",
name: "crm",
tools: [
{ name: "lookup", description: "lookup", inputSchema: {} },
{ name: "orders", description: "orders", inputSchema: {} },
],
},
],
cache: { tools: true },
})
const applied = applyCachePolicy(request)
const namespace = applied.tools[0]
expect(namespace?.type).toBe("namespace")
if (namespace?.type !== "namespace") throw new Error("Expected namespace")
expect(namespace.tools[0]).not.toHaveProperty("cache")
expect(namespace.tools[1]).toHaveProperty("cache", { type: "ephemeral" })
})
it.effect("ttlSeconds in the policy flows through to wire markers", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
+62 -2
View File
@@ -1,8 +1,16 @@
import { describe, expect, test } from "bun:test"
import { Effect, Ref, Schema } from "effect"
import { HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
import { LLM, LLMRequest, Message, ToolCallPart, ToolDefinition, mergeProviderOptions } from "../src/index.js"
import { AnthropicMessages, OpenAIChat } from "../src/protocols.js"
import {
LLM,
LLMRequest,
Message,
ToolCallPart,
ToolDefinition,
ToolNamespace,
mergeProviderOptions,
} from "../src/index.js"
import { AnthropicMessages, OpenAIChat, OpenAIResponses } from "../src/protocols.js"
import { Auth, LLMClient } from "../src/route.js"
import { compileRequest } from "../src/route/client.js"
import { it } from "./lib/effect.js"
@@ -106,6 +114,58 @@ describe("request option precedence", () => {
}),
)
it.effect("deduplicates tools within each namespace", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model: OpenAIResponses.route.model({ id: "gpt-5.4" }),
tools: [
ToolDefinition.make({ name: "crm", description: "Top-level CRM tool", inputSchema: {} }),
ToolNamespace.make({
name: "crm",
description: "CRM tools",
tools: [
ToolDefinition.make({ name: "lookup", description: "old", inputSchema: {} }),
ToolDefinition.make({ name: "search", description: "search", inputSchema: {} }),
ToolDefinition.make({ name: "lookup", description: "new", inputSchema: {} }),
],
}),
ToolNamespace.make({
name: "support",
description: "Support tools",
tools: [ToolDefinition.make({ name: "lookup", description: "support", inputSchema: {} })],
}),
],
}),
)
expect(prepared.body.tools).toEqual([
{
type: "function",
name: "crm",
description: "Top-level CRM tool",
parameters: {},
strict: false,
},
{
type: "namespace",
name: "crm",
description: "CRM tools",
tools: [
{ type: "function", name: "lookup", description: "new", parameters: {}, strict: false },
{ type: "function", name: "search", description: "search", parameters: {}, strict: false },
],
},
{
type: "namespace",
name: "support",
description: "Support tools",
tools: [{ type: "function", name: "lookup", description: "support", parameters: {}, strict: false }],
},
])
}),
)
it.effect("normalizes tool history before protocol lowering", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
+48 -1
View File
@@ -1,5 +1,6 @@
import { describe, expect, test } from "bun:test"
import { CacheHint, LLM, LLMResponse } from "../src/index.js"
import { Schema } from "effect"
import { CacheHint, LLM, LLMResponse, ToolEntry, ToolNamespace } from "../src/index.js"
import * as OpenAIChat from "../src/protocols/openai-chat.js"
import * as OpenAIResponses from "../src/protocols/openai-responses.js"
import {
@@ -17,6 +18,52 @@ const chatRoute = OpenAIChat.route
const responsesRoute = OpenAIResponses.route
describe("llm constructors", () => {
test("normalizes recursive tool namespaces", () => {
const request = LLM.request({
model: LanguageModel.make({ id: "fake-model", provider: "fake", route: responsesRoute }),
tools: [
{
type: "namespace",
name: "crm",
description: "Customer management",
tools: [
{ name: "lookup", description: "Look up a customer", inputSchema: { type: "object" } },
{
type: "namespace",
name: "orders",
tools: [{ name: "list", description: "List orders", inputSchema: { type: "object" } }],
},
],
},
],
})
expect(request.tools[0]).toEqual({
type: "namespace",
name: "crm",
description: "Customer management",
tools: [
expect.objectContaining({ type: "tool", name: "lookup" }),
{
type: "namespace",
name: "orders",
description: undefined,
tools: [expect.objectContaining({ type: "tool", name: "list" })],
},
],
})
expect(request.tools[0]).toEqual(
ToolNamespace.make({
name: "crm",
description: "Customer management",
tools: request.tools[0]!.type === "namespace" ? request.tools[0].tools : [],
}),
)
expect(Schema.decodeUnknownSync(ToolEntry)(Schema.encodeUnknownSync(ToolEntry)(request.tools[0]))).toEqual(
request.tools[0],
)
})
test("builds canonical schema classes from ergonomic input", () => {
const request = LLM.request({
id: "req_1",
@@ -1,6 +1,6 @@
import { expect } from "bun:test"
import { Effect, Schema } from "effect"
import { LLM, LLMRequest, Message } from "../../src/index.js"
import { LLM, LLMRequest, Message, ToolDefinition } from "../../src/index.js"
import { LLMClient, Route } from "../../src/route/client.js"
import { Auth } from "../../src/route/auth.js"
import { Endpoint } from "../../src/route/endpoint.js"
@@ -134,7 +134,12 @@ for (const model of [
[
LLMRequest.update(request, {
tools: [
{ name: "unsupported", description: "Generation only", inputSchema: {}, native: { unsupported: {} } },
ToolDefinition.make({
name: "unsupported",
description: "Generation only",
inputSchema: {},
native: { unsupported: {} },
}),
],
}),
"InvalidRequest",
@@ -326,7 +326,6 @@ describe("Open Responses basic-item lifecycles", () => {
])
}),
)
it.effect("mints an id for a done-only tool that never had one", () =>
Effect.gen(function* () {
const events = yield* collect(
@@ -357,9 +356,16 @@ describe("Open Responses basic-item lifecycles", () => {
const events = yield* collect({ type: "response.output_item.done", item }, completed)
const providerMetadata = { "openai-compatible": { itemId: "fc_1" } }
expect(events.filter((event) => event.type.startsWith("tool-"))).toEqual([
{ type: "tool-input-start", id: "call_1", name: "lookup", providerMetadata },
{ type: "tool-input-end", id: "call_1", name: "lookup", providerMetadata },
{ type: "tool-call", id: "call_1", name: "lookup", input: { query: "weather" }, providerMetadata },
{ type: "tool-input-start", id: "call_1", name: "lookup", namespace: undefined, providerMetadata },
{ type: "tool-input-end", id: "call_1", name: "lookup", namespace: undefined, providerMetadata },
{
type: "tool-call",
id: "call_1",
name: "lookup",
namespace: undefined,
input: { query: "weather" },
providerMetadata,
},
])
expect(events.filter(LLMEvent.is.finish)).toEqual([
{
@@ -171,6 +171,66 @@ describe("Open Responses-compatible route", () => {
}),
)
it.effect("flattens tool namespaces", () =>
Effect.gen(function* () {
const model = configure({ apiKey: "test-key", baseURL: "https://responses.example.test/v1" }).model(
"example-model",
)
const prepared = yield* compileRequest(
LLM.request({
model,
tools: [
{
type: "namespace",
name: "acme",
tools: [
{
type: "namespace",
name: "billing",
tools: [ToolDefinition.make({ name: "lookup", description: "Lookup billing", inputSchema: {} })],
},
ToolDefinition.make({ name: "users", description: "Lookup users", inputSchema: {} }),
],
},
],
}),
)
expect(prepared.body.tools).toEqual([
{
type: "function",
name: "acme_billing_lookup",
description: "Lookup billing",
parameters: {},
strict: false,
},
{ type: "function", name: "acme_users", description: "Lookup users", parameters: {}, strict: false },
])
}),
)
it.effect("flattens tool namespaces in history", () =>
Effect.gen(function* () {
const model = configure({ apiKey: "test-key", baseURL: "https://responses.example.test/v1" }).model(
"example-model",
)
const prepared = yield* compileRequest(
LLM.request({
model,
messages: [
Message.assistant({ type: "tool-call", id: "call_1", name: "lookup", namespace: "crm", input: {} }),
Message.tool({ id: "call_1", name: "lookup", namespace: "crm", result: "done", resultType: "text" }),
],
}),
)
expect(prepared.body.input).toEqual([
{ type: "function_call", call_id: "call_1", name: "crm_lookup", namespace: undefined, arguments: "{}" },
{ type: "function_call_output", call_id: "call_1", output: "done" },
])
}),
)
it.effect("lowers canonical parallel tool control", () =>
Effect.gen(function* () {
const model = configure({
@@ -12,6 +12,7 @@ import {
LanguageModel,
ToolCallPart,
ToolDefinition,
ToolNamespace,
ToolResultPart,
TransportError,
Usage,
@@ -143,6 +144,94 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("lowers tool namespaces without flattening leaf names", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model,
prompt: "Find a customer and their orders.",
tools: [
ToolNamespace.make({
name: "crm",
description: "Customer management",
tools: [
ToolDefinition.make({ name: "lookup", description: "Look up a customer", inputSchema: {} }),
ToolDefinition.make({ name: "orders", description: "List customer orders", inputSchema: {} }),
],
}),
],
}),
)
expect(prepared.body.tools).toEqual([
{
type: "namespace",
name: "crm",
description: "Customer management",
tools: [
{ type: "function", name: "lookup", description: "Look up a customer", parameters: {}, strict: false },
{ type: "function", name: "orders", description: "List customer orders", parameters: {}, strict: false },
],
},
])
}),
)
it.effect("flattens nested levels within a native tool namespace", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model,
tools: [
{
type: "namespace",
name: "crm",
description: "Customer management",
tools: [
{
type: "namespace",
name: "orders",
description: "Order management",
tools: [ToolDefinition.make({ name: "list", description: "List orders", inputSchema: {} })],
},
],
},
],
}),
)
expect(prepared.body.tools).toEqual([
{
type: "namespace",
name: "crm",
description: "Customer management",
tools: [{ type: "function", name: "orders_list", description: "List orders", parameters: {}, strict: false }],
},
])
}),
)
it.effect("defaults tool namespace descriptions", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model,
tools: [
{
type: "namespace",
name: "crm",
tools: [ToolDefinition.make({ name: "lookup", description: "Look up a customer", inputSchema: {} })],
},
],
}),
)
expect(prepared.body.tools).toEqual([
expect.objectContaining({ type: "namespace", name: "crm", description: "Tools in the crm namespace." }),
])
}),
)
it.effect("rejects invalid hosted image generation options locally", () =>
Effect.gen(function* () {
const error = yield* compileRequest(
@@ -2130,6 +2219,71 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("preserves tool namespaces through streaming and history replay", () =>
Effect.gen(function* () {
const item = {
type: "function_call",
id: "fc_1",
call_id: "call_1",
namespace: "crm",
name: "lookup",
arguments: "",
}
const response = yield* LLMClient.generate(request).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{ type: "response.output_item.added", output_index: 0, item },
{
type: "response.function_call_arguments.delta",
output_index: 0,
item_id: "fc_1",
delta: '{"id":"123"}',
},
{
type: "response.output_item.done",
output_index: 0,
item: { ...item, arguments: '{"id":"123"}' },
},
{ type: "response.completed", response: { id: "resp_1" } },
),
),
),
)
const toolEvents = response.events.filter((event) => event.type.startsWith("tool-"))
expect(toolEvents).toEqual([
expect.objectContaining({ type: "tool-input-start", name: "lookup", namespace: "crm" }),
expect.objectContaining({ type: "tool-input-delta", name: "lookup", namespace: "crm" }),
expect.objectContaining({ type: "tool-input-end", name: "lookup", namespace: "crm" }),
expect.objectContaining({ type: "tool-call", name: "lookup", namespace: "crm", input: { id: "123" } }),
])
expect(response.message.content).toEqual([
expect.objectContaining({ type: "tool-call", name: "lookup", namespace: "crm", input: { id: "123" } }),
])
const prepared = yield* compileRequest(
LLM.request({
model,
messages: [
response.message,
Message.tool({ id: "call_1", name: "lookup", namespace: "crm", result: { customer: "Ada" } }),
],
}),
)
expect(prepared.body.input).toEqual([
{
type: "function_call",
id: "fc_1",
call_id: "call_1",
namespace: "crm",
name: "lookup",
arguments: '{"id":"123"}',
},
{ type: "function_call_output", call_id: "call_1", output: '{"customer":"Ada"}' },
])
}),
)
it.effect("routes reasoning summary events by output index", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(request).pipe(
+10 -7
View File
@@ -15,13 +15,7 @@ describe("tool history normalization", () => {
Message.assistant(toolCall("trailing")),
])
expect(normalized.map((message) => message.role)).toEqual([
"assistant",
"tool",
"tool",
"user",
"assistant",
])
expect(normalized.map((message) => message.role)).toEqual(["assistant", "tool", "tool", "user", "assistant"])
expect(normalized[1]?.content[0]).toMatchObject({ type: "tool-result", id: "first", name: "first" })
expect(normalized[2]?.content).toEqual([
{ type: "tool-result", id: "second", name: "second", result: { type: "error", value: "Tool result missing" } },
@@ -74,4 +68,13 @@ describe("tool history normalization", () => {
expect(normalizeToolHistory([orphan, hosted])).toEqual([orphan, hosted])
})
test("uses a matching call as the complete tool identity", () => {
const normalized = normalizeToolHistory([
Message.assistant(ToolCallPart.make({ id: "call_1", name: "lookup", input: {} })),
Message.tool(ToolResultPart.make({ id: "call_1", name: "wrong", namespace: "stale", result: "done" })),
])
expect(normalized[1]?.content[0]).toMatchObject({ name: "lookup", namespace: undefined })
})
})
+22
View File
@@ -7,6 +7,7 @@ import {
LLMEvent,
LLMRequest,
LLMResponse,
ToolCallPart,
ToolChoice,
ToolOutput,
toDefinitions,
@@ -36,6 +37,27 @@ const baseRequest = LLM.request({
})
const weatherFailureCause = new Error("weather lookup denied")
test("dispatches namespaced calls by qualified identity", async () => {
let context: ToolExecuteContext | undefined
const lookup = Tool.make({
description: "Look up a customer.",
parameters: Schema.Struct({}),
success: Schema.String,
execute: (_, value) => {
context = value
return Effect.succeed("customer")
},
})
const call = ToolCallPart.make({ id: "call_1", namespace: "crm", name: "lookup", input: {} })
const result = await Effect.runPromise(
ToolRuntime.dispatch({ "crm.lookup": lookup, lookup: schema_only_weather }, call),
)
expect(result.result).toEqual({ type: "text", value: "customer" })
expect(context).toEqual({ id: "call_1", namespace: "crm", name: "lookup" })
expect(result.events).toEqual([expect.objectContaining({ type: "tool-result", namespace: "crm", name: "lookup" })])
})
const get_weather = Tool.make({
description: "Get current weather for a city.",
parameters: Schema.Struct({ city: Schema.String }),
@@ -1109,6 +1109,8 @@ export async function handler(
authInfo = authInfo!
const cost = centsToMicroCents(totalCostInCent)
// Keep period bounds and persisted timestamps on one snapshot when a queued write crosses a reset boundary.
const trackedAt = new Date()
// For hot workspaces, batch balance/usage updates through Redis to avoid
// row-level lock contention on BillingTable/UserTable. Returns the amount
@@ -1149,7 +1151,7 @@ export async function handler(
if (billingSource === "subscription") {
const plan = authInfo.billing.subscription!.plan
const black = BlackData.getLimits({ plan })
const week = getWeekBounds(new Date())
const week = getWeekBounds(trackedAt)
const rollingWindowSeconds = black.rollingWindow * 3600
return [
db
@@ -1157,11 +1159,17 @@ export async function handler(
.set({
fixedUsage: sql`
CASE
WHEN ${SubscriptionTable.timeFixedUpdated} >= ${week.end} THEN ${SubscriptionTable.fixedUsage}
WHEN ${SubscriptionTable.timeFixedUpdated} >= ${week.start} THEN ${SubscriptionTable.fixedUsage} + ${cost}
ELSE ${cost}
END
`,
timeFixedUpdated: sql`now()`,
timeFixedUpdated: sql`
CASE
WHEN ${SubscriptionTable.timeFixedUpdated} > ${trackedAt} THEN ${SubscriptionTable.timeFixedUpdated}
ELSE ${trackedAt}
END
`,
rollingUsage: sql`
CASE
WHEN UNIX_TIMESTAMP(${SubscriptionTable.timeRollingUpdated}) >= UNIX_TIMESTAMP(now()) - ${rollingWindowSeconds} THEN ${SubscriptionTable.rollingUsage} + ${cost}
@@ -1185,8 +1193,8 @@ export async function handler(
}
if (billingSource === "lite") {
const lite = LiteData.getLimits()
const week = getWeekBounds(new Date())
const month = getMonthlyBounds(new Date(), authInfo.lite!.timeCreated)
const week = getWeekBounds(trackedAt)
const month = getMonthlyBounds(trackedAt, authInfo.lite!.timeCreated)
const rollingWindowSeconds = lite.rollingWindow * 3600
const quotaCost = Math.round(cost * modelInfo.costMultiplier)
return [
@@ -1195,18 +1203,30 @@ export async function handler(
.set({
monthlyUsage: sql`
CASE
WHEN ${LiteTable.timeMonthlyUpdated} >= ${month.end} THEN ${LiteTable.monthlyUsage}
WHEN ${LiteTable.timeMonthlyUpdated} >= ${month.start} THEN ${LiteTable.monthlyUsage} + ${quotaCost}
ELSE ${quotaCost}
END
`,
timeMonthlyUpdated: sql`now()`,
timeMonthlyUpdated: sql`
CASE
WHEN ${LiteTable.timeMonthlyUpdated} > ${trackedAt} THEN ${LiteTable.timeMonthlyUpdated}
ELSE ${trackedAt}
END
`,
weeklyUsage: sql`
CASE
WHEN ${LiteTable.timeWeeklyUpdated} >= ${week.end} THEN ${LiteTable.weeklyUsage}
WHEN ${LiteTable.timeWeeklyUpdated} >= ${week.start} THEN ${LiteTable.weeklyUsage} + ${quotaCost}
ELSE ${quotaCost}
END
`,
timeWeeklyUpdated: sql`now()`,
timeWeeklyUpdated: sql`
CASE
WHEN ${LiteTable.timeWeeklyUpdated} > ${trackedAt} THEN ${LiteTable.timeWeeklyUpdated}
ELSE ${trackedAt}
END
`,
rollingUsage: sql`
CASE
WHEN UNIX_TIMESTAMP(${LiteTable.timeRollingUpdated}) >= UNIX_TIMESTAMP(now()) - ${rollingWindowSeconds} THEN ${LiteTable.rollingUsage} + ${quotaCost}
+3 -2
View File
@@ -416,8 +416,9 @@ function callOptions(
modelID: ID,
optionKey: string,
): LanguageModelV3CallOptions {
const flattened = ProviderShared.flattenToolRequest(request)
return {
prompt: prompt(request),
prompt: prompt(flattened.request),
maxOutputTokens: request.generation?.maxTokens,
temperature: request.generation?.temperature,
stopSequences: request.generation?.stop === undefined ? undefined : [...request.generation.stop],
@@ -426,7 +427,7 @@ function callOptions(
presencePenalty: request.generation?.presencePenalty,
frequencyPenalty: request.generation?.frequencyPenalty,
seed: request.generation?.seed,
tools: request.tools.map(tool),
tools: flattened.tools.map(tool),
toolChoice: toolChoice(request.toolChoice),
headers: request.http?.headers,
providerOptions: requestProviderOptions(request.providerOptions, packageName, modelID, optionKey),
+4 -2
View File
@@ -11,6 +11,7 @@ import {
Message,
type ContentPart,
} from "@opencode-ai/ai"
import { Agent } from "@opencode-ai/schema/agent"
import { SessionError } from "@opencode-ai/schema/session-error"
import { Context, Effect, Layer, Stream } from "effect"
import { Bus } from "../bus.js"
@@ -398,7 +399,8 @@ export const layer = Layer.effect(
kind: "compaction",
scope: {
session: context.session,
agentID: context.agent.id,
agentID: Agent.ID.make("compaction"),
contextAgentID: context.agent.id,
model: context.model,
tools: context.tools,
},
@@ -484,7 +486,7 @@ export const layer = Layer.effect(
const decision = yield* retry({
cause,
error: toSessionError(cause),
agent: context.agent.id,
agent: Agent.ID.make("compaction"),
model: context.model.ref,
hook: prepared.retry,
retry: SessionRunnerRetry.isRetryable(cause),
+18 -34
View File
@@ -1,17 +1,8 @@
export * as SessionModelRequest from "./model-request.js"
import {
GenerationOptions,
type GenerationOptionsFields,
HttpOptions,
LanguageModel,
LLM,
LLMRequest,
Message,
SystemPart,
} from "@opencode-ai/ai"
import { HttpOptions, LanguageModel, LLM, LLMRequest, Message, SystemPart } from "@opencode-ai/ai"
import type { StreamOptions } from "@opencode-ai/ai/route"
import type { SessionRequestKind, SessionRequestOptions } from "@opencode-ai/plugin/effect/session"
import type { SessionRequestKind } from "@opencode-ai/plugin/effect/session"
import type { Agent } from "@opencode-ai/schema/agent"
import type { Model } from "@opencode-ai/schema/model"
import type { Content } from "@opencode-ai/schema/tool"
@@ -35,8 +26,6 @@ const IMAGE_BYTES_TARGET = 15 * 1024 * 1024 // 15 MiB
const IMAGE_REMOVED =
"[This image was removed to reduce the request size and is no longer visible. Do not make claims about its contents from memory. If needed, retrieve it again with an available tool or ask the user to attach it again.]"
const GENERATION_KEYS = new Set(Object.keys(GenerationOptions.fields))
const responsesWebSocketFlag = (providerID: string) =>
`OPENCODE_EXPERIMENTAL_${providerID.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()}_RESPONSES_WEBSOCKET`
@@ -60,10 +49,7 @@ const declineDefect = (cause: Cause.Cause<Tool.Error>) => {
export interface Prepared {
readonly request: LLMRequest
readonly options: StreamOptions
/** Runs retry hooks with this request's kind; the returned event carries the hooked decision. */
readonly retry: (
event: Omit<PluginHooks.Domains["session"]["retry"], "kind">,
) => Effect.Effect<PluginHooks.Domains["session"]["retry"]>
readonly retry: (event: PluginHooks.Domains["session"]["retry"]) => Effect.Effect<void>
/**
* One request-scoped execution operation. Unknown and hook-removed calls
* fail individually through the same seam.
@@ -79,6 +65,8 @@ interface PrepareInput {
readonly scope: {
readonly session: SessionSchema.Info
readonly agentID: Agent.ID
/** Agent whose context an auxiliary request reuses, without changing its request-hook identity. */
readonly contextAgentID?: Agent.ID
readonly model: SessionRunnerModel.Resolved
/** Omitted for requests that carry no tool definitions, such as titles. */
readonly tools?: Tool.Snapshot
@@ -88,6 +76,11 @@ interface PrepareInput {
readonly messages: Array<Message>
}
readonly toolChoice?: LLM.RequestInput["toolChoice"]
/**
* Session context hooks shape the agent conversation. Standalone requests
* such as titles opt out; compaction uses the selected Session context.
*/
readonly contextHooks?: false
/** Stateful Session WebSocket channels require an explicit durable-runner opt-in. */
readonly webSocket?: "session"
}
@@ -312,26 +305,17 @@ export const layer = Layer.effect(
)
// Hooks mutate this record in place: edit descriptions and schemas, rename, or remove.
const definitions = Object.fromEntries(Array.from(given, ([definition, tool]) => [tool.name, definition]))
const draft = {
const context: PluginHooks.Domains["session"]["context"] = {
sessionID: session.id,
agent: input.scope.agentID,
agent: input.scope.contextAgentID ?? input.scope.agentID,
model: resolved.ref,
system: input.transcript.system,
messages: input.transcript.messages,
tools: definitions,
options: {} as SessionRequestOptions,
generation: {},
providerOptions: {},
}
// Titles are not part of the agent conversation and skip context hooks.
const context =
input.kind === "title" ? draft : yield* hooks.trigger("session", "context", { ...draft, kind: input.kind })
// Typed generation keys and provider-semantic keys share one bag in the hook;
// the request keeps them apart.
const generation = Object.fromEntries(
Object.entries(context.options).filter(([key]) => GENERATION_KEYS.has(key)),
) as GenerationOptionsFields
const providerOptions = Object.fromEntries(
Object.entries(context.options).filter(([key]) => !GENERATION_KEYS.has(key)),
)
if (input.contextHooks !== false) yield* hooks.trigger("session", "context", context)
// Match each surviving entry back to its tool, by recognizing a moved definition or
// by key. Identity wins so a definition moved onto another tool's name still executes
// the tool it describes. Entries matching neither were invented by a hook and dropped.
@@ -357,8 +341,8 @@ export const layer = Layer.effect(
messages: boundImages(unsupportedParts(context.messages, resolved.capabilities)),
tools: Array.from(hooked, ([name, tool]) => ({ ...tool, name })),
toolChoice: input.toolChoice,
generation: Object.keys(generation).length === 0 ? undefined : generation,
providerOptions: Object.keys(providerOptions).length === 0 ? undefined : providerOptions,
generation: Object.keys(context.generation).length === 0 ? undefined : context.generation,
providerOptions: Object.keys(context.providerOptions).length === 0 ? undefined : context.providerOptions,
}),
)
const hasHttpHooks =
@@ -389,7 +373,7 @@ export const layer = Layer.effect(
tools
.execute({ ...input, definitions: hooked })
.pipe(Effect.catchCauseFilter(declineDefect, (decline) => Effect.fail(decline)))
const retry: Prepared["retry"] = (event) => hooks.trigger("session", "retry", { ...event, kind: input.kind })
const retry: Prepared["retry"] = (event) => hooks.trigger("session", "retry", event).pipe(Effect.asVoid)
return {
request,
options,
+5 -4
View File
@@ -6,8 +6,8 @@ import { Model } from "@opencode-ai/schema/model"
import { SessionError } from "@opencode-ai/schema/session-error"
import { Clock, Duration, Effect, Pull, Schedule } from "effect"
import { Bus } from "../../bus.js"
import type { PluginHooks } from "../../plugin/hooks.js"
import { SessionEvent } from "../event.js"
import type { SessionModelRequest } from "../model-request.js"
import { SessionMessage } from "../message.js"
import { SessionSchema } from "../schema.js"
@@ -16,7 +16,7 @@ interface Input {
readonly error: SessionError.Error
readonly agent: Agent.ID
readonly model: Model.Ref
readonly hook: SessionModelRequest.Prepared["retry"]
readonly hook: (event: PluginHooks.Domains["session"]["retry"]) => Effect.Effect<void>
readonly retry: boolean
}
@@ -90,14 +90,15 @@ export const policy = (sessionID: SessionSchema.ID) =>
const [, duration] = next
attempt++
const delay = Math.ceil(Duration.toMillis(duration))
const event = yield* input.hook({
const event: PluginHooks.Domains["session"]["retry"] = {
sessionID,
agent: input.agent,
model: input.model,
error: input.error,
attempt,
decision: input.retry ? { retry: true, delay } : { retry: false },
})
}
yield* input.hook(event)
if (!event.decision.retry) return event.decision
const normalized =
Number.isFinite(event.decision.delay) && event.decision.delay >= 0 ? Math.ceil(event.decision.delay) : delay
+1
View File
@@ -70,6 +70,7 @@ export const layer = Layer.effect(
system: input.agent.system ? [SystemPart.make(input.agent.system)] : [],
messages: [Message.user(input.text)],
},
contextHooks: false,
})
yield* llm.stream(prepared.request, prepared.options).pipe(
Stream.runForEach((event) => {
+1
View File
@@ -18,6 +18,7 @@ const jsonSchemas = Effect.runSync(
)
export const definition = (tool: Tool.Info<any, any>): ToolDefinition => ({
type: "tool",
name: effectiveName(tool),
description: tool.description,
inputSchema: inputJsonSchema(tool.input),
+2 -2
View File
@@ -121,11 +121,11 @@ const request = (agent: Agent.ID, messages: Array<Message>): SessionContext => (
sessionID,
agent,
model: { id: Model.ID.make("test-model"), providerID: Provider.ID.make("test") },
kind: "primary",
system: [],
messages,
tools: {},
options: {},
generation: {},
providerOptions: {},
})
type ToolErrorEvent = Extract<ToolHooks["execute.after"], { readonly status: "error" }>
@@ -29,11 +29,11 @@ const context = (id: string, system = fallback): SessionHooks["context"] => ({
sessionID: Session.ID.make("ses_system_prompt"),
agent: Agent.ID.make("build"),
model: Model.Ref.make({ providerID: Provider.ID.make("test"), id: Model.ID.make(id) }),
kind: "primary",
system: [SystemPart.make(system)],
messages: [],
tools: {},
options: {},
generation: {},
providerOptions: {},
})
describe("SystemPromptPlugin", () => {
+5 -5
View File
@@ -2124,19 +2124,19 @@ describe("SessionRunnerLLM", () => {
sessionID,
model: { id: ID.make(s.currentModel.id), providerID: Provider.ID.make(s.currentModel.provider), variant },
})
const hookRequests: Array<{ agent: Agent.ID; kind: string }> = []
const requestAgents: Agent.ID[] = []
yield* hooks.register("session", "context", (event) =>
Effect.sync(() => {
expect(event.agent).toBe(agentID)
expect(event.model.variant).toBe(variant)
event.system.push(SystemPart.make("Hook-provided instructions"))
event.tools.echo.description = "Hook-provided tool description"
event.options.maxTokens = 4_000
event.generation.maxTokens = 4_000
}),
)
yield* hooks.register("session", "model.request", (event) =>
Effect.sync(() => {
hookRequests.push({ agent: event.agent, kind: event.kind })
requestAgents.push(event.agent)
}),
)
yield* s.llm.push(
@@ -2182,7 +2182,7 @@ describe("SessionRunnerLLM", () => {
expect(compact[field]).toEqual(normal[field])
expect(compact.toolChoice).toBeUndefined()
expect(compact.system.map((part) => part.text)).toContain("Review the project carefully.")
expect(hookRequests[2]).toEqual({ agent: agentID, kind: "compaction" })
expect(requestAgents[2]).toBe(Agent.ID.make("compaction"))
expect(s.executions).toEqual(["x".repeat(4_000)])
expect((yield* s.messages).find((message) => message.type === "compaction")).toMatchObject({
model: { id: s.currentModel.id, providerID: s.currentModel.provider, variant },
@@ -2297,7 +2297,7 @@ describe("SessionRunnerLLM", () => {
expect(s.requests).toHaveLength(5)
for (const request of s.requests) expect(request).toEqual(s.requests[0])
expect(retries.map((event) => event.attempt)).toEqual([2, 3, 4, 5])
expect(retries.every((event) => event.sessionID === sessionID && event.kind === "compaction")).toBe(true)
expect(retries.every((event) => event.sessionID === sessionID && event.agent === "compaction")).toBe(true)
expect(retries[3].decision).toEqual({ retry: true, delay: 60_000 })
expect((yield* s.messages).find((message) => message.id === compaction.id)).toMatchObject({
status: "completed",
+1 -1
View File
@@ -105,7 +105,7 @@ for (const fixture of [
agent: Agent.defaultID,
model,
prepared: {
retry: (event) => Effect.succeed({ ...event, kind: "primary" as const }),
retry: () => Effect.void,
request: LLM.request({ model: model.model, prompt: "Run one tool", toolChoice: fixture.toolChoice }),
options: {},
executeTool: () =>
-20
View File
@@ -232,26 +232,6 @@ it.effect("generates a title from the sole user message and renames the session"
}),
)
it.effect("does not run context hooks for title requests", () =>
Effect.gen(function* () {
yield* enableTitleAgent
const sessionID = Session.ID.make("ses_title_context_hook")
yield* insertSession(sessionID)
yield* prompt(sessionID, "Help me debug the failing build")
const hooks = yield* PluginHooks.Service
let calls = 0
yield* hooks.register("session", "context", () => Effect.sync(() => calls++))
yield* hooks.register("session", "model.request", (event) => Effect.sync(() => expect(event.kind).toBe("title")))
const title = yield* SessionTitle.Service
yield* title.generate(sessionID)
expect(requests).toHaveLength(1)
expect(calls).toBe(0)
}),
)
it.effect("uses a small model from the primary provider", () =>
Effect.gen(function* () {
selectedSmall = small
+5
View File
@@ -27,6 +27,7 @@ test("tools are structural values", async () => {
const tool: Info = config
expect(definition(tool)).toEqual({
type: "tool",
name: "foreign",
description: "Foreign tool",
inputSchema: {
@@ -142,6 +143,7 @@ test("portable schemas validate and describe typed tools", async () => {
}
expect(definition(tool)).toEqual({
type: "tool",
name: "portable",
description: "Portable tool",
inputSchema: { type: "object", properties: { count: { type: "string" } } },
@@ -161,6 +163,7 @@ test("Zod schemas validate, transform, and describe typed tools", async () => {
}
expect(definition(tool)).toEqual({
type: "tool",
name: "zod",
description: "Zod tool",
inputSchema: {
@@ -317,6 +320,7 @@ test("raw JSON schemas validate and decode tool input", async () => {
}
expect(definition(tool)).toEqual({
type: "tool",
name: "raw",
description: "Raw tool",
inputSchema: input,
@@ -400,6 +404,7 @@ test("missing external input schemas fall back to an empty schema", () => {
} as unknown as Info
expect(definition(tool)).toEqual({
type: "tool",
name: "external",
description: "External tool",
inputSchema: {},
+12 -21
View File
@@ -18,32 +18,24 @@ export interface SessionPrompt {
delivery: SessionInbox.Delivery
}
export interface SessionContext {
readonly sessionID: Session.ID
readonly agent: Agent.ID
readonly model: Model.Ref
system: Array<SystemPart>
messages: Array<Message>
tools: Record<string, { description: string; input: JsonSchema.JsonSchema }>
/** Request overrides; unset fields retain route and model defaults. */
generation: Types.DeepMutable<GenerationOptionsFields>
providerOptions: Record<string, unknown>
}
/**
* Why a Session request is being made. Auxiliary requests share the Session's
* hook identity but need to be told apart from the agent loop.
*/
export type SessionRequestKind = "primary" | "compaction" | "title" | "generate"
/**
* Request overrides. Typed keys are the protocol-neutral generation settings;
* any other key is passed to the selected protocol as a provider option under its
* semantic name, such as `reasoningEffort` for OpenAI Responses. Unset fields
* retain route and model defaults.
*/
export type SessionRequestOptions = Types.DeepMutable<GenerationOptionsFields> & Record<string, unknown>
export interface SessionContext {
readonly sessionID: Session.ID
readonly agent: Agent.ID
readonly model: Model.Ref
/** Titles do not run context hooks; they will get a dedicated hook. */
readonly kind: Exclude<SessionRequestKind, "title">
system: Array<SystemPart>
messages: Array<Message>
tools: Record<string, { description: string; input: JsonSchema.JsonSchema }>
options: SessionRequestOptions
}
export interface SessionModelRequest {
readonly sessionID: Session.ID
readonly agent: Agent.ID
@@ -76,7 +68,6 @@ export interface SessionRetry {
readonly sessionID: Session.ID
readonly agent: Agent.ID
readonly model: Model.Ref
readonly kind: SessionRequestKind
readonly error: SessionError.Error
readonly attempt: number
decision: SessionRetryDecision
+12 -21
View File
@@ -18,32 +18,24 @@ export interface SessionPrompt {
delivery: SessionInbox.Delivery
}
export interface SessionContext {
readonly sessionID: Session.ID
readonly agent: Agent.ID
readonly model: Model.Ref
system: Array<SystemPart>
messages: Array<Message>
tools: Record<string, { description: string; input: JsonSchema.JsonSchema }>
/** Request overrides; unset fields retain route and model defaults. */
generation: Types.DeepMutable<GenerationOptionsFields>
providerOptions: Record<string, unknown>
}
/**
* Why a Session request is being made. Auxiliary requests share the Session's
* hook identity but need to be told apart from the agent loop.
*/
export type SessionRequestKind = "primary" | "compaction" | "title" | "generate"
/**
* Request overrides. Typed keys are the protocol-neutral generation settings;
* any other key is passed to the selected protocol as a provider option under its
* semantic name, such as `reasoningEffort` for OpenAI Responses. Unset fields
* retain route and model defaults.
*/
export type SessionRequestOptions = Types.DeepMutable<GenerationOptionsFields> & Record<string, unknown>
export interface SessionContext {
readonly sessionID: Session.ID
readonly agent: Agent.ID
readonly model: Model.Ref
/** Titles do not run context hooks; they will get a dedicated hook. */
readonly kind: Exclude<SessionRequestKind, "title">
system: Array<SystemPart>
messages: Array<Message>
tools: Record<string, { description: string; input: JsonSchema.JsonSchema }>
options: SessionRequestOptions
}
export interface SessionModelRequest {
readonly sessionID: Session.ID
readonly agent: Agent.ID
@@ -76,7 +68,6 @@ export interface SessionRetry {
readonly sessionID: Session.ID
readonly agent: Agent.ID
readonly model: Model.Ref
readonly kind: SessionRequestKind
readonly error: SessionError.Error
readonly attempt: number
decision: SessionRetryDecision
+1 -1
View File
@@ -69,7 +69,7 @@ it.live(
)
yield* ctx.session.hook("context", (event) =>
Effect.sync(() => {
event.options.temperature = 0.25
event.generation.temperature = 0.25
}),
)
yield* ctx.tool.transform((editor) =>
@@ -94,7 +94,7 @@ it.live(
)
yield* ctx.session.hook("context", (event) =>
Effect.sync(() => {
event.options.temperature = config.temperature
event.generation.temperature = config.temperature
}),
)
yield* ctx.permission.hook("evaluate", (event) =>
@@ -1089,10 +1089,7 @@ effect: (ctx) =>
### Sessions
Modify assembled system instructions, messages, tools, or request options immediately before model dispatch. The hook
runs for every request that carries the session conversation; `event.kind` is `"primary"`, `"compaction"`, or
`"generate"`. Title requests do not run context hooks.
Typed `options` keys are generation settings; any other key is passed to the protocol as a provider option.
Modify assembled system instructions, messages, or tools immediately before model dispatch.
```ts
effect: (ctx) =>
@@ -1100,10 +1097,8 @@ effect: (ctx) =>
const session = ctx.session
yield* session.hook("context", (event) =>
Effect.sync(() => {
if (event.kind === "compaction") return
event.system.push({ text: "Keep the review focused on correctness." })
delete event.tools.write
event.options.maxTokens = 8_000
}),
)
}),
@@ -1191,13 +1186,10 @@ interface SessionHooks {
type RetryDecision = { retry: false } | { retry: true; delay: number }
type SessionRequestKind = "primary" | "compaction" | "title" | "generate"
interface SessionRetry {
readonly sessionID: string
readonly agent: string
readonly model: { providerID: string; id: string; variant?: string }
readonly kind: SessionRequestKind
readonly error: { type: string; message: string; status?: number }
readonly attempt: number
decision: RetryDecision
@@ -1089,34 +1089,28 @@ Keep prompt hooks retry-safe. They are not an exactly-once side-effect boundary:
#### Model context
Modify assembled system instructions, messages, tools, or request options immediately before model dispatch.
Modify assembled system instructions, messages, tools, generation settings, or provider options immediately before model
dispatch.
```ts
await ctx.session.hook("context", (event) => {
event.system.push({ text: "Keep the review focused on correctness." })
delete event.tools.write
event.options.temperature = 0.2
event.options.maxTokens = 8_000
event.generation.temperature = 0.2
event.generation.maxTokens = 8_000
})
```
Context changes affect only the outgoing model call, not persisted history or configuration. The hook runs for every
request that carries the session conversation; `event.kind` says which flow issued it: `"primary"` for the agent loop,
`"compaction"` for checkpoint summaries, and `"generate"` for transient `ctx.session.generate` calls. Title requests do
not run context hooks. `event.agent` is always the session's selected agent.
Context changes affect only the outgoing model call, not persisted history or
configuration. The hook runs again for subsequent calls such as tool-driven
continuations, transient session generation, and compaction, but not for title requests.
```ts
await ctx.session.hook("context", (event) => {
if (event.kind === "compaction") return
event.messages.push({ role: "user", content: [{ type: "text", text: "Reminder: prefer small diffs." }] })
})
```
Compaction context hooks receive the selected session agent. Its model-request
and HTTP hooks retain the `compaction` agent identity for provider-specific handling.
Request options follow these rules:
Request overrides follow these rules:
- `options` starts empty for each model call; it does not contain resolved model settings.
- Typed keys (`maxTokens`, `temperature`, `topP`, `topK`, `frequencyPenalty`, `presencePenalty`, `seed`, `stop`) are the
protocol-neutral generation settings. Any other key is passed to the selected protocol as a provider option.
- `generation` and `providerOptions` start empty for each model call; they do not contain resolved model settings.
- Hooks run in registration order and see overrides made by earlier hooks.
- Request overrides take precedence over model defaults, which take precedence over route defaults.
- Provider option objects merge recursively; arrays and scalar values replace earlier values.
@@ -1132,7 +1126,7 @@ settings to the matching provider. For example, OpenAI Responses uses `reasoning
await ctx.session.hook(
"context",
(event) => {
event.options.reasoningEffort = "high"
event.providerOptions.reasoningEffort = "high"
},
{ providerID: "openai" },
)
@@ -1228,38 +1222,33 @@ interface SessionHooks {
type RetryDecision = { retry: false } | { retry: true; delay: number }
type SessionRequestKind = "primary" | "compaction" | "title" | "generate"
interface SessionRetryHook {
readonly sessionID: string
readonly agent: string
readonly model: { providerID: string; id: string; variant?: string }
readonly kind: SessionRequestKind
readonly error: { type: string; message: string; status?: number }
readonly attempt: number
decision: RetryDecision
}
type SessionRequestOptions = {
maxTokens?: number
temperature?: number
topP?: number
topK?: number
frequencyPenalty?: number
presencePenalty?: number
seed?: number
stop?: string[]
} & Record<string, unknown>
interface SessionContextHook {
readonly sessionID: string
readonly agent: string
readonly model: { providerID: string; id: string; variant?: string }
readonly kind: Exclude<SessionRequestKind, "title">
system: SystemPart[]
messages: Message[]
tools: Record<string, { description: string; input: JsonSchema }>
options: SessionRequestOptions
generation: {
maxTokens?: number
temperature?: number
topP?: number
topK?: number
frequencyPenalty?: number
presencePenalty?: number
seed?: number
stop?: string[]
}
providerOptions: Record<string, unknown>
}
interface SessionHookContext {
+1 -2
View File
@@ -80,8 +80,7 @@ preserves more recent detail but leaves less room for future work. Larger
V2 uses the session's selected agent, model, and variant to generate the summary.
The request reuses the normal instructions, tool definitions, and structured
history prefix, then appends a user message requesting a checkpoint. Context
hooks run as they do for normal session requests, with `kind` set to
`"compaction"`.
hooks run as they do for normal session requests.
Compaction does not dispatch local tool calls or override tool choice. The
summary must contain at least one heading from the requested template, such as