From d371ea1f01219a90717115b20f93314fd03d5617 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 01:02:18 +0100 Subject: [PATCH] perf(agents): wave-1 tool output contracts for code mode (#110215) * feat(agents): keep output contracts complete across opaque schema leaves * test(agents): add code mode schema-hint hot-path micro-benchmark * test(agents): count raw-first inspection execs per tool in live bench * feat(agents): expand compact output contract hints * feat(agents): declare wave one tool output contracts * docs(tools): list built-in output contracts * fix(agents): bound union scans before literal-union analysis * fix(agents): keep wave-1 contract schemas module-local and type-exact --- config/knip.config.ts | 1 + docs/tools/code-mode.md | 13 +- scripts/repro/tool-schema-hint-bench.ts | 218 ++++++++++++++++++ scripts/repro/tool-surface-live-bench.ts | 85 ++++++- src/agents/tool-schema-hints.test.ts | 52 +++++ src/agents/tool-schema-hints.ts | 23 +- src/agents/tools/agents-list-tool.test.ts | 4 + .../tools/openclaw-delegate-tool.test.ts | 7 + src/agents/tools/openclaw-delegate-tool.ts | 11 + src/agents/tools/screen-tool.test.ts | 13 ++ src/agents/tools/screen-tool.ts | 3 + src/agents/tools/sessions-search-tool.test.ts | 19 ++ src/agents/tools/sessions-search-tool.ts | 32 +++ .../tools/task-suggestion-tools.test.ts | 6 + src/agents/tools/task-suggestion-tools.ts | 6 + src/agents/tools/terminal-tool.test.ts | 12 + src/agents/tools/terminal-tool.ts | 30 +++ 17 files changed, 519 insertions(+), 16 deletions(-) create mode 100644 scripts/repro/tool-schema-hint-bench.ts diff --git a/config/knip.config.ts b/config/knip.config.ts index 3b36a0c0c8e..9e1aabd36b8 100644 --- a/config/knip.config.ts +++ b/config/knip.config.ts @@ -102,6 +102,7 @@ const rootEntries = [ "src/agents/compaction-planning.worker.ts!", "scripts/print-cli-backend-live-metadata.ts!", "scripts/repro/code-mode-namespace-live.ts!", + "scripts/repro/tool-schema-hint-bench.ts!", "scripts/repro/tool-surface-live-bench.ts!", // Workflow/package-script entrypoints are not imported from production modules. "scripts/openclaw-cross-os-release-checks.ts!", diff --git a/docs/tools/code-mode.md b/docs/tools/code-mode.md index e7042de1038..deda539389b 100644 --- a/docs/tools/code-mode.md +++ b/docs/tools/code-mode.md @@ -525,11 +525,14 @@ export default defineToolPlugin({ For `api.registerTool(...)` or a factory tool, put the same `outputSchema` property on the returned `AnyAgentTool` object. -Built-in tools can reuse their owning protocol schema instead of duplicating a -model-only contract. For example, the conversation tools expose the same -Gateway result schemas used by `conversations.list`, `conversations.send`, and -`conversations.turn`. When the quick index declares the fields, one cell can -compose discovery and delivery without a separate inspection turn: +Current built-in contracts include `agents_list`, `conversations_list`, +`conversations_send`, `conversations_turn`, `openclaw`, `screen`, +`sessions_search`, `spawn_task`, and `terminal`. Exact passthroughs can reuse +their owning protocol schema instead of duplicating a model-only contract. For +example, the conversation tools expose the same Gateway result schemas used by +`conversations.list`, `conversations.send`, and `conversations.turn`. When the +quick index declares the fields, one cell can compose discovery and delivery +without a separate inspection turn: ```javascript const listed = await tools.conversations_list({ query: "build bot" }); diff --git a/scripts/repro/tool-schema-hint-bench.ts b/scripts/repro/tool-schema-hint-bench.ts new file mode 100644 index 00000000000..b49d649368d --- /dev/null +++ b/scripts/repro/tool-schema-hint-bench.ts @@ -0,0 +1,218 @@ +#!/usr/bin/env -S node --import tsx +// Micro-benchmark for the Code Mode catalog hot path: schema hint compaction, +// full-catalog quick-index assembly, and declared-output validation. These run +// per agent run attempt, so regressions here tax every Code Mode turn. +import { performance } from "node:perf_hooks"; +import { Type } from "typebox"; +import { applyCodeModeCatalog, createCodeModeTools } from "../../src/agents/code-mode.js"; +import { compactToolInputHint, compactToolOutputHint } from "../../src/agents/tool-schema-hints.js"; +import { + compactToolSearchCatalogEntry, + createToolSearchCatalogRef, +} from "../../src/agents/tool-search.js"; +import { jsonResult, type AnyAgentTool } from "../../src/agents/tools/common.js"; +import { validateJsonSchemaValue } from "../../src/plugins/schema-validator.js"; +import { setPluginToolMeta } from "../../src/plugins/tools.js"; + +const WARMUP_ITERATIONS = 50; +const BATCHES = 7; +const CATALOG_TOOL_COUNT = 72; + +const TypicalInputSchema = Type.Object({ + channel: Type.Optional(Type.String({ minLength: 1 })), + query: Type.Optional(Type.String()), + limit: Type.Optional(Type.Integer({ minimum: 1 })), +}); + +const TypicalOutputSchema = Type.Union([ + Type.Object( + { + status: Type.Literal("replied"), + messageId: Type.String(), + reply: Type.Object( + { + text: Type.String(), + timestamp: Type.Number(), + threadId: Type.Optional(Type.String()), + }, + { additionalProperties: false }, + ), + }, + { additionalProperties: false }, + ), + Type.Object( + { status: Type.Literal("timeout"), messageId: Type.String() }, + { additionalProperties: false }, + ), + Type.Object( + { status: Type.Literal("error"), error: Type.String() }, + { additionalProperties: false }, + ), +]); + +const TypicalOutputValue = { + status: "replied", + messageId: "m-1", + reply: { text: "done", timestamp: 1_700_000_000_000, threadId: "t-9" }, +}; + +function buildAdversarialSchema(): Record { + // Attacker-sized MCP metadata: wide property map, deep nesting, giant enum. + const wide: Record = {}; + for (let index = 0; index < 5_000; index += 1) { + wide[`property_${index}_${"x".repeat(24)}`] = { type: "string" }; + } + let deep: Record = { type: "string" }; + for (let index = 0; index < 40; index += 1) { + deep = { type: "object", properties: { child: deep } }; + } + return { + type: "object", + properties: { + ...wide, + deep, + bigEnum: { enum: Array.from({ length: 10_000 }, (_unused, index) => `value-${index}`) }, + }, + }; +} + +function buildCatalogTools(): AnyAgentTool[] { + const tools: AnyAgentTool[] = []; + for (let index = 0; index < CATALOG_TOOL_COUNT; index += 1) { + const declareOutput = index % 3 === 0; + const name = `bench_tool_${String(index).padStart(2, "0")}`; + const tool = { + name, + label: name, + description: `Benchmark tool ${index} exercising catalog compaction cost.`, + parameters: TypicalInputSchema, + ...(declareOutput ? { outputSchema: TypicalOutputSchema } : {}), + execute: async (_toolCallId: string, _params: unknown) => jsonResult({ ok: true }), + } satisfies AnyAgentTool; + setPluginToolMeta(tool, { pluginId: "bench-hints", optional: true }); + tools.push(tool); + } + return tools; +} + +const CODE_MODE_SESSION = { + sessionId: "bench-hints", + sessionKey: "agent:bench-hints:main", + agentId: "bench", + runId: "run-bench-hints", +}; +const CODE_MODE_CONFIG = { tools: { codeMode: { enabled: true, timeoutMs: 20_000 } } }; + +function applyCodeModeSurface(tools: AnyAgentTool[]) { + const catalogRef = createToolSearchCatalogRef(); + const codeModeTools = createCodeModeTools({ + config: CODE_MODE_CONFIG, + runtimeConfig: CODE_MODE_CONFIG, + ...CODE_MODE_SESSION, + catalogRef, + }); + return { + catalogRef, + applied: applyCodeModeCatalog({ + tools: [...codeModeTools, ...tools], + config: CODE_MODE_CONFIG, + ...CODE_MODE_SESSION, + catalogRef, + }), + }; +} + +type BenchCase = { name: string; iterations: number; run: () => void }; + +function median(values: number[]): number { + const sorted = [...values].toSorted((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)] ?? 0; +} + +function bench(benchCase: BenchCase): { name: string; nsPerOp: number; opsPerSec: number } { + for (let index = 0; index < WARMUP_ITERATIONS; index += 1) { + benchCase.run(); + } + const perBatchNs: number[] = []; + for (let batch = 0; batch < BATCHES; batch += 1) { + const start = performance.now(); + for (let index = 0; index < benchCase.iterations; index += 1) { + benchCase.run(); + } + perBatchNs.push(((performance.now() - start) * 1e6) / benchCase.iterations); + } + const nsPerOp = median(perBatchNs); + return { name: benchCase.name, nsPerOp, opsPerSec: 1e9 / nsPerOp }; +} + +async function main(): Promise { + const adversarial = buildAdversarialSchema(); + const tools = buildCatalogTools(); + const { catalogRef, applied } = applyCodeModeSurface(tools); + const entries = catalogRef.current?.entries ?? []; + if (entries.length < CATALOG_TOOL_COUNT) { + throw new Error(`catalog only registered ${entries.length} entries`); + } + // Warm the validator cache once so the loop measures the steady-state hit. + validateJsonSchemaValue({ + schema: TypicalOutputSchema as never, + cacheKey: "bench:typical-output", + value: TypicalOutputValue, + }); + + const cases: BenchCase[] = [ + { + name: "hint: typical input", + iterations: 20_000, + run: () => void compactToolInputHint(TypicalInputSchema), + }, + { + name: "hint: typical output union", + iterations: 20_000, + run: () => void compactToolOutputHint(TypicalOutputSchema), + }, + { + name: "hint: adversarial 5k-prop schema", + iterations: 200, + run: () => void compactToolInputHint(adversarial), + }, + { + name: `catalog: compact ${CATALOG_TOOL_COUNT} entries`, + iterations: 2_000, + run: () => { + for (const entry of entries) { + compactToolSearchCatalogEntry(entry); + } + }, + }, + { + name: "validate: declared output warm hit", + iterations: 20_000, + run: () => + void validateJsonSchemaValue({ + schema: TypicalOutputSchema as never, + cacheKey: "bench:typical-output", + value: TypicalOutputValue, + }), + }, + { + name: "surface: full applyCodeModeCatalog", + iterations: 500, + run: () => void applyCodeModeSurface(tools), + }, + ]; + + process.stdout.write( + `tool-schema-hint-bench catalogTools=${entries.length} visibleTools=${applied.tools.length}\n`, + ); + for (const benchCase of cases) { + const result = bench(benchCase); + const usPerOp = (result.nsPerOp / 1_000).toFixed(2); + const ops = Math.round(result.opsPerSec).toLocaleString("en-US"); + process.stdout.write( + `${result.name.padEnd(36)} ${usPerOp.padStart(10)} us/op ${ops.padStart(12)} ops/s\n`, + ); + } +} + +await main(); diff --git a/scripts/repro/tool-surface-live-bench.ts b/scripts/repro/tool-surface-live-bench.ts index ea4c5214b67..3d7ccb3f0bb 100755 --- a/scripts/repro/tool-surface-live-bench.ts +++ b/scripts/repro/tool-surface-live-bench.ts @@ -3,6 +3,7 @@ // Tool Search (code/tools), and Code Mode, over a decoy-heavy catalog. import { performance } from "node:perf_hooks"; import { pathToFileURL } from "node:url"; +import { isDeepStrictEqual } from "node:util"; import { Type, type TSchema } from "typebox"; import type { Model } from "../../packages/agent-core/src/llm.js"; import type { AgentEvent, AgentTool } from "../../packages/agent-core/src/types.js"; @@ -118,6 +119,9 @@ function createOrchardService() { let calls = 0; let decoyCalls = 0; const checkedSensorPlots = new Set(); + // Tool results recorded per call so the harness can detect raw-first + // inspection execs (exec output deep-equal to one recorded tool result). + const resultLog: Array<{ tool: string; value: unknown }> = []; const note = () => { calls += 1; }; @@ -128,6 +132,12 @@ function createOrchardService() { get decoyCalls() { return decoyCalls; }, + get resultLog(): ReadonlyArray<{ tool: string; value: unknown }> { + return resultLog; + }, + noteResult(tool: string, value: unknown) { + resultLog.push({ tool, value: structuredClone(value) }); + }, noteDecoy() { decoyCalls += 1; }, @@ -183,6 +193,7 @@ function stringParam(params: Record, key: string): string { } function makeTool( + service: OrchardService, pluginId: string, name: string, description: string, @@ -196,12 +207,13 @@ function makeTool( description, parameters: Type.Object(properties), ...(outputSchema ? { outputSchema } : {}), - execute: async (_toolCallId: string, params: unknown) => - jsonResult( - await execute( - (params && typeof params === "object" ? params : {}) as Record, - ), - ), + execute: async (_toolCallId: string, params: unknown) => { + const value = await execute( + (params && typeof params === "object" ? params : {}) as Record, + ); + service.noteResult(name, value); + return jsonResult(value); + }, } satisfies AnyAgentTool; setPluginToolMeta(tool, { pluginId, optional: true }); return tool; @@ -210,6 +222,7 @@ function makeTool( function createOrchardTools(service: OrchardService): AnyAgentTool[] { return [ makeTool( + service, PLUGIN_ID, "orchard_list_plots", "List orchard plots with crop, hectares, irrigation mode, and yield score.", @@ -218,6 +231,7 @@ function createOrchardTools(service: OrchardService): AnyAgentTool[] { Type.Array(PlotOutputSchema), ), makeTool( + service, PLUGIN_ID, "orchard_get_plot", "Get one orchard plot by id.", @@ -226,6 +240,7 @@ function createOrchardTools(service: OrchardService): AnyAgentTool[] { Type.Union([PlotOutputSchema, Type.Null()]), ), makeTool( + service, PLUGIN_ID, "orchard_list_sensors", "List field sensors, optionally filtered by plot id.", @@ -235,6 +250,7 @@ function createOrchardTools(service: OrchardService): AnyAgentTool[] { Type.Array(SensorOutputSchema), ), makeTool( + service, PLUGIN_ID, "orchard_list_shipments", "List harvest shipments, optionally filtered by buyer.", @@ -244,6 +260,7 @@ function createOrchardTools(service: OrchardService): AnyAgentTool[] { Type.Array(ShipmentOutputSchema), ), makeTool( + service, PLUGIN_ID, "orchard_update_irrigation", "Set the irrigation mode for one plot after its sensors have been read.", @@ -263,7 +280,7 @@ function createDecoyTools(service: OrchardService): AnyAgentTool[] { description: string, properties: Parameters[0] = {}, ) => - makeTool(DECOY_PLUGIN_ID, name, description, properties, () => { + makeTool(service, DECOY_PLUGIN_ID, name, description, properties, () => { service.noteDecoy(); return { error: "decoy tool: not part of the orchard console" }; }); @@ -503,6 +520,46 @@ function parseFirstJson(text: string): unknown { } } +const CODE_EXEC_TOOL_NAMES = new Set(["exec", "tool_search_code"]); + +// An exec that returns one tool's raw result unchanged is a shape-inspection +// turn: the model paid a full model round trip only to observe result fields. +// Output-contract adoption is ranked by how many of these each tool causes. +function countRawInspectionExecs( + messages: readonly unknown[], + resultLog: ReadonlyArray<{ tool: string; value: unknown }>, +): { total: number; byTool: Record } { + const byTool: Record = {}; + let total = 0; + for (const message of messages) { + const record = message as { role?: string; toolName?: string; content?: unknown }; + if (record.role !== "toolResult" || !CODE_EXEC_TOOL_NAMES.has(record.toolName ?? "")) { + continue; + } + let parsed: unknown; + try { + parsed = JSON.parse(textFromMessageContent(record.content)) as unknown; + } catch { + continue; + } + const envelope = parsed as { status?: unknown; value?: unknown }; + // Scalars deep-equal too easily; only structured raw returns count. + const value = envelope.status === "completed" ? envelope.value : undefined; + if (typeof value !== "object" || value === null) { + continue; + } + const matched = new Set( + resultLog.filter((entry) => isDeepStrictEqual(entry.value, value)).map((entry) => entry.tool), + ); + if (matched.size === 1) { + const tool = [...matched][0] as string; + byTool[tool] = (byTool[tool] ?? 0) + 1; + total += 1; + } + } + return { total, byTool }; +} + type RunMetrics = { provider: ProviderId; model: string; @@ -515,6 +572,8 @@ type RunMetrics = { toolCalls: number; serviceCalls: number; decoyCalls: number; + rawInspectionExecs: number; + rawInspectionByTool: Record; toolsExposed: number; tokensIn: number; tokensOut: number; @@ -583,6 +642,7 @@ async function runOne(params: { } } const latencyMs = Math.round(performance.now() - started); + const rawInspections = countRawInspectionExecs(agent.state.messages, service.resultLog); const assistants = agent.state.messages.filter((message) => message.role === "assistant"); const usage = assistants.reduce( (sum, message) => { @@ -654,6 +714,8 @@ async function runOne(params: { toolCalls: counts.toolCalls, serviceCalls: service.calls, decoyCalls: service.decoyCalls, + rawInspectionExecs: rawInspections.total, + rawInspectionByTool: rawInspections.byTool, toolsExposed: tools.length, tokensIn: usage.input, tokensOut: usage.output, @@ -816,13 +878,20 @@ async function main(argv: readonly string[] = process.argv.slice(2)) { turns: entries.reduce((sum, entry) => sum + entry.turns, 0), toolCalls: entries.reduce((sum, entry) => sum + entry.toolCalls, 0), decoyCalls: entries.reduce((sum, entry) => sum + entry.decoyCalls, 0), + rawInspectionExecs: entries.reduce((sum, entry) => sum + entry.rawInspectionExecs, 0), tokensIn: entries.reduce((sum, entry) => sum + entry.tokensIn, 0), tokensOut: entries.reduce((sum, entry) => sum + entry.tokensOut, 0), cacheRead: entries.reduce((sum, entry) => sum + entry.cacheRead, 0), }; }), ); - console.log(JSON.stringify({ results, errors, aggregate }, null, 2)); + const rawInspectionByTool: Record = {}; + for (const entry of results) { + for (const [tool, count] of Object.entries(entry.rawInspectionByTool)) { + rawInspectionByTool[tool] = (rawInspectionByTool[tool] ?? 0) + count; + } + } + console.log(JSON.stringify({ results, errors, aggregate, rawInspectionByTool }, null, 2)); if (errors.length > 0 || results.some((entry) => !entry.ok)) { process.exitCode = 1; } diff --git a/src/agents/tool-schema-hints.test.ts b/src/agents/tool-schema-hints.test.ts index 4854bd7b574..c3400817c34 100644 --- a/src/agents/tool-schema-hints.test.ts +++ b/src/agents/tool-schema-hints.test.ts @@ -52,6 +52,26 @@ describe("tool schema hints", () => { ); }); + it("renders up to eight literal union values", () => { + const values = [ + "env", + "agent", + "defaults", + "model", + "provider", + "implicit", + "session", + "session-key", + ]; + const eight = Type.Union(values.map((value) => Type.Literal(value))); + const nine = Type.Union([...values, "extra"].map((value) => Type.Literal(value))); + + expect(compactToolOutputHint(eight)).toBe( + '"env" | "agent" | "defaults" | "model" | "provider" | "implicit" | "session" | "session-key"', + ); + expect(compactToolOutputHint(nine)).toBeUndefined(); + }); + it("keeps input hints small while allowing larger exact output contracts", () => { const schema = Type.Object( Object.fromEntries( @@ -73,6 +93,38 @@ describe("tool schema hints", () => { expect(outputHint!.length).toBeLessThanOrEqual(600); }); + it("keeps contracts with explicitly opaque leaves complete", () => { + const outputSchema = Type.Object( + { + count: Type.Number(), + messages: Type.Array(Type.Unknown()), + payload: Type.Optional(Type.Unknown()), + }, + { additionalProperties: false }, + ); + + expect(compactToolOutputHint(outputSchema)).toBe( + "{ count: number; messages: Array; payload?: unknown }", + ); + }); + + it("renders a bare top-type schema as unknown without demoting", () => { + expect(compactToolOutputHint(Type.Unknown())).toBe("unknown"); + expect(compactToolOutputHint(Type.Any())).toBe("unknown"); + }); + + it("still fails closed for constrained but untyped leaves", () => { + const outputSchema = Type.Object( + { + id: Type.String(), + blob: { minLength: 1 } as unknown as ReturnType, + }, + { additionalProperties: false }, + ); + + expect(compactToolOutputHint(outputSchema)).toBeUndefined(); + }); + it("includes null in AJV-style nullable output hints", () => { expect(compactToolOutputHint({ type: "string", nullable: true })).toBe("string | null"); expect( diff --git a/src/agents/tool-schema-hints.ts b/src/agents/tool-schema-hints.ts index 69d06b35dfa..8894cc4e49b 100644 --- a/src/agents/tool-schema-hints.ts +++ b/src/agents/tool-schema-hints.ts @@ -8,7 +8,9 @@ const MAX_COMPACT_SCHEMA_PROPERTY_NAME_CHARS = 128; const MAX_COMPACT_INPUT_DEPTH = 4; const MAX_COMPACT_OUTPUT_DEPTH = 6; const MAX_COMPACT_UNION_TYPES = 4; -const MAX_COMPACT_ENUM_VALUES = 6; +// Keeps real literal unions such as agents_list's eight runtime sources renderable, +// while the combined literal text remains independently capped below. +const MAX_COMPACT_ENUM_VALUES = 8; const MAX_COMPACT_ENUM_CHARS = 96; const IDENTIFIER_RE = /^[A-Za-z_$][A-Za-z0-9_$]*$/u; const UNSUPPORTED_SHAPE_KEYWORDS = [ @@ -126,10 +128,13 @@ function compactSchemaUnion( return UNKNOWN_HINT; } const variants = hasAnyOf ? schema.anyOf : schema.oneOf; + // Bound before any per-variant scan: neither the eight-value literal cap nor + // the four-variant structural cap can render a larger union, so oversized + // unions must be rejected in O(1) instead of O(variants). if ( !Array.isArray(variants) || variants.length === 0 || - variants.length > MAX_COMPACT_UNION_TYPES + variants.length > MAX_COMPACT_ENUM_VALUES ) { return UNKNOWN_HINT; } @@ -158,6 +163,9 @@ function compactSchemaUnion( return literalUnion; } } + if (variants.length > MAX_COMPACT_UNION_TYPES) { + return UNKNOWN_HINT; + } const rendered = variants.map((variant) => compactSchemaType(variant, depth + 1, limits)); if (rendered.some((hint) => !hint.complete)) { return UNKNOWN_HINT; @@ -287,7 +295,16 @@ function compactSchemaType( depth = 0, limits: CompactSchemaLimits = INPUT_LIMITS, ): SchemaHint { - if (!isRecord(schema) || depth >= limits.maxDepth) { + if (!isRecord(schema)) { + return UNKNOWN_HINT; + } + // An empty schema is JSON Schema's top type: it accepts any value, so + // `unknown` is its exact rendering, not a truncation. Without this, one + // opaque leaf (Type.Unknown/Type.Any) demotes an otherwise-exact contract. + if (Object.keys(schema).length === 0) { + return completeHint("unknown"); + } + if (depth >= limits.maxDepth) { return UNKNOWN_HINT; } const normalizedNullableSchema = normalizeNullableSchemaForHint(schema); diff --git a/src/agents/tools/agents-list-tool.test.ts b/src/agents/tools/agents-list-tool.test.ts index 84d03132b28..5de7551b5bd 100644 --- a/src/agents/tools/agents-list-tool.test.ts +++ b/src/agents/tools/agents-list-tool.test.ts @@ -2,6 +2,7 @@ // runtime override handling. import { beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import { compactToolOutputHint } from "../tool-schema-hints.js"; import { createAgentsListTool } from "./agents-list-tool.js"; const loadConfigMock = vi.fn<() => OpenClawConfig>(); @@ -61,6 +62,9 @@ describe("agents_list tool", () => { type: "object", required: ["requester", "allowAny", "agents"], }); + expect(compactToolOutputHint(tool.outputSchema)).toBe( + '{ agents: Array<{ configured: boolean; id: string; agentRuntime?: { id: string; source: "env" | "agent" | "defaults" | "model" | "provider" | "implicit" | "session" | "session-key" }; model?: string; name?: string }>; allowAny: boolean; requester: string }', + ); const result = await tool.execute("call", {}); const details = result.details as AgentListDetails; diff --git a/src/agents/tools/openclaw-delegate-tool.test.ts b/src/agents/tools/openclaw-delegate-tool.test.ts index dff09dfd18c..92beec8f655 100644 --- a/src/agents/tools/openclaw-delegate-tool.test.ts +++ b/src/agents/tools/openclaw-delegate-tool.test.ts @@ -1,5 +1,7 @@ +import { Value } from "typebox/value"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { GATEWAY_OWNER_ONLY_CORE_TOOLS } from "../../security/dangerous-tools.js"; +import { compactToolOutputHint } from "../tool-schema-hints.js"; import { callInProcessGatewayTool } from "./in-process-gateway.js"; import { createOpenClawDelegateToolsForRun } from "./openclaw-delegate-tool.js"; @@ -47,6 +49,11 @@ describe("openclaw delegation tool", () => { needsApproval: true, proposalId: "system-agent:proposal-1", }); + expect(tool.outputSchema).toBeDefined(); + expect(Value.Check(tool.outputSchema!, result.details)).toBe(true); + expect(compactToolOutputHint(tool.outputSchema)).toBe( + "{ reply: string; action?: string; needsApproval?: true; proposalId?: string }", + ); expect(tool.catalogMode).toBeUndefined(); }); diff --git a/src/agents/tools/openclaw-delegate-tool.ts b/src/agents/tools/openclaw-delegate-tool.ts index 27ecd982b97..cbb34932bf0 100644 --- a/src/agents/tools/openclaw-delegate-tool.ts +++ b/src/agents/tools/openclaw-delegate-tool.ts @@ -10,6 +10,16 @@ const OpenClawDelegateSchema = Type.Object({ sessionId: Type.Optional(Type.String({ description: "Continue prior OpenClaw talk." })), }); +const OpenClawDelegateOutputSchema = Type.Object( + { + reply: Type.String(), + action: Type.Optional(Type.String()), + needsApproval: Type.Optional(Type.Literal(true)), + proposalId: Type.Optional(Type.String()), + }, + { additionalProperties: false }, +); + type OpenClawDelegateResult = { sessionId: string; reply: string; @@ -40,6 +50,7 @@ function createOpenClawDelegateTool(options?: { description: "Ask system expert. Config, channels, plugins, agents, models/providers, updates. Writes need human approval.", parameters: OpenClawDelegateSchema, + outputSchema: OpenClawDelegateOutputSchema, execute: async (_toolCallId, args) => { const params = (args ?? {}) as Record; const message = readStringParam(params, "message", { required: true }); diff --git a/src/agents/tools/screen-tool.test.ts b/src/agents/tools/screen-tool.test.ts index 229835bce7b..5ea9926f187 100644 --- a/src/agents/tools/screen-tool.test.ts +++ b/src/agents/tools/screen-tool.test.ts @@ -1,5 +1,8 @@ +import { Value } from "typebox/value"; import { describe, expect, it } from "vitest"; import { GATEWAY_CLIENT_CAPS } from "../../../packages/gateway-protocol/src/client-info.js"; +import { UiCommandResultSchema } from "../../../packages/gateway-protocol/src/schema/ui-command.js"; +import { compactToolOutputHint } from "../tool-schema-hints.js"; import type { InProcessGatewayCaller } from "./in-process-gateway.js"; import { createScreenTool } from "./screen-tool.js"; @@ -16,6 +19,16 @@ function createGatewayRecorder() { } describe("screen tool", () => { + it("declares the exact ui.command result contract", async () => { + const { callGateway } = createGatewayRecorder(); + const tool = createScreenTool({ callGateway }); + const result = await tool.execute("contract", { action: "sidebar_show" }); + + expect(tool.outputSchema).toBe(UiCommandResultSchema); + expect(Value.Check(tool.outputSchema!, result.details)).toBe(true); + expect(compactToolOutputHint(tool.outputSchema)).toBe("{ ok: boolean }"); + }); + it("uses a flat action enum and requires the UI capability", () => { const tool = createScreenTool(); expect(tool.requiredClientCaps).toEqual([GATEWAY_CLIENT_CAPS.UI_COMMANDS]); diff --git a/src/agents/tools/screen-tool.ts b/src/agents/tools/screen-tool.ts index a52d8e37bf6..4993f41734b 100644 --- a/src/agents/tools/screen-tool.ts +++ b/src/agents/tools/screen-tool.ts @@ -1,6 +1,8 @@ import { Type } from "typebox"; import { GATEWAY_CLIENT_CAPS } from "../../../packages/gateway-protocol/src/client-info.js"; import type { UiCommand, UiCommandParams } from "../../../packages/gateway-protocol/src/index.js"; +// The tool returns the Gateway result unchanged, so the wire schema remains the single owner. +import { UiCommandResultSchema } from "../../../packages/gateway-protocol/src/schema/ui-command.js"; import type { AnyAgentTool } from "./common.js"; import { jsonResult, readStringParam, ToolInputError } from "./common.js"; import { callInProcessGatewayTool, type InProcessGatewayCaller } from "./in-process-gateway.js"; @@ -101,6 +103,7 @@ export function createScreenTool(opts: ScreenToolOptions = {}): AnyAgentTool { description: "Drive operator web UI. Split panes, focus, panels, sidebar, navigate. Needs connected web client.", parameters: ScreenToolSchema, + outputSchema: UiCommandResultSchema, requiredClientCaps: [GATEWAY_CLIENT_CAPS.UI_COMMANDS], execute: async (_toolCallId, rawArgs) => { const params = rawArgs as Record; diff --git a/src/agents/tools/sessions-search-tool.test.ts b/src/agents/tools/sessions-search-tool.test.ts index 6a49b791f8b..da84e8f2212 100644 --- a/src/agents/tools/sessions-search-tool.test.ts +++ b/src/agents/tools/sessions-search-tool.test.ts @@ -1,7 +1,9 @@ /** sessions_search visibility, bounds, redaction, and input tests. */ +import { Value } from "typebox/value"; import { afterEach, describe, expect, it } from "vitest"; import type { callGateway as gatewayCall } from "../../gateway/call.js"; import { sessionVisibilityGatewayTesting } from "../../plugin-sdk/session-visibility.js"; +import { compactToolOutputHint } from "../tool-schema-hints.js"; import { createSessionsSearchTool } from "./sessions-search-tool.js"; type CallGatewayRequest = Parameters[0]; @@ -82,6 +84,23 @@ afterEach(() => { }); describe("sessions_search tool", () => { + it("declares exact success and error result contracts", async () => { + const tool = createTool({ results: [hit()] }); + const success = await tool.execute("success-contract", { query: "text" }); + const error = await tool.execute("error-contract", { + query: "text", + sessionKey: "01234567-89ab-4def-8123-456789abcdef", + }); + + expect(tool.outputSchema).toBeDefined(); + expect(Value.Check(tool.outputSchema!, success.details)).toBe(true); + expect(error.details).toMatchObject({ status: "error", error: expect.any(String) }); + expect(Value.Check(tool.outputSchema!, error.details)).toBe(true); + expect(compactToolOutputHint(tool.outputSchema)).toBe( + '{ results: Array<{ role: "assistant" | "user"; score: number; sessionKey: string; snippet: string; timestamp: number; messageId?: string; sessionId?: string }>; indexing?: true; truncated?: true } | { error: string; status: "error" | "forbidden" }', + ); + }); + it("rejects empty queries and invalid limits", async () => { const tool = createTool({}); await expect(tool.execute("call-1", { query: " " })).rejects.toThrow( diff --git a/src/agents/tools/sessions-search-tool.ts b/src/agents/tools/sessions-search-tool.ts index 41e068c3d7b..33403a61186 100644 --- a/src/agents/tools/sessions-search-tool.ts +++ b/src/agents/tools/sessions-search-tool.ts @@ -43,6 +43,37 @@ const SessionsSearchToolSchema = Type.Object({ limit: optionalPositiveIntegerSchema({ maximum: SESSIONS_SEARCH_MAX_LIMIT }), }); +const SessionsSearchHitSchema = Type.Object( + { + sessionKey: Type.String(), + timestamp: Type.Number(), + role: Type.Union([Type.Literal("assistant"), Type.Literal("user")]), + snippet: Type.String(), + score: Type.Number(), + sessionId: Type.Optional(Type.String()), + messageId: Type.Optional(Type.String()), + }, + { additionalProperties: false }, +); + +const SessionsSearchOutputSchema = Type.Union([ + Type.Object( + { + results: Type.Array(SessionsSearchHitSchema), + indexing: Type.Optional(Type.Literal(true)), + truncated: Type.Optional(Type.Literal(true)), + }, + { additionalProperties: false }, + ), + Type.Object( + { + status: Type.Union([Type.Literal("error"), Type.Literal("forbidden")]), + error: Type.String(), + }, + { additionalProperties: false }, + ), +]); + type GatewayCaller = typeof callGateway; type GatewaySearchHit = { @@ -299,6 +330,7 @@ export function createSessionsSearchTool(opts?: { displaySummary: SESSIONS_SEARCH_TOOL_DISPLAY_SUMMARY, description: describeSessionsSearchTool(), parameters: SessionsSearchToolSchema, + outputSchema: SessionsSearchOutputSchema, execute: async (_toolCallId, args) => { const params = args as Record; const query = readStringParam(params, "query")?.trim() ?? ""; diff --git a/src/agents/tools/task-suggestion-tools.test.ts b/src/agents/tools/task-suggestion-tools.test.ts index 83b76969f7e..f73f01e44a2 100644 --- a/src/agents/tools/task-suggestion-tools.test.ts +++ b/src/agents/tools/task-suggestion-tools.test.ts @@ -1,4 +1,6 @@ +import { Value } from "typebox/value"; import { describe, expect, it, vi } from "vitest"; +import { compactToolOutputHint } from "../tool-schema-hints.js"; import { createTaskSuggestionTools } from "./task-suggestion-tools.js"; function createTools(gatewayCall = vi.fn()) { @@ -41,6 +43,10 @@ describe("task suggestion tools", () => { expect(result?.content).toEqual([ { type: "text", text: JSON.stringify({ task_id: "task_123" }, null, 2) }, ]); + expect(spawnTask?.outputSchema).toBeDefined(); + expect(Value.Check(spawnTask!.outputSchema!, result?.details)).toBe(true); + expect(compactToolOutputHint(spawnTask?.outputSchema)).toBe("{ task_id: string }"); + expect(tools.find((tool) => tool.name === "dismiss_task")?.outputSchema).toBeUndefined(); }); it("withdraws a pending suggestion", async () => { diff --git a/src/agents/tools/task-suggestion-tools.ts b/src/agents/tools/task-suggestion-tools.ts index 59d0384332c..4022784a5ec 100644 --- a/src/agents/tools/task-suggestion-tools.ts +++ b/src/agents/tools/task-suggestion-tools.ts @@ -40,6 +40,11 @@ const SpawnTaskToolSchema = Type.Object( { additionalProperties: false }, ); +const SpawnTaskOutputSchema = Type.Object( + { task_id: Type.String() }, + { additionalProperties: false }, +); + const DismissTaskToolSchema = Type.Object( { task_id: Type.String({ @@ -73,6 +78,7 @@ export function createTaskSuggestionTools(params: { "Operator suggestion only; does not start work.", ].join(" "), parameters: SpawnTaskToolSchema, + outputSchema: SpawnTaskOutputSchema, execute: async (_toolCallId, args) => { const input = args as Record; const title = readStringParam(input, "title", { required: true }); diff --git a/src/agents/tools/terminal-tool.test.ts b/src/agents/tools/terminal-tool.test.ts index 63f22d45fca..a314c782d40 100644 --- a/src/agents/tools/terminal-tool.test.ts +++ b/src/agents/tools/terminal-tool.test.ts @@ -1,8 +1,10 @@ +import { Value } from "typebox/value"; import { describe, expect, it, vi } from "vitest"; import { TERMINAL_OPEN_DEADLINE_MS } from "../../gateway/terminal/open-deadline.js"; import { TerminalSessionManager } from "../../gateway/terminal/session-manager.js"; import type { spawnTerminalPty } from "../../process/terminal-pty.js"; import { GATEWAY_OWNER_ONLY_CORE_TOOLS } from "../../security/dangerous-tools.js"; +import { compactToolOutputHint } from "../tool-schema-hints.js"; import type { InProcessGatewayCaller } from "./in-process-gateway.js"; import { createTerminalTool } from "./terminal-tool.js"; @@ -89,8 +91,13 @@ describe("terminal tool", () => { callGateway, getGatewayContext: () => makeContext(manager), }); + expect(tool.outputSchema).toBeDefined(); + expect(compactToolOutputHint(tool.outputSchema)).toBe( + "{ sessions: Array<{ agentId: string; attached: boolean; createdAtMs: number; cwd: string; owner: string; sessionId: string; shell: string }> } | { agentId: string; cwd: string; ok: true; sessionId: string; shell: string } | { sessionId: string; text: string } | { ok: boolean }", + ); const opened = await tool.execute("open", { action: "open", command: "echo ready" }); + expect(Value.Check(tool.outputSchema!, opened.details)).toBe(true); const sessionId = (opened.details as { sessionId: string }).sessionId; expect(backend.writes).toEqual(["echo ready\r"]); expect(callGateway).toHaveBeenCalledWith("ui.command", { @@ -106,9 +113,11 @@ describe("terminal tool", () => { backend.emitData("\u001b[31mready\u001b[0m\r\n"); const read = await tool.execute("read", { action: "read", sessionId }); expect(read.details).toEqual({ sessionId, text: "ready\n" }); + expect(Value.Check(tool.outputSchema!, read.details)).toBe(true); const input = await tool.execute("input", { action: "input", sessionId, data: "yes\r" }); expect(input.details).toEqual({ ok: true }); + expect(Value.Check(tool.outputSchema!, input.details)).toBe(true); expect(backend.writes).toEqual(["echo ready\r", "yes\r"]); const resize = await tool.execute("resize", { action: "resize", @@ -117,6 +126,7 @@ describe("terminal tool", () => { rows: 40, }); expect(resize.details).toEqual({ ok: true }); + expect(Value.Check(tool.outputSchema!, resize.details)).toBe(true); expect(backend.resizes).toEqual([[120, 40]]); const list = await tool.execute("list", { action: "list" }); @@ -128,8 +138,10 @@ describe("terminal tool", () => { }), ], }); + expect(Value.Check(tool.outputSchema!, list.details)).toBe(true); const closed = await tool.execute("close", { action: "close", sessionId }); expect(closed.details).toEqual({ ok: true }); + expect(Value.Check(tool.outputSchema!, closed.details)).toBe(true); expect(backend.killed).toBe(true); }); diff --git a/src/agents/tools/terminal-tool.ts b/src/agents/tools/terminal-tool.ts index f5570ee1ff2..465e6b61f34 100644 --- a/src/agents/tools/terminal-tool.ts +++ b/src/agents/tools/terminal-tool.ts @@ -36,6 +36,35 @@ const TerminalToolSchema = Type.Object( { additionalProperties: false }, ); +const TerminalListSessionSchema = Type.Object( + { + sessionId: Type.String(), + agentId: Type.String(), + shell: Type.String(), + cwd: Type.String(), + attached: Type.Boolean(), + owner: Type.String({ pattern: "^agent:.+" }), + createdAtMs: Type.Integer({ minimum: 0 }), + }, + { additionalProperties: false }, +); + +const TerminalToolOutputSchema = Type.Union([ + Type.Object({ sessions: Type.Array(TerminalListSessionSchema) }, { additionalProperties: false }), + Type.Object( + { + ok: Type.Literal(true), + sessionId: Type.String(), + agentId: Type.String(), + cwd: Type.String(), + shell: Type.String(), + }, + { additionalProperties: false }, + ), + Type.Object({ sessionId: Type.String(), text: Type.String() }, { additionalProperties: false }), + Type.Object({ ok: Type.Boolean() }, { additionalProperties: false }), +]); + type TerminalToolGatewayContext = Pick< GatewayRequestContext, "isTerminalEnabled" | "resolveTerminalLaunchPolicy" | "terminalSessions" @@ -119,6 +148,7 @@ export function createTerminalTool(opts: TerminalToolOptions = {}): AnyAgentTool description: "Own terminal on gateway host. open/read/input/close. User sees it in web UI, can type too. read = buffer snapshot.", parameters: TerminalToolSchema, + outputSchema: TerminalToolOutputSchema, execute: async (_toolCallId, rawArgs, signal) => { const params = rawArgs as Record; const action = readStringParam(params, "action", { required: true });