From 10f5a7ac584181214a70aa0c6c5179fa4dcf7b77 Mon Sep 17 00:00:00 2001 From: Ted Li Date: Sat, 18 Jul 2026 17:00:12 -0700 Subject: [PATCH] fix(groq): keep default Llama agent turns within TPM limit (#104904) * fix(groq): recover oversized requests safely * fix(groq): clear parallel tool aliases on fallback * fix(groq): require provenance-aware host * fix(groq): preserve configured request budgets --------- authored-by: @MonkeyLeeT Co-authored-by: Colin --- .../.generated/plugin-sdk-api-baseline.sha256 | 2 +- extensions/groq/index.test.ts | 299 ++++++++++++++++++ extensions/groq/index.ts | 140 ++++++++ ...e-extension-package-boundary-artifacts.mjs | 1 + .../embedded-agent-runner/extra-params.ts | 1 + .../embedded-agent-runner/model.test.ts | 86 +++++ src/agents/embedded-agent-runner/model.ts | 37 ++- src/agents/sessions/model-registry.test.ts | 38 +++ src/agents/sessions/model-registry.ts | 11 +- src/plugins/provider-runtime-model.types.ts | 2 + 10 files changed, 606 insertions(+), 11 deletions(-) diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index f9bb60287c0..aa09b2ce156 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -18,7 +18,7 @@ ec22d7a039fb58d0b8343ad149322960d3d8ca58b3f4c70f2fa8a099f8186d0c module/agent-h 5f63bf587bf3547d59d0dc5d0dc2fee54745aa6edaab4aa3ae700dba03443edb module/agent-harness-tool-runtime 5168648cd946abad8a92822889f13ceacc87ed502314a66190d0b1eb8ebe76ea module/agent-media-payload 56a1fabbb9edb9bc5de8e777f3cfad5924bfc76cc7d22be50ee53aa2a6947765 module/agent-runtime -d24a44ad9030f4657fa339287ac342724a9c0969f2a390b915c82359345a568c module/agent-sessions +fe45da8617366aec94d419f2e0ad5aea0f9f90ceb006bac9d0589cba2b11f078 module/agent-sessions dd9282f1eeadf44db2887599b52d80db7f5fb99c6d9eac720dbf1b77065f2145 module/allow-from 55cea5390d68839ca7768b4a0cc570b17b65fa0fa3bc4d76130ef0f16cb79ede module/allowlist-config-edit 7ddd81bd5f55de9adf64bf4d92d012f24b37b6da0a72805a3a220d8feff24ca3 module/approval-auth-runtime diff --git a/extensions/groq/index.test.ts b/extensions/groq/index.test.ts index 062449ef063..b30080884e1 100644 --- a/extensions/groq/index.test.ts +++ b/extensions/groq/index.test.ts @@ -1,3 +1,8 @@ +import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; +import { + createAssistantMessageEventStream, + type SimpleStreamOptions, +} from "openclaw/plugin-sdk/llm"; // Groq tests cover index plugin behavior. import { capturePluginRegistration } from "openclaw/plugin-sdk/plugin-test-runtime"; import { describe, expect, it } from "vitest"; @@ -5,6 +10,300 @@ import { resolveGroqReasoningCompatPatch } from "./api.js"; import plugin from "./index.js"; describe("groq provider compat", () => { + it("recovers only matching implicit-budget rejections without changing normal tools", async () => { + const [provider] = capturePluginRegistration(plugin).providers; + + const capturePayloads = async ( + extraParams: Record | undefined, + firstError?: string, + initialMaxTokens = 32_768, + onPayload?: SimpleStreamOptions["onPayload"], + streamMaxTokens?: number, + withTools = true, + modelParams?: Record, + maxTokensSource: "configured" | "discovered" | null = "discovered", + ) => { + const payloads: Array> = []; + const baseStreamFn: StreamFn = async (model, _context, options) => { + const payload: Record = { + max_completion_tokens: initialMaxTokens, + }; + if (withTools) { + payload.tools = [{ type: "function", function: { name: "read" } }]; + payload.tool_choice = "auto"; + } + const replacement = await options?.onPayload?.(payload, model); + const finalPayload = + replacement && typeof replacement === "object" + ? (replacement as Record) + : payload; + payloads.push(finalPayload); + const stream = createAssistantMessageEventStream(); + if (payloads.length === 1 && firstError) { + stream.push({ + type: "error", + reason: "error", + error: { stopReason: "error", errorMessage: firstError } as never, + }); + } else { + stream.push({ + type: "done", + reason: "stop", + message: { stopReason: "stop" } as never, + }); + } + stream.end(); + return stream; + }; + const streamFn = provider?.wrapStreamFn?.({ + provider: "groq", + modelId: "llama-3.3-70b-versatile", + extraParams, + model: { + maxTokens: initialMaxTokens, + ...(maxTokensSource ? { maxTokensSource } : {}), + ...(modelParams ? { params: modelParams } : {}), + } as never, + streamFn: baseStreamFn, + } as never); + const streamOptions: SimpleStreamOptions = {}; + if (onPayload) { + streamOptions.onPayload = onPayload; + } + if (streamMaxTokens !== undefined) { + streamOptions.maxTokens = streamMaxTokens; + } + const stream = streamFn?.( + {} as never, + { messages: [], ...(withTools ? { tools: [{ name: "read" }] } : {}) } as never, + streamOptions, + ); + if (stream) { + const resolvedStream = await stream; + for await (const event of resolvedStream) { + void event; + // Drain the wrapper so a matching error can trigger its fallback attempt. + } + } + return payloads; + }; + + const successfulDefault = await capturePayloads(undefined); + expect(successfulDefault).toHaveLength(1); + expect(successfulDefault[0]).toMatchObject({ + max_completion_tokens: 32_768, + tools: [{ type: "function", function: { name: "read" } }], + }); + + const matchingError = + "413 Request too large for model `llama-3.3-70b-versatile` on tokens per minute (TPM): Limit 12000, Requested 30000"; + let payloadHookCalls = 0; + const recovered = await capturePayloads(undefined, matchingError, 32_768, (payload) => { + const record = payload as Record; + payloadHookCalls += 1; + return payloadHookCalls === 1 + ? payload + : { + ...record, + max_tokens: 65_536, + max_completion_tokens: 32_768, + tools: [{ type: "function", function: { name: "restored" } }], + tool_choice: "required", + }; + }); + expect(payloadHookCalls).toBe(2); + expect(recovered).toHaveLength(2); + expect(recovered[0]).toMatchObject({ + max_completion_tokens: 32_768, + tools: [{ type: "function", function: { name: "read" } }], + }); + expect(recovered[1]).toEqual({ max_completion_tokens: 1_024 }); + + const noTools = await capturePayloads( + undefined, + matchingError, + 32_768, + undefined, + undefined, + false, + ); + expect(noTools).toEqual([{ max_completion_tokens: 32_768 }, { max_completion_tokens: 1_024 }]); + + payloadHookCalls = 0; + const alternateAlias = await capturePayloads(undefined, matchingError, 32_768, (payload) => { + const record = payload as Record; + payloadHookCalls += 1; + return payloadHookCalls === 1 + ? payload + : { + ...record, + max_tokens: 65_536, + tools: [{ type: "function", function: { name: "restored" } }], + }; + }); + expect(alternateAlias[1]).toEqual({ max_completion_tokens: 1_024 }); + + payloadHookCalls = 0; + const parallelToolAliasCollision = await capturePayloads( + undefined, + matchingError, + 32_768, + (payload) => { + const record = payload as Record; + payloadHookCalls += 1; + return payloadHookCalls === 1 + ? payload + : { + ...record, + parallel_tool_calls: true, + parallelToolCalls: false, + }; + }, + ); + expect(payloadHookCalls).toBe(2); + expect(parallelToolAliasCollision[1]).toEqual({ max_completion_tokens: 1_024 }); + + for (const unrelatedMessage of [ + "401 Unauthorized", + "400 malformed request", + "413 context length exceeded", + "429 Rate limit reached on tokens per minute (TPM)", + ]) { + const unrelatedError = await capturePayloads(undefined, unrelatedMessage); + expect(unrelatedError).toHaveLength(1); + expect(unrelatedError[0]).toHaveProperty("tools"); + } + + const explicit = { max_completion_tokens: 2_048 }; + const explicitPayloads = await capturePayloads(explicit, matchingError, 2_048); + expect(explicitPayloads).toHaveLength(1); + expect(explicitPayloads[0]).toMatchObject({ + max_completion_tokens: 2_048, + tools: [{ type: "function", function: { name: "read" } }], + tool_choice: "auto", + }); + + for (const modelParams of [ + { maxTokens: 4_096 }, + { max_completion_tokens: 4_096 }, + { max_tokens: 4_096 }, + { extra_body: { max_completion_tokens: 4_096 } }, + { extra_body: { max_tokens: 4_096 } }, + { extraBody: { max_completion_tokens: 4_096 } }, + { extraBody: { max_tokens: 4_096 } }, + ]) { + const resolvedModel = await capturePayloads( + undefined, + matchingError, + 4_096, + undefined, + undefined, + true, + modelParams, + ); + expect(resolvedModel).toHaveLength(1); + expect(resolvedModel[0]).toHaveProperty("max_completion_tokens", 4_096); + } + + const configuredModelBudget = await capturePayloads( + undefined, + matchingError, + 4_096, + undefined, + undefined, + true, + undefined, + "configured", + ); + expect(configuredModelBudget).toHaveLength(1); + expect(configuredModelBudget[0]).toMatchObject({ + max_completion_tokens: 4_096, + tools: [{ type: "function", function: { name: "read" } }], + }); + + const unknownModelBudget = await capturePayloads( + undefined, + matchingError, + 4_096, + undefined, + undefined, + true, + undefined, + null, + ); + expect(unknownModelBudget).toHaveLength(1); + expect(unknownModelBudget[0]).toMatchObject({ + max_completion_tokens: 4_096, + tools: [{ type: "function", function: { name: "read" } }], + }); + + const requestScoped = await capturePayloads(undefined, matchingError, 4_096, undefined, 4_096); + expect(requestScoped).toHaveLength(1); + expect(requestScoped[0]).toMatchObject({ + max_completion_tokens: 4_096, + tools: [{ type: "function", function: { name: "read" } }], + }); + }); + + it("preserves synchronous throws and complete asynchronous error metadata", async () => { + const [provider] = capturePluginRegistration(plugin).providers; + if (!provider) { + throw new Error("expected Groq provider registration"); + } + const model = { + api: "openai-completions", + provider: "groq", + id: "llama-3.3-70b-versatile", + } as never; + const context = { messages: [], tools: [{ name: "read" }] } as never; + const wrap = (streamFn: StreamFn) => + provider.wrapStreamFn?.({ + provider: "groq", + modelId: "llama-3.3-70b-versatile", + model: { maxTokensSource: "discovered" } as never, + streamFn, + } as never); + + const synchronous = wrap(() => { + throw new Error("synchronous failure"); + }); + expect(() => synchronous?.(model, context, {})).toThrow("synchronous failure"); + + const asynchronous = wrap(async () => { + throw new Error("asynchronous failure"); + }); + const events = []; + const stream = asynchronous?.(model, context, {}); + if (stream) { + const resolvedStream = await stream; + for await (const event of resolvedStream) { + events.push(event); + } + } + expect(events).toEqual([ + { + type: "error", + reason: "error", + error: expect.objectContaining({ + api: "openai-completions", + provider: "groq", + model: "llama-3.3-70b-versatile", + stopReason: "error", + errorMessage: "asynchronous failure", + timestamp: expect.any(Number), + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + }), + }, + ]); + }); + it("maps Groq Qwen 3 reasoning to provider-native none/default values", () => { expect(resolveGroqReasoningCompatPatch("qwen/qwen3-32b")).toEqual({ supportsReasoningEffort: true, diff --git a/extensions/groq/index.ts b/extensions/groq/index.ts index f63c84bbaf8..088f33bbcd6 100644 --- a/extensions/groq/index.ts +++ b/extensions/groq/index.ts @@ -1,9 +1,139 @@ +import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; +import { + createAssistantMessageEventStream, + streamSimple, + type AssistantMessageEvent, +} from "openclaw/plugin-sdk/llm"; // Groq plugin entrypoint registers its OpenClaw integration. import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth-api-key"; import { groqMediaUnderstandingProvider } from "./media-understanding-provider.js"; const GROQ_DEFAULT_MODEL_REF = "groq/llama-3.3-70b-versatile"; +const GROQ_DEFAULT_MODEL_ID = "llama-3.3-70b-versatile"; +const GROQ_FALLBACK_MAX_TOKENS = 1_024; + +function hasWireMaxTokens(value: unknown): boolean { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return false; + } + const record = value as Record; + return record.max_completion_tokens !== undefined || record.max_tokens !== undefined; +} + +function hasExplicitMaxTokens(extraParams: Record | undefined): boolean { + return ( + extraParams?.maxTokens !== undefined || + hasWireMaxTokens(extraParams) || + hasWireMaxTokens(extraParams?.extra_body) || + hasWireMaxTokens(extraParams?.extraBody) + ); +} + +function isGroqTpmRequestTooLargeEvent(event: AssistantMessageEvent): boolean { + if (event.type !== "error") { + return false; + } + const message = event.error.errorMessage?.toLowerCase() ?? ""; + return ( + message.includes("413 request too large for model") && + message.includes("tokens per minute (tpm)") && + message.includes("limit ") && + message.includes("requested ") + ); +} + +function wrapGroqOversizedRequestRecovery( + streamFn: StreamFn | undefined, + enabled: boolean, +): StreamFn { + const underlying = streamFn ?? streamSimple; + if (!enabled) { + return underlying; + } + const withoutTools: StreamFn = (model, context, options) => { + const originalOnPayload = options?.onPayload; + return underlying(model, context, { + ...options, + async onPayload(payload, payloadModel) { + // Run configured payload hooks first so they cannot restore the tools or + // output budget that caused this recovery request's TPM rejection. + const replacement = await originalOnPayload?.(payload, payloadModel); + const finalPayload = replacement && typeof replacement === "object" ? replacement : payload; + const record = finalPayload as Record; + delete record.tools; + delete record.tool_choice; + delete record.parallel_tool_calls; + delete record.parallelToolCalls; + delete record.max_tokens; + delete record.max_completion_tokens; + record.max_completion_tokens = GROQ_FALLBACK_MAX_TOKENS; + return finalPayload; + }, + }); + }; + return (model, context, options) => { + // Request-scoped params wrap provider hooks from the outside, so check the + // final stream options here to preserve explicit per-call output budgets. + if (options?.maxTokens !== undefined) { + return underlying(model, context, options); + } + // Invoke before constructing the proxy stream so synchronous provider failures + // retain the caller-visible throw semantics of the underlying transport. + const initial = underlying(model, context, options); + const output = createAssistantMessageEventStream(); + const writable = output as unknown as { push(event: unknown): void; end(): void }; + + void (async () => { + try { + const resolvedInitial = await Promise.resolve(initial); + let forwarded = false; + let retryWithoutTools = false; + for await (const event of resolvedInitial) { + if (!forwarded && !options?.signal?.aborted && isGroqTpmRequestTooLargeEvent(event)) { + retryWithoutTools = true; + break; + } + writable.push(event); + forwarded = true; + } + if (retryWithoutTools) { + const fallback = await Promise.resolve(withoutTools(model, context, options)); + for await (const event of fallback) { + writable.push(event); + } + } + } catch (error) { + writable.push({ + type: "error", + reason: "error", + error: { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "error", + errorMessage: error instanceof Error ? error.message : String(error), + timestamp: Date.now(), + }, + }); + } finally { + writable.end(); + } + })(); + + return output; + }; +} export default definePluginEntry({ id: "groq", @@ -36,6 +166,16 @@ export default definePluginEntry({ }, }), ], + wrapStreamFn: (ctx) => + wrapGroqOversizedRequestRecovery( + ctx.streamFn, + // Older compatible hosts omit this provenance. Only a known discovered default + // is safe to replace; an unknown value could be a user-configured cap. + ctx.modelId === GROQ_DEFAULT_MODEL_ID && + !hasExplicitMaxTokens(ctx.extraParams) && + !hasExplicitMaxTokens(ctx.model?.params) && + ctx.model?.maxTokensSource === "discovered", + ), }); api.registerMediaUnderstandingProvider(groqMediaUnderstandingProvider); }, diff --git a/scripts/prepare-extension-package-boundary-artifacts.mjs b/scripts/prepare-extension-package-boundary-artifacts.mjs index abd1d9bb9b4..4d8ef3a3a61 100644 --- a/scripts/prepare-extension-package-boundary-artifacts.mjs +++ b/scripts/prepare-extension-package-boundary-artifacts.mjs @@ -81,6 +81,7 @@ function listSourceDtsOutputs({ sourceDir, outputPrefix }) { const PLUGIN_SDK_TYPE_INPUTS = [ "tsconfig.json", "src/plugin-sdk", + "src/plugins/provider-runtime-model.types.ts", "src/plugins/types.ts", "src/auto-reply", "packages/ai/src", diff --git a/src/agents/embedded-agent-runner/extra-params.ts b/src/agents/embedded-agent-runner/extra-params.ts index de256602097..f72b8f459e1 100644 --- a/src/agents/embedded-agent-runner/extra-params.ts +++ b/src/agents/embedded-agent-runner/extra-params.ts @@ -200,6 +200,7 @@ function fingerprintPreparedExtraParamsModel(model?: ProviderRuntimeModel): unkn contextTokens: model.contextTokens ?? null, headers: record.headers ?? null, maxTokens: model.maxTokens, + maxTokensSource: model.maxTokensSource ?? null, params: model.params ?? null, requestTimeoutMs: model.requestTimeoutMs ?? null, }; diff --git a/src/agents/embedded-agent-runner/model.test.ts b/src/agents/embedded-agent-runner/model.test.ts index 2dd22b72563..05d71a00509 100644 --- a/src/agents/embedded-agent-runner/model.test.ts +++ b/src/agents/embedded-agent-runner/model.test.ts @@ -263,6 +263,26 @@ function resolveModelForTest( }); } +function mockDiscoveredGroqModel(maxTokensSource?: "configured" | "discovered") { + mockDiscoveredModel(discoverModels, { + provider: "groq", + modelId: "llama-3.3-70b-versatile", + templateModel: { + provider: "groq", + id: "llama-3.3-70b-versatile", + name: "Llama 3.3 70B", + api: "openai-completions", + baseUrl: "https://api.groq.com/openai/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 131_072, + maxTokens: 32_768, + ...(maxTokensSource ? { maxTokensSource } : {}), + }, + }); +} + function resolveModelAsyncForTest( provider: string, modelId: string, @@ -1396,6 +1416,7 @@ describe("resolveModel", () => { expect(model.baseUrl).toBe("https://token-plan-sgp.xiaomimimo.com/v1"); expect(model.contextWindow).toBe(1_048_576); expect(model.maxTokens).toBe(32_000); + expectRecordFields(model, { maxTokensSource: "discovered" }); expect(resolveBundledStaticCatalogModelMock).toHaveBeenCalledWith({ provider: "xiaomi-token-plan", modelId: "mimo-v2.5-pro", @@ -1405,6 +1426,71 @@ describe("resolveModel", () => { }); }); + it("preserves configured maxTokens provenance from model discovery", () => { + mockDiscoveredGroqModel("configured"); + + const result = resolveModelForTest("groq", "llama-3.3-70b-versatile", "/tmp/agent"); + const model = expectResolvedModel(result); + + expectRecordFields(model, { + maxTokens: 32_768, + maxTokensSource: "configured", + }); + }); + + it("marks a provider-level maxTokens override as configured", () => { + mockDiscoveredGroqModel(); + const cfg = { + models: { + providers: { + groq: { + baseUrl: "https://api.groq.com/openai/v1", + api: "openai-completions", + maxTokens: 2_048, + models: [], + }, + }, + }, + } as unknown as OpenClawConfig; + + const result = resolveModelForTest("groq", "llama-3.3-70b-versatile", "/tmp/agent", cfg); + const model = expectResolvedModel(result); + + expectRecordFields(model, { + maxTokens: 2_048, + maxTokensSource: "configured", + }); + }); + + it("marks a configured-model top-level maxTokens override as configured", () => { + mockDiscoveredGroqModel(); + const cfg = { + models: { + providers: { + groq: { + baseUrl: "https://api.groq.com/openai/v1", + api: "openai-completions", + models: [ + { + id: "llama-3.3-70b-versatile", + name: "Llama 3.3 70B", + maxTokens: 4_096, + }, + ], + }, + }, + }, + } as unknown as OpenClawConfig; + + const result = resolveModelForTest("groq", "llama-3.3-70b-versatile", "/tmp/agent", cfg); + const model = expectResolvedModel(result); + + expectRecordFields(model, { + maxTokens: 4_096, + maxTokensSource: "configured", + }); + }); + it("leaves maxTokens undefined when no configured or catalog value is available (regression: #98295)", () => { // Regression for https://github.com/openclaw/openclaw/issues/98295. // A custom provider entry without maxTokens (and no matching bundled diff --git a/src/agents/embedded-agent-runner/model.ts b/src/agents/embedded-agent-runner/model.ts index a4b719b6087..9ec2e342335 100644 --- a/src/agents/embedded-agent-runner/model.ts +++ b/src/agents/embedded-agent-runner/model.ts @@ -599,6 +599,13 @@ function mergeConfiguredRuntimeModelParams(params: { ); } +function markDiscoveredMaxTokensSource(model: ProviderRuntimeModel): ProviderRuntimeModel { + if (model.maxTokens === undefined || model.maxTokensSource !== undefined) { + return model; + } + return { ...model, maxTokensSource: "discovered" }; +} + function applyConfiguredProviderOverrides(params: { provider: string; discoveredModel: ProviderRuntimeModel; @@ -611,7 +618,8 @@ function applyConfiguredProviderOverrides(params: { preferDiscoveredTransport?: boolean; workspaceDir?: string; }): ProviderRuntimeModel { - const { discoveredModel, providerConfig, modelId } = params; + const { providerConfig, modelId } = params; + const discoveredModel = markDiscoveredMaxTokensSource(params.discoveredModel); const manifestAliasTransport = params.manifestAlias.transport; const requestTimeoutMs = resolveProviderRequestTimeoutMs(providerConfig?.timeoutSeconds); const defaultModelParams = findConfiguredAgentModelParams({ @@ -793,8 +801,8 @@ function applyConfiguredProviderOverrides(params: { }); const resolvedContextWindow = metadataOverrideModel?.contextWindow ?? providerConfig.contextWindow; - const resolvedMaxTokens = - metadataOverrideModel?.maxTokens ?? providerConfig.maxTokens ?? discoveredModel.maxTokens; + const configuredMaxTokens = metadataOverrideModel?.maxTokens ?? providerConfig.maxTokens; + const resolvedMaxTokens = configuredMaxTokens ?? discoveredModel.maxTokens; const normalizedResolvedMaxTokens = typeof resolvedMaxTokens === "number" && Number.isFinite(resolvedMaxTokens) ? typeof resolvedContextWindow === "number" && Number.isFinite(resolvedContextWindow) @@ -846,7 +854,13 @@ function applyConfiguredProviderOverrides(params: { providerConfig.contextTokens ?? discoveredModel.contextTokens, ...(normalizedResolvedMaxTokens !== undefined - ? { maxTokens: normalizedResolvedMaxTokens } + ? { + maxTokens: normalizedResolvedMaxTokens, + maxTokensSource: + configuredMaxTokens !== undefined + ? "configured" + : (discoveredModel.maxTokensSource ?? "discovered"), + } : {}), ...(resolvedParams ? { params: resolvedParams } : {}), ...(requestTimeoutMs !== undefined ? { requestTimeoutMs } : {}), @@ -1056,6 +1070,9 @@ function resolveExplicitModelWithRegistry(params: { workspaceDir, model: { ...fallbackInlineMatch, + ...(fallbackInlineMatch.maxTokens !== undefined + ? { maxTokensSource: "configured" as const } + : {}), reasoning: resolveConfiguredModelReasoning({ provider, compat: fallbackInlineMatch.compat, @@ -1376,11 +1393,11 @@ function resolveConfiguredFallbackModel(params: { compat: fallbackCompat, reasoning: metadataModel?.reasoning, }); - const resolvedFallbackMaxTokens = + const configuredFallbackMaxTokens = configuredModel?.maxTokens ?? providerConfig?.maxTokens ?? - providerConfig?.models?.[0]?.maxTokens ?? - staticCatalogModel?.maxTokens; + providerConfig?.models?.[0]?.maxTokens; + const resolvedFallbackMaxTokens = configuredFallbackMaxTokens ?? staticCatalogModel?.maxTokens; return normalizeResolvedModel({ provider, cfg, @@ -1419,7 +1436,11 @@ function resolveConfiguredFallbackModel(params: { // maxTokens is a wire-level output cap, not a context-budget fallback. // Omit an unknown cap so strict providers can apply their own limit. ...(resolvedFallbackMaxTokens !== undefined - ? { maxTokens: resolvedFallbackMaxTokens } + ? { + maxTokens: resolvedFallbackMaxTokens, + maxTokensSource: + configuredFallbackMaxTokens !== undefined ? "configured" : "discovered", + } : {}), ...(resolvedParams ? { params: resolvedParams } : {}), ...(requestTimeoutMs !== undefined ? { requestTimeoutMs } : {}), diff --git a/src/agents/sessions/model-registry.test.ts b/src/agents/sessions/model-registry.test.ts index df3dd1bb870..da7d3413fd4 100644 --- a/src/agents/sessions/model-registry.test.ts +++ b/src/agents/sessions/model-registry.test.ts @@ -226,6 +226,44 @@ describe("ModelRegistry models.json auth", () => { expect(registry.find("zai", "glm-5.1")?.name).toBe("GLM 5.1"); }); + it("tracks explicit max-token provenance across authored and generated catalogs", () => { + const modelsPath = writeModelsJsonWithPluginCatalog({ + root: { + providers: { + custom: { + baseUrl: "https://models.example/v1", + api: "openai-completions", + models: [{ id: "authored-model", maxTokens: 2_048 }], + }, + }, + }, + pluginRelativePath: join("plugins", "zai", PLUGIN_MODEL_CATALOG_FILE), + pluginCatalog: { + generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, + providers: { + zai: { + baseUrl: "https://api.z.ai/api/paas/v4", + api: "openai-completions", + models: [{ id: "catalog-model", maxTokens: 32_768 }], + }, + }, + }, + }); + + const registry = ModelRegistry.create(AuthStorage.inMemory(), modelsPath, { + pluginMetadataSnapshot: pluginOwnerSnapshot("zai", "zai"), + }); + + expect(registry.find("custom", "authored-model")).toMatchObject({ + maxTokens: 2_048, + maxTokensSource: "configured", + }); + expect(registry.find("zai", "catalog-model")).toMatchObject({ + maxTokens: 32_768, + maxTokensSource: "discovered", + }); + }); + it("preserves response-model temperature compatibility from generated catalogs", () => { const modelsPath = writeModelsJsonWithPluginCatalog({ root: { providers: {} }, diff --git a/src/agents/sessions/model-registry.ts b/src/agents/sessions/model-registry.ts index 3a00df7cc9a..bfeabc84ee1 100644 --- a/src/agents/sessions/model-registry.ts +++ b/src/agents/sessions/model-registry.ts @@ -208,6 +208,7 @@ const ModelsConfigSchema = Type.Object({ const validateModelsConfig = Compile(ModelsConfigSchema); type ModelsConfig = Static; +type MaxTokensSource = "configured" | "discovered"; function formatValidationPath(error: TLocalizedValidationError): string { if (error.keyword === "required") { @@ -451,7 +452,12 @@ export class ModelRegistry { } } - const models = this.parseModels(configForUse); + // Root models.json rows are author-owned; generated plugin shards are + // catalog-owned. Preserve that distinction before runtime resolution. + const models = this.parseModels( + configForUse, + options.requireGeneratedCatalog === true ? "discovered" : "configured", + ); const pluginCatalogErrors: string[] = []; if (options.includePluginCatalogs !== false) { for (const pluginCatalog of listPluginModelCatalogFiles(dirname(modelsJsonPath))) { @@ -522,7 +528,7 @@ export class ModelRegistry { } } - private parseModels(config: ModelsConfig): Model[] { + private parseModels(config: ModelsConfig, maxTokensSource: MaxTokensSource): Model[] { const models: Model[] = []; for (const [providerName, providerConfig] of Object.entries(config.providers)) { @@ -566,6 +572,7 @@ export class ModelRegistry { cost: modelDef.cost ?? defaultCost, contextWindow: modelDef.contextWindow ?? 128000, maxTokens: modelDef.maxTokens ?? 16384, + ...(modelDef.maxTokens !== undefined ? { maxTokensSource } : {}), params: modelDef.params, headers: undefined, compat, diff --git a/src/plugins/provider-runtime-model.types.ts b/src/plugins/provider-runtime-model.types.ts index bb3268b413f..7ec40d61714 100644 --- a/src/plugins/provider-runtime-model.types.ts +++ b/src/plugins/provider-runtime-model.types.ts @@ -9,6 +9,8 @@ import type { ModelCompatConfig, ModelMediaInputConfig } from "../config/types.m export type ProviderRuntimeModel = Omit & { compat?: ModelCompatConfig; contextTokens?: number; + /** Host-resolved provenance for the top-level wire output cap. */ + maxTokensSource?: "configured" | "discovered"; params?: Record; requestTimeoutMs?: number; mediaInput?: ModelMediaInputConfig;