mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 02:06:43 +00:00
fix(groq): keep default Llama agent turns within TPM limit (#104904)
* fix(groq): recover oversized requests safely * fix(groq): clear parallel tool aliases on fallback * fix(groq): require provenance-aware host * fix(groq): preserve configured request budgets --------- authored-by: @MonkeyLeeT Co-authored-by: Colin <colin@solvely.net>
This commit is contained in:
Ted Li
@@ -18,7 +18,7 @@ ec22d7a039fb58d0b8343ad149322960d3d8ca58b3f4c70f2fa8a099f8186d0c module/agent-h
|
||||
5f63bf587bf3547d59d0dc5d0dc2fee54745aa6edaab4aa3ae700dba03443edb module/agent-harness-tool-runtime
|
||||
5168648cd946abad8a92822889f13ceacc87ed502314a66190d0b1eb8ebe76ea module/agent-media-payload
|
||||
56a1fabbb9edb9bc5de8e777f3cfad5924bfc76cc7d22be50ee53aa2a6947765 module/agent-runtime
|
||||
d24a44ad9030f4657fa339287ac342724a9c0969f2a390b915c82359345a568c module/agent-sessions
|
||||
fe45da8617366aec94d419f2e0ad5aea0f9f90ceb006bac9d0589cba2b11f078 module/agent-sessions
|
||||
dd9282f1eeadf44db2887599b52d80db7f5fb99c6d9eac720dbf1b77065f2145 module/allow-from
|
||||
55cea5390d68839ca7768b4a0cc570b17b65fa0fa3bc4d76130ef0f16cb79ede module/allowlist-config-edit
|
||||
7ddd81bd5f55de9adf64bf4d92d012f24b37b6da0a72805a3a220d8feff24ca3 module/approval-auth-runtime
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
import type { StreamFn } from "openclaw/plugin-sdk/agent-core";
|
||||
import {
|
||||
createAssistantMessageEventStream,
|
||||
type SimpleStreamOptions,
|
||||
} from "openclaw/plugin-sdk/llm";
|
||||
// Groq tests cover index plugin behavior.
|
||||
import { capturePluginRegistration } from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import { describe, expect, it } from "vitest";
|
||||
@@ -5,6 +10,300 @@ import { resolveGroqReasoningCompatPatch } from "./api.js";
|
||||
import plugin from "./index.js";
|
||||
|
||||
describe("groq provider compat", () => {
|
||||
it("recovers only matching implicit-budget rejections without changing normal tools", async () => {
|
||||
const [provider] = capturePluginRegistration(plugin).providers;
|
||||
|
||||
const capturePayloads = async (
|
||||
extraParams: Record<string, unknown> | undefined,
|
||||
firstError?: string,
|
||||
initialMaxTokens = 32_768,
|
||||
onPayload?: SimpleStreamOptions["onPayload"],
|
||||
streamMaxTokens?: number,
|
||||
withTools = true,
|
||||
modelParams?: Record<string, unknown>,
|
||||
maxTokensSource: "configured" | "discovered" | null = "discovered",
|
||||
) => {
|
||||
const payloads: Array<Record<string, unknown>> = [];
|
||||
const baseStreamFn: StreamFn = async (model, _context, options) => {
|
||||
const payload: Record<string, unknown> = {
|
||||
max_completion_tokens: initialMaxTokens,
|
||||
};
|
||||
if (withTools) {
|
||||
payload.tools = [{ type: "function", function: { name: "read" } }];
|
||||
payload.tool_choice = "auto";
|
||||
}
|
||||
const replacement = await options?.onPayload?.(payload, model);
|
||||
const finalPayload =
|
||||
replacement && typeof replacement === "object"
|
||||
? (replacement as Record<string, unknown>)
|
||||
: payload;
|
||||
payloads.push(finalPayload);
|
||||
const stream = createAssistantMessageEventStream();
|
||||
if (payloads.length === 1 && firstError) {
|
||||
stream.push({
|
||||
type: "error",
|
||||
reason: "error",
|
||||
error: { stopReason: "error", errorMessage: firstError } as never,
|
||||
});
|
||||
} else {
|
||||
stream.push({
|
||||
type: "done",
|
||||
reason: "stop",
|
||||
message: { stopReason: "stop" } as never,
|
||||
});
|
||||
}
|
||||
stream.end();
|
||||
return stream;
|
||||
};
|
||||
const streamFn = provider?.wrapStreamFn?.({
|
||||
provider: "groq",
|
||||
modelId: "llama-3.3-70b-versatile",
|
||||
extraParams,
|
||||
model: {
|
||||
maxTokens: initialMaxTokens,
|
||||
...(maxTokensSource ? { maxTokensSource } : {}),
|
||||
...(modelParams ? { params: modelParams } : {}),
|
||||
} as never,
|
||||
streamFn: baseStreamFn,
|
||||
} as never);
|
||||
const streamOptions: SimpleStreamOptions = {};
|
||||
if (onPayload) {
|
||||
streamOptions.onPayload = onPayload;
|
||||
}
|
||||
if (streamMaxTokens !== undefined) {
|
||||
streamOptions.maxTokens = streamMaxTokens;
|
||||
}
|
||||
const stream = streamFn?.(
|
||||
{} as never,
|
||||
{ messages: [], ...(withTools ? { tools: [{ name: "read" }] } : {}) } as never,
|
||||
streamOptions,
|
||||
);
|
||||
if (stream) {
|
||||
const resolvedStream = await stream;
|
||||
for await (const event of resolvedStream) {
|
||||
void event;
|
||||
// Drain the wrapper so a matching error can trigger its fallback attempt.
|
||||
}
|
||||
}
|
||||
return payloads;
|
||||
};
|
||||
|
||||
const successfulDefault = await capturePayloads(undefined);
|
||||
expect(successfulDefault).toHaveLength(1);
|
||||
expect(successfulDefault[0]).toMatchObject({
|
||||
max_completion_tokens: 32_768,
|
||||
tools: [{ type: "function", function: { name: "read" } }],
|
||||
});
|
||||
|
||||
const matchingError =
|
||||
"413 Request too large for model `llama-3.3-70b-versatile` on tokens per minute (TPM): Limit 12000, Requested 30000";
|
||||
let payloadHookCalls = 0;
|
||||
const recovered = await capturePayloads(undefined, matchingError, 32_768, (payload) => {
|
||||
const record = payload as Record<string, unknown>;
|
||||
payloadHookCalls += 1;
|
||||
return payloadHookCalls === 1
|
||||
? payload
|
||||
: {
|
||||
...record,
|
||||
max_tokens: 65_536,
|
||||
max_completion_tokens: 32_768,
|
||||
tools: [{ type: "function", function: { name: "restored" } }],
|
||||
tool_choice: "required",
|
||||
};
|
||||
});
|
||||
expect(payloadHookCalls).toBe(2);
|
||||
expect(recovered).toHaveLength(2);
|
||||
expect(recovered[0]).toMatchObject({
|
||||
max_completion_tokens: 32_768,
|
||||
tools: [{ type: "function", function: { name: "read" } }],
|
||||
});
|
||||
expect(recovered[1]).toEqual({ max_completion_tokens: 1_024 });
|
||||
|
||||
const noTools = await capturePayloads(
|
||||
undefined,
|
||||
matchingError,
|
||||
32_768,
|
||||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
);
|
||||
expect(noTools).toEqual([{ max_completion_tokens: 32_768 }, { max_completion_tokens: 1_024 }]);
|
||||
|
||||
payloadHookCalls = 0;
|
||||
const alternateAlias = await capturePayloads(undefined, matchingError, 32_768, (payload) => {
|
||||
const record = payload as Record<string, unknown>;
|
||||
payloadHookCalls += 1;
|
||||
return payloadHookCalls === 1
|
||||
? payload
|
||||
: {
|
||||
...record,
|
||||
max_tokens: 65_536,
|
||||
tools: [{ type: "function", function: { name: "restored" } }],
|
||||
};
|
||||
});
|
||||
expect(alternateAlias[1]).toEqual({ max_completion_tokens: 1_024 });
|
||||
|
||||
payloadHookCalls = 0;
|
||||
const parallelToolAliasCollision = await capturePayloads(
|
||||
undefined,
|
||||
matchingError,
|
||||
32_768,
|
||||
(payload) => {
|
||||
const record = payload as Record<string, unknown>;
|
||||
payloadHookCalls += 1;
|
||||
return payloadHookCalls === 1
|
||||
? payload
|
||||
: {
|
||||
...record,
|
||||
parallel_tool_calls: true,
|
||||
parallelToolCalls: false,
|
||||
};
|
||||
},
|
||||
);
|
||||
expect(payloadHookCalls).toBe(2);
|
||||
expect(parallelToolAliasCollision[1]).toEqual({ max_completion_tokens: 1_024 });
|
||||
|
||||
for (const unrelatedMessage of [
|
||||
"401 Unauthorized",
|
||||
"400 malformed request",
|
||||
"413 context length exceeded",
|
||||
"429 Rate limit reached on tokens per minute (TPM)",
|
||||
]) {
|
||||
const unrelatedError = await capturePayloads(undefined, unrelatedMessage);
|
||||
expect(unrelatedError).toHaveLength(1);
|
||||
expect(unrelatedError[0]).toHaveProperty("tools");
|
||||
}
|
||||
|
||||
const explicit = { max_completion_tokens: 2_048 };
|
||||
const explicitPayloads = await capturePayloads(explicit, matchingError, 2_048);
|
||||
expect(explicitPayloads).toHaveLength(1);
|
||||
expect(explicitPayloads[0]).toMatchObject({
|
||||
max_completion_tokens: 2_048,
|
||||
tools: [{ type: "function", function: { name: "read" } }],
|
||||
tool_choice: "auto",
|
||||
});
|
||||
|
||||
for (const modelParams of [
|
||||
{ maxTokens: 4_096 },
|
||||
{ max_completion_tokens: 4_096 },
|
||||
{ max_tokens: 4_096 },
|
||||
{ extra_body: { max_completion_tokens: 4_096 } },
|
||||
{ extra_body: { max_tokens: 4_096 } },
|
||||
{ extraBody: { max_completion_tokens: 4_096 } },
|
||||
{ extraBody: { max_tokens: 4_096 } },
|
||||
]) {
|
||||
const resolvedModel = await capturePayloads(
|
||||
undefined,
|
||||
matchingError,
|
||||
4_096,
|
||||
undefined,
|
||||
undefined,
|
||||
true,
|
||||
modelParams,
|
||||
);
|
||||
expect(resolvedModel).toHaveLength(1);
|
||||
expect(resolvedModel[0]).toHaveProperty("max_completion_tokens", 4_096);
|
||||
}
|
||||
|
||||
const configuredModelBudget = await capturePayloads(
|
||||
undefined,
|
||||
matchingError,
|
||||
4_096,
|
||||
undefined,
|
||||
undefined,
|
||||
true,
|
||||
undefined,
|
||||
"configured",
|
||||
);
|
||||
expect(configuredModelBudget).toHaveLength(1);
|
||||
expect(configuredModelBudget[0]).toMatchObject({
|
||||
max_completion_tokens: 4_096,
|
||||
tools: [{ type: "function", function: { name: "read" } }],
|
||||
});
|
||||
|
||||
const unknownModelBudget = await capturePayloads(
|
||||
undefined,
|
||||
matchingError,
|
||||
4_096,
|
||||
undefined,
|
||||
undefined,
|
||||
true,
|
||||
undefined,
|
||||
null,
|
||||
);
|
||||
expect(unknownModelBudget).toHaveLength(1);
|
||||
expect(unknownModelBudget[0]).toMatchObject({
|
||||
max_completion_tokens: 4_096,
|
||||
tools: [{ type: "function", function: { name: "read" } }],
|
||||
});
|
||||
|
||||
const requestScoped = await capturePayloads(undefined, matchingError, 4_096, undefined, 4_096);
|
||||
expect(requestScoped).toHaveLength(1);
|
||||
expect(requestScoped[0]).toMatchObject({
|
||||
max_completion_tokens: 4_096,
|
||||
tools: [{ type: "function", function: { name: "read" } }],
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves synchronous throws and complete asynchronous error metadata", async () => {
|
||||
const [provider] = capturePluginRegistration(plugin).providers;
|
||||
if (!provider) {
|
||||
throw new Error("expected Groq provider registration");
|
||||
}
|
||||
const model = {
|
||||
api: "openai-completions",
|
||||
provider: "groq",
|
||||
id: "llama-3.3-70b-versatile",
|
||||
} as never;
|
||||
const context = { messages: [], tools: [{ name: "read" }] } as never;
|
||||
const wrap = (streamFn: StreamFn) =>
|
||||
provider.wrapStreamFn?.({
|
||||
provider: "groq",
|
||||
modelId: "llama-3.3-70b-versatile",
|
||||
model: { maxTokensSource: "discovered" } as never,
|
||||
streamFn,
|
||||
} as never);
|
||||
|
||||
const synchronous = wrap(() => {
|
||||
throw new Error("synchronous failure");
|
||||
});
|
||||
expect(() => synchronous?.(model, context, {})).toThrow("synchronous failure");
|
||||
|
||||
const asynchronous = wrap(async () => {
|
||||
throw new Error("asynchronous failure");
|
||||
});
|
||||
const events = [];
|
||||
const stream = asynchronous?.(model, context, {});
|
||||
if (stream) {
|
||||
const resolvedStream = await stream;
|
||||
for await (const event of resolvedStream) {
|
||||
events.push(event);
|
||||
}
|
||||
}
|
||||
expect(events).toEqual([
|
||||
{
|
||||
type: "error",
|
||||
reason: "error",
|
||||
error: expect.objectContaining({
|
||||
api: "openai-completions",
|
||||
provider: "groq",
|
||||
model: "llama-3.3-70b-versatile",
|
||||
stopReason: "error",
|
||||
errorMessage: "asynchronous failure",
|
||||
timestamp: expect.any(Number),
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
}),
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("maps Groq Qwen 3 reasoning to provider-native none/default values", () => {
|
||||
expect(resolveGroqReasoningCompatPatch("qwen/qwen3-32b")).toEqual({
|
||||
supportsReasoningEffort: true,
|
||||
|
||||
@@ -1,9 +1,139 @@
|
||||
import type { StreamFn } from "openclaw/plugin-sdk/agent-core";
|
||||
import {
|
||||
createAssistantMessageEventStream,
|
||||
streamSimple,
|
||||
type AssistantMessageEvent,
|
||||
} from "openclaw/plugin-sdk/llm";
|
||||
// Groq plugin entrypoint registers its OpenClaw integration.
|
||||
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth-api-key";
|
||||
import { groqMediaUnderstandingProvider } from "./media-understanding-provider.js";
|
||||
|
||||
const GROQ_DEFAULT_MODEL_REF = "groq/llama-3.3-70b-versatile";
|
||||
const GROQ_DEFAULT_MODEL_ID = "llama-3.3-70b-versatile";
|
||||
const GROQ_FALLBACK_MAX_TOKENS = 1_024;
|
||||
|
||||
function hasWireMaxTokens(value: unknown): boolean {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
||||
return false;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
return record.max_completion_tokens !== undefined || record.max_tokens !== undefined;
|
||||
}
|
||||
|
||||
function hasExplicitMaxTokens(extraParams: Record<string, unknown> | undefined): boolean {
|
||||
return (
|
||||
extraParams?.maxTokens !== undefined ||
|
||||
hasWireMaxTokens(extraParams) ||
|
||||
hasWireMaxTokens(extraParams?.extra_body) ||
|
||||
hasWireMaxTokens(extraParams?.extraBody)
|
||||
);
|
||||
}
|
||||
|
||||
function isGroqTpmRequestTooLargeEvent(event: AssistantMessageEvent): boolean {
|
||||
if (event.type !== "error") {
|
||||
return false;
|
||||
}
|
||||
const message = event.error.errorMessage?.toLowerCase() ?? "";
|
||||
return (
|
||||
message.includes("413 request too large for model") &&
|
||||
message.includes("tokens per minute (tpm)") &&
|
||||
message.includes("limit ") &&
|
||||
message.includes("requested ")
|
||||
);
|
||||
}
|
||||
|
||||
function wrapGroqOversizedRequestRecovery(
|
||||
streamFn: StreamFn | undefined,
|
||||
enabled: boolean,
|
||||
): StreamFn {
|
||||
const underlying = streamFn ?? streamSimple;
|
||||
if (!enabled) {
|
||||
return underlying;
|
||||
}
|
||||
const withoutTools: StreamFn = (model, context, options) => {
|
||||
const originalOnPayload = options?.onPayload;
|
||||
return underlying(model, context, {
|
||||
...options,
|
||||
async onPayload(payload, payloadModel) {
|
||||
// Run configured payload hooks first so they cannot restore the tools or
|
||||
// output budget that caused this recovery request's TPM rejection.
|
||||
const replacement = await originalOnPayload?.(payload, payloadModel);
|
||||
const finalPayload = replacement && typeof replacement === "object" ? replacement : payload;
|
||||
const record = finalPayload as Record<string, unknown>;
|
||||
delete record.tools;
|
||||
delete record.tool_choice;
|
||||
delete record.parallel_tool_calls;
|
||||
delete record.parallelToolCalls;
|
||||
delete record.max_tokens;
|
||||
delete record.max_completion_tokens;
|
||||
record.max_completion_tokens = GROQ_FALLBACK_MAX_TOKENS;
|
||||
return finalPayload;
|
||||
},
|
||||
});
|
||||
};
|
||||
return (model, context, options) => {
|
||||
// Request-scoped params wrap provider hooks from the outside, so check the
|
||||
// final stream options here to preserve explicit per-call output budgets.
|
||||
if (options?.maxTokens !== undefined) {
|
||||
return underlying(model, context, options);
|
||||
}
|
||||
// Invoke before constructing the proxy stream so synchronous provider failures
|
||||
// retain the caller-visible throw semantics of the underlying transport.
|
||||
const initial = underlying(model, context, options);
|
||||
const output = createAssistantMessageEventStream();
|
||||
const writable = output as unknown as { push(event: unknown): void; end(): void };
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const resolvedInitial = await Promise.resolve(initial);
|
||||
let forwarded = false;
|
||||
let retryWithoutTools = false;
|
||||
for await (const event of resolvedInitial) {
|
||||
if (!forwarded && !options?.signal?.aborted && isGroqTpmRequestTooLargeEvent(event)) {
|
||||
retryWithoutTools = true;
|
||||
break;
|
||||
}
|
||||
writable.push(event);
|
||||
forwarded = true;
|
||||
}
|
||||
if (retryWithoutTools) {
|
||||
const fallback = await Promise.resolve(withoutTools(model, context, options));
|
||||
for await (const event of fallback) {
|
||||
writable.push(event);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
writable.push({
|
||||
type: "error",
|
||||
reason: "error",
|
||||
error: {
|
||||
role: "assistant",
|
||||
content: [],
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "error",
|
||||
errorMessage: error instanceof Error ? error.message : String(error),
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
writable.end();
|
||||
}
|
||||
})();
|
||||
|
||||
return output;
|
||||
};
|
||||
}
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "groq",
|
||||
@@ -36,6 +166,16 @@ export default definePluginEntry({
|
||||
},
|
||||
}),
|
||||
],
|
||||
wrapStreamFn: (ctx) =>
|
||||
wrapGroqOversizedRequestRecovery(
|
||||
ctx.streamFn,
|
||||
// Older compatible hosts omit this provenance. Only a known discovered default
|
||||
// is safe to replace; an unknown value could be a user-configured cap.
|
||||
ctx.modelId === GROQ_DEFAULT_MODEL_ID &&
|
||||
!hasExplicitMaxTokens(ctx.extraParams) &&
|
||||
!hasExplicitMaxTokens(ctx.model?.params) &&
|
||||
ctx.model?.maxTokensSource === "discovered",
|
||||
),
|
||||
});
|
||||
api.registerMediaUnderstandingProvider(groqMediaUnderstandingProvider);
|
||||
},
|
||||
|
||||
@@ -81,6 +81,7 @@ function listSourceDtsOutputs({ sourceDir, outputPrefix }) {
|
||||
const PLUGIN_SDK_TYPE_INPUTS = [
|
||||
"tsconfig.json",
|
||||
"src/plugin-sdk",
|
||||
"src/plugins/provider-runtime-model.types.ts",
|
||||
"src/plugins/types.ts",
|
||||
"src/auto-reply",
|
||||
"packages/ai/src",
|
||||
|
||||
@@ -200,6 +200,7 @@ function fingerprintPreparedExtraParamsModel(model?: ProviderRuntimeModel): unkn
|
||||
contextTokens: model.contextTokens ?? null,
|
||||
headers: record.headers ?? null,
|
||||
maxTokens: model.maxTokens,
|
||||
maxTokensSource: model.maxTokensSource ?? null,
|
||||
params: model.params ?? null,
|
||||
requestTimeoutMs: model.requestTimeoutMs ?? null,
|
||||
};
|
||||
|
||||
@@ -263,6 +263,26 @@ function resolveModelForTest(
|
||||
});
|
||||
}
|
||||
|
||||
function mockDiscoveredGroqModel(maxTokensSource?: "configured" | "discovered") {
|
||||
mockDiscoveredModel(discoverModels, {
|
||||
provider: "groq",
|
||||
modelId: "llama-3.3-70b-versatile",
|
||||
templateModel: {
|
||||
provider: "groq",
|
||||
id: "llama-3.3-70b-versatile",
|
||||
name: "Llama 3.3 70B",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://api.groq.com/openai/v1",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 131_072,
|
||||
maxTokens: 32_768,
|
||||
...(maxTokensSource ? { maxTokensSource } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function resolveModelAsyncForTest(
|
||||
provider: string,
|
||||
modelId: string,
|
||||
@@ -1396,6 +1416,7 @@ describe("resolveModel", () => {
|
||||
expect(model.baseUrl).toBe("https://token-plan-sgp.xiaomimimo.com/v1");
|
||||
expect(model.contextWindow).toBe(1_048_576);
|
||||
expect(model.maxTokens).toBe(32_000);
|
||||
expectRecordFields(model, { maxTokensSource: "discovered" });
|
||||
expect(resolveBundledStaticCatalogModelMock).toHaveBeenCalledWith({
|
||||
provider: "xiaomi-token-plan",
|
||||
modelId: "mimo-v2.5-pro",
|
||||
@@ -1405,6 +1426,71 @@ describe("resolveModel", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves configured maxTokens provenance from model discovery", () => {
|
||||
mockDiscoveredGroqModel("configured");
|
||||
|
||||
const result = resolveModelForTest("groq", "llama-3.3-70b-versatile", "/tmp/agent");
|
||||
const model = expectResolvedModel(result);
|
||||
|
||||
expectRecordFields(model, {
|
||||
maxTokens: 32_768,
|
||||
maxTokensSource: "configured",
|
||||
});
|
||||
});
|
||||
|
||||
it("marks a provider-level maxTokens override as configured", () => {
|
||||
mockDiscoveredGroqModel();
|
||||
const cfg = {
|
||||
models: {
|
||||
providers: {
|
||||
groq: {
|
||||
baseUrl: "https://api.groq.com/openai/v1",
|
||||
api: "openai-completions",
|
||||
maxTokens: 2_048,
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
|
||||
const result = resolveModelForTest("groq", "llama-3.3-70b-versatile", "/tmp/agent", cfg);
|
||||
const model = expectResolvedModel(result);
|
||||
|
||||
expectRecordFields(model, {
|
||||
maxTokens: 2_048,
|
||||
maxTokensSource: "configured",
|
||||
});
|
||||
});
|
||||
|
||||
it("marks a configured-model top-level maxTokens override as configured", () => {
|
||||
mockDiscoveredGroqModel();
|
||||
const cfg = {
|
||||
models: {
|
||||
providers: {
|
||||
groq: {
|
||||
baseUrl: "https://api.groq.com/openai/v1",
|
||||
api: "openai-completions",
|
||||
models: [
|
||||
{
|
||||
id: "llama-3.3-70b-versatile",
|
||||
name: "Llama 3.3 70B",
|
||||
maxTokens: 4_096,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
|
||||
const result = resolveModelForTest("groq", "llama-3.3-70b-versatile", "/tmp/agent", cfg);
|
||||
const model = expectResolvedModel(result);
|
||||
|
||||
expectRecordFields(model, {
|
||||
maxTokens: 4_096,
|
||||
maxTokensSource: "configured",
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves maxTokens undefined when no configured or catalog value is available (regression: #98295)", () => {
|
||||
// Regression for https://github.com/openclaw/openclaw/issues/98295.
|
||||
// A custom provider entry without maxTokens (and no matching bundled
|
||||
|
||||
@@ -599,6 +599,13 @@ function mergeConfiguredRuntimeModelParams(params: {
|
||||
);
|
||||
}
|
||||
|
||||
function markDiscoveredMaxTokensSource(model: ProviderRuntimeModel): ProviderRuntimeModel {
|
||||
if (model.maxTokens === undefined || model.maxTokensSource !== undefined) {
|
||||
return model;
|
||||
}
|
||||
return { ...model, maxTokensSource: "discovered" };
|
||||
}
|
||||
|
||||
function applyConfiguredProviderOverrides(params: {
|
||||
provider: string;
|
||||
discoveredModel: ProviderRuntimeModel;
|
||||
@@ -611,7 +618,8 @@ function applyConfiguredProviderOverrides(params: {
|
||||
preferDiscoveredTransport?: boolean;
|
||||
workspaceDir?: string;
|
||||
}): ProviderRuntimeModel {
|
||||
const { discoveredModel, providerConfig, modelId } = params;
|
||||
const { providerConfig, modelId } = params;
|
||||
const discoveredModel = markDiscoveredMaxTokensSource(params.discoveredModel);
|
||||
const manifestAliasTransport = params.manifestAlias.transport;
|
||||
const requestTimeoutMs = resolveProviderRequestTimeoutMs(providerConfig?.timeoutSeconds);
|
||||
const defaultModelParams = findConfiguredAgentModelParams({
|
||||
@@ -793,8 +801,8 @@ function applyConfiguredProviderOverrides(params: {
|
||||
});
|
||||
const resolvedContextWindow =
|
||||
metadataOverrideModel?.contextWindow ?? providerConfig.contextWindow;
|
||||
const resolvedMaxTokens =
|
||||
metadataOverrideModel?.maxTokens ?? providerConfig.maxTokens ?? discoveredModel.maxTokens;
|
||||
const configuredMaxTokens = metadataOverrideModel?.maxTokens ?? providerConfig.maxTokens;
|
||||
const resolvedMaxTokens = configuredMaxTokens ?? discoveredModel.maxTokens;
|
||||
const normalizedResolvedMaxTokens =
|
||||
typeof resolvedMaxTokens === "number" && Number.isFinite(resolvedMaxTokens)
|
||||
? typeof resolvedContextWindow === "number" && Number.isFinite(resolvedContextWindow)
|
||||
@@ -846,7 +854,13 @@ function applyConfiguredProviderOverrides(params: {
|
||||
providerConfig.contextTokens ??
|
||||
discoveredModel.contextTokens,
|
||||
...(normalizedResolvedMaxTokens !== undefined
|
||||
? { maxTokens: normalizedResolvedMaxTokens }
|
||||
? {
|
||||
maxTokens: normalizedResolvedMaxTokens,
|
||||
maxTokensSource:
|
||||
configuredMaxTokens !== undefined
|
||||
? "configured"
|
||||
: (discoveredModel.maxTokensSource ?? "discovered"),
|
||||
}
|
||||
: {}),
|
||||
...(resolvedParams ? { params: resolvedParams } : {}),
|
||||
...(requestTimeoutMs !== undefined ? { requestTimeoutMs } : {}),
|
||||
@@ -1056,6 +1070,9 @@ function resolveExplicitModelWithRegistry(params: {
|
||||
workspaceDir,
|
||||
model: {
|
||||
...fallbackInlineMatch,
|
||||
...(fallbackInlineMatch.maxTokens !== undefined
|
||||
? { maxTokensSource: "configured" as const }
|
||||
: {}),
|
||||
reasoning: resolveConfiguredModelReasoning({
|
||||
provider,
|
||||
compat: fallbackInlineMatch.compat,
|
||||
@@ -1376,11 +1393,11 @@ function resolveConfiguredFallbackModel(params: {
|
||||
compat: fallbackCompat,
|
||||
reasoning: metadataModel?.reasoning,
|
||||
});
|
||||
const resolvedFallbackMaxTokens =
|
||||
const configuredFallbackMaxTokens =
|
||||
configuredModel?.maxTokens ??
|
||||
providerConfig?.maxTokens ??
|
||||
providerConfig?.models?.[0]?.maxTokens ??
|
||||
staticCatalogModel?.maxTokens;
|
||||
providerConfig?.models?.[0]?.maxTokens;
|
||||
const resolvedFallbackMaxTokens = configuredFallbackMaxTokens ?? staticCatalogModel?.maxTokens;
|
||||
return normalizeResolvedModel({
|
||||
provider,
|
||||
cfg,
|
||||
@@ -1419,7 +1436,11 @@ function resolveConfiguredFallbackModel(params: {
|
||||
// maxTokens is a wire-level output cap, not a context-budget fallback.
|
||||
// Omit an unknown cap so strict providers can apply their own limit.
|
||||
...(resolvedFallbackMaxTokens !== undefined
|
||||
? { maxTokens: resolvedFallbackMaxTokens }
|
||||
? {
|
||||
maxTokens: resolvedFallbackMaxTokens,
|
||||
maxTokensSource:
|
||||
configuredFallbackMaxTokens !== undefined ? "configured" : "discovered",
|
||||
}
|
||||
: {}),
|
||||
...(resolvedParams ? { params: resolvedParams } : {}),
|
||||
...(requestTimeoutMs !== undefined ? { requestTimeoutMs } : {}),
|
||||
|
||||
@@ -226,6 +226,44 @@ describe("ModelRegistry models.json auth", () => {
|
||||
expect(registry.find("zai", "glm-5.1")?.name).toBe("GLM 5.1");
|
||||
});
|
||||
|
||||
it("tracks explicit max-token provenance across authored and generated catalogs", () => {
|
||||
const modelsPath = writeModelsJsonWithPluginCatalog({
|
||||
root: {
|
||||
providers: {
|
||||
custom: {
|
||||
baseUrl: "https://models.example/v1",
|
||||
api: "openai-completions",
|
||||
models: [{ id: "authored-model", maxTokens: 2_048 }],
|
||||
},
|
||||
},
|
||||
},
|
||||
pluginRelativePath: join("plugins", "zai", PLUGIN_MODEL_CATALOG_FILE),
|
||||
pluginCatalog: {
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {
|
||||
zai: {
|
||||
baseUrl: "https://api.z.ai/api/paas/v4",
|
||||
api: "openai-completions",
|
||||
models: [{ id: "catalog-model", maxTokens: 32_768 }],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const registry = ModelRegistry.create(AuthStorage.inMemory(), modelsPath, {
|
||||
pluginMetadataSnapshot: pluginOwnerSnapshot("zai", "zai"),
|
||||
});
|
||||
|
||||
expect(registry.find("custom", "authored-model")).toMatchObject({
|
||||
maxTokens: 2_048,
|
||||
maxTokensSource: "configured",
|
||||
});
|
||||
expect(registry.find("zai", "catalog-model")).toMatchObject({
|
||||
maxTokens: 32_768,
|
||||
maxTokensSource: "discovered",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves response-model temperature compatibility from generated catalogs", () => {
|
||||
const modelsPath = writeModelsJsonWithPluginCatalog({
|
||||
root: { providers: {} },
|
||||
|
||||
@@ -208,6 +208,7 @@ const ModelsConfigSchema = Type.Object({
|
||||
const validateModelsConfig = Compile(ModelsConfigSchema);
|
||||
|
||||
type ModelsConfig = Static<typeof ModelsConfigSchema>;
|
||||
type MaxTokensSource = "configured" | "discovered";
|
||||
|
||||
function formatValidationPath(error: TLocalizedValidationError): string {
|
||||
if (error.keyword === "required") {
|
||||
@@ -451,7 +452,12 @@ export class ModelRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
const models = this.parseModels(configForUse);
|
||||
// Root models.json rows are author-owned; generated plugin shards are
|
||||
// catalog-owned. Preserve that distinction before runtime resolution.
|
||||
const models = this.parseModels(
|
||||
configForUse,
|
||||
options.requireGeneratedCatalog === true ? "discovered" : "configured",
|
||||
);
|
||||
const pluginCatalogErrors: string[] = [];
|
||||
if (options.includePluginCatalogs !== false) {
|
||||
for (const pluginCatalog of listPluginModelCatalogFiles(dirname(modelsJsonPath))) {
|
||||
@@ -522,7 +528,7 @@ export class ModelRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
private parseModels(config: ModelsConfig): Model[] {
|
||||
private parseModels(config: ModelsConfig, maxTokensSource: MaxTokensSource): Model[] {
|
||||
const models: Model[] = [];
|
||||
|
||||
for (const [providerName, providerConfig] of Object.entries(config.providers)) {
|
||||
@@ -566,6 +572,7 @@ export class ModelRegistry {
|
||||
cost: modelDef.cost ?? defaultCost,
|
||||
contextWindow: modelDef.contextWindow ?? 128000,
|
||||
maxTokens: modelDef.maxTokens ?? 16384,
|
||||
...(modelDef.maxTokens !== undefined ? { maxTokensSource } : {}),
|
||||
params: modelDef.params,
|
||||
headers: undefined,
|
||||
compat,
|
||||
|
||||
@@ -9,6 +9,8 @@ import type { ModelCompatConfig, ModelMediaInputConfig } from "../config/types.m
|
||||
export type ProviderRuntimeModel = Omit<Model, "compat"> & {
|
||||
compat?: ModelCompatConfig;
|
||||
contextTokens?: number;
|
||||
/** Host-resolved provenance for the top-level wire output cap. */
|
||||
maxTokensSource?: "configured" | "discovered";
|
||||
params?: Record<string, unknown>;
|
||||
requestTimeoutMs?: number;
|
||||
mediaInput?: ModelMediaInputConfig;
|
||||
|
||||
Reference in New Issue
Block a user