fix(agents): classify embedded provider business denials for fallback (#84814)

Summary:
- The PR classifies selected embedded agent provider-denial error payloads through the shared failover matcher ... 1/current-ak auth matching, preserves guarded non-fallback cases, and covers fallback progression in tests.
- PR surface: Source +34, Tests +166. Total +200 across 5 files.
- Reproducibility: yes. Current main is source-reproducible: a non-GPT embedded result whose only signal is CE ... returns null from the classifier, and the fallback wrapper treats null classification as candidate success.

Automerge notes:
- PR branch already contained follow-up commit before automerge: fix(agents): classify embedded provider business denials for fallback
- PR branch already contained follow-up commit before automerge: fix(clawsweeper): address review for automerge-openclaw-openclaw-8304…

Validation:
- ClawSweeper review passed for head e266beac93.
- Required merge gates passed before the squash merge.

Prepared head SHA: e266beac93
Review: https://github.com/openclaw/openclaw/pull/84814#issuecomment-4505010446

Co-authored-by: Stellar鱼 <2182712990@qq.com>
Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com>
Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com>
Approved-by: takhoffman
Co-authored-by: takhoffman <781889+takhoffman@users.noreply.github.com>
This commit is contained in:
clawsweeper[bot]
2026-05-30 00:34:28 +00:00
committed by GitHub
co-authored by Stellar鱼 clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com> takhoffman
parent aada44fca5
commit 18f94fc83a
5 changed files with 200 additions and 0 deletions
@@ -1412,6 +1412,8 @@ describe("classifyFailoverReason provider messages", () => {
// Auth errors
expect(classifyFailoverReason("无权访问该模型")).toBe("auth");
expect(classifyFailoverReason("403 您无权访问glm-5.1。")).toBe("auth");
expect(classifyFailoverReason("当前ak因违规请求被禁止访问该模型")).toBe("auth");
expect(classifyFailoverReason('{"success":false,"code":"CE-011"}')).toBe("auth");
expect(classifyFailoverReason("认证失败")).toBe("auth");
expect(classifyFailoverReason("鉴权失败,请检查API Key")).toBe("auth");
expect(classifyFailoverReason("密钥无效")).toBe("auth");
@@ -47,6 +47,8 @@ const CJK_AUTH_ERROR_PATTERNS = [
"鉴权失败",
"密钥无效",
"apikey 无效",
/(?:当前\s*ak|ce-011).*?(?:违规请求|禁止访问)|(?:违规请求|禁止访问).*?(?:当前\s*ak|ce-011)/i,
/\bce-011\b/i,
] as const satisfies readonly ErrorPattern[];
const ZAI_BILLING_CODE_1311_RE = /"code"\s*:\s*1311\b/;
@@ -19,4 +19,119 @@ describe("classifyEmbeddedAgentRunResultForModelFallback", () => {
}),
).toBeNull();
});
it("classifies provider business-denial error payloads as fallback-worthy", () => {
const result = classifyEmbeddedAgentRunResultForModelFallback({
provider: "zai",
model: "glm-5.1",
result: {
payloads: [
{
isError: true,
text: '{"success":false,"code":"CE-011","message":"当前ak因违规请求被禁止访问该模型"}',
},
],
meta: {
durationMs: 42,
},
},
});
expect(result).toEqual({
message:
'zai/glm-5.1 ended with a provider error: {"success":false,"code":"CE-011","message":"当前ak因违规请求被禁止访问该模型"}',
reason: "auth",
code: "embedded_error_payload",
rawError: '{"success":false,"code":"CE-011","message":"当前ak因违规请求被禁止访问该模型"}',
});
});
it("preserves hook block results with auth-like error payload text", () => {
const result = classifyEmbeddedAgentRunResultForModelFallback({
provider: "custom",
model: "gpt-5.5",
result: {
payloads: [
{
isError: true,
text: "Access denied by policy",
},
],
meta: {
durationMs: 42,
error: {
kind: "hook_block",
message: "Access denied by policy",
},
},
},
});
expect(result).toBeNull();
});
it("uses provider-scoped failover matching for business-denial payloads", () => {
const result = classifyEmbeddedAgentRunResultForModelFallback({
provider: "openrouter",
model: "claude-3.5-sonnet",
result: {
payloads: [
{
isError: true,
text: "Key limit exceeded",
},
],
meta: {
durationMs: 42,
},
},
});
expect(result).toEqual({
message: "openrouter/claude-3.5-sonnet ended with a provider error: Key limit exceeded",
reason: "billing",
code: "embedded_error_payload",
rawError: "Key limit exceeded",
});
});
it("does not retry unclassified non-GPT error payloads", () => {
const result = classifyEmbeddedAgentRunResultForModelFallback({
provider: "custom",
model: "llama-3.1",
result: {
payloads: [
{
isError: true,
text: "the model produced an application-level error",
},
],
meta: {
durationMs: 42,
},
},
});
expect(result).toBeNull();
});
it("does not retry non-business transport error payloads", () => {
const result = classifyEmbeddedAgentRunResultForModelFallback({
provider: "custom",
model: "llama-3.1",
result: {
payloads: [
{
isError: true,
text: "HTTP 500: internal server error",
},
],
meta: {
durationMs: 42,
},
},
});
expect(result).toBeNull();
});
});
@@ -1,4 +1,6 @@
import { isSilentReplyPayloadText } from "../../auto-reply/tokens.js";
import { classifyFailoverReason } from "../embedded-agent-helpers/errors.js";
import type { FailoverReason } from "../embedded-agent-helpers/types.js";
import { isGpt5ModelId } from "../gpt5-prompt-overlay.js";
import type { ModelFallbackResultClassification } from "../model-fallback.js";
import { hasOutboundDeliveryEvidence, hasVisibleAgentPayload } from "./delivery-evidence.js";
@@ -55,6 +57,24 @@ function classifyHarnessResult(params: {
}
}
function classifyBusinessDenialErrorPayloadReason(
errorText: string,
provider: string,
): Extract<FailoverReason, "auth" | "auth_permanent" | "billing"> | null {
if (!errorText.trim()) {
return null;
}
const failoverReason = classifyFailoverReason(errorText, { provider });
switch (failoverReason) {
case "auth":
case "auth_permanent":
case "billing":
return failoverReason;
default:
return null;
}
}
export function classifyEmbeddedAgentRunResultForModelFallback(params: {
provider: string;
model: string;
@@ -79,6 +99,9 @@ export function classifyEmbeddedAgentRunResultForModelFallback(params: {
if (hasOutboundDeliveryEvidence(params.result)) {
return null;
}
if (params.result.meta.error?.kind === "hook_block") {
return null;
}
const harnessClassification = classifyHarnessResult({
provider: params.provider,
@@ -101,6 +124,15 @@ export function classifyEmbeddedAgentRunResultForModelFallback(params: {
code: "incomplete_result",
};
}
const failoverReason = classifyBusinessDenialErrorPayloadReason(errorText, params.provider);
if (failoverReason) {
return {
message: `${params.provider}/${params.model} ended with a provider error: ${errorText}`,
reason: failoverReason,
code: "embedded_error_payload",
rawError: errorText,
};
}
if (!isGpt5ModelId(params.model)) {
return null;
+49
View File
@@ -1223,6 +1223,55 @@ describe("runWithModelFallback", () => {
expect(result.attempts[0]?.code).toBe("empty_result");
});
it("continues fallback after embedded provider business-denial payloads", async () => {
const cfg = makeCfg({
agents: {
defaults: {
model: {
primary: "zai/glm-5.1",
fallbacks: ["openai/gpt-5.5"],
},
},
},
});
const rawError =
'{"success":false,"code":"CE-011","message":"当前ak因违规请求被禁止访问该模型"}';
const run = vi
.fn()
.mockResolvedValueOnce({
payloads: [{ text: rawError, isError: true }],
meta: { durationMs: 1 },
} satisfies EmbeddedAgentRunResult)
.mockResolvedValueOnce({
payloads: [{ text: "fallback ok" }],
meta: { durationMs: 1 },
} satisfies EmbeddedAgentRunResult);
const result = await runWithModelFallback<EmbeddedAgentRunResult>({
cfg,
provider: "zai",
model: "glm-5.1",
run,
classifyResult: ({ provider, model, result }) =>
classifyEmbeddedAgentRunResultForModelFallback({
provider,
model,
result,
}),
});
expect(result.result.payloads).toEqual([{ text: "fallback ok" }]);
expect(run).toHaveBeenCalledTimes(2);
expect(requireMockCall(run, 1, "fallback run")).toEqual(["openai", "gpt-5.5"]);
expect(result.attempts[0]).toMatchObject({
provider: "zai",
model: "glm-5.1",
reason: "auth",
code: "embedded_error_payload",
error: rawError,
});
});
it("surfaces classified terminal results when no fallback remains", async () => {
const cfg = makeCfg({
agents: {