mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
fix(agents): classify embedded provider business denials for fallback (#84814)
Summary: - The PR classifies selected embedded agent provider-denial error payloads through the shared failover matcher ... 1/current-ak auth matching, preserves guarded non-fallback cases, and covers fallback progression in tests. - PR surface: Source +34, Tests +166. Total +200 across 5 files. - Reproducibility: yes. Current main is source-reproducible: a non-GPT embedded result whose only signal is CE ... returns null from the classifier, and the fallback wrapper treats null classification as candidate success. Automerge notes: - PR branch already contained follow-up commit before automerge: fix(agents): classify embedded provider business denials for fallback - PR branch already contained follow-up commit before automerge: fix(clawsweeper): address review for automerge-openclaw-openclaw-8304… Validation: - ClawSweeper review passed for heade266beac93. - Required merge gates passed before the squash merge. Prepared head SHA:e266beac93Review: https://github.com/openclaw/openclaw/pull/84814#issuecomment-4505010446 Co-authored-by: Stellar鱼 <2182712990@qq.com> Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com> Approved-by: takhoffman Co-authored-by: takhoffman <781889+takhoffman@users.noreply.github.com>
This commit is contained in:
co-authored by
Stellar鱼
clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com>
clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com>
takhoffman
parent
aada44fca5
commit
18f94fc83a
@@ -1412,6 +1412,8 @@ describe("classifyFailoverReason provider messages", () => {
|
||||
// Auth errors
|
||||
expect(classifyFailoverReason("无权访问该模型")).toBe("auth");
|
||||
expect(classifyFailoverReason("403 您无权访问glm-5.1。")).toBe("auth");
|
||||
expect(classifyFailoverReason("当前ak因违规请求被禁止访问该模型")).toBe("auth");
|
||||
expect(classifyFailoverReason('{"success":false,"code":"CE-011"}')).toBe("auth");
|
||||
expect(classifyFailoverReason("认证失败")).toBe("auth");
|
||||
expect(classifyFailoverReason("鉴权失败,请检查API Key")).toBe("auth");
|
||||
expect(classifyFailoverReason("密钥无效")).toBe("auth");
|
||||
|
||||
@@ -47,6 +47,8 @@ const CJK_AUTH_ERROR_PATTERNS = [
|
||||
"鉴权失败",
|
||||
"密钥无效",
|
||||
"apikey 无效",
|
||||
/(?:当前\s*ak|ce-011).*?(?:违规请求|禁止访问)|(?:违规请求|禁止访问).*?(?:当前\s*ak|ce-011)/i,
|
||||
/\bce-011\b/i,
|
||||
] as const satisfies readonly ErrorPattern[];
|
||||
|
||||
const ZAI_BILLING_CODE_1311_RE = /"code"\s*:\s*1311\b/;
|
||||
|
||||
@@ -19,4 +19,119 @@ describe("classifyEmbeddedAgentRunResultForModelFallback", () => {
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("classifies provider business-denial error payloads as fallback-worthy", () => {
|
||||
const result = classifyEmbeddedAgentRunResultForModelFallback({
|
||||
provider: "zai",
|
||||
model: "glm-5.1",
|
||||
result: {
|
||||
payloads: [
|
||||
{
|
||||
isError: true,
|
||||
text: '{"success":false,"code":"CE-011","message":"当前ak因违规请求被禁止访问该模型"}',
|
||||
},
|
||||
],
|
||||
meta: {
|
||||
durationMs: 42,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
message:
|
||||
'zai/glm-5.1 ended with a provider error: {"success":false,"code":"CE-011","message":"当前ak因违规请求被禁止访问该模型"}',
|
||||
reason: "auth",
|
||||
code: "embedded_error_payload",
|
||||
rawError: '{"success":false,"code":"CE-011","message":"当前ak因违规请求被禁止访问该模型"}',
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves hook block results with auth-like error payload text", () => {
|
||||
const result = classifyEmbeddedAgentRunResultForModelFallback({
|
||||
provider: "custom",
|
||||
model: "gpt-5.5",
|
||||
result: {
|
||||
payloads: [
|
||||
{
|
||||
isError: true,
|
||||
text: "Access denied by policy",
|
||||
},
|
||||
],
|
||||
meta: {
|
||||
durationMs: 42,
|
||||
error: {
|
||||
kind: "hook_block",
|
||||
message: "Access denied by policy",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("uses provider-scoped failover matching for business-denial payloads", () => {
|
||||
const result = classifyEmbeddedAgentRunResultForModelFallback({
|
||||
provider: "openrouter",
|
||||
model: "claude-3.5-sonnet",
|
||||
result: {
|
||||
payloads: [
|
||||
{
|
||||
isError: true,
|
||||
text: "Key limit exceeded",
|
||||
},
|
||||
],
|
||||
meta: {
|
||||
durationMs: 42,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
message: "openrouter/claude-3.5-sonnet ended with a provider error: Key limit exceeded",
|
||||
reason: "billing",
|
||||
code: "embedded_error_payload",
|
||||
rawError: "Key limit exceeded",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not retry unclassified non-GPT error payloads", () => {
|
||||
const result = classifyEmbeddedAgentRunResultForModelFallback({
|
||||
provider: "custom",
|
||||
model: "llama-3.1",
|
||||
result: {
|
||||
payloads: [
|
||||
{
|
||||
isError: true,
|
||||
text: "the model produced an application-level error",
|
||||
},
|
||||
],
|
||||
meta: {
|
||||
durationMs: 42,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("does not retry non-business transport error payloads", () => {
|
||||
const result = classifyEmbeddedAgentRunResultForModelFallback({
|
||||
provider: "custom",
|
||||
model: "llama-3.1",
|
||||
result: {
|
||||
payloads: [
|
||||
{
|
||||
isError: true,
|
||||
text: "HTTP 500: internal server error",
|
||||
},
|
||||
],
|
||||
meta: {
|
||||
durationMs: 42,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { isSilentReplyPayloadText } from "../../auto-reply/tokens.js";
|
||||
import { classifyFailoverReason } from "../embedded-agent-helpers/errors.js";
|
||||
import type { FailoverReason } from "../embedded-agent-helpers/types.js";
|
||||
import { isGpt5ModelId } from "../gpt5-prompt-overlay.js";
|
||||
import type { ModelFallbackResultClassification } from "../model-fallback.js";
|
||||
import { hasOutboundDeliveryEvidence, hasVisibleAgentPayload } from "./delivery-evidence.js";
|
||||
@@ -55,6 +57,24 @@ function classifyHarnessResult(params: {
|
||||
}
|
||||
}
|
||||
|
||||
function classifyBusinessDenialErrorPayloadReason(
|
||||
errorText: string,
|
||||
provider: string,
|
||||
): Extract<FailoverReason, "auth" | "auth_permanent" | "billing"> | null {
|
||||
if (!errorText.trim()) {
|
||||
return null;
|
||||
}
|
||||
const failoverReason = classifyFailoverReason(errorText, { provider });
|
||||
switch (failoverReason) {
|
||||
case "auth":
|
||||
case "auth_permanent":
|
||||
case "billing":
|
||||
return failoverReason;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function classifyEmbeddedAgentRunResultForModelFallback(params: {
|
||||
provider: string;
|
||||
model: string;
|
||||
@@ -79,6 +99,9 @@ export function classifyEmbeddedAgentRunResultForModelFallback(params: {
|
||||
if (hasOutboundDeliveryEvidence(params.result)) {
|
||||
return null;
|
||||
}
|
||||
if (params.result.meta.error?.kind === "hook_block") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const harnessClassification = classifyHarnessResult({
|
||||
provider: params.provider,
|
||||
@@ -101,6 +124,15 @@ export function classifyEmbeddedAgentRunResultForModelFallback(params: {
|
||||
code: "incomplete_result",
|
||||
};
|
||||
}
|
||||
const failoverReason = classifyBusinessDenialErrorPayloadReason(errorText, params.provider);
|
||||
if (failoverReason) {
|
||||
return {
|
||||
message: `${params.provider}/${params.model} ended with a provider error: ${errorText}`,
|
||||
reason: failoverReason,
|
||||
code: "embedded_error_payload",
|
||||
rawError: errorText,
|
||||
};
|
||||
}
|
||||
|
||||
if (!isGpt5ModelId(params.model)) {
|
||||
return null;
|
||||
|
||||
@@ -1223,6 +1223,55 @@ describe("runWithModelFallback", () => {
|
||||
expect(result.attempts[0]?.code).toBe("empty_result");
|
||||
});
|
||||
|
||||
it("continues fallback after embedded provider business-denial payloads", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "zai/glm-5.1",
|
||||
fallbacks: ["openai/gpt-5.5"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const rawError =
|
||||
'{"success":false,"code":"CE-011","message":"当前ak因违规请求被禁止访问该模型"}';
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
payloads: [{ text: rawError, isError: true }],
|
||||
meta: { durationMs: 1 },
|
||||
} satisfies EmbeddedAgentRunResult)
|
||||
.mockResolvedValueOnce({
|
||||
payloads: [{ text: "fallback ok" }],
|
||||
meta: { durationMs: 1 },
|
||||
} satisfies EmbeddedAgentRunResult);
|
||||
|
||||
const result = await runWithModelFallback<EmbeddedAgentRunResult>({
|
||||
cfg,
|
||||
provider: "zai",
|
||||
model: "glm-5.1",
|
||||
run,
|
||||
classifyResult: ({ provider, model, result }) =>
|
||||
classifyEmbeddedAgentRunResultForModelFallback({
|
||||
provider,
|
||||
model,
|
||||
result,
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.result.payloads).toEqual([{ text: "fallback ok" }]);
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
expect(requireMockCall(run, 1, "fallback run")).toEqual(["openai", "gpt-5.5"]);
|
||||
expect(result.attempts[0]).toMatchObject({
|
||||
provider: "zai",
|
||||
model: "glm-5.1",
|
||||
reason: "auth",
|
||||
code: "embedded_error_payload",
|
||||
error: rawError,
|
||||
});
|
||||
});
|
||||
|
||||
it("surfaces classified terminal results when no fallback remains", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
|
||||
Reference in New Issue
Block a user