feat: pass structured provider error signals to hooks

Summary:
- Pass provider status/code/type descriptors through failover hook classification.
- Keep structured provider hook dispatch scoped, while preserving legacy broad message-hook fallback for unresolved custom provider ids.
- Isolate long commands/infra Vitest lanes in fork workers and update config expectations.

Verification:
- node scripts/run-vitest.mjs src/agents/embedded-agent-helpers/errors-provider-structured-signals.test.ts src/agents/failover-error.test.ts
- OPENCLAW_VITEST_MAX_WORKERS=1 node scripts/run-vitest.mjs src/plugins/provider-runtime.test.ts
- node scripts/run-vitest.mjs src/agents/embedded-agent-helpers/errors-provider-structured-signals.test.ts src/agents/embedded-agent-helpers/provider-error-patterns.test.ts src/agents/failover-error.test.ts src/plugins/provider-runtime.test.ts test/vitest-projects-config.test.ts test/vitest-scoped-config.test.ts src/infra/vitest-config.test.ts
- pnpm tsgo:prod
- autoreview --mode branch --base origin/main --no-web-search --thinking low
- GitHub required dependency-guard: pass
- GitHub Real behavior proof: pass
- GitHub broad CI/checks visible on PR: pass

Co-authored-by: Jason (Json) <fuller-stack-dev@users.noreply.github.com>
This commit is contained in:
Jason (Json)
2026-05-30 22:14:46 +01:00
committed by GitHub
co-authored by Jason
parent dbd3e10312
commit 89975eea24
11 changed files with 398 additions and 33 deletions
@@ -0,0 +1,126 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { resolveFailoverReasonFromError } from "../failover-error.js";
import { classifyFailoverSignal } from "./errors.js";
const providerRuntimeMocks = vi.hoisted(() => ({
classifyProviderPluginError: vi.fn(),
}));
vi.mock("./provider-error-patterns.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("./provider-error-patterns.js")>();
return {
...actual,
classifyProviderPluginError: providerRuntimeMocks.classifyProviderPluginError,
};
});
describe("provider failover hook structured signals", () => {
beforeEach(() => {
providerRuntimeMocks.classifyProviderPluginError.mockReset();
});
it("lets provider hooks refine ambiguous auth statuses from stable codes", () => {
providerRuntimeMocks.classifyProviderPluginError.mockImplementation((context) => {
if (
context.provider === "demo-provider" &&
context.status === 403 &&
context.code === "PROVIDER_RATE_LIMITED"
) {
return "rate_limit";
}
return context.provider === "demo-provider" &&
context.status === 403 &&
context.code === "PROVIDER_QUOTA_EXHAUSTED"
? "billing"
: undefined;
});
expect(
classifyFailoverSignal({
provider: "demo-provider",
status: 403,
code: "PROVIDER_QUOTA_EXHAUSTED",
message: "Forbidden",
}),
).toEqual({ kind: "reason", reason: "billing" });
expect(
classifyFailoverSignal({
provider: "demo-provider",
status: 403,
code: "PROVIDER_RATE_LIMITED",
message: "Forbidden",
}),
).toEqual({ kind: "reason", reason: "rate_limit" });
expect(
classifyFailoverSignal({
provider: "other-provider",
status: 403,
code: "PROVIDER_QUOTA_EXHAUSTED",
message: "Forbidden",
}),
).toEqual({ kind: "reason", reason: "auth" });
});
it("does not call the direct provider hook for unstructured classified messages", () => {
expect(
classifyFailoverSignal({
provider: "demo-provider",
message: "invalid_api_key",
}),
).toEqual({ kind: "reason", reason: "auth" });
expect(providerRuntimeMocks.classifyProviderPluginError).not.toHaveBeenCalled();
});
it("does not treat message-parsed HTTP prefixes as structured provider descriptors", () => {
providerRuntimeMocks.classifyProviderPluginError.mockReturnValue("billing");
expect(
classifyFailoverSignal({
provider: "demo-provider",
message: "403 concurrency limit breached",
}),
).toEqual({ kind: "reason", reason: "auth" });
expect(providerRuntimeMocks.classifyProviderPluginError).not.toHaveBeenCalled();
});
it("passes nested provider error types through failover error normalization", () => {
providerRuntimeMocks.classifyProviderPluginError.mockImplementation((context) => {
return context.provider === "demo-provider" &&
context.errorType === "PROVIDER_QUOTA_EXHAUSTED"
? "billing"
: undefined;
});
expect(
resolveFailoverReasonFromError({
provider: "demo-provider",
status: 403,
type: "error",
error: {
type: "PROVIDER_QUOTA_EXHAUSTED",
message: "Forbidden",
},
}),
).toBe("billing");
});
it("does not promote generic SDK type strings as structured provider descriptors", () => {
providerRuntimeMocks.classifyProviderPluginError.mockReturnValue("billing");
expect(
resolveFailoverReasonFromError({
provider: "demo-provider",
type: "api_error",
message: "unclassified provider failure",
}),
).toBeNull();
expect(
resolveFailoverReasonFromError({
provider: "demo-provider",
message: "unclassified provider failure",
detail: { type: "api_error" },
}),
).toBeNull();
expect(providerRuntimeMocks.classifyProviderPluginError).not.toHaveBeenCalled();
});
});
+54 -6
View File
@@ -34,6 +34,7 @@ import {
matchesFormatErrorPattern,
} from "./failover-matches.js";
import {
classifyProviderPluginError,
classifyProviderSpecificError,
matchesProviderContextOverflow,
} from "./provider-error-patterns.js";
@@ -264,6 +265,7 @@ type PaymentRequiredFailoverReason = Extract<FailoverReason, "billing" | "rate_l
export type FailoverSignal = {
status?: number;
code?: string;
errorType?: string;
message?: string;
provider?: string;
};
@@ -633,16 +635,30 @@ export function classifyFailoverReasonFromHttpStatus(
message?: string,
opts?: { provider?: string },
): FailoverReason | null {
const hasProviderStatusSignal = Boolean(opts?.provider && typeof status === "number");
const messageClassification = message
? classifyFailoverClassificationFromMessage(message, opts?.provider)
? classifyFailoverClassificationFromMessage(message, opts?.provider, {
includeProviderPluginHooks: !hasProviderStatusSignal,
})
: null;
const providerPluginReason = hasProviderStatusSignal
? classifyProviderPluginError({
errorMessage: message ?? "",
provider: opts?.provider,
status,
})
: null;
const effectiveMessageClassification = providerPluginReason
? toReasonClassification(providerPluginReason)
: messageClassification;
return failoverReasonFromClassification(
classifyFailoverClassificationFromHttpStatus(
status,
message,
messageClassification,
effectiveMessageClassification,
status,
opts?.provider,
{ preserveProviderSignalClassification: providerPluginReason !== null },
),
);
}
@@ -653,6 +669,7 @@ function classifyFailoverClassificationFromHttpStatus(
messageClassification: FailoverClassification | null,
explicitStatus: number | undefined,
provider?: string,
opts?: { preserveProviderSignalClassification?: boolean },
): FailoverClassification | null {
const messageReason = failoverReasonFromClassification(messageClassification);
if (typeof status !== "number" || !Number.isFinite(status)) {
@@ -685,6 +702,9 @@ function classifyFailoverClassificationFromHttpStatus(
return toReasonClassification("rate_limit");
}
if (status === 401 || status === 403) {
if (opts?.preserveProviderSignalClassification && messageClassification) {
return messageClassification;
}
if (message && isAuthPermanentErrorMessage(message)) {
return toReasonClassification("auth_permanent");
}
@@ -868,6 +888,7 @@ function isExactUnknownNoDetailsError(raw: string): boolean {
function classifyFailoverClassificationFromMessage(
raw: string,
provider?: string,
opts?: { includeProviderPluginHooks?: boolean },
): FailoverClassification | null {
if (isImageDimensionErrorMessage(raw)) {
return null;
@@ -960,7 +981,10 @@ function classifyFailoverClassificationFromMessage(
return toReasonClassification("timeout");
}
// Provider-specific patterns as a final catch (Bedrock, Groq, Together AI, etc.)
const providerSpecific = classifyProviderSpecificError(raw);
const providerSpecific = classifyProviderSpecificError(
{ errorMessage: raw, provider },
{ includePluginHooks: opts?.includeProviderPluginHooks },
);
if (providerSpecific) {
return toReasonClassification(providerSpecific);
}
@@ -969,6 +993,8 @@ function classifyFailoverClassificationFromMessage(
export function classifyFailoverSignal(signal: FailoverSignal): FailoverClassification | null {
const inferredStatus = inferSignalStatus(signal);
const explicitStatus =
typeof signal.status === "number" && Number.isFinite(signal.status) ? signal.status : undefined;
if (
signal.message &&
isTransportHtmlErrorStatus(inferredStatus) &&
@@ -976,9 +1002,30 @@ export function classifyFailoverSignal(signal: FailoverSignal): FailoverClassifi
) {
return toReasonClassification("timeout");
}
const hasStructuredProviderSignal = Boolean(
signal.provider &&
(explicitStatus !== undefined || signal.code !== undefined || signal.errorType !== undefined),
);
const messageClassification = signal.message
? classifyFailoverClassificationFromMessage(signal.message, signal.provider)
? classifyFailoverClassificationFromMessage(signal.message, signal.provider, {
includeProviderPluginHooks: !hasStructuredProviderSignal,
})
: null;
const providerPluginReason =
hasStructuredProviderSignal &&
signal.provider &&
(signal.message || signal.code || signal.errorType || typeof inferredStatus === "number")
? classifyProviderPluginError({
errorMessage: signal.message ?? "",
provider: signal.provider,
status: explicitStatus,
code: signal.code,
errorType: signal.errorType,
})
: null;
const effectiveMessageClassification = providerPluginReason
? toReasonClassification(providerPluginReason)
: messageClassification;
const codeReason = classifyFailoverReasonFromCode(signal.code);
if (codeReason === "auth_permanent") {
return toReasonClassification(codeReason);
@@ -986,9 +1033,10 @@ export function classifyFailoverSignal(signal: FailoverSignal): FailoverClassifi
const statusClassification = classifyFailoverClassificationFromHttpStatus(
inferredStatus,
signal.message,
messageClassification,
effectiveMessageClassification,
signal.status,
signal.provider,
{ preserveProviderSignalClassification: providerPluginReason !== null },
);
if (statusClassification) {
return statusClassification;
@@ -996,7 +1044,7 @@ export function classifyFailoverSignal(signal: FailoverSignal): FailoverClassifi
if (codeReason) {
return toReasonClassification(codeReason);
}
return messageClassification;
return effectiveMessageClassification;
}
export function classifyProviderRuntimeFailureKind(
@@ -77,7 +77,8 @@ export const PROVIDER_SPECIFIC_PATTERNS: readonly ProviderErrorPattern[] = [
type ProviderRuntimeHooks = {
classifyProviderFailoverReasonWithPlugin: (params: {
context: { errorMessage: string };
provider?: string;
context: ProviderSpecificErrorContext;
}) => FailoverReason | null;
matchesProviderContextOverflowWithPlugin: (params: {
context: { errorMessage: string };
@@ -105,8 +106,8 @@ function resolveProviderRuntimeHooks(): ProviderRuntimeHooks | null {
"../../plugins/provider-runtime.js",
) as unknown as ProviderRuntimeHooks;
cachedProviderRuntimeHooks = {
classifyProviderFailoverReasonWithPlugin: ({ context }) =>
loaded.classifyProviderFailoverReasonWithPlugin({ context }) ?? null,
classifyProviderFailoverReasonWithPlugin: ({ provider, context }) =>
loaded.classifyProviderFailoverReasonWithPlugin({ provider, context }) ?? null,
matchesProviderContextOverflowWithPlugin: loaded.matchesProviderContextOverflowWithPlugin,
};
} catch {
@@ -122,6 +123,25 @@ function looksLikeProviderContextOverflowCandidate(errorMessage: string): boolea
);
}
export type ProviderSpecificErrorContext = {
provider?: string;
modelId?: string;
errorMessage: string;
status?: number;
code?: string;
errorType?: string;
};
function normalizeProviderSpecificErrorContext(
input: string | ProviderSpecificErrorContext,
): ProviderSpecificErrorContext {
return typeof input === "string" ? { errorMessage: input } : input;
}
type ProviderSpecificErrorOptions = {
includePluginHooks?: boolean;
};
/**
* Check if an error message matches any provider-specific context overflow pattern.
* Called from `isContextOverflowError()` to catch provider-specific wording.
@@ -138,21 +158,36 @@ export function matchesProviderContextOverflow(errorMessage: string): boolean {
);
}
export function classifyProviderPluginError(
input: string | ProviderSpecificErrorContext,
): FailoverReason | null {
const context = normalizeProviderSpecificErrorContext(input);
const runtimeHooks = resolveProviderRuntimeHooks();
return (
runtimeHooks?.classifyProviderFailoverReasonWithPlugin({
provider: context.provider,
context,
}) ?? null
);
}
/**
* Try to classify an error using provider-specific patterns.
* Returns null if no provider-specific pattern matches (fall through to generic classification).
*/
export function classifyProviderSpecificError(errorMessage: string): FailoverReason | null {
const runtimeHooks = resolveProviderRuntimeHooks();
const pluginReason =
runtimeHooks?.classifyProviderFailoverReasonWithPlugin({
context: { errorMessage },
}) ?? null;
if (pluginReason) {
return pluginReason;
export function classifyProviderSpecificError(
input: string | ProviderSpecificErrorContext,
opts?: ProviderSpecificErrorOptions,
): FailoverReason | null {
const context = normalizeProviderSpecificErrorContext(input);
if (opts?.includePluginHooks !== false) {
const pluginReason = classifyProviderPluginError(context);
if (pluginReason) {
return pluginReason;
}
}
for (const pattern of PROVIDER_SPECIFIC_PATTERNS) {
if (pattern.test.test(errorMessage)) {
if (pattern.test.test(context.errorMessage)) {
return pattern.reason;
}
}
+42
View File
@@ -169,6 +169,46 @@ function getErrorCode(err: unknown): string | undefined {
return findErrorProperty(err, readDirectErrorCode);
}
function isStableProviderErrorType(value: string): boolean {
if (
/^(?:api|authentication|invalid_request|not_found|overloaded|permission|rate_limit|server)_error$/i.test(
value,
)
) {
return false;
}
return /^[A-Z][A-Z0-9_:-]*$/.test(value);
}
function readDirectErrorType(err: unknown): string | undefined {
if (!err || typeof err !== "object") {
return undefined;
}
const directType = (err as { errorType?: unknown }).errorType;
if (typeof directType === "string") {
const trimmed = directType.trim();
return trimmed && isStableProviderErrorType(trimmed) ? trimmed : undefined;
}
const detailType = (err as { detail?: { type?: unknown } }).detail?.type;
if (typeof detailType === "string") {
const trimmed = detailType.trim();
return trimmed && isStableProviderErrorType(trimmed) ? trimmed : undefined;
}
const type = (err as { type?: unknown }).type;
if (typeof type === "string") {
const trimmed = type.trim();
if (!trimmed || /^(?:error|exception)$/i.test(trimmed)) {
return undefined;
}
return isStableProviderErrorType(trimmed) ? trimmed : undefined;
}
return undefined;
}
function getErrorType(err: unknown): string | undefined {
return findErrorProperty(err, readDirectErrorType);
}
function readDirectProvider(err: unknown): string | undefined {
if (!err || typeof err !== "object") {
return undefined;
@@ -216,6 +256,7 @@ function normalizeDirectErrorSignal(err: unknown): FailoverSignal {
return {
status: readDirectStatusCode(err),
code: readDirectErrorCode(err),
errorType: readDirectErrorType(err),
message: message || undefined,
provider: readDirectProvider(err),
};
@@ -333,6 +374,7 @@ function normalizeErrorSignal(err: unknown, providerHint?: string): FailoverSign
return {
status: getStatusCode(err),
code: getErrorCode(err),
errorType: getErrorType(err),
message: message || undefined,
provider: getProvider(err) ?? providerHint,
};
+69 -2
View File
@@ -1453,8 +1453,12 @@ describe("provider-runtime", () => {
auth: [],
matchesContextOverflowError: ({ errorMessage }) =>
/\bcontent_filter\b.*\btoo long\b/i.test(errorMessage),
classifyFailoverReason: ({ errorMessage }) =>
/\bquota exceeded\b/i.test(errorMessage) ? "rate_limit" : undefined,
classifyFailoverReason: ({ errorMessage, code, status }) => {
if (status === 403 && code === "PROVIDER_QUOTA_EXHAUSTED") {
return "billing";
}
return /\bquota exceeded\b/i.test(errorMessage) ? "rate_limit" : undefined;
},
},
]);
@@ -1476,6 +1480,69 @@ describe("provider-runtime", () => {
},
}),
).toBe("rate_limit");
expect(
classifyProviderFailoverReasonWithPlugin({
provider: "azure-openai-responses",
context: {
provider: "azure-openai-responses",
errorMessage: "Forbidden",
status: 403,
code: "PROVIDER_QUOTA_EXHAUSTED",
},
}),
).toBe("billing");
});
it("falls back to all failover hooks when a provider owner cannot be resolved", () => {
const classifyFailoverReason = vi.fn(({ errorMessage }) =>
/\bconcurrency limit breached\b/i.test(errorMessage) ? "rate_limit" : undefined,
);
resolvePluginProvidersMock.mockReturnValue([
{
id: "together",
label: "Together",
auth: [],
classifyFailoverReason,
},
]);
expect(
classifyProviderFailoverReasonWithPlugin({
provider: "my-together",
context: {
provider: "my-together",
errorMessage: "concurrency limit breached",
},
}),
).toBe("rate_limit");
expect(classifyFailoverReason).toHaveBeenCalledWith({
provider: "my-together",
errorMessage: "concurrency limit breached",
});
});
it("does not broad-scan failover hooks for unresolved providers with structured descriptors", () => {
const classifyFailoverReason = vi.fn(() => "overloaded" as const);
resolvePluginProvidersMock.mockReturnValue([
{
id: "mantle",
label: "Mantle",
auth: [],
classifyFailoverReason,
},
]);
expect(
classifyProviderFailoverReasonWithPlugin({
provider: "my-openai-compatible",
context: {
provider: "my-openai-compatible",
status: 403,
errorMessage: "service unavailable",
},
}),
).toBeUndefined();
expect(classifyFailoverReason).not.toHaveBeenCalled();
});
it("resolves stream wrapper hooks through hook-only aliases without provider ownership", () => {
+32 -10
View File
@@ -631,11 +631,7 @@ export function matchesProviderContextOverflowWithPlugin(params: {
env?: NodeJS.ProcessEnv;
context: ProviderFailoverErrorContext;
}): boolean {
const plugins = params.provider
? [resolveProviderHookPlugin({ ...params, provider: params.provider })].filter(
(plugin): plugin is ProviderPlugin => Boolean(plugin),
)
: resolveProviderPluginsForHooks(params);
const plugins = resolveProviderPluginsForScopedHook(params);
for (const plugin of plugins) {
if (plugin.matchesContextOverflowError?.(params.context)) {
return true;
@@ -651,11 +647,7 @@ export function classifyProviderFailoverReasonWithPlugin(params: {
env?: NodeJS.ProcessEnv;
context: ProviderFailoverErrorContext;
}) {
const plugins = params.provider
? [resolveProviderHookPlugin({ ...params, provider: params.provider })].filter(
(plugin): plugin is ProviderPlugin => Boolean(plugin),
)
: resolveProviderPluginsForHooks(params);
const plugins = resolveProviderPluginsForScopedHook(params);
for (const plugin of plugins) {
const reason = plugin.classifyFailoverReason?.(params.context);
if (reason) {
@@ -665,6 +657,36 @@ export function classifyProviderFailoverReasonWithPlugin(params: {
return undefined;
}
function resolveProviderPluginsForScopedHook(params: {
provider?: string;
config?: OpenClawConfig;
workspaceDir?: string;
env?: NodeJS.ProcessEnv;
context: ProviderFailoverErrorContext;
}): ProviderPlugin[] {
if (!params.provider) {
return resolveProviderPluginsForHooks(params);
}
const plugin = resolveProviderHookPlugin({ ...params, provider: params.provider });
if (plugin) {
return [plugin];
}
if (hasStructuredFailoverDescriptor(params.context)) {
return [];
}
// Custom provider ids may only name their canonical API in config, and the
// legacy message classifier only has the runtime id here. Preserve its old
// broad hook scan for descriptor-free messages, but do not let unrelated
// hooks override structured HTTP/auth signals.
return resolveProviderPluginsForHooks(params);
}
function hasStructuredFailoverDescriptor(context: ProviderFailoverErrorContext): boolean {
return (
context.status !== undefined || context.code !== undefined || context.errorType !== undefined
);
}
export function formatProviderAuthProfileApiKeyWithPlugin(params: {
provider: string;
config?: OpenClawConfig;
+3
View File
@@ -960,6 +960,9 @@ export type ProviderFailoverErrorContext = {
provider?: string;
modelId?: string;
errorMessage: string;
status?: number;
code?: string;
errorType?: string;
};
/**
+1 -1
View File
@@ -97,7 +97,7 @@ describe("projects vitest config", () => {
expect(createAgentsSupportVitestConfig().test.pool).toBe("threads");
expect(createAgentsToolsVitestConfig().test.pool).toBe("threads");
expect(createCommandsLightVitestConfig().test.pool).toBe("threads");
expect(createCommandsVitestConfig().test.pool).toBe("threads");
expect(createCommandsVitestConfig().test.pool).toBe("forks");
expect(createPluginSdkLightVitestConfig().test.pool).toBe("threads");
expect(createUnitFastVitestConfig().test.pool).toBe("threads");
expect(createContractsVitestConfig(pluginContractPatterns).test.pool).toBe("threads");
+20 -2
View File
@@ -129,6 +129,24 @@ function expectThreadedIsolatedRunner(config: {
expect(testConfig.runner).toBeUndefined();
}
function expectForkedNonIsolatedRunner(config: {
test?: { pool?: unknown; isolate?: unknown; runner?: unknown };
}) {
const testConfig = requireTestConfig(config);
expect(testConfig.pool).toBe("forks");
expect(testConfig.isolate).toBe(false);
expect(normalizeConfigPath(testConfig.runner)).toBe("test/non-isolated-runner.ts");
}
function expectForkedIsolatedRunner(config: {
test?: { pool?: unknown; isolate?: unknown; runner?: unknown };
}) {
const testConfig = requireTestConfig(config);
expect(testConfig.pool).toBe("forks");
expect(testConfig.isolate).toBe(true);
expect(testConfig.runner).toBeUndefined();
}
describe("resolveVitestIsolation", () => {
it("aliases private QA plugin SDK subpaths for source tests only", () => {
for (const subpath of PRIVATE_PLUGIN_SDK_SUBPATHS) {
@@ -384,10 +402,10 @@ describe("scoped vitest configs", () => {
expectThreadedNonIsolatedRunner(config);
}
expectThreadedNonIsolatedRunner(defaultCommandsConfig);
expectForkedNonIsolatedRunner(defaultCommandsConfig);
expectThreadedNonIsolatedRunner(defaultUiConfig);
expectThreadedIsolatedRunner(defaultInfraConfig);
expectForkedIsolatedRunner(defaultInfraConfig);
});
it("keeps the process lane off the openclaw runtime setup", () => {
+2
View File
@@ -6,7 +6,9 @@ export function createCommandsVitestConfig(env?: Record<string, string | undefin
dir: "src/commands",
env,
exclude: commandsLightTestFiles,
fileParallelism: false,
name: "commands",
pool: "forks",
});
}
+2
View File
@@ -6,9 +6,11 @@ export function createInfraVitestConfig(env?: Record<string, string | undefined>
dir: "src",
env,
exclude: boundaryTestFiles,
fileParallelism: false,
isolate: true,
name: "infra",
passWithNoTests: true,
pool: "forks",
});
}