fix(agents): recover cli context overflow sessions

Fixes #98897
This commit is contained in:
Ayaan Zaidi
2026-07-01 22:39:24 -07:00
parent b60803dfaa
commit 52cabf877d
17 changed files with 221 additions and 16 deletions
@@ -125,6 +125,7 @@ const CronFailoverReasonSchema = Type.Union([
Type.Literal("billing"),
Type.Literal("server_error"),
Type.Literal("timeout"),
Type.Literal("context_overflow"),
Type.Literal("model_not_found"),
Type.Literal("session_expired"),
Type.Literal("empty_response"),
+118 -1
View File
@@ -661,6 +661,67 @@ describe("runCliAgent reliability", () => {
expect(supervisorSpawnMock).toHaveBeenCalledTimes(1);
});
it("does not retry context overflow after a confirmed message send", async () => {
supervisorSpawnMock.mockClear();
supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => {
const input = args[0] as Parameters<ReturnType<typeof getProcessSupervisor>["spawn"]>[0];
const captureHandle = markMcpLoopbackToolCallStarted({
captureKey: input.env?.OPENCLAW_MCP_CLI_CAPTURE_KEY ?? "",
toolName: "message",
args: {
action: "send",
channel: "telegram",
target: "chat123",
message: "sent before overflow",
},
});
if (!captureHandle) {
throw new Error("Expected message delivery capture");
}
recordMcpLoopbackToolCallResult({
captureHandle,
toolName: "message",
args: {
action: "send",
channel: "telegram",
target: "chat123",
message: "sent before overflow",
},
result: { status: "sent" },
isError: false,
});
markMcpLoopbackToolCallFinished(captureHandle);
return createManagedRun({
reason: "exit",
exitCode: 1,
exitSignal: null,
durationMs: 150,
stdout: "",
stderr: "Prompt is too long",
timedOut: false,
noOutputTimedOut: false,
});
});
const context = buildPreparedContext({
sessionKey: "agent:main:delivered-overflow",
runId: "run-delivered-overflow",
cliSessionId: "stale-cli-session",
provider: "claude-cli",
model: "opus",
openClawHistoryPrompt: CLI_RESEED_PROMPT,
});
context.mcpDeliveryCapture = true;
const result = await runPreparedCliAgent(context);
expect(result.payloads).toBeUndefined();
expect(result.didSendViaMessagingTool).toBe(true);
expect(result.messagingToolSentTexts).toEqual(["sent before overflow"]);
expect(result.meta.executionTrace?.attempts?.[0]?.result).toBe("error");
expect(result.meta.agentMeta?.clearCliSessionBinding).toBe(true);
expect(supervisorSpawnMock).toHaveBeenCalledTimes(1);
});
it("preserves first-turn delivery through cleanup without binding the OpenClaw session id", async () => {
supervisorSpawnMock.mockClear();
supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => {
@@ -1390,6 +1451,48 @@ describe("runCliAgent reliability", () => {
expect(clearBeforeRetry).not.toHaveBeenCalled();
});
it("does not fresh retry context overflow when the run timeout budget is exhausted", async () => {
supervisorSpawnMock.mockClear();
const clearBeforeRetry = vi.fn(async () => true);
supervisorSpawnMock.mockResolvedValueOnce(
createManagedRun({
reason: "exit",
exitCode: 1,
exitSignal: null,
durationMs: 150,
stdout: "",
stderr: "Prompt is too long",
timedOut: false,
noOutputTimedOut: false,
}),
);
const context = buildPreparedContext({
sessionKey: "agent:main:expired-overflow-budget",
runId: "run-expired-overflow-budget",
cliSessionId: "stale-cli-session",
provider: "claude-cli",
model: "opus",
openClawHistoryPrompt: CLI_RESEED_PROMPT,
});
const expiredBudgetContext = {
...context,
started: Date.now() - context.params.timeoutMs - 1,
};
await expect(
runPreparedCliAgent({
...expiredBudgetContext,
params: {
...expiredBudgetContext.params,
onBeforeFreshCliSessionRetry: clearBeforeRetry,
},
}),
).rejects.toThrow("Prompt is too long");
expect(supervisorSpawnMock).toHaveBeenCalledTimes(1);
expect(clearBeforeRetry).not.toHaveBeenCalled();
});
it("does not fresh retry a no-output timeout after CLI diagnostic output", async () => {
supervisorSpawnMock.mockClear();
enqueueSystemEventMock.mockClear();
@@ -1468,7 +1571,7 @@ describe("runCliAgent reliability", () => {
expect(clearBeforeRetry).not.toHaveBeenCalled();
});
it.each(["timeout", "unknown"] as const)(
it.each(["timeout", "unknown", "context_overflow"] as const)(
"retries a fresh CLI session after recoverable %s failover without a failed agent_end",
async (reason) => {
const hookRunner = {
@@ -1500,6 +1603,18 @@ describe("runCliAgent reliability", () => {
noOutputTimedOut: true,
});
}
if (spawnCount === 1 && reason === "context_overflow") {
return createManagedRun({
reason: "exit",
exitCode: 1,
exitSignal: null,
durationMs: 150,
stdout: "",
stderr: "Prompt is too long",
timedOut: false,
noOutputTimedOut: false,
});
}
if (spawnCount === 1) {
return createManagedRun({
reason: "exit",
@@ -1552,6 +1667,8 @@ describe("runCliAgent reliability", () => {
});
expect(result.payloads).toEqual([{ text: "hello from fresh cli" }]);
expect(result.meta.finalPromptText).toContain("User: earlier context");
expect(result.meta.finalPromptText).toContain("<next_user_message>");
expect(supervisorSpawnMock).toHaveBeenCalledTimes(2);
expect(events).toEqual(["spawn-1", `clear-${reason}`, "spawn-2"]);
if (reason === "timeout") {
+58
View File
@@ -3003,6 +3003,64 @@ ${JSON.stringify({
);
});
it("marks Claude live stderr context overflows as retryable", async () => {
let stdoutListener: ((chunk: string) => void) | undefined;
let resolveExit: ((exit: RunExit) => void) | undefined;
const exited = new Promise<RunExit>((resolve) => {
resolveExit = resolve;
});
const stdin = {
write: vi.fn((dataValue: string, cb?: (err?: Error | null) => void) => {
stdoutListener?.(
JSON.stringify({ type: "system", subtype: "init", session_id: "live-overflow" }) + "\n",
);
cb?.();
resolveExit?.({
reason: "exit",
exitCode: 1,
exitSignal: null,
durationMs: 1,
stdout: "",
stderr: "Prompt is too long",
timedOut: false,
noOutputTimedOut: false,
});
}),
end: vi.fn(),
};
supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => {
const input = (args[0] ?? {}) as { onStdout?: (chunk: string) => void };
stdoutListener = input.onStdout;
return {
runId: "live-overflow-run",
pid: 2345,
startedAtMs: Date.now(),
stdin,
wait: vi.fn(() => exited),
cancel: vi.fn(),
};
});
await expectRejectsWithFields(
executePreparedCliRun(
buildPreparedCliRunContext({
provider: "claude-cli",
model: "sonnet",
runId: "run-live-overflow",
backend: {
liveSession: "claude-stdio",
},
}),
),
{
name: "FailoverError",
reason: "context_overflow",
code: "cli_context_overflow",
status: 413,
},
);
});
it("fails when Claude exits before a live turn starts", async () => {
supervisorSpawnMock.mockImplementationOnce(async () => ({
runId: "live-run",
+2
View File
@@ -100,6 +100,8 @@ function shouldRetryFreshCliSessionAfterFailover(params: {
return params.error.code === "cli_unknown_empty_failure";
case "timeout":
return params.error.code === "cli_no_output_timeout";
case "context_overflow":
return params.error.code === "cli_context_overflow";
default:
return false;
}
@@ -833,11 +833,13 @@ function createResultError(
const result = typeof parsed.result === "string" ? parsed.result.trim() : "";
const message = extractCliErrorMessage(raw) ?? (result || "Claude CLI failed.");
const reason = classifyFailoverReason(message, { provider: session.providerId }) ?? "unknown";
const code = reason === "context_overflow" ? "cli_context_overflow" : undefined;
return new FailoverError(message, {
reason,
provider: session.providerId,
model: session.modelId,
status: resolveFailoverStatus(reason),
code,
});
}
@@ -1012,6 +1014,7 @@ function handleClaudeExit(session: ClaudeLiveSession, exitCode: number | null):
return;
}
const reason = classifyFailoverReason(message, { provider: session.providerId }) ?? "unknown";
const code = reason === "context_overflow" ? "cli_context_overflow" : undefined;
failTurn(
session,
new FailoverError(message, {
@@ -1019,6 +1022,7 @@ function handleClaudeExit(session: ClaudeLiveSession, exitCode: number | null):
provider: session.providerId,
model: session.modelId,
status: resolveFailoverStatus(reason),
code,
}),
);
}
+8 -6
View File
@@ -1300,12 +1300,14 @@ export async function executePreparedCliRun(
reason = reason ?? "unknown";
const status = resolveFailoverStatus(reason);
const retryCode =
reason === "unknown" &&
result.reason === "exit" &&
errorCandidates.length === 0 &&
!observedCliActivity
? "cli_unknown_empty_failure"
: undefined;
reason === "context_overflow"
? "cli_context_overflow"
: reason === "unknown" &&
result.reason === "exit" &&
errorCandidates.length === 0 &&
!observedCliActivity
? "cli_unknown_empty_failure"
: undefined;
throw new FailoverError(err, {
reason,
provider: params.provider,
@@ -611,6 +611,12 @@ describe("extractObservedOverflowTokenCount", () => {
});
});
describe("classifyFailoverReason context overflow", () => {
it("maps prompt overflow to the closed failover reason", () => {
expect(classifyFailoverReason("Prompt is too long")).toBe("context_overflow");
});
});
describe("isTransientHttpError", () => {
it("returns true for retryable 5xx status codes", () => {
expect(isTransientHttpError("499 Client Closed Request")).toBe(true);
@@ -671,13 +677,13 @@ describe("classifyFailoverReasonFromHttpStatus", () => {
).toBe("rate_limit");
});
it("does not force HTTP 400 context-overflow payloads into format", () => {
it("classifies HTTP 400 context-overflow payloads without using format", () => {
expect(
classifyFailoverReasonFromHttpStatus(
400,
"INVALID_ARGUMENT: input exceeds the maximum number of tokens",
),
).toBeNull();
).toBe("context_overflow");
});
it("lets OpenRouter billing-classified HTTP 401 responses bypass generic auth", () => {
@@ -803,12 +809,12 @@ describe("classifyFailoverReason HTTP 410 handling", () => {
expect(classifyFailoverReason("HTTP 404: insufficient credits")).toBe("billing");
});
it("does not map HTTP 404 plus context-overflow text to model_not_found", () => {
it("maps HTTP 404 plus context-overflow text to context_overflow", () => {
expect(
classifyFailoverReason(
"HTTP 404: INVALID_ARGUMENT: input exceeds the maximum number of tokens",
),
).toBeNull();
).toBe("context_overflow");
});
it("keeps raw HTTP 400 wrappers aligned with structured provider classification", () => {
@@ -819,7 +825,7 @@ describe("classifyFailoverReason HTTP 410 handling", () => {
classifyFailoverReason(
"HTTP 400: INVALID_ARGUMENT: input exceeds the maximum number of tokens",
),
).toBeNull();
).toBe("context_overflow");
});
it("classifies OpenAI Responses unknown-no-details message distinctly", () => {
+4 -1
View File
@@ -731,7 +731,10 @@ function toReasonClassification(reason: FailoverReason): FailoverClassification
function failoverReasonFromClassification(
classification: FailoverClassification | null,
): FailoverReason | null {
return classification?.kind === "reason" ? classification.reason : null;
if (!classification) {
return null;
}
return classification.kind === "reason" ? classification.reason : "context_overflow";
}
export function isTransientHttpError(raw: string): boolean {
@@ -11,6 +11,7 @@ export type FailoverReason =
| "billing"
| "server_error"
| "timeout"
| "context_overflow"
| "model_not_found"
| "session_expired"
| "empty_response"
@@ -34,6 +34,7 @@ export function resolveAuthProfileFailureReason(params: {
(params.failoverReason === "rate_limit" && params.transientRateLimit === true))) ||
params.failoverReason === "server_error" ||
params.failoverReason === "empty_response" ||
params.failoverReason === "context_overflow" ||
params.failoverReason === "format"
) {
return null;
+2 -2
View File
@@ -607,13 +607,13 @@ describe("failover-error", () => {
).toBe("rate_limit");
});
it("does not misclassify structured HTTP 400 context overflow payloads as format", () => {
it("classifies structured HTTP 400 context overflow payloads without using format", () => {
expect(
resolveFailoverReasonFromError({
status: 400,
message: "INVALID_ARGUMENT: input exceeds the maximum number of tokens",
}),
).toBeNull();
).toBe("context_overflow");
});
it("keeps context overflow first-class in the shared signal classifier", () => {
+6 -1
View File
@@ -107,6 +107,8 @@ export function resolveFailoverStatus(reason: FailoverReason): number | undefine
return 403;
case "timeout":
return 408;
case "context_overflow":
return 413;
case "format":
return 400;
case "model_not_found":
@@ -461,7 +463,10 @@ export function isSignalTimeoutReason(reason: unknown): boolean {
function failoverReasonFromClassification(
classification: FailoverClassification | null,
): FailoverReason | null {
return classification?.kind === "reason" ? classification.reason : null;
if (!classification) {
return null;
}
return classification.kind === "reason" ? classification.reason : "context_overflow";
}
function normalizeErrorSignal(err: unknown, providerHint?: string): FailoverSignal {
+1
View File
@@ -41,6 +41,7 @@ export type AgentRuntimeFailoverReason =
| "billing"
| "server_error"
| "timeout"
| "context_overflow"
| "model_not_found"
| "session_expired"
| "empty_response"
@@ -108,6 +108,7 @@ describe("cron protocol conformance", () => {
"billing",
"server_error",
"timeout",
"context_overflow",
"model_not_found",
"session_expired",
"empty_response",
+1
View File
@@ -62,6 +62,7 @@ describe("cron run log errorReason", () => {
"timeout",
"model_not_found",
"session_expired",
"context_overflow",
"empty_response",
"no_error_details",
"unclassified",
+1
View File
@@ -17,6 +17,7 @@ const CRON_FAILOVER_REASONS = new Set<FailoverReason>([
"timeout",
"model_not_found",
"session_expired",
"context_overflow",
"empty_response",
"no_error_details",
"unclassified",
+1
View File
@@ -16,6 +16,7 @@ const ERROR_TYPE_BY_REASON: Partial<Record<FailoverReason, string>> = {
auth: "authentication_error",
auth_permanent: "permission_error",
billing: "insufficient_quota",
context_overflow: "invalid_request_error",
format: "invalid_request_error",
model_not_found: "invalid_request_error",
overloaded: "api_error",