mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
fix(claude-cli): max-turn stops omit run and recovery context (#94130)
* fix(anthropic): handle max_turns stop reason and surface retry-limit details to users * fix(anthropic): classify max_turns as length-limited * fix(anthropic): preserve max_turns diagnostics * fix(anthropic): preserve max_turns side-effect warning * fix(claude-cli): max-turn stops omit run and recovery context * fix(claude-cli): preserve max-turn terminal boundaries * refactor(claude-cli): centralize terminal error conversion Co-authored-by: zhang-guiping <zhang.guiping@xydigit.com> * fix(claude-cli): warn before retrying max-turn failures Co-authored-by: zhang-guiping <zhang.guiping@xydigit.com> * fix(claude-cli): preserve max-turn stops through wrappers Co-authored-by: zhang-guiping <zhang.guiping@xydigit.com> * fix(claude-cli): prioritize terminal max-turn output --------- Co-authored-by: 张贵萍 <zhangguiping@zhangguipingdeMac-mini.local> Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
co-authored by
张贵萍
Peter Steinberger
parent
e119d050e5
commit
fb48df4a0f
@@ -34,6 +34,7 @@ Docs: https://docs.openclaw.ai
|
||||
|
||||
- **Native app connection and relay reliability:** keep Android disconnects stopped across Activity recreation, fail remote camera commands without opening permission prompts, refresh mobile node registration after capability changes, surface iOS onboarding connection failures, cancel stale Talk owners on session switches, reject invalid Watch acknowledgments, preserve Watch events received during startup, and prevent older agent overview requests from replacing newer gateway state.
|
||||
- **Gateway source watch:** hand the configured port off from the installed service before starting the tmux watcher, preserve failed panes for attach/capture, and keep explicit alternate-port watches side by side with the managed Gateway.
|
||||
- **Claude CLI max-turn diagnostics:** preserve terminal max-turn results with OpenClaw and Claude session context, warn when tool actions may already have run, and stop unsafe auth-profile or model replay for potentially side-effecting turns. (#94130) Thanks @zhangguiping-xydt.
|
||||
- **Provider network retries:** align provider read/poll/download and agent-wait recovery for transient connection errors, retry bounded provider `ENOTFOUND` failures while leaving gateway `ENOTFOUND` and non-idempotent create operations fail-fast. (#101496) Thanks @xialonglee.
|
||||
- **Session retry classification:** stop permanent provider errors whose identifiers or payload details merely contain 429/5xx digit sequences from re-sending full context, and share bounded rate-limit-window parsing across retry paths. (#105258) Thanks @destire-mio.
|
||||
- **LINE directive templates:** suppress confirms and buttons with blank required fields or unlabeled actions while preserving valid titleless buttons and surrounding reply text. (#105520) Thanks @edenfunf.
|
||||
|
||||
@@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
createCliJsonlStreamingParser,
|
||||
extractCliErrorMessage,
|
||||
formatCliOutputError,
|
||||
parseCliJson,
|
||||
parseCliJsonl,
|
||||
parseCliOutput,
|
||||
@@ -37,6 +38,32 @@ describe("supportsCliJsonlToolEvents", () => {
|
||||
});
|
||||
|
||||
describe("parseCliJson", () => {
|
||||
it("preserves Claude max-turn terminal context in JSON mode", () => {
|
||||
const result = parseCliJson(
|
||||
JSON.stringify({
|
||||
type: "result",
|
||||
subtype: "error_max_turns",
|
||||
session_id: "session-json-max-turns",
|
||||
terminal_reason: "max_turns",
|
||||
errors: ["Reached maximum number of turns (3)"],
|
||||
}),
|
||||
{
|
||||
command: "claude",
|
||||
output: "json",
|
||||
sessionIdFields: ["session_id"],
|
||||
},
|
||||
"claude-cli",
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
text: "",
|
||||
sessionId: "session-json-max-turns",
|
||||
usage: undefined,
|
||||
errorText: "Reached maximum number of turns (3)",
|
||||
terminalFailure: { reason: "max_turns", limit: 3 },
|
||||
});
|
||||
});
|
||||
|
||||
it("classifies Claude is_error JSON results as provider errors", () => {
|
||||
const result = parseCliJson(
|
||||
JSON.stringify({
|
||||
@@ -930,12 +957,16 @@ describe("parseCliJsonl", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("uses Claude error subtypes when result text is absent", () => {
|
||||
it("preserves Claude max-turn terminal context for actionable run errors", () => {
|
||||
const result = parseCliJsonl(
|
||||
JSON.stringify({
|
||||
type: "result",
|
||||
subtype: "error_max_turns",
|
||||
session_id: "session-max-turns",
|
||||
num_turns: 2,
|
||||
stop_reason: "tool_use",
|
||||
terminal_reason: "max_turns",
|
||||
errors: ["Reached maximum number of turns (1)"],
|
||||
}),
|
||||
{
|
||||
command: "claude",
|
||||
@@ -949,8 +980,72 @@ describe("parseCliJsonl", () => {
|
||||
text: "",
|
||||
sessionId: "session-max-turns",
|
||||
usage: undefined,
|
||||
errorText: "Claude CLI result subtype error_max_turns.",
|
||||
errorText: "Reached maximum number of turns (1)",
|
||||
terminalFailure: {
|
||||
reason: "max_turns",
|
||||
limit: 1,
|
||||
},
|
||||
});
|
||||
expect(
|
||||
formatCliOutputError(result!, {
|
||||
runId: "run-max-turns",
|
||||
sessionId: "openclaw-session-max-turns",
|
||||
}),
|
||||
).toBe(
|
||||
"Claude CLI stopped after reaching the maximum number of turns (limit: 1). " +
|
||||
"OpenClaw run: run-max-turns. OpenClaw session: openclaw-session-max-turns. " +
|
||||
"Claude session: session-max-turns. Tool actions may already have run; verify their effects before retrying. " +
|
||||
"Retry with a higher --max-turns value or a narrower task.",
|
||||
);
|
||||
});
|
||||
|
||||
it("warns that terminal_reason-only max-turn results may have run tools", () => {
|
||||
const result = parseCliJsonl(
|
||||
JSON.stringify({
|
||||
type: "result",
|
||||
session_id: "session-terminal-reason-only",
|
||||
terminal_reason: "max_turns",
|
||||
}),
|
||||
{
|
||||
command: "claude",
|
||||
output: "jsonl",
|
||||
sessionIdFields: ["session_id"],
|
||||
},
|
||||
"claude-cli",
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
text: "",
|
||||
sessionId: "session-terminal-reason-only",
|
||||
usage: undefined,
|
||||
errorText: "Reached maximum number of turns.",
|
||||
terminalFailure: { reason: "max_turns" },
|
||||
});
|
||||
expect(formatCliOutputError(result!)).toBe(
|
||||
"Claude CLI stopped after reaching the maximum number of turns. " +
|
||||
"Claude session: session-terminal-reason-only. " +
|
||||
"Tool actions may already have run; verify their effects before retrying. " +
|
||||
"Retry with a higher --max-turns value or a narrower task.",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not apply Claude terminal semantics to an explicit Gemini dialect", () => {
|
||||
const result = parseCliJsonl(
|
||||
JSON.stringify({
|
||||
type: "result",
|
||||
subtype: "error_max_turns",
|
||||
terminal_reason: "max_turns",
|
||||
errors: ["Reached maximum number of turns (1)"],
|
||||
}),
|
||||
{
|
||||
command: "claude",
|
||||
output: "jsonl",
|
||||
jsonlDialect: "gemini-stream-json",
|
||||
},
|
||||
"claude-cli",
|
||||
);
|
||||
|
||||
expect(result?.terminalFailure).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+117
-1
@@ -34,6 +34,11 @@ type CliProcessDiagnostics = {
|
||||
useResume: boolean;
|
||||
};
|
||||
|
||||
export type CliTerminalFailure = {
|
||||
reason: "max_turns";
|
||||
limit?: number;
|
||||
};
|
||||
|
||||
/** Normalized result from a CLI-backed model provider turn. */
|
||||
export type CliOutput = {
|
||||
text: string;
|
||||
@@ -41,6 +46,7 @@ export type CliOutput = {
|
||||
sessionId?: string;
|
||||
usage?: CliUsage;
|
||||
errorText?: string;
|
||||
terminalFailure?: CliTerminalFailure;
|
||||
diagnostics?: {
|
||||
process?: CliProcessDiagnostics;
|
||||
};
|
||||
@@ -54,6 +60,36 @@ export type CliOutput = {
|
||||
yielded?: true;
|
||||
};
|
||||
|
||||
function normalizeCliContextValue(value: string | undefined): string | undefined {
|
||||
const normalized = value?.trim().replace(/\s+/g, " ");
|
||||
return normalized ? normalized.slice(0, 200) : undefined;
|
||||
}
|
||||
|
||||
export function formatCliOutputError(
|
||||
output: CliOutput,
|
||||
attribution: { runId?: string; sessionId?: string } = {},
|
||||
): string {
|
||||
if (output.terminalFailure?.reason !== "max_turns") {
|
||||
return output.errorText || "CLI failed.";
|
||||
}
|
||||
|
||||
const runId = normalizeCliContextValue(attribution.runId);
|
||||
const sessionId = normalizeCliContextValue(attribution.sessionId);
|
||||
const cliSessionId = normalizeCliContextValue(output.sessionId);
|
||||
const context = [
|
||||
runId ? `OpenClaw run: ${runId}.` : undefined,
|
||||
sessionId ? `OpenClaw session: ${sessionId}.` : undefined,
|
||||
cliSessionId ? `Claude session: ${cliSessionId}.` : undefined,
|
||||
].filter((entry): entry is string => Boolean(entry));
|
||||
const limit = output.terminalFailure.limit;
|
||||
return [
|
||||
`Claude CLI stopped after reaching the maximum number of turns${limit ? ` (limit: ${limit})` : ""}.`,
|
||||
...context,
|
||||
"Tool actions may already have run; verify their effects before retrying.",
|
||||
"Retry with a higher --max-turns value or a narrower task.",
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
export const CLI_STREAM_JSON_DEFAULT_MAX_TURN_RAW_CHARS = 8 * 1024 * 1024;
|
||||
const CLI_STREAM_JSON_MIN_TURN_RAW_CHARS = 1_024;
|
||||
const CLI_STREAM_JSON_MAX_CONFIGURABLE_TURN_RAW_CHARS = 64 * 1024 * 1024;
|
||||
@@ -121,6 +157,16 @@ function isGeminiStreamJsonDialect(params: {
|
||||
);
|
||||
}
|
||||
|
||||
function isClaudeStreamJsonDialect(params: {
|
||||
backend: CliBackendConfig;
|
||||
providerId: string;
|
||||
}): boolean {
|
||||
if (params.backend.jsonlDialect) {
|
||||
return params.backend.jsonlDialect === "claude-stream-json";
|
||||
}
|
||||
return isClaudeCliProvider(params.providerId);
|
||||
}
|
||||
|
||||
function isStreamJsonDialect(params: { backend: CliBackendConfig; providerId: string }): boolean {
|
||||
return supportsCliJsonlToolEvents(params);
|
||||
}
|
||||
@@ -379,6 +425,57 @@ function collectExplicitCliErrorText(parsed: Record<string, unknown>): string {
|
||||
return "";
|
||||
}
|
||||
|
||||
function readClaudeMaxTurnsFailure(
|
||||
parsed: Record<string, unknown>,
|
||||
): CliTerminalFailure | undefined {
|
||||
const subtype = typeof parsed.subtype === "string" ? parsed.subtype.trim() : "";
|
||||
const terminalReason =
|
||||
typeof parsed.terminal_reason === "string" ? parsed.terminal_reason.trim() : "";
|
||||
if (subtype !== "error_max_turns" && terminalReason !== "max_turns") {
|
||||
return undefined;
|
||||
}
|
||||
const errors = Array.isArray(parsed.errors) ? parsed.errors : [];
|
||||
for (const error of errors) {
|
||||
if (typeof error !== "string") {
|
||||
continue;
|
||||
}
|
||||
const match = error.match(/maximum number of turns\s*\((\d+)\)/i);
|
||||
if (match) {
|
||||
const limit = Number.parseInt(match[1] ?? "", 10);
|
||||
if (Number.isSafeInteger(limit) && limit > 0) {
|
||||
return {
|
||||
reason: "max_turns",
|
||||
limit,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
return { reason: "max_turns" };
|
||||
}
|
||||
|
||||
function readClaudeMaxTurnsErrorText(parsed: Record<string, unknown>): string | undefined {
|
||||
if (!Array.isArray(parsed.errors)) {
|
||||
return undefined;
|
||||
}
|
||||
for (const error of parsed.errors) {
|
||||
if (typeof error === "string" && error.trim()) {
|
||||
return error.trim();
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolveCliTerminalErrorText(
|
||||
parsed: Record<string, unknown>,
|
||||
terminalFailure: CliTerminalFailure | undefined,
|
||||
): string {
|
||||
const explicitErrorText = collectExplicitCliErrorText(parsed);
|
||||
return (
|
||||
((terminalFailure ? readClaudeMaxTurnsErrorText(parsed) : undefined) ?? explicitErrorText) ||
|
||||
(terminalFailure ? "Reached maximum number of turns." : "")
|
||||
);
|
||||
}
|
||||
|
||||
function pickCliSessionId(
|
||||
parsed: Record<string, unknown>,
|
||||
backend: CliBackendConfig,
|
||||
@@ -481,6 +578,21 @@ export function parseCliJson(
|
||||
for (const parsed of parsedRecords) {
|
||||
sessionId = pickCliSessionId(parsed, backend) ?? sessionId;
|
||||
usage = readCliUsage(parsed) ?? usage;
|
||||
const terminalFailure = isClaudeStreamJsonDialect({
|
||||
backend,
|
||||
providerId: providerId ?? "",
|
||||
})
|
||||
? readClaudeMaxTurnsFailure(parsed)
|
||||
: undefined;
|
||||
if (terminalFailure) {
|
||||
return {
|
||||
text: "",
|
||||
sessionId,
|
||||
usage,
|
||||
errorText: resolveCliTerminalErrorText(parsed, terminalFailure),
|
||||
terminalFailure,
|
||||
};
|
||||
}
|
||||
const subtype = typeof parsed.subtype === "string" ? parsed.subtype.trim() : "";
|
||||
const shouldClassifyError =
|
||||
parsed.is_error === true ||
|
||||
@@ -531,13 +643,17 @@ function parseClaudeCliJsonlResult(params: {
|
||||
return null;
|
||||
}
|
||||
if (typeof params.parsed.type === "string" && params.parsed.type === "result") {
|
||||
const errorText = collectExplicitCliErrorText(params.parsed);
|
||||
const terminalFailure = isClaudeStreamJsonDialect(params)
|
||||
? readClaudeMaxTurnsFailure(params.parsed)
|
||||
: undefined;
|
||||
const errorText = resolveCliTerminalErrorText(params.parsed, terminalFailure);
|
||||
if (errorText) {
|
||||
return {
|
||||
text: "",
|
||||
sessionId: params.sessionId,
|
||||
usage: params.usage,
|
||||
errorText,
|
||||
...(terminalFailure ? { terminalFailure } : {}),
|
||||
};
|
||||
}
|
||||
if (typeof params.parsed.result !== "string") {
|
||||
|
||||
@@ -4108,6 +4108,71 @@ ${JSON.stringify({
|
||||
);
|
||||
});
|
||||
|
||||
it("surfaces Claude live max-turn results with run and session recovery context", async () => {
|
||||
let stdoutListener: ((chunk: string) => void) | undefined;
|
||||
const stdin = {
|
||||
write: vi.fn((dataValue: string, cb?: (err?: Error | null) => void) => {
|
||||
stdoutListener?.(
|
||||
[
|
||||
JSON.stringify({
|
||||
type: "system",
|
||||
subtype: "init",
|
||||
session_id: "live-max-turns",
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "result",
|
||||
subtype: "error_max_turns",
|
||||
session_id: "live-max-turns",
|
||||
num_turns: 2,
|
||||
stop_reason: "tool_use",
|
||||
terminal_reason: "max_turns",
|
||||
errors: ["Reached maximum number of turns (1)"],
|
||||
}),
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
cb?.();
|
||||
}),
|
||||
end: vi.fn(),
|
||||
};
|
||||
supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => {
|
||||
const input = (args[0] ?? {}) as { onStdout?: (chunk: string) => void };
|
||||
stdoutListener = input.onStdout;
|
||||
return {
|
||||
runId: "live-run",
|
||||
pid: 2345,
|
||||
startedAtMs: Date.now(),
|
||||
stdin,
|
||||
wait: vi.fn(() => new Promise(() => {})),
|
||||
cancel: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
await expectRejectsWithFields(
|
||||
executePreparedCliRun(
|
||||
buildPreparedCliRunContext({
|
||||
provider: "claude-cli",
|
||||
model: "sonnet",
|
||||
runId: "run-live-max-turns",
|
||||
backend: {
|
||||
liveSession: "claude-stdio",
|
||||
},
|
||||
}),
|
||||
),
|
||||
{
|
||||
name: "FailoverError",
|
||||
message:
|
||||
"Claude CLI stopped after reaching the maximum number of turns (limit: 1). " +
|
||||
"OpenClaw run: run-live-max-turns. OpenClaw session: s1. " +
|
||||
"Claude session: live-max-turns. Tool actions may already have run; verify their effects before retrying. " +
|
||||
"Retry with a higher --max-turns value or a narrower task.",
|
||||
sessionId: "s1",
|
||||
reason: "unknown",
|
||||
code: "cli_max_turns",
|
||||
rawError: "Reached maximum number of turns (1)",
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("marks Claude live stderr context overflows as retryable", async () => {
|
||||
let stdoutListener: ((chunk: string) => void) | undefined;
|
||||
let resolveExit: ((exit: RunExit) => void) | undefined;
|
||||
|
||||
@@ -44,6 +44,7 @@ import { resolveCliToolTerminalReason } from "../run-termination.js";
|
||||
import { prepareCliBundleMcpCaptureAttempt } from "./bundle-mcp.js";
|
||||
import { buildClaudeOwnerKey } from "./helpers.js";
|
||||
import { cliBackendLog, formatCliBackendOutputDigest } from "./log.js";
|
||||
import { createCliOutputFailoverError } from "./output-error.js";
|
||||
import type { PreparedCliRunContext } from "./types.js";
|
||||
|
||||
type ProcessSupervisor = ReturnType<
|
||||
@@ -840,19 +841,6 @@ function parseClaudeLiveJsonLine(
|
||||
return isRecord(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function createParsedOutputError(session: ClaudeLiveSession, output: CliOutput): FailoverError {
|
||||
const message = output.errorText || "Claude CLI failed.";
|
||||
const reason = classifyFailoverReason(message, { provider: session.providerId }) ?? "unknown";
|
||||
const code = reason === "context_overflow" ? "cli_context_overflow" : undefined;
|
||||
return new FailoverError(message, {
|
||||
reason,
|
||||
provider: session.providerId,
|
||||
model: session.modelId,
|
||||
status: resolveFailoverStatus(reason),
|
||||
code,
|
||||
});
|
||||
}
|
||||
|
||||
function writeClaudeLiveControlResponse(session: ClaudeLiveSession, response: unknown): void {
|
||||
const stdin = session.managedRun.stdin;
|
||||
if (!stdin) {
|
||||
@@ -963,7 +951,16 @@ function handleClaudeLiveLine(session: ClaudeLiveSession, line: string): void {
|
||||
fallbackSessionId: turn.sessionId,
|
||||
});
|
||||
if (output.errorText) {
|
||||
failTurn(session, createParsedOutputError(session, output));
|
||||
const error = createCliOutputFailoverError({
|
||||
output,
|
||||
provider: session.providerId,
|
||||
model: session.modelId,
|
||||
runId: turn.diagnosticRefs.runId,
|
||||
sessionId: turn.diagnosticRefs.sessionId,
|
||||
});
|
||||
if (error) {
|
||||
failTurn(session, error);
|
||||
}
|
||||
scheduleIdleClose(session);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
} from "../../infra/diagnostic-events.js";
|
||||
import type { getProcessSupervisor } from "../../process/supervisor/index.js";
|
||||
import { createManagedRun, supervisorSpawnMock } from "../cli-runner.test-support.js";
|
||||
import { findCliMaxTurnsError } from "../failover-error.js";
|
||||
import { getCliMessagingDeliveryEvidence } from "./delivery-evidence.js";
|
||||
import { executePreparedCliRun } from "./execute.js";
|
||||
import type { PreparedCliRunContext } from "./types.js";
|
||||
@@ -69,7 +70,7 @@ function recordMcpLoopbackToolCallResult(params: {
|
||||
}
|
||||
|
||||
function buildPreparedCliRunContext(params: {
|
||||
output: "jsonl" | "text";
|
||||
output: "json" | "jsonl" | "text";
|
||||
provider?: string;
|
||||
runId?: string;
|
||||
beforeExecution?: () => Promise<void>;
|
||||
@@ -404,6 +405,186 @@ describe("executePreparedCliRun supervisor output capture", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("surfaces Claude max-turn results with run and session recovery context", async () => {
|
||||
const stdout = `${JSON.stringify({
|
||||
type: "result",
|
||||
subtype: "error_max_turns",
|
||||
session_id: "claude-session-max-turns",
|
||||
num_turns: 2,
|
||||
stop_reason: "tool_use",
|
||||
terminal_reason: "max_turns",
|
||||
errors: ["Reached maximum number of turns (1)"],
|
||||
})}\n`;
|
||||
|
||||
supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => {
|
||||
const input = args[0] as SupervisorSpawnInput;
|
||||
input.onStdout?.(stdout);
|
||||
return createManagedRun({
|
||||
reason: "exit",
|
||||
exitCode: 1,
|
||||
exitSignal: null,
|
||||
durationMs: 50,
|
||||
stdout: input.captureOutput === false ? "" : stdout,
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
noOutputTimedOut: false,
|
||||
});
|
||||
});
|
||||
|
||||
await expect(
|
||||
executePreparedCliRun(
|
||||
buildPreparedCliRunContext({
|
||||
output: "jsonl",
|
||||
provider: "claude-cli",
|
||||
runId: "run-max-turns",
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
name: "FailoverError",
|
||||
message:
|
||||
"Claude CLI stopped after reaching the maximum number of turns (limit: 1). " +
|
||||
"OpenClaw run: run-max-turns. OpenClaw session: session-1. " +
|
||||
"Claude session: claude-session-max-turns. Tool actions may already have run; verify their effects before retrying. " +
|
||||
"Retry with a higher --max-turns value or a narrower task.",
|
||||
sessionId: "session-1",
|
||||
reason: "unknown",
|
||||
code: "cli_max_turns",
|
||||
rawError: "Reached maximum number of turns (1)",
|
||||
});
|
||||
});
|
||||
|
||||
it("surfaces Claude max-turn results from JSON output", async () => {
|
||||
const stdout = JSON.stringify({
|
||||
type: "result",
|
||||
subtype: "error_max_turns",
|
||||
session_id: "claude-json-max-turns",
|
||||
terminal_reason: "max_turns",
|
||||
errors: ["Reached maximum number of turns (2)"],
|
||||
});
|
||||
supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => {
|
||||
const input = args[0] as SupervisorSpawnInput;
|
||||
input.onStdout?.(stdout);
|
||||
return createManagedRun({
|
||||
reason: "exit",
|
||||
exitCode: 1,
|
||||
exitSignal: null,
|
||||
durationMs: 50,
|
||||
stdout: input.captureOutput === false ? "" : stdout,
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
noOutputTimedOut: false,
|
||||
});
|
||||
});
|
||||
|
||||
await expect(
|
||||
executePreparedCliRun(
|
||||
buildPreparedCliRunContext({
|
||||
output: "json",
|
||||
provider: "claude-cli",
|
||||
runId: "run-json-max-turns",
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
name: "FailoverError",
|
||||
code: "cli_max_turns",
|
||||
rawError: "Reached maximum number of turns (2)",
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["no-output-timeout", true],
|
||||
["overall-timeout", false],
|
||||
] as const)(
|
||||
"keeps a terminal max-turn result ahead of a later %s",
|
||||
async (reason, noOutputTimedOut) => {
|
||||
const stdout = `${JSON.stringify({
|
||||
type: "result",
|
||||
subtype: "error_max_turns",
|
||||
session_id: `claude-${reason}`,
|
||||
terminal_reason: "max_turns",
|
||||
errors: ["Reached maximum number of turns (1)"],
|
||||
})}\n`;
|
||||
supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => {
|
||||
const input = args[0] as SupervisorSpawnInput;
|
||||
input.onStdout?.(stdout);
|
||||
return createManagedRun({
|
||||
reason,
|
||||
exitCode: null,
|
||||
exitSignal: "SIGTERM",
|
||||
durationMs: 1_000,
|
||||
stdout: input.captureOutput === false ? "" : stdout,
|
||||
stderr: "",
|
||||
timedOut: true,
|
||||
noOutputTimedOut,
|
||||
});
|
||||
});
|
||||
|
||||
await expect(
|
||||
executePreparedCliRun(
|
||||
buildPreparedCliRunContext({ output: "jsonl", provider: "claude-cli" }),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
name: "FailoverError",
|
||||
code: "cli_max_turns",
|
||||
rawError: "Reached maximum number of turns (1)",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves max-turn failure through fork successor persistence errors", async () => {
|
||||
const stdout = `${JSON.stringify({
|
||||
type: "result",
|
||||
subtype: "error_max_turns",
|
||||
session_id: "fork-successor",
|
||||
terminal_reason: "max_turns",
|
||||
errors: ["Reached maximum number of turns (1)"],
|
||||
})}\n`;
|
||||
supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => {
|
||||
const input = args[0] as SupervisorSpawnInput;
|
||||
input.onStdout?.(stdout);
|
||||
return createManagedRun({
|
||||
reason: "exit",
|
||||
exitCode: 1,
|
||||
exitSignal: null,
|
||||
durationMs: 50,
|
||||
stdout: input.captureOutput === false ? "" : stdout,
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
noOutputTimedOut: false,
|
||||
});
|
||||
});
|
||||
const persistenceError = new Error("fork successor persistence failed");
|
||||
const persistCliSessionForkSuccessor = vi.fn().mockRejectedValue(persistenceError);
|
||||
const restoreCliSessionFork = vi.fn().mockResolvedValue(undefined);
|
||||
const context = buildPreparedCliRunContext({
|
||||
output: "jsonl",
|
||||
provider: "claude-cli",
|
||||
runId: "run-fork-max-turns",
|
||||
});
|
||||
context.preparedBackend.backend.resumeArgs = ["--resume", "{sessionId}"];
|
||||
context.preparedBackend.backend.forkArg = "--fork-session";
|
||||
context.params.forkCliSessionOnResume = true;
|
||||
context.params.claimCliSessionFork = vi.fn().mockResolvedValue(true);
|
||||
context.params.persistCliSessionForkSuccessor = persistCliSessionForkSuccessor;
|
||||
context.params.restoreCliSessionFork = restoreCliSessionFork;
|
||||
|
||||
let failure: unknown;
|
||||
try {
|
||||
await executePreparedCliRun(context, "fork-source");
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
}
|
||||
|
||||
expect(failure).toBeInstanceOf(AggregateError);
|
||||
expect((failure as AggregateError).errors).toEqual([
|
||||
expect.objectContaining({ code: "cli_max_turns" }),
|
||||
persistenceError,
|
||||
]);
|
||||
expect(findCliMaxTurnsError(failure)).toMatchObject({ code: "cli_max_turns" });
|
||||
expect(persistCliSessionForkSuccessor).toHaveBeenCalledWith("fork-successor");
|
||||
expect(restoreCliSessionFork).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("still streams every JSONL stdout chunk with supervisor capture disabled", async () => {
|
||||
// Streaming events are emitted from live chunks, not from the final captured
|
||||
// stdout string, so users still see deltas when captureOutput is false.
|
||||
|
||||
@@ -104,6 +104,7 @@ import {
|
||||
formatCliBackendOutputDigest,
|
||||
LEGACY_CLAUDE_CLI_LOG_OUTPUT_ENV,
|
||||
} from "./log.js";
|
||||
import { createCliOutputFailoverError } from "./output-error.js";
|
||||
import type { CliReusableSession, PreparedCliRunContext } from "./types.js";
|
||||
|
||||
const executeDeps = {
|
||||
@@ -1767,6 +1768,35 @@ export async function executePreparedCliRun(
|
||||
}
|
||||
}
|
||||
|
||||
const streamedJsonlOutput =
|
||||
outputMode === "jsonl" ? (streamingParser?.getOutput() ?? null) : null;
|
||||
const parsedStructuredOutput =
|
||||
streamedJsonlOutput ??
|
||||
(outputMode === "json" && !stdoutParseExceeded
|
||||
? parseCliOutput({
|
||||
raw: stdout,
|
||||
backend,
|
||||
providerId: context.backendResolved.id,
|
||||
outputMode,
|
||||
fallbackSessionId: resolvedSessionId,
|
||||
})
|
||||
: null);
|
||||
// A completed terminal record is authoritative even if the CLI hangs
|
||||
// afterward. Reclassifying it as a timeout could replay completed tools.
|
||||
if (parsedStructuredOutput?.terminalFailure) {
|
||||
const terminalError = createCliOutputFailoverError({
|
||||
output: parsedStructuredOutput,
|
||||
provider: params.provider,
|
||||
model: context.modelId,
|
||||
runId: params.runId,
|
||||
sessionId: params.sessionId,
|
||||
lane: params.lane,
|
||||
});
|
||||
if (terminalError) {
|
||||
throw terminalError;
|
||||
}
|
||||
}
|
||||
|
||||
if (result.exitCode !== 0 || result.reason !== "exit") {
|
||||
if (result.reason === "no-output-timeout" || result.noOutputTimedOut) {
|
||||
const timeoutReason = `CLI produced no output for ${Math.round(noOutputTimeoutMs / 1000)}s and was terminated.`;
|
||||
@@ -1876,9 +1906,6 @@ export async function executePreparedCliRun(
|
||||
});
|
||||
}
|
||||
|
||||
const streamedJsonlOutput =
|
||||
outputMode === "jsonl" ? (streamingParser?.getOutput() ?? null) : null;
|
||||
|
||||
if (stdoutParseExceeded && !streamedJsonlOutput) {
|
||||
throw new FailoverError(
|
||||
`CLI stdout exceeded ${CLI_RUNNER_OUTPUT_PARSE_BYTES} bytes; refusing to parse truncated output.`,
|
||||
@@ -1894,7 +1921,7 @@ export async function executePreparedCliRun(
|
||||
}
|
||||
|
||||
const parsed =
|
||||
streamedJsonlOutput ??
|
||||
parsedStructuredOutput ??
|
||||
parseCliOutput({
|
||||
raw: stdout,
|
||||
backend,
|
||||
@@ -1902,19 +1929,16 @@ export async function executePreparedCliRun(
|
||||
outputMode,
|
||||
fallbackSessionId: resolvedSessionId,
|
||||
});
|
||||
if (parsed.errorText) {
|
||||
const reason =
|
||||
classifyFailoverReason(parsed.errorText, { provider: params.provider }) ?? "unknown";
|
||||
const code = reason === "context_overflow" ? "cli_context_overflow" : undefined;
|
||||
throw new FailoverError(parsed.errorText, {
|
||||
reason,
|
||||
provider: params.provider,
|
||||
model: context.modelId,
|
||||
sessionId: params.sessionId,
|
||||
lane: params.lane,
|
||||
status: resolveFailoverStatus(reason),
|
||||
code,
|
||||
});
|
||||
const parsedError = createCliOutputFailoverError({
|
||||
output: parsed,
|
||||
provider: params.provider,
|
||||
model: context.modelId,
|
||||
runId: params.runId,
|
||||
sessionId: params.sessionId,
|
||||
lane: params.lane,
|
||||
});
|
||||
if (parsedError) {
|
||||
throw parsedError;
|
||||
}
|
||||
const rawText = parsed.text;
|
||||
cliBackendLog.info(
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { formatCliOutputError, type CliOutput } from "../cli-output.js";
|
||||
import { classifyFailoverReason } from "../embedded-agent-helpers.js";
|
||||
import { FailoverError, resolveFailoverStatus } from "../failover-error.js";
|
||||
|
||||
export function createCliOutputFailoverError(params: {
|
||||
output: CliOutput;
|
||||
provider: string;
|
||||
model: string;
|
||||
runId?: string;
|
||||
sessionId?: string;
|
||||
lane?: string;
|
||||
}): FailoverError | undefined {
|
||||
if (!params.output.errorText) {
|
||||
return undefined;
|
||||
}
|
||||
const message = formatCliOutputError(params.output, {
|
||||
runId: params.runId,
|
||||
sessionId: params.sessionId,
|
||||
});
|
||||
const reason = classifyFailoverReason(message, { provider: params.provider }) ?? "unknown";
|
||||
const code =
|
||||
params.output.terminalFailure?.reason === "max_turns"
|
||||
? "cli_max_turns"
|
||||
: reason === "context_overflow"
|
||||
? "cli_context_overflow"
|
||||
: undefined;
|
||||
return new FailoverError(message, {
|
||||
reason,
|
||||
provider: params.provider,
|
||||
model: params.model,
|
||||
sessionId: params.sessionId,
|
||||
lane: params.lane,
|
||||
status: resolveFailoverStatus(reason),
|
||||
code,
|
||||
rawError: params.output.errorText,
|
||||
});
|
||||
}
|
||||
@@ -3931,6 +3931,7 @@ async function runEmbeddedAgentInternal(
|
||||
aborted,
|
||||
externalAbort,
|
||||
fallbackConfigured,
|
||||
failoverCode: promptErrorDetails.code,
|
||||
failoverFailure: promptFailoverFailure,
|
||||
failoverReason: promptFailoverReason,
|
||||
harnessOwnsTransport: pluginHarnessOwnsTransport,
|
||||
@@ -3972,6 +3973,7 @@ async function runEmbeddedAgentInternal(
|
||||
aborted,
|
||||
externalAbort,
|
||||
fallbackConfigured,
|
||||
failoverCode: promptErrorDetails.code,
|
||||
failoverFailure: promptFailoverFailure,
|
||||
failoverReason: promptFailoverReason,
|
||||
harnessOwnsTransport: pluginHarnessOwnsTransport,
|
||||
|
||||
@@ -95,6 +95,24 @@ describe("resolveRunFailoverDecision", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("surfaces max-turn prompt failures without profile rotation or model fallback", () => {
|
||||
expect(
|
||||
resolveRunFailoverDecision({
|
||||
stage: "prompt",
|
||||
aborted: false,
|
||||
externalAbort: false,
|
||||
fallbackConfigured: true,
|
||||
failoverCode: "cli_max_turns",
|
||||
failoverFailure: true,
|
||||
failoverReason: "unknown",
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
action: "surface_error",
|
||||
reason: "unknown",
|
||||
});
|
||||
});
|
||||
|
||||
it("surfaces prompt run-budget timeouts instead of model fallback (#60388)", () => {
|
||||
expect(
|
||||
resolveRunFailoverDecision({
|
||||
|
||||
@@ -47,6 +47,7 @@ type PromptDecisionParams = {
|
||||
aborted: boolean;
|
||||
externalAbort: boolean;
|
||||
fallbackConfigured: boolean;
|
||||
failoverCode?: string;
|
||||
failoverFailure: boolean;
|
||||
failoverReason: FailoverReason | null;
|
||||
harnessOwnsTransport?: boolean;
|
||||
@@ -177,6 +178,14 @@ export function resolveRunFailoverDecision(params: RunFailoverDecisionParams): R
|
||||
}
|
||||
|
||||
if (params.stage === "prompt") {
|
||||
if (params.failoverCode === "cli_max_turns") {
|
||||
// A CLI may have completed tool actions before reaching this terminal
|
||||
// limit. Replaying against another profile/model could repeat effects.
|
||||
return {
|
||||
action: "surface_error",
|
||||
reason: params.failoverReason,
|
||||
};
|
||||
}
|
||||
if (params.externalAbort) {
|
||||
return {
|
||||
action: "surface_error",
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
coerceToFailoverError,
|
||||
describeFailoverError,
|
||||
FailoverError,
|
||||
findCliMaxTurnsError,
|
||||
isNonProviderRuntimeCoordinationError,
|
||||
isSignalTimeoutReason,
|
||||
isTimeoutError,
|
||||
@@ -61,6 +62,19 @@ const OPENAI_SERVER_ERROR_PAYLOAD =
|
||||
'Codex error: {"type":"error","error":{"type":"server_error","code":"server_error","message":"An error occurred while processing your request."},"sequence_number":2}';
|
||||
|
||||
describe("failover-error", () => {
|
||||
it("finds CLI max-turn failures through aggregate wrappers", () => {
|
||||
const maxTurns = new FailoverError("max turns", {
|
||||
reason: "unknown",
|
||||
code: "cli_max_turns",
|
||||
});
|
||||
const aggregate = new AggregateError(
|
||||
[new Error("fork successor persistence failed"), { error: maxTurns }],
|
||||
"CLI turn and persistence failed",
|
||||
);
|
||||
|
||||
expect(findCliMaxTurnsError(aggregate)).toBe(maxTurns);
|
||||
});
|
||||
|
||||
it("infers failover reason from HTTP status", () => {
|
||||
expect(resolveFailoverReasonFromError({ status: 402 })).toBe("billing");
|
||||
// Anthropic Claude Max plan surfaces rate limits as HTTP 402 (#30484)
|
||||
|
||||
@@ -90,6 +90,34 @@ export function isFailoverError(err: unknown): err is FailoverError {
|
||||
);
|
||||
}
|
||||
|
||||
export function findCliMaxTurnsError(
|
||||
err: unknown,
|
||||
seen: Set<object> = new Set(),
|
||||
): FailoverError | undefined {
|
||||
if (isFailoverError(err) && err.code === "cli_max_turns") {
|
||||
return err;
|
||||
}
|
||||
if (!err || typeof err !== "object" || seen.has(err)) {
|
||||
return undefined;
|
||||
}
|
||||
// Fork persistence can aggregate a terminal run error with its own failure.
|
||||
// Keep max-turn replay protection intact across those wrapper boundaries.
|
||||
seen.add(err);
|
||||
const candidate = err as { error?: unknown; cause?: unknown; errors?: unknown };
|
||||
const nested = [
|
||||
candidate.error,
|
||||
candidate.cause,
|
||||
...(Array.isArray(candidate.errors) ? candidate.errors : []),
|
||||
];
|
||||
for (const value of nested) {
|
||||
const found = findCliMaxTurnsError(value, seen);
|
||||
if (found) {
|
||||
return found;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Map a failover reason to the closest HTTP-like status code. */
|
||||
export function resolveFailoverStatus(reason: FailoverReason): number | undefined {
|
||||
switch (reason) {
|
||||
|
||||
@@ -712,6 +712,55 @@ describe("runWithModelFallback", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("does not replay CLI max-turn failures on configured fallback models", async () => {
|
||||
const failure = new FailoverError(
|
||||
"Claude CLI stopped after reaching the maximum number of turns (limit: 1). Tool actions may already have run; verify their effects before retrying.",
|
||||
{
|
||||
provider: "claude-cli",
|
||||
model: "sonnet",
|
||||
reason: "unknown",
|
||||
code: "cli_max_turns",
|
||||
},
|
||||
);
|
||||
const run = vi.fn().mockRejectedValue(failure);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg: makeDiagnosticFallbackConfig(["anthropic/claude-opus-4-7"]),
|
||||
provider: "claude-cli",
|
||||
model: "sonnet",
|
||||
run,
|
||||
}),
|
||||
).rejects.toBe(failure);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not replay aggregate failures containing a CLI max-turn stop", async () => {
|
||||
const maxTurns = new FailoverError("max turns", {
|
||||
provider: "claude-cli",
|
||||
model: "sonnet",
|
||||
reason: "unknown",
|
||||
code: "cli_max_turns",
|
||||
});
|
||||
const failure = new AggregateError(
|
||||
[maxTurns, new Error("fork successor persistence failed")],
|
||||
"CLI turn failed and its fork successor could not be persisted",
|
||||
);
|
||||
const run = vi.fn().mockRejectedValue(failure);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg: makeDiagnosticFallbackConfig(["anthropic/claude-opus-4-7"]),
|
||||
provider: "claude-cli",
|
||||
model: "sonnet",
|
||||
run,
|
||||
}),
|
||||
).rejects.toBe(failure);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("uses the opt-in auth skip cache on the second turn for the same session", async () => {
|
||||
const previous = process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS;
|
||||
process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS = "60000";
|
||||
|
||||
@@ -40,6 +40,7 @@ import {
|
||||
buildProviderReauthCommand,
|
||||
coerceToFailoverError,
|
||||
describeFailoverError,
|
||||
findCliMaxTurnsError,
|
||||
isFailoverError,
|
||||
isNonProviderRuntimeCoordinationError,
|
||||
resolveModelFallbackError,
|
||||
@@ -1804,6 +1805,11 @@ async function runWithModelFallbackInternal<T>(
|
||||
return attemptRun.success;
|
||||
}
|
||||
const err = attemptRun.error;
|
||||
// Max-turn termination can follow successful tool actions. Stop before
|
||||
// candidate fallback so the user can verify effects before any replay.
|
||||
if (findCliMaxTurnsError(err)) {
|
||||
throw err;
|
||||
}
|
||||
if (
|
||||
!attemptRun.classifiedResult &&
|
||||
params.canFallbackAfterError &&
|
||||
|
||||
@@ -7079,6 +7079,36 @@ describe("runAgentTurnWithFallback", () => {
|
||||
expect(terminalFailureEvent).toBeDefined();
|
||||
});
|
||||
|
||||
it("surfaces CLI max-turn recovery context at normal verbosity", async () => {
|
||||
const recoveryText =
|
||||
"Claude CLI stopped after reaching the maximum number of turns (limit: 1). " +
|
||||
"OpenClaw run: run-max-turns. OpenClaw session: session-1. Claude session: claude-session-1. " +
|
||||
"Tool actions may already have run; verify their effects before retrying. " +
|
||||
"Retry with a higher --max-turns value or a narrower task.";
|
||||
const maxTurns = new FailoverError(recoveryText, {
|
||||
reason: "unknown",
|
||||
code: "cli_max_turns",
|
||||
provider: "claude-cli",
|
||||
model: "sonnet",
|
||||
});
|
||||
state.runEmbeddedAgentMock.mockRejectedValueOnce(
|
||||
new AggregateError(
|
||||
[maxTurns, new Error("fork successor persistence failed")],
|
||||
"CLI turn failed and its fork successor could not be persisted",
|
||||
),
|
||||
);
|
||||
|
||||
const runAgentTurnWithFallback = await getRunAgentTurnWithFallback();
|
||||
const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams());
|
||||
|
||||
expect(result.kind).toBe("final");
|
||||
if (result.kind === "final") {
|
||||
expect(result.payload.isError).toBe(true);
|
||||
expect(result.payload.text).toBe(recoveryText);
|
||||
expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT);
|
||||
}
|
||||
});
|
||||
|
||||
it("uses heartbeat failure copy for raw external errors during heartbeat runs", async () => {
|
||||
state.runEmbeddedAgentMock.mockRejectedValueOnce(
|
||||
new Error('Command lane "main" task timed out after 120000ms'),
|
||||
|
||||
@@ -49,7 +49,7 @@ import { isMessagingToolSendAction } from "../../agents/embedded-agent-messaging
|
||||
import { mergeEmbeddedAgentRunResultForModelFallbackExhaustion } from "../../agents/embedded-agent-runner/result-fallback-classifier.js";
|
||||
import type { RunEmbeddedAgentParams } from "../../agents/embedded-agent-runner/run/params.js";
|
||||
import { runEmbeddedAgent } from "../../agents/embedded-agent.js";
|
||||
import { isFailoverError } from "../../agents/failover-error.js";
|
||||
import { findCliMaxTurnsError, isFailoverError } from "../../agents/failover-error.js";
|
||||
import type { FastModeAutoProgressState } from "../../agents/fast-mode.js";
|
||||
import { resolveAgentHarnessPolicy } from "../../agents/harness/policy.js";
|
||||
import { ensureSelectedAgentHarnessPlugin } from "../../agents/harness/runtime-plugin.js";
|
||||
@@ -792,6 +792,13 @@ function buildExternalRunFailureReply(
|
||||
if (authProfileFailoverFailure) {
|
||||
return { text: authProfileFailoverFailure, isGenericRunnerFailure: false };
|
||||
}
|
||||
const cliMaxTurnsError = findCliMaxTurnsError(error);
|
||||
if (cliMaxTurnsError) {
|
||||
return {
|
||||
text: sanitizeUserFacingText(cliMaxTurnsError.message, { errorContext: true }),
|
||||
isGenericRunnerFailure: false,
|
||||
};
|
||||
}
|
||||
const providerRequestError = classifyProviderRequestError(error ?? normalizedMessage);
|
||||
if (providerRequestError) {
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user