fix(codex): session permissions persist across resumed turns (#111136)

* test: harden Codex compaction command oracle

* test: accept omitted Codex command exit code

* fix(codex): apply stored session permissions

* test: harden Codex gateway stress evidence

* test: accept successful Codex command retries

* test: classify Codex evidence as live helper
This commit is contained in:
Peter Steinberger
2026-07-18 19:56:42 -07:00
committed by GitHub
parent bdfe2a4e60
commit 208036afdc
10 changed files with 1112 additions and 84 deletions
+1
View File
@@ -73,6 +73,7 @@ Docs: https://docs.openclaw.ai
- **Control UI chat transcripts:** preserve loaded history across session and pane returns, bound automatic backscroll loading, virtualize long transcripts, retain hidden native run boundaries, and keep prepends, streaming, and responsive layouts from flickering or jumping. Thanks @shakkernerd.
- **Codex dynamic tool outcomes:** use the shared tool-result failure contract for arbitrary lifecycle metadata, preventing successful Skill Workshop results from being displayed and persisted as failed calls. Fixes #107684. Thanks @shakkernerd.
- **Codex `/status` context freshness:** consume exact per-response usage from Codex app servers that emit `rawResponse/completed`; when exact usage is unavailable or omitted, keep context unknown instead of reusing cumulative lifetime totals. (#107813) Thanks @wuqxuan.
- **Codex resumed permissions:** apply stored per-session approval and sandbox overrides to primary resumed harness turns so `/codex permissions` survives later messages and gateway restarts.
- **Nested resource ignores:** honor slash-free patterns and escaped literal exclamation marks in nested ignore files during skill and resource discovery. Thanks @moguangyu5-design.
- **Proxy bypass precedence:** honor blank lower-case `no_proxy` values shadowing upper-case `NO_PROXY` consistently with Undici, and reuse the canonical matcher for Telegram fallback selection.
- **Tokenjuice exec compaction:** avoid retaining raw command output inside compacted middleware metadata, preventing large successful compactions from failing the middleware details-size guard.
@@ -45,6 +45,23 @@ import {
import { getLeasedSharedCodexAppServerClient } from "./shared-client.js";
import { rotateOversizedCodexAppServerStartupBinding } from "./startup-binding.js";
function applyStoredBindingPermissions(params: {
appServer: ReturnType<typeof resolveCodexBindingAppServerConnection>["appServer"];
binding: CodexAppServerThreadBinding | undefined;
execPolicyTouched: boolean;
}) {
if (params.execPolicyTouched || params.binding?.connectionScope === "supervision") {
return params.appServer;
}
// `/codex permissions` owns per-session policy. Explicit OpenClaw exec config
// and supervised private connections remain authoritative when present.
return {
...params.appServer,
approvalPolicy: params.binding?.approvalPolicy ?? params.appServer.approvalPolicy,
sandbox: params.binding?.sandbox ?? params.appServer.sandbox,
};
}
export async function prepareCodexAttemptConnection({ params, options }: CodexRunAttemptInput) {
const attemptStartedAt = Date.now();
const profilerEnabled = isCodexAppServerProfilerEnabled(params.config);
@@ -135,16 +152,20 @@ export async function prepareCodexAttemptConnection({ params, options }: CodexRu
);
}
const resolveRuntimeOptionsForBinding = (selection: { modelProvider?: string; model?: string }) =>
resolveCodexBindingAppServerConnection({
applyStoredBindingPermissions({
appServer: resolveCodexBindingAppServerConnection({
binding: startupBinding,
pluginConfig,
execPolicy,
modelProvider: selection.modelProvider,
model: selection.model,
config: params.config,
agentDir,
openClawSandboxActive: sandbox?.enabled === true,
}).appServer,
binding: startupBinding,
pluginConfig,
execPolicy,
modelProvider: selection.modelProvider,
model: selection.model,
config: params.config,
agentDir,
openClawSandboxActive: sandbox?.enabled === true,
}).appServer;
execPolicyTouched: execPolicy.touched,
});
const initialStartupBindingHadInactiveThreadBootstrap =
isInactiveThreadBootstrapBinding(startupBinding);
const preparedAuthRoute = usesSupervisionConnection
@@ -327,16 +348,20 @@ export async function prepareCodexAttemptConnection({ params, options }: CodexRu
modelProvider?: string;
model?: string;
}) =>
resolveCodexBindingAppServerConnection({
applyStoredBindingPermissions({
appServer: resolveCodexBindingAppServerConnection({
binding: mutable.startupBinding,
pluginConfig,
execPolicy,
modelProvider: selection.modelProvider,
model: selection.model,
config: params.config,
agentDir,
openClawSandboxActive: sandbox?.enabled === true,
}).appServer,
binding: mutable.startupBinding,
pluginConfig,
execPolicy,
modelProvider: selection.modelProvider,
model: selection.model,
config: params.config,
agentDir,
openClawSandboxActive: sandbox?.enabled === true,
}).appServer;
execPolicyTouched: execPolicy.touched,
});
return {
params,
options,
@@ -3680,6 +3680,32 @@ describe("runCodexAppServerAttempt", () => {
expect(startParams?.sandbox).toBe("danger-full-access");
});
it("applies stored session permissions to resumed harness turns", async () => {
const sessionFile = path.join(tempDir, "session.jsonl");
const workspaceDir = path.join(tempDir, "workspace");
await writeExistingBinding(sessionFile, workspaceDir, {
approvalPolicy: "never",
sandbox: "danger-full-access",
});
const harness = createResumeHarness();
const run = runCodexAppServerAttempt(createParams(sessionFile, workspaceDir), {
pluginConfig: { appServer: { mode: "guardian" } },
});
await harness.waitForMethod("turn/start");
await harness.completeTurn({ threadId: "thread-existing", turnId: "turn-1" });
await run;
const resumeParams = harness.requests.find((request) => request.method === "thread/resume")
?.params as Record<string, unknown> | undefined;
const turnParams = harness.requests.find((request) => request.method === "turn/start")
?.params as Record<string, unknown> | undefined;
expect(resumeParams?.approvalPolicy).toBe("never");
expect(resumeParams?.sandbox).toBe("danger-full-access");
expect(turnParams?.approvalPolicy).toBe("never");
expect(turnParams?.sandboxPolicy).toEqual({ type: "dangerFullAccess" });
});
it("keeps normalized full exec mode unpromoted when OpenClaw tool policy exists", async () => {
initializeGlobalHookRunner(
createMockPluginRegistry([{ hookName: "before_tool_call", handler: vi.fn() }]),
@@ -0,0 +1,231 @@
// Live-only command evidence shared by the gateway stress test and its unit coverage.
import {
isExpectedNativeCommand,
requireSuccessfulNativeCommandExecution,
} from "./gateway-codex-harness.live-helpers.js";
function readPersistedToolCalls(
message: unknown,
commandMarker: string,
expectedCommand: string,
): Array<{ id: string; matches: boolean }> {
if (!message || typeof message !== "object") {
return [];
}
const record = message as { role?: unknown; content?: unknown };
if (record.role !== "assistant" || !Array.isArray(record.content)) {
return [];
}
const calls: Array<{ id: string; matches: boolean }> = [];
for (const item of record.content) {
if (!item || typeof item !== "object") {
continue;
}
const content = item as {
type?: unknown;
id?: unknown;
name?: unknown;
arguments?: unknown;
};
if (
content.type !== "toolCall" ||
content.name !== "bash" ||
typeof content.id !== "string" ||
!content.arguments ||
typeof content.arguments !== "object"
) {
continue;
}
const command = (content.arguments as { command?: unknown }).command;
calls.push({
id: content.id,
matches:
typeof command === "string" &&
command.includes(commandMarker) &&
isExpectedNativeCommand(command, expectedCommand),
});
}
return calls;
}
function readPersistedTextContent(content: unknown): string | undefined {
if (!Array.isArray(content)) {
return undefined;
}
const blocks = content.flatMap((item) => {
if (!item || typeof item !== "object") {
return [];
}
const block = item as { text?: unknown };
return typeof block.text === "string" ? [block.text] : [];
});
return blocks.length > 0 ? blocks.join("\n") : undefined;
}
function isSuccessfulPersistedResult(record: { details?: unknown; isError?: unknown }): boolean {
if (record.isError !== false) {
return false;
}
if (!record.details || typeof record.details !== "object") {
return true;
}
const details = record.details as {
exitCode?: unknown;
status?: unknown;
timedOut?: unknown;
};
return (
(details.exitCode === undefined || details.exitCode === null || details.exitCode === 0) &&
(details.status === undefined || details.status === "completed") &&
details.timedOut !== true
);
}
/** Requires durable history to prove one successful native bash execution and its large output. */
export function requireSuccessfulPersistedNativeCommandExecution(
messages: readonly unknown[],
params: {
commandMarker: string;
expectedCommand: string;
minimumOutputChars: number;
toolCallId?: string;
},
): { callIndex: number; resultIndex: number; toolCallId: string } {
const calls: Array<{ id: string; callIndex: number; matches: boolean }> = [];
for (let callIndex = 0; callIndex < messages.length; callIndex += 1) {
for (const call of readPersistedToolCalls(
messages[callIndex],
params.commandMarker,
params.expectedCommand,
)) {
calls.push({ ...call, callIndex });
}
}
const observedResults: unknown[] = [];
for (let resultIndex = 0; resultIndex < messages.length; resultIndex += 1) {
const message = messages[resultIndex];
if (!message || typeof message !== "object") {
continue;
}
const record = message as {
role?: unknown;
toolCallId?: unknown;
isError?: unknown;
content?: unknown;
details?: unknown;
};
if (record.role !== "toolResult" || typeof record.toolCallId !== "string") {
continue;
}
if (params.toolCallId !== undefined && record.toolCallId !== params.toolCallId) {
continue;
}
const content = readPersistedTextContent(record.content);
const originalLengths =
typeof content === "string"
? Array.from(content.matchAll(/original (\d+) chars/gu), (match) => Number(match[1]))
: [];
const hasMarker = typeof content === "string" && content.includes(params.commandMarker);
const associatedCall = calls.find((call) => call.id === record.toolCallId);
const callIndex = associatedCall?.matches === true ? associatedCall.callIndex : -1;
if (
(associatedCall?.matches === true ||
(associatedCall === undefined && params.toolCallId !== undefined)) &&
isSuccessfulPersistedResult(record) &&
hasMarker &&
originalLengths.some((length) => length >= params.minimumOutputChars)
) {
return { callIndex, resultIndex, toolCallId: record.toolCallId };
}
if (hasMarker || callIndex >= 0) {
observedResults.push({
resultIndex,
toolCallId: record.toolCallId,
isError: record.isError,
details:
record.details && typeof record.details === "object"
? {
status: (record.details as { status?: unknown }).status,
exitCode: (record.details as { exitCode?: unknown }).exitCode,
timedOut: (record.details as { timedOut?: unknown }).timedOut,
}
: undefined,
originalLengths,
...(typeof content === "string"
? {
contentLength: content.length,
}
: {}),
});
}
}
throw new Error(
`persisted native bash command for marker ${params.commandMarker} has no successful large result; observed=${JSON.stringify(observedResults)}`,
);
}
function hasPersistedToolResult(messages: readonly unknown[], toolCallId: string): boolean {
return messages.some(
(message) =>
message !== null &&
typeof message === "object" &&
(message as { role?: unknown }).role === "toolResult" &&
(message as { toolCallId?: unknown }).toolCallId === toolCallId,
);
}
/** Proves the large command either survived in history or was replaced by later compaction. */
export function requireSuccessfulNativeCommandCompactionEvidence(params: {
commandMarker: string;
events: readonly unknown[];
expectedCommand: string;
messages: readonly unknown[];
minimumOutputChars: number;
}): { source: "compacted-event" | "persisted-history" } {
let requestEvidence: ReturnType<typeof requireSuccessfulNativeCommandExecution>;
try {
requestEvidence = requireSuccessfulNativeCommandExecution(params.events, params);
} catch (eventError) {
throw new Error(
`large native command has no successful request-local evidence: ${String(eventError)}`,
{
cause: eventError,
},
);
}
let persistedError: unknown;
try {
requireSuccessfulPersistedNativeCommandExecution(params.messages, {
...params,
toolCallId: requestEvidence.itemId,
});
return { source: "persisted-history" };
} catch (error) {
persistedError = error;
}
if (hasPersistedToolResult(params.messages, requestEvidence.itemId)) {
throw new Error(
`durable result for successful request-local command failed validation: ${String(persistedError)}`,
{ cause: persistedError },
);
}
const compactedAfterResult = params.events.some(
(event, index) =>
index > requestEvidence.resultIndex &&
event !== null &&
typeof event === "object" &&
(event as { stream?: unknown }).stream === "compaction" &&
(event as { data?: { phase?: unknown; completed?: unknown } }).data?.phase === "end" &&
(event as { data?: { phase?: unknown; completed?: unknown } }).data?.completed === true,
);
if (!compactedAfterResult) {
throw new Error(
`successful request-local command result was not followed by compaction; persisted=${String(persistedError)}`,
);
}
return { source: "compacted-event" };
}
@@ -0,0 +1,55 @@
import { describe, expect, it } from "vitest";
import { requireSuccessfulNativeCommandExecution } from "./gateway-codex-harness.live-helpers.js";
describe("Codex gateway command retry evidence", () => {
it("accepts a successful retry after an earlier matching command fails", () => {
const expectedCommand = "node -e OPENCLAW-RETRY";
const events = [
{
stream: "tool",
data: {
phase: "start",
name: "bash",
itemId: "item-first",
args: { command: expectedCommand },
},
},
{
stream: "tool",
data: {
phase: "result",
itemId: "item-first",
status: "completed",
isError: true,
result: { exitCode: 1 },
},
},
{
stream: "tool",
data: {
phase: "start",
name: "bash",
itemId: "item-retry",
args: { command: expectedCommand },
},
},
{
stream: "tool",
data: {
phase: "result",
itemId: "item-retry",
status: "completed",
isError: false,
result: { exitCode: 0 },
},
},
];
expect(
requireSuccessfulNativeCommandExecution(events, {
commandMarker: "OPENCLAW-RETRY",
expectedCommand,
}),
).toEqual({ itemId: "item-retry", resultIndex: 3, startIndex: 2 });
});
});
@@ -3,6 +3,11 @@
*/
import { describe, expect, it } from "vitest";
import {
requireSuccessfulNativeCommandCompactionEvidence,
requireSuccessfulPersistedNativeCommandExecution,
} from "./gateway-codex-harness.command-evidence.live-helpers.js";
import {
buildCodexHarnessLargeOutputCommand,
EXPECTED_CODEX_MODELS_COMMAND_TEXT,
EXPECTED_CODEX_STATUS_COMMAND_TEXT,
isExpectedCodexModelsCommandText,
@@ -10,12 +15,15 @@ import {
isExpectedYieldedAgentTimeout,
isRetryableCodexHarnessLiveError,
isStrictExpectedCodexModelsCommandText,
requireSuccessfulNativeCommandExecution,
shouldUseCodexHarnessSubagentOnlyFastPath,
} from "./gateway-codex-harness.live-helpers.js";
const includesExpectedCodexModelsCommandText = (text: string) =>
EXPECTED_CODEX_MODELS_COMMAND_TEXT.some((expectedText) => text.includes(expectedText));
const shellSingleQuote = (value: string) => `'${value.replaceAll("'", `'\\''`)}'`;
function expectExpectedCodexModelsCommandText(text: string): void {
expect(includesExpectedCodexModelsCommandText(text)).toBe(true);
}
@@ -31,6 +39,18 @@ function expectStrictCodexModelsCommandText(text: string): void {
}
describe("gateway codex harness live helpers", () => {
it("builds an exact large-output command without escape-sensitive newlines", () => {
const command = buildCodexHarnessLargeOutputCommand({
commandMarker: "OPENCLAW-LARGE-OUTPUT-ABC",
outputBytes: 1_000_000,
});
expect(command).toContain('"OPENCLAW-LARGE-OUTPUT-ABC|"');
expect(command).toContain(".slice(0,1000000)");
expect(command).not.toContain("\\n");
expect(command).not.toContain("\n");
});
it("keeps combined stress probes out of the subagent-only fast path", () => {
const base = {
chatImageProbe: false,
@@ -67,6 +87,521 @@ describe("gateway codex harness live helpers", () => {
expect(isRetryableCodexHarnessLiveError(error)).toBe(false);
});
it("matches a successful wrapped native command by its per-turn marker", () => {
const expectedCommand = `node -e 'console.log("OPENCLAW-LARGE-OUTPUT-ABC")'`;
const wrappedCommand = `node -e "console.log(\\"OPENCLAW-LARGE-OUTPUT-ABC\\")"`;
const events = [
{
stream: "tool",
data: {
phase: "start",
name: "bash",
itemId: "item-1",
args: { command: `/bin/bash -lc ${shellSingleQuote(wrappedCommand)}` },
},
},
{
stream: "tool",
data: {
phase: "result",
itemId: "item-1",
status: "completed",
isError: false,
result: { exitCode: 0 },
},
},
];
expect(
requireSuccessfulNativeCommandExecution(events, {
commandMarker: "OPENCLAW-LARGE-OUTPUT-ABC",
expectedCommand,
}),
).toEqual({ itemId: "item-1", resultIndex: 1, startIndex: 0 });
});
it("rejects a successful command that only echoes the expected command", () => {
const expectedCommand = `node -e 'console.log("OPENCLAW-LARGE-OUTPUT-ABC")'`;
expect(() =>
requireSuccessfulNativeCommandExecution(
[
{
stream: "tool",
data: {
phase: "start",
name: "bash",
itemId: "item-echo",
args: { command: `echo ${shellSingleQuote(expectedCommand)}` },
},
},
{
stream: "tool",
data: {
phase: "result",
itemId: "item-echo",
status: "completed",
isError: false,
result: { exitCode: 0 },
},
},
],
{
commandMarker: "OPENCLAW-LARGE-OUTPUT-ABC",
expectedCommand,
},
),
).toThrow("missing native bash command start for marker OPENCLAW-LARGE-OUTPUT-ABC");
});
it("accepts a completed native command when Codex omits or nulls its optional exit code", () => {
const expectedCommand = "node -e OPENCLAW-NO-EXIT-CODE";
for (const result of [{ status: "completed" }, { status: "completed", exitCode: null }]) {
const events = [
{
stream: "tool",
data: {
phase: "start",
name: "bash",
itemId: "item-no-exit-code",
args: { command: expectedCommand },
},
},
{
stream: "tool",
data: {
phase: "result",
itemId: "item-no-exit-code",
status: "completed",
isError: false,
result,
},
},
];
expect(
requireSuccessfulNativeCommandExecution(events, {
commandMarker: "OPENCLAW-NO-EXIT-CODE",
expectedCommand,
}),
).toEqual({ itemId: "item-no-exit-code", resultIndex: 1, startIndex: 0 });
}
});
it("reports a missing native command start explicitly", () => {
expect(() =>
requireSuccessfulNativeCommandExecution([], {
commandMarker: "OPENCLAW-MISSING",
expectedCommand: "node -e OPENCLAW-MISSING",
}),
).toThrow("missing native bash command start for marker OPENCLAW-MISSING");
});
it("reports a missing native command item id explicitly", () => {
expect(() =>
requireSuccessfulNativeCommandExecution(
[
{
stream: "tool",
data: {
phase: "start",
name: "bash",
args: { command: "node -e OPENCLAW-NO-ITEM" },
},
},
],
{
commandMarker: "OPENCLAW-NO-ITEM",
expectedCommand: "node -e OPENCLAW-NO-ITEM",
},
),
).toThrow("native bash command start for marker OPENCLAW-NO-ITEM has no itemId");
});
it("reports a missing successful native command result explicitly", () => {
expect(() =>
requireSuccessfulNativeCommandExecution(
[
{
stream: "tool",
data: {
phase: "start",
name: "bash",
itemId: "item-failed",
args: { command: "node -e OPENCLAW-FAILED" },
},
},
{
stream: "tool",
data: {
phase: "result",
itemId: "item-failed",
status: "completed",
isError: true,
result: { exitCode: 1 },
},
},
],
{
commandMarker: "OPENCLAW-FAILED",
expectedCommand: "node -e OPENCLAW-FAILED",
},
),
).toThrow(
"native bash command item-failed for marker OPENCLAW-FAILED has no successful result",
);
});
it("bounds failed-command diagnostics to the matching item without raw output", () => {
const secretOutput = "sensitive-command-output";
let message = "";
try {
requireSuccessfulNativeCommandExecution(
[
{
stream: "tool",
data: {
phase: "start",
name: "bash",
itemId: "item-failed",
args: { command: "node -e OPENCLAW-FAILED" },
},
},
{
stream: "tool",
data: {
phase: "result",
itemId: "item-other",
status: "completed",
isError: true,
result: { stderr: secretOutput, exitCode: 1 },
},
},
{
stream: "tool",
data: {
phase: "result",
itemId: "item-failed",
status: "completed",
isError: true,
result: { stdout: secretOutput, exitCode: 1 },
},
},
],
{
commandMarker: "OPENCLAW-FAILED",
expectedCommand: "node -e OPENCLAW-FAILED",
},
);
} catch (error) {
message = error instanceof Error ? error.message : String(error);
}
expect(message).toContain('"itemId":"item-failed"');
expect(message).toContain(`"stdoutChars":${secretOutput.length}`);
expect(message).not.toContain("item-other");
expect(message).not.toContain(secretOutput);
});
it("requires a successful marker-bearing large result in durable history", () => {
const expectedCommand = `node -e 'console.log("OPENCLAW-PERSISTED")'`;
const messages = [
{
role: "assistant",
content: [
{
type: "toolCall",
id: "persisted-call",
name: "bash",
arguments: { command: expectedCommand },
},
],
},
{
role: "toolResult",
toolCallId: "persisted-call",
isError: false,
content: [
{
type: "text",
text: "OPENCLAW-PERSISTED",
},
{
type: "text",
text: "...(truncated: original 2000 chars)",
},
],
},
];
expect(
requireSuccessfulPersistedNativeCommandExecution(messages, {
commandMarker: "OPENCLAW-PERSISTED",
expectedCommand,
minimumOutputChars: 1_000,
}),
).toEqual({ callIndex: 0, resultIndex: 1, toolCallId: "persisted-call" });
expect(() =>
requireSuccessfulPersistedNativeCommandExecution(messages.slice(1), {
commandMarker: "OPENCLAW-PERSISTED",
expectedCommand,
minimumOutputChars: 1_000,
}),
).toThrow("has no successful large result");
expect(
requireSuccessfulPersistedNativeCommandExecution(messages.slice(1), {
commandMarker: "OPENCLAW-PERSISTED",
expectedCommand,
minimumOutputChars: 1_000,
toolCallId: "persisted-call",
}),
).toEqual({ callIndex: -1, resultIndex: 0, toolCallId: "persisted-call" });
});
it("rejects echoed or failed native commands in durable history", () => {
const expectedCommand = `node -e 'console.log("OPENCLAW-PERSISTED")'`;
const echoedMessages = [
{
role: "assistant",
content: [
{
type: "toolCall",
id: "echoed-call",
name: "bash",
arguments: { command: `echo ${shellSingleQuote(expectedCommand)}` },
},
],
},
{
role: "toolResult",
toolCallId: "echoed-call",
isError: false,
content: [
{
type: "text",
text: "OPENCLAW-PERSISTED\n...(truncated: original 2000 chars)",
},
],
},
];
expect(() =>
requireSuccessfulPersistedNativeCommandExecution(echoedMessages, {
commandMarker: "OPENCLAW-PERSISTED",
expectedCommand,
minimumOutputChars: 1_000,
}),
).toThrow("has no successful large result");
const failedMessages = [
{
role: "assistant",
content: [
{
type: "toolCall",
id: "failed-call",
name: "bash",
arguments: { command: expectedCommand },
},
],
},
{
role: "toolResult",
toolCallId: "failed-call",
isError: true,
content: [
{
type: "text",
text: "OPENCLAW-PERSISTED\n...(truncated: original 2000 chars)",
},
],
},
];
expect(() =>
requireSuccessfulPersistedNativeCommandExecution(failedMessages, {
commandMarker: "OPENCLAW-PERSISTED",
expectedCommand,
minimumOutputChars: 1_000,
}),
).toThrow(
"persisted native bash command for marker OPENCLAW-PERSISTED has no successful large result",
);
const nonzeroMessages = [
failedMessages[0],
{
...failedMessages[1],
isError: false,
details: { status: "completed", exitCode: 17 },
},
];
expect(() =>
requireSuccessfulPersistedNativeCommandExecution(nonzeroMessages, {
commandMarker: "OPENCLAW-PERSISTED",
expectedCommand,
minimumOutputChars: 1_000,
}),
).toThrow("has no successful large result");
});
it("accepts successful request-local evidence when compaction removed durable history", () => {
const expectedCommand = "node -e OPENCLAW-COMPACTED";
const events = [
{
stream: "tool",
data: {
phase: "start",
name: "bash",
itemId: "compacted-command",
args: { command: expectedCommand },
},
},
{
stream: "tool",
data: {
phase: "result",
itemId: "compacted-command",
status: "completed",
isError: false,
result: { exitCode: 0 },
},
},
{ stream: "compaction", data: { phase: "end", completed: true } },
];
expect(
requireSuccessfulNativeCommandCompactionEvidence({
commandMarker: "OPENCLAW-COMPACTED",
events,
expectedCommand,
messages: [],
minimumOutputChars: 1_000,
}),
).toEqual({ source: "compacted-event" });
expect(() =>
requireSuccessfulNativeCommandCompactionEvidence({
commandMarker: "OPENCLAW-COMPACTED",
events: events.slice(0, 2),
expectedCommand,
messages: [],
minimumOutputChars: 1_000,
}),
).toThrow("successful request-local command result was not followed by compaction");
expect(() =>
requireSuccessfulNativeCommandCompactionEvidence({
commandMarker: "OPENCLAW-COMPACTED",
events,
expectedCommand,
messages: [
{
role: "toolResult",
toolCallId: "compacted-command",
isError: true,
content: [
{
type: "text",
text: "OPENCLAW-COMPACTED\n...(truncated: original 2000 chars)",
},
],
},
],
minimumOutputChars: 1_000,
}),
).toThrow("durable result for successful request-local command failed validation");
});
it("rejects large durable output from a different marker-bearing command", () => {
const expectedCommand = "node -e OPENCLAW-EXACT";
const messages = [
{
role: "assistant",
content: [
{
type: "toolCall",
id: "mismatched-call",
name: "bash",
arguments: { command: `echo ${shellSingleQuote(expectedCommand)}` },
},
],
},
{
role: "toolResult",
toolCallId: "mismatched-call",
isError: false,
content: [
{
type: "text",
text: "OPENCLAW-EXACT\n...(truncated: original 2000 chars)",
},
],
},
];
expect(() =>
requireSuccessfulNativeCommandCompactionEvidence({
commandMarker: "OPENCLAW-EXACT",
events: [],
expectedCommand,
messages,
minimumOutputChars: 1_000,
}),
).toThrow("has no successful request-local evidence");
});
it("ties a result-only durable row to the exact request-local item id", () => {
const expectedCommand = "node -e OPENCLAW-RESULT-ONLY";
const events = [
{
stream: "tool",
data: {
phase: "start",
name: "bash",
itemId: "exact-call",
args: { command: expectedCommand },
},
},
{
stream: "tool",
data: {
phase: "result",
itemId: "exact-call",
status: "completed",
isError: false,
result: { exitCode: 0 },
},
},
];
const resultOnlyMessage = (toolCallId: string) => ({
role: "toolResult",
toolCallId,
isError: false,
content: [
{
type: "text",
text: "OPENCLAW-RESULT-ONLY\n...(truncated: original 2000 chars)",
},
],
});
expect(
requireSuccessfulNativeCommandCompactionEvidence({
commandMarker: "OPENCLAW-RESULT-ONLY",
events,
expectedCommand,
messages: [resultOnlyMessage("different-call"), resultOnlyMessage("exact-call")],
minimumOutputChars: 1_000,
}),
).toEqual({ source: "persisted-history" });
expect(() =>
requireSuccessfulNativeCommandCompactionEvidence({
commandMarker: "OPENCLAW-RESULT-ONLY",
events,
expectedCommand,
messages: [resultOnlyMessage("different-call")],
minimumOutputChars: 1_000,
}),
).toThrow("successful request-local command result was not followed by compaction");
});
it("accepts only paused yielded agent timeouts for native subagent delivery", () => {
expect(
isExpectedYieldedAgentTimeout({
@@ -1,3 +1,6 @@
import { extractShellWrapperInlineCommand } from "../infra/shell-wrapper-resolution.js";
import { splitShellArgs } from "../utils/shell-argv.js";
/**
* Text matchers shared by live Codex harness tests.
*
@@ -105,6 +108,161 @@ export function shouldUseCodexHarnessSubagentOnlyFastPath(params: {
);
}
type CodexHarnessToolEventData = {
args?: unknown;
isError?: unknown;
itemId?: unknown;
name?: unknown;
phase?: unknown;
result?: unknown;
status?: unknown;
};
function readCodexHarnessToolEventData(event: unknown): CodexHarnessToolEventData | undefined {
if (!event || typeof event !== "object" || (event as { stream?: unknown }).stream !== "tool") {
return undefined;
}
const data = (event as { data?: unknown }).data;
return data && typeof data === "object" ? (data as CodexHarnessToolEventData) : undefined;
}
function summarizeNativeCommandResult(result: unknown): Record<string, unknown> {
if (!result || typeof result !== "object") {
return { type: result === null ? "null" : typeof result };
}
const record = result as Record<string, unknown>;
const summary: Record<string, unknown> = {};
for (const key of ["exitCode", "durationMs", "status"] as const) {
const value = record[key];
if (typeof value === "number" || typeof value === "boolean") {
summary[key] = value;
}
}
for (const key of ["stdout", "stderr", "output", "aggregatedOutput"] as const) {
const value = record[key];
if (typeof value === "string") {
summary[`${key}Chars`] = value.length;
}
}
return summary;
}
function shellArgvMatches(actual: readonly string[], expected: readonly string[]): boolean {
return actual.length === expected.length && actual.every((arg, index) => arg === expected[index]);
}
export function isExpectedNativeCommand(command: string, expectedCommand: string): boolean {
const commandArgv = splitShellArgs(command);
const expectedArgv = splitShellArgs(expectedCommand);
if (!commandArgv || !expectedArgv) {
return false;
}
if (shellArgvMatches(commandArgv, expectedArgv)) {
return true;
}
const wrappedCommand = extractShellWrapperInlineCommand(commandArgv);
const wrappedArgv = wrappedCommand ? splitShellArgs(wrappedCommand) : null;
return wrappedArgv ? shellArgvMatches(wrappedArgv, expectedArgv) : false;
}
export function buildCodexHarnessLargeOutputCommand(params: {
commandMarker: string;
outputBytes: number;
}): string {
if (!/^[A-Z0-9-]+$/u.test(params.commandMarker)) {
throw new Error(
"large-output command marker must contain only uppercase letters, digits, and hyphens",
);
}
if (!Number.isSafeInteger(params.outputBytes) || params.outputBytes <= 0) {
throw new Error("large-output byte count must be a positive safe integer");
}
const filler = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
return `node -e 'const p="${filler}";process.stdout.write(("${params.commandMarker}|"+p.repeat(Math.ceil(${params.outputBytes}/p.length))).slice(0,${params.outputBytes}))'`;
}
/** Requires one successful native bash execution carrying the per-turn command marker. */
export function requireSuccessfulNativeCommandExecution(
events: readonly unknown[],
params: { commandMarker: string; expectedCommand: string },
): { itemId: string; resultIndex: number; startIndex: number } {
const starts = events.flatMap((event, startIndex) => {
const data = readCodexHarnessToolEventData(event);
if (data?.phase !== "start" || data.name !== "bash" || !data.args) {
return [];
}
const command = (data.args as { command?: unknown }).command;
const matches =
typeof command === "string" &&
command.includes(params.commandMarker) &&
isExpectedNativeCommand(command, params.expectedCommand);
return matches ? [{ itemId: data.itemId, startIndex }] : [];
});
if (starts.length === 0) {
throw new Error(`missing native bash command start for marker ${params.commandMarker}`);
}
const validStarts = starts.filter(
(start): start is { itemId: string; startIndex: number } =>
typeof start.itemId === "string" && start.itemId.length > 0,
);
if (validStarts.length === 0) {
throw new Error(`native bash command start for marker ${params.commandMarker} has no itemId`);
}
for (const { itemId, startIndex } of validStarts) {
const resultIndex = events.findIndex((event, index) => {
const data = readCodexHarnessToolEventData(event);
const result = data?.result;
return (
index > startIndex &&
data?.phase === "result" &&
data.itemId === itemId &&
data.status === "completed" &&
data.isError === false &&
result !== null &&
typeof result === "object" &&
// App-server commandExecution.exitCode is optional and nullable. Completed +
// !isError is authoritative when Codex omits it; a present nonzero code still fails.
((result as { exitCode?: unknown }).exitCode === undefined ||
(result as { exitCode?: unknown }).exitCode === null ||
(result as { exitCode?: unknown }).exitCode === 0)
);
});
if (resultIndex >= 0) {
return { itemId, resultIndex, startIndex };
}
}
const startByItemId = new Map(validStarts.map((start) => [start.itemId, start.startIndex]));
const observedResults = events.flatMap((event, index) => {
const data = readCodexHarnessToolEventData(event);
const startIndex =
typeof data?.itemId === "string" ? startByItemId.get(data.itemId) : undefined;
if (
startIndex === undefined ||
index <= startIndex ||
data?.phase !== "result" ||
typeof data.itemId !== "string"
) {
return [];
}
return [
{
index,
phase: data.phase,
itemId: data.itemId,
status: data.status,
isError: data.isError,
result: summarizeNativeCommandResult(data.result),
},
];
});
throw new Error(
`native bash command ${validStarts.map((start) => start.itemId).join(",")} for marker ${params.commandMarker} has no successful result; observed=${JSON.stringify(observedResults)}`,
);
}
const HEALTHY_CODEX_MODELS_COMMAND_TEXT = [
"Codex models:",
"Available Codex models",
+25 -59
View File
@@ -23,7 +23,9 @@ import {
connectTestGatewayClient,
ensurePairedTestGatewayClientIdentity,
} from "./gateway-cli-backend.live-helpers.js";
import { requireSuccessfulNativeCommandCompactionEvidence } from "./gateway-codex-harness.command-evidence.live-helpers.js";
import {
buildCodexHarnessLargeOutputCommand,
EXPECTED_CODEX_MODELS_COMMAND_TEXT,
EXPECTED_CODEX_STATUS_COMMAND_TEXT,
isExpectedCodexStatusCommandText,
@@ -851,13 +853,23 @@ async function verifyCodexCompactionStress(params: {
minimum: 0,
sessionKey: params.sessionKey,
});
await requestCodexCommandText({
client: params.client,
command: "/codex permissions yolo",
events: params.events,
expectedText: "Codex permissions set to full access.",
sessionKey: params.sessionKey,
});
const outputLines = Math.ceil(CODEX_HARNESS_LARGE_OUTPUT_BYTES / 90);
let completedCompactions = 0;
let reportedCompactions = 0;
for (let turn = 1; turn <= CODEX_HARNESS_COMPACTION_STRESS_TURNS; turn += 1) {
const acknowledgement = `CODEX-LARGE-OUTPUT-${turn}-OK`;
const largeOutputCommand = `node -e 'for(let i=0;i<${outputLines};i++){console.log(i.toString(36).padStart(8,"0")+"-"+((i*2654435761)>>>0).toString(16).padStart(8,"0")+"-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")}'`;
const commandMarker = `OPENCLAW-CODEX-LARGE-OUTPUT-${turn}-${randomBytes(6).toString("hex").toUpperCase()}`;
const largeOutputCommand = buildCodexHarnessLargeOutputCommand({
commandMarker,
outputBytes: CODEX_HARNESS_LARGE_OUTPUT_BYTES,
});
const { text, events, compactionCount } = await requestAgentTextWithEvents({
client: params.client,
eventPrefixes: ["codex_app_server.", "compaction", "tool"],
@@ -880,67 +892,18 @@ async function verifyCodexCompactionStress(params: {
).length;
completedCompactions += turnCompletedCompactions;
reportedCompactions += compactionCount;
const commandStartIndex = events.findIndex((event) => {
if (
event.stream !== "tool" ||
event.data?.phase !== "start" ||
event.data?.name !== "bash" ||
!event.data?.args ||
typeof event.data.args !== "object"
) {
return false;
}
const command = (event.data.args as { command?: unknown }).command;
// Codex may preserve the command text or wrap it in a login shell.
return (
typeof command === "string" &&
command.includes("node -e") &&
command.includes(`i<${outputLines}`) &&
command.includes("2654435761") &&
command.includes("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")
);
});
const commandItemId = events[commandStartIndex]?.data?.itemId;
const commandResultIndex = events.findIndex((event, index) => {
const result = event.data?.result;
return (
index > commandStartIndex &&
event.stream === "tool" &&
event.data?.phase === "result" &&
event.data?.itemId === commandItemId &&
event.data?.status === "completed" &&
event.data?.isError === false &&
result !== null &&
typeof result === "object" &&
(result as { exitCode?: unknown }).exitCode === 0
);
});
expect(
commandResultIndex,
`large-output turn did not successfully complete the exact native command; events=${JSON.stringify(events)}`,
).toBeGreaterThan(commandStartIndex);
const history: { messages?: unknown[] } = await params.client.request("chat.history", {
sessionKey: params.sessionKey,
limit: 100,
});
const serialized = JSON.stringify(history.messages ?? []);
const originalLengths = Array.from(serialized.matchAll(/original (\d+) chars/gu), (match) =>
Number(match[1]),
);
const hasTruncatedToolResult = originalLengths.some((length) => length > 10_000);
const postCommandCompaction = events.some(
(event, index) =>
index > commandResultIndex &&
event.stream === "compaction" &&
event.data?.phase === "end" &&
event.data?.completed === true,
);
// Native compaction can replace the command row after the successful large-output result.
expect(
hasTruncatedToolResult || postCommandCompaction,
`expected a truncated large native tool result or its later native compaction; lengths=${JSON.stringify(originalLengths)}`,
).toBe(true);
const historyMessages = history.messages ?? [];
requireSuccessfulNativeCommandCompactionEvidence({
commandMarker,
events,
expectedCommand: largeOutputCommand,
messages: historyMessages,
minimumOutputChars: Math.floor(CODEX_HARNESS_LARGE_OUTPUT_BYTES * 0.95),
});
}
expect(completedCompactions, "expected at least one native automatic compaction").toBeGreaterThan(
@@ -1230,6 +1193,9 @@ async function verifyCodexGuardianProbe(params: {
const review = assertGuardianReviewCompleted({
events: deniedResult.events,
label: "ask-back probe",
// The strict projection path is proved above. Codex may refuse this risky
// prompt before creating a review, so its explicit ask-back is also valid.
requireEvents: false,
});
// The approve/deny call is Codex policy-owned and may change independently.
// OpenClaw's strict projection contract is covered by the allow probe above.
+14
View File
@@ -8,10 +8,24 @@ import {
buildLiveCronProbeMessage,
createLiveCronProbeSpec,
isClaudeLikeLiveAgent,
resolveOpenClawCliProcessArgs,
shouldRunLiveImageProbe,
} from "./live-agent-probes.js";
describe("live-agent-probes", () => {
it("uses the source runner when packaged CLI output is absent", () => {
expect(resolveOpenClawCliProcessArgs(["cron", "list"], false)).toEqual([
"scripts/run-node.mjs",
"cron",
"list",
]);
expect(resolveOpenClawCliProcessArgs(["cron", "list"], true)).toEqual([
"openclaw.mjs",
"cron",
"list",
]);
});
it("only special-cases Claude-like retry prompts", () => {
expect(isClaudeLikeLiveAgent("claude")).toBe(true);
expect(isClaudeLikeLiveAgent("claude-cli")).toBe(true);
+24 -7
View File
@@ -1,6 +1,8 @@
// Gateway live agent probe helpers.
// Builds prompts and verification helpers for live image and cron probe tests.
import { randomBytes } from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import {
resolveExpiresAtMsFromDurationSeconds,
resolveTimestampMsToIsoString,
@@ -33,6 +35,14 @@ type LiveCronProbeSpec = {
argsJson: string;
};
/** Selects the packaged launcher when built, otherwise the canonical source runner. */
export function resolveOpenClawCliProcessArgs(
args: readonly string[],
hasBuildOutput: boolean,
): string[] {
return [hasBuildOutput ? "openclaw.mjs" : "scripts/run-node.mjs", ...args];
}
/** Return true for live agents that expose Claude-style MCP tool names. */
export function isClaudeLikeLiveAgent(raw: string): boolean {
const normalized = normalizeOptionalLowercaseString(raw);
@@ -149,13 +159,20 @@ export async function runOpenClawCliJson<T>(args: string[], env: NodeJS.ProcessE
const cliArgs = args.includes("--timeout")
? args
: [...args, "--timeout", String(OPENCLAW_CLI_GATEWAY_TIMEOUT_MS)];
const { stdout, stderr } = await runExec(process.execPath, ["openclaw.mjs", ...cliArgs], {
baseEnv: childEnv,
cwd: process.cwd(),
logOutput: false,
maxBuffer: 1024 * 1024,
timeoutMs: OPENCLAW_CLI_CHILD_TIMEOUT_MS,
});
const hasBuildOutput = ["entry.js", "entry.mjs"].some((entry) =>
fs.existsSync(path.join(process.cwd(), "dist", entry)),
);
const { stdout, stderr } = await runExec(
process.execPath,
resolveOpenClawCliProcessArgs(cliArgs, hasBuildOutput),
{
baseEnv: childEnv,
cwd: process.cwd(),
logOutput: false,
maxBuffer: 1024 * 1024,
timeoutMs: OPENCLAW_CLI_CHILD_TIMEOUT_MS,
},
);
const trimmed = stdout.trim();
if (!trimmed) {
throw new Error(