furyhawk/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-21 02:06:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(codex): persist automatic compaction history (#110587)

Browse Source 
* fix(codex): persist automatic compaction counts

* test(codex): stress compaction and restart recovery

* test(codex): bind compaction waves to fresh counts

* chore: keep Codex release note in PR
This commit is contained in:
Peter Steinberger
2026-07-18 10:37:25 +01:00
committed by GitHub
parent 609e5d2318
commit f14947d256
9 changed files with 510 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/help/testing-live.md
+22
View File
@@ -334,6 +334,17 @@ Docker notes:
history. Override the bounded counts with
`OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS` (1-20) and
`OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS` (1-10).
- Optional fan-out stress: set `OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE=1`
and `OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT` (1-12). The harness starts
every child concurrently, waits for every terminal run, and verifies each
unique child reply and native thread identity.
- Optional compaction stress: `OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS=1`
generates bounded native tool output, requires automatic compaction events,
verifies the persisted compaction count and hidden-marker recall, restarts
the Gateway and physical Codex app-server, then repeats the output and
compaction wave. Tune the bounded work with
`OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS` (1-8) and
`OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES` (100000-1000000).
- Optional loop-relay opt-out probe:
`OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY=1`
- The requested thinking preference may map to the nearest effort advertised
@@ -367,6 +378,17 @@ Restart and history stress:
```bash
OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS=1 \
pnpm test:docker:live-codex-harness
```
Fan-out, large-output, compaction, and restart stress:
```bash
OPENCLAW_LIVE_CODEX_HARNESS_AUTH=api-key \
OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE=1 \
OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT=8 \
OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS=1 \
OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS=1 \
pnpm test:docker:live-codex-harness
```
extensions/codex/src/app-server/event-projector.test.ts
+4 -1
View File
@@ -2998,7 +2998,10 @@ describe("CodexAppServerEventProjector", () => {
expect(JSON.stringify(result.messagesSnapshot[2])).toContain("Codex plan");
expect(JSON.stringify(result.messagesSnapshot[2])).toContain("next");
expect(JSON.stringify(result.messagesSnapshot[2])).toContain("[in_progress] patch");
expect(requireRecord(result.itemLifecycle, "item lifecycle").compactionCount).toBe(1);
expect(result.compactionCount).toBe(1);
expect(requireRecord(result.itemLifecycle, "item lifecycle")).not.toHaveProperty(
"compactionCount",
);
expect(onContextCompacted).toHaveBeenCalledOnce();
});
extensions/codex/src/app-server/event-projector.ts
+3 -3
View File
@@ -434,6 +434,9 @@ export class CodexAppServerEventProjector {
successfulCronAdds: toolTelemetry.successfulCronAdds,
cloudCodeAssistFormatError: false,
attemptUsage: projectedUsage,
...(this.completedCompactionCount > 0
? { compactionCount: this.completedCompactionCount }
: {}),
replayMetadata: {
hadPotentialSideEffects,
replaySafe: !hadPotentialSideEffects,
@@ -442,9 +445,6 @@ export class CodexAppServerEventProjector {
startedCount: this.activeItemIds.size + this.completedItemIds.size,
completedCount: this.completedItemIds.size,
activeCount: this.activeItemIds.size,
...(this.completedCompactionCount > 0
? { compactionCount: this.completedCompactionCount }
: {}),
},
yieldDetected: options?.yieldDetected || false,
didSendDeterministicApprovalPrompt:
extensions/codex/src/app-server/run-attempt-start.ts
+6 -1
View File
@@ -159,7 +159,12 @@ export async function startCodexAttemptRuntime(resources: CodexAttemptResources)
state.codexSandboxPolicy = startupResult.sandboxPolicy;
void emitCodexAppServerEvent(params, {
stream: "codex_app_server.lifecycle",
data: { phase: "thread_ready", threadId: state.thread.threadId },
data: {
phase: "thread_ready",
threadId: state.thread.threadId,
action: state.thread.lifecycle.action,
clientId: state.client.getInstanceId(),
},
});
} catch (error) {
activateNativePreToolUseFailureFallback();
scripts/test-live-codex-harness-docker.sh
+12
View File
@@ -367,11 +367,17 @@ echo "==> Chat image probe: ${OPENCLAW_LIVE_CODEX_HARNESS_CHAT_IMAGE_PROBE:-0}"
echo "==> Image probe: ${OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE:-1}"
echo "==> MCP probe: ${OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE:-1}"
echo "==> Subagent probe: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE:-1}"
echo "==> Subagent count: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT:-1}"
echo "==> Subagent-only fast path: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY:-auto}"
echo "==> Guardian probe: ${OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE:-1}"
echo "==> Code-mode-only probe: ${OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY:-0}"
echo "==> Loop relay disabled: ${OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY:-0}"
echo "==> Resume stress: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS:-0}"
echo "==> Resume stress history turns: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS:-4}"
echo "==> Resume stress restarts: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}"
echo "==> Compaction stress: ${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS:-0}"
echo "==> Compaction stress turns: ${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS:-4}"
echo "==> Large output bytes: ${OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES:-300000}"
echo "==> Auth mode: $CODEX_HARNESS_AUTH_MODE"
echo "==> Profile file: $PROFILE_STATUS"
echo "==> CI-safe Codex config: ${OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG:-1}"
@@ -401,9 +407,14 @@ DOCKER_RUN_ARGS+=(--rm -t \
-e OPENCLAW_LIVE_CODEX_HARNESS_AUTH="$CODEX_HARNESS_AUTH_MODE" \
-e OPENCLAW_LIVE_CODEX_HARNESS=1 \
-e OPENCLAW_LIVE_CODEX_HARNESS_CHAT_IMAGE_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_CHAT_IMAGE_PROBE:-0}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY="${OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY:-0}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS:-0}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS:-4}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_DEBUG="${OPENCLAW_LIVE_CODEX_HARNESS_DEBUG:-}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY="${OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY:-0}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE:-1}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE:-1}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES="${OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES:-300000}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE:-1}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_MODEL="${OPENCLAW_LIVE_CODEX_HARNESS_MODEL:-openai/gpt-5.6-luna}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_TARGETS="${OPENCLAW_LIVE_CODEX_HARNESS_TARGETS:-}" \
@@ -416,6 +427,7 @@ DOCKER_RUN_ARGS+=(--rm -t \
-e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_SETUP_TIMEOUT_SECONDS="$CODEX_HARNESS_SETUP_TIMEOUT_SECONDS" \
-e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY:-}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT:-1}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE:-1}" \
-e OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG="${OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG:-1}" \
-e OPENCLAW_LIVE_CODEX_CLI_PACKAGE_SPEC="$CODEX_CLI_PACKAGE_SPEC" \
src/gateway/gateway-codex-harness.live-helpers.test.ts
+25
View File
@@ -10,6 +10,7 @@ import {
isExpectedYieldedAgentTimeout,
isRetryableCodexHarnessLiveError,
isStrictExpectedCodexModelsCommandText,
shouldUseCodexHarnessSubagentOnlyFastPath,
} from "./gateway-codex-harness.live-helpers.js";
const includesExpectedCodexModelsCommandText = (text: string) =>
@@ -30,6 +31,30 @@ function expectStrictCodexModelsCommandText(text: string): void {
}
describe("gateway codex harness live helpers", () => {
it("keeps combined stress probes out of the subagent-only fast path", () => {
const base = {
chatImageProbe: false,
codeModeOnly: false,
compactionStress: false,
explicitOptOut: false,
guardianProbe: false,
imageProbe: false,
mcpProbe: false,
resumeStress: false,
subagentProbe: true,
};
expect(shouldUseCodexHarnessSubagentOnlyFastPath(base)).toBe(true);
expect(shouldUseCodexHarnessSubagentOnlyFastPath({ ...base, resumeStress: true })).toBe(false);
expect(shouldUseCodexHarnessSubagentOnlyFastPath({ ...base, compactionStress: true })).toBe(
false,
);
expect(shouldUseCodexHarnessSubagentOnlyFastPath({ ...base, codeModeOnly: true })).toBe(false);
expect(shouldUseCodexHarnessSubagentOnlyFastPath({ ...base, explicitOptOut: true })).toBe(
false,
);
});
it("classifies sessions.list timeouts as retryable live Codex errors", () => {
const error = new Error("gateway request timeout for sessions.list");
src/gateway/gateway-codex-harness.live-helpers.ts
+24
View File
@@ -81,6 +81,30 @@ export const EXPECTED_CODEX_MODELS_COMMAND_TEXT = [
"Current OpenClaw session status reports the active model as:",
] as const;
export function shouldUseCodexHarnessSubagentOnlyFastPath(params: {
chatImageProbe: boolean;
codeModeOnly: boolean;
compactionStress: boolean;
explicitOptOut: boolean;
guardianProbe: boolean;
imageProbe: boolean;
mcpProbe: boolean;
resumeStress: boolean;
subagentProbe: boolean;
}): boolean {
return (
params.subagentProbe &&
!params.chatImageProbe &&
!params.codeModeOnly &&
!params.compactionStress &&
!params.guardianProbe &&
!params.imageProbe &&
!params.mcpProbe &&
!params.resumeStress &&
!params.explicitOptOut
);
}
const HEALTHY_CODEX_MODELS_COMMAND_TEXT = [
"Codex models:",
"Available Codex models",
src/gateway/gateway-codex-harness.live.test.ts
+396 -81
View File
@@ -30,6 +30,7 @@ import {
isExpectedYieldedAgentTimeout,
isRetryableCodexHarnessLiveError,
isStrictExpectedCodexModelsCommandText,
shouldUseCodexHarnessSubagentOnlyFastPath,
} from "./gateway-codex-harness.live-helpers.js";
import {
assertCronJobMatches,
@@ -63,13 +64,6 @@ const CODEX_HARNESS_CODE_MODE_ONLY = isTruthyEnvValue(
const CODEX_HARNESS_DISABLE_LOOP_RELAY = isTruthyEnvValue(
process.env.OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY,
);
const CODEX_HARNESS_SUBAGENT_ONLY =
CODEX_HARNESS_SUBAGENT_PROBE &&
!CODEX_HARNESS_CHAT_IMAGE_PROBE &&
!CODEX_HARNESS_IMAGE_PROBE &&
!CODEX_HARNESS_MCP_PROBE &&
!CODEX_HARNESS_GUARDIAN_PROBE &&
process.env.OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY !== "0";
const CODEX_HARNESS_REQUIRE_GUARDIAN_EVENTS = isTruthyEnvValue(
process.env.OPENCLAW_LIVE_CODEX_HARNESS_REQUIRE_GUARDIAN_EVENTS,
);
@@ -88,6 +82,40 @@ const CODEX_HARNESS_RESUME_STRESS_RESTARTS = resolveBoundedPositiveIntEnv(
3,
10,
);
const CODEX_HARNESS_COMPACTION_STRESS = isTruthyEnvValue(
process.env.OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS,
);
const CODEX_HARNESS_COMPACTION_STRESS_TURNS = resolveBoundedPositiveIntEnv(
"OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS",
process.env.OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS,
4,
8,
);
const CODEX_HARNESS_LARGE_OUTPUT_BYTES = resolveBoundedPositiveIntEnv(
"OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES",
process.env.OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES,
300_000,
1_000_000,
100_000,
);
const CODEX_HARNESS_SUBAGENT_COUNT = resolveBoundedPositiveIntEnv(
"OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT",
process.env.OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT,
1,
12,
);
const CODEX_HARNESS_SUBAGENT_ONLY = shouldUseCodexHarnessSubagentOnlyFastPath({
chatImageProbe: CODEX_HARNESS_CHAT_IMAGE_PROBE,
codeModeOnly: CODEX_HARNESS_CODE_MODE_ONLY,
compactionStress: CODEX_HARNESS_COMPACTION_STRESS,
explicitOptOut: process.env.OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY === "0",
guardianProbe: CODEX_HARNESS_GUARDIAN_PROBE,
imageProbe: CODEX_HARNESS_IMAGE_PROBE,
mcpProbe: CODEX_HARNESS_MCP_PROBE,
resumeStress: CODEX_HARNESS_RESUME_STRESS,
subagentProbe: CODEX_HARNESS_SUBAGENT_PROBE,
});
const CODEX_HARNESS_RESTART_STRESS = CODEX_HARNESS_RESUME_STRESS || CODEX_HARNESS_COMPACTION_STRESS;
const CODEX_HARNESS_REQUEST_TIMEOUT_MS = resolveLiveTimeoutMs(
process.env.OPENCLAW_LIVE_CODEX_HARNESS_REQUEST_TIMEOUT_MS,
300_000,
@@ -103,7 +131,7 @@ const CODEX_HARNESS_THINKING = resolveCodexHarnessThinkingLevel(
);
const describeLive = LIVE && CODEX_HARNESS_LIVE ? describe : describe.skip;
const describeDisabled = LIVE && !CODEX_HARNESS_LIVE ? describe : describe.skip;
const CODEX_HARNESS_TIMEOUT_MS = CODEX_HARNESS_RESUME_STRESS ? 1_800_000 : 900_000;
const CODEX_HARNESS_TIMEOUT_MS = CODEX_HARNESS_RESTART_STRESS ? 3_600_000 : 900_000;
const DEFAULT_CODEX_MODEL = "openai/gpt-5.6-luna";
const GATEWAY_CONNECT_TIMEOUT_MS = 60_000;
const CODEX_HARNESS_REASONING_EFFORTS = [
@@ -132,6 +160,8 @@ type CapturedAgentEvent = {
};
const observedCodexThreadIds = new Map<string, string>();
const observedCodexClientIds = new Map<string, string>();
const observedCodexThreadActions = new Map<string, string>();
type GuardianPluginApprovalDecision = "allow-once" | "deny";
type CodexHarnessThinkingLevel =
@@ -154,6 +184,7 @@ function resolveBoundedPositiveIntEnv(
raw: string | undefined,
fallback: number,
max: number,
min = 1,
): number {
if (raw === undefined || raw.trim() === "") {
return fallback;
@@ -162,8 +193,8 @@ function resolveBoundedPositiveIntEnv(
throw new Error(`invalid ${name}: ${raw}`);
}
const parsed = Number(raw);
if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > max) {
throw new Error(`${name} must be between 1 and ${max}`);
if (!Number.isSafeInteger(parsed) || parsed < min || parsed > max) {
throw new Error(`${name} must be between ${min} and ${max}`);
}
return parsed;
}
@@ -392,6 +423,7 @@ async function assertCodexHarnessTranscriptModelIdentity(params: {
async function writeLiveGatewayConfig(params: {
codexAppServerMode?: "guardian" | "yolo";
codeModeOnly?: boolean;
compactionStress?: boolean;
loopDetectionPreToolUseRelay?: boolean;
configPath: string;
modelKey: string;
@@ -414,6 +446,23 @@ async function writeLiveGatewayConfig(params: {
config: {
appServer: {
mode: params.codexAppServerMode ?? "yolo",
...(params.compactionStress
? {
args: [
"-c",
"model_auto_compact_token_limit_scope=body_after_prefix",
"-c",
// One truncated 300 KB tool result is only a few thousand
// tokens after Codex applies its transcript output cap.
"model_auto_compact_token_limit=4000",
"-c",
"tool_output_token_limit=10000",
"app-server",
"--listen",
"stdio://",
],
}
: {}),
...(params.codeModeOnly === true ? { codeModeOnly: true } : {}),
...(params.loopDetectionPreToolUseRelay === false
? { loopDetectionPreToolUseRelay: false }
@@ -430,6 +479,11 @@ async function writeLiveGatewayConfig(params: {
workspace: params.workspace,
skipBootstrap: true,
timeoutSeconds: CODEX_HARNESS_AGENT_TIMEOUT_SECONDS,
maxConcurrent: Math.max(4, CODEX_HARNESS_SUBAGENT_COUNT + 1),
subagents: {
maxConcurrent: CODEX_HARNESS_SUBAGENT_COUNT,
maxChildrenPerAgent: CODEX_HARNESS_SUBAGENT_COUNT,
},
thinkingDefault: CODEX_HARNESS_THINKING,
model: { primary: params.modelKey },
models: { [params.modelKey]: { agentRuntime: { id: "codex" } } },
@@ -454,17 +508,18 @@ async function requestAgentTextWithEvents(params: {
acceptYieldedTimeout?: boolean;
client: GatewayClient;
eventPrefix?: string;
eventPrefixes?: readonly string[];
includeAllSessions?: boolean;
message: string;
sessionKey: string;
}): Promise<{ text: string; events: CapturedAgentEvent[] }> {
}): Promise<{ text: string; events: CapturedAgentEvent[]; compactionCount: number }> {
const { extractPayloadText } = await import("./test-helpers.agent-results.js");
const { onAgentEvent } = await import("../infra/agent-events.js");
const events: CapturedAgentEvent[] = [];
const eventPrefix = params.eventPrefix ?? "codex_app_server.guardian";
const eventPrefixes = params.eventPrefixes ?? [params.eventPrefix ?? "codex_app_server.guardian"];
const unsubscribe = onAgentEvent((event) => {
if (
!event.stream.startsWith(eventPrefix) ||
!eventPrefixes.some((prefix) => event.stream.startsWith(prefix)) ||
(!params.includeAllSessions && event.sessionKey && event.sessionKey !== params.sessionKey)
) {
return;
@@ -493,7 +548,14 @@ async function requestAgentTextWithEvents(params: {
if (payload?.status !== "ok" && !acceptedYieldedTimeout) {
throw new Error(`agent status=${String(payload?.status)} payload=${JSON.stringify(payload)}`);
}
return { text: extractPayloadText(payload.result), events };
const result = payload.result as
| { meta?: { agentMeta?: { compactionCount?: number } } }
| undefined;
return {
text: extractPayloadText(payload.result),
events,
compactionCount: Math.max(0, result?.meta?.agentMeta?.compactionCount ?? 0),
};
} finally {
unsubscribe();
}
@@ -501,7 +563,7 @@ async function requestAgentTextWithEvents(params: {
async function requestAgentText(params: {
client: GatewayClient;
expectedToken: string;
expectedReply: string;
message: string;
sessionKey: string;
}): Promise<string> {
@@ -511,7 +573,16 @@ async function requestAgentText(params: {
message: params.message,
sessionKey: params.sessionKey,
});
expect(text).toContain(params.expectedToken);
expect(text).toContain(params.expectedReply);
recordCodexAttemptIdentity({ events, sessionKey: params.sessionKey });
return text;
}
function recordCodexAttemptIdentity(params: {
events: CapturedAgentEvent[];
sessionKey: string;
}): void {
const { events } = params;
const turnStarting = events.find(
(event) =>
event.stream === "codex_app_server.lifecycle" && event.data?.phase === "turn_starting",
@@ -539,7 +610,15 @@ async function requestAgentText(params: {
`expected Codex thread_ready identity for ${params.sessionKey}; events=${JSON.stringify(events)}`,
).toBe(true);
observedCodexThreadIds.set(params.sessionKey, threadId as string);
return text;
const clientId = threadReady?.data?.clientId;
expect(
typeof clientId === "string" && clientId.trim().length > 0,
`expected Codex client identity for ${params.sessionKey}; events=${JSON.stringify(events)}`,
).toBe(true);
observedCodexClientIds.set(params.sessionKey, clientId as string);
const action = threadReady?.data?.action;
expect(["started", "resumed", "forked"]).toContain(action);
observedCodexThreadActions.set(params.sessionKey, action as string);
}
async function verifyCodexCodeModeOnlyDynamicToolProbe(params: {
@@ -726,6 +805,127 @@ function formatAssistantTextPreview(texts: string[], maxChars = 800): string {
return combined.length > maxChars ? `${combined.slice(0, maxChars)}...` : combined;
}
async function readCodexHarnessCompactionCount(params: {
client: GatewayClient;
events: EventFrame[];
minimum: number;
sessionKey: string;
}): Promise<number> {
const statusText = await requestCodexCommandText({
client: params.client,
command: "/status",
events: params.events,
expectedText: "Compactions:",
sessionKey: params.sessionKey,
});
const count = Number(/Compactions:\s*(\d+)/u.exec(statusText)?.[1]);
expect(
count,
`session ${params.sessionKey} did not report ${params.minimum} persisted compactions`,
).toBeGreaterThanOrEqual(params.minimum);
return count;
}
async function verifyCodexCompactionStress(params: {
client: GatewayClient;
events: EventFrame[];
sessionKey: string;
}): Promise<{ hiddenMarker: string; persistedCount: number }> {
const hiddenMarker = `CODEX-DURABLE-${randomBytes(6).toString("hex").toUpperCase()}`;
await requestAgentText({
client: params.client,
sessionKey: params.sessionKey,
expectedReply: hiddenMarker,
message: [
`Remember this as durable slot A: ${hiddenMarker}`,
`Reply exactly ${hiddenMarker} and nothing else.`,
].join("\n"),
});
const baselineCount = await readCodexHarnessCompactionCount({
client: params.client,
events: params.events,
minimum: 0,
sessionKey: params.sessionKey,
});
const outputLines = Math.ceil(CODEX_HARNESS_LARGE_OUTPUT_BYTES / 90);
let completedCompactions = 0;
let reportedCompactions = 0;
for (let turn = 1; turn <= CODEX_HARNESS_COMPACTION_STRESS_TURNS; turn += 1) {
const acknowledgement = `CODEX-LARGE-OUTPUT-${turn}-${randomBytes(3)
.toString("hex")
.toUpperCase()}`;
const { text, events, compactionCount } = await requestAgentTextWithEvents({
client: params.client,
eventPrefixes: ["codex_app_server.", "compaction"],
sessionKey: params.sessionKey,
message: [
"Large-output compaction probe.",
"Use the native exec_command tool exactly once.",
`Run this exact command: node -e 'for(let i=0;i<${outputLines};i++){console.log(i.toString(36).padStart(8,"0")+"-"+((i*2654435761)>>>0).toString(16).padStart(8,"0")+"-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")}'`,
"Set max_output_tokens to 10000.",
`After the tool completes, reply exactly ${acknowledgement} and nothing else.`,
].join("\n"),
});
expect(text).toContain(acknowledgement);
recordCodexAttemptIdentity({ events, sessionKey: params.sessionKey });
completedCompactions += events.filter(
(event) =>
event.stream === "compaction" &&
event.data?.phase === "end" &&
event.data?.completed === true,
).length;
reportedCompactions += compactionCount;
const history: { messages?: unknown[] } = await params.client.request("chat.history", {
sessionKey: params.sessionKey,
limit: 100,
});
const serialized = JSON.stringify(history.messages ?? []);
const originalLengths = Array.from(serialized.matchAll(/original (\d+) chars/gu), (match) =>
Number(match[1]),
);
expect(
originalLengths.some((length) => length > 10_000),
`expected a truncated large native tool result; lengths=${JSON.stringify(originalLengths)}`,
).toBe(true);
}
expect(completedCompactions, "expected at least one native automatic compaction").toBeGreaterThan(
0,
);
expect(reportedCompactions, "agent result dropped native automatic compactions").toBe(
completedCompactions,
);
// `/status` stops in the local command handler (`shouldContinue: false`), so
// these snapshots cannot introduce an unobserved native Codex compaction.
const persistedCount = await readCodexHarnessCompactionCount({
client: params.client,
events: params.events,
minimum: baselineCount + completedCompactions,
sessionKey: params.sessionKey,
});
expect(
persistedCount - baselineCount,
"persisted session count did not match this wave's native compactions",
).toBe(completedCompactions);
const recalled = await requestAgentText({
client: params.client,
sessionKey: params.sessionKey,
expectedReply: hiddenMarker,
message: "Reply with exactly the value stored in durable slot A and nothing else.",
});
expect(recalled.trim()).toBe(hiddenMarker);
logCodexLiveStep("compaction-stress:complete", {
baselineCount,
completedCompactions,
outputBytes: CODEX_HARNESS_LARGE_OUTPUT_BYTES,
outputTurns: CODEX_HARNESS_COMPACTION_STRESS_TURNS,
persistedCount,
});
return { hiddenMarker, persistedCount };
}
async function waitForAssistantText(params: {
client: GatewayClient;
sessionKey: string;
@@ -930,7 +1130,6 @@ async function verifyCodexGuardianProbe(params: {
const allowReview = assertGuardianReviewCompleted({
events: allowResult.events,
label: "allow probe",
requireEvents: false,
});
const allowStatus = findGuardianReviewStatus(allowResult.events);
if (allowStatus === "denied") {
@@ -977,7 +1176,6 @@ async function verifyCodexGuardianProbe(params: {
const review = assertGuardianReviewCompleted({
events: deniedResult.events,
label: "ask-back probe",
requireEvents: false,
});
// The approve/deny call is Codex policy-owned and may change independently.
// OpenClaw's strict projection contract is covered by the allow probe above.
@@ -1067,24 +1265,21 @@ async function verifyCodexCronMcpProbe(params: {
}
}
function hasCodexAppServerLifecycleEvent(params: {
childSessionKey: string;
events: CapturedAgentEvent[];
}): boolean {
return params.events.some(
(event) =>
event.sessionKey === params.childSessionKey && event.stream === "codex_app_server.lifecycle",
);
}
async function waitForCodexSubagentStarted(params: {
childSessionKey: string;
events: CapturedAgentEvent[];
}): Promise<void> {
}): Promise<string> {
const deadline = Date.now() + CODEX_HARNESS_REQUEST_TIMEOUT_MS;
while (Date.now() < deadline) {
if (hasCodexAppServerLifecycleEvent(params)) {
return;
const threadReady = params.events.find(
(event) =>
event.sessionKey === params.childSessionKey &&
event.stream === "codex_app_server.lifecycle" &&
event.data?.phase === "thread_ready" &&
typeof event.data.threadId === "string",
);
if (threadReady && typeof threadReady.data?.threadId === "string") {
return threadReady.data.threadId;
}
await delay(2_000);
}
@@ -1158,36 +1353,84 @@ async function verifyCodexSubagentProbe(params: {
}
},
});
const spawnResult = await spawnSubagentDirect(
{
task: `Reply exactly ${expectedToken} and nothing else.`,
agentId: "dev",
thinking: CODEX_HARNESS_THINKING,
mode: "run",
cleanup: "keep",
context: "isolated",
lightContext: true,
expectsCompletionMessage: false,
runTimeoutSeconds: CODEX_HARNESS_AGENT_TIMEOUT_SECONDS,
},
{
agentSessionKey: params.sessionKey,
},
const probeReplies = [
expectedToken,
...Array.from(
{ length: CODEX_HARNESS_SUBAGENT_COUNT - 1 },
(_, index) => `CODEX-SUBAGENT-${index + 2}-${randomUUID().slice(0, 6).toUpperCase()}`,
),
];
const probes = probeReplies.map((expectedReply, index) => ({ expectedReply, index }));
const spawnResults = await Promise.all(
probes.map(async (probe) => ({
expectedReply: probe.expectedReply,
index: probe.index,
result: await spawnSubagentDirect(
{
task: `Reply exactly ${probe.expectedReply} and nothing else.`,
agentId: "dev",
thinking: CODEX_HARNESS_THINKING,
mode: "run",
cleanup: "keep",
context: "isolated",
lightContext: true,
expectsCompletionMessage: false,
runTimeoutSeconds: CODEX_HARNESS_AGENT_TIMEOUT_SECONDS,
},
{
agentSessionKey: params.sessionKey,
},
),
})),
);
if (spawnResult.status !== "accepted") {
throw new Error(
`Codex subagent spawn failed: ${JSON.stringify(spawnResult)} trace=${JSON.stringify(gatewayTrace)}`,
);
for (const probe of spawnResults) {
if (probe.result.status !== "accepted") {
throw new Error(
`Codex subagent ${probe.index + 1} spawn failed: ${JSON.stringify(probe.result)} trace=${JSON.stringify(gatewayTrace)}`,
);
}
if (!probe.result.childSessionKey?.includes(":subagent:") || !probe.result.runId) {
throw new Error(
`subagent spawn did not return child/run identities: ${JSON.stringify(probe.result)}`,
);
}
}
const childSessionKey = spawnResult.childSessionKey;
if (!childSessionKey?.includes(":subagent:")) {
throw new Error(
`subagent spawn did not return a child session key: ${JSON.stringify(spawnResult)}`,
);
}
await waitForCodexSubagentStarted({
childSessionKey,
events,
expect(new Set(spawnResults.map((probe) => probe.result.childSessionKey)).size).toBe(
spawnResults.length,
);
expect(new Set(spawnResults.map((probe) => probe.result.runId)).size).toBe(spawnResults.length);
const threadIds = await Promise.all(
spawnResults.map((probe) =>
waitForCodexSubagentStarted({
childSessionKey: probe.result.childSessionKey as string,
events,
}),
),
);
expect(new Set(threadIds).size).toBe(threadIds.length);
await Promise.all(
spawnResults.map((probe) =>
waitForChatAgentRunOk(params.client, probe.result.runId as string),
),
);
await Promise.all(
spawnResults.map(async (probe) => {
const history: { messages?: unknown[] } = await params.client.request("chat.history", {
sessionKey: probe.result.childSessionKey,
limit: 20,
});
const replies = extractAssistantTexts(history.messages ?? []);
expect(
replies.some((text) => text.trim() === probe.expectedReply),
`subagent ${probe.index + 1} missing exact reply ${probe.expectedReply}; replies=${formatAssistantTextPreview(replies)}`,
).toBe(true);
}),
);
logCodexLiveStep("subagent-fanout:complete", {
count: spawnResults.length,
uniqueThreads: new Set(threadIds).size,
});
} finally {
const { testing: subagentSpawnTesting } =
@@ -1306,6 +1549,7 @@ describeLive("gateway live (Codex harness)", () => {
workspace,
codexAppServerMode: CODEX_HARNESS_GUARDIAN_PROBE ? "guardian" : "yolo",
codeModeOnly: CODEX_HARNESS_CODE_MODE_ONLY,
compactionStress: CODEX_HARNESS_COMPACTION_STRESS,
...(CODEX_HARNESS_DISABLE_LOOP_RELAY ? { loopDetectionPreToolUseRelay: false } : {}),
});
const deviceIdentity = await ensurePairedTestGatewayClientIdentity({
@@ -1318,7 +1562,14 @@ describeLive("gateway live (Codex harness)", () => {
let guardianPluginApprovalDecision: GuardianPluginApprovalDecision | undefined;
let activeApprovalClient: GatewayClient | undefined;
let resumeStressState:
| { lastMarker: string; sessionKey: string; threadId: string }
| {
clientId: string;
hiddenMarker?: string;
lastMarker: string;
persistedCompactionCount: number;
sessionKey: string;
threadId: string;
}
| undefined;
const maybeResolveGuardianPluginApproval = (event: EventFrame): void => {
const decision = guardianPluginApprovalDecision;
@@ -1345,6 +1596,8 @@ describeLive("gateway live (Codex harness)", () => {
};
logCodexLiveStep("config-written", { configPath, modelKey, port });
observedCodexThreadIds.clear();
observedCodexClientIds.clear();
observedCodexThreadActions.clear();
try {
server = await startGatewayServer(port, {
@@ -1411,7 +1664,7 @@ describeLive("gateway live (Codex harness)", () => {
const firstText = await requestAgentText({
client: activeClient,
sessionKey,
expectedToken: firstToken,
expectedReply: firstToken,
message: `Reply with exactly ${firstToken} and nothing else.`,
});
expect(firstText).toContain(firstToken);
@@ -1422,7 +1675,7 @@ describeLive("gateway live (Codex harness)", () => {
const secondText = await requestAgentText({
client: activeClient,
sessionKey,
expectedToken: secondToken,
expectedReply: secondToken,
message: `Reply with exactly ${secondToken} and nothing else. Do not repeat ${firstToken}.`,
});
expect(secondText).toContain(secondToken);
@@ -1529,31 +1782,51 @@ describeLive("gateway live (Codex harness)", () => {
});
logCodexLiveStep("guardian-probe:done");
}
if (CODEX_HARNESS_RESUME_STRESS) {
const compactionStressState = CODEX_HARNESS_COMPACTION_STRESS
? await verifyCodexCompactionStress({
client: activeClient,
events: gatewayEvents,
sessionKey,
})
: undefined;
if (CODEX_HARNESS_RESTART_STRESS) {
const threadId = observedCodexThreadIds.get(sessionKey);
if (!threadId) {
throw new Error("Codex resume stress did not observe a thread identity");
}
const clientId = observedCodexClientIds.get(sessionKey);
if (!clientId) {
throw new Error("Codex resume stress did not observe a client identity");
}
let lastMarker = "";
for (
let historyTurn = 1;
historyTurn <= CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS;
historyTurn += 1
) {
const historyTurns = CODEX_HARNESS_RESUME_STRESS
? CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS
: 1;
for (let historyTurn = 1; historyTurn <= historyTurns; historyTurn += 1) {
lastMarker = `CODEX-HISTORY-${historyTurn}-${randomBytes(3)
.toString("hex")
.toUpperCase()}`;
await requestAgentText({
client: activeClient,
sessionKey,
expectedToken: lastMarker,
message: `Reply with exactly ${lastMarker} and nothing else.`,
expectedReply: lastMarker,
message: [
`Replace durable resume slot B with ${lastMarker}.`,
`Reply with exactly ${lastMarker} and nothing else.`,
].join(" "),
});
expect(observedCodexThreadIds.get(sessionKey)).toBe(threadId);
}
resumeStressState = { lastMarker, sessionKey, threadId };
resumeStressState = {
clientId,
hiddenMarker: compactionStressState?.hiddenMarker,
lastMarker,
persistedCompactionCount: compactionStressState?.persistedCount ?? 0,
sessionKey,
threadId,
};
logCodexLiveStep("resume-stress:history-ready", {
historyTurns: CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS + 2,
historyTurns: historyTurns + 2,
threadId,
});
}
@@ -1583,7 +1856,7 @@ describeLive("gateway live (Codex harness)", () => {
}
}
}
if (CODEX_HARNESS_RESUME_STRESS) {
if (CODEX_HARNESS_RESTART_STRESS) {
if (!resumeStressState) {
throw new Error("Codex resume stress did not seed a thread");
}
@@ -1622,25 +1895,67 @@ describeLive("gateway live (Codex harness)", () => {
const nextMarker = `CODEX-RESTART-${restart}-${randomBytes(3)
.toString("hex")
.toUpperCase()}`;
const priorClientId = resumeStressState.clientId;
const replyInstructions = [
"Read durable resume slot B before replacing it.",
"Reply with exactly the requested lines and nothing else.",
"First line: the current value of durable resume slot B.",
`Second line: ${nextMarker}`,
`Then replace durable resume slot B with ${nextMarker}.`,
...(resumeStressState.hiddenMarker
? ["Third line: the value stored in durable slot A."]
: []),
];
const resumedText = await requestAgentText({
client,
sessionKey: resumeStressState.sessionKey,
expectedToken: nextMarker,
message: [
"Reply with exactly two lines and nothing else.",
"First line: the final marker from the immediately preceding assistant message.",
`Second line: ${nextMarker}`,
].join(" "),
expectedReply: nextMarker,
message: replyInstructions.join(" "),
});
expect(resumedText.trim()).toBe(`${resumeStressState.lastMarker}\n${nextMarker}`);
const expectedResumeText = [
resumeStressState.lastMarker,
nextMarker,
...(resumeStressState.hiddenMarker ? [resumeStressState.hiddenMarker] : []),
].join("\n");
expect(resumedText.trim()).toBe(expectedResumeText);
const resumedThreadId = observedCodexThreadIds.get(resumeStressState.sessionKey);
expect(resumedThreadId).toBe(resumeStressState.threadId);
expect(observedCodexThreadActions.get(resumeStressState.sessionKey)).toBe("resumed");
const resumedClientId = observedCodexClientIds.get(resumeStressState.sessionKey);
expect(resumedClientId).toBeTruthy();
expect(resumedClientId).not.toBe(priorClientId);
resumeStressState.clientId = resumedClientId as string;
resumeStressState.lastMarker = nextMarker;
if (resumeStressState.persistedCompactionCount > 0) {
await readCodexHarnessCompactionCount({
client,
events: gatewayEvents,
minimum: resumeStressState.persistedCompactionCount,
sessionKey: resumeStressState.sessionKey,
});
}
logCodexLiveStep("resume-stress:restart-complete", {
restart,
action: observedCodexThreadActions.get(resumeStressState.sessionKey),
threadId: resumedThreadId,
});
}
if (CODEX_HARNESS_SUBAGENT_PROBE && !CODEX_HARNESS_SUBAGENT_ONLY) {
await verifyCodexSubagentProbe({
client,
sessionKey: resumeStressState.sessionKey,
});
}
if (CODEX_HARNESS_COMPACTION_STRESS) {
const continued = await verifyCodexCompactionStress({
client,
events: gatewayEvents,
sessionKey: resumeStressState.sessionKey,
});
expect(continued.persistedCount).toBeGreaterThan(
resumeStressState.persistedCompactionCount,
);
}
}
} finally {
try {
test/scripts/test-live-codex-harness-docker.test.ts
+18
View File
@@ -110,6 +110,24 @@ describe("scripts/test-live-codex-harness-docker.sh", () => {
expect(script).toContain(
'-e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}"',
);
expect(script).toContain(
'-e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT:-1}"',
);
expect(script).toContain(
'-e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS:-0}"',
);
expect(script).toContain(
'-e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS:-4}"',
);
expect(script).toContain(
'-e OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES="${OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES:-300000}"',
);
expect(script).toContain(
'-e OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY="${OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY:-0}"',
);
expect(script).toContain(
'-e OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY="${OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY:-0}"',
);
});
it("installs the plugin-pinned Codex CLI package for app-server proof", () => {