test(qa): prove Codex cold install product path (#109267)

This commit is contained in:
Dallin Romney
2026-07-16 13:10:41 -07:00
committed by GitHub
parent a9f82837a1
commit b5eb8977dc
11 changed files with 132 additions and 306 deletions
@@ -1654,7 +1654,6 @@ describe("qa cli runtime", () => {
"auth-profile-codex-mixed-profiles",
"auth-profile-doctor-migration-safety",
"codex-plugin-cold-install",
"codex-plugin-install-race",
"codex-plugin-pinned-new",
"codex-plugin-pinned-old",
"runtime-first-hour-20-turn",
@@ -12,7 +12,6 @@ import {
import {
CODEX_PLUGIN_CURRENT_VERSION,
CODEX_PLUGIN_LIFECYCLE_MESSAGES,
createCodexPluginInstallGate,
evaluateCodexPluginLifecycle,
seedCodexPluginAt,
snapshotCodexPluginState,
@@ -148,31 +147,6 @@ describe("codex plugin lifecycle: pinned-new codex plugin with old OpenClaw", ()
});
});
describe("codex plugin lifecycle: install racing first agent turn", () => {
it("gates the first turn on install completion without sleeps, lost tokens, or duplicate responses", async () => {
const gate = createCodexPluginInstallGate();
const turn = gate.runFirstTurnAfterInstall({
inputTokens: 17,
run: () => "QA_CODEX_PLUGIN_TURN_OK",
});
expect(gate.events).toEqual(["agent-turn:waiting-for-codex-plugin"]);
gate.markInstalled();
await expect(turn).resolves.toEqual({
text: "QA_CODEX_PLUGIN_TURN_OK",
inputTokens: 17,
responseCount: 1,
});
expect(gate.events).toEqual([
"agent-turn:waiting-for-codex-plugin",
"codex-plugin:installed",
"agent-turn:started",
"agent-turn:completed",
]);
});
});
describe("codex plugin lifecycle: doctor migration safety matrix", () => {
it.each([
{
@@ -44,12 +44,6 @@ type CodexPluginPackageJson = {
};
};
type CodexPluginInstallGateResult = {
text: string;
inputTokens: number;
responseCount: number;
};
function codexPluginDir(agentDir: string) {
return path.join(agentDir, "plugins", CODEX_PLUGIN_ID);
}
@@ -251,41 +245,3 @@ export function evaluateCodexPluginLifecycle(params: {
removedRuntimePins,
};
}
export function createCodexPluginInstallGate() {
const events: string[] = [];
let installed = false;
let resolveInstall: (() => void) | undefined;
const installedPromise = new Promise<void>((resolve) => {
resolveInstall = resolve;
});
return {
events,
markInstalled() {
if (installed) {
return;
}
installed = true;
events.push("codex-plugin:installed");
resolveInstall?.();
},
async runFirstTurnAfterInstall(params: {
inputTokens: number;
run: () => string | Promise<string>;
}): Promise<CodexPluginInstallGateResult> {
if (!installed) {
events.push("agent-turn:waiting-for-codex-plugin");
await installedPromise;
}
events.push("agent-turn:started");
const text = await params.run();
events.push("agent-turn:completed");
return {
text,
inputTokens: params.inputTokens,
responseCount: 1,
};
},
};
}
+10 -6
View File
@@ -284,6 +284,7 @@ describe("qa scenario catalog", () => {
it("routes Docker runtime scenarios through the shared lane adapter", () => {
const scenarioLanes = [
["codex-plugin-cold-install", "codex-on-demand"],
["openai-compatible-chat-tools", "openai-chat-tools"],
["openai-web-search-minimal", "openai-web-search-minimal"],
["openwebui-openai-compatible", "openwebui"],
@@ -326,7 +327,6 @@ describe("qa scenario catalog", () => {
"auth-profile-codex-mixed-profiles",
"auth-profile-doctor-migration-safety",
"codex-plugin-cold-install",
"codex-plugin-install-race",
"codex-plugin-pinned-new",
"codex-plugin-pinned-old",
"plugin-manifest-contract-health",
@@ -562,17 +562,21 @@ describe("qa scenario catalog", () => {
]);
});
it("loads the Codex plugin lifecycle fixture scenarios into the standard runtime tier", () => {
const scenarioIds = [
"codex-plugin-cold-install",
"codex-plugin-install-race",
it("loads Codex plugin lifecycle scenarios into the standard runtime tier", () => {
const coldInstall = readQaScenarioById("codex-plugin-cold-install");
expect(coldInstall.runtimeParityTier).toBe("standard");
expect(coldInstall.coverage?.primary).toContain("runtime.codex-plugin.lifecycle");
expect(coldInstall.coverage?.secondary).toBeUndefined();
expect(coldInstall.execution.kind).toBe("script");
const fixtureScenarioIds = [
"codex-plugin-pinned-old",
"codex-plugin-pinned-new",
"auth-profile-codex-mixed-profiles",
"auth-profile-doctor-migration-safety",
];
for (const scenarioId of scenarioIds) {
for (const scenarioId of fixtureScenarioIds) {
const scenario = readQaScenarioById(scenarioId);
expect(scenario.runtimeParityTier).toBe("standard");
expect(scenario.coverage?.primary.length).toBeGreaterThan(0);
@@ -372,7 +372,7 @@ describe("scenario-flow-runner", () => {
},
{
assert: {
expr: 'typeof plugin.createCodexPluginInstallGate === "function"',
expr: 'typeof plugin.evaluateCodexPluginLifecycle === "function"',
},
},
],
@@ -386,92 +386,6 @@ describe("scenario-flow-runner", () => {
expect(result.steps[0]?.details).toBe("loaded");
});
it("can hold a gated promise across later flow actions", async () => {
const result = await runScenarioFlow({
api: {
state: createQaBusState(),
scenario: {
id: "qa-gated-promise",
title: "qa-gated-promise",
sourcePath: "qa/scenarios/qa-gated-promise.yaml",
surface: "test",
objective: "test",
successCriteria: ["test"],
execution: { kind: "flow" },
},
config: { expectedText: "QA_CODEX_PLUGIN_TURN_OK" },
runScenario: async (
_name: string,
steps: Array<{ name: string; run: () => Promise<string | void> }>,
) => {
const stepResults = [];
for (const step of steps) {
const details = await step.run();
stepResults.push({
name: step.name,
status: "pass" as const,
...(details !== undefined ? { details } : {}),
});
}
return {
name: "qa-gated-promise",
status: "pass" as const,
steps: stepResults,
};
},
},
scenarioTitle: "qa-gated-promise",
flow: {
steps: [
{
name: "uses deferred promise wrapper",
actions: [
{
set: "plugin",
value: {
expr: 'await qaImport("./codex-plugin.fixture.js")',
},
},
{
set: "gate",
value: {
expr: "plugin.createCodexPluginInstallGate()",
},
},
{
set: "turn",
value: {
expr: "({ promise: gate.runFirstTurnAfterInstall({ inputTokens: 17, run: () => config.expectedText }) })",
},
},
{
assert: {
expr: 'JSON.stringify(gate.events) === JSON.stringify(["agent-turn:waiting-for-codex-plugin"])',
},
},
{ call: "gate.markInstalled" },
{
set: "completed",
value: {
expr: "await turn.promise",
},
},
{
assert: {
expr: "completed.text === config.expectedText && completed.responseCount === 1 && completed.inputTokens === 17",
},
},
],
detailsExpr: "completed.text",
},
],
},
});
expect(result.status).toBe("pass");
expect(result.steps[0]?.details).toBe("QA_CODEX_PLUGIN_TURN_OK");
});
it.each([
{
scenarioId: "channel-chat-baseline",
@@ -6,85 +6,29 @@ scenario:
runtimeParityTier: standard
runtimeParityUsage:
expectation: not-applicable
reason: Local plugin-lifecycle fixture only; no assistant turn runs.
reason: Packaged CLI/plugin-lifecycle proof only; no assistant turn runs.
coverage:
primary:
- runtime.codex-plugin.lifecycle
secondary:
- runtime.doctor-repair
objective: Verify a clean home that needs the Codex runtime reports a clear missing-plugin remediation, installs through doctor repair, and retries through Codex OAuth instead of OpenAI API-key auth.
objective: Verify a package-installed OpenClaw in a clean Docker home downloads the official Codex runtime on demand during OpenAI onboarding and records a loadable managed install.
successCriteria:
- Missing Codex plugin emits the exact remediation string asserted by the fixture test.
- Doctor repair seeds the Codex plugin before retrying the agent turn.
- The retry uses the openai OAuth profile and never routes through the openai API-key profile.
- The prepared OpenClaw tarball and clean home contain neither @openclaw/codex nor @openai/codex before onboarding.
- Non-interactive OpenAI onboarding emits exactly one successful JSON terminal document and downloads @openclaw/codex into the managed npm root.
- The canonical SQLite plugin index records the npm install while openclaw.json contains no legacy plugins.installs record.
- The installed @openai/codex entrypoint executes its native Codex binary, reports a Codex CLI version, and the Codex plugin loads with the codex agent harness.
docsRefs:
- docs/cli/doctor.md
- docs/cli/plugins.md
- docs/plugins/install-overrides.md
codeRefs:
- extensions/qa-lab/src/codex-plugin.fixture.ts
- extensions/qa-lab/src/auth-profile.fixture.ts
- extensions/qa-lab/src/codex-plugin-lifecycle.test.ts
- src/commands/codex-runtime-plugin-install.ts
- src/plugins/installed-plugin-index-records.ts
- scripts/e2e/codex-on-demand-docker.sh
- scripts/e2e/lib/codex-on-demand/assertions.mjs
execution:
kind: flow
summary: Exercise the Codex lifecycle fixture for missing plugin repair and retry auth routing.
config:
remediation: Codex plugin is required for Codex runtime. Run "openclaw doctor --fix" to install @openclaw/codex, then retry.
flow:
steps:
- name: validates cold-install repair routing
actions:
- set: auth
value:
expr: await qaImport("./auth-profile.fixture.js")
- set: plugin
value:
expr: await qaImport("./codex-plugin.fixture.js")
- set: tmpRoot
value:
expr: await fs.mkdtemp(path.join(env.gateway?.workspaceDir ?? "/tmp", "qa-codex-cold-"))
- set: agentDir
value:
expr: path.join(tmpRoot, "agents", "qa", "agent")
- try:
actions:
- call: plugin.seedCodexPluginAt
args:
- missing
- ref: agentDir
- call: auth.seedAuthProfiles
args:
- mixed
- ref: agentDir
- set: missing
value:
expr: "plugin.evaluateCodexPluginLifecycle({ plugin: await plugin.snapshotCodexPluginState(agentDir), auth: await auth.snapshotAuthProfiles(agentDir), hostVersion: plugin.CODEX_PLUGIN_CURRENT_VERSION })"
- assert:
expr: "missing.status === 'repair-required'"
message:
expr: "`expected repair-required, got ${JSON.stringify(missing)}`"
- assert:
expr: "missing.remediation === config.remediation"
message: missing Codex plugin remediation drifted
- assert:
expr: "missing.selectedAuthProfileId === auth.QA_CODEX_OAUTH_PROFILE_ID"
message: missing-plugin repair must keep Codex OAuth selected
- call: plugin.seedCodexPluginAt
args:
- current
- ref: agentDir
- set: repaired
value:
expr: "plugin.evaluateCodexPluginLifecycle({ plugin: await plugin.snapshotCodexPluginState(agentDir), auth: await auth.snapshotAuthProfiles(agentDir), hostVersion: plugin.CODEX_PLUGIN_CURRENT_VERSION })"
- assert:
expr: "repaired.status === 'ready' && repaired.tokenRoute === 'codex-oauth'"
message:
expr: "`expected repaired Codex OAuth route, got ${JSON.stringify(repaired)}`"
finally:
- call: fs.rm
args:
- ref: tmpRoot
- recursive: true
force: true
detailsExpr: "`missing=${missing.status} repaired=${repaired.status} route=${repaired.tokenRoute}`"
kind: script
path: test/e2e/qa-lab/runtime/docker-e2e-lane.ts
summary: Runs the package-backed codex-on-demand Docker lane and verifies canonical install records, managed dependencies, plugin loading, and the terminal onboarding result.
timeoutMs: 1800000
args:
- --lane
- codex-on-demand
@@ -1,64 +0,0 @@
title: Codex plugin install race
scenario:
id: codex-plugin-install-race
surface: runtime
runtimeParityTier: standard
runtimeParityUsage:
expectation: not-applicable
reason: Local install-ordering fixture only; no assistant turn runs.
coverage:
primary:
- runtime.codex-plugin.lifecycle
secondary:
- runtime.turn-ordering
objective: Verify first agent turns wait on Codex plugin installation through deterministic ordering primitives, without sleep-based race assertions, lost tokens, or duplicate responses.
successCriteria:
- The first turn records a waiting event before the install completion event.
- The turn starts exactly once after the install completion event.
- Input-token accounting survives the gate and responseCount remains 1.
docsRefs:
- docs/cli/plugins.md
codeRefs:
- extensions/qa-lab/src/codex-plugin.fixture.ts
- extensions/qa-lab/src/codex-plugin-lifecycle.test.ts
execution:
kind: flow
summary: Exercise the deterministic install-vs-first-turn gate.
config:
expectedResponseCount: 1
expectedText: QA_CODEX_PLUGIN_TURN_OK
flow:
steps:
- name: validates deterministic install-race gate
actions:
- set: plugin
value:
expr: await qaImport("./codex-plugin.fixture.js")
- set: gate
value:
expr: plugin.createCodexPluginInstallGate()
- set: turn
value:
expr: "({ promise: gate.runFirstTurnAfterInstall({ inputTokens: 17, run: () => config.expectedText }) })"
- assert:
expr: "JSON.stringify(gate.events) === JSON.stringify(['agent-turn:waiting-for-codex-plugin'])"
message:
expr: "`expected first turn to wait, got ${JSON.stringify(gate.events)}`"
- call: gate.markInstalled
- set: completed
value:
expr: await turn.promise
- assert:
expr: "completed.text === config.expectedText && completed.responseCount === config.expectedResponseCount && completed.inputTokens === 17"
message:
expr: "`unexpected completed turn: ${JSON.stringify(completed)}`"
- assert:
expr: "JSON.stringify(gate.events) === JSON.stringify(['agent-turn:waiting-for-codex-plugin', 'codex-plugin:installed', 'agent-turn:started', 'agent-turn:completed'])"
message:
expr: "`unexpected install ordering: ${JSON.stringify(gate.events)}`"
- assert:
expr: "config.expectedResponseCount === 1"
message: "first turn must produce one response"
detailsExpr: "`expected=${completed.text} count=${completed.responseCount}`"
+28 -2
View File
@@ -1,4 +1,5 @@
// Assertions for Codex on-demand plugin E2E scenarios.
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { DatabaseSync } from "node:sqlite";
@@ -15,9 +16,16 @@ import {
} from "../codex-install-utils.mjs";
const cfg = readJson(configPath());
const onboard = readJson("/tmp/openclaw-onboard.json");
const inspect = readJson("/tmp/openclaw-codex-inspect.json");
const records = readInstallRecords(cfg.plugins?.installs);
const codexRecord = records.codex || inspect.install;
const records = readInstallRecords();
const codexRecord = records.codex;
if (onboard.ok !== true || onboard.mode !== "local" || onboard.authChoice !== "openai-api-key") {
throw new Error(`unexpected onboarding terminal result: ${JSON.stringify(onboard)}`);
}
if (cfg.plugins?.installs !== undefined) {
throw new Error("codex install record remained in config instead of the canonical SQLite index");
}
if (!codexRecord) {
throw new Error(`missing codex install record: ${JSON.stringify(records)}`);
}
@@ -69,6 +77,24 @@ if (!fs.existsSync(codexBin)) {
throw new Error(`missing managed Codex binary: ${codexBin}`);
}
assertPathInside(npmRoot, codexBin, "managed Codex binary");
const codexVersion = spawnSync(process.execPath, [codexBin, "--version"], {
encoding: "utf8",
maxBuffer: 64 * 1024,
timeout: 15_000,
windowsHide: true,
});
const codexVersionStdout = codexVersion.stdout?.trim() ?? "";
const codexVersionStderr = codexVersion.stderr?.trim() ?? "";
if (codexVersion.error || codexVersion.status !== 0) {
const failure = codexVersion.error?.message ?? `exit status ${String(codexVersion.status)}`;
const output = codexVersionStderr || codexVersionStdout || "no output";
throw new Error(`managed Codex --version failed (${failure}): ${output}`);
}
if (!/^codex-cli\s+\S+$/u.test(codexVersionStdout)) {
throw new Error(
`unexpected managed Codex --version output: ${JSON.stringify(codexVersionStdout)}`,
);
}
const list = readJson("/tmp/openclaw-plugins-list.json");
const plugin = (list.plugins || []).find((entry) => entry.id === "codex");
@@ -13,6 +13,7 @@ describe("QA Docker E2E lane fixture", () => {
expect(listQaDockerE2eLaneNames()).toEqual(
expect.arrayContaining([
"agent-bundle-mcp-tools",
"codex-on-demand",
"system-agent-first-run",
"gateway-network",
"release-plugin-marketplace",
@@ -44,6 +45,10 @@ describe("QA Docker E2E lane fixture", () => {
});
it("resolves lane-specific environment overlays at run time", () => {
expect(resolveQaDockerE2eLane("codex-on-demand", {}).script).toBe(
"scripts/e2e/codex-on-demand-docker.sh",
);
const updateMigration = resolveQaDockerE2eLane("update-migration", {
OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC: "openclaw@custom",
OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: "custom-scenario",
@@ -28,6 +28,9 @@ const QA_DOCKER_E2E_LANES = {
"bundled-plugin-install-uninstall": {
script: "scripts/e2e/bundled-plugin-install-uninstall-docker.sh",
},
"codex-on-demand": {
script: "scripts/e2e/codex-on-demand-docker.sh",
},
"system-agent-first-run": {
script: "scripts/e2e/system-agent-first-run-docker.sh",
},
+68 -3
View File
@@ -27,6 +27,7 @@ const tmpFixtureFiles = [
"/tmp/openclaw-codex-inspect.json",
"/tmp/openclaw-codex-plugin-inspect.json",
"/tmp/openclaw-codex-plugins-list.json",
"/tmp/openclaw-onboard.json",
"/tmp/openclaw-plugins-list.json",
];
@@ -542,13 +543,17 @@ function createCodexInstallFixture(root: string) {
});
const codexBin = path.join(openAiCodexRoot, "bin", "codex.js");
mkdirSync(path.dirname(codexBin), { recursive: true });
writeFileSync(codexBin, "#!/usr/bin/env node\n", { mode: 0o755 });
writeFileSync(codexBin, '#!/usr/bin/env node\nconsole.log("codex-cli 0.0.0-test");\n', {
mode: 0o755,
});
chmodSync(codexBin, 0o755);
writeJson(path.join(stateDir, "openclaw.json"), {
agents: { defaults: { model: { primary: "openai/gpt-5.6" } } },
models: { providers: { openai: { agentRuntime: { id: "codex" } } } },
plugins: {
installs: {
});
writePluginInstallIndexForE2E(
{
installRecords: {
codex: {
installPath,
source: "npm",
@@ -556,6 +561,12 @@ function createCodexInstallFixture(root: string) {
},
},
},
{ stateDir },
);
writeJson("/tmp/openclaw-onboard.json", {
ok: true,
mode: "local",
authChoice: "openai-api-key",
});
writeJson("/tmp/openclaw-codex-inspect.json", {
plugin: { id: "codex", status: "loaded", agentHarnessIds: ["codex"] },
@@ -683,6 +694,32 @@ describe("Codex install helpers", () => {
expect(result.stderr).toBe("");
});
it("rejects on-demand fixtures without the canonical SQLite install record", () => {
const root = makeTempDir(tempDirs, "openclaw-codex-on-demand-no-index-");
createCodexInstallFixture(root);
rmSync(path.join(root, "state", "state", "openclaw.sqlite"), { force: true });
const result = runCodexOnDemandAssertions(root);
expect(result.status).not.toBe(0);
expect(result.stderr).toContain("missing codex install record");
});
it("rejects duplicate onboarding terminal JSON documents", () => {
const root = makeTempDir(tempDirs, "openclaw-codex-on-demand-duplicate-terminal-");
createCodexInstallFixture(root);
writeFileSync(
"/tmp/openclaw-onboard.json",
`${JSON.stringify({ ok: true })}\n${JSON.stringify({ ok: true })}\n`,
"utf8",
);
const result = runCodexOnDemandAssertions(root);
expect(result.status).not.toBe(0);
expect(result.stderr).toContain("Unexpected non-whitespace character after JSON");
});
it("accepts SQLite-backed session and Codex binding state in the npm live assertion", () => {
const root = makeTempDir(tempDirs, "openclaw-codex-npm-live-");
const fixture = createCodexNpmPluginLiveFixture(root);
@@ -931,4 +968,32 @@ describe("Codex install helpers", () => {
expect(result.status).not.toBe(0);
expect(result.stderr).toContain("missing managed Codex binary:");
});
it("rejects a present managed Codex wrapper when its native executable is unavailable", () => {
const root = makeTempDir(tempDirs, "openclaw-codex-on-demand-broken-native-");
createCodexInstallFixture(root);
const codexBin = path.join(
root,
"state",
"npm",
"projects",
"codex",
"node_modules",
"@openai",
"codex",
"bin",
"codex.js",
);
writeFileSync(
codexBin,
'#!/usr/bin/env node\nconsole.error("Missing optional dependency @openai/codex-linux-x64");\nprocess.exit(1);\n',
{ mode: 0o755 },
);
const result = runCodexOnDemandAssertions(root);
expect(result.status).not.toBe(0);
expect(result.stderr).toContain("managed Codex --version failed (exit status 1)");
expect(result.stderr).toContain("Missing optional dependency @openai/codex-linux-x64");
});
});