furyhawk/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-21 02:06:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(test): require startup bench report overlap

Browse Source
This commit is contained in:
Vincent Koc
2026-06-06 20:53:08 +02:00
parent cd806101cd
commit f2677d55ec
2 changed files with 170 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/test-cli-startup-bench-budget.mjs
+19
View File
@@ -154,15 +154,34 @@ const current = readJsonFile(resolveCurrentReportPath());
const baselineCases = indexCases(baseline);
const currentCases = indexCases(current);
const shouldRequireEveryBaselineCase = opts.preset === "all";
const matchedBaselineCaseIds = [...baselineCases.keys()].filter((id) => currentCases.has(id));
let failed = false;
if (currentCases.size === 0) {
console.error(
`[test-cli-startup-bench-budget] current report has no cases for preset ${opts.preset}`,
);
failed = true;
}
for (const [id] of baselineCases) {
if (shouldRequireEveryBaselineCase && !currentCases.has(id)) {
console.error(`[test-cli-startup-bench-budget] missing current case ${String(id)}`);
failed = true;
}
}
if (
!opts.skipBaseline &&
!shouldRequireEveryBaselineCase &&
baselineCases.size > 0 &&
matchedBaselineCaseIds.length === 0
) {
console.error(
`[test-cli-startup-bench-budget] no current cases matched the baseline for preset ${opts.preset}`,
);
failed = true;
}
for (const currentCase of currentCases.values()) {
const samples = currentCase.samples ?? [];
test/scripts/cli-startup-bench-spawner.test.ts
+151
View File
@@ -104,6 +104,157 @@ describe("CLI startup benchmark script spawners", () => {
}
});
it("rejects narrowed preset reports with no matching current cases", () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-empty-test-"));
try {
const baselinePath = path.join(tmpDir, "baseline.json");
const reportPath = path.join(tmpDir, "current.json");
fs.writeFileSync(
baselinePath,
JSON.stringify({
primary: {
cases: [
{
id: "gatewayStatusJson",
name: "gateway status --json",
samples: [{ ms: 10, firstOutputMs: 5, maxRssMb: 10, exitCode: 0, signal: null }],
summary: {
durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
firstOutputMs: { avg: 5, p50: 5, p95: 5, min: 5, max: 5 },
maxRssMb: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
},
},
],
},
}),
);
fs.writeFileSync(reportPath, JSON.stringify({ primary: { cases: [] } }));
const result = spawnSync(
process.execPath,
[
"scripts/test-cli-startup-bench-budget.mjs",
"--baseline",
baselinePath,
"--report",
reportPath,
"--preset",
"real",
],
{ cwd: process.cwd(), encoding: "utf8" },
);
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"[test-cli-startup-bench-budget] current report has no cases for preset real",
);
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
it("rejects narrowed preset reports with unrelated current cases when baseline checks run", () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-overlap-test-"));
try {
const baselinePath = path.join(tmpDir, "baseline.json");
const reportPath = path.join(tmpDir, "current.json");
const makeCase = (id: string, name: string) => ({
id,
name,
samples: [{ ms: 10, firstOutputMs: 5, maxRssMb: 10, exitCode: 0, signal: null }],
summary: {
durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
firstOutputMs: { avg: 5, p50: 5, p95: 5, min: 5, max: 5 },
maxRssMb: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
},
});
fs.writeFileSync(
baselinePath,
JSON.stringify({ primary: { cases: [makeCase("fixtureOnly", "fixture only")] } }),
);
fs.writeFileSync(
reportPath,
JSON.stringify({ primary: { cases: [makeCase("targetOnly", "target only")] } }),
);
const result = spawnSync(
process.execPath,
[
"scripts/test-cli-startup-bench-budget.mjs",
"--baseline",
baselinePath,
"--report",
reportPath,
"--preset",
"real",
],
{
cwd: process.cwd(),
encoding: "utf8",
env: {
...process.env,
OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH: "1",
},
},
);
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"[test-cli-startup-bench-budget] no current cases matched the baseline for preset real",
);
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
it("allows skip-baseline reports without fixture case overlap", () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-skip-test-"));
try {
const baselinePath = path.join(tmpDir, "baseline.json");
const reportPath = path.join(tmpDir, "current.json");
const makeCase = (id: string, name: string) => ({
id,
name,
samples: [{ ms: 10, firstOutputMs: 5, maxRssMb: 10, exitCode: 0, signal: null }],
summary: {
durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
firstOutputMs: { avg: 5, p50: 5, p95: 5, min: 5, max: 5 },
maxRssMb: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
},
});
fs.writeFileSync(
baselinePath,
JSON.stringify({ primary: { cases: [makeCase("fixtureOnly", "fixture only")] } }),
);
fs.writeFileSync(
reportPath,
JSON.stringify({ primary: { cases: [makeCase("targetOnly", "target only")] } }),
);
const result = spawnSync(
process.execPath,
[
"scripts/test-cli-startup-bench-budget.mjs",
"--baseline",
baselinePath,
"--report",
reportPath,
"--preset",
"real",
"--skip-baseline",
],
{ cwd: process.cwd(), encoding: "utf8" },
);
expect(result.status).toBe(0);
expect(result.stderr).not.toContain("no current cases matched the baseline");
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
it("skips x64 startup budgets on noncanonical architectures", () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-arch-test-"));
try {