diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index dac7f8dae0f..926ac2c8336 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -50,7 +50,7 @@ on: kova_ref: description: openclaw/Kova Git ref to install required: false - default: 99b4b5c70fac2b13c48550c1d9bed09b795f0186 + default: ff755f6afcad9664a80e9ff29e15bbd29d9e1fa6 type: string dispatch_id: description: Optional parent workflow dispatch identifier @@ -153,7 +153,7 @@ jobs: include_filters: "scenario:agent-cold-warm-message" expected_release_entries: "agent-cold-warm-message:mock-openai-provider" env: - KOVA_REF: ${{ inputs.kova_ref || '99b4b5c70fac2b13c48550c1d9bed09b795f0186' }} + KOVA_REF: ${{ inputs.kova_ref || 'ff755f6afcad9664a80e9ff29e15bbd29d9e1fa6' }} KOVA_HOME: ${{ github.workspace }}/.artifacts/kova/home/${{ matrix.lane }} PERFORMANCE_HELPER_DIR: ${{ github.workspace }}/.artifacts/performance-workflow REPORT_DIR: ${{ github.workspace }}/.artifacts/kova/reports/${{ matrix.lane }} @@ -318,40 +318,6 @@ jobs: chmod 0755 "$HOME/.local/bin/kova" echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - name: Prepare live OpenAI candidate state - if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }} - shell: bash - run: | - set -euo pipefail - node - <<'NODE' - const fs = require("node:fs"); - const path = require("node:path"); - const file = path.join(process.env.KOVA_SRC, "states/mock-openai-provider.json"); - const state = JSON.parse(fs.readFileSync(file, "utf8")); - if (state.auth?.mode !== "mock") { - throw new Error(`expected mock-openai-provider auth.mode=mock, got ${state.auth?.mode}`); - } - - // The release profile pairs the live agent scenario with this otherwise mock-only state. - // This ephemeral checkout must honor the lane's explicit --auth live selection. - state.auth.mode = "default"; - state.auth.reason = "Honor the workflow lane's explicit run-level auth selection."; - state.setup = [ - ...(state.setup ?? []), - { - id: "force-openclaw-agent-runtime", - title: "Force OpenClaw Agent Runtime", - intent: "Keep live provider requests inside the OpenClaw harness for timeline evidence.", - afterPhase: "provision", - commands: [ - "ocm @{env} -- config set models.providers.openai.agentRuntime.id openclaw" - ], - evidence: ["OpenAI provider runtime is pinned to the OpenClaw harness"] - } - ]; - fs.writeFileSync(file, `${JSON.stringify(state, null, 2)}\n`, "utf8"); - NODE - - name: Configure OCM local workspace dependencies if: steps.lane.outputs.run == 'true' shell: bash @@ -488,6 +454,11 @@ jobs: report_json="$(node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-selector.mjs" --report-dir "$REPORT_DIR")" report_md="${report_json%.json}.md" + echo "status=$status" >> "$GITHUB_OUTPUT" + echo "report_json=$report_json" >> "$GITHUB_OUTPUT" + echo "report_md=$report_md" >> "$GITHUB_OUTPUT" + + set +e node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-workflow-evidence.mjs" \ --plan "$KOVA_PLAN_JSON" \ --report "$report_json" \ @@ -497,9 +468,11 @@ jobs: --include "$INCLUDE_FILTERS" \ --auth "$AUTH_MODE" \ --model "$PERFORMANCE_MODEL_ID" + evidence_status=$? + set -e effective_status="$status" - if [[ "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then + if [[ "$evidence_status" == "0" && "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then if node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "$report_json" then effective_status=0 @@ -509,16 +482,28 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" fi fi - echo "status=$status" >> "$GITHUB_OUTPUT" echo "effective_status=$effective_status" >> "$GITHUB_OUTPUT" - echo "report_json=$report_json" >> "$GITHUB_OUTPUT" - echo "report_md=$report_md" >> "$GITHUB_OUTPUT" + set +e kova report bundle "$report_json" --output-dir "$BUNDLE_DIR" --json | tee "$BUNDLE_DIR/bundle.json" + bundle_status=${PIPESTATUS[0]} + set -e summary_path="$SUMMARY_DIR/${LANE_ID}.md" summary_args=(node "$PERFORMANCE_HELPER_DIR/scripts/kova-ci-summary.mjs" --report "$report_json" --output "$summary_path" --lane "$LANE_ID") + set +e "${summary_args[@]}" + summary_status=$? + set -e + if [[ "$summary_status" != "0" ]]; then + cat > "$summary_path" <> "$summary_path" <> "$GITHUB_STEP_SUMMARY" + if [[ "$evidence_status" != "0" || "$bundle_status" != "0" || "$summary_status" != "0" ]]; then + exit 1 + fi if [[ "$FAIL_ON_REGRESSION" == "true" && "$effective_status" != "0" ]]; then exit "$effective_status" fi diff --git a/scripts/kova-ci-summary.mjs b/scripts/kova-ci-summary.mjs index 2af9d039fae..f74f24f9fde 100644 --- a/scripts/kova-ci-summary.mjs +++ b/scripts/kova-ci-summary.mjs @@ -32,7 +32,6 @@ const keyMetricIds = [ ]; const rssMetricIds = ["peakRssMb", "resourcePeakGatewayRssMb"]; const cpuMetricIds = ["cpuPercentMax"]; - const reportPath = path.resolve(args.report); const report = JSON.parse(await readFile(reportPath, "utf8")); const invalidReport = validateKovaSummaryReport(report); @@ -79,32 +78,39 @@ function renderSummary(reportLocal, options) { const groups = Array.isArray(reportLocal.performance?.groups) ? reportLocal.performance.groups : []; - if (groups.length > 0) { + const metricRows = []; + for (const group of groups) { + for (const metricId of keyMetricIds) { + const metric = group.metrics?.[metricId]; + if (!hasPositiveSampleCount(metric)) { + continue; + } + metricRows.push( + [ + value(group.scenario), + value(group.state), + value(metric.title || metricId), + formatMetric(metric.median, metric.unit), + formatMetric(metric.p95, metric.unit), + formatMetric(metric.max, metric.unit), + ] + .join(" | ") + .replace(/^/, "| ") + .replace(/$/, " |"), + ); + } + } + if (metricRows.length > 0) { lines.push("## Key metrics"); lines.push(""); lines.push("| Scenario | State | Metric | Median | p95 | Max |"); lines.push("| --- | --- | --- | ---: | ---: | ---: |"); - for (const group of groups) { - for (const metricId of keyMetricIds) { - const metric = group.metrics?.[metricId]; - if (!hasPositiveSampleCount(metric)) { - continue; - } - lines.push( - [ - value(group.scenario), - value(group.state), - value(metric.title || metricId), - formatMetric(metric.median, metric.unit), - formatMetric(metric.p95, metric.unit), - formatMetric(metric.max, metric.unit), - ] - .join(" | ") - .replace(/^/, "| ") - .replace(/$/, " |"), - ); - } - } + lines.push(...metricRows); + lines.push(""); + } else if (groups.length > 0) { + lines.push("## Key metrics"); + lines.push(""); + lines.push("- No sampled key metrics were available; inspect the blocking records below."); lines.push(""); } @@ -180,7 +186,11 @@ function validateKovaSummaryReport(reportLocal) { if (records.length === 0 && groups.length === 0) { return "missing records or performance groups"; } - if (groups.length > 0 && !hasExplicitResourceCollectionSkip(reportLocal)) { + if ( + groups.length > 0 && + reportHasOnlyPassingStatuses(statuses) && + !hasExplicitResourceCollectionSkip(reportLocal) + ) { if (!groups.some((group) => hasSampledMetric(group, rssMetricIds))) { return "missing sampled RSS metric in performance groups"; } @@ -191,16 +201,20 @@ function validateKovaSummaryReport(reportLocal) { return null; } +function reportHasOnlyPassingStatuses(statuses) { + const populated = Object.entries(statuses).filter(([, count]) => Number(count) > 0); + return ( + populated.length > 0 && populated.every(([status]) => status.trim().toUpperCase() === "PASS") + ); +} + function hasExplicitResourceCollectionSkip(reportLocal) { const reason = reportLocal.performance?.resourceCollectionSkippedReason; return typeof reason === "string" && reason.trim().length > 0; } function hasSampledMetric(group, metricIds) { - return metricIds.some((metricId) => { - const metric = group?.metrics?.[metricId]; - return hasPositiveSampleCount(metric); - }); + return metricIds.some((metricId) => hasPositiveSampleCount(group?.metrics?.[metricId])); } function hasPositiveSampleCount(metric) { diff --git a/test/scripts/kova-ci-summary.test.ts b/test/scripts/kova-ci-summary.test.ts index 538118eee7b..9af2849b1de 100644 --- a/test/scripts/kova-ci-summary.test.ts +++ b/test/scripts/kova-ci-summary.test.ts @@ -128,7 +128,37 @@ describe("scripts/kova-ci-summary", () => { expect(output).toContain("| gateway | clean | CPU max | 12 % | 12 % | 12 % |"); }); - it("rejects performance summaries without resource metrics", () => { + it("renders blocked reports without resource metrics", () => { + const { output, result } = runSummary({ + performance: { + repeat: 1, + groups: [ + { + metrics: { + timeToHealthReadyMs: { + count: 1, + max: 30, + median: 20, + p95: 30, + title: "Health ready", + unit: "ms", + }, + }, + scenario: "gateway", + state: "clean", + }, + ], + }, + records: [{ scenario: "gateway", state: "clean", status: "BLOCKED" }], + summary: { statuses: { BLOCKED: 1 } }, + }); + + expect(result.status).toBe(0); + expect(output).toContain("| gateway | clean | Health ready | 20 ms | 30 ms | 30 ms |"); + expect(output).toContain("| gateway | clean | BLOCKED |"); + }); + + it("rejects successful reports without resource metrics", () => { const { result } = runSummary({ performance: { repeat: 1, @@ -149,8 +179,8 @@ describe("scripts/kova-ci-summary", () => { }, ], }, - records: [{ scenario: "gateway", state: "clean", status: "pass" }], - summary: { statuses: { pass: 1 } }, + records: [{ scenario: "gateway", state: "clean", status: "PASS" }], + summary: { statuses: { PASS: 1 } }, }); expect(result.status).toBe(1); @@ -159,8 +189,8 @@ describe("scripts/kova-ci-summary", () => { ); }); - it("rejects performance summaries without CPU metrics", () => { - const { result } = runSummary({ + it("renders available metrics from failed reports when CPU samples are absent", () => { + const { output, result } = runSummary({ performance: { repeat: 1, groups: [ @@ -188,8 +218,38 @@ describe("scripts/kova-ci-summary", () => { }, ], }, - records: [{ scenario: "gateway", state: "clean", status: "pass" }], - summary: { statuses: { pass: 1 } }, + records: [{ scenario: "gateway", state: "clean", status: "FAIL" }], + summary: { statuses: { FAIL: 1 } }, + }); + + expect(result.status).toBe(0); + expect(output).toContain("| gateway | clean | Gateway RSS | 256 MB | 256 MB | 256 MB |"); + expect(output).toContain("| gateway | clean | FAIL |"); + }); + + it("rejects successful reports without CPU metrics", () => { + const { result } = runSummary({ + performance: { + repeat: 1, + groups: [ + { + metrics: { + resourcePeakGatewayRssMb: { + count: 1, + max: 256, + median: 256, + p95: 256, + title: "Gateway RSS", + unit: "MB", + }, + }, + scenario: "gateway", + state: "clean", + }, + ], + }, + records: [{ scenario: "gateway", state: "clean", status: "PASS" }], + summary: { statuses: { PASS: 1 } }, }); expect(result.status).toBe(1); @@ -198,8 +258,8 @@ describe("scripts/kova-ci-summary", () => { ); }); - it("rejects malformed resource metric counts instead of treating them as sampled", () => { - const { result } = runSummary({ + it("omits malformed resource metrics instead of rejecting failure evidence", () => { + const { output, result } = runSummary({ performance: { repeat: 1, groups: [ @@ -227,14 +287,12 @@ describe("scripts/kova-ci-summary", () => { }, ], }, - records: [{ scenario: "gateway", state: "clean", status: "pass" }], - summary: { statuses: { pass: 1 } }, + records: [{ scenario: "gateway", state: "clean", status: "FAIL" }], + summary: { statuses: { FAIL: 1 } }, }); - expect(result.status).toBe(1); - expect(result.stderr).toContain( - "invalid Kova report: missing sampled RSS metric in performance groups", - ); + expect(result.status).toBe(0); + expect(output).toContain("No sampled key metrics were available"); }); it("omits key metric rows with invalid sample counts", () => { diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index 90decd858bb..7f0b16881a4 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -82,7 +82,7 @@ describe("OpenClaw performance workflow", () => { it("pins the Kova evaluator with release validation contracts", () => { const workflow = readFileSync(WORKFLOW, "utf8"); - const kovaRef = "99b4b5c70fac2b13c48550c1d9bed09b795f0186"; + const kovaRef = "ff755f6afcad9664a80e9ff29e15bbd29d9e1fa6"; const install = findStep("Install OCM and Kova"); const installRun = install.run ?? ""; @@ -666,25 +666,34 @@ esac expect(sanity.run).not.toContain("--include scenario:fresh-install"); }); - it("makes the live lane use live auth through the OpenClaw runtime", () => { - const override = findStep("Prepare live OpenAI candidate state"); + it("uses Kova's explicit live auth contract without rewriting its state registry", () => { + const workflow = readWorkflow(); + const stepNames = workflow.jobs?.kova?.steps?.map((step) => step.name) ?? []; + const runKova = findStep("Run Kova"); - expect(override.if).toContain("matrix.live == 'true'"); - expect(override.run).toContain("states/mock-openai-provider.json"); - expect(override.run).toContain('state.auth?.mode !== "mock"'); - expect(override.run).toContain('state.auth.mode = "default"'); - expect(override.run).toContain( - "This ephemeral checkout must honor the lane's explicit --auth live selection.", + expect(stepNames).not.toContain("Prepare live OpenAI candidate state"); + expect(runKova.run).toContain('--auth "$AUTH_MODE"'); + expect(runKova.run).toContain('args+=(--model "$PERFORMANCE_MODEL_ID")'); + expect(JSON.stringify(workflow)).not.toContain("states/mock-openai-provider.json"); + }); + + it("finalizes Kova artifacts before failing evidence integrity", () => { + const run = findStep("Run Kova").run ?? ""; + const evidence = run.indexOf("scripts/lib/kova-workflow-evidence.mjs"); + const bundle = run.indexOf('kova report bundle "$report_json"'); + const summary = run.indexOf("scripts/kova-ci-summary.mjs"); + const integrityExit = run.indexOf( + 'if [[ "$evidence_status" != "0" || "$bundle_status" != "0" || "$summary_status" != "0" ]]', ); - expect(override.run).toContain( - 'state.auth.reason = "Honor the workflow lane\'s explicit run-level auth selection."', - ); - expect(override.run).toContain('id: "force-openclaw-agent-runtime"'); - expect(override.run).toContain('afterPhase: "provision"'); - expect(override.run).toContain( - "ocm @{env} -- config set models.providers.openai.agentRuntime.id openclaw", - ); - expect(override.run).not.toContain("agents.defaults.agentRuntime"); + + expect(evidence).toBeGreaterThan(-1); + expect(bundle).toBeGreaterThan(evidence); + expect(summary).toBeGreaterThan(bundle); + expect(integrityExit).toBeGreaterThan(summary); + expect(run).toContain("evidence_status=$?"); + expect(run).toContain("bundle_status=${PIPESTATUS[0]}"); + expect(run).toContain("summary_status=$?"); + expect(run).toContain("Summary generation failed with status ${summary_status}"); }); it("runs the trusted lane evidence validator before tolerating gate failures", () => {