furyhawk/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-21 02:06:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): preserve Kova failure evidence

Browse Source
This commit is contained in:
Vincent Koc
2026-07-11 15:57:55 +08:00
committed by Vincent Koc
parent 5219c9353d
commit aef26f6020
4 changed files with 170 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/openclaw-performance.yml
+28 -40
View File
@@ -50,7 +50,7 @@ on:
kova_ref:
description: openclaw/Kova Git ref to install
required: false
default: 99b4b5c70fac2b13c48550c1d9bed09b795f0186
default: ff755f6afcad9664a80e9ff29e15bbd29d9e1fa6
type: string
dispatch_id:
description: Optional parent workflow dispatch identifier
@@ -153,7 +153,7 @@ jobs:
include_filters: "scenario:agent-cold-warm-message"
expected_release_entries: "agent-cold-warm-message:mock-openai-provider"
env:
KOVA_REF: ${{ inputs.kova_ref || '99b4b5c70fac2b13c48550c1d9bed09b795f0186' }}
KOVA_REF: ${{ inputs.kova_ref || 'ff755f6afcad9664a80e9ff29e15bbd29d9e1fa6' }}
KOVA_HOME: ${{ github.workspace }}/.artifacts/kova/home/${{ matrix.lane }}
PERFORMANCE_HELPER_DIR: ${{ github.workspace }}/.artifacts/performance-workflow
REPORT_DIR: ${{ github.workspace }}/.artifacts/kova/reports/${{ matrix.lane }}
@@ -318,40 +318,6 @@ jobs:
chmod 0755 "$HOME/.local/bin/kova"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Prepare live OpenAI candidate state
if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}
shell: bash
run: |
set -euo pipefail
node - <<'NODE'
const fs = require("node:fs");
const path = require("node:path");
const file = path.join(process.env.KOVA_SRC, "states/mock-openai-provider.json");
const state = JSON.parse(fs.readFileSync(file, "utf8"));
if (state.auth?.mode !== "mock") {
throw new Error(`expected mock-openai-provider auth.mode=mock, got ${state.auth?.mode}`);
}
// The release profile pairs the live agent scenario with this otherwise mock-only state.
// This ephemeral checkout must honor the lane's explicit --auth live selection.
state.auth.mode = "default";
state.auth.reason = "Honor the workflow lane's explicit run-level auth selection.";
state.setup = [
...(state.setup ?? []),
{
id: "force-openclaw-agent-runtime",
title: "Force OpenClaw Agent Runtime",
intent: "Keep live provider requests inside the OpenClaw harness for timeline evidence.",
afterPhase: "provision",
commands: [
"ocm @{env} -- config set models.providers.openai.agentRuntime.id openclaw"
],
evidence: ["OpenAI provider runtime is pinned to the OpenClaw harness"]
}
];
fs.writeFileSync(file, `${JSON.stringify(state, null, 2)}\n`, "utf8");
NODE
- name: Configure OCM local workspace dependencies
if: steps.lane.outputs.run == 'true'
shell: bash
@@ -488,6 +454,11 @@ jobs:
report_json="$(node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-selector.mjs" --report-dir "$REPORT_DIR")"
report_md="${report_json%.json}.md"
echo "status=$status" >> "$GITHUB_OUTPUT"
echo "report_json=$report_json" >> "$GITHUB_OUTPUT"
echo "report_md=$report_md" >> "$GITHUB_OUTPUT"
set +e
node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-workflow-evidence.mjs" \
--plan "$KOVA_PLAN_JSON" \
--report "$report_json" \
@@ -497,9 +468,11 @@ jobs:
--include "$INCLUDE_FILTERS" \
--auth "$AUTH_MODE" \
--model "$PERFORMANCE_MODEL_ID"
evidence_status=$?
set -e
effective_status="$status"
if [[ "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then
if [[ "$evidence_status" == "0" && "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then
if node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "$report_json"
then
effective_status=0
@@ -509,16 +482,28 @@ jobs:
} >> "$GITHUB_STEP_SUMMARY"
fi
fi
echo "status=$status" >> "$GITHUB_OUTPUT"
echo "effective_status=$effective_status" >> "$GITHUB_OUTPUT"
echo "report_json=$report_json" >> "$GITHUB_OUTPUT"
echo "report_md=$report_md" >> "$GITHUB_OUTPUT"
set +e
kova report bundle "$report_json" --output-dir "$BUNDLE_DIR" --json | tee "$BUNDLE_DIR/bundle.json"
bundle_status=${PIPESTATUS[0]}
set -e
summary_path="$SUMMARY_DIR/${LANE_ID}.md"
summary_args=(node "$PERFORMANCE_HELPER_DIR/scripts/kova-ci-summary.mjs" --report "$report_json" --output "$summary_path" --lane "$LANE_ID")
set +e
"${summary_args[@]}"
summary_status=$?
set -e
if [[ "$summary_status" != "0" ]]; then
cat > "$summary_path" <<EOF
# OpenClaw Performance Report
- Lane: ${LANE_ID}
- Kova report: ${report_json}
- Summary generation failed with status ${summary_status}; inspect the Kova report and lane log.
EOF
fi
cat >> "$summary_path" <<EOF
## Test scope
@@ -539,6 +524,9 @@ jobs:
EOF
cat "$summary_path" >> "$GITHUB_STEP_SUMMARY"
if [[ "$evidence_status" != "0" || "$bundle_status" != "0" || "$summary_status" != "0" ]]; then
exit 1
fi
if [[ "$FAIL_ON_REGRESSION" == "true" && "$effective_status" != "0" ]]; then
exit "$effective_status"
fi
scripts/kova-ci-summary.mjs
+42 -28
View File
@@ -32,7 +32,6 @@ const keyMetricIds = [
];
const rssMetricIds = ["peakRssMb", "resourcePeakGatewayRssMb"];
const cpuMetricIds = ["cpuPercentMax"];
const reportPath = path.resolve(args.report);
const report = JSON.parse(await readFile(reportPath, "utf8"));
const invalidReport = validateKovaSummaryReport(report);
@@ -79,32 +78,39 @@ function renderSummary(reportLocal, options) {
const groups = Array.isArray(reportLocal.performance?.groups)
? reportLocal.performance.groups
: [];
if (groups.length > 0) {
const metricRows = [];
for (const group of groups) {
for (const metricId of keyMetricIds) {
const metric = group.metrics?.[metricId];
if (!hasPositiveSampleCount(metric)) {
continue;
}
metricRows.push(
[
value(group.scenario),
value(group.state),
value(metric.title || metricId),
formatMetric(metric.median, metric.unit),
formatMetric(metric.p95, metric.unit),
formatMetric(metric.max, metric.unit),
]
.join(" | ")
.replace(/^/, "| ")
.replace(/$/, " |"),
);
}
}
if (metricRows.length > 0) {
lines.push("## Key metrics");
lines.push("");
lines.push("| Scenario | State | Metric | Median | p95 | Max |");
lines.push("| --- | --- | --- | ---: | ---: | ---: |");
for (const group of groups) {
for (const metricId of keyMetricIds) {
const metric = group.metrics?.[metricId];
if (!hasPositiveSampleCount(metric)) {
continue;
}
lines.push(
[
value(group.scenario),
value(group.state),
value(metric.title || metricId),
formatMetric(metric.median, metric.unit),
formatMetric(metric.p95, metric.unit),
formatMetric(metric.max, metric.unit),
]
.join(" | ")
.replace(/^/, "| ")
.replace(/$/, " |"),
);
}
}
lines.push(...metricRows);
lines.push("");
} else if (groups.length > 0) {
lines.push("## Key metrics");
lines.push("");
lines.push("- No sampled key metrics were available; inspect the blocking records below.");
lines.push("");
}
@@ -180,7 +186,11 @@ function validateKovaSummaryReport(reportLocal) {
if (records.length === 0 && groups.length === 0) {
return "missing records or performance groups";
}
if (groups.length > 0 && !hasExplicitResourceCollectionSkip(reportLocal)) {
if (
groups.length > 0 &&
reportHasOnlyPassingStatuses(statuses) &&
!hasExplicitResourceCollectionSkip(reportLocal)
) {
if (!groups.some((group) => hasSampledMetric(group, rssMetricIds))) {
return "missing sampled RSS metric in performance groups";
}
@@ -191,16 +201,20 @@ function validateKovaSummaryReport(reportLocal) {
return null;
}
function reportHasOnlyPassingStatuses(statuses) {
const populated = Object.entries(statuses).filter(([, count]) => Number(count) > 0);
return (
populated.length > 0 && populated.every(([status]) => status.trim().toUpperCase() === "PASS")
);
}
function hasExplicitResourceCollectionSkip(reportLocal) {
const reason = reportLocal.performance?.resourceCollectionSkippedReason;
return typeof reason === "string" && reason.trim().length > 0;
}
function hasSampledMetric(group, metricIds) {
return metricIds.some((metricId) => {
const metric = group?.metrics?.[metricId];
return hasPositiveSampleCount(metric);
});
return metricIds.some((metricId) => hasPositiveSampleCount(group?.metrics?.[metricId]));
}
function hasPositiveSampleCount(metric) {
test/scripts/kova-ci-summary.test.ts
+73 -15
View File
@@ -128,7 +128,37 @@ describe("scripts/kova-ci-summary", () => {
expect(output).toContain("| gateway | clean | CPU max | 12 % | 12 % | 12 % |");
});
it("rejects performance summaries without resource metrics", () => {
it("renders blocked reports without resource metrics", () => {
const { output, result } = runSummary({
performance: {
repeat: 1,
groups: [
{
metrics: {
timeToHealthReadyMs: {
count: 1,
max: 30,
median: 20,
p95: 30,
title: "Health ready",
unit: "ms",
},
},
scenario: "gateway",
state: "clean",
},
],
},
records: [{ scenario: "gateway", state: "clean", status: "BLOCKED" }],
summary: { statuses: { BLOCKED: 1 } },
});
expect(result.status).toBe(0);
expect(output).toContain("| gateway | clean | Health ready | 20 ms | 30 ms | 30 ms |");
expect(output).toContain("| gateway | clean | BLOCKED |");
});
it("rejects successful reports without resource metrics", () => {
const { result } = runSummary({
performance: {
repeat: 1,
@@ -149,8 +179,8 @@ describe("scripts/kova-ci-summary", () => {
},
],
},
records: [{ scenario: "gateway", state: "clean", status: "pass" }],
summary: { statuses: { pass: 1 } },
records: [{ scenario: "gateway", state: "clean", status: "PASS" }],
summary: { statuses: { PASS: 1 } },
});
expect(result.status).toBe(1);
@@ -159,8 +189,8 @@ describe("scripts/kova-ci-summary", () => {
);
});
it("rejects performance summaries without CPU metrics", () => {
const { result } = runSummary({
it("renders available metrics from failed reports when CPU samples are absent", () => {
const { output, result } = runSummary({
performance: {
repeat: 1,
groups: [
@@ -188,8 +218,38 @@ describe("scripts/kova-ci-summary", () => {
},
],
},
records: [{ scenario: "gateway", state: "clean", status: "pass" }],
summary: { statuses: { pass: 1 } },
records: [{ scenario: "gateway", state: "clean", status: "FAIL" }],
summary: { statuses: { FAIL: 1 } },
});
expect(result.status).toBe(0);
expect(output).toContain("| gateway | clean | Gateway RSS | 256 MB | 256 MB | 256 MB |");
expect(output).toContain("| gateway | clean | FAIL |");
});
it("rejects successful reports without CPU metrics", () => {
const { result } = runSummary({
performance: {
repeat: 1,
groups: [
{
metrics: {
resourcePeakGatewayRssMb: {
count: 1,
max: 256,
median: 256,
p95: 256,
title: "Gateway RSS",
unit: "MB",
},
},
scenario: "gateway",
state: "clean",
},
],
},
records: [{ scenario: "gateway", state: "clean", status: "PASS" }],
summary: { statuses: { PASS: 1 } },
});
expect(result.status).toBe(1);
@@ -198,8 +258,8 @@ describe("scripts/kova-ci-summary", () => {
);
});
it("rejects malformed resource metric counts instead of treating them as sampled", () => {
const { result } = runSummary({
it("omits malformed resource metrics instead of rejecting failure evidence", () => {
const { output, result } = runSummary({
performance: {
repeat: 1,
groups: [
@@ -227,14 +287,12 @@ describe("scripts/kova-ci-summary", () => {
},
],
},
records: [{ scenario: "gateway", state: "clean", status: "pass" }],
summary: { statuses: { pass: 1 } },
records: [{ scenario: "gateway", state: "clean", status: "FAIL" }],
summary: { statuses: { FAIL: 1 } },
});
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"invalid Kova report: missing sampled RSS metric in performance groups",
);
expect(result.status).toBe(0);
expect(output).toContain("No sampled key metrics were available");
});
it("omits key metric rows with invalid sample counts", () => {
test/scripts/openclaw-performance-workflow.test.ts
+27 -18
View File
@@ -82,7 +82,7 @@ describe("OpenClaw performance workflow", () => {
it("pins the Kova evaluator with release validation contracts", () => {
const workflow = readFileSync(WORKFLOW, "utf8");
const kovaRef = "99b4b5c70fac2b13c48550c1d9bed09b795f0186";
const kovaRef = "ff755f6afcad9664a80e9ff29e15bbd29d9e1fa6";
const install = findStep("Install OCM and Kova");
const installRun = install.run ?? "";
@@ -666,25 +666,34 @@ esac
expect(sanity.run).not.toContain("--include scenario:fresh-install");
});
it("makes the live lane use live auth through the OpenClaw runtime", () => {
const override = findStep("Prepare live OpenAI candidate state");
it("uses Kova's explicit live auth contract without rewriting its state registry", () => {
const workflow = readWorkflow();
const stepNames = workflow.jobs?.kova?.steps?.map((step) => step.name) ?? [];
const runKova = findStep("Run Kova");
expect(override.if).toContain("matrix.live == 'true'");
expect(override.run).toContain("states/mock-openai-provider.json");
expect(override.run).toContain('state.auth?.mode !== "mock"');
expect(override.run).toContain('state.auth.mode = "default"');
expect(override.run).toContain(
"This ephemeral checkout must honor the lane's explicit --auth live selection.",
expect(stepNames).not.toContain("Prepare live OpenAI candidate state");
expect(runKova.run).toContain('--auth "$AUTH_MODE"');
expect(runKova.run).toContain('args+=(--model "$PERFORMANCE_MODEL_ID")');
expect(JSON.stringify(workflow)).not.toContain("states/mock-openai-provider.json");
});
it("finalizes Kova artifacts before failing evidence integrity", () => {
const run = findStep("Run Kova").run ?? "";
const evidence = run.indexOf("scripts/lib/kova-workflow-evidence.mjs");
const bundle = run.indexOf('kova report bundle "$report_json"');
const summary = run.indexOf("scripts/kova-ci-summary.mjs");
const integrityExit = run.indexOf(
'if [[ "$evidence_status" != "0" || "$bundle_status" != "0" || "$summary_status" != "0" ]]',
);
expect(override.run).toContain(
'state.auth.reason = "Honor the workflow lane\'s explicit run-level auth selection."',
);
expect(override.run).toContain('id: "force-openclaw-agent-runtime"');
expect(override.run).toContain('afterPhase: "provision"');
expect(override.run).toContain(
"ocm @{env} -- config set models.providers.openai.agentRuntime.id openclaw",
);
expect(override.run).not.toContain("agents.defaults.agentRuntime");
expect(evidence).toBeGreaterThan(-1);
expect(bundle).toBeGreaterThan(evidence);
expect(summary).toBeGreaterThan(bundle);
expect(integrityExit).toBeGreaterThan(summary);
expect(run).toContain("evidence_status=$?");
expect(run).toContain("bundle_status=${PIPESTATUS[0]}");
expect(run).toContain("summary_status=$?");
expect(run).toContain("Summary generation failed with status ${summary_status}");
});
it("runs the trusted lane evidence validator before tolerating gate failures", () => {