fix(ci): require an identical workflow harness for evidence reuse

Codex review round three: a prior green run may have executed with
different lane definitions; the resolver now compares the candidate
run head SHA's .github/workflows tree against the current workflow
ref before accepting its manifest.
This commit is contained in:
Peter Steinberger
2026-07-10 00:21:25 -07:00
parent e8a634f714
commit 494987fa6e
3 changed files with 88 additions and 8 deletions
@@ -303,6 +303,7 @@ jobs:
}')"
bash workflow/scripts/github/find-reusable-release-validation.sh \
--target-sha "$TARGET_SHA" \
--workflow-sha "$GITHUB_SHA" \
--release-profile "$RELEASE_PROFILE" \
--run-release-soak "$RUN_RELEASE_SOAK" \
--inputs-json "$inputs_json" \
@@ -9,6 +9,7 @@ set -euo pipefail
REPO="${GH_REPO:-}"
WORKFLOW_FILE="full-release-validation.yml"
TARGET_SHA=""
WORKFLOW_SHA=""
RELEASE_PROFILE=""
RUN_RELEASE_SOAK="false"
INPUTS_JSON=""
@@ -21,15 +22,17 @@ PREFLIGHT="${SCRIPT_DIR}/../release-preflight.mjs"
usage() {
cat >&2 <<'EOF'
Usage: find-reusable-release-validation.sh --target-sha <sha> --release-profile <beta|stable|full> \
--inputs-json <json> [--run-release-soak <true|false>] [--repo <owner/repo>] \
[--repo-dir <path>] [--workflow <file>] [--max-candidates <n>] [--github-output <file>]
Usage: find-reusable-release-validation.sh --target-sha <sha> --workflow-sha <sha> \
--release-profile <beta|stable|full> --inputs-json <json> \
[--run-release-soak <true|false>] [--repo <owner/repo>] [--repo-dir <path>] \
[--workflow <file>] [--max-candidates <n>] [--github-output <file>]
Scans recent successful Full Release Validation runs for a validation manifest
whose targetSha differs from --target-sha only by release metadata paths, whose
recorded lane-selection inputs match --inputs-json exactly, and whose recorded
child runs are still green. Writes reuse=true plus evidence_* outputs when
found; reuse=false otherwise.
recorded lane-selection inputs match --inputs-json exactly, whose harness
(.github/workflows tree at the run's head SHA) matches --workflow-sha, and
whose recorded child runs are still green. Writes reuse=true plus evidence_*
outputs when found; reuse=false otherwise.
EOF
}
@@ -39,6 +42,10 @@ while [[ $# -gt 0 ]]; do
TARGET_SHA="${2:-}"
shift 2
;;
--workflow-sha)
WORKFLOW_SHA="${2:-}"
shift 2
;;
--release-profile)
RELEASE_PROFILE="${2:-}"
shift 2
@@ -113,6 +120,10 @@ if [[ ! "$TARGET_SHA" =~ ^[0-9a-f]{40}$ ]]; then
echo "Expected --target-sha to be a full lowercase commit SHA; got: ${TARGET_SHA}" >&2
exit 2
fi
if [[ ! "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]]; then
echo "Expected --workflow-sha to be a full lowercase commit SHA; got: ${WORKFLOW_SHA}" >&2
exit 2
fi
if [[ -z "$REPO" ]]; then
echo "Expected --repo <owner/repo> or GH_REPO." >&2
exit 2
@@ -134,11 +145,23 @@ if ! (cd "$REPO_DIR" && node "$PREFLIGHT" --macos-versions-only >&2); then
no_reuse "target version metadata is inconsistent"
fi
# Evidence must come from an identical harness: prior runs may have executed
# with different lane definitions. Compare the .github/workflows tree of the
# candidate run's head SHA against the current workflow ref's tree.
workflows_tree_for() {
gh api "repos/${REPO}/contents/.github/workflows?ref=$1" \
--jq '[.[] | {path, sha}] | sort_by(.path) | tostring'
}
current_workflows_tree=""
if ! current_workflows_tree="$(workflows_tree_for "$WORKFLOW_SHA")" || [[ -z "$current_workflows_tree" ]]; then
no_reuse "could not resolve the current workflow tree"
fi
runs_json=""
if ! runs_json="$(
gh api -X GET "repos/${REPO}/actions/workflows/${WORKFLOW_FILE}/runs" \
-F status=success -F event=workflow_dispatch -F per_page="$MAX_CANDIDATES" \
--jq '[.workflow_runs[] | {id, html_url}]'
--jq '[.workflow_runs[] | {id, html_url, head_sha}]'
)"; then
no_reuse "could not list prior successful validation runs"
fi
@@ -154,6 +177,16 @@ trap 'rm -rf "$work_dir"' EXIT
for ((index = 0; index < run_count; index += 1)); do
run_id="$(jq -r ".[${index}].id" <<< "$runs_json")"
run_url="$(jq -r ".[${index}].html_url" <<< "$runs_json")"
run_head_sha="$(jq -r ".[${index}].head_sha // \"\"" <<< "$runs_json")"
if [[ "$run_head_sha" != "$WORKFLOW_SHA" ]]; then
candidate_workflows_tree=""
if ! candidate_workflows_tree="$(workflows_tree_for "$run_head_sha")" \
|| [[ -z "$candidate_workflows_tree" || "$candidate_workflows_tree" != "$current_workflows_tree" ]]; then
echo "[evidence-reuse] run ${run_id}: harness workflows differ from the current workflow ref; skipping" >&2
continue
fi
fi
artifact_id=""
if ! artifact_id="$(
@@ -98,11 +98,15 @@ fi
exec cat "\${fixture}.json"
`;
const WORKFLOW_SHA = "f".repeat(40);
const CANDIDATE_HEAD_SHA = "e".repeat(40);
interface FixtureOptions {
runId?: string;
manifest?: Record<string, unknown>;
compare?: { base: string; head: string; status: string; files: string[] } | undefined;
childRunStates?: Record<string, string>;
candidateWorkflowsTree?: { path: string; sha: string }[];
}
function setUpFixtures(options: FixtureOptions): { fixtures: string; binDir: string } {
@@ -121,9 +125,27 @@ function setUpFixtures(options: FixtureOptions): { fixtures: string; binDir: str
"repos_openclaw_openclaw_actions_workflows_full-release-validation.yml_runs.json",
),
JSON.stringify({
workflow_runs: [{ id: Number(runId), html_url: `https://example.test/runs/${runId}` }],
workflow_runs: [
{
id: Number(runId),
html_url: `https://example.test/runs/${runId}`,
head_sha: CANDIDATE_HEAD_SHA,
},
],
}),
);
const workflowsTree = [{ path: ".github/workflows/full-release-validation.yml", sha: "aa11" }];
writeFileSync(
join(fixtures, `repos_openclaw_openclaw_contents_.github_workflows_ref=${WORKFLOW_SHA}.json`),
JSON.stringify(workflowsTree),
);
writeFileSync(
join(
fixtures,
`repos_openclaw_openclaw_contents_.github_workflows_ref=${CANDIDATE_HEAD_SHA}.json`,
),
JSON.stringify(options.candidateWorkflowsTree ?? workflowsTree),
);
if (options.manifest) {
writeFileSync(
join(fixtures, `repos_openclaw_openclaw_actions_runs_${runId}_artifacts_per_page=100.json`),
@@ -201,6 +223,8 @@ function runResolver(args: {
SCRIPT_PATH,
"--target-sha",
args.targetSha,
"--workflow-sha",
WORKFLOW_SHA,
"--release-profile",
args.releaseProfile,
"--run-release-soak",
@@ -417,6 +441,28 @@ describe("scripts/github/find-reusable-release-validation.sh", () => {
expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" });
});
it("rejects evidence from a run with a different workflow harness", () => {
const { origin, priorSha } = createRepoPair();
const clone = cloneHead(origin);
const { fixtures, binDir } = setUpFixtures({
manifest: manifestFor(priorSha),
childRunStates: HEALTHY_CHILDREN,
candidateWorkflowsTree: [
{ path: ".github/workflows/full-release-validation.yml", sha: "bb22" },
],
});
const result = runResolver({
repoDir: clone,
targetSha: priorSha,
releaseProfile: "stable",
fixtures,
binDir,
});
expect(result.status).toBe(0);
expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" });
});
it("rejects targets whose version stamps are internally inconsistent", () => {
const { origin, priorSha } = createRepoPair({ plistBuildVersion: "2026061000" });
const clone = cloneHead(origin);