perf(ci): gate prompt-snapshot regeneration on the generator's blast radius (#109083)

check-prompt-snapshots costs 200-310s per PR regenerating real prompt
stacks, but snapshots only change when the generator's import graph or
its fixtures do. The manifest now computes that reach (same pattern as
QA smoke gating: always-run surface regex incl. the codex extension
whose test API loads via a dynamic module id, plus import-graph walk
from the snapshot helper; deletions and null diffs fail safe to
running). Unaffected PR diffs skip the lane with an explicit log line;
pushes and dispatches always run.
This commit is contained in:
Peter Steinberger
2026-07-16 08:06:06 -07:00
committed by GitHub
parent 28323b4480
commit f3a2f9f8a7
4 changed files with 88 additions and 2 deletions
+18 -2
View File
@@ -104,6 +104,7 @@ jobs:
run_checks_fast: ${{ steps.manifest.outputs.run_checks_fast }}
historical_target: ${{ steps.manifest.outputs.historical_target }}
run_qa_smoke_ci: ${{ steps.manifest.outputs.run_qa_smoke_ci }}
run_prompt_snapshots: ${{ steps.manifest.outputs.run_prompt_snapshots }}
checks_fast_core_matrix: ${{ steps.manifest.outputs.checks_fast_core_matrix }}
run_plugin_contracts_shards: ${{ steps.manifest.outputs.run_plugin_contracts_shards }}
plugin_contracts_matrix: ${{ steps.manifest.outputs.plugin_contracts_matrix }}
@@ -554,6 +555,13 @@ jobs:
eventName !== "pull_request" ||
typeof changedNodeTestPlan.hasQaSmokeAffectingChange !== "function" ||
changedNodeTestPlan.hasQaSmokeAffectingChange(changedPaths);
// Prompt snapshots only change when the generator's import graph or
// its fixtures do; unaffected PR diffs skip the regeneration lane.
const changedScopeHasPromptSnapshotImpact =
changedPaths === null ||
eventName !== "pull_request" ||
typeof changedNodeTestPlan.hasPromptSnapshotAffectingChange !== "function" ||
changedNodeTestPlan.hasPromptSnapshotAffectingChange(changedPaths);
const runBuildArtifacts = runNodeFull && changedScopeHasBuildImpact;
const runQaSmokeCi =
runNodeFull &&
@@ -614,6 +622,7 @@ jobs:
historical_target: historicalTarget,
compatibility_target: compatibilityTarget,
run_qa_smoke_ci: runQaSmokeCi,
run_prompt_snapshots: runNodeFull && changedScopeHasPromptSnapshotImpact,
checks_fast_core_matrix: createMatrix(checksFastCoreTasks),
run_plugin_contracts_shards: runPluginContractShards,
plugin_contracts_matrix: createMatrix(
@@ -2021,6 +2030,7 @@ jobs:
- name: Run additional check shard
env:
ADDITIONAL_CHECK_GROUP: ${{ matrix.group }}
RUN_PROMPT_SNAPSHOTS: ${{ needs.preflight.outputs.run_prompt_snapshots }}
OPENCLAW_ADDITIONAL_BOUNDARY_SHARD: ${{ matrix.boundary_shard || '' }}
OPENCLAW_ADDITIONAL_BOUNDARY_CONCURRENCY: 4
OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY: 6
@@ -2051,8 +2061,14 @@ jobs:
prompt-snapshots)
# No presence fallback: the boundary runner previously invoked
# this unconditionally, and silent success would drop snapshot
# drift coverage.
run_check "prompt:snapshots:check" pnpm prompt:snapshots:check
# drift coverage. The manifest gates the lane on the generator's
# import graph and fixtures; diffs outside both cannot change
# generated snapshots.
if [ "$RUN_PROMPT_SNAPSHOTS" != "true" ]; then
echo "[skip] changed scope cannot affect generated prompt snapshots"
else
run_check "prompt:snapshots:check" pnpm prompt:snapshots:check
fi
;;
session-accessor-boundary)
if [ ! -f scripts/check-session-accessor-boundary.mjs ]; then
@@ -14,4 +14,9 @@ export function createChangedNodeTestShards(
export function hasBuildArtifactAffectingChange(changedPaths: string[]): boolean;
export function hasPromptSnapshotAffectingChange(
changedPaths: string[],
options?: { cwd?: string },
): boolean;
export function hasQaSmokeAffectingChange(changedPaths: string[]): boolean;
+36
View File
@@ -91,6 +91,42 @@ export function hasQaSmokeAffectingChange(changedPaths, options = {}) {
return hasImportGraphImpactOnTargets(sourcePaths, [QA_SMOKE_RUNTIME_ENTRY], cwd);
}
// Surfaces the prompt-snapshot check exercises outside its generator's
// relative import graph: the snapshot fixtures and generator scripts, the
// codex extension (its test API loads through a dynamic bundled-plugin module
// id the graph walk cannot see), and the gate's own orchestration — changes
// to the gate must not be able to skip the gated lane.
const PROMPT_SNAPSHOT_SURFACE_RE =
/^(?:test\/(?:helpers\/agents|fixtures\/agents\/prompt-snapshots)|extensions\/codex)\/|^scripts\/(?:generate-prompt-snapshots\.ts|prompt-snapshot-files\.[cm]?[jt]s)$|^scripts\/lib\/ci-changed-node-test-plan\.mjs$|^\.github\/(?:workflows\/ci\.yml$|actions\/)|^(?:package\.json|pnpm-lock\.yaml|npm-shrinkwrap\.json|pnpm-workspace\.yaml)$/u;
// The generator renders real prompt-layer stacks, so its runtime blast radius
// is the snapshot helper's import graph (auto-reply prompts, channel typing,
// plugin-sdk agent harness, codex catalog fixtures).
const PROMPT_SNAPSHOT_ENTRY = "test/helpers/agents/happy-path-prompt-snapshots.ts";
/**
* True when a changed path can influence generated prompt snapshots: it
* touches the snapshot surface directly, or the generator's import graph
* reaches it. Diffs outside both cannot change generator output, so the
* manifest may skip the check lane.
*/
export function hasPromptSnapshotAffectingChange(changedPaths, options = {}) {
const cwd = options.cwd ?? process.cwd();
if (changedPaths.some((changedPath) => PROMPT_SNAPSHOT_SURFACE_RE.test(changedPath))) {
return true;
}
const sourcePaths = changedPaths.filter(
(changedPath) => changedPath.startsWith("src/") && !isTestFileTarget(changedPath),
);
if (sourcePaths.length === 0) {
return false;
}
// Deleted sources cannot be graphed; fail safe to running the check.
if (sourcePaths.some((changedPath) => !existsSync(path.join(cwd, changedPath)))) {
return true;
}
return hasImportGraphImpactOnTargets(sourcePaths, [PROMPT_SNAPSHOT_ENTRY], cwd);
}
function createBoundaryShard() {
// Boundary tests scan the source tree (including test files) and build
// their own fixtures; they do not consume the built dist artifact. When the
@@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest";
import {
createChangedNodeTestShards,
hasBuildArtifactAffectingChange,
hasPromptSnapshotAffectingChange,
hasQaSmokeAffectingChange,
} from "../../scripts/lib/ci-changed-node-test-plan.mjs";
import { listGitTrackedFiles } from "../../src/test-utils/repo-files.js";
@@ -80,6 +81,34 @@ describe("CI changed Node test plan", () => {
expect(hasQaSmokeAffectingChange(["src/infra/definitely-deleted-module.ts"])).toBe(true);
});
it("classifies prompt-snapshot impact by surface and generator import graph", () => {
// Inside the generator's import graph -> regenerated output can change.
expect(hasPromptSnapshotAffectingChange(["src/auto-reply/reply/prompt-prelude.ts"])).toBe(true);
// The codex extension loads through a dynamic bundled-plugin module id the
// graph walk cannot see; it stays on the always-run surface.
expect(hasPromptSnapshotAffectingChange(["extensions/codex/src/index.ts"])).toBe(true);
expect(
hasPromptSnapshotAffectingChange([
"test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/README.md",
]),
).toBe(true);
expect(hasPromptSnapshotAffectingChange(["scripts/generate-prompt-snapshots.ts"])).toBe(true);
// The gate's own orchestration must not be able to skip the gated lane.
expect(hasPromptSnapshotAffectingChange([".github/workflows/ci.yml"])).toBe(true);
expect(hasPromptSnapshotAffectingChange(["scripts/lib/ci-changed-node-test-plan.mjs"])).toBe(
true,
);
// Outside the surface and the generator graph -> the lane may skip.
expect(hasPromptSnapshotAffectingChange(["ui/src/app.ts"])).toBe(false);
expect(hasPromptSnapshotAffectingChange(["extensions/discord/src/index.ts"])).toBe(false);
expect(hasPromptSnapshotAffectingChange(["docs/index.md"])).toBe(false);
expect(hasPromptSnapshotAffectingChange(["test/scripts/ci-node-test-plan.test.ts"])).toBe(
false,
);
// Deleted source files cannot be graphed; fail safe to running the check.
expect(hasPromptSnapshotAffectingChange(["src/infra/definitely-deleted-module.ts"])).toBe(true);
});
it("fails safe to the full plan for broad changes", () => {
expect(createChangedNodeTestShards(["package.json"])).toBeNull();
});