mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-20 09:47:53 +00:00
fix(ci): validate sticky importer restores (#111444)
This commit is contained in:
@@ -325,13 +325,19 @@ runs:
|
||||
|
||||
sticky_marker="$STICKY_ROOT/.openclaw-deps-fingerprint"
|
||||
sticky_fingerprint=""
|
||||
sticky_fingerprint_matches="false"
|
||||
sticky_snapshot_matches="false"
|
||||
if [ "$STICKY_DISK" = "true" ] && [ -f "$sticky_marker" ]; then
|
||||
sticky_fingerprint="$(<"$sticky_marker")"
|
||||
fi
|
||||
if [ "$STICKY_DISK" = "true" ] && [ -n "$sticky_fingerprint" ] &&
|
||||
[ "$sticky_fingerprint" = "${OPENCLAW_STICKY_DEPS_FINGERPRINT:?}" ]; then
|
||||
sticky_snapshot_matches="true"
|
||||
sticky_fingerprint_matches="true"
|
||||
if bash "$GITHUB_ACTION_PATH/sticky-importers.sh" restore "$STICKY_ROOT" "$GITHUB_WORKSPACE"; then
|
||||
sticky_snapshot_matches="true"
|
||||
else
|
||||
echo "::warning::Sticky dependency fingerprint matches, but restored importer contents are incomplete; reinstalling"
|
||||
fi
|
||||
fi
|
||||
if [ "$STICKY_DISK" = "true" ] && [ "$STICKY_WRITER" != "true" ] &&
|
||||
[ "$sticky_snapshot_matches" != "true" ]; then
|
||||
@@ -345,7 +351,7 @@ runs:
|
||||
mkdir -p "$ephemeral_store"
|
||||
export PNPM_CONFIG_STORE_DIR="$ephemeral_store"
|
||||
echo "PNPM_CONFIG_STORE_DIR=$ephemeral_store" >> "$GITHUB_ENV"
|
||||
echo "Sticky dependency snapshot is stale; using runner-local storage for this read-only run"
|
||||
echo "Sticky dependency snapshot is unusable; using runner-local storage for this read-only run"
|
||||
fi
|
||||
|
||||
install_args=(
|
||||
@@ -389,10 +395,11 @@ runs:
|
||||
export NODE_PATH="$PNPM_CONFIG_MODULES_DIR${NODE_PATH:+:$NODE_PATH}"
|
||||
fi
|
||||
if [ "$sticky_snapshot_matches" = "true" ]; then
|
||||
bash "$GITHUB_ACTION_PATH/sticky-importers.sh" restore "$STICKY_ROOT" "$GITHUB_WORKSPACE"
|
||||
echo "Sticky dependency snapshot matches the install fingerprint; skipping pnpm install"
|
||||
echo "Sticky dependency snapshot matches the install fingerprint and importer contents; skipping pnpm install"
|
||||
else
|
||||
if [ "$STICKY_DISK" = "true" ] && [ -n "$sticky_fingerprint" ]; then
|
||||
if [ "$sticky_fingerprint_matches" = "true" ]; then
|
||||
echo "Sticky dependency snapshot importer contents are incomplete; reinstalling"
|
||||
elif [ "$STICKY_DISK" = "true" ] && [ -n "$sticky_fingerprint" ]; then
|
||||
echo "Sticky dependency snapshot is stale (disk: $sticky_fingerprint, want: $OPENCLAW_STICKY_DEPS_FINGERPRINT); reinstalling"
|
||||
fi
|
||||
# A stale marker must not survive a mid-install failure: the commit
|
||||
@@ -435,7 +442,7 @@ runs:
|
||||
fi
|
||||
|
||||
if [ "$STICKY_DISK" = "true" ]; then
|
||||
# This step already establishes an exact fingerprint match or finishes
|
||||
# This step already establishes a content-validated snapshot or finishes
|
||||
# a frozen-lockfile install. pnpm 11's redundant pre-run check treats
|
||||
# our intentionally pruned plugin-local node_modules as stale and can
|
||||
# launch concurrent implicit installs when later CI steps fan out.
|
||||
|
||||
@@ -39,6 +39,7 @@ const INSTALL_INPUT_FILES = [
|
||||
"pnpmfile.cjs",
|
||||
".github/actions/setup-node-env/dependency-fingerprint.mjs",
|
||||
".github/actions/setup-node-env/sticky-importers.sh",
|
||||
".github/actions/setup-node-env/verify-importers.mjs",
|
||||
"scripts/postinstall-bundled-plugins.mjs",
|
||||
"scripts/lib/package-dist-imports.mjs",
|
||||
"scripts/preinstall-package-manager-warning.mjs",
|
||||
|
||||
@@ -5,7 +5,30 @@ mode="${1:?mode is required}"
|
||||
sticky_root="${2:?sticky root is required}"
|
||||
workspace="${3:?workspace is required}"
|
||||
archive="$sticky_root/importer-node-modules.tar"
|
||||
archive_checksum="$sticky_root/.openclaw-importer-archive.sha256"
|
||||
importer_manifest="$sticky_root/importer-node-modules.manifest"
|
||||
marker="$sticky_root/.openclaw-deps-fingerprint"
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
archive_sha256() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum -- "$1" | awk '{print $1}'
|
||||
else
|
||||
shasum -a 256 "$1" | awk '{print $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
clear_importers() {
|
||||
(
|
||||
cd "$workspace"
|
||||
find . \( -type d -o -type l \) -name node_modules -prune \
|
||||
! -path ./node_modules -exec rm -rf -- {} +
|
||||
)
|
||||
}
|
||||
|
||||
verify_importers() {
|
||||
node "$script_dir/verify-importers.mjs" "$workspace" "$1"
|
||||
}
|
||||
|
||||
case "$mode" in
|
||||
capture)
|
||||
@@ -13,26 +36,66 @@ case "$mode" in
|
||||
mkdir -p "$sticky_root"
|
||||
list_file="$(mktemp)"
|
||||
temp_archive="$archive.tmp.$$"
|
||||
temp_checksum="$archive_checksum.tmp.$$"
|
||||
temp_manifest="$importer_manifest.tmp.$$"
|
||||
temp_marker="$marker.tmp.$$"
|
||||
cleanup() {
|
||||
rm -f "$list_file" "$temp_archive"
|
||||
rm -f "$list_file" "$temp_archive" "$temp_checksum" "$temp_manifest" "$temp_marker"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
# Do not publish a fingerprint for an install whose importer resolution is
|
||||
# already falling through to a wrong hoisted version.
|
||||
rm -f "$marker" "$archive_checksum" "$importer_manifest"
|
||||
(
|
||||
cd "$workspace"
|
||||
find . -type d -name node_modules -prune ! -path ./node_modules -print0 >"$list_file"
|
||||
find . \( -type d -o -type l \) -name node_modules -prune \
|
||||
! -path ./node_modules -print0 >"$list_file"
|
||||
tar --create --file "$temp_archive" --null --files-from "$list_file"
|
||||
)
|
||||
tr '\0' '\n' <"$list_file" >"$temp_manifest"
|
||||
# Record the exact importer set and check its live resolution before a
|
||||
# writer can publish the marker.
|
||||
verify_importers "$temp_manifest"
|
||||
{
|
||||
archive_sha256 "$temp_archive"
|
||||
archive_sha256 "$temp_manifest"
|
||||
} >"$temp_checksum"
|
||||
mv "$temp_archive" "$archive"
|
||||
# The marker lands last: a fingerprint is only ever visible next to the
|
||||
# importer archive it describes, so consumers cannot restore a torn pair.
|
||||
printf '%s\n' "$fingerprint" > "$marker"
|
||||
mv "$temp_manifest" "$importer_manifest"
|
||||
mv "$temp_checksum" "$archive_checksum"
|
||||
# The marker lands last. Consumers also verify the archive bytes and every
|
||||
# registry-backed importer resolution before trusting this snapshot.
|
||||
printf '%s\n' "$fingerprint" >"$temp_marker"
|
||||
mv "$temp_marker" "$marker"
|
||||
;;
|
||||
restore)
|
||||
if [[ ! -f "$archive" ]]; then
|
||||
echo "::error::sticky importer node_modules archive is missing: $archive" >&2
|
||||
if [[ ! -f "$archive" || ! -f "$archive_checksum" || ! -f "$importer_manifest" ]]; then
|
||||
echo "sticky importer archive, manifest, or checksum is missing under $sticky_root" >&2
|
||||
exit 1
|
||||
fi
|
||||
expected_archive_checksum="$(sed -n '1p' "$archive_checksum" | tr -d '[:space:]')"
|
||||
expected_manifest_checksum="$(sed -n '2p' "$archive_checksum" | tr -d '[:space:]')"
|
||||
actual_archive_checksum="$(archive_sha256 "$archive")"
|
||||
actual_manifest_checksum="$(archive_sha256 "$importer_manifest")"
|
||||
if [[ ! "$expected_archive_checksum" =~ ^[a-f0-9]{64}$ ]] || \
|
||||
[[ ! "$expected_manifest_checksum" =~ ^[a-f0-9]{64}$ ]] || \
|
||||
[[ "$actual_archive_checksum" != "$expected_archive_checksum" ]] || \
|
||||
[[ "$actual_manifest_checksum" != "$expected_manifest_checksum" ]]; then
|
||||
echo "sticky importer archive or manifest checksum mismatch" >&2
|
||||
exit 1
|
||||
fi
|
||||
# A restored archive is authoritative for checkout-local importer links.
|
||||
# Clear first so entries absent from the archive cannot survive from a
|
||||
# reused workspace, and clear again when validation rejects the restore.
|
||||
clear_importers
|
||||
if ! tar --extract --file "$archive" --directory "$workspace"; then
|
||||
clear_importers
|
||||
exit 1
|
||||
fi
|
||||
if ! verify_importers "$importer_manifest"; then
|
||||
clear_importers
|
||||
exit 1
|
||||
fi
|
||||
tar --extract --file "$archive" --directory "$workspace"
|
||||
;;
|
||||
*)
|
||||
echo "unsupported sticky importer mode: $mode" >&2
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { readFileSync, realpathSync, statSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import YAML from "yaml";
|
||||
|
||||
const DEPENDENCY_FIELDS = [
|
||||
{ name: "dependencies", optional: false },
|
||||
{ name: "devDependencies", optional: false },
|
||||
{ name: "optionalDependencies", optional: true },
|
||||
];
|
||||
const MAX_REPORTED_MISMATCHES = 12;
|
||||
|
||||
function readImporters(workspace) {
|
||||
const lockfilePath = path.join(workspace, "pnpm-lock.yaml");
|
||||
const lockfile = YAML.parse(readFileSync(lockfilePath, "utf8"));
|
||||
if (!lockfile?.importers || typeof lockfile.importers !== "object") {
|
||||
throw new Error(`${lockfilePath} does not contain importers`);
|
||||
}
|
||||
return lockfile.importers;
|
||||
}
|
||||
|
||||
function registryResolution(dependencyName, resolution) {
|
||||
if (typeof resolution !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const locator = resolution.split("(", 1)[0];
|
||||
if (locator.includes(":")) {
|
||||
return undefined;
|
||||
}
|
||||
const versionSeparator = locator.startsWith("@")
|
||||
? locator.indexOf("@", locator.indexOf("/") + 1)
|
||||
: locator.indexOf("@");
|
||||
if (versionSeparator > 0) {
|
||||
return {
|
||||
packageName: locator.slice(0, versionSeparator),
|
||||
snapshotKey: resolution,
|
||||
version: locator.slice(versionSeparator + 1),
|
||||
};
|
||||
}
|
||||
return {
|
||||
packageName: dependencyName,
|
||||
snapshotKey: `${dependencyName}@${resolution}`,
|
||||
version: locator,
|
||||
};
|
||||
}
|
||||
|
||||
function packageNameParts(packageName) {
|
||||
const parts = packageName.split("/");
|
||||
const valid = packageName.startsWith("@")
|
||||
? parts.length === 2 && parts.every(Boolean)
|
||||
: parts.length === 1 && parts[0] !== "";
|
||||
if (!valid || parts.some((part) => part === "." || part === "..")) {
|
||||
throw new Error(`invalid dependency name from pnpm lockfile: ${packageName}`);
|
||||
}
|
||||
return parts;
|
||||
}
|
||||
|
||||
function isWithinWorkspace(workspace, candidate) {
|
||||
const relative = path.relative(workspace, candidate);
|
||||
return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== "..");
|
||||
}
|
||||
|
||||
function findInstalledManifest({ dependencyName, projectPath, workspace }) {
|
||||
const dependencyParts = packageNameParts(dependencyName);
|
||||
let current = projectPath;
|
||||
for (;;) {
|
||||
const candidate = path.join(current, "node_modules", ...dependencyParts, "package.json");
|
||||
try {
|
||||
if (statSync(candidate).isFile()) {
|
||||
return candidate;
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT" && error.code !== "ENOTDIR") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (current === workspace) {
|
||||
return undefined;
|
||||
}
|
||||
const parent = path.dirname(current);
|
||||
if (parent === current || !isWithinWorkspace(workspace, parent)) {
|
||||
return undefined;
|
||||
}
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
|
||||
function parseManifest(manifestPath) {
|
||||
try {
|
||||
return JSON.parse(readFileSync(manifestPath, "utf8"));
|
||||
} catch (error) {
|
||||
throw new Error(`could not read ${manifestPath}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function relativeDisplayPath(workspace, absolutePath) {
|
||||
const relative = path.relative(workspace, absolutePath);
|
||||
return relative || ".";
|
||||
}
|
||||
|
||||
function normalizeLocation(location) {
|
||||
return location.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function readHoistedResolutions(workspace) {
|
||||
const modulesMetadataPath = path.join(workspace, "node_modules", ".modules.yaml");
|
||||
const metadata = YAML.parse(readFileSync(modulesMetadataPath, "utf8"));
|
||||
if (!metadata?.hoistedLocations || typeof metadata.hoistedLocations !== "object") {
|
||||
throw new Error(`${modulesMetadataPath} does not contain hoistedLocations`);
|
||||
}
|
||||
const byLocation = new Map();
|
||||
const bySnapshotKey = new Map();
|
||||
for (const [snapshotKey, locations] of Object.entries(metadata.hoistedLocations)) {
|
||||
if (!Array.isArray(locations)) {
|
||||
throw new Error(`invalid hoistedLocations entry for ${snapshotKey}`);
|
||||
}
|
||||
for (const location of locations) {
|
||||
if (typeof location !== "string") {
|
||||
throw new Error(`invalid hoisted location for ${snapshotKey}`);
|
||||
}
|
||||
const normalized = normalizeLocation(location);
|
||||
const keys = byLocation.get(normalized) ?? new Set();
|
||||
keys.add(snapshotKey);
|
||||
byLocation.set(normalized, keys);
|
||||
const snapshotLocations = bySnapshotKey.get(snapshotKey) ?? new Set();
|
||||
snapshotLocations.add(normalized);
|
||||
bySnapshotKey.set(snapshotKey, snapshotLocations);
|
||||
}
|
||||
}
|
||||
return { byLocation, bySnapshotKey };
|
||||
}
|
||||
|
||||
function readCapturedImporters(workspace, manifestPath) {
|
||||
const importers = new Map();
|
||||
for (const entry of readFileSync(manifestPath, "utf8").split("\n")) {
|
||||
if (!entry) {
|
||||
continue;
|
||||
}
|
||||
const modulesPath = path.resolve(workspace, entry);
|
||||
if (
|
||||
!isWithinWorkspace(workspace, modulesPath) ||
|
||||
path.basename(modulesPath) !== "node_modules" ||
|
||||
modulesPath === path.join(workspace, "node_modules")
|
||||
) {
|
||||
throw new Error(`invalid importer manifest entry: ${entry}`);
|
||||
}
|
||||
const projectPath = path.dirname(modulesPath);
|
||||
const importerPath = relativeDisplayPath(workspace, projectPath);
|
||||
importers.set(importerPath, { modulesPath });
|
||||
}
|
||||
return importers;
|
||||
}
|
||||
|
||||
function isPrunedImporter(importerPath) {
|
||||
// postinstall-bundled-plugins.mjs deliberately removes every plugin source
|
||||
// node_modules tree; installed plugins own those dependencies separately.
|
||||
return importerPath.startsWith("extensions/");
|
||||
}
|
||||
|
||||
function verifyImporters(workspace, manifestPath) {
|
||||
const importers = readImporters(workspace);
|
||||
const hoistedResolutions = readHoistedResolutions(workspace);
|
||||
const capturedImporters = readCapturedImporters(workspace, manifestPath);
|
||||
const mismatches = [];
|
||||
let checked = 0;
|
||||
|
||||
for (const [importerPath, { modulesPath }] of capturedImporters) {
|
||||
if (!importers[importerPath] || typeof importers[importerPath] !== "object") {
|
||||
throw new Error(`importer manifest entry is absent from pnpm-lock.yaml: ${importerPath}`);
|
||||
}
|
||||
try {
|
||||
if (!statSync(modulesPath).isDirectory()) {
|
||||
mismatches.push(`${importerPath}: captured node_modules path is not a directory`);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT" && error.code !== "ENOTDIR") {
|
||||
throw error;
|
||||
}
|
||||
mismatches.push(`${importerPath}: captured node_modules directory is missing`);
|
||||
}
|
||||
}
|
||||
|
||||
// The lockfile, not the captured manifest, owns the validation universe. A
|
||||
// missing importer must still be checked against the version Node falls back to.
|
||||
for (const [importerPath, importer] of Object.entries(importers)) {
|
||||
if (isPrunedImporter(importerPath)) {
|
||||
continue;
|
||||
}
|
||||
const projectPath = path.resolve(workspace, importerPath);
|
||||
if (!isWithinWorkspace(workspace, projectPath)) {
|
||||
throw new Error(`pnpm lockfile contains an importer outside the workspace: ${importerPath}`);
|
||||
}
|
||||
for (const field of DEPENDENCY_FIELDS) {
|
||||
const dependencies = importer[field.name] ?? {};
|
||||
for (const [dependencyName, expected] of Object.entries(dependencies)) {
|
||||
// Workspace, file, and git locators do not map directly to installed
|
||||
// manifest versions; registry versions and aliases do.
|
||||
const expectedResolution = registryResolution(dependencyName, expected?.version);
|
||||
if (!expectedResolution) {
|
||||
continue;
|
||||
}
|
||||
const manifestPath = findInstalledManifest({ dependencyName, projectPath, workspace });
|
||||
const importerDisplay = relativeDisplayPath(workspace, projectPath);
|
||||
if (!manifestPath) {
|
||||
if (field.optional) {
|
||||
continue;
|
||||
}
|
||||
mismatches.push(
|
||||
`${importerDisplay}: ${dependencyName} ${expectedResolution.version} is not resolvable`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
checked += 1;
|
||||
const installedLocation = normalizeLocation(
|
||||
relativeDisplayPath(workspace, path.dirname(manifestPath)),
|
||||
);
|
||||
const expectedLocation = normalizeLocation(
|
||||
path.join(
|
||||
importerPath === "." ? "" : importerPath,
|
||||
"node_modules",
|
||||
...packageNameParts(dependencyName),
|
||||
),
|
||||
);
|
||||
const exactImporterSlot = hoistedResolutions.bySnapshotKey
|
||||
.get(expectedResolution.snapshotKey)
|
||||
?.has(expectedLocation);
|
||||
const installedSnapshotKeys = hoistedResolutions.byLocation.get(installedLocation);
|
||||
// Hoisted pnpm installs may intentionally share a different peer-context
|
||||
// variant from the root. Exact identity is required when pnpm metadata
|
||||
// says this importer owns the lockfile snapshot in its local slot.
|
||||
if (exactImporterSlot && !installedSnapshotKeys?.has(expectedResolution.snapshotKey)) {
|
||||
const actualKeys = installedSnapshotKeys
|
||||
? [...installedSnapshotKeys].toSorted().join(", ")
|
||||
: "<missing metadata>";
|
||||
mismatches.push(
|
||||
`${importerDisplay}: ${dependencyName} expected pnpm snapshot ${expectedResolution.snapshotKey}, resolved ${actualKeys} from ${installedLocation}`,
|
||||
);
|
||||
}
|
||||
const actual = parseManifest(manifestPath);
|
||||
if (
|
||||
actual.name !== expectedResolution.packageName ||
|
||||
actual.version !== expectedResolution.version
|
||||
) {
|
||||
const resolvedFrom = relativeDisplayPath(
|
||||
workspace,
|
||||
realpathSync(path.dirname(manifestPath)),
|
||||
);
|
||||
mismatches.push(
|
||||
`${importerDisplay}: ${dependencyName} expected ${expectedResolution.packageName}@${expectedResolution.version}, resolved ${actual.name ?? "<missing>"}@${actual.version ?? "<missing>"} from ${resolvedFrom}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (mismatches.length > 0) {
|
||||
const visible = mismatches.slice(0, MAX_REPORTED_MISMATCHES);
|
||||
const remainder = mismatches.length - visible.length;
|
||||
const suffix = remainder > 0 ? `\n... and ${remainder} more` : "";
|
||||
throw new Error(
|
||||
`sticky importer dependency validation failed (${mismatches.length} mismatch${mismatches.length === 1 ? "" : "es"}):\n${visible.join("\n")}${suffix}`,
|
||||
);
|
||||
}
|
||||
console.log(`Verified ${checked} registry-backed importer dependency resolutions`);
|
||||
}
|
||||
|
||||
const workspace = path.resolve(process.argv[2] ?? process.cwd());
|
||||
const manifestPath = path.resolve(process.argv[3] ?? "");
|
||||
try {
|
||||
if (!process.argv[3]) {
|
||||
throw new Error("importer manifest path is required");
|
||||
}
|
||||
verifyImporters(workspace, manifestPath);
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
}
|
||||
@@ -12,6 +12,7 @@ function bundledPluginFile(pluginId: string, relativePath: string, suffix = ""):
|
||||
const repositoryScriptEntries = [
|
||||
// setup-node-env invokes this helper from composite-action YAML.
|
||||
".github/actions/setup-node-env/dependency-fingerprint.mjs!",
|
||||
".github/actions/setup-node-env/verify-importers.mjs!",
|
||||
".github/actions/register-bind-mount-cleanup/main.cjs!",
|
||||
".github/actions/register-bind-mount-cleanup/post.cjs!",
|
||||
"apps/android/scripts/build-release-artifacts.ts!",
|
||||
|
||||
+1
-1
@@ -116,7 +116,7 @@ The slowest Node test families are split or balanced so each job stays small wit
|
||||
- The full Node matrix admits the consistently slow serial tooling, auto-reply command shards, and broad core-fast cache writer first. This keeps the 28-job cap while preventing critical-path work and the next run's transform seed from slipping into a later wave.
|
||||
- Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard.
|
||||
- Linux Node shard jobs persist Vitest's experimental filesystem module cache through the upstream Actions cache API, which Blacksmith transparently accelerates on its runners. Every CI shard is restore-only and unpacks the protected seed into its own runner-local root; the shard wrapper then gives concurrent Vitest processes separate live subdirectories. Only the non-cancelling daily or explicitly dispatched warmer saves a new immutable archive, so pull requests cannot publish transforms or mint per-PR cache families. A transform-input fingerprint clears incompatible lockfile, package, tsconfig, and Vitest-config generations. The protected writer scans and prunes its restored cache to 75% after it exceeds 2 GiB. Vitest hashes module id, source content, environment, and resolved transform config, so ordinary partial source changes keep unchanged entries warm while changed modules miss safely. Coarse restore prefixes bridge workflow runs; normal Actions cache LRU and inactivity eviction bound old immutable archives.
|
||||
- Trusted Linux Node jobs also bind the pnpm store and `node_modules` from one protected dependency disk per supported Node line. Package manifests, install settings, runner platform, and the exact Node patch stay out of the disk key; an exact runtime and install-input fingerprint decides whether a job reuses the tree or reinstalls and refreshes the same disk. Manifests are canonicalized before hashing. The audited direct root hooks retain only pnpm's install lifecycle scripts, so formatting and ordinary test/build script edits keep the warm dependency tree; unaudited lifecycle-hook drift fails closed until its source inputs join the fingerprint contract. Dependency, package-manager, hook-source, and lockfile changes always invalidate the snapshot. A pull request whose read-only snapshot has a different fingerprint detaches the workspace bind and installs into runner-local storage, avoiding slow writes to a clone it cannot publish. Sticky cold installs disable pnpm's inner fetch retries and make up to three bounded full-install attempts from the progressively warmed store; a timeout remains a failure. After an exact restore or frozen-lockfile install, setup disables pnpm's redundant pre-run dependency check: the repository intentionally prunes plugin-local `node_modules`, which pnpm otherwise treats as stale and repairs through unsafe concurrent implicit installs during shard fanout. Canonical main preflight is the sole writer and measures the store on every refresh, running `pnpm store prune` only after retired package versions push it above 8 GiB. Blacksmith snapshot publication is asynchronous even after a writer job completes, so the first run after a fresh key or fingerprint can remain cold; later exact-marker restores are the rollout proof. Required CI jobs and pull requests get disposable clones, so dependency changes do not create new disks, competing snapshots, or a cache lock that can cancel builds.
|
||||
- Trusted Linux Node jobs also bind the pnpm store and `node_modules` from one protected dependency disk per supported Node line. Package manifests, install settings, runner platform, and the exact Node patch stay out of the disk key; an exact runtime and install-input fingerprint decides whether a job reuses the tree or reinstalls and refreshes the same disk. Manifests are canonicalized before hashing. The audited direct root hooks retain only pnpm's install lifecycle scripts, so formatting and ordinary test/build script edits keep the warm dependency tree; unaudited lifecycle-hook drift fails closed until its source inputs join the fingerprint contract. Dependency, package-manager, hook-source, and lockfile changes always invalidate the snapshot. A matching fingerprint is necessary but not sufficient: setup also checks the importer archive and manifest checksums, then verifies registry-backed lockfile dependencies retained by postinstall against the package manifests Node resolves from their importers. Missing or stale importer content falls back to a fresh install instead of serving the root hoist. A pull request whose read-only snapshot is unusable detaches the workspace bind and installs into runner-local storage, avoiding slow writes to a clone it cannot publish. Sticky cold installs disable pnpm's inner fetch retries and make up to three bounded full-install attempts from the progressively warmed store; a timeout remains a failure. After a content-validated restore or frozen-lockfile install, setup disables pnpm's redundant pre-run dependency check: the repository intentionally prunes plugin-local `node_modules`, which pnpm otherwise treats as stale and repairs through unsafe concurrent implicit installs during shard fanout. Canonical main preflight is the sole writer and measures the store on every refresh, running `pnpm store prune` only after retired package versions push it above 8 GiB. Blacksmith snapshot publication is asynchronous even after a writer job completes, so the first run after a fresh key or fingerprint can remain cold; later content-validated exact-marker restores are the rollout proof. Required CI jobs and pull requests get disposable clones, so dependency changes do not create new disks, competing snapshots, or a cache lock that can cancel builds.
|
||||
- Node shard and build-artifact jobs also restore Node's portable on-disk compile cache through immutable Actions caches. Independent `test` and `build` namespaces prevent their writers from replacing each other's archives: the scheduled test warmer owns the protected test seed, while `build-artifacts` may publish at most one protected build archive per UTC day from trusted `main` pushes. PR and ordinary test jobs only read protected snapshots, so feature-branch bytecode never enters the shared seed and PR traffic creates no cache archives. This reuses V8 bytecode for Node-loaded orchestration, build tooling, and external dependencies across different checkout paths, including when only part of the source graph changes. Vitest child processes disable an inherited compile cache because coverage can be enabled inside dynamic configs and V8 coverage can lose source-position precision when scripts are deserialized from bytecode.
|
||||
- The build-artifact job also persists content-fingerprinted `build-all` step outputs. CI's self-built plugin SDK declarations hash the complete repository-owned TypeScript/JSON source graph, exclude installed and generated directories, and restore both flat declarations and package bridges after `tsdown` clears `dist`. Documentation, workflow, plugin, and other changes outside that graph can reuse the declaration snapshot; source changes rebuild it before the export gate runs.
|
||||
- Full declaration builds split `tsdown` into AI, workspace-package, and unified groups. Each group caches declarations only, then still rebuilds runtime JavaScript before restoring those declarations. Core or plugin changes therefore invalidate only the large unified graph, while workspace-package changes conservatively invalidate every dependent declaration group. Public full builds generally use an immutable Actions cache; coarse restore keys seed partial changes, per-group content fingerprints reject stale data, and GitHub's cache quota evicts old generations. The weekly Node 22 lane instead publishes a 14-day artifact after successful `main` runs and restores only artifacts whose immutable producer identity resolves to that workflow on `main`, avoiding quota churn without allowing PR code to write a shared cache. Private-QA declarations are never persisted in Actions caches because cache namespaces are not confidentiality boundaries.
|
||||
|
||||
@@ -720,6 +720,10 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([
|
||||
".github/actions/setup-node-env/sticky-importers.sh",
|
||||
["test/scripts/ci-workflow-guards.test.ts"],
|
||||
],
|
||||
[
|
||||
".github/actions/setup-node-env/verify-importers.mjs",
|
||||
["test/scripts/ci-workflow-guards.test.ts"],
|
||||
],
|
||||
[
|
||||
".github/actions/setup-pnpm-store-cache/action.yml",
|
||||
["test/scripts/package-acceptance-workflow.test.ts", "test/scripts/ci-workflow-guards.test.ts"],
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Ci Workflow Guards tests cover ci workflow guards script behavior.
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
@@ -7,7 +8,6 @@ import {
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
readlinkSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
@@ -2112,17 +2112,21 @@ describe("ci workflow guards", () => {
|
||||
expect(installStep.run).toContain(
|
||||
'[ "$sticky_fingerprint" = "${OPENCLAW_STICKY_DEPS_FINGERPRINT:?}" ]',
|
||||
);
|
||||
expect(installStep.run).toContain('sticky_fingerprint_matches="true"');
|
||||
expect(installStep.run).toContain(
|
||||
"Sticky dependency fingerprint matches, but restored importer contents are incomplete; reinstalling",
|
||||
);
|
||||
expect(installStep.run).toContain('[ "$STICKY_WRITER" != "true" ]');
|
||||
expect(installStep.run).toContain('sudo umount "$GITHUB_WORKSPACE/node_modules"');
|
||||
expect(installStep.run).toContain('ephemeral_store="${RUNNER_TEMP:?}/openclaw-pnpm-store"');
|
||||
expect(installStep.run).toContain(
|
||||
"Sticky dependency snapshot is stale; using runner-local storage for this read-only run",
|
||||
"Sticky dependency snapshot is unusable; using runner-local storage for this read-only run",
|
||||
);
|
||||
expect(installStep.run).toContain(
|
||||
'bash "$GITHUB_ACTION_PATH/sticky-importers.sh" restore "$STICKY_ROOT" "$GITHUB_WORKSPACE"',
|
||||
);
|
||||
expect(installStep.run).toContain(
|
||||
"Sticky dependency snapshot matches the install fingerprint; skipping pnpm install",
|
||||
"Sticky dependency snapshot matches the install fingerprint and importer contents; skipping pnpm install",
|
||||
);
|
||||
expect(installStep.run).toContain("timeout --signal=TERM --kill-after=15s 4m");
|
||||
expect(installStep.run).toContain('pnpm "${install_args[@]}" --config.fetch-retries=0');
|
||||
@@ -2141,7 +2145,7 @@ describe("ci workflow guards", () => {
|
||||
'bash "$GITHUB_ACTION_PATH/sticky-importers.sh" capture "$STICKY_ROOT" "$GITHUB_WORKSPACE" "$OPENCLAW_STICKY_DEPS_FINGERPRINT"',
|
||||
),
|
||||
);
|
||||
// The exact snapshot fingerprint or successful install already owns
|
||||
// The content-validated snapshot or successful install already owns
|
||||
// dependency validation. pnpm's redundant check sees intentionally pruned
|
||||
// plugin importers as stale, so it must not mutate during shard fanout.
|
||||
const disableImplicitInstall =
|
||||
@@ -2258,27 +2262,126 @@ describe("ci workflow guards", () => {
|
||||
try {
|
||||
const workspace = path.join(root, "workspace");
|
||||
const stickyRoot = path.join(root, "sticky");
|
||||
const importerRoot = path.join(workspace, "packages", "example");
|
||||
const rootModules = path.join(workspace, "node_modules");
|
||||
const importerModules = path.join(workspace, "packages", "example", "node_modules");
|
||||
const importerModules = path.join(importerRoot, "node_modules");
|
||||
const rootDependency = path.join(rootModules, "ipaddr.js");
|
||||
const rootOptionalDependency = path.join(rootModules, "optional-ipaddr");
|
||||
const importerDependency = path.join(importerModules, "ipaddr.js");
|
||||
const helper = path.resolve(".github/actions/setup-node-env/sticky-importers.sh");
|
||||
mkdirSync(path.join(rootModules, "shared"), { recursive: true });
|
||||
mkdirSync(importerModules, { recursive: true });
|
||||
const lockfile = [
|
||||
"lockfileVersion: '9.0'",
|
||||
"importers:",
|
||||
" packages/example:",
|
||||
" dependencies:",
|
||||
" ipaddr.js:",
|
||||
" specifier: 2.4.0",
|
||||
" version: 2.4.0",
|
||||
" aliased-ipaddr:",
|
||||
" specifier: npm:ipaddr.js@2.4.0",
|
||||
" version: ipaddr.js@2.4.0",
|
||||
" local-helper:",
|
||||
" specifier: file:../local-helper",
|
||||
" version: file:../local-helper",
|
||||
" optionalDependencies:",
|
||||
" optional-ipaddr:",
|
||||
" specifier: npm:ipaddr.js@2.4.0",
|
||||
" version: ipaddr.js@2.4.0",
|
||||
" unsupported-optional:",
|
||||
" specifier: 3.0.0",
|
||||
" version: 3.0.0",
|
||||
"",
|
||||
].join("\n");
|
||||
mkdirSync(workspace, { recursive: true });
|
||||
writeFileSync(path.join(workspace, "pnpm-lock.yaml"), lockfile, "utf8");
|
||||
mkdirSync(rootDependency, { recursive: true });
|
||||
mkdirSync(rootOptionalDependency, { recursive: true });
|
||||
mkdirSync(importerDependency, { recursive: true });
|
||||
writeFileSync(
|
||||
path.join(rootDependency, "package.json"),
|
||||
JSON.stringify({ name: "ipaddr.js", version: "1.9.1" }),
|
||||
"utf8",
|
||||
);
|
||||
writeFileSync(
|
||||
path.join(rootOptionalDependency, "package.json"),
|
||||
JSON.stringify({ name: "ipaddr.js", version: "1.9.1" }),
|
||||
"utf8",
|
||||
);
|
||||
writeFileSync(
|
||||
path.join(importerDependency, "package.json"),
|
||||
JSON.stringify({ name: "ipaddr.js", version: "2.4.0" }),
|
||||
"utf8",
|
||||
);
|
||||
for (const dependencyName of ["aliased-ipaddr", "optional-ipaddr"]) {
|
||||
const dependencyRoot = path.join(importerModules, dependencyName);
|
||||
mkdirSync(dependencyRoot, { recursive: true });
|
||||
writeFileSync(
|
||||
path.join(dependencyRoot, "package.json"),
|
||||
JSON.stringify({ name: "ipaddr.js", version: "2.4.0" }),
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
writeFileSync(
|
||||
path.join(rootModules, ".modules.yaml"),
|
||||
JSON.stringify({
|
||||
hoistedLocations: {
|
||||
"ipaddr.js@1.9.1": ["node_modules/ipaddr.js", "node_modules/optional-ipaddr"],
|
||||
"ipaddr.js@2.4.0": [
|
||||
"packages/example/node_modules/ipaddr.js",
|
||||
"packages/example/node_modules/aliased-ipaddr",
|
||||
"packages/example/node_modules/optional-ipaddr",
|
||||
],
|
||||
},
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
writeFileSync(path.join(rootModules, "root-sentinel"), "before", "utf8");
|
||||
symlinkSync("../../../node_modules/shared", path.join(importerModules, "shared"));
|
||||
|
||||
execFileSync("bash", [helper, "capture", stickyRoot, workspace, "fingerprint-a"]);
|
||||
rmSync(importerModules, { recursive: true });
|
||||
writeFileSync(path.join(rootModules, "root-sentinel"), "after", "utf8");
|
||||
execFileSync("bash", [helper, "restore", stickyRoot, workspace]);
|
||||
|
||||
expect(readlinkSync(path.join(importerModules, "shared"))).toBe(
|
||||
"../../../node_modules/shared",
|
||||
);
|
||||
expect(
|
||||
JSON.parse(readFileSync(path.join(importerDependency, "package.json"), "utf8")),
|
||||
).toMatchObject({ version: "2.4.0" });
|
||||
expect(readFileSync(path.join(rootModules, "root-sentinel"), "utf8")).toBe("after");
|
||||
expect(readFileSync(path.join(stickyRoot, ".openclaw-deps-fingerprint"), "utf8")).toBe(
|
||||
"fingerprint-a\n",
|
||||
);
|
||||
expect(() => execFileSync("bash", [helper, "capture", stickyRoot, workspace])).toThrow();
|
||||
|
||||
// Recreate the reported failure shape: a marker-matching archive can be
|
||||
// structurally valid yet omit the importer-local override, causing Node
|
||||
// to fall through to the stale root-hoisted version.
|
||||
rmSync(importerModules, { recursive: true });
|
||||
const archive = path.join(stickyRoot, "importer-node-modules.tar");
|
||||
execFileSync("tar", ["--create", "--file", archive, "--files-from", "/dev/null"]);
|
||||
const manifest = path.join(stickyRoot, "importer-node-modules.manifest");
|
||||
const archiveChecksum = createHash("sha256").update(readFileSync(archive)).digest("hex");
|
||||
const manifestChecksum = createHash("sha256").update(readFileSync(manifest)).digest("hex");
|
||||
writeFileSync(
|
||||
path.join(stickyRoot, ".openclaw-importer-archive.sha256"),
|
||||
`${archiveChecksum}\n${manifestChecksum}\n`,
|
||||
"utf8",
|
||||
);
|
||||
const failedRestore = spawnSync("bash", [helper, "restore", stickyRoot, workspace], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
expect(failedRestore.status).toBe(1);
|
||||
expect(failedRestore.stderr).toContain(
|
||||
"ipaddr.js expected ipaddr.js@2.4.0, resolved ipaddr.js@1.9.1",
|
||||
);
|
||||
expect(existsSync(importerModules)).toBe(false);
|
||||
|
||||
const failedCapture = spawnSync(
|
||||
"bash",
|
||||
[helper, "capture", stickyRoot, workspace, "fingerprint-b"],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
expect(failedCapture.status).toBe(1);
|
||||
expect(failedCapture.stderr).toContain(
|
||||
"ipaddr.js expected ipaddr.js@2.4.0, resolved ipaddr.js@1.9.1",
|
||||
);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
@@ -1972,6 +1972,10 @@ describe("scripts/test-projects changed-target routing", () => {
|
||||
".github/actions/setup-node-env/dependency-fingerprint.mjs",
|
||||
["test/scripts/ci-workflow-guards.test.ts"],
|
||||
],
|
||||
[
|
||||
".github/actions/setup-node-env/verify-importers.mjs",
|
||||
["test/scripts/ci-workflow-guards.test.ts"],
|
||||
],
|
||||
[
|
||||
".github/actions/setup-pnpm-store-cache/action.yml",
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user