From 2ba272bda686dfbc2c497241ff3abbb6c491bf1a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 09:20:14 +0100 Subject: [PATCH] improve(ci): warm import-heavy Vitest graphs (#110557) * perf(ci): broaden Vitest transform cache seed * fix(ci): declare cache slot clone helper --- .github/workflows/vitest-cache-warm.yml | 27 +++++++++++-- scripts/ci-run-node-test-shard.d.mts | 2 + scripts/ci-run-node-test-shard.mjs | 42 ++++++++++++++++++++- test/scripts/ci-run-node-test-shard.test.ts | 19 ++++++++++ test/scripts/ci-workflow-guards.test.ts | 3 ++ 5 files changed, 88 insertions(+), 5 deletions(-) diff --git a/.github/workflows/vitest-cache-warm.yml b/.github/workflows/vitest-cache-warm.yml index b456d195d20..e96b2a65ed6 100644 --- a/.github/workflows/vitest-cache-warm.yml +++ b/.github/workflows/vitest-cache-warm.yml @@ -45,14 +45,33 @@ jobs: import { appendFileSync } from "node:fs"; import { createNodeTestShards } from "./scripts/lib/ci-node-test-plan.mjs"; - // The unit-fast graph is striped across jobs; warm the union of its - // configs as whole suites so the seed covers every stripe's imports. - const shards = createNodeTestShards().filter((candidate) => + // Warm whole configs for the striped unit-fast graph plus the import- + // bound graphs that remained cold in measured protected-cache readers. + const additionalShardNames = new Set([ + "agentic-agents-embedded", + "agentic-gateway-methods", + "auto-reply-reply-commands-3", + ]); + const allShards = createNodeTestShards(); + const coreShards = allShards.filter((candidate) => candidate.shardName.startsWith("core-unit-fast"), ); - if (shards.length === 0) { + if (coreShards.length === 0) { throw new Error("core-unit-fast cache seed shards are missing"); } + const additionalShards = allShards.filter((candidate) => + additionalShardNames.has(candidate.shardName), + ); + const foundAdditionalShardNames = new Set( + additionalShards.map((shard) => shard.shardName), + ); + const missingShardNames = [...additionalShardNames].filter( + (name) => !foundAdditionalShardNames.has(name), + ); + if (missingShardNames.length > 0) { + throw new Error(`cache seed shards are missing: ${missingShardNames.join(", ")}`); + } + const shards = [...coreShards, ...additionalShards]; const configs = [...new Set(shards.flatMap((shard) => shard.configs))]; appendFileSync( process.env.GITHUB_ENV, diff --git a/scripts/ci-run-node-test-shard.d.mts b/scripts/ci-run-node-test-shard.d.mts index 54e1dc390f9..524a6892162 100644 --- a/scripts/ci-run-node-test-shard.d.mts +++ b/scripts/ci-run-node-test-shard.d.mts @@ -35,6 +35,8 @@ export function buildChildEnv( export function pruneFsModuleCache(root: string, maxBytes?: number): FsModuleCachePruneResult; +export function clonePersistentCacheSlots(root: string | undefined, concurrency: number): number; + export function resolveShardChildCommand( args: string[], nodeExecPath?: string, diff --git a/scripts/ci-run-node-test-shard.mjs b/scripts/ci-run-node-test-shard.mjs index 42ec776b91d..d9ef2a504fa 100644 --- a/scripts/ci-run-node-test-shard.mjs +++ b/scripts/ci-run-node-test-shard.mjs @@ -2,7 +2,17 @@ // list of packed group plans. Extracted from .github/workflows/ci.yml so the // execution policy is unit-testable and plans can run concurrently. import { spawn } from "node:child_process"; -import { existsSync, mkdtempSync, readdirSync, statSync, unlinkSync, writeFileSync } from "node:fs"; +import { + constants, + cpSync, + existsSync, + mkdtempSync, + readdirSync, + rmSync, + statSync, + unlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { StringDecoder } from "node:string_decoder"; @@ -150,6 +160,30 @@ export function pruneFsModuleCache(root, maxBytes = FS_MODULE_CACHE_MAX_BYTES) { return { beforeBytes, afterBytes: totalBytes, removedFiles }; } +export function clonePersistentCacheSlots(root, concurrency) { + if (!root || concurrency <= 1) { + return 0; + } + const seed = join(root, "vitest-cache-0"); + if (!existsSync(seed)) { + return 0; + } + + let clonedSlots = 0; + for (let cacheSlot = 1; cacheSlot < concurrency; cacheSlot += 1) { + const destination = join(root, `vitest-cache-${cacheSlot}`); + rmSync(destination, { force: true, recursive: true }); + // Clone before workers start. Reflinks make the common Linux path cheap; + // unsupported filesystems transparently fall back to a regular copy. + cpSync(seed, destination, { + mode: constants.COPYFILE_FICLONE, + recursive: true, + }); + clonedSlots += 1; + } + return clonedSlots; +} + const MAX_PENDING_LINE_CHARS = 1_000_000; function relayChildStream(stream, label) { @@ -227,6 +261,12 @@ export async function runShardPlans(plans, options = {}) { const scratchDir = options.scratchDir ?? mkdtempSync(join(tmpdir(), "openclaw-node-shard-")); const persistentCacheRoot = baseEnv[FS_MODULE_CACHE_PATH_ENV_KEY]?.trim(); const nodeCompileCacheRoot = baseEnv[NODE_COMPILE_CACHE_PATH_ENV_KEY]?.trim(); + const clonedCacheSlots = clonePersistentCacheSlots(persistentCacheRoot, concurrency); + if (clonedCacheSlots > 0) { + process.stdout.write( + `[shard:cache] cloned restored Vitest seed into ${clonedCacheSlots} isolated lane(s)\n`, + ); + } let nextIndex = 0; let exitCode = 0; diff --git a/test/scripts/ci-run-node-test-shard.test.ts b/test/scripts/ci-run-node-test-shard.test.ts index 2b2bbd97950..e039c15900d 100644 --- a/test/scripts/ci-run-node-test-shard.test.ts +++ b/test/scripts/ci-run-node-test-shard.test.ts @@ -14,6 +14,7 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { buildChildEnv, + clonePersistentCacheSlots, pruneFsModuleCache, resolveShardChildCommand, resolveShardPlans, @@ -218,6 +219,24 @@ describe("scripts/ci-run-node-test-shard.mjs", () => { ]); }); + it("clones a restored persistent seed into every concurrent cache slot", () => { + const persistentRoot = makeScratchDir(); + const seed = path.join(persistentRoot, "vitest-cache-0"); + mkdirSync(seed, { recursive: true }); + writeFileSync(path.join(seed, "transform"), "cached", "utf8"); + const staleSlot = path.join(persistentRoot, "vitest-cache-1"); + mkdirSync(staleSlot, { recursive: true }); + writeFileSync(path.join(staleSlot, "stale"), "old", "utf8"); + + expect(clonePersistentCacheSlots(persistentRoot, 3)).toBe(2); + for (const cacheSlot of [1, 2]) { + expect( + readFileSync(path.join(persistentRoot, `vitest-cache-${cacheSlot}`, "transform"), "utf8"), + ).toBe("cached"); + } + expect(existsSync(path.join(staleSlot, "stale"))).toBe(false); + }); + it("prunes oldest transform entries while preserving Vitest metadata", () => { const persistentRoot = makeScratchDir(); const slot = path.join(persistentRoot, "vitest-cache-0"); diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index cf7ebc82329..bf27e4f1954 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -2527,6 +2527,9 @@ describe("ci workflow guards", () => { expect(checkoutStep.with).toBeUndefined(); expect(warmerSource).toContain('cron: "17 8 * * *"'); expect(warmerSource).toContain('candidate.shardName.startsWith("core-unit-fast")'); + expect(warmerSource).toContain('"agentic-agents-embedded"'); + expect(warmerSource).toContain('"agentic-gateway-methods"'); + expect(warmerSource).toContain('"auto-reply-reply-commands-3"'); expect(warmerSetup.with).toMatchObject({ "node-compile-cache-scope": "test", "save-actions-cache": "true",