diff --git a/.github/actions/setup-node-env/action.yml b/.github/actions/setup-node-env/action.yml index 1066e6835bb..31cd9ba8a14 100644 --- a/.github/actions/setup-node-env/action.yml +++ b/.github/actions/setup-node-env/action.yml @@ -62,6 +62,12 @@ inputs: snapshots. required: false default: "false" + build-all-cache-scope: + description: > + Namespace for restoring and saving build-all's content-addressed step cache. + Leave empty to disable; use only for declaration builds with public inputs. + required: false + default: "" runs: using: composite steps: @@ -363,6 +369,17 @@ runs: fi fi + - name: Restore and save build-all cache + if: inputs.build-all-cache-scope != '' + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: .artifacts/build-all-cache + # Exact keys deduplicate concurrent jobs. Coarse restore supplies the + # newest declaration groups; build-all rehashes every group's inputs. + key: ${{ github.repository }}-build-all-v1-${{ inputs.build-all-cache-scope }}-${{ runner.os }}-${{ runner.arch }}-node-${{ inputs.node-version }}-${{ hashFiles('package.json', 'pnpm-lock.yaml', 'npm-shrinkwrap.json', 'tsconfig*.json', 'tsdown*.config.ts', 'scripts/build-all.mjs', 'scripts/tsdown-build.mjs', 'scripts/lib/tsdown-*.mjs', 'scripts/lib/plugin-sdk-*', 'scripts/lib/bundled-plugin-*', 'scripts/lib/optional-bundled-clusters.mjs', 'src/**', 'packages/**', 'extensions/**') }} + restore-keys: | + ${{ github.repository }}-build-all-v1-${{ inputs.build-all-cache-scope }}-${{ runner.os }}-${{ runner.arch }}-node-${{ inputs.node-version }}- + - name: Save pnpm store cache if: ${{ inputs.install-deps == 'true' && inputs.use-actions-cache == 'true' && inputs.save-actions-cache == 'true' && runner.os != 'Windows' && steps.setup-pnpm.outputs.store-cache-hit != 'true' }} uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7d08bff711..b6e9b1cf744 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1744,6 +1744,7 @@ jobs: with: node-version: "22.22.3" install-bun: "false" + build-all-cache-scope: full - name: Configure Node test resources run: echo "OPENCLAW_VITEST_MAX_WORKERS=2" >> "$GITHUB_ENV" diff --git a/.github/workflows/macos-release.yml b/.github/workflows/macos-release.yml index f7884bd7dae..86c6bf08b38 100644 --- a/.github/workflows/macos-release.yml +++ b/.github/workflows/macos-release.yml @@ -53,6 +53,7 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} install-bun: "false" + build-all-cache-scope: full - name: Build run: pnpm build diff --git a/.github/workflows/node22-compat.yml b/.github/workflows/node22-compat.yml index 15c6ed08ad6..97d87910e61 100644 --- a/.github/workflows/node22-compat.yml +++ b/.github/workflows/node22-compat.yml @@ -22,6 +22,7 @@ jobs: with: node-version: "22.22.3" install-bun: "false" + build-all-cache-scope: full - name: Configure Node test resources run: echo "OPENCLAW_VITEST_MAX_WORKERS=2" >> "$GITHUB_ENV" diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 4985f04450e..a9d26161a1c 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -787,6 +787,7 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} install-bun: "true" + build-all-cache-scope: full - name: Build dist for repo E2E env: @@ -830,6 +831,7 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} install-bun: "true" + build-all-cache-scope: full - name: Build dist for special E2E if: | diff --git a/.github/workflows/openclaw-npm-release.yml b/.github/workflows/openclaw-npm-release.yml index 80472c048d6..a554c0c4645 100644 --- a/.github/workflows/openclaw-npm-release.yml +++ b/.github/workflows/openclaw-npm-release.yml @@ -133,6 +133,7 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} install-bun: "true" + build-all-cache-scope: full - name: Validate npm release request env: diff --git a/docs/ci.md b/docs/ci.md index 4b6d47fed1b..559beaefe8b 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -117,6 +117,7 @@ The slowest Node test families are split or balanced so each job stays small wit - Linux Node shard jobs persist Vitest's experimental filesystem module cache. Trusted Blacksmith jobs use PR-scoped writable overlays seeded from the protected snapshot only when their transform-input generations match; GitHub-hosted and fork jobs use an `actions/cache` fallback with coarse restore prefixes. The planner marks the broad `core-unit-fast` graph as the single writer without coupling cache ownership to matrix order, while every other job restores a private read-only clone. Concurrent Vitest workers retain separate live directories. A transform-input fingerprint clears incompatible lockfile, package, tsconfig, and Vitest-config generations inside stable sticky keys. Only the writer scans and prunes the cache to 75% after it exceeds 2 GiB. A non-cancelling daily or default-branch repository-dispatch warmer prevents rapid `main` pushes from starving the protected seed, and closed PR cache archives are deleted. - Node shard and build-artifact jobs also restore Node's portable on-disk compile cache. Independent `test` and `build` namespaces prevent their writers from replacing each other's snapshots: the scheduled test warmer owns the protected test seed, while `build-artifacts` publishes the protected build seed only from trusted `main` pushes. PR jobs read protected snapshots without publishing feature-branch bytecode; fallback archives remain PR-scoped. This reuses V8 bytecode for Node-loaded orchestration, build tooling, and external dependencies across different checkout paths, including when only part of the source graph changes. Vitest child processes disable an inherited compile cache because coverage can be enabled inside dynamic configs and V8 coverage can lose source-position precision when scripts are deserialized from bytecode. - The build-artifact job also persists content-fingerprinted `build-all` step outputs. CI's self-built plugin SDK declarations hash the complete repository-owned TypeScript/JSON source graph, exclude installed and generated directories, and restore both flat declarations and package bridges after `tsdown` clears `dist`. Documentation, workflow, plugin, and other changes outside that graph can reuse the declaration snapshot; source changes rebuild it before the export gate runs. +- Full declaration builds split `tsdown` into AI, workspace-package, and unified groups. Each group caches declarations only, then still rebuilds runtime JavaScript before restoring those declarations. Core or plugin changes therefore invalidate only the large unified graph, while workspace-package changes conservatively invalidate every dependent declaration group. Public full builds use an immutable Actions cache; coarse restore keys seed partial changes, per-group content fingerprints reject stale data, and GitHub's cache quota evicts old generations. Private-QA declarations are never persisted in Actions caches because cache namespaces are not confidentiality boundaries. - `check-additional-*` stripes the supplemental boundary guard list (`scripts/run-additional-boundary-checks.mjs`) into one prompt-heavy shard (`check-additional-boundaries-a`, which includes the Codex prompt snapshot drift check) and one combined shard for the remaining stripes (`check-additional-boundaries-bcd`), each running independent guards concurrently and printing per-check timings. Package-boundary compile/canary work stays together, and runtime topology architecture runs separately from the gateway watch coverage embedded in `build-artifacts`. - Gateway watch, channel tests, and the core support-boundary shard run concurrently inside `build-artifacts` after `dist/` and `dist-runtime/` are already built. diff --git a/scripts/build-all.d.mts b/scripts/build-all.d.mts index 749e0591944..1aa026fff40 100644 --- a/scripts/build-all.d.mts +++ b/scripts/build-all.d.mts @@ -4,6 +4,7 @@ export type BuildCacheEntry = | string | { path: string; + excludeDirectories?: string[]; extensions?: string[]; recursive?: boolean; }; @@ -20,6 +21,9 @@ export type BuildAllStep = { inputs: BuildCacheEntry[]; outputs: BuildCacheEntry[]; restore?: "always"; + runOnHit?: { + env?: NodeJS.ProcessEnv; + }; }; }; @@ -68,6 +72,7 @@ export function resolveBuildAllStep( windowsVerbatimArguments?: boolean; }; }; +export function resolveBuildAllStepOnCacheHit(step: BuildAllStep): BuildAllStep | null; export function resolveBuildAllStepCacheState( step: BuildAllStep, params?: { rootDir?: string; fs?: typeof fs; env?: NodeJS.ProcessEnv }, @@ -86,6 +91,16 @@ export function restoreBuildAllStepCacheOutputs( cacheState: BuildAllCacheState, params?: { rootDir?: string; fs?: typeof fs }, ): boolean; +export function finalizeBuildAllStepCache( + step: BuildAllStep, + cacheState: BuildAllCacheState, + params?: { + rootDir?: string; + fs?: typeof fs; + env?: NodeJS.ProcessEnv; + reusedCache?: boolean; + }, +): boolean; export function formatBuildAllDuration(durationMs: number): string; export function formatBuildAllTimingSummary( timings: Array<{ label: string; durationMs: number; status: string }>, diff --git a/scripts/build-all.mjs b/scripts/build-all.mjs index f2588f4b1fe..4f36a149ce2 100644 --- a/scripts/build-all.mjs +++ b/scripts/build-all.mjs @@ -9,11 +9,74 @@ import { performance } from "node:perf_hooks"; import { pathToFileURL } from "node:url"; import prettyMilliseconds from "pretty-ms"; import { pluginSdkEntrypoints } from "./lib/plugin-sdk-entries.mjs"; +import { + TSDOWN_PACKAGE_CONFIG_GROUP, + TSDOWN_UNIFIED_CONFIG_GROUP, +} from "./lib/tsdown-config-groups.mjs"; +import { + TSDOWN_PACKAGE_OUTPUT_ROOTS, + tsdownPackageOutputRoot, +} from "./lib/tsdown-output-roots.mjs"; import { resolvePnpmRunner } from "./pnpm-runner.mjs"; const nodeBin = process.execPath; const FULL_GIT_COMMIT_RE = /^[0-9a-f]{40}$/iu; const BUILD_CACHE_VERSION = 4; +const TSDOWN_DECLARATION_EXTENSIONS = [".d.ts", ".d.mts", ".d.cts"]; +const TSDOWN_SOURCE_EXTENSIONS = [ + ".cjs", + ".cts", + ".js", + ".json", + ".json5", + ".mjs", + ".mts", + ".ts", + ".tsx", + ".yaml", + ".yml", +]; +const TSDOWN_AI_OUTPUT_ROOT = tsdownPackageOutputRoot("ai"); +const TSDOWN_MAIN_PACKAGE_OUTPUT_ROOTS = TSDOWN_PACKAGE_OUTPUT_ROOTS.filter( + (root) => root !== TSDOWN_AI_OUTPUT_ROOT, +); +const TSDOWN_DECLARATION_TOOL_INPUTS = [ + "package.json", + "pnpm-lock.yaml", + "npm-shrinkwrap.json", + "tsconfig.json", + "scripts/tsdown-build.mjs", + "scripts/lib/bundled-plugin-build-entries.mjs", + "scripts/lib/bundled-plugin-paths.mjs", + "scripts/lib/optional-bundled-clusters.mjs", + "scripts/lib/plugin-sdk-entries.mjs", + "scripts/lib/plugin-sdk-entrypoints.json", + "scripts/lib/plugin-sdk-private-local-only-subpaths.json", + "scripts/lib/plugin-sdk-deprecated-public-subpaths.json", + "scripts/lib/plugin-sdk-deprecated-barrel-subpaths.json", + "scripts/lib/tsdown-config-groups.mjs", + "scripts/lib/tsdown-output-roots.mjs", +]; +const TSDOWN_PACKAGES_CACHE_INPUT = { + path: "packages", + extensions: TSDOWN_SOURCE_EXTENSIONS, + excludeDirectories: ["dist", "node_modules"], +}; +const TSDOWN_UNIFIED_CACHE_INPUTS = [ + { + path: "src", + extensions: TSDOWN_SOURCE_EXTENSIONS, + excludeDirectories: ["dist", "node_modules"], + }, + { + path: "extensions", + extensions: TSDOWN_SOURCE_EXTENSIONS, + excludeDirectories: ["dist", "node_modules"], + }, + TSDOWN_PACKAGES_CACHE_INPUT, +]; +const declarationCacheOutputs = (roots) => + roots.map((root) => ({ path: root, extensions: TSDOWN_DECLARATION_EXTENSIONS })); const PLUGIN_SDK_ENTRY_DTS_CACHE_ENV = [ "OPENCLAW_BUILD_PRIVATE_QA", "OPENCLAW_PLUGIN_SDK_CANONICAL_DTS", @@ -65,6 +128,68 @@ const PNPM_STEP_NODE_FALLBACKS = new Map([ export const BUILD_ALL_STEPS = [ { label: "plugins:assets:build", kind: "pnpm", pnpmArgs: ["plugins:assets:build"] }, { label: "tsdown", kind: "node", args: ["scripts/tsdown-build.mjs"] }, + { + label: "tsdown-ai", + kind: "node", + args: ["scripts/tsdown-build.mjs", "--config", "tsdown.ai.config.ts"], + cache: { + env: ["OPENCLAW_RUN_NODE_SKIP_DTS_BUILD"], + inputs: [ + ...TSDOWN_DECLARATION_TOOL_INPUTS, + "tsdown.ai.config.ts", + TSDOWN_PACKAGES_CACHE_INPUT, + ], + outputs: declarationCacheOutputs([TSDOWN_AI_OUTPUT_ROOT]), + restore: "always", + runOnHit: { + env: { OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "1" }, + }, + }, + }, + { + label: "tsdown-packages", + kind: "node", + args: [ + "scripts/tsdown-build.mjs", + "--config", + "tsdown.config.ts", + "--filter", + TSDOWN_PACKAGE_CONFIG_GROUP, + ], + cache: { + env: ["OPENCLAW_RUN_NODE_SKIP_DTS_BUILD"], + inputs: [...TSDOWN_DECLARATION_TOOL_INPUTS, "tsdown.config.ts", TSDOWN_PACKAGES_CACHE_INPUT], + outputs: declarationCacheOutputs(TSDOWN_MAIN_PACKAGE_OUTPUT_ROOTS), + restore: "always", + runOnHit: { + env: { OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "1" }, + }, + }, + }, + { + label: "tsdown-unified", + kind: "node", + args: [ + "scripts/tsdown-build.mjs", + "--config", + "tsdown.config.ts", + "--filter", + TSDOWN_UNIFIED_CONFIG_GROUP, + ], + cache: { + env: ["OPENCLAW_BUILD_PRIVATE_QA", "OPENCLAW_RUN_NODE_SKIP_DTS_BUILD"], + inputs: [ + ...TSDOWN_DECLARATION_TOOL_INPUTS, + "tsdown.config.ts", + ...TSDOWN_UNIFIED_CACHE_INPUTS, + ], + outputs: declarationCacheOutputs(["dist"]), + restore: "always", + runOnHit: { + env: { OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "1" }, + }, + }, + }, { label: "check-cli-bootstrap-imports", kind: "node", @@ -142,7 +267,24 @@ export const BUILD_ALL_STEPS = [ ]; export const BUILD_ALL_PROFILES = { - full: BUILD_ALL_STEPS.map((step) => step.label), + full: [ + "plugins:assets:build", + "tsdown-ai", + "tsdown-packages", + "tsdown-unified", + "check-cli-bootstrap-imports", + "plugins:assets:copy", + "runtime-postbuild", + "build-stamp", + "runtime-postbuild-stamp", + "write-plugin-sdk-entry-dts", + "check-plugin-sdk-exports", + "copy-hook-metadata", + "copy-export-html-templates", + "ui:build", + "write-build-info", + "write-cli-startup-metadata", + ], ciArtifacts: [ "plugins:assets:build", "tsdown", @@ -197,7 +339,7 @@ export const BUILD_ALL_PROFILES = { export const BUILD_ALL_PROFILE_STEP_ENV = { full: { - tsdown: { + "tsdown-unified": { OPENCLAW_PRESERVE_CLI_STARTUP_METADATA: "1", }, }, @@ -411,6 +553,16 @@ export function resolveBuildAllStep(step, params = {}) { }; } +export function resolveBuildAllStepOnCacheHit(step) { + if (!step.cache?.runOnHit) { + return null; + } + return { + ...step, + env: Object.assign({}, step.env, step.cache.runOnHit.env), + }; +} + function normalizeCacheEntry(entry) { if (typeof entry === "string") { return { path: entry }; @@ -638,6 +790,18 @@ export function restoreBuildAllStepCacheOutputs(cacheState, params = {}) { return true; } +export function finalizeBuildAllStepCache(step, cacheState, params = {}) { + if (params.reusedCache) { + return restoreBuildAllStepCacheOutputs(cacheState, params); + } + writeBuildAllStepCacheStamp( + step, + resolveBuildAllStepCacheStampState(step, cacheState, params), + params, + ); + return true; +} + export function formatBuildAllDuration(durationMs) { const clampedMs = Math.max(0, durationMs); const roundedMs = @@ -691,15 +855,22 @@ if (isMainModule()) { for (const step of resolveBuildAllSteps(args.profile)) { const startedAt = performance.now(); const cacheState = resolveBuildAllStepCacheState(step, { env: buildEnv }); + let stepToRun = step; + let reusedCache = false; if (process.env.OPENCLAW_BUILD_CACHE !== "0" && cacheState.fresh) { restoreBuildAllStepCacheOutputs(cacheState); - const durationMs = performance.now() - startedAt; - timings.push({ label: step.label, status: "cached", durationMs }); - console.error(`[build-all] ${step.label} (cached) ${formatBuildAllDuration(durationMs)}`); - continue; + const cacheHitStep = resolveBuildAllStepOnCacheHit(step); + if (!cacheHitStep) { + const durationMs = performance.now() - startedAt; + timings.push({ label: step.label, status: "cached", durationMs }); + console.error(`[build-all] ${step.label} (cached) ${formatBuildAllDuration(durationMs)}`); + continue; + } + reusedCache = true; + stepToRun = cacheHitStep; } - console.error(`[build-all] ${step.label}`); - const invocation = resolveBuildAllStep(step, { env: buildEnv }); + console.error(`[build-all] ${step.label}${reusedCache ? " (cached declarations)" : ""}`); + const invocation = resolveBuildAllStep(stepToRun, { env: buildEnv }); const result = spawnSync(invocation.command, invocation.args, invocation.options); const durationMs = performance.now() - startedAt; if (typeof result.status === "number") { @@ -711,11 +882,10 @@ if (isMainModule()) { exitCode = result.status; break; } - writeBuildAllStepCacheStamp( - step, - resolveBuildAllStepCacheStampState(step, cacheState, { env: buildEnv }), - ); - timings.push({ label: step.label, status: "ran", durationMs }); + // Runtime-only tsdown cleans its output roots. Cache hits restore + // declarations again after that pass so the full build stays complete. + finalizeBuildAllStepCache(step, cacheState, { env: buildEnv, reusedCache }); + timings.push({ label: step.label, status: reusedCache ? "reused" : "ran", durationMs }); console.error(`[build-all] ${step.label} done in ${formatBuildAllDuration(durationMs)}`); continue; } diff --git a/scripts/tsdown-build.d.mts b/scripts/tsdown-build.d.mts index 3da4ef4bb52..326aa2eabdd 100644 --- a/scripts/tsdown-build.d.mts +++ b/scripts/tsdown-build.d.mts @@ -2,9 +2,15 @@ /** * Removes build output roots while preserving explicitly protected artifacts. */ -export function cleanTsdownOutputRoots(params?: Record): void; +export function cleanTsdownOutputRoots(params?: { + cwd?: string; + env?: NodeJS.ProcessEnv; + fs?: typeof import("node:fs"); + roots?: string[]; +}): void; export function pruneStaleRootChunkFiles(params?: Record): void; export function listTsdownOutputRoots(): string[]; +export function resolveTsdownCleanOutputRoots(args?: string[]): string[]; export function pruneUntrackedGeneratedSourceDeclarations(params?: Record): number; export function pruneSourceCheckoutBundledPluginNodeModules(params?: Record): void; export function parseTsdownBuildArgs(argv: unknown): { diff --git a/scripts/tsdown-build.mjs b/scripts/tsdown-build.mjs index c9d5db79b5a..3ff5cdfb848 100644 --- a/scripts/tsdown-build.mjs +++ b/scripts/tsdown-build.mjs @@ -12,7 +12,10 @@ import { TSDOWN_PACKAGE_CONFIG_GROUP, TSDOWN_UNIFIED_CONFIG_GROUP, } from "./lib/tsdown-config-groups.mjs"; -import { TSDOWN_PACKAGE_OUTPUT_ROOTS } from "./lib/tsdown-output-roots.mjs"; +import { + TSDOWN_PACKAGE_OUTPUT_ROOTS, + tsdownPackageOutputRoot, +} from "./lib/tsdown-output-roots.mjs"; import { resolveWindowsTaskkillPath } from "./lib/windows-taskkill.mjs"; import { resolvePnpmRunner } from "./pnpm-runner.mjs"; import { @@ -87,7 +90,7 @@ export function cleanTsdownOutputRoots(params = {}) { const cwd = params.cwd ?? process.cwd(); const fsImpl = params.fs ?? fs; const env = params.env ?? process.env; - const roots = listTsdownOutputRoots(); + const roots = params.roots ?? listTsdownOutputRoots(); const protectedDeclarationPaths = env[RUN_NODE_SKIP_DTS_BUILD_ENV] === "1" ? listExistingDeclarationOutputPaths({ @@ -227,6 +230,52 @@ export function listTsdownOutputRoots() { return [...ROOT_TSDOWN_OUTPUT_ROOTS, ...TSDOWN_PACKAGE_OUTPUT_ROOTS]; } +function readForwardedOption(args, names) { + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + for (const name of names) { + if (arg === name) { + return args[index + 1]; + } + if (arg.startsWith(`${name}=`)) { + return arg.slice(name.length + 1); + } + } + } + return undefined; +} + +/** Limits cleanup to the output roots owned by an explicitly filtered build. */ +export function resolveTsdownCleanOutputRoots(args = []) { + const config = readForwardedOption(args, ["--config", "-c"]); + const filter = readForwardedOption(args, ["--filter", "-F"]); + const configPath = config ? path.resolve(config) : undefined; + const aiConfigPath = path.resolve("tsdown.ai.config.ts"); + const mainConfigPath = path.resolve("tsdown.config.ts"); + const aiRoot = tsdownPackageOutputRoot("ai"); + const packageRoots = TSDOWN_PACKAGE_OUTPUT_ROOTS.filter((root) => root !== aiRoot); + + if (configPath === aiConfigPath) { + return [aiRoot]; + } + if (configPath === mainConfigPath) { + if (filter === TSDOWN_PACKAGE_CONFIG_GROUP) { + return packageRoots; + } + if (filter === TSDOWN_UNIFIED_CONFIG_GROUP) { + return [...ROOT_TSDOWN_OUTPUT_ROOTS]; + } + return [...ROOT_TSDOWN_OUTPUT_ROOTS, ...packageRoots]; + } + if (!config && filter === TSDOWN_PACKAGE_CONFIG_GROUP) { + return [aiRoot, ...packageRoots]; + } + if (!config && filter === TSDOWN_UNIFIED_CONFIG_GROUP) { + return [aiRoot, ...ROOT_TSDOWN_OUTPUT_ROOTS]; + } + return listTsdownOutputRoots(); +} + export function pruneUntrackedGeneratedSourceDeclarations(params = {}) { const cwd = params.cwd ?? process.cwd(); const fsImpl = params.fs ?? fs; @@ -908,7 +957,7 @@ if (isMainModule()) { pruneSourceCheckoutBundledPluginNodeModules(); pruneUntrackedGeneratedSourceDeclarations(); pruneStaleRuntimeSymlinks(); - cleanTsdownOutputRoots(); + cleanTsdownOutputRoots({ roots: resolveTsdownCleanOutputRoots(args.forwardedArgs) }); const invocations = resolveTsdownBuildInvocations({ args: args.forwardedArgs }); let result; for (const [index, invocation] of invocations.entries()) { diff --git a/test/scripts/build-all.test.ts b/test/scripts/build-all.test.ts index a9044056724..a723123e67b 100644 --- a/test/scripts/build-all.test.ts +++ b/test/scripts/build-all.test.ts @@ -10,6 +10,7 @@ import { BUILD_ALL_PROFILE_STEP_ENV, BUILD_ALL_STEPS, buildAllUsage, + finalizeBuildAllStepCache, formatBuildAllDuration, formatBuildAllTimingSummary, parseBuildAllArgs, @@ -17,6 +18,7 @@ import { resolveBuildAllStepCacheState, resolveBuildAllStepCacheStampState, resolveBuildAllStep, + resolveBuildAllStepOnCacheHit, resolveBuildAllSteps, restoreBuildAllStepCacheOutputs, writeBuildAllStepCacheStamp, @@ -317,11 +319,56 @@ describe("resolveBuildAllSteps", () => { expect(result.stderr).not.toContain("at "); }); - it("keeps the full profile aligned with the declared steps", () => { - expect(resolveBuildAllSteps("full").map((step) => step.label)).toEqual( - BUILD_ALL_STEPS.map((step) => step.label), + it("uses declaration-cache groups only for the full build", () => { + expect(resolveBuildAllSteps("full").map((step) => step.label)).toEqual([ + "plugins:assets:build", + "tsdown-ai", + "tsdown-packages", + "tsdown-unified", + "check-cli-bootstrap-imports", + "plugins:assets:copy", + "runtime-postbuild", + "build-stamp", + "runtime-postbuild-stamp", + "write-plugin-sdk-entry-dts", + "check-plugin-sdk-exports", + "copy-hook-metadata", + "copy-export-html-templates", + "ui:build", + "write-build-info", + "write-cli-startup-metadata", + ]); + expect(BUILD_ALL_PROFILES.ciArtifacts).toContain("tsdown"); + expect(BUILD_ALL_PROFILES.ciArtifacts).not.toContain("tsdown-unified"); + }); + + it("rebuilds runtime JS while reusing fresh declaration groups", () => { + const ai = getBuildAllStep("tsdown-ai"); + const packages = getBuildAllStep("tsdown-packages"); + const unified = getBuildAllStep("tsdown-unified"); + + expect(ai.args).toEqual(["scripts/tsdown-build.mjs", "--config", "tsdown.ai.config.ts"]); + expect(packages.args).toEqual( + expect.arrayContaining(["--config", "tsdown.config.ts", "--filter", "openclaw-packages"]), ); - expect(BUILD_ALL_PROFILES.full).toEqual(BUILD_ALL_STEPS.map((step) => step.label)); + expect(unified.args).toEqual( + expect.arrayContaining(["--config", "tsdown.config.ts", "--filter", "openclaw-unified"]), + ); + for (const step of [ai, packages, unified]) { + expect(step.cache?.restore).toBe("always"); + expect(step.cache?.env).toContain("OPENCLAW_RUN_NODE_SKIP_DTS_BUILD"); + expect(step.cache?.runOnHit?.env).toEqual({ OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "1" }); + expect(resolveBuildAllStepOnCacheHit(step)?.env).toMatchObject({ + OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "1", + }); + expect(step.cache?.outputs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ extensions: [".d.ts", ".d.mts", ".d.cts"] }), + ]), + ); + } + expect(unified.cache?.env).toContain("OPENCLAW_BUILD_PRIVATE_QA"); + expect(resolveBuildAllStepOnCacheHit(getBuildAllStep("copy-export-html-templates"))).toBeNull(); }); it("uses a runtime artifact plus plugin SDK export profile for ci artifacts", () => { @@ -427,7 +474,15 @@ describe("resolveBuildAllSteps", () => { }); it("preserves startup metadata only for profiles that regenerate it", () => { - for (const profile of ["full", "ciArtifacts", "sourcePerformance", "cliStartup"]) { + const fullTsdown = resolveBuildAllSteps("full").find((step) => step.label === "tsdown-unified"); + if (!fullTsdown) { + throw new Error("Missing full tsdown-unified step"); + } + expect(resolveBuildAllStep(fullTsdown, { env: {} }).options.env).toMatchObject({ + OPENCLAW_PRESERVE_CLI_STARTUP_METADATA: "1", + }); + + for (const profile of ["ciArtifacts", "sourcePerformance", "cliStartup"]) { const tsdown = resolveBuildAllSteps(profile).find((step) => step.label === "tsdown"); if (!tsdown) { throw new Error(`Missing ${profile} tsdown step`); @@ -549,7 +604,8 @@ describe("resolveBuildAllSteps", () => { it("copies generated plugin assets before runtime postbuild snapshots static outputs", () => { for (const profile of ["full", "ciArtifacts", "qaRuntime", "sourcePerformance"]) { const labels = resolveBuildAllSteps(profile).map((step) => step.label); - expect(labels.indexOf("plugins:assets:copy")).toBeGreaterThan(labels.indexOf("tsdown")); + const lastTsdown = profile === "full" ? "tsdown-unified" : "tsdown"; + expect(labels.indexOf("plugins:assets:copy")).toBeGreaterThan(labels.indexOf(lastTsdown)); expect(labels.indexOf("runtime-postbuild")).toBeGreaterThan( labels.indexOf("plugins:assets:copy"), ); @@ -572,10 +628,11 @@ describe("resolveBuildAllSteps", () => { it("includes ui:build in the full and ciArtifacts profiles after runtime postbuild", () => { for (const profile of ["full", "ciArtifacts"]) { const labels = resolveBuildAllSteps(profile).map((step) => step.label); + const lastTsdown = profile === "full" ? "tsdown-unified" : "tsdown"; expect(labels).toContain("ui:build"); // Control UI bundling must run after tsdown clears dist so that // dist/control-ui survives `pnpm build` without a second command. - expect(labels.indexOf("ui:build")).toBeGreaterThan(labels.indexOf("tsdown")); + expect(labels.indexOf("ui:build")).toBeGreaterThan(labels.indexOf(lastTsdown)); expect(labels.indexOf("ui:build")).toBeGreaterThan(labels.indexOf("runtime-postbuild-stamp")); // ui:build must run before write-build-info so the build manifest can // see the final dist/control-ui assets. @@ -661,6 +718,61 @@ describe("build-all timing output", () => { }); describe("resolveBuildAllStepCacheState", () => { + it("invalidates only declaration groups that depend on the changed module", () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-tsdown-group-cache-")); + const ai = getBuildAllStep("tsdown-ai"); + const packages = getBuildAllStep("tsdown-packages"); + const unified = getBuildAllStep("tsdown-unified"); + const sourcePath = path.join(rootDir, "src/index.ts"); + const aiSourcePath = path.join(rootDir, "packages/ai/src/index.ts"); + const packageSourcePath = path.join(rootDir, "packages/net-policy/src/index.ts"); + const fixtures = [ + ["package.json", "{}"], + ["src/index.ts", "export const core = 1;"], + ["extensions/example/index.ts", "export const extension = 1;"], + ["packages/ai/src/index.ts", "export const ai = 1;"], + ["packages/net-policy/src/index.ts", "export const net = 1;"], + ["packages/ai/dist/index.d.ts", "export declare const ai = 1;"], + ["packages/net-policy/dist/index.d.ts", "export declare const net = 1;"], + ["dist/index.d.ts", "export declare const core = 1;"], + ] as const; + + try { + for (const [relativePath, contents] of fixtures) { + const filePath = path.join(rootDir, relativePath); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, contents); + } + for (const step of [ai, packages, unified]) { + const state = resolveBuildAllStepCacheState(step, { rootDir }); + writeBuildAllStepCacheStamp( + step, + resolveBuildAllStepCacheStampState(step, state, { rootDir }), + { rootDir }, + ); + } + + fs.writeFileSync(sourcePath, "export const core = 2;"); + expect(resolveBuildAllStepCacheState(ai, { rootDir }).fresh).toBe(true); + expect(resolveBuildAllStepCacheState(packages, { rootDir }).fresh).toBe(true); + expect(resolveBuildAllStepCacheState(unified, { rootDir }).fresh).toBe(false); + + fs.writeFileSync(sourcePath, "export const core = 1;"); + fs.writeFileSync(packageSourcePath, "export const net = 2;"); + expect(resolveBuildAllStepCacheState(ai, { rootDir }).fresh).toBe(false); + expect(resolveBuildAllStepCacheState(packages, { rootDir }).fresh).toBe(false); + expect(resolveBuildAllStepCacheState(unified, { rootDir }).fresh).toBe(false); + + fs.writeFileSync(packageSourcePath, "export const net = 1;"); + fs.writeFileSync(aiSourcePath, "export const ai = 2;"); + expect(resolveBuildAllStepCacheState(ai, { rootDir }).fresh).toBe(false); + expect(resolveBuildAllStepCacheState(packages, { rootDir }).fresh).toBe(false); + expect(resolveBuildAllStepCacheState(unified, { rootDir }).fresh).toBe(false); + } finally { + fs.rmSync(rootDir, { force: true, recursive: true }); + } + }); + it("marks cacheable steps fresh when the input signature matches", () => { withBuildCacheFixture(({ rootDir, step }) => { const cacheState = resolveBuildAllStepCacheState(step, { rootDir }); @@ -882,6 +994,27 @@ describe("resolveBuildAllStepCacheState", () => { }); }); + it("restores always-restore outputs after a cache-hit command cleans them", () => { + withBuildCacheFixture(({ rootDir, outputPath, step }) => { + const alwaysRestoreStep = { + ...step, + cache: { + ...step.cache, + restore: "always" as const, + }, + }; + const cacheState = resolveBuildAllStepCacheState(alwaysRestoreStep, { rootDir }); + writeBuildAllStepCacheStamp(alwaysRestoreStep, cacheState, { rootDir }); + const restorable = resolveBuildAllStepCacheState(alwaysRestoreStep, { rootDir }); + + fs.rmSync(outputPath); + expect( + finalizeBuildAllStepCache(alwaysRestoreStep, restorable, { rootDir, reusedCache: true }), + ).toBe(true); + expect(fs.readFileSync(outputPath, "utf8")).toBe("output"); + }); + }); + it("can cache only direct directory files for generated flat outputs", () => { withBuildCacheFixture(({ rootDir, step }) => { const nestedPath = path.join(rootDir, "dist/nested/output.d.ts"); diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 9c379b41966..ac4fc69e8cd 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -2008,6 +2008,44 @@ describe("ci workflow guards", () => { ); }); + it("persists content-validated public full-build declarations", () => { + const action = parse(readFileSync(".github/actions/setup-node-env/action.yml", "utf8")); + const installStep = action.runs.steps.find( + (step: WorkflowStep) => step.name === "Install dependencies", + ); + const cacheStep = action.runs.steps.find( + (step: WorkflowStep) => step.name === "Restore and save build-all cache", + ); + + expect(action.inputs["build-all-cache-scope"].default).toBe(""); + expect(cacheStep).toMatchObject({ + if: "inputs.build-all-cache-scope != ''", + uses: "actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae", + with: { path: ".artifacts/build-all-cache" }, + }); + expect(cacheStep.with.key).toContain("build-all-v1-${{ inputs.build-all-cache-scope }}"); + expect(cacheStep.with.key).toContain("${{ runner.os }}-${{ runner.arch }}"); + expect(cacheStep.with.key).toContain("scripts/lib/optional-bundled-clusters.mjs"); + expect(cacheStep.with.key).toContain("'src/**', 'packages/**', 'extensions/**'"); + expect(cacheStep.with["restore-keys"]).not.toContain("hashFiles"); + expect(action.runs.steps.indexOf(installStep)).toBeLessThan( + action.runs.steps.indexOf(cacheStep), + ); + + const privateQaWorkflows = [ + ".github/workflows/mantis-discord-smoke.yml", + ".github/workflows/mantis-discord-status-reactions.yml", + ".github/workflows/mantis-discord-thread-attachment.yml", + ".github/workflows/mantis-slack-desktop-smoke.yml", + ".github/workflows/mantis-telegram-live.yml", + ".github/workflows/qa-live-transports-convex.yml", + ]; + for (const workflowPath of privateQaWorkflows) { + const source = readFileSync(workflowPath, "utf8"); + expect(source, workflowPath).not.toContain("build-all-cache-scope:"); + } + }); + it("restores importer-local node_modules from sticky snapshots", () => { const root = mkdtempSync(path.join(tmpdir(), "openclaw-sticky-importers-")); try { diff --git a/test/scripts/tsdown-build.test.ts b/test/scripts/tsdown-build.test.ts index 37c50603d06..321583c14f3 100644 --- a/test/scripts/tsdown-build.test.ts +++ b/test/scripts/tsdown-build.test.ts @@ -16,6 +16,7 @@ import { pruneUntrackedGeneratedSourceDeclarations, resolveTsdownBuildInvocation, resolveTsdownBuildInvocations, + resolveTsdownCleanOutputRoots, runTsdownBuildInvocation, signalTsdownBuildProcessTree, } from "../../scripts/tsdown-build.mjs"; @@ -466,6 +467,36 @@ describe("resolveTsdownBuildInvocation", () => { }); }); + it("limits cleanup to the explicitly selected declaration group", () => { + expect(resolveTsdownCleanOutputRoots(["--config", "tsdown.ai.config.ts"])).toEqual([ + "packages/ai/dist", + ]); + expect( + resolveTsdownCleanOutputRoots([ + "--config", + "tsdown.config.ts", + "--filter", + "openclaw-packages", + ]), + ).toEqual(expect.arrayContaining(["packages/agent-core/dist", "packages/net-policy/dist"])); + expect( + resolveTsdownCleanOutputRoots(["--config=tsdown.config.ts", "--filter=openclaw-packages"]), + ).not.toContain("packages/ai/dist"); + expect(resolveTsdownCleanOutputRoots(["-c=tsdown.config.ts", "-F=openclaw-unified"])).toEqual([ + "dist", + "dist-runtime", + ]); + expect( + resolveTsdownCleanOutputRoots([ + "--config", + "configs/tsdown.config.ts", + "--filter", + "openclaw-packages", + ]), + ).toEqual(listTsdownOutputRoots()); + expect(resolveTsdownCleanOutputRoots(["--format", "esm"])).toEqual(listTsdownOutputRoots()); + }); + it("keeps source-checkout prune best-effort", () => { const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); const rmSync = vi.spyOn(fs, "rmSync"); @@ -569,6 +600,21 @@ describe("resolveTsdownBuildInvocation", () => { await expect(fsPromises.readFile(unrelatedFile, "utf8")).resolves.toBe("keep\n"); }); + it("cleans only selected tsdown output roots", async () => { + const rootDir = createTempDir("openclaw-tsdown-selected-clean-"); + const aiFile = path.join(rootDir, "packages", "ai", "dist", "stale.js"); + const coreFile = path.join(rootDir, "dist", "keep.js"); + await fsPromises.mkdir(path.dirname(aiFile), { recursive: true }); + await fsPromises.mkdir(path.dirname(coreFile), { recursive: true }); + await fsPromises.writeFile(aiFile, "stale\n"); + await fsPromises.writeFile(coreFile, "keep\n"); + + cleanTsdownOutputRoots({ cwd: rootDir, roots: ["packages/ai/dist"] }); + + await expectPathMissing(aiFile); + await expect(fsPromises.readFile(coreFile, "utf8")).resolves.toBe("keep\n"); + }); + it("removes CLI startup metadata during default tsdown clean", async () => { const rootDir = createTempDir("openclaw-tsdown-clean-metadata-default-"); const metadataFile = path.join(rootDir, "dist", "cli-startup-metadata.json");