From 87fbc19fe373bed4bcf1e8a0ca2c5ea4eb0b6611 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 19 Jul 2026 08:58:43 -0700 Subject: [PATCH] fix: complete lint toolchain reuse in worktrees (#111467) * fix(scripts): complete worktree lint toolchain * fix(scripts): prepare complete lint pipeline --- package.json | 2 +- scripts/lib/local-heavy-check-runtime.d.mts | 11 +++++ scripts/lib/local-heavy-check-runtime.mjs | 40 ++++++++++++++++ ...extension-package-boundary-artifacts.d.mts | 6 +++ ...e-extension-package-boundary-artifacts.mjs | 25 +++++++++- scripts/run-lint.mjs | 39 ++++++++++++++++ scripts/run-oxlint-shards.mjs | 7 ++- scripts/run-oxlint.mjs | 12 ++--- scripts/run-tsgo.mjs | 2 + scripts/test-projects.test-support.mjs | 1 + .../scripts/local-heavy-check-runtime.test.ts | 46 +++++++++++++++++++ ...tension-package-boundary-artifacts.test.ts | 28 +++++++++++ test/scripts/run-oxlint.test.ts | 20 ++++++-- 13 files changed, 226 insertions(+), 13 deletions(-) create mode 100644 scripts/run-lint.mjs diff --git a/package.json b/package.json index 41f0d0ad6e4..400d00d6f8b 100644 --- a/package.json +++ b/package.json @@ -1695,7 +1695,7 @@ "ios:version:check": "node --import tsx scripts/ios-sync-versioning.ts --check", "ios:version:sync": "node --import tsx scripts/ios-sync-versioning.ts --write", "leak:embedded-run": "node --import tsx --expose-gc scripts/embedded-run-abort-leak.ts", - "lint": "pnpm lint:ui:i18n && node scripts/run-oxlint-shards.mjs", + "lint": "node scripts/run-lint.mjs", "lint:agent:ingress-owner": "node scripts/check-ingress-agent-owner-context.mjs", "lint:all": "node scripts/run-oxlint.mjs", "lint:apps": "pnpm lint:swift", diff --git a/scripts/lib/local-heavy-check-runtime.d.mts b/scripts/lib/local-heavy-check-runtime.d.mts index 6bee302ec8b..5fa0cee1949 100644 --- a/scripts/lib/local-heavy-check-runtime.d.mts +++ b/scripts/lib/local-heavy-check-runtime.d.mts @@ -15,6 +15,17 @@ export function resolveRepoToolBinPath( resolveCommonDir?: (cwd: string) => string | null; }, ): string; +/** Link a dependency-less worktree to the primary checkout toolchain selected above. */ +export function ensureRepoToolNodeModulesLink( + toolPath: string, + options?: { + cwd?: string; + fileExists?: (candidate: string) => boolean; + resolveCommonDir?: (cwd: string) => string | null; + symlink?: (target: string, path: string, type?: string) => void; + platform?: NodeJS.Platform; + }, +): string | null; /** Apply local tsgo defaults for declaration skipping, caching, throttling, and profiling. */ export function applyLocalTsgoPolicy( args: string[], diff --git a/scripts/lib/local-heavy-check-runtime.mjs b/scripts/lib/local-heavy-check-runtime.mjs index 1be77628295..fea66f54752 100644 --- a/scripts/lib/local-heavy-check-runtime.mjs +++ b/scripts/lib/local-heavy-check-runtime.mjs @@ -59,6 +59,46 @@ export function resolveRepoToolBinPath( return fileExists(primaryPath) ? primaryPath : localPath; } +/** Link a dependency-less worktree to the primary checkout toolchain selected above. */ +export function ensureRepoToolNodeModulesLink( + toolPath, + { + cwd = process.cwd(), + fileExists = fs.existsSync, + resolveCommonDir = resolveGitCommonDir, + symlink = fs.symlinkSync, + platform = process.platform, + } = {}, +) { + const localNodeModules = path.resolve(cwd, "node_modules"); + if (fileExists(localNodeModules)) { + return localNodeModules; + } + + const commonDir = resolveCommonDir(cwd); + if (!commonDir || path.basename(commonDir) !== ".git") { + return null; + } + + const primaryNodeModules = path.join(path.dirname(commonDir), "node_modules"); + const toolNodeModules = path.dirname(path.dirname(path.resolve(toolPath))); + if (toolNodeModules !== path.resolve(primaryNodeModules) || !fileExists(primaryNodeModules)) { + return null; + } + + try { + // Match run-vitest.mjs's hydrated-toolchain behavior: keep one stable link + // so compilers can resolve imports from worktree source paths. + symlink(primaryNodeModules, localNodeModules, platform === "win32" ? "junction" : "dir"); + } catch (error) { + // Another local runner may have installed the same stable link concurrently. + if (!fileExists(localNodeModules)) { + throw error; + } + } + return localNodeModules; +} + function hasFlag(args, name) { return args.some((arg) => arg === name || arg.startsWith(`${name}=`)); } diff --git a/scripts/prepare-extension-package-boundary-artifacts.d.mts b/scripts/prepare-extension-package-boundary-artifacts.d.mts index f64cc9ce7bb..acac304a152 100644 --- a/scripts/prepare-extension-package-boundary-artifacts.d.mts +++ b/scripts/prepare-extension-package-boundary-artifacts.d.mts @@ -1,3 +1,9 @@ +/** Resolve tsx's loader through the selected checkout toolchain. */ +export function resolveTsxImportSpecifier(options?: { + resolveTool?: (toolName: string) => string; + createRequireFrom?: (filename: string) => { resolve(packageName: string): string }; + ensureToolchain?: (toolPath: string) => string | null; +}): string; /** * Lists entry-shim artifacts written by scripts/write-plugin-sdk-entry-dts.ts. */ diff --git a/scripts/prepare-extension-package-boundary-artifacts.mjs b/scripts/prepare-extension-package-boundary-artifacts.mjs index 4d8ef3a3a61..de92e44d455 100644 --- a/scripts/prepare-extension-package-boundary-artifacts.mjs +++ b/scripts/prepare-extension-package-boundary-artifacts.mjs @@ -2,8 +2,14 @@ // boundary imports resolve through public package surfaces. import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; +import { createRequire } from "node:module"; import path, { resolve } from "node:path"; -import { isLocalCheckEnabled } from "./lib/local-heavy-check-runtime.mjs"; +import { pathToFileURL } from "node:url"; +import { + ensureRepoToolNodeModulesLink, + isLocalCheckEnabled, + resolveRepoToolBinPath, +} from "./lib/local-heavy-check-runtime.mjs"; import { parsePositiveInt } from "./lib/numeric-options.mjs"; import { pluginSdkEntrypoints, publicPluginSdkEntrypoints } from "./lib/plugin-sdk-entries.mjs"; import { resolveWindowsTaskkillPath } from "./lib/windows-taskkill.mjs"; @@ -31,6 +37,17 @@ let exitingAfterParentSignal = false; let parentSignalExitCode = 1; let parentSignalExitTimer; +/** Resolve tsx's loader through the selected checkout toolchain. */ +export function resolveTsxImportSpecifier({ + resolveTool = resolveRepoToolBinPath, + createRequireFrom = createRequire, + ensureToolchain = ensureRepoToolNodeModulesLink, +} = {}) { + const tsxBinPath = resolveTool("tsx"); + ensureToolchain(tsxBinPath); + return pathToFileURL(createRequireFrom(tsxBinPath).resolve("tsx")).href; +} + function listPackageDtsOutputsFromExports({ packageDir, outputPrefix }) { const packageJson = JSON.parse( fs.readFileSync(path.join(repoRoot, "packages", packageDir, "package.json"), "utf8"), @@ -1016,7 +1033,11 @@ async function main(argv = process.argv.slice(2)) { if (mode === "all" && (!entryShimsFresh || prerequisiteSteps.length > 0)) { await runNodeStep( "plugin-sdk boundary root shims", - ["--import", "tsx", resolve(repoRoot, "scripts/write-plugin-sdk-entry-dts.ts")], + [ + "--import", + resolveTsxImportSpecifier(), + resolve(repoRoot, "scripts/write-plugin-sdk-entry-dts.ts"), + ], ROOT_SHIMS_TIMEOUT_MS, { env: { NODE_OPTIONS: ROOT_SHIMS_NODE_OPTIONS } }, ); diff --git a/scripts/run-lint.mjs b/scripts/run-lint.mjs new file mode 100644 index 00000000000..f6dc242bad1 --- /dev/null +++ b/scripts/run-lint.mjs @@ -0,0 +1,39 @@ +// Runs the complete lint pipeline after preparing a linked-worktree toolchain. +import { spawnSync } from "node:child_process"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + ensureRepoToolNodeModulesLink, + resolveRepoToolBinPath, +} from "./lib/local-heavy-check-runtime.mjs"; + +function run(command, args, options) { + const result = spawnSync(command, args, options); + if (result.error) { + throw result.error; + } + return result.status ?? 1; +} + +const oxlintPath = resolveRepoToolBinPath("oxlint"); +const tsxPath = resolveRepoToolBinPath("tsx"); +ensureRepoToolNodeModulesLink(oxlintPath); +const tsxImportSpecifier = pathToFileURL(createRequire(tsxPath).resolve("tsx")).href; + +// Invoke the pre-step directly: running pnpm through a linked node_modules can +// reconcile the owning checkout's dependency tree instead of merely running it. +const uiI18nStatus = run( + process.execPath, + ["--import", tsxImportSpecifier, path.resolve("scripts", "control-ui-i18n-verify.ts"), "verify"], + { env: process.env, stdio: "inherit" }, +); +if (uiI18nStatus !== 0) { + process.exitCode = uiI18nStatus; +} else { + process.exitCode = run( + process.execPath, + [path.resolve("scripts", "run-oxlint-shards.mjs"), ...process.argv.slice(2)], + { env: process.env, stdio: "inherit" }, + ); +} diff --git a/scripts/run-oxlint-shards.mjs b/scripts/run-oxlint-shards.mjs index b997eca5970..03c867946fe 100644 --- a/scripts/run-oxlint-shards.mjs +++ b/scripts/run-oxlint-shards.mjs @@ -3,10 +3,11 @@ import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import pMap from "p-map"; import { acquireLocalHeavyCheckLockSync, + ensureRepoToolNodeModulesLink, resolveLocalHeavyCheckEnv, + resolveRepoToolBinPath, shouldAcquireLocalHeavyCheckLockForOxlint, } from "./lib/local-heavy-check-runtime.mjs"; @@ -258,6 +259,7 @@ export async function main(extraArgs = process.argv.slice(2), runtimeEnv = proce const selectedShards = filterOxlintShards(shards, shardArgs.only); try { + ensureRepoToolNodeModulesLink(resolveRepoToolBinPath("oxlint")); const prepareResult = spawnSync( process.execPath, [path.resolve("scripts", "prepare-extension-package-boundary-artifacts.mjs")], @@ -390,6 +392,9 @@ export function resolveOxlintShardConcurrency({ } async function runShards({ concurrency, entries, env, extraArgs, runner }) { + // Dependency-less worktrees establish their primary-checkout toolchain link + // before this lazy import, avoiding a top-level package-resolution failure. + const { default: pMap } = await import("p-map"); const results = await pMap( entries, async (shard) => { diff --git a/scripts/run-oxlint.mjs b/scripts/run-oxlint.mjs index d19dd237526..b22619d5454 100644 --- a/scripts/run-oxlint.mjs +++ b/scripts/run-oxlint.mjs @@ -230,6 +230,11 @@ export async function main(argv = process.argv.slice(2), runtimeEnv = process.en : applyLocalOxlintPolicy(oxlintArgs, localEnv); const sparseTargets = filterSparseMissingOxlintTargets(policyArgs); const finalArgs = sparseTargets.args; + const oxlintPath = resolveRepoToolBinPath("oxlint"); + const needsArtifactPreparation = + !focusedConfig && + env.OPENCLAW_OXLINT_SKIP_PREPARE !== "1" && + shouldPrepareExtensionPackageBoundaryArtifacts(finalArgs); if (sparseTargets.skippedTargets.length > 0) { console.error( `[oxlint] sparse checkout is missing tracked target(s); skipping ${sparseTargets.skippedTargets.join(", ")}`, @@ -261,15 +266,10 @@ export async function main(argv = process.argv.slice(2), runtimeEnv = process.en : () => {}; try { - if ( - !focusedConfig && - env.OPENCLAW_OXLINT_SKIP_PREPARE !== "1" && - shouldPrepareExtensionPackageBoundaryArtifacts(finalArgs) - ) { + if (needsArtifactPreparation) { await prepareExtensionPackageBoundaryArtifacts(env); } - const oxlintPath = resolveRepoToolBinPath("oxlint"); const status = await runManagedCommand({ bin: oxlintPath, args: finalArgs, diff --git a/scripts/run-tsgo.mjs b/scripts/run-tsgo.mjs index 4423bcb87e6..072ac46e1dd 100644 --- a/scripts/run-tsgo.mjs +++ b/scripts/run-tsgo.mjs @@ -6,6 +6,7 @@ import { readFlagValue } from "./lib/arg-utils.mjs"; import { acquireLocalHeavyCheckLockSync, applyLocalTsgoPolicy, + ensureRepoToolNodeModulesLink, resolveLocalHeavyCheckEnv, resolveRepoToolBinPath, shouldAcquireLocalHeavyCheckLockForTsgo, @@ -48,6 +49,7 @@ try { process.exitCode = 1; } } else { + ensureRepoToolNodeModulesLink(tsgoPath); const tsgo = createManagedCommandInvocation({ args: finalArgs, bin: tsgoPath, diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index 8d466e53774..ef94463ccdf 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -1535,6 +1535,7 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ ], ["scripts/run-oxlint.mjs", ["test/scripts/run-oxlint.test.ts"]], ["scripts/run-oxlint-shards.mjs", ["test/scripts/run-oxlint.test.ts"]], + ["scripts/run-lint.mjs", ["test/scripts/run-oxlint.test.ts"]], ["scripts/run-with-env.mjs", ["test/scripts/run-with-env.test.ts"]], ["scripts/run-node.mjs", ["src/infra/run-node.test.ts"]], [ diff --git a/test/scripts/local-heavy-check-runtime.test.ts b/test/scripts/local-heavy-check-runtime.test.ts index b1cd7a37066..1236ad0d5a8 100644 --- a/test/scripts/local-heavy-check-runtime.test.ts +++ b/test/scripts/local-heavy-check-runtime.test.ts @@ -7,6 +7,7 @@ import { acquireLocalHeavyCheckLockSync, applyLocalOxlintPolicy, applyLocalTsgoPolicy, + ensureRepoToolNodeModulesLink, resolveLocalHeavyCheckEnv, resolveRepoToolBinPath, shouldAcquireLocalHeavyCheckLockForOxlint, @@ -64,6 +65,51 @@ describe("local-heavy-check-runtime", () => { ).toBe(localPath); }); + it("links dependency-less worktrees to the selected checkout's modules", () => { + const primaryRoot = createTempDir("openclaw-primary-toolchain-"); + const cwd = path.join(primaryRoot, ".codex", "worktrees", "task", "openclaw"); + const commonDir = path.join(primaryRoot, ".git"); + const primaryTsgo = path.join(primaryRoot, "node_modules", ".bin", "tsgo"); + const primaryNodeModules = path.join(primaryRoot, "node_modules"); + const localNodeModules = path.join(cwd, "node_modules"); + fs.mkdirSync(path.dirname(primaryTsgo), { recursive: true }); + fs.mkdirSync(cwd, { recursive: true }); + + expect( + ensureRepoToolNodeModulesLink(primaryTsgo, { + cwd, + resolveCommonDir: () => commonDir, + }), + ).toBe(localNodeModules); + expect(fs.realpathSync(localNodeModules)).toBe(fs.realpathSync(primaryNodeModules)); + + // The stable link is idempotent for concurrent and later local runners. + expect( + ensureRepoToolNodeModulesLink(primaryTsgo, { + cwd, + resolveCommonDir: () => commonDir, + }), + ).toBe(localNodeModules); + }); + + it("leaves existing worktree node_modules directories locally owned", () => { + const primaryRoot = createTempDir("openclaw-primary-toolchain-"); + const commonDir = path.join(primaryRoot, ".git"); + const primaryTsgo = path.join(primaryRoot, "node_modules", ".bin", "tsgo"); + const cwd = path.join(primaryRoot, "worktree"); + const localNodeModules = path.join(cwd, "node_modules"); + fs.mkdirSync(path.dirname(primaryTsgo), { recursive: true }); + fs.mkdirSync(localNodeModules, { recursive: true }); + + ensureRepoToolNodeModulesLink(primaryTsgo, { + cwd, + resolveCommonDir: () => commonDir, + }); + + expect(fs.lstatSync(localNodeModules).isDirectory()).toBe(true); + expect(fs.lstatSync(localNodeModules).isSymbolicLink()).toBe(false); + }); + it("reenables local heavy-check policy for local wrapper entrypoints", () => { expect(resolveLocalHeavyCheckEnv({ OPENCLAW_LOCAL_CHECK: "0", PATH: "/usr/bin" })).toEqual({ OPENCLAW_LOCAL_CHECK: "1", diff --git a/test/scripts/prepare-extension-package-boundary-artifacts.test.ts b/test/scripts/prepare-extension-package-boundary-artifacts.test.ts index e6dd2c54203..b7557395882 100644 --- a/test/scripts/prepare-extension-package-boundary-artifacts.test.ts +++ b/test/scripts/prepare-extension-package-boundary-artifacts.test.ts @@ -16,6 +16,7 @@ import { parseMode, resolveBoundaryEntryShimRequiredOutputs, resolveBoundaryRootShimsTimeoutMs, + resolveTsxImportSpecifier, runNodeStep, runNodeSteps, runNodeStepsInParallel, @@ -100,6 +101,33 @@ async function waitForProcessExit( } describe("prepare-extension-package-boundary-artifacts", () => { + it("resolves the tsx loader from the selected checkout toolchain", () => { + const tsxBinPath = "/primary/node_modules/.bin/tsx"; + const loaderPath = "/primary/node_modules/tsx/dist/loader.mjs"; + + expect( + resolveTsxImportSpecifier({ + resolveTool: (toolName) => { + expect(toolName).toBe("tsx"); + return tsxBinPath; + }, + ensureToolchain: (toolPath) => { + expect(toolPath).toBe(tsxBinPath); + return "/worktree/node_modules"; + }, + createRequireFrom: (filename) => { + expect(filename).toBe(tsxBinPath); + return { + resolve(packageName) { + expect(packageName).toBe("tsx"); + return loaderPath; + }, + }; + }, + }), + ).toBe(pathToFileURL(loaderPath).href); + }); + it("prefixes each completed line and flushes the trailing partial line", () => { let output = ""; const writer = createPrefixedOutputWriter("boundary", { diff --git a/test/scripts/run-oxlint.test.ts b/test/scripts/run-oxlint.test.ts index bfb6eb53376..936ed8de40c 100644 --- a/test/scripts/run-oxlint.test.ts +++ b/test/scripts/run-oxlint.test.ts @@ -69,9 +69,7 @@ describe("run-oxlint", () => { const shardedLintRunner = readFileSync("scripts/run-oxlint-shards.mjs", "utf8"); expect(packageJson.scripts.check).toBe("node scripts/check.mjs"); - expect(packageJson.scripts.lint).toBe( - "pnpm lint:ui:i18n && node scripts/run-oxlint-shards.mjs", - ); + expect(packageJson.scripts.lint).toBe("node scripts/run-lint.mjs"); expect(packageJson.scripts["lint:core"]).toBe( "node scripts/run-oxlint-shards.mjs --only=core --split-core", ); @@ -82,6 +80,22 @@ describe("run-oxlint", () => { expect(shardedLintRunner).toContain('OPENCLAW_OXLINT_SKIP_PREPARE: "1"'); }); + it("prepares the worktree toolchain before the complete lint pre-step", () => { + const packageJson = JSON.parse(readFileSync("package.json", "utf8")) as { + scripts: Record; + }; + const lintRunner = readFileSync("scripts/run-lint.mjs", "utf8"); + + expect(packageJson.scripts.lint).toBe("node scripts/run-lint.mjs"); + expect(lintRunner.indexOf("ensureRepoToolNodeModulesLink(")).toBeGreaterThan(-1); + expect( + lintRunner.indexOf('path.resolve("scripts", "control-ui-i18n-verify.ts")'), + ).toBeGreaterThan(lintRunner.indexOf("ensureRepoToolNodeModulesLink(")); + expect(lintRunner.indexOf('path.resolve("scripts", "run-oxlint-shards.mjs")')).toBeGreaterThan( + lintRunner.indexOf('path.resolve("scripts", "control-ui-i18n-verify.ts")'), + ); + }); + it("holds one parent heavy-check lock for sharded lint runs", () => { const shardedLintRunner = readFileSync("scripts/run-oxlint-shards.mjs", "utf8"); const skipLockIndex = shardedLintRunner.indexOf('env.OPENCLAW_OXLINT_SKIP_LOCK === "1"');