fix(scripts): restore Testbox lease claims to the originating repo after delegated runs (#110869)

* fix(scripts): restore Testbox lease claims to the originating repo after delegated runs Delegated Blacksmith Testbox runs execute from throwaway sparse-sync checkouts, so Crabbox stamps retained lease claims with that temporary cwd. Restore only claims still pointing at the child checkout after the run on both success and failure, preserving foreign claims and every other field. This repairs the post-run artifact instead of weakening the pre-run ownership guard. * fix(scripts): restore claims for delegated runs that create retained leases
2026-07-21 10:16:44 +00:00 · 2026-07-18 19:49:04 +01:00
parent 453444636c
commit a0cc7d11fb
2 changed files with 363 additions and 3 deletions
@@ -19,6 +19,7 @@ import {
 } from "node:fs";
 import { homedir, tmpdir } from "node:os";
 import { delimiter, dirname, extname, isAbsolute, relative, resolve } from "node:path";
+import { StringDecoder } from "node:string_decoder";
 import { fileURLToPath } from "node:url";
 import {
  canonicalProviderName,
@@ -33,6 +34,7 @@ import { resolvePathEnvKey, resolveWindowsCmdExePath } from "./windows-cmd-helpe

 const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 const CRABBOX_METADATA_PROBE_TIMEOUT_MS = 5_000;
+const MAX_TIMING_JSON_LINE_CHARS = 1024 * 1024;
 const REMOTE_CHANGED_GATE_BUNDLE_FILE = ".openclaw-crabbox-changed-gate.bundle";
 // A cold Crabbox (first call after an upgrade, or one on a loaded machine) can
 // exceed the snappy default probe timeout while it renders `run --help` or does
@@ -940,12 +942,20 @@ function blacksmithTestboxPrivateKeyPath(id) {

 // Crabbox claims bind raw Testbox ids to one repo before remote execution.
 // Check the same sidecar so a dependency exit bug cannot make refusal green.
-function blacksmithTestboxClaimRepoRoot(id) {
+function blacksmithTestboxClaimPath(id) {
+  return resolve(blacksmithTestboxClaimsDir(), `${id}.json`);
+}
+
+function blacksmithTestboxClaimsDir() {
  const configuredStateRoot = process.env.XDG_STATE_HOME?.trim();
  const stateDir = configuredStateRoot
    ? resolve(configuredStateRoot, "crabbox")
    : resolve(crabboxConfigDir(), "state");
-  const claimPath = resolve(stateDir, "claims", `${id}.json`);
+  return resolve(stateDir, "claims");
+}
+
+function blacksmithTestboxClaimRepoRoot(id) {
+  const claimPath = blacksmithTestboxClaimPath(id);
  if (!pathExists(claimPath)) {
    return "";
  }
@@ -986,6 +996,93 @@ function enforceCrabboxOwnedBlacksmithLease(commandArgs) {
  }
 }

+function restoreTemporaryBlacksmithTestboxClaimPath(claimPath) {
+  const original = readFileSync(claimPath, "utf8");
+  const claim = JSON.parse(original);
+  if (!claim || typeof claim !== "object" || claim.repoRoot !== childCwd) {
+    return;
+  }
+  claim.repoRoot = repoRoot;
+  const temporaryPath = `${claimPath}.tmp-${process.pid}-${Date.now()}`;
+  try {
+    writeFileSync(temporaryPath, `${JSON.stringify(claim)}\n`, {
+      encoding: "utf8",
+      mode: 0o600,
+    });
+    if (readFileSync(claimPath, "utf8") !== original) {
+      return;
+    }
+    renameSync(temporaryPath, claimPath);
+  } finally {
+    rmSync(temporaryPath, { force: true });
+  }
+}
+
+function restoreTemporaryBlacksmithTestboxClaim(commandArgs, capturedLeaseId) {
+  if (childCwd === repoRoot) {
+    return;
+  }
+
+  const explicitLeaseId = commandArgs[0] === "run" ? optionValue(commandArgs, "--id") : "";
+  const exactLeaseId = explicitLeaseId || capturedLeaseId;
+  const canCreateRetainedLease =
+    commandArgs[0] === "warmup" ||
+    (commandArgs[0] === "run" &&
+      (hasOption(commandArgs, "--keep") || hasOption(commandArgs, "--keep-on-failure")));
+  let claimPaths = [];
+  if (exactLeaseId) {
+    claimPaths = [blacksmithTestboxClaimPath(exactLeaseId)];
+  } else if (canCreateRetainedLease) {
+    try {
+      const claimsDir = blacksmithTestboxClaimsDir();
+      if (pathExists(claimsDir)) {
+        claimPaths = readdirSync(claimsDir)
+          .filter((entry) => entry.endsWith(".json"))
+          .map((entry) => resolve(claimsDir, entry));
+      }
+    } catch (error) {
+      console.error(
+        `[crabbox] warning: failed to inspect temporary Testbox claims: ${error instanceof Error ? error.message : String(error)}`,
+      );
+      return;
+    }
+  } else {
+    return;
+  }
+
+  for (const claimPath of claimPaths) {
+    if (!pathExists(claimPath)) {
+      continue;
+    }
+    try {
+      restoreTemporaryBlacksmithTestboxClaimPath(claimPath);
+    } catch (error) {
+      console.error(
+        `[crabbox] warning: failed to restore temporary Testbox claim: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+}
+
+function observeBlacksmithTimingJSONLine(line) {
+  const value = line.trim();
+  if (!value.startsWith("{") || !value.endsWith("}")) {
+    return;
+  }
+  try {
+    const report = JSON.parse(value);
+    if (
+      canonicalProviderName(report?.provider) === "blacksmith-testbox" &&
+      typeof report.leaseId === "string" &&
+      report.leaseId.startsWith("tbx_")
+    ) {
+      capturedBlacksmithLeaseId = report.leaseId;
+    }
+  } catch {
+    // Human stderr may contain brace-delimited non-JSON lines.
+  }
+}
+
 function preserveTemporaryCrabboxRuns() {
  if (childCwd === repoRoot) {
    return;
@@ -3482,6 +3579,7 @@ let stopFullCheckoutKeepalive = () => {};
 let cleanupDone = false;
 let remoteChangedGateBase = "";
 let remoteChangedGateAlias = "";
+let capturedBlacksmithLeaseId = "";
 const scriptBootstrap = prepareAwsMacosScriptStdinBootstrap(normalizedArgs, provider);
 normalizedArgs = scriptBootstrap.args;
 const scriptStdinPrepared = scriptBootstrap.prepared;
@@ -3520,6 +3618,11 @@ function cleanupOnce() {
  stopFullCheckoutKeepalive();
  wsl2ScriptBootstrap.cleanup();
  scriptBootstrap.cleanup();
+  if (canonicalProvider === "blacksmith-testbox") {
+    // Crabbox stamps claims with its cwd. Delegated runs use a throwaway sync checkout,
+    // so restore the real repo or every later reuse needs --reclaim.
+    restoreTemporaryBlacksmithTestboxClaim(normalizedArgs, capturedBlacksmithLeaseId);
+  }
  preserveTemporaryCrabboxRuns();
  cleanupChildCwd();
 }
@@ -3626,6 +3729,8 @@ if (fullCheckout) {
  }
 }
 const childInvocation = spawnInvocation(binary, childArgs, childEnv, process.platform);
+const captureBlacksmithTimingJSON =
+  canonicalProvider === "blacksmith-testbox" && hasOption(normalizedArgs, "--timing-json");
 // Fast-fail hint context: run --id reuse dies in under a second when the
 // lease hit its idle timeout, with only a bare nonzero exit from the binary.
 const reusedRunLeaseId = normalizedArgs[0] === "run" ? optionValue(normalizedArgs, "--id") : "";
@@ -3633,11 +3738,53 @@ const childStartedAtMs = Date.now();
 const FAST_FAIL_HINT_WINDOW_MS = 15_000;
 const child = spawn(childInvocation.command, childInvocation.args, {
  cwd: childCwd,
-  stdio: "inherit",
+  stdio: ["inherit", "inherit", captureBlacksmithTimingJSON ? "pipe" : "inherit"],
  detached: process.platform !== "win32",
  env: childEnv,
  windowsVerbatimArguments: childInvocation.windowsVerbatimArguments,
 });
+if (child.stderr) {
+  const decoder = new StringDecoder("utf8");
+  let pending = "";
+  let discardingOversizedLine = false;
+  const observeText = (text) => {
+    let remaining = text;
+    while (remaining) {
+      const newline = remaining.indexOf("\n");
+      const fragment = newline >= 0 ? remaining.slice(0, newline) : remaining;
+      if (!discardingOversizedLine) {
+        pending += fragment;
+        if (pending.length > MAX_TIMING_JSON_LINE_CHARS) {
+          pending = "";
+          discardingOversizedLine = true;
+        }
+      }
+      if (newline < 0) {
+        return;
+      }
+      if (!discardingOversizedLine) {
+        observeBlacksmithTimingJSONLine(pending);
+      }
+      pending = "";
+      discardingOversizedLine = false;
+      remaining = remaining.slice(newline + 1);
+    }
+  };
+  child.stderr.on("data", (chunk) => {
+    const canContinue = process.stderr.write(chunk);
+    observeText(decoder.write(chunk));
+    if (!canContinue) {
+      child.stderr.pause();
+      process.stderr.once("drain", () => child.stderr?.resume());
+    }
+  });
+  child.stderr.on("end", () => {
+    observeText(decoder.end());
+    if (pending && !discardingOversizedLine) {
+      observeBlacksmithTimingJSONLine(pending);
+    }
+  });
+}
 const childKillGraceMs = resolveChildKillGraceMs(process.env);
 let childForceKillTimer;
 let childTreeShutdownStarted = false;
@@ -55,6 +55,19 @@ function writeFakeCrabbox(binDir: string, helpText: string): string {
  mkdirSync(binDir, { recursive: true });
  const crabboxPath = path.join(binDir, "crabbox");
  const helperPath = path.join(binDir, "fake-crabbox-json.cjs");
+  const stampClaimScript = [
+    "const fs = require('node:fs');",
+    "const claimPaths = [process.env.OPENCLAW_FAKE_CRABBOX_CLAIM_PATH, process.env.OPENCLAW_FAKE_CRABBOX_EXTRA_CLAIM_PATH].filter(Boolean);",
+    "for (const claimPath of claimPaths) {",
+    "  const claim = fs.existsSync(claimPath) ? JSON.parse(fs.readFileSync(claimPath, 'utf8')) : { leaseID: process.env.OPENCLAW_FAKE_CRABBOX_TIMING_LEASE_ID };",
+    "  claim.repoRoot = process.env.OPENCLAW_FAKE_CRABBOX_CLAIM_REPO_ROOT || process.cwd();",
+    "  fs.mkdirSync(require('node:path').dirname(claimPath), { recursive: true });",
+    "  fs.writeFileSync(claimPath, JSON.stringify(claim) + '\\n', 'utf8');",
+    "}",
+    "if (process.env.OPENCLAW_FAKE_CRABBOX_TIMING_LEASE_ID) {",
+    "  process.stderr.write(JSON.stringify({ provider: 'blacksmith-testbox', leaseId: process.env.OPENCLAW_FAKE_CRABBOX_TIMING_LEASE_ID, exitCode: 0 }) + '\\n');",
+    "}",
+  ].join("\n");

  if (process.platform !== "win32") {
    const signalIgnoringDescendantScript = [
@@ -78,6 +91,9 @@ function writeFakeCrabbox(binDir: string, helpText: string): string {
      `  printf "%s" ${shellSingleQuote(helpText)}`,
      "  exit 0",
      "fi",
+      'if { [ "$1" = "run" ] || [ "$1" = "warmup" ]; } && { [ -n "${OPENCLAW_FAKE_CRABBOX_CLAIM_PATH:-}" ] || [ -n "${OPENCLAW_FAKE_CRABBOX_TIMING_LEASE_ID:-}" ]; }; then',
+      `  ${shellSingleQuote(process.execPath)} --eval ${shellSingleQuote(stampClaimScript)}`,
+      "fi",
      'if [ "$1" = "run" ] && [ -n "${OPENCLAW_FAKE_CRABBOX_RUN_STATUS:-}" ] && [ "$OPENCLAW_FAKE_CRABBOX_RUN_STATUS" != "0" ]; then',
      '  printf "%s\\n" "fake run failure" >&2',
      '  exit "$OPENCLAW_FAKE_CRABBOX_RUN_STATUS"',
@@ -247,6 +263,7 @@ function writeFakeCrabbox(binDir: string, helpText: string): string {
    `  process.stdout.write(${JSON.stringify(helpText)});`,
    "  process.exit(0);",
    "}",
+    `if (args[0] === "run" || args[0] === "warmup") { ${stampClaimScript} }`,
    'if (args[0] === "run" && Number.parseInt(process.env.OPENCLAW_FAKE_CRABBOX_RUN_STATUS || "0", 10) !== 0) {',
    "  process.stderr.write('fake run failure\\n');",
    "  process.exit(Number.parseInt(process.env.OPENCLAW_FAKE_CRABBOX_RUN_STATUS, 10));",
@@ -944,6 +961,202 @@ describe("scripts/crabbox-wrapper", () => {
    expect(parseFakeCrabboxOutput(reclaimed).args).toContain("--reclaim");
  });

+  it.each([
+    { label: "successful", status: 0 },
+    { label: "failed", status: 7 },
+  ])("restores delegated Blacksmith claims after $label runs", ({ status }) => {
+    const home = mkdtempSync(path.join(tmpdir(), "openclaw-crabbox-home-"));
+    tempDirs.push(home);
+    const id = `tbx_restore_${status}`;
+    const keyPath = path.join(testCrabboxConfigDir(home), "testboxes", id, "id_ed25519");
+    mkdirSync(path.dirname(keyPath), { recursive: true });
+    writeFileSync(keyPath, "fake test key\n", "utf8");
+    const stateRoot = path.join(home, ".local", "state");
+    const claimPath = path.join(stateRoot, "crabbox", "claims", `${id}.json`);
+    mkdirSync(path.dirname(claimPath), { recursive: true });
+    const originalClaim = {
+      leaseID: id,
+      repoRoot,
+      owner: "preserved-owner",
+      metadata: { keep: true },
+    };
+    writeFileSync(claimPath, `${JSON.stringify(originalClaim)}\n`, "utf8");
+
+    const result = runWrapper(
+      "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n",
+      ["run", "--provider", "blacksmith-testbox", "--id", id, "--", "echo ok"],
+      {
+        env: {
+          ...testHomeEnv(home),
+          XDG_STATE_HOME: stateRoot,
+          OPENCLAW_FAKE_CRABBOX_CLAIM_PATH: claimPath,
+          ...(status > 0 ? { OPENCLAW_FAKE_CRABBOX_RUN_STATUS: String(status) } : {}),
+        },
+        gitResponses: {
+          [GIT_CONFIG_SPARSE_KEY]: { stdout: "true\n" },
+          [GIT_STATUS_PORCELAIN_KEY]: { stdout: "" },
+        },
+      },
+    );
+
+    expect(result.status).toBe(status);
+    expect(JSON.parse(readFileSync(claimPath, "utf8"))).toEqual(originalClaim);
+  });
+
+  it("restores a created delegated Blacksmith claim by captured timing lease id", () => {
+    const home = mkdtempSync(path.join(tmpdir(), "openclaw-crabbox-home-"));
+    tempDirs.push(home);
+    const stateRoot = path.join(home, ".local", "state");
+    const claimsDir = path.join(stateRoot, "crabbox", "claims");
+    const id = "tbx_created_timing";
+    const claimPath = path.join(claimsDir, `${id}.json`);
+    const decoyPath = path.join(claimsDir, "tbx_created_decoy.json");
+    const originalClaim = { leaseID: id, repoRoot, metadata: { keep: true } };
+    const decoyClaim = { leaseID: "tbx_created_decoy", repoRoot };
+    mkdirSync(claimsDir, { recursive: true });
+    writeFileSync(claimPath, `${JSON.stringify(originalClaim)}\n`, "utf8");
+    writeFileSync(decoyPath, `${JSON.stringify(decoyClaim)}\n`, "utf8");
+
+    const result = runWrapper(
+      "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n",
+      ["run", "--provider", "blacksmith-testbox", "--keep", "--timing-json", "--", "echo ok"],
+      {
+        env: {
+          ...testHomeEnv(home),
+          XDG_STATE_HOME: stateRoot,
+          OPENCLAW_FAKE_CRABBOX_CLAIM_PATH: claimPath,
+          OPENCLAW_FAKE_CRABBOX_EXTRA_CLAIM_PATH: decoyPath,
+          OPENCLAW_FAKE_CRABBOX_TIMING_LEASE_ID: id,
+        },
+        gitResponses: {
+          [GIT_CONFIG_SPARSE_KEY]: { stdout: "true\n" },
+          [GIT_STATUS_PORCELAIN_KEY]: { stdout: "" },
+        },
+      },
+    );
+
+    expect(result.status).toBe(0);
+    expect(JSON.parse(readFileSync(claimPath, "utf8"))).toEqual(originalClaim);
+    expect(JSON.parse(readFileSync(decoyPath, "utf8"))).toEqual({
+      ...decoyClaim,
+      repoRoot: parseFakeCrabboxOutput(result).cwd,
+    });
+  });
+
+  it("restores created delegated Blacksmith claims from the temporary checkout fallback", () => {
+    const home = mkdtempSync(path.join(tmpdir(), "openclaw-crabbox-home-"));
+    tempDirs.push(home);
+    const stateRoot = path.join(home, ".local", "state");
+    const claimsDir = path.join(stateRoot, "crabbox", "claims");
+    const claimPath = path.join(claimsDir, "tbx_created_fallback.json");
+    const siblingPath = path.join(claimsDir, "tbx_created_sibling.json");
+    const foreignPath = path.join(claimsDir, "tbx_foreign_fallback.json");
+    const createdClaim = { leaseID: "tbx_created_fallback", repoRoot, owner: "created" };
+    const siblingClaim = { leaseID: "tbx_created_sibling", repoRoot, owner: "sibling" };
+    const foreignClaim = {
+      leaseID: "tbx_foreign_fallback",
+      repoRoot: "/tmp/genuinely-foreign-repo",
+    };
+    mkdirSync(claimsDir, { recursive: true });
+    writeFileSync(claimPath, `${JSON.stringify(createdClaim)}\n`, "utf8");
+    writeFileSync(siblingPath, `${JSON.stringify(siblingClaim)}\n`, "utf8");
+    writeFileSync(foreignPath, `${JSON.stringify(foreignClaim)}\n`, "utf8");
+
+    const result = runWrapper(
+      "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n",
+      ["run", "--provider", "blacksmith-testbox", "--keep", "--", "echo ok"],
+      {
+        env: {
+          ...testHomeEnv(home),
+          XDG_STATE_HOME: stateRoot,
+          OPENCLAW_FAKE_CRABBOX_CLAIM_PATH: claimPath,
+          OPENCLAW_FAKE_CRABBOX_EXTRA_CLAIM_PATH: siblingPath,
+        },
+        gitResponses: {
+          [GIT_CONFIG_SPARSE_KEY]: { stdout: "true\n" },
+          [GIT_STATUS_PORCELAIN_KEY]: { stdout: "" },
+        },
+      },
+    );
+
+    expect(result.status).toBe(0);
+    expect(JSON.parse(readFileSync(claimPath, "utf8"))).toEqual(createdClaim);
+    expect(JSON.parse(readFileSync(siblingPath, "utf8"))).toEqual(siblingClaim);
+    expect(JSON.parse(readFileSync(foreignPath, "utf8"))).toEqual(foreignClaim);
+  });
+
+  it("restores a failed delegated Blacksmith claim kept on failure", () => {
+    const home = mkdtempSync(path.join(tmpdir(), "openclaw-crabbox-home-"));
+    tempDirs.push(home);
+    const stateRoot = path.join(home, ".local", "state");
+    const claimPath = path.join(stateRoot, "crabbox", "claims", "tbx_created_failure.json");
+    const originalClaim = {
+      leaseID: "tbx_created_failure",
+      repoRoot,
+      metadata: { keepOnFailure: true },
+    };
+    mkdirSync(path.dirname(claimPath), { recursive: true });
+    writeFileSync(claimPath, `${JSON.stringify(originalClaim)}\n`, "utf8");
+
+    const result = runWrapper(
+      "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n",
+      ["run", "--provider", "blacksmith-testbox", "--keep-on-failure", "--", "false"],
+      {
+        env: {
+          ...testHomeEnv(home),
+          XDG_STATE_HOME: stateRoot,
+          OPENCLAW_FAKE_CRABBOX_CLAIM_PATH: claimPath,
+          OPENCLAW_FAKE_CRABBOX_RUN_STATUS: "7",
+        },
+        gitResponses: {
+          [GIT_CONFIG_SPARSE_KEY]: { stdout: "true\n" },
+          [GIT_STATUS_PORCELAIN_KEY]: { stdout: "" },
+        },
+      },
+    );
+
+    expect(result.status).toBe(7);
+    expect(JSON.parse(readFileSync(claimPath, "utf8"))).toEqual(originalClaim);
+  });
+
+  it("leaves genuinely foreign delegated Blacksmith claims untouched", () => {
+    const home = mkdtempSync(path.join(tmpdir(), "openclaw-crabbox-home-"));
+    tempDirs.push(home);
+    const id = "tbx_foreign_claim";
+    const keyPath = path.join(testCrabboxConfigDir(home), "testboxes", id, "id_ed25519");
+    mkdirSync(path.dirname(keyPath), { recursive: true });
+    writeFileSync(keyPath, "fake test key\n", "utf8");
+    const stateRoot = path.join(home, ".local", "state");
+    const claimPath = path.join(stateRoot, "crabbox", "claims", `${id}.json`);
+    mkdirSync(path.dirname(claimPath), { recursive: true });
+    const foreignClaim = {
+      leaseID: id,
+      repoRoot: "/tmp/genuinely-foreign-repo",
+      owner: "foreign-owner",
+    };
+    writeFileSync(claimPath, `${JSON.stringify({ ...foreignClaim, repoRoot })}\n`, "utf8");
+
+    const result = runWrapper(
+      "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n",
+      ["run", "--provider", "blacksmith-testbox", "--id", id, "--", "echo ok"],
+      {
+        env: {
+          ...testHomeEnv(home),
+          XDG_STATE_HOME: stateRoot,
+          OPENCLAW_FAKE_CRABBOX_CLAIM_PATH: claimPath,
+          OPENCLAW_FAKE_CRABBOX_CLAIM_REPO_ROOT: foreignClaim.repoRoot,
+        },
+        gitResponses: {
+          [GIT_CONFIG_SPARSE_KEY]: { stdout: "true\n" },
+          [GIT_STATUS_PORCELAIN_KEY]: { stdout: "" },
+        },
+      },
+    );
+
+    expect(result.status).toBe(0);
+    expect(JSON.parse(readFileSync(claimPath, "utf8"))).toEqual(foreignClaim);
+  });
+
  it("lets Crabbox resolve reusable Testbox slugs", () => {
    const home = mkdtempSync(path.join(tmpdir(), "openclaw-crabbox-home-"));
    tempDirs.push(home);