From 1fe3a5f55783f5c1c75f68392b30a0a22443dccc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 15 Jul 2026 08:52:00 +0100 Subject: [PATCH] fix(skills): shorten Testbox SSH socket paths --- .agents/skills/autoreview/SKILL.md | 6 + .agents/skills/autoreview/scripts/autoreview | 28 +- .../tests/test_autoreview_hardening.py | 47 + .agents/skills/crabbox/SKILL.md | 1001 +++++------------ 4 files changed, 341 insertions(+), 741 deletions(-) diff --git a/.agents/skills/autoreview/SKILL.md b/.agents/skills/autoreview/SKILL.md index 2223d16f29d..f9ae3a5db62 100644 --- a/.agents/skills/autoreview/SKILL.md +++ b/.agents/skills/autoreview/SKILL.md @@ -235,6 +235,12 @@ happen before the test shell starts: OPENCLAW_TESTBOX=1 "$AUTOREVIEW" --parallel-tests "pnpm check:changed" ``` +On POSIX, the helper puts this isolated Testbox home under the short, sticky +system `/tmp`; Blacksmith creates an SSH control socket below that home, and a +long macOS `TMPDIR` can exceed the Unix-socket path limit. With an older helper, +prefix the outer autoreview process with `TMPDIR=/tmp`. Setting `TMPDIR` inside +the quoted test command is too late because the isolated home already exists. + This is the narrow trusted-maintainer-code exception: it stages only the Blacksmith credential file into the temporary home so the command can delegate remotely. Never use this credential-hydrated path for untrusted contributor or fork code. Run other diff --git a/.agents/skills/autoreview/scripts/autoreview b/.agents/skills/autoreview/scripts/autoreview index ca3b8775c29..cd7b8f8a45f 100755 --- a/.agents/skills/autoreview/scripts/autoreview +++ b/.agents/skills/autoreview/scripts/autoreview @@ -968,6 +968,29 @@ def safe_temp_root(repo: Path) -> Path: return root +def parallel_test_temp_root(repo: Path) -> Path: + root = safe_temp_root(repo) + if os.name == "nt" or not env_truthy("OPENCLAW_TESTBOX"): + return root + + # Blacksmith stores its SSH control socket below HOME. macOS temp roots can + # already consume the Unix-socket path budget before the socket name. + try: + short_root = Path("/tmp").resolve(strict=True) + short_root_stat = short_root.stat() + except OSError as exc: + raise SystemExit(f"unable to resolve short Testbox temp root: {exc}") from exc + if not stat.S_ISDIR(short_root_stat.st_mode): + raise SystemExit("short Testbox temp root must be a directory") + if is_within(short_root, repo.resolve()): + raise SystemExit("short Testbox temp root must be outside the reviewed repository") + if short_root_stat.st_mode & stat.S_IWOTH and not ( + short_root_stat.st_mode & stat.S_ISVTX + ): + raise SystemExit("world-writable Testbox temp root must have the sticky bit set") + return short_root + + def safe_proxy_url(value: str) -> bool: try: candidate = value if "://" in value else f"http://{value}" @@ -8971,7 +8994,10 @@ def start_parallel_tests( ) -> tuple[subprocess.Popen, float]: print(f"tests: {command}") test_home = Path( - tempfile.mkdtemp(prefix="autoreview-test-home-", dir=safe_temp_root(repo)) + tempfile.mkdtemp( + prefix="autoreview-test-home-", + dir=parallel_test_temp_root(repo), + ) ) try: env = safe_test_env(repo, test_home) diff --git a/.agents/skills/autoreview/tests/test_autoreview_hardening.py b/.agents/skills/autoreview/tests/test_autoreview_hardening.py index 10688ddb253..5815c52b51b 100644 --- a/.agents/skills/autoreview/tests/test_autoreview_hardening.py +++ b/.agents/skills/autoreview/tests/test_autoreview_hardening.py @@ -4149,6 +4149,53 @@ class AutoreviewHardeningTests(unittest.TestCase): ): self.helper["safe_temp_root"](repo) + @unittest.skipIf(os.name == "nt", "POSIX Testbox temp-root behavior") + def test_testbox_parallel_test_temp_root_stays_within_socket_limit(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + repo = init_repo(root) + long_temp = root / ("macos-temp-root-" + "x" * 96) + long_temp.mkdir() + + with mock.patch.object( + tempfile, + "gettempdir", + return_value=str(long_temp), + ), mock.patch.dict( + os.environ, + {"OPENCLAW_TESTBOX": "1"}, + ): + selected = self.helper["parallel_test_temp_root"](repo) + + self.assertEqual(selected, Path("/tmp").resolve()) + socket_path = ( + selected + / ("autoreview-test-home-" + "x" * 8) + / ".blacksmith" + / "c" + / "6d146d2f25180c1d.sock" + ) + self.assertLess(len(os.fsencode(socket_path)), 104) + + def test_parallel_test_temp_root_keeps_configured_root_without_testbox(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + repo = init_repo(root) + configured_temp = root / "configured-temp" + configured_temp.mkdir() + + with mock.patch.object( + tempfile, + "gettempdir", + return_value=str(configured_temp), + ), mock.patch.dict( + os.environ, + {"OPENCLAW_TESTBOX": "0"}, + ): + selected = self.helper["parallel_test_temp_root"](repo) + + self.assertEqual(selected, configured_temp.resolve()) + def test_claude_fable_alias_requires_fable_safe_mode_version(self) -> None: args = argparse.Namespace( claude_bin="claude", diff --git a/.agents/skills/crabbox/SKILL.md b/.agents/skills/crabbox/SKILL.md index 92635a8d3bc..832f9566fba 100644 --- a/.agents/skills/crabbox/SKILL.md +++ b/.agents/skills/crabbox/SKILL.md @@ -1,102 +1,141 @@ --- name: crabbox -description: Use the Crabbox wrapper for OpenClaw remote validation across Linux, macOS, Windows, and WSL2, including delegated Blacksmith Testbox proof. Report the actual provider and id. +description: "Crabbox/Testbox remote proof for OpenClaw: trusted-source routing, untrusted isolation, Linux/macOS/Windows/WSL2, live E2E, desktop, diagnostics, cleanup." --- # Crabbox -OpenClaw agent sessions use the Crabbox wrapper for heavy proof: larger test -suites, builds, typechecks, lint fan-out, broad gates, CI-parity checks, -secrets, hosted services, Docker/E2E/package lanes, reusable boxes, sync -timing, logs/results, cache inspection, and lease cleanup. For trusted source, -one/few focused tests stay local when the existing dependency install is ready. -Untrusted repository tooling never runs locally, regardless of proof size. +Remote OpenClaw proof. Heavy tests. Builds. Typecheck/lint fan-out. Docker. +Packages. Live providers. Desktop. Cross-OS. -Crabbox is the transport/orchestration surface. The actual backend can be: +Backends: -- brokered AWS Crabbox: direct provider, `provider=aws`, lease ids like - `cbx_...`, `syncDelegated=false` -- Blacksmith Testbox through Crabbox: delegated provider, - `provider=blacksmith-testbox`, ids like `tbx_...`, `syncDelegated=true` +- `blacksmith-testbox`: trusted maintainer source. Prepared CI. `tbx_...`. +- `aws`: direct brokered Crabbox. Fresh PRs. Custom sync/env/capture. `cbx_...`. +- `local-container`: Docker fallback. Not remote proof. +- `ssh`: existing operator host. macOS/Windows/WSL2. -Blacksmith Testbox through the Crabbox wrapper is the default OpenClaw agent -backend for trusted maintainer heavy proof and heavy `pnpm` gates. The configured -Blacksmith workflow hydrates provider and agent credentials, so never sync or -run untrusted contributor/fork code there. Use secretless fork CI or -sanitized direct AWS Crabbox for untrusted source. Do not describe -Blacksmith runs as "AWS Crabbox"; report them as Testbox-through-Crabbox with -the `tbx_...` id and Actions run. +Always report provider, id, run URL, command, result. Never call Testbox “AWS +Crabbox.” -Pass `--provider aws` when the task specifically needs direct AWS Crabbox -behavior, persistent direct-provider leases, `--fresh-pr`, `--full-resync`, -environment forwarding, capture/download support, or provider comparison. Use -`--provider blacksmith-testbox` for the default OpenClaw agent path. +## Route First -## First Checks +Source trust before test size. -- Run from the repo root. Crabbox sync mirrors the current checkout. -- Check the wrapper and providers before remote work: +- Trusted + one/few focused tests + ready deps: local. +- Trusted + heavy proof: Blacksmith Testbox. +- Untrusted contributor/fork: secretless fork CI or sanitized direct AWS. +- Never untrusted code on credential-hydrated Testbox. +- Never run untrusted repo wrapper/config locally. +- No speculative warmup. Acquire when first heavy command ready. Reuse id. Stop. + +Need direct AWS semantics? Pass `--provider aws`. Need normal trusted OpenClaw +heavy proof? Pass `--provider blacksmith-testbox`. + +## Preflight + +Run from repo root. ```sh command -v crabbox ../crabbox/bin/crabbox --version -pnpm crabbox:run -- --help | sed -n '1,120p' -../crabbox/bin/crabbox desktop launch --help -../crabbox/bin/crabbox webvnc --help +node scripts/crabbox-wrapper.mjs run --help | sed -n '1,100p' +command -v blacksmith +blacksmith --version ``` -- OpenClaw scripts prefer `../crabbox/bin/crabbox` when present. The user PATH - shim can be stale. -- Check `.crabbox.yaml` for the provider default. Omitting `--provider` - means Blacksmith Testbox through Crabbox for normal Linux paths; the wrapper - selects Azure for unqualified Windows/WSL2 runs when the local Crabbox - binary advertises Azure. Pass `--provider aws` for direct brokered AWS runs. -- The brokered AWS image is a Linux developer image in `eu-west-1`; the repo - config pins hot `eu-west-1a/b/c` placement so Fast Snapshot Restore can apply. - If warmup drifts well past the minute-scale path, verify image promotion, - region/AZ placement, and FSR state before blaming OpenClaw. -- For trusted OpenClaw computationally intensive work, use the repo wrapper - with `--provider blacksmith-testbox` or the repo Testbox helpers. -- Treat contributor/fork source as untrusted unless a maintainer explicitly - approves credentialed execution after review. Run untrusted source only in - secretless fork CI or sanitized direct AWS Crabbox. For every untrusted AWS - run, launch an installed trusted Crabbox binary from a clean trusted `main` - checkout and fetch the remote PR with `--fresh-pr`; never execute the - untrusted checkout's wrapper or config locally. Set - `CRABBOX_ENV_ALLOW=CI` to replace the repo's `OPENCLAW_*`/`NODE_OPTIONS` - allowlist, pass `--provider aws --no-hydrate`, and use a fresh temporary - remote `HOME` on a newly warmed lease dedicated to that untrusted source. - Unset `CRABBOX_AWS_INSTANCE_PROFILE` and fail closed unless resolved - `aws.instanceProfile` is empty. Before install/test, use trusted absolute-path - tools to require an IMDSv2 token, prove the IAM credentials endpoint returns - 404, and verify remote `git rev-parse HEAD` equals the full reviewed PR head - SHA. Bind the lease to that SHA; stop and rewarm when the head changes. Do not - inherit Tailscale: unset every `CRABBOX_TAILSCALE*` override, force - `--network public --tailscale=false`, clear exit-node/LAN flags, and require - `crabbox inspect` to report public networking with no Tailscale state before - uploading any script. Execute PR code only through trusted - `scripts/crabbox-untrusted-bootstrap.sh`, uploaded from clean `main` alongside - `--fresh-pr`; it installs pinned Node/pnpm and rejects a changed PR - `packageManager` pin before install. Never reuse a trusted or previously - hydrated lease. If the broker cannot provide - the no-role proof or no remote PR exists, use secretless fork CI. Never use - `hydrate-github` or a credential-hydrated Testbox workflow for untrusted code. -- Cold Testbox acquisition and hydration often take about a minute. Do not - pre-warm for anticipated work. After confirming source trust, acquire a box - lazily when the first heavy command is ready. Reuse the returned `tbx_...` - with `--provider blacksmith-testbox --id ` for later heavy commands, - and stop it before handoff. For untrusted heavy proof, switch to a clean - trusted `main` checkout and lazily warm with the installed binary after the - empty-instance-profile check below. -- Run untrusted source only with the sanitized form below. The explicit - allowlist prevents locally exported `OPENCLAW_*` credentials from crossing - the SSH boundary; `--no-hydrate` and temporary `HOME` prevent auth-profile - reuse: +Set checked binary once. PATH copy may be stale. ```sh +if [ -x ../crabbox/bin/crabbox ]; then + export CRABBOX=../crabbox/bin/crabbox +else + export CRABBOX="$(command -v crabbox)" +fi +"$CRABBOX" --version +``` + +Read `.crabbox.yaml`; never guess provider default. + +No binary? Clean sibling checkout only: + +```sh +if [ -n "$(git -C ../crabbox status --short)" ]; then + git -C ../crabbox status --short + exit 1 +fi +git -C ../crabbox pull --ff-only +mkdir -p ../crabbox/bin +(cd ../crabbox && go build -o bin/crabbox ./cmd/crabbox) +../crabbox/bin/crabbox --version +``` + +Dirty/missing/nonstandard sibling: stop. No overwrite. + +## Trusted Testbox + +One-shot heavy gate: + +```sh +node scripts/crabbox-wrapper.mjs run \ + --provider blacksmith-testbox \ + --timing-json -- \ + CI=1 NODE_OPTIONS=--max-old-space-size=4096 \ + OPENCLAW_TEST_PROJECTS_PARALLEL=6 \ + OPENCLAW_VITEST_MAX_WORKERS=1 \ + OPENCLAW_TESTBOX=1 OPENCLAW_TESTBOX_REMOTE_RUN=1 \ + pnpm check:changed +``` + +Several commands: warm once, save id, reuse, stop. + +```sh +node scripts/crabbox-wrapper.mjs warmup \ + --provider blacksmith-testbox --keep --timing-json +node scripts/crabbox-wrapper.mjs run \ + --provider blacksmith-testbox --id --timing-json -- \ + OPENCLAW_TESTBOX=1 OPENCLAW_TESTBOX_REMOTE_RUN=1 \ + pnpm test +blacksmith testbox stop --id +``` + +Rules: + +- One lease, one active command. No sync/reclaim during run. +- Sync current checkout every run. `--no-sync` only unchanged intentional rerun. +- `--reclaim` only deliberate checkout-path ownership transfer. +- Base/head change: stop. Rewarm. No stale-lease override. +- Raw SHA unreliable for `warmup --ref`; use branch/tag. +- `blacksmith testbox list` hides states. Use `list --all` or + `status --id `. +- Testbox status/stop: `--id`. No status `--json`. +- Delegated provider rejects `--fresh-pr`, `--full-resync`, `--script*`, + `--env-helper`, capture/download flags. + +Autoreview parallel tests: + +- Current helper: short POSIX test home. Nothing extra. +- Old helper + macOS `ControlPath too long`: put `TMPDIR=/tmp` on outer process. + +```sh +TMPDIR=/tmp OPENCLAW_TESTBOX=1 "$AUTOREVIEW" \ + --parallel-tests "pnpm check:changed" +``` + +- Do not put `TMPDIR` inside quoted test command. Home already created. + +## Untrusted AWS + +Clean trusted `main` checkout. Installed trusted Crabbox binary. Fresh lease per +reviewed full head SHA. No instance role. No Tailscale. No hydration. Only `CI` +forwarded. Trusted bootstrap uploaded beside `--fresh-pr`. + +```sh +cd env -u CRABBOX_AWS_INSTANCE_PROFILE \ - crabbox config show --json | \ + "$CRABBOX" config show --json | \ jq -e '.aws.instanceProfile == ""' >/dev/null + env -u CRABBOX_AWS_INSTANCE_PROFILE \ -u CRABBOX_TAILSCALE \ -u CRABBOX_TAILSCALE_AUTH_KEY \ @@ -105,727 +144,209 @@ env -u CRABBOX_AWS_INSTANCE_PROFILE \ -u CRABBOX_TAILSCALE_EXIT_NODE_ALLOW_LAN_ACCESS \ -u CRABBOX_TAILSCALE_HOSTNAME_TEMPLATE \ -u CRABBOX_TAILSCALE_TAGS \ - crabbox warmup \ - --provider aws \ - --network public \ - --tailscale=false \ + "$CRABBOX" warmup \ + --provider aws --network public --tailscale=false \ --tailscale-exit-node= \ --tailscale-exit-node-allow-lan-access=false \ - --keep \ - --timing-json -crabbox inspect --provider aws --id --json | \ + --keep --timing-json + +"$CRABBOX" inspect --provider aws --id --json | \ jq -e '.network == "public" and .tailscale == null' >/dev/null + env -u CRABBOX_AWS_INSTANCE_PROFILE \ CRABBOX_ENV_ALLOW=CI \ - crabbox run \ - --provider aws \ - --id \ + "$CRABBOX" run \ + --provider aws --id \ --fresh-pr \ - --no-hydrate \ - --timing-json \ + --no-hydrate --timing-json \ --script scripts/crabbox-untrusted-bootstrap.sh -- \ - /usr/local/bin/pnpm test -# After all proof: + /usr/local/bin/pnpm test + env -u CRABBOX_AWS_INSTANCE_PROFILE \ - crabbox stop --provider aws + "$CRABBOX" stop --provider aws ``` -- Always report the actual provider and id. `cbx_...` means AWS Crabbox; - `tbx_...` means Blacksmith Testbox through Crabbox. If the output only says - `blacksmith testbox list`, use `blacksmith testbox list --all` before - concluding no box exists. -- If a warm direct-provider lease smells stale, retry with `--full-resync` - (alias `--fresh-sync`) before replacing the lease. This resets the remote - workdir, skips the fingerprint fast path, reseeds Git when possible, and - uploads the checkout from scratch. -- For live/provider bugs, use the configured secret workflow before downgrading - to mocks. Copy only the exact needed key into the remote process environment - for that one command. Do not print it, do not sync it as a repo file, and do - not leave it in remote shell history or logs. If no secret-safe injection path - is available, say true live provider auth is blocked instead of silently using - a fake key. -- Run one/few targeted edit-loop tests locally with - `node scripts/run-vitest.mjs` when the existing dependency install is ready. - If the proof fans out, becomes expensive, lacks ready dependencies, or needs - OS/package/Docker/service behavior, acquire a remote box selected by source - trust. -- Do not treat inherited shell env as operator intent. In particular, - `OPENCLAW_LOCAL_CHECK_MODE=throttled` from the local shell is not permission - to move broad `pnpm check:changed`, `pnpm test:changed`, full `pnpm test`, or - lint/typecheck fan-out onto the laptop. -- Only use `OPENCLAW_LOCAL_CHECK_MODE=throttled|full` when the user explicitly - asks for local proof in the current task. If Testbox is queued or capacity is - constrained, report the blocker; do not silently move heavy work onto the - laptop. +Bootstrap proves IMDSv2 IAM credential endpoint returns 404, verifies full SHA, +unsets `NODE_OPTIONS`, pins Node/pnpm, checks package-manager pin, isolates +`HOME`, installs, tests. -## macOS And Windows Targets +Head moved? Stop. Rewarm. No reuse across revisions. No remote PR or no-role +proof unavailable? Secretless fork CI. No exceptions. -Use these only when the task needs an existing non-Linux host. OpenClaw broad -Linux validation uses the repo Crabbox config unless a provider is explicitly -requested. +## Direct AWS -Native brokered Windows is available for Windows-specific proof. Prefer Azure -for Windows/WSL2 when the subscription has quota or credits and the local -Crabbox binary advertises Azure. Keep broad Linux gates on Linux/Testbox unless -the bug is Windows-specific, and only force AWS when the operator asks for the -older AWS developer image/cache path or Azure is unavailable: +Trusted direct run: ```sh -pnpm crabbox:warmup -- \ - --target windows \ - --windows-mode wsl2 \ - --timing-json -``` - -The hydrate workflow assumes Docker should already be baked into Linux images -and only installs it as a fallback. Do not add per-run Docker installs to proof -commands unless the image probe shows Docker is actually missing. - -When the user explicitly asks for brokered macOS runners, use Crabbox AWS -macOS only after confirming the deployed coordinator supports EC2 Mac host -lifecycle/image routes and the operator has AWS EC2 Mac Dedicated Host quota -and IAM. Prefer `CRABBOX_HOST_ID` for a known Crabbox-managed Dedicated Host, -or run the no-spend preflight first: - -```sh -crabbox admin hosts quota --provider aws --target macos --region eu-west-1 --type mac2.metal --json -crabbox admin hosts allocate --provider aws --target macos --region eu-west-1 --type mac2.metal --dry-run --json -CRABBOX_MACOS_TYPES=all scripts/macos-host-region-preflight.sh -``` - -Do not silently substitute AWS macOS for normal OpenClaw Linux proof. Report -paid-host blockers as quota, IAM, coordinator deployment, or host availability -instead of falling back to local macOS. - -Crabbox supports static SSH targets: - -```sh -../crabbox/bin/crabbox run --provider ssh --target macos --static-host mac-studio.local -- xcodebuild test -../crabbox/bin/crabbox run --provider ssh --target windows --windows-mode normal --static-host win-dev.local -- pwsh -NoProfile -Command "dotnet test" -../crabbox/bin/crabbox run --provider ssh --target windows --windows-mode wsl2 --static-host win-dev.local -- pnpm test -``` - -- `target=macos` and `target=windows --windows-mode wsl2` use the POSIX SSH, - bash, Git, rsync, and tar contract. -- Native Windows uses OpenSSH, PowerShell, Git, and tar; sync is manifest tar - archive transfer into `static.workRoot`. Direct native Windows runs support - `--script*`, `--env-from-profile`, `--preflight`, and PowerShell `--shell`. -- `crabbox actions hydrate/register` are Linux-only today; use plain - `crabbox run` loops for static macOS and Windows hosts. -- Live proof needs a reachable, operator-managed SSH host. Without one, verify - with `../crabbox/bin/crabbox run --help`, config/flag tests, and the Crabbox - Go test suite. - -## Direct Brokered AWS Backend - -Use this when the task needs direct AWS Crabbox semantics rather than the -prepared Blacksmith Testbox CI environment. - -Changed gate: - -```sh -pnpm crabbox:run -- \ +node scripts/crabbox-wrapper.mjs run \ --provider aws \ - --idle-timeout 90m \ - --ttl 240m \ - --timing-json \ + --idle-timeout 90m --ttl 240m --timing-json \ --shell -- \ "pnpm test:changed" ``` -Full suite: +Focused: ```sh -pnpm crabbox:run -- \ - --provider aws \ - --idle-timeout 90m \ - --ttl 240m \ - --timing-json \ - --shell -- \ - "pnpm verify" -``` - -Use `pnpm verify` when you need check plus full Vitest proof. It emits -`CRABBOX_PHASE:check` and `CRABBOX_PHASE:test`, making Crabbox summaries show -which stage failed. Use plain `pnpm test` only when check proof is already -covered or intentionally skipped. - -Focused rerun: - -```sh -pnpm crabbox:run -- \ - --provider aws \ - --idle-timeout 90m \ - --ttl 240m \ - --timing-json \ - --shell -- \ +node scripts/crabbox-wrapper.mjs run \ + --provider aws --timing-json --shell -- \ "pnpm test " ``` -Read the JSON summary. Useful fields: +Stale sync: retry `--full-resync` once. Still bad: fresh lease. One-shot should +stop itself; after failure/interruption verify `"$CRABBOX" list --provider aws`. -- `provider`: `aws` -- `leaseId`: `cbx_...` -- `syncDelegated`: `false` -- `commandPhases`: populated when the command prints `CRABBOX_PHASE:` -- `commandMs` / `totalMs` -- `exitCode` - -Crabbox should stop one-shot AWS leases automatically after the run. Verify -cleanup when a run fails, is interrupted, or the command output is unclear: +Broker auth, not cloud keys: ```sh -../crabbox/bin/crabbox list --provider aws +"$CRABBOX" config show +"$CRABBOX" doctor +"$CRABBOX" whoami +"$CRABBOX" login --url https://crabbox.openclaw.ai --provider aws ``` -## Blacksmith Testbox Through Crabbox +Normal validation asking for AWS keys usually means wrong path. -Use this for OpenClaw maintainer broad/heavy `pnpm` gates when the prepared CI -environment is the right proof surface: +## Fresh PR / Container + +`--fresh-pr `: clean remote checkout. Add `--apply-local-patch` +only for intentional local fixup. Direct providers only. + +No remote provider? Local Docker fallback: ```sh node scripts/crabbox-wrapper.mjs run \ - --provider blacksmith-testbox \ - --blacksmith-org openclaw \ - --blacksmith-workflow .github/workflows/ci-check-testbox.yml \ - --blacksmith-job check \ - --blacksmith-ref main \ - --idle-timeout 90m \ - --ttl 240m \ - --timing-json \ - -- \ - corepack pnpm check:changed + --provider local-container \ + --local-container-image node:24-bookworm \ + --no-hydrate --fresh-pr openclaw/openclaw#123 \ + --timing-json --shell -- \ + "corepack pnpm install --frozen-lockfile --store-dir .pnpm-store && \ + corepack pnpm test " ``` -Read the JSON summary and the Testbox line. Useful fields: +Report `local-container`; not AWS/Testbox. `ERR_PNPM_EXDEV`: keep `--no-hydrate` +and repo-local store. -- `provider`: `blacksmith-testbox` -- `leaseId`: `tbx_...` -- `syncDelegated`: `true` -- `syncPhases`: delegated/skipped because Blacksmith owns checkout/sync -- Actions run URL/id from the Testbox output -- `exitCode` +## Observability -Use provider-backed cache volumes only for rebuildable caches, not secrets or -checkout state. On Blacksmith, Crabbox forwards them as sticky disks: +Prefer built-ins: + +- `--preflight`: target/workspace/tool probes. +- `--debug --timing-json`: sync, command, total timing. +- `--script ` / `--script-stdin`: safe multiline direct-provider command. +- `--allow-env NAME` + `--env-from-profile `: exact direct-provider env. +- `CRABBOX_ENV_ALLOW=NAME,...`: exact ambient env allowlist. +- `--capture-stdout`, `--capture-stderr`: direct-provider local capture. +- `--capture-on-fail`: test artifacts. Treat as secret-bearing until reviewed. +- `--keep-on-failure`: retain failed lease for debugging. +- `--results-auto` / `--junit `: structured failure digest. +- `CRABBOX_PHASE:` lines: phase timing. + +Secrets: exact key only. One command. Never print. Never repo file. Never shell +history. No safe injection path? Report live auth blocked. No fake-key upgrade to +“live proof.” + +## Real E2E + +“Test in Crabbox” means user path, not merely remote unit tests. + +1. Reproduce entrypoint when feasible. +2. Patch. Narrow local test. +3. Remote install/update/onboard/Gateway/channel/agent-turn path. +4. Record provider, id, command, environment shape, redacted secret source, + observed result. +5. Cleanup. + +Route: + +- Install/package: pack tarball; install like user; matching Docker/package lane. +- Provider/auth: real provider. Scrub unrelated provider vars. +- Channel: setup, config, Gateway, send/receive, redacted logs. +- Gateway/session/tool: real CLI or RPC; inspect state/API result. +- Parser/config: focused tests enough only when OS/package/service cannot matter. + +Before/after: same Testbox when practical. Detached temp worktrees under `/tmp`. +Never checkout refs in synced root. Full-screen CLI: real PTY. Interactive Clack: +exact arrows/Enter; raw search typing can lie. + +Isolate mutable state: `OPENCLAW_STATE_DIR=$(mktemp -d)`. Test-only local plugin +artifacts may use `OPENCLAW_ALLOW_PLUGIN_INSTALL_OVERRIDES=1`; never call them +official/trusted installs. + +## Desktop / Cross-OS + +Static hosts: ```sh -node scripts/crabbox-wrapper.mjs run \ - --provider blacksmith-testbox \ - --cache-volume pnpm-store=openclaw-node24-pnpm-lock:/tmp/openclaw-pnpm-store \ - --timing-json \ - -- \ - corepack pnpm check:changed +"$CRABBOX" run --provider ssh --target macos \ + --static-host mac-studio.local -- xcodebuild test +"$CRABBOX" run --provider ssh --target windows --windows-mode normal \ + --static-host win-dev.local -- pwsh -NoProfile -Command "dotnet test" +"$CRABBOX" run --provider ssh --target windows --windows-mode wsl2 \ + --static-host win-dev.local -- pnpm test ``` -The selected provider must advertise cache-volume support. If not, omit -`--cache-volume` and rely on kept-lease caches. +Windows/WSL2: prefer Azure when advertised/configured. Native Windows uses +OpenSSH + PowerShell + Git + tar. Actions hydration Linux-only. -`blacksmith testbox list` may hide hydrating or ready boxes. Use: +Brokered macOS: paid EC2 Mac. First quota/no-spend preflight. No silent +substitution for Linux proof. ```sh -blacksmith testbox list --all -blacksmith testbox status +"$CRABBOX" admin hosts quota --provider aws --target macos \ + --region eu-west-1 --type mac2.metal --json +"$CRABBOX" admin hosts allocate --provider aws --target macos \ + --region eu-west-1 --type mac2.metal --dry-run --json ``` -## Observability Flags - -Use these on debugging runs before inventing ad hoc logging: - -- `--preflight`: prints run context, workspace mode, SSH target, remote user/cwd, - and target-specific tool probes. Defaults cover `git`, `tar`, `node`, `npm`, - `corepack`, `pnpm`, `yarn`, `bun`, `docker`, plus POSIX - `sudo`/`apt`/`bubblewrap` and native Windows - `powershell`/`execution_policy`/`longpaths`/`temp`/`pwsh`. Add - `--preflight-tools node,bun,docker`, `CRABBOX_PREFLIGHT_TOOLS`, or repo - `run.preflightTools` to replace the list. `default` expands built-ins; `none` - prints only the workspace summary. Preflight is diagnostic only; install - toolchains through Actions hydration, images, devcontainer/Nix/mise/asdf, or - the run script. On `blacksmith-testbox`, this prints a delegated-unsupported - note because the workflow owns setup. -- `CRABBOX_ENV_ALLOW=NAME,...`: forwards only listed local env vars for direct - providers and prints `set len=N secret=true` style summaries. On - `blacksmith-testbox`, env forwarding is unsupported; put secrets in the - Testbox workflow instead. -- `--env-from-profile ` plus `--allow-env NAME`: loads simple - `export NAME=value` / `NAME=value` lines from a local profile without - executing it, then forwards only allowlisted names. `--allow-env` is - repeatable and comma-separated. Profile values override ambient allowlisted - env values for that run. Direct POSIX, WSL2, and native Windows runs are - supported; delegated providers are not. Crabbox probes the uploaded profile - remotely and prints redacted presence/length metadata before the command. -- `--env-helper `: with `--env-from-profile` on POSIX SSH targets, - persists `.crabbox/env/` and `.crabbox/env/.env` so follow-up - commands on the same lease can run through `./.crabbox/env/ `. - Use only on leases you control; the profile stays until cleanup, lease reset, - or `--full-resync`. -- `--script ` / `--script-stdin`: upload a local script into - `.crabbox/scripts/` and execute it on the remote box. Shebang scripts execute - directly on POSIX; scripts without a shebang run through `bash`. Native - Windows uploads run through Windows PowerShell, and Crabbox appends `.ps1` - when needed. Arguments after `--` become script args. -- `--fresh-pr owner/repo#123|URL|number`: skip dirty local sync and create a - fresh remote checkout of the GitHub PR. Bare numbers use the current repo's - GitHub origin. Add `--apply-local-patch` only when the current local - `git diff --binary HEAD` should be applied on top of that PR checkout. -- `--full-resync` / `--fresh-sync`: reset a stale direct-provider workdir - before syncing. Use after sync fingerprints look wrong, SSH times out before - sync, or rsync watchdog output suggests it. It is redundant with - `--fresh-pr`, incompatible with `--no-sync`, and unsupported by delegated - providers. -- `--capture-stdout ` / `--capture-stderr `: write remote streams to - local files and keep binary/noisy output out of retained logs. Parent - directories must already exist. These are direct-provider only. -- `--capture-on-fail`: on non-zero direct-provider exits, downloads - `.crabbox/captures/*.tar.gz` with `test-results`, `playwright-report`, - `coverage`, JUnit XML, and nearby logs. Treat as secret-bearing until reviewed. -- `--keep-on-failure`: leave a failed one-shot lease alive for live debugging - until idle/TTL expiry. Useful on direct providers and delegated one-shots. -- `--timing-json`: final machine-readable timing. Add - `echo CRABBOX_PHASE:install`, `CRABBOX_PHASE:test`, etc. in long shell - commands; direct providers and Blacksmith Testbox both report them as - `commandPhases`. - -Live-provider debug template for direct AWS/Hetzner leases: +Human desktop: WebVNC preferred. ```sh -mkdir -p .crabbox/logs -pnpm crabbox:run -- --provider aws \ - --preflight \ - --allow-env OPENAI_API_KEY,OPENAI_BASE_URL \ - --timing-json \ - --capture-stdout .crabbox/logs/live-provider.stdout.log \ - --capture-stderr .crabbox/logs/live-provider.stderr.log \ - --capture-on-fail \ - --shell -- \ - "echo CRABBOX_PHASE:install; pnpm install --frozen-lockfile; echo CRABBOX_PHASE:test; pnpm test:live" +"$CRABBOX" warmup --provider hetzner --desktop --browser --keep +"$CRABBOX" desktop launch --provider hetzner --id \ + --browser --url https://example.com --webvnc --open --take-control +"$CRABBOX" desktop doctor --provider hetzner --id +"$CRABBOX" webvnc status --provider hetzner --id +"$CRABBOX" artifacts collect --id --all --output artifacts/ ``` -Do not pass `--capture-*`, `--download`, `--checksum`, `--force-sync-large`, or -`--sync-only` to delegated providers. Also do not pass `--script*`, -`--fresh-pr`, `--full-resync`, or `--env-helper` there. Crabbox rejects these -because the provider owns sync or command transport. `--keep-on-failure` is OK -for delegated one-shots when you need to inspect a failed lease. - -## Efficient Bug E2E Verification - -Use the smallest Crabbox lane that proves the reported user path, not just the -touched code. Aim for one after-fix E2E proof before commenting, closing, or -opening a PR for a user-visible bug. - -When the user says "test in Crabbox", do not simply copy tests to the remote -box and run them there. Crabbox is for remote real-scenario proof: copy or -install OpenClaw as the user would, run the same setup/update/CLI/Gateway/API -call that failed, and capture behavior from that entrypoint. For regressions or -bug reports, prove the broken state first when feasible, then run the same -scenario after the fix. - -Pick the lane by symptom: - -- Docker/setup/install bug: build a package tarball and run the matching - `scripts/e2e/*-docker.sh` or package script. This proves npm packaging, - install paths, runtime deps, config writes, and container behavior. -- Provider/model/auth bug: prefer true live E2E. Use the configured secret - workflow, then inject the single needed key into Crabbox if needed. Scrub - unrelated provider env vars in the child command so interactive defaults do - not drift to another provider. If only a dummy key is used, label the proof - narrowly, e.g. "UI/install path only; live provider auth not exercised." -- Channel delivery bug: use the channel Docker/live lane when available; include - setup, config, gateway start, send/receive or agent-turn proof, and redacted - logs. -- Gateway/session/tool bug: prefer an end-to-end CLI or Gateway RPC command that - creates real state and inspects the resulting files/API output. -- Pure parser/config bug: targeted tests may be enough, but still run a - Crabbox command when OS, package, Docker, secrets, or service lifecycle could - change behavior. - -Efficient flow: - -1. Reproduce or prove the pre-fix symptom from the real user-facing entrypoint - when feasible. If the issue cannot be reproduced, capture the exact command - and observed behavior instead. -2. For trusted source, patch locally and run narrow tests locally when - dependencies are ready and the proof remains bounded. Never run untrusted - repository tooling locally. -3. Lazily acquire one Crabbox when heavy proof is needed, then run an E2E - command that starts from the user-facing entrypoint: - package install, Docker setup, onboarding, channel add, gateway start, or - agent turn as appropriate. -4. Record proof as: Testbox id, command, environment shape, redacted secret - source, and copied success/failure output. -5. If the issue says "cannot reproduce", ask for the missing config/log fields - that would distinguish the tested path from the reporter's path. - -Keep it efficient: - -- Reuse existing E2E scripts and helper assertions before writing ad hoc shell. -- Use `--script ` or `--script-stdin` for multi-line E2E commands instead - of quote-heavy `--shell` strings on direct SSH providers. -- Use `--fresh-pr ` when validating an upstream PR in isolation from the - local dirty tree. Add `--apply-local-patch` only when testing a local fixup on - top of that PR. -- Use `--full-resync` before replacing a warmed direct-provider lease when the - remote workdir or sync fingerprint appears stale. -- After the first heavy proof acquires a remote box, reuse it across later - heavy commands. Use a one-shot when a single late proof is the task's only - remote command. -- Prefer `OPENCLAW_CURRENT_PACKAGE_TGZ` with Docker/package lanes when testing a - candidate tarball; prefer the repo's package helper instead of direct source - execution when the bug might be packaging/install related. -- Keep secrets redacted. It is fine to report key presence, source, and length; - never print secret values. -- Include `--timing-json` on broad or flaky runs when command duration or sync - behavior matters. - -Before/after PR proof on delegated Testbox: - -- For PRs that should prove "broken before, fixed after", compare base and PR - on the same Testbox when practical. Fetch both refs, create detached temp - worktrees under `/tmp`, install in each, then run the same harness twice. -- Do not checkout base/PR refs in the synced repo root. Delegated Testbox sync - may leave the root dirty with local files; `git checkout` can abort or mix - proof state. -- Temp harness files under `/tmp` do not resolve repo packages by default. Put - the harness inside the worktree, or in ESM use - `createRequire(path.join(process.cwd(), "package.json"))` before requiring - workspace deps such as `@lydell/node-pty`. -- For full-screen TUI/CLI bugs, a PTY harness is stronger than helper-only - assertions. Use a real PTY, wait for visible lifecycle markers, send input, - then send control keys and assert process exit/stuck behavior. -- When validating a rebased local branch before push, remember delegated sync - usually validates synced file content on a detached dirty checkout, not a - remote commit object. Record the local head SHA, changed files, Testbox id, - and final success markers; after pushing, ensure the pushed SHA has the same - file content. -- If GitHub CI is still queued but the exact changed content passed Testbox - `pnpm check:changed`, `pnpm check:test-types`, and the real E2E proof, it is - reasonable to merge once required checks allow it. Note any still-running - unrelated shards in the proof comment instead of waiting forever. - -Interactive CLI/onboarding: - -- For full-screen or prompt-heavy CLI flows, run the target command inside tmux - on the Crabbox and drive it with `tmux send-keys`; capture proof with - `tmux capture-pane`, redacted through `sed`. -- Prefer deterministic arrow navigation over search typing for Clack-style - searchable selects. Raw `send-keys -l openai` may not trigger filtering in a - tmux pane; inspect option order locally or on-box and send exact Down/Enter - sequences. -- Isolate mutable state with `OPENCLAW_STATE_DIR=$(mktemp -d)`. Plugin npm - installs live under that state dir (`npm/node_modules/...`), not under - `OPENCLAW_CONFIG_DIR`. Verify downloads by checking the state dir, package - lock, and installed package metadata. -- To test automatic setup installs against local package artifacts, use - `OPENCLAW_ALLOW_PLUGIN_INSTALL_OVERRIDES=1` plus - `OPENCLAW_PLUGIN_INSTALL_OVERRIDES='{"plugin-id":"npm-pack:/tmp/plugin.tgz"}'`. - Pack with `npm pack`, set an isolated `OPENCLAW_STATE_DIR`, and verify the - package under `npm/node_modules`. Overrides are test-only and must not be - treated as official/trusted-source installs. -- For OpenAI/Codex onboarding proof, the useful markers are the UI line - `Installed Codex plugin`, `npm/node_modules/@openclaw/codex`, and the - package-lock entry showing the bundled `@openai/codex` dependency. A dummy - OpenAI-shaped key can prove only UI/install behavior; it is not live auth. - -## Reuse And Keepalive - -Agent code tasks should acquire one remote box lazily when the first heavy -proof is ready, then reuse it for later heavy commands. One-shot runs remain -appropriate for a single late proof. - -Reuse the lease, not stale source. Each command must sync the current checkout; -use `--no-sync` only to rerun an unchanged, already-synced tree intentionally. -Untrusted reuse still requires `CRABBOX_ENV_ALLOW=CI`, -`--no-hydrate`, and a fresh temporary remote `HOME` on every command. Reuse -only a fresh lease dedicated to the same untrusted source; never a trusted or -previously hydrated lease. Launch from the clean trusted `main` checkout and -use `--fresh-pr` plus the same reviewed-SHA check on every run. Keep -`CRABBOX_AWS_INSTANCE_PROFILE` unset for warmup, run, and cleanup. The lease is -valid only for that reviewed SHA; stop and rewarm after any head change. - -If Crabbox returns a reusable id or you intentionally keep a lease: +Before handoff, prove CLI/app from neutral `~`: ```sh -node scripts/crabbox-wrapper.mjs run --provider --id --timing-json --shell -- "corepack pnpm test " +"$CRABBOX" run --id --shell -- \ + "cd ~ && command -v && --version" ``` -Stop boxes you created before handoff: - -```sh -pnpm crabbox:stop -- -blacksmith testbox stop --id -``` - -## Interactive Desktop And WebVNC - -Prefer WebVNC for human inspection because the browser portal can preload the -lease VNC password and avoids a native VNC client's copy/paste/password dance. -Use native `crabbox vnc` only when WebVNC is unavailable, the browser portal is -broken, or the user explicitly wants a local VNC client. - -Common desktop flow: - -```sh -../crabbox/bin/crabbox warmup --provider hetzner --desktop --browser --class standard --idle-timeout 60m --ttl 240m -../crabbox/bin/crabbox desktop launch --provider hetzner --id --browser --url https://example.com --webvnc --open --take-control -``` - -Useful WebVNC commands: - -```sh -../crabbox/bin/crabbox webvnc --provider hetzner --id --open --take-control -../crabbox/bin/crabbox webvnc daemon start --provider hetzner --id --open --take-control -../crabbox/bin/crabbox webvnc daemon status --provider hetzner --id -../crabbox/bin/crabbox webvnc daemon stop --provider hetzner --id -../crabbox/bin/crabbox webvnc status --provider hetzner --id -../crabbox/bin/crabbox webvnc reset --provider hetzner --id --open --take-control -../crabbox/bin/crabbox desktop doctor --provider hetzner --id -../crabbox/bin/crabbox desktop click --provider hetzner --id --x 640 --y 420 -../crabbox/bin/crabbox desktop paste --provider hetzner --id --text "user@example.com" -../crabbox/bin/crabbox desktop key --provider hetzner --id ctrl+l -../crabbox/bin/crabbox artifacts collect --id --all --output artifacts/ -../crabbox/bin/crabbox artifacts publish --dir artifacts/ --pr -``` - -`desktop launch --webvnc --open` is usually the nicest one-shot: it starts the -browser/app inside the visible session, bridges the lease into the authenticated -WebVNC portal, and opens the portal. Keep browsers windowed for human QA; use -`--fullscreen` only for capture/video workflows. -For human handoff, include `--take-control` so the opened portal viewer gets -keyboard/mouse control automatically instead of landing as an observer. - -Human handoff preflight: - -- Do not assume a visible desktop or launched browser means the repo CLI/app is - installed, built, or on the interactive terminal's `PATH`. -- Before handing WebVNC to a human tester, prove the expected command from the - same kept lease and from a neutral directory such as `~`. -- If the handoff needs repo-local code, sync/build/link it explicitly on that - lease. Source-tree CLIs often need build output before a symlink works. -- Prefer a real `command -v && --version` - check over a repo-root-only `pnpm ...` command. - -Generic handoff repair pattern: - -```sh -../crabbox/bin/crabbox run --id --full-resync --shell -- \ - "set -euo pipefail - pnpm install --frozen-lockfile - pnpm build - sudo ln -sf \"\$PWD/\" /usr/local/bin/ - cd ~ - command -v - --version" -``` - -## If Crabbox Fails - -Keep the fallback narrow. First decide whether the failure is Crabbox itself, -the brokered AWS lease, Blacksmith/Testbox, repo hydration, sync, or the test -command. - -Fast checks: - -```sh -command -v crabbox -../crabbox/bin/crabbox --version -pnpm crabbox:run -- --help | sed -n '1,140p' -../crabbox/bin/crabbox doctor -command -v blacksmith -blacksmith --version -blacksmith testbox list -``` - -Common Crabbox-only failures: - -- Provider missing or old CLI: use `../crabbox/bin/crabbox` from the sibling - repo, or update/install Crabbox before retrying. -- Bad local config: inspect `.crabbox.yaml`, `crabbox config show`, and - `crabbox whoami`; normal OpenClaw agent proof should use Blacksmith Testbox. - Direct AWS is an explicit fallback and must use brokered auth, not raw keys. -- Slug/claim confusion: use the raw `cbx_...` / `tbx_...` id, or run one-shot - without `--id`. -- Sync/timing bug: add `--debug --timing-json`; capture the final JSON and the - printed Actions URL. Large sync warnings now include top source directories - by file count and a hint to update `.crabboxignore` / `sync.exclude`; inspect - those before reaching for `--force-sync-large`. Quiet rsync watchdogs and SSH - timeouts now print `next_action=` hints; follow them, usually `--full-resync` - first and a fresh lease second. -- Cleanup uncertainty: run `crabbox list --provider aws`; for explicit - Blacksmith runs, use `blacksmith testbox list` and stop only boxes you - created. -- Testbox queued/capacity pressure: do not retry Blacksmith repeatedly. Rerun - once with `--provider aws` when direct AWS still proves the requested - surface, or report the Blacksmith blocker if Testbox itself is required. - -If brokered AWS cannot dispatch, sync, attach, or stop, retry once with -`--debug` and `--timing-json`: - -```sh -pnpm crabbox:run -- --provider aws --debug --timing-json -- \ - pnpm test:changed -``` - -Full suite: - -```sh -pnpm crabbox:run -- --provider aws --debug --timing-json -- \ - pnpm test -``` - -Auth fallback, only when `blacksmith` says auth is missing: - -```sh -blacksmith auth login --non-interactive --organization openclaw -``` - -Raw Blacksmith footguns: - -- Run from repo root. The CLI syncs the current directory. -- Save the returned `tbx_...` id in the session. -- Reuse that id for focused reruns; stop it before handoff. -- Raw commit SHAs are not reliable `warmup --ref` refs; use a branch or tag. -- Treat `blacksmith testbox list` as cleanup diagnostics, not a shared reusable - queue. - -Use Blacksmith Testbox through Crabbox by default for OpenClaw agent tests and -heavy work. If Blacksmith is down or quota-limited, do not keep probing it; -switch to direct AWS only when that backend proves the same surface, and note -the delegated-provider outage. - -## Blacksmith Backend Notes - -Crabbox Blacksmith backend delegates setup to: - -- org: `openclaw` -- workflow: `.github/workflows/ci-check-testbox.yml` -- job: `check` -- ref: `main` unless testing a branch/tag intentionally - -The hydration workflow owns checkout, Node/pnpm setup, dependency install, -secrets, ready marker, and keepalive. Crabbox owns dispatch, sync, SSH command -execution, timing, logs/results, cleanup, and cache-volume requests. Blacksmith -implements cache volumes as sticky disks. - -Minimal Blacksmith-backed Crabbox run, from repo root: - -```sh -pnpm crabbox:run -- --provider blacksmith-testbox --timing-json -- \ - corepack pnpm test:changed -``` - -Use direct Blacksmith only when Crabbox is the broken layer and you are -isolating a Crabbox bug. Prefer direct `blacksmith testbox list` for cleanup -diagnostics, not as a reusable work queue. - -Important Blacksmith footguns: - -- Always run from repo root. The CLI syncs the current directory. -- Raw commit SHAs are not reliable `warmup --ref` refs; use a branch or tag. -- If auth is missing and browser auth is acceptable: - -```sh -blacksmith auth login --non-interactive --organization openclaw -``` - -## Brokered AWS Fallback - -Use direct AWS when Testbox is unavailable, when the task needs direct-provider -semantics, or when an explicit backend comparison is required. The repo -`.crabbox.yaml` defaults to Blacksmith Testbox, so pass `--provider aws`. - -```sh -pnpm crabbox:warmup -- --provider aws --class beast --market on-demand --idle-timeout 90m -pnpm crabbox:hydrate -- --provider aws --id -pnpm crabbox:run -- --provider aws --id --timing-json --shell -- "pnpm test:changed" -pnpm crabbox:stop -- --provider aws -``` - -Install/auth for owned Crabbox if needed: - -```sh -brew install openclaw/tap/crabbox -crabbox login --url https://crabbox.openclaw.ai --provider aws -``` - -New users should self-resolve broker auth before anyone asks for AWS keys: - -```sh -crabbox config show -crabbox doctor -crabbox whoami -``` - -- If broker auth is missing, run `crabbox login --url https://crabbox.openclaw.ai --provider aws`. -- If the CLI asks for `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, or AWS - profile setup during normal OpenClaw validation, assume the agent selected - the wrong path. Use brokered `crabbox login` or an existing brokered lease - before asking the user for cloud credentials. -- Ask for AWS keys only for explicit direct-provider/account administration, - not for normal brokered OpenClaw proof. -- Trusted automation may still use - `printf '%s' "$CRABBOX_COORDINATOR_TOKEN" | crabbox login --url https://crabbox.openclaw.ai --provider aws --token-stdin`. - -macOS config lives at: - -```text -~/Library/Application Support/crabbox/config.yaml -``` - -It should include `broker.url`, `broker.token`, and usually `provider: aws` -for OpenClaw lanes. Let that config drive normal validation. - -### Interactive Desktop / WebVNC - -For human desktop demos, prefer `webvnc` over native `vnc` and keep the remote -desktop visible/windowed. Do not fullscreen the remote browser or hide the XFCE -panel/window chrome unless the explicit goal is video/capture output. After -launch, verify a screenshot shows the desktop panel plus browser title bar. If -Chrome is fullscreen, toggle it back with: - -```sh -crabbox run --id --shell -- 'DISPLAY=:99 xdotool search --onlyvisible --class google-chrome windowactivate key F11' -``` - -## Diagnostics - -```sh -crabbox status --id --wait -crabbox inspect --id --json -crabbox sync-plan -crabbox history --limit 20 -crabbox history --lease -crabbox attach -crabbox events --json -crabbox logs -crabbox results -crabbox cache stats --id -crabbox cache volumes -crabbox ssh --id -blacksmith testbox list -``` - -Use `--debug` on `run` when measuring sync timing. -Use `--timing-json` on warmup, hydrate, and run when comparing backends. -Use `--market spot|on-demand` only on AWS warmup/one-shot runs. +Visible desktop alone proves nothing. Keep browser windowed unless capture task. +Never commit proof assets to product repo. ## Failure Triage -- Crabbox cannot find provider: verify `../crabbox/bin/crabbox --help` lists - the provider selected by `.crabbox.yaml`; update Crabbox before falling back. -- Hydration stuck or failed: open the printed GitHub Actions run URL and inspect - the hydration step. -- Sync failed: rerun with `--debug`; check changed-file count and whether the - checkout is dirty. -- Command failed: rerun only the failing shard/file first. Do not rerun a full - suite until the focused failure is understood. -- Cleanup uncertain: `crabbox list --provider aws`; for explicit Blacksmith - runs, use `blacksmith testbox list` and stop owned `tbx_...` leases you - created. -- Crabbox broken but Blacksmith works: use the direct Blacksmith fallback above, - then file/fix the Crabbox issue. +Identify layer: wrapper, provider, hydration, sync, SSH, command. + +```sh +"$CRABBOX" doctor +"$CRABBOX" status --id --wait +"$CRABBOX" inspect --id --json +"$CRABBOX" history --limit 20 +"$CRABBOX" logs +"$CRABBOX" results +blacksmith testbox list --all +blacksmith testbox status --id +``` + +- Provider/CLI old: use sibling binary; update it. +- Config/auth: `config show`, `doctor`, `whoami`. +- Sync quiet/stale: `--debug --timing-json`, then `--full-resync` once. +- Testbox capacity: no retry storm. Use AWS only if equivalent proof. +- Command failure: read phase, failed test, JUnit, skipped shell segment. Focused + rerun first. +- Cleanup unclear: list exact provider. Stop only owned ids. +- Wrapper broken, Blacksmith healthy: direct Blacksmith only to isolate wrapper. + +Crabbox stop wrapper: no `--timing-json`. ## Boundary -Do not add OpenClaw-specific setup to Crabbox itself. Put repo setup in the -hydration workflow and keep Crabbox generic around lease, sync, command -execution, logs/results, timing, and cleanup. +Crabbox stays generic: lease, sync, command, logs, results, timing, cleanup. +OpenClaw setup belongs hydration workflow/repo scripts.