fix(agents): retry transient filesystem races when reading workspace bootstrap files (#100910)

* fix(agents): retry transient filesystem races when reading workspace bootstrap files

* fix(agents): retry transient boundary resolution

---------

Co-authored-by: Vincent Koc <25068+vincentkoc@users.noreply.github.com>

src/agents/workspace.test.ts

@@ -1,6 +1,7 @@
 // Workspace tests cover bootstrap seeding, attestation safety, bootstrap file
 // filtering, and setup-completion state for agent workspaces.
 import { createHash } from "node:crypto";
+import syncFs from "node:fs";
 import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
@@ -890,6 +891,144 @@ describe("loadWorkspaceBootstrapFiles", () => {
       await fs.rm(rootDir, { recursive: true, force: true });
     }
   });
+
+  it("retries a transient bootstrap read instead of dropping the file for the turn", async () => {
+    const tempDir = await makeTempWorkspace("openclaw-workspace-");
+    await writeWorkspaceFile({
+      dir: tempDir,
+      name: DEFAULT_AGENTS_FILENAME,
+      content: "# AGENTS.md\n\nkeep me\n",
+    });
+
+    const originalReadFileSync = syncFs.readFileSync.bind(syncFs);
+    let readFileSyncCalls = 0;
+    let threwTransient = false;
+    const readSpy = vi.spyOn(syncFs, "readFileSync").mockImplementation(((
+      target: unknown,
+      options: unknown,
+    ) => {
+      readFileSyncCalls += 1;
+      if (!threwTransient) {
+        threwTransient = true;
+        throw Object.assign(new Error("Unknown system error -11: read"), {
+          code: "EAGAIN",
+          errno: -11,
+        });
+      }
+      return originalReadFileSync(target as never, options as never);
+    }) as typeof syncFs.readFileSync);
+
+    try {
+      const files = await loadWorkspaceBootstrapFiles(tempDir);
+      expect(threwTransient).toBe(true);
+      // The retry re-opens and reads again, so the failed first read is followed
+      // by at least one more successful read.
+      expect(readFileSyncCalls).toBeGreaterThanOrEqual(2);
+      const agents = files.find((file) => file.name === DEFAULT_AGENTS_FILENAME);
+      expect(agents?.missing).toBe(false);
+      expect(agents?.content).toContain("keep me");
+    } finally {
+      readSpy.mockRestore();
+    }
+  });
+
+  it("marks a bootstrap file missing after transient read retries are exhausted", async () => {
+    const tempDir = await makeTempWorkspace("openclaw-workspace-");
+    await writeWorkspaceFile({
+      dir: tempDir,
+      name: DEFAULT_AGENTS_FILENAME,
+      content: "# AGENTS.md\n",
+    });
+
+    const readSpy = vi.spyOn(syncFs, "readFileSync").mockImplementation((() => {
+      throw Object.assign(new Error("Unknown system error -11: read"), {
+        code: "EAGAIN",
+        errno: -11,
+      });
+    }) as typeof syncFs.readFileSync);
+
+    try {
+      // Unlike the template check, this reader returns an io failure (not a
+      // throw) when the budget is exhausted, so the file surfaces as missing.
+      const files = await loadWorkspaceBootstrapFiles(tempDir);
+      const agents = files.find((file) => file.name === DEFAULT_AGENTS_FILENAME);
+      expect(agents?.missing).toBe(true);
+      expect(agents?.content).toBeUndefined();
+    } finally {
+      readSpy.mockRestore();
+    }
+  });
+
+  it("retries a transient boundary-resolution failure before dropping a bootstrap file", async () => {
+    const tempDir = await makeTempWorkspace("openclaw-workspace-");
+    await writeWorkspaceFile({
+      dir: tempDir,
+      name: DEFAULT_AGENTS_FILENAME,
+      content: "# AGENTS.md\n\nboundary retry\n",
+    });
+
+    const agentsPath = path.join(tempDir, DEFAULT_AGENTS_FILENAME);
+    const originalLstat = syncFs.promises.lstat.bind(syncFs.promises);
+    let agentsLstatAttempts = 0;
+    const lstatSpy = vi.spyOn(syncFs.promises, "lstat").mockImplementation((async (
+      target: unknown,
+      options?: unknown,
+    ) => {
+      if (String(target) === agentsPath && ++agentsLstatAttempts === 1) {
+        throw Object.assign(new Error("Unknown system error -11: lstat"), {
+          code: "EAGAIN",
+          errno: -11,
+        });
+      }
+      return await originalLstat(target as never, options as never);
+    }) as typeof syncFs.promises.lstat);
+
+    try {
+      const files = await loadWorkspaceBootstrapFiles(tempDir);
+      expect(agentsLstatAttempts).toBe(2);
+      const agents = files.find((file) => file.name === DEFAULT_AGENTS_FILENAME);
+      expect(agents?.missing).toBe(false);
+      expect(agents?.content).toContain("boundary retry");
+    } finally {
+      lstatSpy.mockRestore();
+    }
+  });
+
+  it("retries a transient open failure before dropping a bootstrap file", async () => {
+    const tempDir = await makeTempWorkspace("openclaw-workspace-");
+    await writeWorkspaceFile({
+      dir: tempDir,
+      name: DEFAULT_AGENTS_FILENAME,
+      content: "# AGENTS.md\n\nopen retry\n",
+    });
+
+    // openRootFile reports a transient open failure as reason "io"; the reader
+    // must retry it (re-open) rather than drop the file for the turn.
+    const originalOpenSync = syncFs.openSync.bind(syncFs);
+    let threwTransientOpen = false;
+    const openSpy = vi.spyOn(syncFs, "openSync").mockImplementation(((
+      ...args: Parameters<typeof syncFs.openSync>
+    ) => {
+      if (!threwTransientOpen) {
+        threwTransientOpen = true;
+        throw Object.assign(new Error("Unknown system error -11: open"), {
+          code: "EAGAIN",
+          errno: -11,
+        });
+      }
+      return originalOpenSync(...args);
+    }) as typeof syncFs.openSync);
+
+    try {
+      const files = await loadWorkspaceBootstrapFiles(tempDir);
+      expect(threwTransientOpen).toBe(true);
+      const agents = files.find((file) => file.name === DEFAULT_AGENTS_FILENAME);
+      expect(agents?.missing).toBe(false);
+      expect(agents?.content).toContain("open retry");
+    } finally {
+      openSpy.mockRestore();
+    }
+  });
 });
 
 describe("filterBootstrapFilesForSession", () => {

src/agents/workspace.ts

@@ -77,33 +77,58 @@ async function readWorkspaceFileWithGuards(params: {
   filePath: string;
   workspaceDir: string;
 }): Promise<WorkspaceGuardedReadResult> {
-  const opened = await openRootFile({
-    absolutePath: params.filePath,
-    rootPath: params.workspaceDir,
-    boundaryLabel: "workspace root",
-    maxBytes: MAX_WORKSPACE_BOOTSTRAP_FILE_BYTES,
-  });
-  if (!opened.ok) {
-    workspaceFileCache.delete(params.filePath);
-    return opened;
-  }
-
-  const identity = workspaceFileIdentity(opened.stat, opened.path);
-  const cached = workspaceFileCache.get(params.filePath);
-  if (cached && cached.identity === identity) {
-    syncFs.closeSync(opened.fd);
-    return { ok: true, content: cached.content };
-  }
-
   try {
-    const content = syncFs.readFileSync(opened.fd, "utf-8");
-    workspaceFileCache.set(params.filePath, { content, identity });
-    return { ok: true, content };
+    // A transient FS race (EAGAIN/EWOULDBLOCK/EINTR under load) on the open or
+    // read must not drop the agent's bootstrap file for the turn — this reader
+    // runs every turn for AGENTS/SOUL/HEARTBEAT/etc. Retry the whole open+read so
+    // each attempt uses a fresh fd (retrying readFileSync on the same fd could
+    // return truncated content after a partial read); the inode-identity guard
+    // in openRootFile still protects against a swapped file between attempts.
+    return await retryAsync(
+      async () => {
+        const opened = await openRootFile({
+          absolutePath: params.filePath,
+          rootPath: params.workspaceDir,
+          boundaryLabel: "workspace root",
+          maxBytes: MAX_WORKSPACE_BOOTSTRAP_FILE_BYTES,
+        });
+        if (!opened.ok) {
+          // Boundary resolution can report transient IO as "validation", while
+          // pinned open failures use "io". Classify the underlying error so
+          // deterministic path and validation failures still return unchanged.
+          if (isTransientWorkspaceReadError(opened.error)) {
+            throw opened.error;
+          }
+          workspaceFileCache.delete(params.filePath);
+          return opened;
+        }
+
+        const identity = workspaceFileIdentity(opened.stat, opened.path);
+        const cached = workspaceFileCache.get(params.filePath);
+        if (cached && cached.identity === identity) {
+          syncFs.closeSync(opened.fd);
+          return { ok: true, content: cached.content };
+        }
+
+        try {
+          const content = syncFs.readFileSync(opened.fd, "utf-8");
+          workspaceFileCache.set(params.filePath, { content, identity });
+          return { ok: true, content };
+        } finally {
+          syncFs.closeSync(opened.fd);
+        }
+      },
+      {
+        attempts: 3,
+        minDelayMs: 50,
+        maxDelayMs: 50,
+        shouldRetry: (err) => isTransientWorkspaceReadError(err),
+      },
+    );
   } catch (error) {
+    // Non-transient read failure, or transient retries exhausted.
     workspaceFileCache.delete(params.filePath);
     return { ok: false, reason: "io", error };
-  } finally {
-    syncFs.closeSync(opened.fd);
   }
 }