fix(gateway): avoid macOS startup freeze during worker reconciliation (#111533)

* fix(gateway): warm system CA before worker reconciliation

* fix(gateway): tolerate denied worker warmup

src/daemon/gateway-heap.ts

@@ -1,6 +1,5 @@
 /** Adaptive Node heap policy for the managed Gateway service. */
 import os from "node:os";
-import { parseNodeOptionsTokens } from "../infra/node-options.js";
 
 const MEBIBYTE_BYTES = 1024 * 1024;
 const GATEWAY_HEAP_FLOOR_MIB = 2048;
@@ -66,6 +65,45 @@ function resolveGatewayHeapLimit(params: GatewayHeapMemoryInputs = {}): GatewayH
   };
 }
 
+function parseNodeOptionsTokens(nodeOptions: string): string[] | null {
+  // Match Node's NODE_OPTIONS splitter: space delimiters, double quotes, and
+  // backslash escapes only inside quotes. Other shell quoting does not apply.
+  const tokens: string[] = [];
+  let token = "";
+  let inQuotes = false;
+  let tokenStarted = false;
+  for (let index = 0; index < nodeOptions.length; index += 1) {
+    let char = nodeOptions[index];
+    if (char === "\\" && inQuotes) {
+      index += 1;
+      if (index >= nodeOptions.length) {
+        return null;
+      }
+      char = nodeOptions[index];
+    } else if (char === " " && !inQuotes) {
+      if (tokenStarted) {
+        tokens.push(token);
+        token = "";
+        tokenStarted = false;
+      }
+      continue;
+    } else if (char === '"') {
+      inQuotes = !inQuotes;
+      tokenStarted = true;
+      continue;
+    }
+    token += char;
+    tokenStarted = true;
+  }
+  if (inQuotes) {
+    return null;
+  }
+  if (tokenStarted) {
+    tokens.push(token);
+  }
+  return tokens;
+}
+
 function parseMaxOldSpaceSizeMiB(nodeOptions: string | undefined): number | null {
   if (!nodeOptions) {
     return null;

src/gateway/server-startup-post-attach.test.ts

@@ -1501,42 +1501,6 @@ describe("startGatewayPostAttachRuntime", () => {
     );
   });
 
-  it("warms the macOS system CA cache before channel startup", async () => {
-    let finishWarmup: (() => void) | undefined;
-    const warmSystemCa = vi.fn(
-      () =>
-        new Promise<void>((resolve) => {
-          finishWarmup = resolve;
-        }),
-    );
-    const startChannels = vi.fn(async () => {});
-    const sidecars = startGatewaySidecars({
-      cfg: { hooks: { internal: { enabled: false } } } as never,
-      pluginRegistry: createPostAttachParams().pluginRegistry,
-      defaultWorkspaceDir: "/tmp/openclaw-workspace",
-      deps: {} as never,
-      startChannels,
-      warmSystemCa,
-      log: { warn: vi.fn() },
-      logHooks: {
-        info: vi.fn(),
-        warn: vi.fn(),
-        error: vi.fn(),
-      },
-      logChannels: {
-        info: vi.fn(),
-        error: vi.fn(),
-      },
-    });
-
-    await waitForGatewayTestState(() => expect(warmSystemCa).toHaveBeenCalledTimes(1));
-    expect(startChannels).not.toHaveBeenCalled();
-
-    finishWarmup?.();
-    await sidecars;
-    expect(startChannels).toHaveBeenCalledTimes(1);
-  });
-
   it("starts and reports plugin services after channel startup completes", async () => {
     await withEnvAsync(
       { OPENCLAW_SKIP_CHANNELS: undefined, OPENCLAW_SKIP_PROVIDERS: undefined },
@@ -2208,12 +2172,21 @@ describe("startGatewayPostAttachRuntime", () => {
     expect(startGatewaySidecarsValue).toHaveBeenCalledTimes(1);
   });
 
-  it("reconciles worker placement before starting channels and sidecars", async () => {
+  it("warms the CA cache before worker placement and sidecar startup", async () => {
+    let finishWarmup: (() => void) | undefined;
+    const warmupReady = new Promise<void>((resolve) => {
+      finishWarmup = resolve;
+    });
     let finishReconcile: (() => void) | undefined;
     const reconcileReady = new Promise<void>((resolve) => {
       finishReconcile = resolve;
     });
     const startupOrder: string[] = [];
+    const warmSystemCa = vi.fn(async () => {
+      startupOrder.push("ca-warmup");
+      await warmupReady;
+      startupOrder.push("ca-ready");
+    });
     const workerSidecar = { stop: vi.fn() };
     const startWorkerEnvironmentRuntime = vi.fn(async () => {
       startupOrder.push("worker-reconcile");
@@ -2231,7 +2204,7 @@ describe("startGatewayPostAttachRuntime", () => {
     const onGatewayLifetimeSidecars = vi.fn();
     const unavailableGatewayMethods = new Set<string>(STARTUP_UNAVAILABLE_GATEWAY_METHODS);
 
-    await startGatewayPostAttachRuntime(
+    const runtimePromise = startGatewayPostAttachRuntime(
       {
         ...createPostAttachParams(),
         unavailableGatewayMethods,
@@ -2241,21 +2214,37 @@ describe("startGatewayPostAttachRuntime", () => {
       },
       createPostAttachRuntimeDeps({
         startGatewaySidecars: startGatewaySidecarsValue,
+        warmSystemCa,
       }),
     );
 
+    await waitForGatewayTestState(() => {
+      expect(warmSystemCa).toHaveBeenCalledTimes(1);
+    });
+    expect(startWorkerEnvironmentRuntime).not.toHaveBeenCalled();
+    expect(startGatewaySidecarsValue).not.toHaveBeenCalled();
+    expect(startupOrder).toEqual(["ca-warmup"]);
+
+    finishWarmup?.();
+    await runtimePromise;
     await waitForGatewayTestState(() => {
       expect(startWorkerEnvironmentRuntime).toHaveBeenCalledTimes(1);
     });
     expect(startGatewaySidecarsValue).not.toHaveBeenCalled();
-    expect(startupOrder).toEqual(["worker-reconcile"]);
+    expect(startupOrder).toEqual(["ca-warmup", "ca-ready", "worker-reconcile"]);
     expect([...unavailableGatewayMethods]).toEqual([...STARTUP_UNAVAILABLE_GATEWAY_METHODS]);
 
     finishReconcile?.();
     await waitForGatewayTestState(() => {
       expect(startGatewaySidecarsValue).toHaveBeenCalledTimes(1);
     });
-    expect(startupOrder).toEqual(["worker-reconcile", "worker-ready", "gateway-sidecars"]);
+    expect(startupOrder).toEqual([
+      "ca-warmup",
+      "ca-ready",
+      "worker-reconcile",
+      "worker-ready",
+      "gateway-sidecars",
+    ]);
     expect([...unavailableGatewayMethods]).toEqual([]);
     expect(onGatewayLifetimeSidecars).toHaveBeenCalledWith(expect.arrayContaining([workerSidecar]));
   });
@@ -2557,6 +2546,7 @@ function createPostAttachRuntimeDeps(
     refreshLatestUpdateRestartSentinel: hoisted.refreshLatestUpdateRestartSentinel,
     scheduleGatewayUpdateCheck: hoisted.scheduleGatewayUpdateCheck,
     startGatewaySidecars: vi.fn(async () => ({ pluginServices: null, postReadySidecars: [] })),
+    warmSystemCa: vi.fn(async () => {}),
     startGatewayTailscaleExposure: hoisted.startGatewayTailscaleExposure,
     ...overrides,
   };

src/gateway/server-startup-post-attach.ts

@@ -612,16 +612,9 @@ export async function startGatewaySidecars(params: {
   logChannels: { info: (msg: string) => void; error: (msg: string) => void };
   startupTrace?: GatewayStartupTrace;
   startupOutcomes?: GatewayStartupOutcomeRecorder;
-  warmSystemCa?: typeof warmMacOSSystemCaOffMainThread;
 }) {
   const postReadySidecars: GatewayPostReadySidecarHandle[] = [];
 
-  // Node initializes the process-wide macOS Keychain CA cache synchronously. Warm it in a
-  // worker before any provider TLS so a slow trustd lookup cannot freeze gateway HTTP/WS.
-  await measureStartup(params.startupTrace, "sidecars.system-ca", () =>
-    (params.warmSystemCa ?? warmMacOSSystemCaOffMainThread)({ log: params.log }),
-  );
-
   const internalHooksConfigured = hasConfiguredInternalHooks(params.cfg);
   await measureStartup(params.startupTrace, "sidecars.internal-hooks", async () => {
     try {
@@ -917,6 +910,7 @@ type GatewayPostAttachRuntimeDeps = {
     ...args: Parameters<typeof scheduleGatewayUpdateCheck>
   ) => Awaitable<ReturnType<typeof scheduleGatewayUpdateCheck>>;
   startGatewaySidecars: typeof startGatewaySidecars;
+  warmSystemCa: typeof warmMacOSSystemCaOffMainThread;
   startGatewayTailscaleExposure: (
     ...args: Parameters<typeof startGatewayTailscaleExposure>
   ) => ReturnType<typeof startGatewayTailscaleExposure>;
@@ -931,6 +925,7 @@ const defaultGatewayPostAttachRuntimeDeps: GatewayPostAttachRuntimeDeps = {
   scheduleGatewayUpdateCheck: async (...args) =>
     (await import("../infra/update-startup.js")).scheduleGatewayUpdateCheck(...args),
   startGatewaySidecars,
+  warmSystemCa: warmMacOSSystemCaOffMainThread,
   startGatewayTailscaleExposure: async (...args) =>
     (await import("./server-tailscale.js")).startGatewayTailscaleExposure(...args),
 };
@@ -1074,6 +1069,14 @@ export async function startGatewayPostAttachRuntime(
   },
   runtimeDeps: GatewayPostAttachRuntimeDeps = defaultGatewayPostAttachRuntimeDeps,
 ) {
+  if (!params.minimalTestGateway) {
+    // The HTTP server is already attached, so keep health probes responsive while the worker
+    // resolves Node's effective default CA set before any plugin or worker provider can use TLS.
+    await measureStartup(params.startupTrace, "post-attach.system-ca", () =>
+      runtimeDeps.warmSystemCa({ log: params.log }),
+    );
+  }
+
   let pluginRegistry = params.pluginRegistry;
   let startupPluginsLoaded = false;
   let startupPluginsLoadPromise: Promise<{

src/gateway/system-ca-warmup.test.ts

@@ -11,91 +11,32 @@ describe("warmMacOSSystemCaOffMainThread", () => {
     vi.useRealTimers();
   });
 
-  it.each([
-    ["env only", "darwin", { NODE_USE_SYSTEM_CA: "1" }, [], true],
-    ["dash flag", "darwin", {}, ["--use-system-ca"], true],
-    ["underscore flag", "darwin", {}, ["--use_system_ca"], true],
-    ["equals-form flag", "darwin", {}, ["--use_system_ca=false"], true],
-    ["OpenSSL CA alone", "darwin", {}, ["--use-openssl-ca"], false],
-    [
-      "bundled CA does not suppress env system CA",
-      "darwin",
-      { NODE_USE_SYSTEM_CA: "1" },
-      ["--use-bundled-ca"],
-      true,
-    ],
-    [
-      "dash negation overrides env",
-      "darwin",
-      { NODE_USE_SYSTEM_CA: "1" },
-      ["--no-use-system-ca"],
-      false,
-    ],
-    [
-      "equals-form negation overrides env",
-      "darwin",
-      { NODE_USE_SYSTEM_CA: "1" },
-      ["--no-use-system-ca=false"],
-      false,
-    ],
-    [
-      "underscore negation overrides env",
-      "darwin",
-      { NODE_USE_SYSTEM_CA: "1" },
-      ["--no_use_system_ca"],
-      false,
-    ],
-    [
-      "last conflicting flag enables",
-      "darwin",
-      {},
-      ["--no-use-system-ca", "--use_system_ca"],
-      true,
-    ],
-    [
-      "last conflicting flag disables",
-      "darwin",
-      {},
-      ["--use-system-ca", "--no_use_system_ca"],
-      false,
-    ],
-    ["NODE_OPTIONS flag", "darwin", { NODE_OPTIONS: '"--use_system_ca"' }, [], true],
-    [
-      "NODE_OPTIONS negation overrides env",
-      "darwin",
-      { NODE_USE_SYSTEM_CA: "1", NODE_OPTIONS: "--no-use-system-ca" },
-      [],
-      false,
-    ],
-    [
-      "execArgv overrides NODE_OPTIONS",
-      "darwin",
-      { NODE_OPTIONS: "--use-system-ca" },
-      ["--no-use-system-ca"],
-      false,
-    ],
-    ["system CA disabled", "darwin", { NODE_USE_SYSTEM_CA: "0" }, [], false],
-    ["non-macOS", "linux", { NODE_USE_SYSTEM_CA: "1" }, [], false],
-  ] as const)(
-    "%s: warmup runs iff system CA is effectively enabled on macOS",
-    async (_name, platform, env, execArgv, shouldWarm) => {
-      const worker = new FakeWorker();
-      const createWorker = vi.fn(() => worker);
-      const warmup = warmMacOSSystemCaOffMainThread({
-        platform,
-        env: { ...env },
-        execArgv: [...execArgv],
-        createWorker,
-      });
+  it("lets Node resolve the effective default CA set on macOS", async () => {
+    const worker = new FakeWorker();
+    const createWorker = vi.fn((_source: string) => worker);
+    const warmup = warmMacOSSystemCaOffMainThread({
+      platform: "darwin",
+      env: {},
+      createWorker,
+    });
 
-      if (shouldWarm) {
-        worker.emit("message", { ok: true, certificateCount: 42 });
-      }
-      await warmup;
-      expect(createWorker).toHaveBeenCalledTimes(shouldWarm ? 1 : 0);
-      expect(worker.unref).toHaveBeenCalledTimes(shouldWarm ? 1 : 0);
-    },
-  );
+    worker.emit("message", { ok: true, certificateCount: 42 });
+    await warmup;
+
+    expect(createWorker).toHaveBeenCalledOnce();
+    const workerSource = createWorker.mock.calls[0]?.[0];
+    expect(workerSource).toContain('getCACertificates("default")');
+    expect(workerSource).not.toContain('getCACertificates("system")');
+    expect(worker.unref).toHaveBeenCalledOnce();
+  });
+
+  it("skips the warmup outside macOS", async () => {
+    const createWorker = vi.fn(() => new FakeWorker());
+
+    await warmMacOSSystemCaOffMainThread({ platform: "linux", env: {}, createWorker });
+
+    expect(createWorker).not.toHaveBeenCalled();
+  });
 
   it("waits for the worker while leaving the main event loop available", async () => {
     vi.useFakeTimers();
@@ -103,7 +44,7 @@ describe("warmMacOSSystemCaOffMainThread", () => {
     const log = { warn: vi.fn() };
     const warmup = warmMacOSSystemCaOffMainThread({
       platform: "darwin",
-      env: { NODE_USE_SYSTEM_CA: "1" },
+      env: {},
       warningMs: 10,
       log,
       createWorker: vi.fn(() => worker),
@@ -117,7 +58,7 @@ describe("warmMacOSSystemCaOffMainThread", () => {
 
     expect(mainTurnRan).toBe(true);
     expect(log.warn).toHaveBeenCalledWith(
-      "macOS system CA warmup is still waiting for Keychain trust settings; channel startup remains deferred",
+      "macOS CA warmup is still waiting for default trust settings; gateway post-attach startup remains deferred",
     );
     expect(worker.unref).toHaveBeenCalledOnce();
 
@@ -125,11 +66,31 @@ describe("warmMacOSSystemCaOffMainThread", () => {
     await warmup;
   });
 
+  it("falls back to lazy CA loading when Node denies worker-thread permission", async () => {
+    const permissionError = Object.assign(new Error("worker permission denied"), {
+      code: "ERR_ACCESS_DENIED",
+    });
+    const log = { warn: vi.fn() };
+
+    await warmMacOSSystemCaOffMainThread({
+      platform: "darwin",
+      env: { NODE_USE_SYSTEM_CA: "0" },
+      log,
+      createWorker: vi.fn(() => {
+        throw permissionError;
+      }),
+    });
+
+    expect(log.warn).toHaveBeenCalledWith(
+      "macOS CA warmup skipped because Node denied worker-thread permission; trust settings will load lazily",
+    );
+  });
+
   it("fails closed when the worker cannot populate the cache", async () => {
     const worker = new FakeWorker();
     const warmup = warmMacOSSystemCaOffMainThread({
       platform: "darwin",
-      env: { NODE_USE_SYSTEM_CA: "1" },
+      env: {},
       createWorker: vi.fn(() => worker),
     });
 

src/gateway/system-ca-warmup.ts

@@ -1,7 +1,6 @@
 import type { EventEmitter } from "node:events";
 import { Worker, type WorkerOptions } from "node:worker_threads";
 import { isVitestRuntimeEnv } from "../infra/env.js";
-import { parseNodeOptionsTokens } from "../infra/node-options.js";
 
 const SYSTEM_CA_WARMUP_WARNING_MS = 10_000;
 const SYSTEM_CA_WORKER_SOURCE = String.raw`
@@ -9,7 +8,7 @@ const SYSTEM_CA_WORKER_SOURCE = String.raw`
   const { parentPort } = require("node:worker_threads");
 
   try {
-    const certificateCount = getCACertificates("system").length;
+    const certificateCount = getCACertificates("default").length;
     parentPort.postMessage({ ok: true, certificateCount });
   } catch (error) {
     parentPort.postMessage({
@@ -27,7 +26,6 @@ type SystemCaWarmupWorker = Pick<EventEmitter, "once" | "removeAllListeners"> &
 
 type SystemCaWarmupOptions = {
   env?: NodeJS.ProcessEnv;
-  execArgv?: readonly string[];
   platform?: NodeJS.Platform;
   log?: { warn: (message: string) => void };
   warningMs?: number;
@@ -36,35 +34,6 @@ type SystemCaWarmupOptions = {
 
 type SystemCaWarmupMessage = { ok: true; certificateCount: number } | { ok: false; error: string };
 
-function applySystemCaRuntimeOptions(enabled: boolean, args: readonly string[]): boolean {
-  let result = enabled;
-  for (const arg of args) {
-    const match = /^--(?:(no)[-_])?use[-_]system[-_]ca(?:=.*)?$/u.exec(arg);
-    if (match) {
-      result = match[1] === undefined;
-    }
-  }
-  return result;
-}
-
-function resolveEffectiveSystemCaEnabled(params: {
-  env: NodeJS.ProcessEnv;
-  execArgv: readonly string[];
-}): boolean {
-  // Node v26.5.0 gates macOS Keychain reads solely on effective --use-system-ca /
-  // NODE_USE_SYSTEM_CA selection. --use-bundled-ca is additive, while --use-openssl-ca
-  // cannot coexist with system CA (startup exits 9), so neither overrides this gate.
-  // Scope: reads the env var, NODE_OPTIONS, and execArgv — the paths OpenClaw uses
-  // (the gateway enables system CA via NODE_USE_SYSTEM_CA by default). The experimental
-  // --experimental-config-file `nodeOptions.use-system-ca` source is intentionally not
-  // parsed here; if it is ever the only enabler the warmup simply no-ops and the process
-  // degrades to Node's original lazy (pre-fix) CA load — no regression, just unimproved.
-  const fromEnvironment = params.env.NODE_USE_SYSTEM_CA === "1";
-  const nodeOptions = parseNodeOptionsTokens(params.env.NODE_OPTIONS ?? "");
-  const afterNodeOptions = applySystemCaRuntimeOptions(fromEnvironment, nodeOptions ?? []);
-  return applySystemCaRuntimeOptions(afterNodeOptions, params.execArgv);
-}
-
 function isSystemCaWarmupMessage(value: unknown): value is SystemCaWarmupMessage {
   if (!value || typeof value !== "object" || Array.isArray(value)) {
     return false;
@@ -75,31 +44,50 @@ function isSystemCaWarmupMessage(value: unknown): value is SystemCaWarmupMessage
     : message.ok === false && typeof message.error === "string";
 }
 
-/** Populate Node's process-wide macOS system CA cache without blocking the gateway event loop. */
+function isWorkerPermissionDenied(error: unknown): boolean {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    error.code === "ERR_ACCESS_DENIED"
+  );
+}
+
+/** Warm Node's effective default CA set without blocking the gateway event loop on macOS. */
 export async function warmMacOSSystemCaOffMainThread(
   options: SystemCaWarmupOptions = {},
 ): Promise<void> {
   const env = options.env ?? process.env;
-  const execArgv = options.execArgv ?? process.execArgv;
   const platform = options.platform ?? process.platform;
-  const usesSystemCa = resolveEffectiveSystemCaEnabled({ env, execArgv });
   if (
     platform !== "darwin" ||
-    !usesSystemCa ||
     (options.env === undefined && options.platform === undefined && isVitestRuntimeEnv(env))
   ) {
     return;
   }
 
-  const worker = (
-    options.createWorker ?? ((source, workerOptions) => new Worker(source, workerOptions))
-  )(SYSTEM_CA_WORKER_SOURCE, { eval: true });
+  let worker: SystemCaWarmupWorker;
+  try {
+    worker = (
+      options.createWorker ?? ((source, workerOptions) => new Worker(source, workerOptions))
+    )(SYSTEM_CA_WORKER_SOURCE, { eval: true });
+  } catch (error) {
+    // Node's permission model can deny Worker construction. Fall back to Node's lazy CA
+    // loading instead of turning an optional event-loop warmup into a startup requirement.
+    if (!isWorkerPermissionDenied(error)) {
+      throw error;
+    }
+    options.log?.warn(
+      "macOS CA warmup skipped because Node denied worker-thread permission; trust settings will load lazily",
+    );
+    return;
+  }
 
   await new Promise<void>((resolve, reject) => {
     let settled = false;
     const warningTimer = setTimeout(() => {
       options.log?.warn(
-        "macOS system CA warmup is still waiting for Keychain trust settings; channel startup remains deferred",
+        "macOS CA warmup is still waiting for default trust settings; gateway post-attach startup remains deferred",
       );
     }, options.warningMs ?? SYSTEM_CA_WARMUP_WARNING_MS);
     warningTimer.unref?.();

src/infra/node-options.ts

@@ -1,38 +0,0 @@
-export function parseNodeOptionsTokens(nodeOptions: string): string[] | null {
-  // Match Node's NODE_OPTIONS splitter: space delimiters, double quotes, and
-  // backslash escapes only inside quotes. Other shell quoting does not apply.
-  const tokens: string[] = [];
-  let token = "";
-  let inQuotes = false;
-  let tokenStarted = false;
-  for (let index = 0; index < nodeOptions.length; index += 1) {
-    let char = nodeOptions[index];
-    if (char === "\\" && inQuotes) {
-      index += 1;
-      if (index >= nodeOptions.length) {
-        return null;
-      }
-      char = nodeOptions[index];
-    } else if (char === " " && !inQuotes) {
-      if (tokenStarted) {
-        tokens.push(token);
-        token = "";
-        tokenStarted = false;
-      }
-      continue;
-    } else if (char === '"') {
-      inQuotes = !inQuotes;
-      tokenStarted = true;
-      continue;
-    }
-    token += char;
-    tokenStarted = true;
-  }
-  if (inQuotes) {
-    return null;
-  }
-  if (tokenStarted) {
-    tokens.push(token);
-  }
-  return tokens;
-}