fix(gateway): surface systemd start-limit exhaustion (#98291)

* fix(gateway): surface systemd start-limit exhaustion

* fix(gateway): avoid start-limit false positive for exit 78
This commit is contained in:
Masato Hoshino
2026-07-01 01:01:15 -07:00
committed by GitHub
parent a9945d1e7d
commit 76f9b6d837
8 changed files with 450 additions and 10 deletions
+61
View File
@@ -727,4 +727,65 @@ describe("printDaemonStatus", () => {
expect(errors).not.toContain("systemd user services unavailable");
expect(errors).not.toContain("run loginctl enable-linger");
});
it("warns that systemd gave up restarting a crash-looped gateway", () => {
const platform = vi.spyOn(process, "platform", "get").mockReturnValue("linux");
try {
printDaemonStatus(
{
service: {
label: "systemd",
loaded: true,
loadedText: "loaded",
notLoadedText: "not loaded",
runtime: {
status: "stopped",
state: "failed",
systemd: { result: "exit-code", nRestarts: 5, startLimitBurst: 5 },
},
},
extraServices: [],
},
{ json: false },
);
} finally {
platform.mockRestore();
}
const errors = runtime.error.mock.calls.map(([line]) => line).join("\n");
expect(errors).toContain("systemd stopped restarting the gateway after repeated crashes");
expect(errors).not.toContain("likely exited immediately");
});
it("keeps the generic stopped message after a config exit (78) despite a stale restart count", () => {
// RestartPreventExitStatus=78 stopped systemd deliberately; a leftover
// NRestarts must not surface start-limit recovery guidance here.
const platform = vi.spyOn(process, "platform", "get").mockReturnValue("linux");
try {
printDaemonStatus(
{
service: {
label: "systemd",
loaded: true,
loadedText: "loaded",
notLoadedText: "not loaded",
runtime: {
status: "stopped",
state: "failed",
lastExitStatus: 78,
systemd: { result: "exit-code", nRestarts: 5, startLimitBurst: 5 },
},
},
extraServices: [],
},
{ json: false },
);
} finally {
platform.mockRestore();
}
const errors = runtime.error.mock.calls.map(([line]) => line).join("\n");
expect(errors).toContain("Service is loaded but not running (likely exited immediately).");
expect(errors).not.toContain("systemd stopped restarting the gateway");
});
});
+11 -1
View File
@@ -10,6 +10,7 @@ import {
resolveGatewayRestartLogPath,
resolveGatewaySupervisorLogPaths,
} from "../../daemon/restart-logs.js";
import { isSystemdStartLimitHit } from "../../daemon/service-runtime.js";
import {
isSystemdUnavailableDetail,
renderSystemdUnavailableHints,
@@ -366,8 +367,17 @@ export function printDaemonStatus(status: DaemonStatus, opts: { json: boolean; d
defaultRuntime.error(errorText(hint));
}
} else if (service.loaded && service.runtime?.status === "stopped") {
const startLimitHit = process.platform === "linux" && isSystemdStartLimitHit(service.runtime);
defaultRuntime.error(
errorText("Service is loaded but not running (likely exited immediately)."),
errorText(
startLimitHit
? // systemd gave up restarting after repeated crashes; sending the operator
// to restart (which now clears the failed latch) beats "exited immediately".
`systemd stopped restarting the gateway after repeated crashes; run ${formatCliCommand(
"openclaw gateway restart",
)} or inspect logs.`
: "Service is loaded but not running (likely exited immediately).",
),
);
for (const hint of renderRuntimeHints(
service.runtime,
+58
View File
@@ -58,6 +58,64 @@ describe("buildGatewayRuntimeHints", () => {
expect(hints.join("\n")).not.toContain("systemd user services are unavailable");
});
it("guides recovery when systemd hit its restart start limit (crash loop)", () => {
// Real give-up shape: process kept failing (Result=exit-code) until NRestarts
// reached StartLimitBurst and systemd stopped restarting.
const text = buildGatewayRuntimeHints(
{
status: "stopped",
state: "failed",
systemd: { result: "exit-code", nRestarts: 5, startLimitBurst: 5 },
},
{ platform: "linux", env: {} },
).join("\n");
expect(text).toContain("systemd stopped restarting the gateway after repeated crashes");
expect(text).toContain("openclaw gateway restart");
expect(text).not.toContain("likely exited immediately");
});
it("keeps the generic stopped hint for a single failed exit below the start limit", () => {
const text = buildGatewayRuntimeHints(
{
status: "stopped",
state: "failed",
systemd: { result: "exit-code", nRestarts: 1, startLimitBurst: 5 },
},
{ platform: "linux", env: {} },
).join("\n");
expect(text).toContain("likely exited immediately");
expect(text).not.toContain("systemd stopped restarting the gateway");
});
it("keeps the generic stopped hint after a config exit (78) despite a stale restart count", () => {
// RestartPreventExitStatus=78 stopped systemd on purpose; the leftover
// NRestarts must not flip the hint to start-limit recovery guidance.
const text = buildGatewayRuntimeHints(
{
status: "stopped",
state: "failed",
lastExitStatus: 78,
systemd: { result: "exit-code", nRestarts: 5, startLimitBurst: 5 },
},
{ platform: "linux", env: {} },
).join("\n");
expect(text).toContain("likely exited immediately");
expect(text).not.toContain("systemd stopped restarting the gateway");
});
it("keeps the generic stopped hint for an ordinary cleanly-stopped service", () => {
const text = buildGatewayRuntimeHints(
{ status: "stopped", state: "inactive" },
{ platform: "linux", env: {} },
).join("\n");
expect(text).toContain("likely exited immediately");
expect(text).not.toContain("systemd stopped restarting the gateway");
});
it("does not warn for normal systemd cgroup metrics", () => {
expect(
buildGatewayRuntimeHints(
+11 -1
View File
@@ -11,6 +11,7 @@ import { buildPlatformRuntimeLogHints } from "../daemon/runtime-hints.js";
import {
getSystemdCgroupHygieneSummary,
isSystemdCgroupHygieneRisk,
isSystemdStartLimitHit,
type GatewayServiceRuntime,
} from "../daemon/service-runtime.js";
import {
@@ -104,7 +105,16 @@ export function buildGatewayRuntimeHints(
return hints;
}
if (runtime.status === "stopped") {
hints.push("Service is loaded but not running (likely exited immediately).");
if (platform === "linux" && isSystemdStartLimitHit(runtime)) {
// start-limit-hit means systemd gave up restarting after repeated crashes;
// a plain "exited immediately" hint would hide that recovery needs a restart.
hints.push(
"systemd stopped restarting the gateway after repeated crashes.",
`Recover with: ${formatCliCommand("openclaw gateway restart", env)}, then inspect logs if it keeps crashing.`,
);
} else {
hints.push("Service is loaded but not running (likely exited immediately).");
}
if (fileLog) {
hints.push(`File logs: ${fileLog}`);
}
+110
View File
@@ -0,0 +1,110 @@
// Tests for systemd supervision-state classification helpers.
import { describe, expect, it } from "vitest";
import { isSystemdStartLimitHit } from "./service-runtime.js";
describe("isSystemdStartLimitHit", () => {
it("detects a crash loop where the restart counter reached StartLimitBurst", () => {
// Real systemd 249 give-up: process kept exiting non-zero so Result stays
// exit-code; NRestarts hitting StartLimitBurst is the give-up signal.
expect(
isSystemdStartLimitHit({
status: "stopped",
state: "failed",
systemd: { result: "exit-code", nRestarts: 5, startLimitBurst: 5 },
}),
).toBe(true);
});
it("detects a crash loop when the last exit was a non-config code", () => {
// exit 1 is a normal crash, not the RestartPreventExitStatus=78 no-restart
// path, so a counter that reached StartLimitBurst is real start-limit exhaustion.
expect(
isSystemdStartLimitHit({
status: "stopped",
state: "failed",
lastExitStatus: 1,
systemd: { result: "exit-code", nRestarts: 3, startLimitBurst: 3 },
}),
).toBe(true);
});
it("detects Result=start-limit-hit even when restart counters are absent", () => {
expect(
isSystemdStartLimitHit({
status: "stopped",
state: "failed",
systemd: { result: "start-limit-hit" },
}),
).toBe(true);
});
it("does not counter-detect the deliberate EX_CONFIG (78) no-restart exit", () => {
// RestartPreventExitStatus=78 stops systemd on purpose; the NRestarts left
// over from earlier crashes is stale and must not read as start-limit exhaustion.
expect(
isSystemdStartLimitHit({
status: "stopped",
state: "failed",
lastExitStatus: 78,
systemd: { result: "exit-code", nRestarts: 5, startLimitBurst: 5 },
}),
).toBe(false);
});
it("keeps Result=start-limit-hit authoritative even after a config (78) exit", () => {
// The explicit systemd give-up signal wins regardless of the last exit code.
expect(
isSystemdStartLimitHit({
status: "stopped",
state: "failed",
lastExitStatus: 78,
systemd: { result: "start-limit-hit", nRestarts: 5, startLimitBurst: 5 },
}),
).toBe(true);
});
it("does not flag a single failed exit below the start limit", () => {
expect(
isSystemdStartLimitHit({
status: "stopped",
state: "failed",
systemd: { result: "exit-code", nRestarts: 1, startLimitBurst: 5 },
}),
).toBe(false);
});
it("does not flag a running unit even if its lifetime restart count is high", () => {
expect(
isSystemdStartLimitHit({
status: "running",
state: "active",
systemd: { result: "success", nRestarts: 9, startLimitBurst: 5 },
}),
).toBe(false);
});
it("does not flag when rate limiting is disabled (StartLimitBurst=0)", () => {
expect(
isSystemdStartLimitHit({
status: "stopped",
state: "failed",
systemd: { result: "exit-code", nRestarts: 9, startLimitBurst: 0 },
}),
).toBe(false);
});
it("does not counter-detect when StartLimitBurst is missing from the probe", () => {
expect(
isSystemdStartLimitHit({
status: "stopped",
state: "failed",
systemd: { result: "exit-code", nRestarts: 9 },
}),
).toBe(false);
});
it("returns false without systemd supervision data or runtime", () => {
expect(isSystemdStartLimitHit({ status: "stopped", state: "failed" })).toBe(false);
expect(isSystemdStartLimitHit(undefined)).toBe(false);
});
});
+57 -1
View File
@@ -1,12 +1,18 @@
/** Shared daemon runtime status types and systemd cgroup hygiene helpers. */
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
/** systemd cgroup fields used to spot unhealthy gateway service supervision. */
/** systemd supervision fields used to spot unhealthy or given-up gateway service state. */
export type GatewayServiceSystemdRuntime = {
unit?: string;
killMode?: string;
tasksCurrent?: number;
memoryCurrent?: number;
// systemd `Result` (e.g. success, exit-code, start-limit-hit) plus the restart
// counter and configured StartLimitBurst. Together they detect a crash-loop
// give-up that the collapsed `status` string and `Result` alone cannot.
result?: string;
nRestarts?: number;
startLimitBurst?: number;
};
export type GatewayServiceRuntime = {
@@ -29,6 +35,14 @@ export type GatewayServiceRuntime = {
export const SYSTEMD_TASKS_CURRENT_WARNING_THRESHOLD = 200;
export const SYSTEMD_MEMORY_CURRENT_WARNING_BYTES = 2 * 1024 * 1024 * 1024;
// EX_CONFIG (78) from sysexits.h. The generated systemd unit pins
// RestartPreventExitStatus=78 (see systemd-unit.ts) so the gateway's
// config-error / duplicate-lock exit (gateway-cli run) deliberately stops
// without a restart. A last exit of 78 therefore means systemd gave up on
// purpose, not that it exhausted StartLimitBurst, so any accumulated NRestarts
// is stale from earlier crashes and must not drive start-limit detection.
const SYSTEMD_NO_RESTART_EXIT_STATUS = 78;
export function isRiskySystemdKillMode(value: string | undefined): boolean {
const normalized = normalizeLowercaseStringOrEmpty(value);
return normalized === "process" || normalized === "none";
@@ -83,3 +97,45 @@ export function getSystemdCgroupHygieneSummary(
export function isSystemdCgroupHygieneRisk(runtime?: GatewayServiceSystemdRuntime): boolean {
return getSystemdCgroupHygieneSummary(runtime) !== null;
}
/**
* True when systemd has stopped auto-restarting the gateway because it crashed
* faster than StartLimitBurst/StartLimitIntervalSec allows. Unlike an ordinary
* stopped/exited unit, this terminal latch needs an explicit `reset-failed` +
* restart to recover, so status/doctor must surface it instead of the generic
* "exited immediately" message.
*
* Detection: the unit is `failed` and either systemd reported the give-up
* directly (Result=start-limit-hit, the start-was-refused-before-exec case) or
* the restart counter reached the configured burst. The counter path is the
* common one: once the gateway process has actually run and exited non-zero,
* systemd keeps Result=exit-code and never overwrites it with start-limit-hit
* (verified against systemd 249), so Result alone misses real crash loops.
*
* The counter path is guarded against the deliberate no-restart exit: a last
* exit of 78 (EX_CONFIG, held back by RestartPreventExitStatus=78) means
* systemd stopped on purpose, so a stale NRestarts left over from earlier
* crashes must not be mistaken for start-limit exhaustion. The explicit
* Result=start-limit-hit signal stays authoritative regardless of exit status.
*/
export function isSystemdStartLimitHit(runtime?: GatewayServiceRuntime): boolean {
if (!runtime || normalizeLowercaseStringOrEmpty(runtime.state) !== "failed") {
return false;
}
const systemd = runtime.systemd;
if (!systemd) {
return false;
}
if (normalizeLowercaseStringOrEmpty(systemd.result) === "start-limit-hit") {
return true;
}
if (runtime.lastExitStatus === SYSTEMD_NO_RESTART_EXIT_STATUS) {
return false;
}
return (
typeof systemd.startLimitBurst === "number" &&
systemd.startLimitBurst > 0 &&
typeof systemd.nRestarts === "number" &&
systemd.nRestarts >= systemd.startLimitBurst
);
}
+105 -6
View File
@@ -740,10 +740,15 @@ describe("system-scope gateway unit detection (openclaw#87577)", () => {
it("restartSystemdService restarts the system unit directly when running as root", async () => {
mockUnitFileLayout({ system: "/etc/systemd/system/openclaw-gateway.service" });
mockEffectiveUid(0);
execFileMock.mockImplementationOnce((_cmd, args, _opts, cb) => {
expect(args).toEqual(["restart", GATEWAY_SERVICE]);
cb(null, "", "");
});
execFileMock
.mockImplementationOnce((_cmd, args, _opts, cb) => {
expect(args).toEqual(["reset-failed", GATEWAY_SERVICE]);
cb(null, "", "");
})
.mockImplementationOnce((_cmd, args, _opts, cb) => {
expect(args).toEqual(["restart", GATEWAY_SERVICE]);
cb(null, "", "");
});
const { stdout, write } = createWritableStreamMock();
const result = await restartSystemdService({ stdout, env: { HOME: TEST_MANAGED_HOME } });
expect(result).toEqual({ outcome: "completed" });
@@ -804,6 +809,31 @@ describe("systemd runtime parsing", () => {
});
});
it("parses Result and the restart counter for crash-loop give-up detection", () => {
// Real systemd 249 give-up shape: a crash-looped unit keeps Result=exit-code
// (start-limit-hit never overwrites an exec failure), so the counter reaching
// StartLimitBurst is what flags the give-up.
const output = [
"ActiveState=failed",
"SubState=failed",
"Result=exit-code",
"NRestarts=5",
"StartLimitBurst=5",
"MainPID=0",
"ExecMainStatus=1",
"ExecMainCode=exited",
].join("\n");
expect(parseSystemdShow(output)).toEqual({
activeState: "failed",
subState: "failed",
result: "exit-code",
nRestarts: 5,
startLimitBurst: 5,
execMainStatus: 1,
execMainCode: "exited",
});
});
it("rejects pid and exit status values with junk suffixes", () => {
const output = [
"ActiveState=inactive",
@@ -855,7 +885,7 @@ describe("readSystemdServiceRuntime", () => {
GATEWAY_SERVICE,
"--no-page",
"--property",
"Id,ActiveState,SubState,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent",
"Id,ActiveState,SubState,Result,NRestarts,StartLimitBurst,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent",
);
cb(
null,
@@ -889,6 +919,41 @@ describe("readSystemdServiceRuntime", () => {
},
});
});
it("carries the supervision counters through a crash-looped failed unit", async () => {
execFileMock
.mockImplementationOnce((_cmd, args, _opts, cb) => {
assertUserSystemctlArgs(args, "status");
cb(null, "", "");
})
.mockImplementationOnce((_cmd, _args, _opts, cb) => {
cb(
null,
[
"Id=openclaw-gateway.service",
"ActiveState=failed",
"SubState=failed",
"Result=exit-code",
"NRestarts=5",
"StartLimitBurst=5",
"MainPID=0",
"ExecMainStatus=1",
"ExecMainCode=exited",
].join("\n"),
"",
);
});
const runtime = await readSystemdServiceRuntime({ HOME: TEST_MANAGED_HOME });
// ActiveState=failed collapses to the generic "stopped" status, so the raw
// state + restart counters are what let callers detect the crash-loop give-up.
expect(runtime.status).toBe("stopped");
expect(runtime.state).toBe("failed");
expect(runtime.systemd).toMatchObject({
result: "exit-code",
nRestarts: 5,
startLimitBurst: 5,
});
});
});
describe("resolveSystemdUserUnitPath", () => {
@@ -2008,14 +2073,25 @@ describe("systemd service control", () => {
});
});
it("restarts a profile-specific user unit", async () => {
it("runs reset-failed before restarting a profile-specific user unit", async () => {
const restartSequence: string[] = [];
execFileMock
.mockImplementationOnce((_cmd, _args, _opts, cb) => cb(null, "", ""))
.mockImplementationOnce((_cmd, args, _opts, cb) => {
assertUserSystemctlArgs(args, "reset-failed", "openclaw-gateway-work.service");
// args[0] is the "--user" scope flag; the systemctl verb is args[1].
restartSequence.push(args[1] ?? "");
cb(null, "", "");
})
.mockImplementationOnce((_cmd, args, _opts, cb) => {
assertUserSystemctlArgs(args, "restart", "openclaw-gateway-work.service");
restartSequence.push(args[1] ?? "");
cb(null, "", "");
});
await assertRestartSuccess({ OPENCLAW_PROFILE: "work" });
// reset-failed must clear any start-limit-hit latch before the restart so a
// crash-looped unit can recover.
expect(restartSequence).toEqual(["reset-failed", "restart"]);
});
it("surfaces stop failures with systemctl detail", async () => {
@@ -2062,6 +2138,10 @@ describe("systemd service control", () => {
assertMachineUserSystemctlArgs(args, "debian", "status");
cb(null, "", "");
})
.mockImplementationOnce((_cmd, args, _opts, cb) => {
assertMachineUserSystemctlArgs(args, "debian", "reset-failed", GATEWAY_SERVICE);
cb(null, "", "");
})
.mockImplementationOnce((_cmd, args, _opts, cb) => {
assertMachineRestartArgs(args);
cb(null, "", "");
@@ -2076,6 +2156,10 @@ describe("systemd service control", () => {
assertUserSystemctlArgs(args, "status");
cb(null, "", "");
})
.mockImplementationOnce((_cmd, args, _opts, cb) => {
assertUserSystemctlArgs(args, "reset-failed", GATEWAY_SERVICE);
cb(null, "", "");
})
.mockImplementationOnce((_cmd, args, _opts, cb) => {
assertUserSystemctlArgs(args, "restart", GATEWAY_SERVICE);
cb(null, "", "");
@@ -2096,6 +2180,10 @@ describe("systemd service control", () => {
assertUserSystemctlArgs(args, "status");
cb(null, "", "");
})
.mockImplementationOnce((_cmd, args, _opts, cb) => {
assertUserSystemctlArgs(args, "reset-failed", GATEWAY_SERVICE);
cb(null, "", "");
})
.mockImplementationOnce((_cmd, args, _opts, cb) => {
assertUserSystemctlArgs(args, "restart", GATEWAY_SERVICE);
cb(null, "", "");
@@ -2117,6 +2205,17 @@ describe("systemd service control", () => {
assertMachineUserSystemctlArgs(args, "debian", "status");
cb(null, "", "");
})
.mockImplementationOnce((_cmd, args, _opts, cb) => {
assertUserSystemctlArgs(args, "reset-failed", GATEWAY_SERVICE);
const err = createExecFileError("Failed to connect to user scope bus", {
stderr: "Failed to connect to user scope bus",
});
cb(err, "", "");
})
.mockImplementationOnce((_cmd, args, _opts, cb) => {
assertMachineUserSystemctlArgs(args, "debian", "reset-failed", GATEWAY_SERVICE);
cb(null, "", "");
})
.mockImplementationOnce((_cmd, args, _opts, cb) => {
assertUserSystemctlArgs(args, "restart", GATEWAY_SERVICE);
const err = createExecFileError("Failed to connect to user scope bus", {
+37 -1
View File
@@ -480,6 +480,9 @@ type SystemdServiceInfo = {
mainPid?: number;
execMainStatus?: number;
execMainCode?: string;
result?: string;
nRestarts?: number;
startLimitBurst?: number;
unit?: string;
killMode?: string;
tasksCurrent?: number;
@@ -515,6 +518,24 @@ export function parseSystemdShow(output: string): SystemdServiceInfo {
if (execMainCode) {
info.execMainCode = execMainCode;
}
const result = entries.result;
if (result) {
info.result = result;
}
const nRestartsValue = entries.nrestarts;
if (nRestartsValue) {
const nRestarts = parseStrictInteger(nRestartsValue);
if (nRestarts !== undefined) {
info.nRestarts = nRestarts;
}
}
const startLimitBurstValue = entries.startlimitburst;
if (startLimitBurstValue) {
const startLimitBurst = parseStrictInteger(startLimitBurstValue);
if (startLimitBurst !== undefined) {
info.startLimitBurst = startLimitBurst;
}
}
const unit = entries.id;
if (unit) {
info.unit = unit;
@@ -1184,6 +1205,13 @@ async function runSystemdServiceAction(params: {
`${unitName} is a system-scope unit (${installed.unitPath}); run \`sudo systemctl ${params.action} ${unitName}\` to ${params.action} it`,
);
}
if (params.action === "restart") {
// systemd latches a unit into failed/start-limit-hit after it crashes faster
// than StartLimitBurst allows and then stops auto-restarting it. Clear the
// latch first so an operator restart can recover a crash-looped gateway;
// reset-failed is idempotent and a no-op on a healthy unit.
await execSystemctl(["reset-failed", unitName], env);
}
const res = await execSystemctl([params.action, unitName], env);
if (res.code !== 0) {
throw new Error(`systemctl ${params.action} failed: ${res.stderr || res.stdout}`.trim());
@@ -1192,6 +1220,11 @@ async function runSystemdServiceAction(params: {
return;
}
await assertSystemdAvailable(env);
if (params.action === "restart") {
// Clear any failed/start-limit-hit latch before restart so a crash-looped
// gateway recovers (see system-scope branch above). Idempotent on healthy units.
await execSystemctlUser(env, ["reset-failed", unitName]);
}
const res = await execSystemctlUser(env, [params.action, unitName]);
if (res.code !== 0) {
throw new Error(`systemctl ${params.action} failed: ${res.stderr || res.stdout}`.trim());
@@ -1264,7 +1297,7 @@ export async function readSystemdServiceRuntime(
unitName,
"--no-page",
"--property",
"Id,ActiveState,SubState,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent",
"Id,ActiveState,SubState,Result,NRestarts,StartLimitBurst,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent",
];
const res =
installed?.scope === "system"
@@ -1294,6 +1327,9 @@ export async function readSystemdServiceRuntime(
killMode: parsed.killMode,
tasksCurrent: parsed.tasksCurrent,
memoryCurrent: parsed.memoryCurrent,
result: parsed.result,
nRestarts: parsed.nRestarts,
startLimitBurst: parsed.startLimitBurst,
},
};
}