mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
fix(gateway): surface systemd start-limit exhaustion (#98291)
* fix(gateway): surface systemd start-limit exhaustion * fix(gateway): avoid start-limit false positive for exit 78
This commit is contained in:
@@ -727,4 +727,65 @@ describe("printDaemonStatus", () => {
|
||||
expect(errors).not.toContain("systemd user services unavailable");
|
||||
expect(errors).not.toContain("run loginctl enable-linger");
|
||||
});
|
||||
|
||||
it("warns that systemd gave up restarting a crash-looped gateway", () => {
|
||||
const platform = vi.spyOn(process, "platform", "get").mockReturnValue("linux");
|
||||
try {
|
||||
printDaemonStatus(
|
||||
{
|
||||
service: {
|
||||
label: "systemd",
|
||||
loaded: true,
|
||||
loadedText: "loaded",
|
||||
notLoadedText: "not loaded",
|
||||
runtime: {
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
systemd: { result: "exit-code", nRestarts: 5, startLimitBurst: 5 },
|
||||
},
|
||||
},
|
||||
extraServices: [],
|
||||
},
|
||||
{ json: false },
|
||||
);
|
||||
} finally {
|
||||
platform.mockRestore();
|
||||
}
|
||||
|
||||
const errors = runtime.error.mock.calls.map(([line]) => line).join("\n");
|
||||
expect(errors).toContain("systemd stopped restarting the gateway after repeated crashes");
|
||||
expect(errors).not.toContain("likely exited immediately");
|
||||
});
|
||||
|
||||
it("keeps the generic stopped message after a config exit (78) despite a stale restart count", () => {
|
||||
// RestartPreventExitStatus=78 stopped systemd deliberately; a leftover
|
||||
// NRestarts must not surface start-limit recovery guidance here.
|
||||
const platform = vi.spyOn(process, "platform", "get").mockReturnValue("linux");
|
||||
try {
|
||||
printDaemonStatus(
|
||||
{
|
||||
service: {
|
||||
label: "systemd",
|
||||
loaded: true,
|
||||
loadedText: "loaded",
|
||||
notLoadedText: "not loaded",
|
||||
runtime: {
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
lastExitStatus: 78,
|
||||
systemd: { result: "exit-code", nRestarts: 5, startLimitBurst: 5 },
|
||||
},
|
||||
},
|
||||
extraServices: [],
|
||||
},
|
||||
{ json: false },
|
||||
);
|
||||
} finally {
|
||||
platform.mockRestore();
|
||||
}
|
||||
|
||||
const errors = runtime.error.mock.calls.map(([line]) => line).join("\n");
|
||||
expect(errors).toContain("Service is loaded but not running (likely exited immediately).");
|
||||
expect(errors).not.toContain("systemd stopped restarting the gateway");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
resolveGatewayRestartLogPath,
|
||||
resolveGatewaySupervisorLogPaths,
|
||||
} from "../../daemon/restart-logs.js";
|
||||
import { isSystemdStartLimitHit } from "../../daemon/service-runtime.js";
|
||||
import {
|
||||
isSystemdUnavailableDetail,
|
||||
renderSystemdUnavailableHints,
|
||||
@@ -366,8 +367,17 @@ export function printDaemonStatus(status: DaemonStatus, opts: { json: boolean; d
|
||||
defaultRuntime.error(errorText(hint));
|
||||
}
|
||||
} else if (service.loaded && service.runtime?.status === "stopped") {
|
||||
const startLimitHit = process.platform === "linux" && isSystemdStartLimitHit(service.runtime);
|
||||
defaultRuntime.error(
|
||||
errorText("Service is loaded but not running (likely exited immediately)."),
|
||||
errorText(
|
||||
startLimitHit
|
||||
? // systemd gave up restarting after repeated crashes; sending the operator
|
||||
// to restart (which now clears the failed latch) beats "exited immediately".
|
||||
`systemd stopped restarting the gateway after repeated crashes; run ${formatCliCommand(
|
||||
"openclaw gateway restart",
|
||||
)} or inspect logs.`
|
||||
: "Service is loaded but not running (likely exited immediately).",
|
||||
),
|
||||
);
|
||||
for (const hint of renderRuntimeHints(
|
||||
service.runtime,
|
||||
|
||||
@@ -58,6 +58,64 @@ describe("buildGatewayRuntimeHints", () => {
|
||||
expect(hints.join("\n")).not.toContain("systemd user services are unavailable");
|
||||
});
|
||||
|
||||
it("guides recovery when systemd hit its restart start limit (crash loop)", () => {
|
||||
// Real give-up shape: process kept failing (Result=exit-code) until NRestarts
|
||||
// reached StartLimitBurst and systemd stopped restarting.
|
||||
const text = buildGatewayRuntimeHints(
|
||||
{
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
systemd: { result: "exit-code", nRestarts: 5, startLimitBurst: 5 },
|
||||
},
|
||||
{ platform: "linux", env: {} },
|
||||
).join("\n");
|
||||
|
||||
expect(text).toContain("systemd stopped restarting the gateway after repeated crashes");
|
||||
expect(text).toContain("openclaw gateway restart");
|
||||
expect(text).not.toContain("likely exited immediately");
|
||||
});
|
||||
|
||||
it("keeps the generic stopped hint for a single failed exit below the start limit", () => {
|
||||
const text = buildGatewayRuntimeHints(
|
||||
{
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
systemd: { result: "exit-code", nRestarts: 1, startLimitBurst: 5 },
|
||||
},
|
||||
{ platform: "linux", env: {} },
|
||||
).join("\n");
|
||||
|
||||
expect(text).toContain("likely exited immediately");
|
||||
expect(text).not.toContain("systemd stopped restarting the gateway");
|
||||
});
|
||||
|
||||
it("keeps the generic stopped hint after a config exit (78) despite a stale restart count", () => {
|
||||
// RestartPreventExitStatus=78 stopped systemd on purpose; the leftover
|
||||
// NRestarts must not flip the hint to start-limit recovery guidance.
|
||||
const text = buildGatewayRuntimeHints(
|
||||
{
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
lastExitStatus: 78,
|
||||
systemd: { result: "exit-code", nRestarts: 5, startLimitBurst: 5 },
|
||||
},
|
||||
{ platform: "linux", env: {} },
|
||||
).join("\n");
|
||||
|
||||
expect(text).toContain("likely exited immediately");
|
||||
expect(text).not.toContain("systemd stopped restarting the gateway");
|
||||
});
|
||||
|
||||
it("keeps the generic stopped hint for an ordinary cleanly-stopped service", () => {
|
||||
const text = buildGatewayRuntimeHints(
|
||||
{ status: "stopped", state: "inactive" },
|
||||
{ platform: "linux", env: {} },
|
||||
).join("\n");
|
||||
|
||||
expect(text).toContain("likely exited immediately");
|
||||
expect(text).not.toContain("systemd stopped restarting the gateway");
|
||||
});
|
||||
|
||||
it("does not warn for normal systemd cgroup metrics", () => {
|
||||
expect(
|
||||
buildGatewayRuntimeHints(
|
||||
|
||||
@@ -11,6 +11,7 @@ import { buildPlatformRuntimeLogHints } from "../daemon/runtime-hints.js";
|
||||
import {
|
||||
getSystemdCgroupHygieneSummary,
|
||||
isSystemdCgroupHygieneRisk,
|
||||
isSystemdStartLimitHit,
|
||||
type GatewayServiceRuntime,
|
||||
} from "../daemon/service-runtime.js";
|
||||
import {
|
||||
@@ -104,7 +105,16 @@ export function buildGatewayRuntimeHints(
|
||||
return hints;
|
||||
}
|
||||
if (runtime.status === "stopped") {
|
||||
hints.push("Service is loaded but not running (likely exited immediately).");
|
||||
if (platform === "linux" && isSystemdStartLimitHit(runtime)) {
|
||||
// start-limit-hit means systemd gave up restarting after repeated crashes;
|
||||
// a plain "exited immediately" hint would hide that recovery needs a restart.
|
||||
hints.push(
|
||||
"systemd stopped restarting the gateway after repeated crashes.",
|
||||
`Recover with: ${formatCliCommand("openclaw gateway restart", env)}, then inspect logs if it keeps crashing.`,
|
||||
);
|
||||
} else {
|
||||
hints.push("Service is loaded but not running (likely exited immediately).");
|
||||
}
|
||||
if (fileLog) {
|
||||
hints.push(`File logs: ${fileLog}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
// Tests for systemd supervision-state classification helpers.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { isSystemdStartLimitHit } from "./service-runtime.js";
|
||||
|
||||
describe("isSystemdStartLimitHit", () => {
|
||||
it("detects a crash loop where the restart counter reached StartLimitBurst", () => {
|
||||
// Real systemd 249 give-up: process kept exiting non-zero so Result stays
|
||||
// exit-code; NRestarts hitting StartLimitBurst is the give-up signal.
|
||||
expect(
|
||||
isSystemdStartLimitHit({
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
systemd: { result: "exit-code", nRestarts: 5, startLimitBurst: 5 },
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("detects a crash loop when the last exit was a non-config code", () => {
|
||||
// exit 1 is a normal crash, not the RestartPreventExitStatus=78 no-restart
|
||||
// path, so a counter that reached StartLimitBurst is real start-limit exhaustion.
|
||||
expect(
|
||||
isSystemdStartLimitHit({
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
lastExitStatus: 1,
|
||||
systemd: { result: "exit-code", nRestarts: 3, startLimitBurst: 3 },
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("detects Result=start-limit-hit even when restart counters are absent", () => {
|
||||
expect(
|
||||
isSystemdStartLimitHit({
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
systemd: { result: "start-limit-hit" },
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("does not counter-detect the deliberate EX_CONFIG (78) no-restart exit", () => {
|
||||
// RestartPreventExitStatus=78 stops systemd on purpose; the NRestarts left
|
||||
// over from earlier crashes is stale and must not read as start-limit exhaustion.
|
||||
expect(
|
||||
isSystemdStartLimitHit({
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
lastExitStatus: 78,
|
||||
systemd: { result: "exit-code", nRestarts: 5, startLimitBurst: 5 },
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps Result=start-limit-hit authoritative even after a config (78) exit", () => {
|
||||
// The explicit systemd give-up signal wins regardless of the last exit code.
|
||||
expect(
|
||||
isSystemdStartLimitHit({
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
lastExitStatus: 78,
|
||||
systemd: { result: "start-limit-hit", nRestarts: 5, startLimitBurst: 5 },
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("does not flag a single failed exit below the start limit", () => {
|
||||
expect(
|
||||
isSystemdStartLimitHit({
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
systemd: { result: "exit-code", nRestarts: 1, startLimitBurst: 5 },
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does not flag a running unit even if its lifetime restart count is high", () => {
|
||||
expect(
|
||||
isSystemdStartLimitHit({
|
||||
status: "running",
|
||||
state: "active",
|
||||
systemd: { result: "success", nRestarts: 9, startLimitBurst: 5 },
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does not flag when rate limiting is disabled (StartLimitBurst=0)", () => {
|
||||
expect(
|
||||
isSystemdStartLimitHit({
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
systemd: { result: "exit-code", nRestarts: 9, startLimitBurst: 0 },
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does not counter-detect when StartLimitBurst is missing from the probe", () => {
|
||||
expect(
|
||||
isSystemdStartLimitHit({
|
||||
status: "stopped",
|
||||
state: "failed",
|
||||
systemd: { result: "exit-code", nRestarts: 9 },
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false without systemd supervision data or runtime", () => {
|
||||
expect(isSystemdStartLimitHit({ status: "stopped", state: "failed" })).toBe(false);
|
||||
expect(isSystemdStartLimitHit(undefined)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,18 @@
|
||||
/** Shared daemon runtime status types and systemd cgroup hygiene helpers. */
|
||||
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
|
||||
|
||||
/** systemd cgroup fields used to spot unhealthy gateway service supervision. */
|
||||
/** systemd supervision fields used to spot unhealthy or given-up gateway service state. */
|
||||
export type GatewayServiceSystemdRuntime = {
|
||||
unit?: string;
|
||||
killMode?: string;
|
||||
tasksCurrent?: number;
|
||||
memoryCurrent?: number;
|
||||
// systemd `Result` (e.g. success, exit-code, start-limit-hit) plus the restart
|
||||
// counter and configured StartLimitBurst. Together they detect a crash-loop
|
||||
// give-up that the collapsed `status` string and `Result` alone cannot.
|
||||
result?: string;
|
||||
nRestarts?: number;
|
||||
startLimitBurst?: number;
|
||||
};
|
||||
|
||||
export type GatewayServiceRuntime = {
|
||||
@@ -29,6 +35,14 @@ export type GatewayServiceRuntime = {
|
||||
export const SYSTEMD_TASKS_CURRENT_WARNING_THRESHOLD = 200;
|
||||
export const SYSTEMD_MEMORY_CURRENT_WARNING_BYTES = 2 * 1024 * 1024 * 1024;
|
||||
|
||||
// EX_CONFIG (78) from sysexits.h. The generated systemd unit pins
|
||||
// RestartPreventExitStatus=78 (see systemd-unit.ts) so the gateway's
|
||||
// config-error / duplicate-lock exit (gateway-cli run) deliberately stops
|
||||
// without a restart. A last exit of 78 therefore means systemd gave up on
|
||||
// purpose, not that it exhausted StartLimitBurst, so any accumulated NRestarts
|
||||
// is stale from earlier crashes and must not drive start-limit detection.
|
||||
const SYSTEMD_NO_RESTART_EXIT_STATUS = 78;
|
||||
|
||||
export function isRiskySystemdKillMode(value: string | undefined): boolean {
|
||||
const normalized = normalizeLowercaseStringOrEmpty(value);
|
||||
return normalized === "process" || normalized === "none";
|
||||
@@ -83,3 +97,45 @@ export function getSystemdCgroupHygieneSummary(
|
||||
export function isSystemdCgroupHygieneRisk(runtime?: GatewayServiceSystemdRuntime): boolean {
|
||||
return getSystemdCgroupHygieneSummary(runtime) !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when systemd has stopped auto-restarting the gateway because it crashed
|
||||
* faster than StartLimitBurst/StartLimitIntervalSec allows. Unlike an ordinary
|
||||
* stopped/exited unit, this terminal latch needs an explicit `reset-failed` +
|
||||
* restart to recover, so status/doctor must surface it instead of the generic
|
||||
* "exited immediately" message.
|
||||
*
|
||||
* Detection: the unit is `failed` and either systemd reported the give-up
|
||||
* directly (Result=start-limit-hit, the start-was-refused-before-exec case) or
|
||||
* the restart counter reached the configured burst. The counter path is the
|
||||
* common one: once the gateway process has actually run and exited non-zero,
|
||||
* systemd keeps Result=exit-code and never overwrites it with start-limit-hit
|
||||
* (verified against systemd 249), so Result alone misses real crash loops.
|
||||
*
|
||||
* The counter path is guarded against the deliberate no-restart exit: a last
|
||||
* exit of 78 (EX_CONFIG, held back by RestartPreventExitStatus=78) means
|
||||
* systemd stopped on purpose, so a stale NRestarts left over from earlier
|
||||
* crashes must not be mistaken for start-limit exhaustion. The explicit
|
||||
* Result=start-limit-hit signal stays authoritative regardless of exit status.
|
||||
*/
|
||||
export function isSystemdStartLimitHit(runtime?: GatewayServiceRuntime): boolean {
|
||||
if (!runtime || normalizeLowercaseStringOrEmpty(runtime.state) !== "failed") {
|
||||
return false;
|
||||
}
|
||||
const systemd = runtime.systemd;
|
||||
if (!systemd) {
|
||||
return false;
|
||||
}
|
||||
if (normalizeLowercaseStringOrEmpty(systemd.result) === "start-limit-hit") {
|
||||
return true;
|
||||
}
|
||||
if (runtime.lastExitStatus === SYSTEMD_NO_RESTART_EXIT_STATUS) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
typeof systemd.startLimitBurst === "number" &&
|
||||
systemd.startLimitBurst > 0 &&
|
||||
typeof systemd.nRestarts === "number" &&
|
||||
systemd.nRestarts >= systemd.startLimitBurst
|
||||
);
|
||||
}
|
||||
|
||||
+105
-6
@@ -740,10 +740,15 @@ describe("system-scope gateway unit detection (openclaw#87577)", () => {
|
||||
it("restartSystemdService restarts the system unit directly when running as root", async () => {
|
||||
mockUnitFileLayout({ system: "/etc/systemd/system/openclaw-gateway.service" });
|
||||
mockEffectiveUid(0);
|
||||
execFileMock.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
expect(args).toEqual(["restart", GATEWAY_SERVICE]);
|
||||
cb(null, "", "");
|
||||
});
|
||||
execFileMock
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
expect(args).toEqual(["reset-failed", GATEWAY_SERVICE]);
|
||||
cb(null, "", "");
|
||||
})
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
expect(args).toEqual(["restart", GATEWAY_SERVICE]);
|
||||
cb(null, "", "");
|
||||
});
|
||||
const { stdout, write } = createWritableStreamMock();
|
||||
const result = await restartSystemdService({ stdout, env: { HOME: TEST_MANAGED_HOME } });
|
||||
expect(result).toEqual({ outcome: "completed" });
|
||||
@@ -804,6 +809,31 @@ describe("systemd runtime parsing", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("parses Result and the restart counter for crash-loop give-up detection", () => {
|
||||
// Real systemd 249 give-up shape: a crash-looped unit keeps Result=exit-code
|
||||
// (start-limit-hit never overwrites an exec failure), so the counter reaching
|
||||
// StartLimitBurst is what flags the give-up.
|
||||
const output = [
|
||||
"ActiveState=failed",
|
||||
"SubState=failed",
|
||||
"Result=exit-code",
|
||||
"NRestarts=5",
|
||||
"StartLimitBurst=5",
|
||||
"MainPID=0",
|
||||
"ExecMainStatus=1",
|
||||
"ExecMainCode=exited",
|
||||
].join("\n");
|
||||
expect(parseSystemdShow(output)).toEqual({
|
||||
activeState: "failed",
|
||||
subState: "failed",
|
||||
result: "exit-code",
|
||||
nRestarts: 5,
|
||||
startLimitBurst: 5,
|
||||
execMainStatus: 1,
|
||||
execMainCode: "exited",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects pid and exit status values with junk suffixes", () => {
|
||||
const output = [
|
||||
"ActiveState=inactive",
|
||||
@@ -855,7 +885,7 @@ describe("readSystemdServiceRuntime", () => {
|
||||
GATEWAY_SERVICE,
|
||||
"--no-page",
|
||||
"--property",
|
||||
"Id,ActiveState,SubState,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent",
|
||||
"Id,ActiveState,SubState,Result,NRestarts,StartLimitBurst,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent",
|
||||
);
|
||||
cb(
|
||||
null,
|
||||
@@ -889,6 +919,41 @@ describe("readSystemdServiceRuntime", () => {
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("carries the supervision counters through a crash-looped failed unit", async () => {
|
||||
execFileMock
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
assertUserSystemctlArgs(args, "status");
|
||||
cb(null, "", "");
|
||||
})
|
||||
.mockImplementationOnce((_cmd, _args, _opts, cb) => {
|
||||
cb(
|
||||
null,
|
||||
[
|
||||
"Id=openclaw-gateway.service",
|
||||
"ActiveState=failed",
|
||||
"SubState=failed",
|
||||
"Result=exit-code",
|
||||
"NRestarts=5",
|
||||
"StartLimitBurst=5",
|
||||
"MainPID=0",
|
||||
"ExecMainStatus=1",
|
||||
"ExecMainCode=exited",
|
||||
].join("\n"),
|
||||
"",
|
||||
);
|
||||
});
|
||||
const runtime = await readSystemdServiceRuntime({ HOME: TEST_MANAGED_HOME });
|
||||
// ActiveState=failed collapses to the generic "stopped" status, so the raw
|
||||
// state + restart counters are what let callers detect the crash-loop give-up.
|
||||
expect(runtime.status).toBe("stopped");
|
||||
expect(runtime.state).toBe("failed");
|
||||
expect(runtime.systemd).toMatchObject({
|
||||
result: "exit-code",
|
||||
nRestarts: 5,
|
||||
startLimitBurst: 5,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveSystemdUserUnitPath", () => {
|
||||
@@ -2008,14 +2073,25 @@ describe("systemd service control", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("restarts a profile-specific user unit", async () => {
|
||||
it("runs reset-failed before restarting a profile-specific user unit", async () => {
|
||||
const restartSequence: string[] = [];
|
||||
execFileMock
|
||||
.mockImplementationOnce((_cmd, _args, _opts, cb) => cb(null, "", ""))
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
assertUserSystemctlArgs(args, "reset-failed", "openclaw-gateway-work.service");
|
||||
// args[0] is the "--user" scope flag; the systemctl verb is args[1].
|
||||
restartSequence.push(args[1] ?? "");
|
||||
cb(null, "", "");
|
||||
})
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
assertUserSystemctlArgs(args, "restart", "openclaw-gateway-work.service");
|
||||
restartSequence.push(args[1] ?? "");
|
||||
cb(null, "", "");
|
||||
});
|
||||
await assertRestartSuccess({ OPENCLAW_PROFILE: "work" });
|
||||
// reset-failed must clear any start-limit-hit latch before the restart so a
|
||||
// crash-looped unit can recover.
|
||||
expect(restartSequence).toEqual(["reset-failed", "restart"]);
|
||||
});
|
||||
|
||||
it("surfaces stop failures with systemctl detail", async () => {
|
||||
@@ -2062,6 +2138,10 @@ describe("systemd service control", () => {
|
||||
assertMachineUserSystemctlArgs(args, "debian", "status");
|
||||
cb(null, "", "");
|
||||
})
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
assertMachineUserSystemctlArgs(args, "debian", "reset-failed", GATEWAY_SERVICE);
|
||||
cb(null, "", "");
|
||||
})
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
assertMachineRestartArgs(args);
|
||||
cb(null, "", "");
|
||||
@@ -2076,6 +2156,10 @@ describe("systemd service control", () => {
|
||||
assertUserSystemctlArgs(args, "status");
|
||||
cb(null, "", "");
|
||||
})
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
assertUserSystemctlArgs(args, "reset-failed", GATEWAY_SERVICE);
|
||||
cb(null, "", "");
|
||||
})
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
assertUserSystemctlArgs(args, "restart", GATEWAY_SERVICE);
|
||||
cb(null, "", "");
|
||||
@@ -2096,6 +2180,10 @@ describe("systemd service control", () => {
|
||||
assertUserSystemctlArgs(args, "status");
|
||||
cb(null, "", "");
|
||||
})
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
assertUserSystemctlArgs(args, "reset-failed", GATEWAY_SERVICE);
|
||||
cb(null, "", "");
|
||||
})
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
assertUserSystemctlArgs(args, "restart", GATEWAY_SERVICE);
|
||||
cb(null, "", "");
|
||||
@@ -2117,6 +2205,17 @@ describe("systemd service control", () => {
|
||||
assertMachineUserSystemctlArgs(args, "debian", "status");
|
||||
cb(null, "", "");
|
||||
})
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
assertUserSystemctlArgs(args, "reset-failed", GATEWAY_SERVICE);
|
||||
const err = createExecFileError("Failed to connect to user scope bus", {
|
||||
stderr: "Failed to connect to user scope bus",
|
||||
});
|
||||
cb(err, "", "");
|
||||
})
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
assertMachineUserSystemctlArgs(args, "debian", "reset-failed", GATEWAY_SERVICE);
|
||||
cb(null, "", "");
|
||||
})
|
||||
.mockImplementationOnce((_cmd, args, _opts, cb) => {
|
||||
assertUserSystemctlArgs(args, "restart", GATEWAY_SERVICE);
|
||||
const err = createExecFileError("Failed to connect to user scope bus", {
|
||||
|
||||
+37
-1
@@ -480,6 +480,9 @@ type SystemdServiceInfo = {
|
||||
mainPid?: number;
|
||||
execMainStatus?: number;
|
||||
execMainCode?: string;
|
||||
result?: string;
|
||||
nRestarts?: number;
|
||||
startLimitBurst?: number;
|
||||
unit?: string;
|
||||
killMode?: string;
|
||||
tasksCurrent?: number;
|
||||
@@ -515,6 +518,24 @@ export function parseSystemdShow(output: string): SystemdServiceInfo {
|
||||
if (execMainCode) {
|
||||
info.execMainCode = execMainCode;
|
||||
}
|
||||
const result = entries.result;
|
||||
if (result) {
|
||||
info.result = result;
|
||||
}
|
||||
const nRestartsValue = entries.nrestarts;
|
||||
if (nRestartsValue) {
|
||||
const nRestarts = parseStrictInteger(nRestartsValue);
|
||||
if (nRestarts !== undefined) {
|
||||
info.nRestarts = nRestarts;
|
||||
}
|
||||
}
|
||||
const startLimitBurstValue = entries.startlimitburst;
|
||||
if (startLimitBurstValue) {
|
||||
const startLimitBurst = parseStrictInteger(startLimitBurstValue);
|
||||
if (startLimitBurst !== undefined) {
|
||||
info.startLimitBurst = startLimitBurst;
|
||||
}
|
||||
}
|
||||
const unit = entries.id;
|
||||
if (unit) {
|
||||
info.unit = unit;
|
||||
@@ -1184,6 +1205,13 @@ async function runSystemdServiceAction(params: {
|
||||
`${unitName} is a system-scope unit (${installed.unitPath}); run \`sudo systemctl ${params.action} ${unitName}\` to ${params.action} it`,
|
||||
);
|
||||
}
|
||||
if (params.action === "restart") {
|
||||
// systemd latches a unit into failed/start-limit-hit after it crashes faster
|
||||
// than StartLimitBurst allows and then stops auto-restarting it. Clear the
|
||||
// latch first so an operator restart can recover a crash-looped gateway;
|
||||
// reset-failed is idempotent and a no-op on a healthy unit.
|
||||
await execSystemctl(["reset-failed", unitName], env);
|
||||
}
|
||||
const res = await execSystemctl([params.action, unitName], env);
|
||||
if (res.code !== 0) {
|
||||
throw new Error(`systemctl ${params.action} failed: ${res.stderr || res.stdout}`.trim());
|
||||
@@ -1192,6 +1220,11 @@ async function runSystemdServiceAction(params: {
|
||||
return;
|
||||
}
|
||||
await assertSystemdAvailable(env);
|
||||
if (params.action === "restart") {
|
||||
// Clear any failed/start-limit-hit latch before restart so a crash-looped
|
||||
// gateway recovers (see system-scope branch above). Idempotent on healthy units.
|
||||
await execSystemctlUser(env, ["reset-failed", unitName]);
|
||||
}
|
||||
const res = await execSystemctlUser(env, [params.action, unitName]);
|
||||
if (res.code !== 0) {
|
||||
throw new Error(`systemctl ${params.action} failed: ${res.stderr || res.stdout}`.trim());
|
||||
@@ -1264,7 +1297,7 @@ export async function readSystemdServiceRuntime(
|
||||
unitName,
|
||||
"--no-page",
|
||||
"--property",
|
||||
"Id,ActiveState,SubState,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent",
|
||||
"Id,ActiveState,SubState,Result,NRestarts,StartLimitBurst,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent",
|
||||
];
|
||||
const res =
|
||||
installed?.scope === "system"
|
||||
@@ -1294,6 +1327,9 @@ export async function readSystemdServiceRuntime(
|
||||
killMode: parsed.killMode,
|
||||
tasksCurrent: parsed.tasksCurrent,
|
||||
memoryCurrent: parsed.memoryCurrent,
|
||||
result: parsed.result,
|
||||
nRestarts: parsed.nRestarts,
|
||||
startLimitBurst: parsed.startLimitBurst,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user