mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
Surface config hot-reload watcher status in health (#99267)
* fix(gateway): surface config hot-reload watcher status in health Thread the config reloader's existing hotReloadStatus() accessor through the managed reloader handle (previously only stop() survived the handoff) and into openclaw health as an optional configReload.hotReloadStatus field, so operators can tell when the watcher has permanently disabled itself after exhausting retries instead of silently running with stale config. * fix(gateway): keep configReload.hotReloadStatus fresh on cached health responses Thread getConfigReloaderHotReloadStatus through GatewayRequestContext (mirroring the existing getEventLoopHealth pattern) so the health RPC's cache-hit merge path picks up a live watcher disable within the same request instead of waiting up to HEALTH_REFRESH_INTERVAL_MS for the background refresh to catch up. * fix(health): move config reload status type to leaf contract Move GatewayHotReloadStatus out of config-reload.ts into a leaf config-reload-status.types.ts so health/request-context/runtime-handles callers no longer pull the full config-reload implementation into the shared server-method type graph (ClawSweeper-flagged architecture cycle). * test(gateway): update config reloader fixture --------- Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
This commit is contained in:
co-authored by
Vincent Koc
parent
ba9700d59a
commit
e7158a581d
@@ -585,6 +585,28 @@ describe("getHealthSnapshot", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("omits configReload when no config reloader status is supplied", async () => {
|
||||
testConfig = { session: { store: "/tmp/x" } };
|
||||
testStore = {};
|
||||
|
||||
const snap = await getHealthSnapshot({ timeoutMs: 10, probe: false });
|
||||
|
||||
expect(snap.configReload).toBeUndefined();
|
||||
});
|
||||
|
||||
it("surfaces a disabled config hot-reload watcher in the health snapshot", async () => {
|
||||
testConfig = { session: { store: "/tmp/x" } };
|
||||
testStore = {};
|
||||
|
||||
const snap = await getHealthSnapshot({
|
||||
timeoutMs: 10,
|
||||
probe: false,
|
||||
configReloadHotReloadStatus: "disabled",
|
||||
});
|
||||
|
||||
expect(snap.configReload).toEqual({ hotReloadStatus: "disabled" });
|
||||
});
|
||||
|
||||
it("skips telegram probe when not configured", async () => {
|
||||
testConfig = { session: { store: "/tmp/x" } };
|
||||
testStore = {
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
import { formatHealthCheckFailure } from "./health-format.js";
|
||||
import type { HealthSummary } from "./health.js";
|
||||
import {
|
||||
formatConfigReloadHealthLine,
|
||||
formatContextEngineHealthLine,
|
||||
formatDeliveryQueueHealthLine,
|
||||
formatHealthChannelLines,
|
||||
@@ -205,6 +206,51 @@ describe("healthCommand", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("surfaces a disabled config hot-reload watcher in JSON output", async () => {
|
||||
const snapshot = createHealthSummary({
|
||||
channels: {},
|
||||
channelOrder: [],
|
||||
channelLabels: {},
|
||||
});
|
||||
snapshot.configReload = { hotReloadStatus: "disabled" };
|
||||
callGatewayMock.mockResolvedValueOnce(snapshot);
|
||||
|
||||
await healthCommand({ json: true, timeoutMs: 5000, config: {} }, runtime as never);
|
||||
|
||||
const parsed = JSON.parse(requireFirstRuntimeLog()) as HealthSummary;
|
||||
expect(parsed.configReload).toEqual({ hotReloadStatus: "disabled" });
|
||||
});
|
||||
|
||||
it("prints the config hot-reload disabled line in text output", async () => {
|
||||
const snapshot = createHealthSummary({
|
||||
channels: {},
|
||||
channelOrder: [],
|
||||
channelLabels: {},
|
||||
});
|
||||
snapshot.configReload = { hotReloadStatus: "disabled" };
|
||||
callGatewayMock.mockResolvedValueOnce(snapshot);
|
||||
|
||||
await healthCommand({ json: false, timeoutMs: 5000, config: {} }, runtime as never);
|
||||
|
||||
const output = stripAnsi(runtime.log.mock.calls.map((c) => String(c[0])).join("\n"));
|
||||
expect(output).toContain("Config hot reload: disabled");
|
||||
});
|
||||
|
||||
it("omits the config hot-reload line in text output when the reloader is active", async () => {
|
||||
const snapshot = createHealthSummary({
|
||||
channels: {},
|
||||
channelOrder: [],
|
||||
channelLabels: {},
|
||||
});
|
||||
snapshot.configReload = { hotReloadStatus: "active" };
|
||||
callGatewayMock.mockResolvedValueOnce(snapshot);
|
||||
|
||||
await healthCommand({ json: false, timeoutMs: 5000, config: {} }, runtime as never);
|
||||
|
||||
const output = stripAnsi(runtime.log.mock.calls.map((c) => String(c[0])).join("\n"));
|
||||
expect(output).not.toContain("Config hot reload");
|
||||
});
|
||||
|
||||
it("prints the rich text summary and verbose gateway details", async () => {
|
||||
const recent = [
|
||||
{ key: "main", updatedAt: Date.now() - 60_000, age: 60_000 },
|
||||
@@ -537,6 +583,42 @@ describe("formatDeliveryQueueHealthLine", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatConfigReloadHealthLine", () => {
|
||||
it("reports a disabled config hot-reload watcher", () => {
|
||||
const summary = createHealthSummary({
|
||||
channels: {},
|
||||
channelOrder: [],
|
||||
channelLabels: {},
|
||||
});
|
||||
summary.configReload = { hotReloadStatus: "disabled" };
|
||||
|
||||
expect(formatConfigReloadHealthLine(summary)).toBe(
|
||||
"Config hot reload: disabled (watcher retries exhausted; restart the gateway to restore it)",
|
||||
);
|
||||
});
|
||||
|
||||
it("stays silent while the config hot-reload watcher is active", () => {
|
||||
const summary = createHealthSummary({
|
||||
channels: {},
|
||||
channelOrder: [],
|
||||
channelLabels: {},
|
||||
});
|
||||
summary.configReload = { hotReloadStatus: "active" };
|
||||
|
||||
expect(formatConfigReloadHealthLine(summary)).toBeNull();
|
||||
});
|
||||
|
||||
it("stays silent when no config reloader is running", () => {
|
||||
const summary = createHealthSummary({
|
||||
channels: {},
|
||||
channelOrder: [],
|
||||
channelLabels: {},
|
||||
});
|
||||
|
||||
expect(formatConfigReloadHealthLine(summary)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatHealthCheckFailure", () => {
|
||||
it("keeps non-rich output stable", () => {
|
||||
const err = new Error("gateway closed (1006 abnormal closure): no close reason");
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
DEFAULT_CHANNEL_STALE_EVENT_THRESHOLD_MS,
|
||||
evaluateChannelHealth,
|
||||
} from "../gateway/channel-health-policy.js";
|
||||
import type { GatewayHotReloadStatus } from "../gateway/config-reload-status.types.js";
|
||||
import { isGatewaySecretRefUnavailableError } from "../gateway/credentials.js";
|
||||
import { getGatewayModelPricingHealth } from "../gateway/model-pricing-cache-state.js";
|
||||
import { isGatewayModelPricingEnabled } from "../gateway/model-pricing-config.js";
|
||||
@@ -285,6 +286,14 @@ export function formatDeliveryQueueHealthLine(
|
||||
return `Delivery queue: warning (dead-lettered entries — ${counts}${oldestNote})`;
|
||||
}
|
||||
|
||||
/** Formats config hot-reload watcher degradation for text health output. */
|
||||
export function formatConfigReloadHealthLine(summary: HealthSummary): string | null {
|
||||
if (summary.configReload?.hotReloadStatus !== "disabled") {
|
||||
return null;
|
||||
}
|
||||
return "Config hot reload: disabled (watcher retries exhausted; restart the gateway to restore it)";
|
||||
}
|
||||
|
||||
const resolveHeartbeatSummary = (cfg: OpenClawConfig, agentId: string) =>
|
||||
resolveHeartbeatSummaryForAgent(cfg, agentId);
|
||||
|
||||
@@ -504,6 +513,7 @@ export async function getHealthSnapshot(params?: {
|
||||
includeSensitive?: boolean;
|
||||
runtimeSnapshot?: ChannelRuntimeSnapshot;
|
||||
eventLoop?: HealthSummary["eventLoop"];
|
||||
configReloadHotReloadStatus?: GatewayHotReloadStatus;
|
||||
}): Promise<HealthSummary> {
|
||||
const timeoutMs = params?.timeoutMs;
|
||||
const cfg = await readRuntimeHealthConfig();
|
||||
@@ -710,6 +720,9 @@ export async function getHealthSnapshot(params?: {
|
||||
...(pluginHealth ? { plugins: pluginHealth } : {}),
|
||||
...(contextEngineHealth ? { contextEngines: contextEngineHealth } : {}),
|
||||
...(deliveryQueueHealth ? { deliveryQueues: deliveryQueueHealth } : {}),
|
||||
...(params?.configReloadHotReloadStatus
|
||||
? { configReload: { hotReloadStatus: params.configReloadHotReloadStatus } }
|
||||
: {}),
|
||||
modelPricing: getGatewayModelPricingHealth({ enabled: isGatewayModelPricingEnabled(cfg) }),
|
||||
channels,
|
||||
channelOrder,
|
||||
@@ -949,6 +962,10 @@ export async function healthCommand(
|
||||
if (deliveryQueueLine) {
|
||||
runtime.log(styleHealthChannelLine(deliveryQueueLine, rich));
|
||||
}
|
||||
const configReloadLine = formatConfigReloadHealthLine(summary);
|
||||
if (configReloadLine) {
|
||||
runtime.log(styleHealthChannelLine(configReloadLine, rich));
|
||||
}
|
||||
for (const plugin of displayPlugins) {
|
||||
const channelSummary = summary.channels?.[plugin.id];
|
||||
if (!channelSummary || channelSummary.linked !== true) {
|
||||
|
||||
@@ -68,6 +68,11 @@ export type DeliveryQueueHealthSummary = {
|
||||
type ModelPricingHealthSummary =
|
||||
import("../gateway/model-pricing-cache-state.js").GatewayModelPricingHealth;
|
||||
|
||||
/** Config hot-reload watcher status, present only when a reloader is running. */
|
||||
export type ConfigReloadHealthSummary = {
|
||||
hotReloadStatus: import("../gateway/config-reload-status.types.js").GatewayHotReloadStatus;
|
||||
};
|
||||
|
||||
/** Full gateway health payload consumed by `openclaw health`. */
|
||||
export type HealthSummary = {
|
||||
ok: true;
|
||||
@@ -78,6 +83,7 @@ export type HealthSummary = {
|
||||
contextEngines?: ContextEngineHealthSummary;
|
||||
deliveryQueues?: DeliveryQueueHealthSummary;
|
||||
modelPricing?: ModelPricingHealthSummary;
|
||||
configReload?: ConfigReloadHealthSummary;
|
||||
channels: Record<string, ChannelHealthSummary>;
|
||||
channelOrder: string[];
|
||||
channelLabels: Record<string, string>;
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
// Leaf contract for the config hot-reload watcher's terminal status.
|
||||
// Kept separate from config-reload.ts so callers that only need the status
|
||||
// shape (health summaries, request context, runtime handles) do not pull in
|
||||
// the full config-reload implementation.
|
||||
|
||||
// Hot-reload stays "active" while a watcher is live. It flips to "disabled" only
|
||||
// after watcher re-creation fails past the retry budget, so operators/callers
|
||||
// can detect silent degradation instead of assuming reloads still fire.
|
||||
export type GatewayHotReloadStatus = "active" | "disabled";
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
type GatewayReloadPlan,
|
||||
} from "./config-reload-plan.js";
|
||||
import { resolveGatewayReloadSettings } from "./config-reload-settings.js";
|
||||
import type { GatewayHotReloadStatus } from "./config-reload-status.types.js";
|
||||
|
||||
export {
|
||||
buildGatewayReloadPlan,
|
||||
@@ -90,11 +91,6 @@ function isNoopReloadPlan(plan: GatewayReloadPlan): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
// Hot-reload stays "active" while a watcher is live. It flips to "disabled" only
|
||||
// after watcher re-creation fails past the retry budget, so operators/callers
|
||||
// can detect silent degradation instead of assuming reloads still fire.
|
||||
export type GatewayHotReloadStatus = "active" | "disabled";
|
||||
|
||||
type GatewayConfigReloader = {
|
||||
stop: () => Promise<void>;
|
||||
hotReloadStatus: () => GatewayHotReloadStatus;
|
||||
|
||||
@@ -6,6 +6,7 @@ import { buildDeliveryQueueHealthSummary } from "../../commands/health.js";
|
||||
import type { ChannelHealthSummary, HealthSummary } from "../../commands/health.types.js";
|
||||
import { getStatusSummary } from "../../commands/status.js";
|
||||
import { listContextEngineQuarantines } from "../../context-engine/registry.js";
|
||||
import type { GatewayHotReloadStatus } from "../config-reload-status.types.js";
|
||||
import { getGatewayModelPricingHealth } from "../model-pricing-cache-state.js";
|
||||
import type { ChannelRuntimeSnapshot } from "../server-channel-runtime.types.js";
|
||||
import { HEALTH_REFRESH_INTERVAL_MS } from "../server-constants.js";
|
||||
@@ -92,6 +93,7 @@ function cachedHealthDiffersFromRuntime(
|
||||
function mergeCachedHealthRuntimeState(params: {
|
||||
cached: HealthSummary;
|
||||
eventLoop?: HealthSummary["eventLoop"];
|
||||
configReloadHotReloadStatus?: GatewayHotReloadStatus;
|
||||
}): HealthSummary {
|
||||
const {
|
||||
contextEngines: _cachedContextEngines,
|
||||
@@ -122,6 +124,9 @@ function mergeCachedHealthRuntimeState(params: {
|
||||
? { contextEngines: { quarantined: quarantinedContextEngines } }
|
||||
: {}),
|
||||
...(deliveryQueues ? { deliveryQueues } : {}),
|
||||
...(params.configReloadHotReloadStatus
|
||||
? { configReload: { hotReloadStatus: params.configReloadHotReloadStatus } }
|
||||
: {}),
|
||||
modelPricing: getGatewayModelPricingHealth({
|
||||
enabled: params.cached.modelPricing?.state !== "disabled",
|
||||
}),
|
||||
@@ -159,6 +164,7 @@ export const healthHandlers: GatewayRequestHandlers = {
|
||||
mergeCachedHealthRuntimeState({
|
||||
cached,
|
||||
eventLoop: context.getEventLoopHealth?.(),
|
||||
configReloadHotReloadStatus: context.getConfigReloaderHotReloadStatus?.(),
|
||||
}),
|
||||
undefined,
|
||||
{ cached: true },
|
||||
|
||||
@@ -4962,6 +4962,87 @@ describe("gateway healthHandlers.health cache freshness", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("merges a live disabled config hot-reload status into cached health responses", async () => {
|
||||
const cached = {
|
||||
ok: true,
|
||||
ts: Date.now(),
|
||||
durationMs: 1,
|
||||
channels: {},
|
||||
channelOrder: [],
|
||||
channelLabels: {},
|
||||
heartbeatSeconds: 0,
|
||||
defaultAgentId: "main",
|
||||
agents: [],
|
||||
sessions: { path: "/tmp/sessions.json", count: 0, recent: [] },
|
||||
configReload: { hotReloadStatus: "active" },
|
||||
};
|
||||
const respond = vi.fn();
|
||||
const refreshHealthSnapshot = vi.fn().mockResolvedValue(cached);
|
||||
const getConfigReloaderHotReloadStatus = vi.fn(() => "disabled" as const);
|
||||
|
||||
await healthHandlers.health({
|
||||
req: {} as never,
|
||||
params: {} as never,
|
||||
respond: respond as never,
|
||||
context: {
|
||||
getHealthCache: () => cached,
|
||||
refreshHealthSnapshot,
|
||||
getRuntimeSnapshot: () => ({ channels: {}, channelAccounts: {} }),
|
||||
getConfigReloaderHotReloadStatus,
|
||||
logHealth: { error: vi.fn() },
|
||||
} as never,
|
||||
client: { connect: { role: "operator", scopes: ["operator.read"] } } as never,
|
||||
isWebchatConnect: () => false,
|
||||
});
|
||||
|
||||
const payload = mockCallArg(respond, 0, 1) as
|
||||
| { configReload?: { hotReloadStatus?: string } }
|
||||
| undefined;
|
||||
// The cache-hit merge must reflect the live "disabled" flip immediately,
|
||||
// not the stale "active" value from the cached snapshot — otherwise
|
||||
// operators wait up to HEALTH_REFRESH_INTERVAL_MS to see a watcher that
|
||||
// has already permanently given up.
|
||||
expect(payload?.configReload?.hotReloadStatus).toBe("disabled");
|
||||
expect(mockCallArg(respond, 0, 3)).toEqual({ cached: true });
|
||||
});
|
||||
|
||||
it("preserves the cached config hot-reload status when no live accessor is available", async () => {
|
||||
const cached = {
|
||||
ok: true,
|
||||
ts: Date.now(),
|
||||
durationMs: 1,
|
||||
channels: {},
|
||||
channelOrder: [],
|
||||
channelLabels: {},
|
||||
heartbeatSeconds: 0,
|
||||
defaultAgentId: "main",
|
||||
agents: [],
|
||||
sessions: { path: "/tmp/sessions.json", count: 0, recent: [] },
|
||||
configReload: { hotReloadStatus: "disabled" },
|
||||
};
|
||||
const respond = vi.fn();
|
||||
const refreshHealthSnapshot = vi.fn().mockResolvedValue(cached);
|
||||
|
||||
await healthHandlers.health({
|
||||
req: {} as never,
|
||||
params: {} as never,
|
||||
respond: respond as never,
|
||||
context: {
|
||||
getHealthCache: () => cached,
|
||||
refreshHealthSnapshot,
|
||||
getRuntimeSnapshot: () => ({ channels: {}, channelAccounts: {} }),
|
||||
logHealth: { error: vi.fn() },
|
||||
} as never,
|
||||
client: { connect: { role: "operator", scopes: ["operator.read"] } } as never,
|
||||
isWebchatConnect: () => false,
|
||||
});
|
||||
|
||||
const payload = mockCallArg(respond, 0, 1) as
|
||||
| { configReload?: { hotReloadStatus?: string } }
|
||||
| undefined;
|
||||
expect(payload?.configReload?.hotReloadStatus).toBe("disabled");
|
||||
});
|
||||
|
||||
it("refreshes cached health when a runtime account is missing from the cached account summary", async () => {
|
||||
const cached = {
|
||||
ok: true,
|
||||
|
||||
@@ -15,6 +15,7 @@ import type { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import type { WizardSession } from "../../wizard/session.js";
|
||||
import type { AgentRuntimeIdentity } from "../agent-runtime-identity-token.js";
|
||||
import type { ChatAbortControllerEntry } from "../chat-abort.js";
|
||||
import type { GatewayHotReloadStatus } from "../config-reload-status.types.js";
|
||||
import type { ExecApprovalManager, ExecApprovalRecord } from "../exec-approval-manager.js";
|
||||
import type { GatewayMethodRegistryView } from "../methods/descriptor.js";
|
||||
import type { NodeRegistry } from "../node-registry.js";
|
||||
@@ -140,6 +141,7 @@ export type GatewayRequestContext = {
|
||||
purgeWizardSession: (id: string) => void;
|
||||
getRuntimeSnapshot: () => ChannelRuntimeSnapshot;
|
||||
getEventLoopHealth?: () => GatewayEventLoopHealth | undefined;
|
||||
getConfigReloaderHotReloadStatus?: () => GatewayHotReloadStatus | undefined;
|
||||
startChannel: (
|
||||
channel: import("../../channels/plugins/types.public.js").ChannelId,
|
||||
accountId?: string,
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* Proves `startManagedGatewayConfigReloader` forwards the underlying watcher's
|
||||
* live `hotReloadStatus()` accessor on its returned handle instead of only
|
||||
* `stop`. Before this test, the returned handle dropped the accessor, so
|
||||
* `openclaw health` had no live signal to surface even though the watcher
|
||||
* itself already tracked "active"/"disabled" correctly.
|
||||
*/
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { GatewayPluginReloadResult } from "./server-reload-handlers.js";
|
||||
import { startManagedGatewayConfigReloader } from "./server-reload-handlers.js";
|
||||
|
||||
const hoisted = vi.hoisted(() => ({
|
||||
hotReloadStatus: { current: "active" as "active" | "disabled" },
|
||||
stop: vi.fn(async () => {}),
|
||||
}));
|
||||
|
||||
vi.mock("./config-reload.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("./config-reload.js")>("./config-reload.js");
|
||||
return {
|
||||
...actual,
|
||||
startGatewayConfigReloader: vi.fn(() => ({
|
||||
stop: hoisted.stop,
|
||||
hotReloadStatus: () => hoisted.hotReloadStatus.current,
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
describe("startManagedGatewayConfigReloader hotReloadStatus plumbing", () => {
|
||||
it("forwards the live watcher accessor instead of dropping it", async () => {
|
||||
const initialConfig = { session: { store: "/tmp/sessions.json" } } as OpenClawConfig;
|
||||
const reloader = startManagedGatewayConfigReloader({
|
||||
minimalTestGateway: false,
|
||||
initialConfig,
|
||||
initialCompareConfig: initialConfig,
|
||||
initialInternalWriteHash: null,
|
||||
watchPath: "/tmp/openclaw.json",
|
||||
readSnapshot: vi.fn() as never,
|
||||
promoteSnapshot: vi.fn(async () => true) as never,
|
||||
subscribeToWrites: vi.fn(() => () => {}) as never,
|
||||
deps: {} as never,
|
||||
broadcast: vi.fn(),
|
||||
getState: () => ({
|
||||
hooksConfig: {} as never,
|
||||
hookClientIpConfig: {} as never,
|
||||
heartbeatRunner: { stop: vi.fn(), updateConfig: vi.fn() } as never,
|
||||
cronState: {
|
||||
cron: { start: vi.fn(async () => {}), stop: vi.fn() },
|
||||
storePath: "/tmp/cron.json",
|
||||
cronEnabled: false,
|
||||
} as never,
|
||||
channelHealthMonitor: null,
|
||||
}),
|
||||
setState: vi.fn(),
|
||||
startChannel: vi.fn(async () => {}),
|
||||
stopChannel: vi.fn(async () => {}),
|
||||
reloadPlugins: vi.fn(
|
||||
async (): Promise<GatewayPluginReloadResult> => ({
|
||||
restartChannels: new Set(),
|
||||
activeChannels: new Set(),
|
||||
}),
|
||||
),
|
||||
logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
logChannels: { info: vi.fn(), error: vi.fn() },
|
||||
logCron: { error: vi.fn() },
|
||||
logReload: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
channelManager: {} as never,
|
||||
activateRuntimeSecrets: vi.fn(async (config: OpenClawConfig) => ({
|
||||
sourceConfig: config,
|
||||
config,
|
||||
authStores: [],
|
||||
warnings: [],
|
||||
webTools: {},
|
||||
})) as never,
|
||||
resolveSharedGatewaySessionGenerationForConfig: () => undefined,
|
||||
sharedGatewaySessionGenerationState: { current: undefined, required: null },
|
||||
reconcileTerminalSessions: vi.fn(),
|
||||
commitTerminalConfig: vi.fn(),
|
||||
clients: [],
|
||||
});
|
||||
|
||||
expect(reloader.hotReloadStatus).toBeTypeOf("function");
|
||||
expect(reloader.hotReloadStatus?.()).toBe("active");
|
||||
|
||||
// Flip the underlying watcher's live state without recreating the managed
|
||||
// handle — a copied/snapshotted value would stay stuck on "active".
|
||||
hoisted.hotReloadStatus.current = "disabled";
|
||||
expect(reloader.hotReloadStatus?.()).toBe("disabled");
|
||||
|
||||
await reloader.stop();
|
||||
expect(hoisted.stop).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -44,6 +44,7 @@ import { resolveHooksConfig } from "./hooks.js";
|
||||
import { buildGatewayCronService, type GatewayCronState } from "./server-cron.js";
|
||||
import { applyGatewayLaneConcurrency } from "./server-lanes.js";
|
||||
import { markGatewayModelCatalogStaleForReload } from "./server-model-catalog.js";
|
||||
import type { GatewayConfigReloaderHandle } from "./server-runtime-handles.js";
|
||||
import {
|
||||
type GatewayChannelManager,
|
||||
startGatewayChannelHealthMonitor,
|
||||
@@ -735,7 +736,9 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
|
||||
return { applyHotReload, requestGatewayRestart };
|
||||
}
|
||||
|
||||
export function startManagedGatewayConfigReloader(params: ManagedGatewayConfigReloaderParams) {
|
||||
export function startManagedGatewayConfigReloader(
|
||||
params: ManagedGatewayConfigReloaderParams,
|
||||
): GatewayConfigReloaderHandle {
|
||||
if (params.minimalTestGateway) {
|
||||
return { stop: async () => {} };
|
||||
}
|
||||
@@ -901,5 +904,6 @@ export function startManagedGatewayConfigReloader(params: ManagedGatewayConfigRe
|
||||
abortActiveGmailRestart();
|
||||
await configReloader.stop();
|
||||
},
|
||||
hotReloadStatus: configReloader.hotReloadStatus,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -11,12 +11,13 @@ import {
|
||||
function makeContextParams(
|
||||
overrides: Partial<GatewayRequestContextParams> = {},
|
||||
): GatewayRequestContextParams {
|
||||
const runtimeState: Pick<GatewayServerLiveState, "cronState"> = {
|
||||
const runtimeState: Pick<GatewayServerLiveState, "cronState" | "configReloader"> = {
|
||||
cronState: {
|
||||
cron: { start: vi.fn(), stop: vi.fn() } as never,
|
||||
storePath: "/tmp/cron",
|
||||
cronEnabled: true,
|
||||
},
|
||||
configReloader: { stop: vi.fn(async () => {}) },
|
||||
};
|
||||
return {
|
||||
deps: {} as never,
|
||||
@@ -88,12 +89,13 @@ describe("createGatewayRequestContext", () => {
|
||||
it("reads cron state live from runtime state", () => {
|
||||
const cronA = { start: vi.fn(), stop: vi.fn() } as never;
|
||||
const cronB = { start: vi.fn(), stop: vi.fn() } as never;
|
||||
const runtimeState: Pick<GatewayServerLiveState, "cronState"> = {
|
||||
const runtimeState: Pick<GatewayServerLiveState, "cronState" | "configReloader"> = {
|
||||
cronState: {
|
||||
cron: cronA,
|
||||
storePath: "/tmp/cron-a",
|
||||
cronEnabled: true,
|
||||
},
|
||||
configReloader: { stop: vi.fn(async () => {}) },
|
||||
};
|
||||
|
||||
const context = createGatewayRequestContext(makeContextParams({ runtimeState }));
|
||||
@@ -111,6 +113,33 @@ describe("createGatewayRequestContext", () => {
|
||||
expect(context.cronStorePath).toBe("/tmp/cron-b");
|
||||
});
|
||||
|
||||
it("reads config hot-reload status live from runtime state", () => {
|
||||
const runtimeState: Pick<GatewayServerLiveState, "cronState" | "configReloader"> = {
|
||||
cronState: {
|
||||
cron: { start: vi.fn(), stop: vi.fn() } as never,
|
||||
storePath: "/tmp/cron",
|
||||
cronEnabled: true,
|
||||
},
|
||||
configReloader: { stop: vi.fn(async () => {}) },
|
||||
};
|
||||
|
||||
const context = createGatewayRequestContext(makeContextParams({ runtimeState }));
|
||||
|
||||
expect(context.getConfigReloaderHotReloadStatus?.()).toBeUndefined();
|
||||
|
||||
runtimeState.configReloader = {
|
||||
stop: vi.fn(async () => {}),
|
||||
hotReloadStatus: () => "active",
|
||||
};
|
||||
expect(context.getConfigReloaderHotReloadStatus?.()).toBe("active");
|
||||
|
||||
runtimeState.configReloader = {
|
||||
stop: vi.fn(async () => {}),
|
||||
hotReloadStatus: () => "disabled",
|
||||
};
|
||||
expect(context.getConfigReloaderHotReloadStatus?.()).toBe("disabled");
|
||||
});
|
||||
|
||||
it("invalidateClientsForDevice sets the flag on matching clients without closing the socket", () => {
|
||||
const target = {
|
||||
connId: "conn-target",
|
||||
|
||||
@@ -14,7 +14,7 @@ type GatewayRequestContextClient = GatewayClient & {
|
||||
|
||||
export type GatewayRequestContextParams = {
|
||||
deps: GatewayRequestContext["deps"];
|
||||
runtimeState: Pick<GatewayServerLiveState, "cronState">;
|
||||
runtimeState: Pick<GatewayServerLiveState, "cronState" | "configReloader">;
|
||||
getRuntimeConfig: GatewayRequestContext["getRuntimeConfig"];
|
||||
resolveTerminalLaunchPolicy: GatewayRequestContext["resolveTerminalLaunchPolicy"];
|
||||
isTerminalEnabled: GatewayRequestContext["isTerminalEnabled"];
|
||||
@@ -212,6 +212,7 @@ export function createGatewayRequestContext(
|
||||
purgeWizardSession: params.purgeWizardSession,
|
||||
getRuntimeSnapshot: params.getRuntimeSnapshot,
|
||||
getEventLoopHealth: params.getEventLoopHealth,
|
||||
getConfigReloaderHotReloadStatus: () => params.runtimeState.configReloader.hotReloadStatus?.(),
|
||||
startChannel: params.startChannel,
|
||||
stopChannel: params.stopChannel,
|
||||
markChannelLoggedOut: params.markChannelLoggedOut,
|
||||
|
||||
@@ -3,12 +3,17 @@
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { HeartbeatRunner } from "../infra/heartbeat-runner.js";
|
||||
import type { ChannelHealthMonitor } from "./channel-health-monitor.js";
|
||||
import type { GatewayHotReloadStatus } from "./config-reload-status.types.js";
|
||||
import type { GatewayPostReadySidecarHandle } from "./server-startup-post-attach.js";
|
||||
|
||||
// Mutable server handles track timers, sidecars, subscriptions, and service
|
||||
// cleanup hooks that shutdown/reload code must stop exactly once.
|
||||
type GatewayConfigReloaderHandle = {
|
||||
// `hotReloadStatus` is omitted (not defaulted to "active") when no real
|
||||
// watcher is running, so health can distinguish "no reloader" from "reloader
|
||||
// active" instead of guessing.
|
||||
export type GatewayConfigReloaderHandle = {
|
||||
stop: () => Promise<void>;
|
||||
hotReloadStatus?: () => GatewayHotReloadStatus;
|
||||
};
|
||||
|
||||
/** Mutable handles owned by a running gateway server process. */
|
||||
|
||||
@@ -1027,6 +1027,7 @@ export async function startGatewayServer(
|
||||
...optsResult,
|
||||
getRuntimeSnapshot,
|
||||
getEventLoopHealth: readinessEventLoopHealth.snapshot,
|
||||
getConfigReloaderHotReloadStatus: () => runtimeState?.configReloader.hotReloadStatus?.(),
|
||||
});
|
||||
const stopRegisteredPostReadySidecars = async () => {
|
||||
const postReadySidecars = runtimeState.postReadySidecars;
|
||||
|
||||
@@ -18,6 +18,7 @@ function healthSnapshotCallArg(index = 0) {
|
||||
includeSensitive?: boolean;
|
||||
probe?: boolean;
|
||||
runtimeSnapshot?: unknown;
|
||||
configReloadHotReloadStatus?: unknown;
|
||||
}
|
||||
| undefined;
|
||||
}
|
||||
@@ -103,6 +104,25 @@ describe("refreshGatewayHealthSnapshot", () => {
|
||||
expect(Object.hasOwn(healthSnapshotCallArg(1) ?? {}, "eventLoop")).toBe(false);
|
||||
});
|
||||
|
||||
it("passes the config reloader hot-reload status only when the hook returns one", async () => {
|
||||
const healthState = await loadHealthState();
|
||||
|
||||
await healthState.refreshGatewayHealthSnapshot({
|
||||
probe: false,
|
||||
getConfigReloaderHotReloadStatus: () => "disabled",
|
||||
});
|
||||
await healthState.refreshGatewayHealthSnapshot({
|
||||
probe: true,
|
||||
getConfigReloaderHotReloadStatus: () => undefined,
|
||||
});
|
||||
|
||||
expect(getHealthSnapshotMock).toHaveBeenCalledTimes(2);
|
||||
expect(healthSnapshotCallArg()?.configReloadHotReloadStatus).toBe("disabled");
|
||||
expect(Object.hasOwn(healthSnapshotCallArg(1) ?? {}, "configReloadHotReloadStatus")).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("captures runtime snapshots for completed refreshes and guards snapshot failures", async () => {
|
||||
const healthState = await loadHealthState();
|
||||
const runtimeSnapshot = {
|
||||
|
||||
@@ -9,6 +9,7 @@ import { listSystemPresence } from "../../infra/system-presence.js";
|
||||
import { getUpdateAvailable } from "../../infra/update-startup.js";
|
||||
import { normalizeMainKey } from "../../routing/session-key.js";
|
||||
import { resolveGatewayAuth } from "../auth.js";
|
||||
import type { GatewayHotReloadStatus } from "../config-reload-status.types.js";
|
||||
import type { ChannelRuntimeSnapshot } from "../server-channel-runtime.types.js";
|
||||
import type { GatewayEventLoopHealth } from "./event-loop-health.js";
|
||||
|
||||
@@ -79,6 +80,7 @@ export async function refreshGatewayHealthSnapshot(opts?: {
|
||||
includeSensitive?: boolean;
|
||||
getRuntimeSnapshot?: () => ChannelRuntimeSnapshot;
|
||||
getEventLoopHealth?: () => GatewayEventLoopHealth | undefined;
|
||||
getConfigReloaderHotReloadStatus?: () => GatewayHotReloadStatus | undefined;
|
||||
}) {
|
||||
const includeSensitive = opts?.includeSensitive === true;
|
||||
let refresh = includeSensitive ? sensitiveHealthRefresh : healthRefresh;
|
||||
@@ -91,11 +93,13 @@ export async function refreshGatewayHealthSnapshot(opts?: {
|
||||
runtimeSnapshot = undefined;
|
||||
}
|
||||
const eventLoop = opts?.getEventLoopHealth?.();
|
||||
const configReloadHotReloadStatus = opts?.getConfigReloaderHotReloadStatus?.();
|
||||
const snap = await getHealthSnapshot({
|
||||
probe: opts?.probe,
|
||||
includeSensitive,
|
||||
runtimeSnapshot,
|
||||
...(eventLoop ? { eventLoop } : {}),
|
||||
...(configReloadHotReloadStatus ? { configReloadHotReloadStatus } : {}),
|
||||
});
|
||||
if (!includeSensitive) {
|
||||
healthCache = snap;
|
||||
|
||||
Reference in New Issue
Block a user