test(release): align built-in timing proofs (#111483)

This commit is contained in:
Peter Steinberger
2026-07-19 09:42:16 -07:00
committed by GitHub
parent a70f7702a9
commit 7aedca21b2
13 changed files with 155 additions and 40 deletions
@@ -43,13 +43,13 @@ describe("session tab cleanup timer", () => {
const onWarn = vi.fn(); const onWarn = vi.fn();
const stop = startTrackedBrowserTabCleanupTimer({ onWarn }); const stop = startTrackedBrowserTabCleanupTimer({ onWarn });
await vi.advanceTimersByTimeAsync(60_000); await vi.advanceTimersByTimeAsync(300_000);
await vi.waitFor(() => await vi.waitFor(() =>
expect(onWarn).toHaveBeenCalledWith( expect(onWarn).toHaveBeenCalledWith(
"failed to sweep tracked browser tabs: Error: sqlite read failed", "failed to sweep tracked browser tabs: Error: sqlite read failed",
), ),
); );
await vi.advanceTimersByTimeAsync(60_000); await vi.advanceTimersByTimeAsync(300_000);
await vi.waitFor(() => expect(registryMocks.sweepTrackedBrowserTabs).toHaveBeenCalledTimes(2)); await vi.waitFor(() => expect(registryMocks.sweepTrackedBrowserTabs).toHaveBeenCalledTimes(2));
await stop(); await stop();
@@ -519,17 +519,19 @@ describe("processDiscordMessage ack reactions", () => {
outcome: "done", outcome: "done",
timingKey: "doneHoldMs", timingKey: "doneHoldMs",
configuredHoldMs: 2_000, configuredHoldMs: 2_000,
builtInHoldMs: DEFAULT_TIMING.doneHoldMs,
terminalEmoji: DEFAULT_EMOJIS.done, terminalEmoji: DEFAULT_EMOJIS.done,
}, },
{ {
outcome: "error", outcome: "error",
timingKey: "errorHoldMs", timingKey: "errorHoldMs",
configuredHoldMs: 4_000, configuredHoldMs: 4_000,
builtInHoldMs: DEFAULT_TIMING.errorHoldMs,
terminalEmoji: DEFAULT_EMOJIS.error, terminalEmoji: DEFAULT_EMOJIS.error,
}, },
] as const)( ] as const)(
"uses configured statusReactions.timing.$timingKey for $outcome cleanup", "uses built-in statusReactions.timing.$timingKey for $outcome cleanup",
async ({ outcome, timingKey, configuredHoldMs, terminalEmoji }) => { async ({ outcome, timingKey, configuredHoldMs, builtInHoldMs, terminalEmoji }) => {
vi.useFakeTimers(); vi.useFakeTimers();
dispatchInboundMessage.mockImplementationOnce(async (params?: DispatchInboundParams) => { dispatchInboundMessage.mockImplementationOnce(async (params?: DispatchInboundParams) => {
if (outcome === "done") { if (outcome === "done") {
@@ -559,7 +561,7 @@ describe("processDiscordMessage ack reactions", () => {
await runProcessDiscordMessage(ctx); await runProcessDiscordMessage(ctx);
expect(getReactionEmojis()).toContain(terminalEmoji); expect(getReactionEmojis()).toContain(terminalEmoji);
await vi.advanceTimersByTimeAsync(configuredHoldMs - 1); await vi.advanceTimersByTimeAsync(builtInHoldMs - 1);
expect(sendMocks.removeReactionDiscord).not.toHaveBeenCalledWith( expect(sendMocks.removeReactionDiscord).not.toHaveBeenCalledWith(
expect.anything(), expect.anything(),
expect.anything(), expect.anything(),
@@ -806,7 +806,7 @@ describe("monitorDiscordProvider", () => {
}); });
}); });
it("forwards custom eventQueue config from discord config to the Discord client", async () => { it("ignores removed custom eventQueue config", async () => {
resolveDiscordAccountMock.mockReturnValue({ resolveDiscordAccountMock.mockReturnValue({
accountId: "default", accountId: "default",
token: "MTIz.abc.def", token: "MTIz.abc.def",
@@ -825,7 +825,7 @@ describe("monitorDiscordProvider", () => {
}); });
const eventQueue = getConstructedEventQueue(); const eventQueue = getConstructedEventQueue();
expect(eventQueue?.listenerTimeout).toBe(300_000); expect(eventQueue?.listenerTimeout).toBe(120_000);
}); });
it("does not pass eventQueue.listenerTimeout into the message run queue", async () => { it("does not pass eventQueue.listenerTimeout into the message run queue", async () => {
@@ -104,6 +104,7 @@ function readSessionStoreEntries(storePath: string): Record<string, SessionEntry
async function seedDreamingTranscriptEvent(params: { async function seedDreamingTranscriptEvent(params: {
sessionId: string; sessionId: string;
sessionKey: string;
storePath: string; storePath: string;
timestampMs: number; timestampMs: number;
runId?: string; runId?: string;
@@ -111,7 +112,7 @@ async function seedDreamingTranscriptEvent(params: {
await appendSqliteSessionTranscriptEventForTest({ await appendSqliteSessionTranscriptEventForTest({
agentId: "main", agentId: "main",
sessionId: params.sessionId, sessionId: params.sessionId,
sessionKey: `agent:main:dreaming-narrative-fixture:${params.sessionId}`, sessionKey: params.sessionKey,
storePath: params.storePath, storePath: params.storePath,
event: { event: {
type: "metadata", type: "metadata",
@@ -866,7 +867,8 @@ describe("generateAndAppendDreamNarrative", () => {
const updatedAt = Date.now(); const updatedAt = Date.now();
await seedSessionStore(storePath, { await seedSessionStore(storePath, {
"agent:main:dreaming-narrative-light-1": { "agent:main:dreaming-narrative-light-1": {
sessionId: "missing", sessionId: "orphan",
sessionFile: orphanPath,
updatedAt, updatedAt,
}, },
"agent:main:kept-session": { "agent:main:kept-session": {
@@ -886,12 +888,14 @@ describe("generateAndAppendDreamNarrative", () => {
}); });
await seedDreamingTranscriptEvent({ await seedDreamingTranscriptEvent({
sessionId: "orphan", sessionId: "orphan",
sessionKey: "agent:main:dreaming-narrative-light-1",
storePath, storePath,
timestampMs: Date.now() - 600_000, timestampMs: Date.now() - 600_000,
runId: "dreaming-narrative-light-123", runId: "dreaming-narrative-light-123",
}); });
await seedDreamingTranscriptEvent({ await seedDreamingTranscriptEvent({
sessionId: "still-live", sessionId: "still-live",
sessionKey: "agent:main:kept-session",
storePath, storePath,
timestampMs: Date.now(), timestampMs: Date.now(),
runId: "dreaming-narrative-light-keep", runId: "dreaming-narrative-light-keep",
@@ -969,12 +973,14 @@ describe("generateAndAppendDreamNarrative", () => {
}); });
await seedDreamingTranscriptEvent({ await seedDreamingTranscriptEvent({
sessionId: "orphan-dreaming", sessionId: "orphan-dreaming",
sessionKey: "agent:main:dreaming-narrative-deep-orphan",
storePath, storePath,
timestampMs: Date.now() - 600_000, timestampMs: Date.now() - 600_000,
runId: "dreaming-narrative-deep-orphan", runId: "dreaming-narrative-deep-orphan",
}); });
await seedDreamingTranscriptEvent({ await seedDreamingTranscriptEvent({
sessionId: "live-dreaming", sessionId: "live-dreaming",
sessionKey: "agent:main:dreaming-narrative-deep-live",
storePath, storePath,
timestampMs: Date.now(), timestampMs: Date.now(),
runId: "dreaming-narrative-deep-live", runId: "dreaming-narrative-deep-live",
@@ -11,6 +11,8 @@ import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vites
type WatchIgnoredFn = (watchPath: string, stats?: { isDirectory?: () => boolean }) => boolean; type WatchIgnoredFn = (watchPath: string, stats?: { isDirectory?: () => boolean }) => boolean;
const BUILT_IN_WATCH_DEBOUNCE_MS = 1_500;
const { const {
createdChokidarWatchers, createdChokidarWatchers,
createdNativeWatchers, createdNativeWatchers,
@@ -380,7 +382,7 @@ describe("memory watcher config", () => {
.mockResolvedValue(undefined); .mockResolvedValue(undefined);
createdChokidarWatchers[0]?.emit(event); createdChokidarWatchers[0]?.emit(event);
await vi.advanceTimersByTimeAsync(25); await vi.advanceTimersByTimeAsync(BUILT_IN_WATCH_DEBOUNCE_MS);
expect(syncSpy).toHaveBeenCalledWith({ reason: "watch" }); expect(syncSpy).toHaveBeenCalledWith({ reason: "watch" });
}, },
@@ -407,7 +409,7 @@ describe("memory watcher config", () => {
(w) => w.dir === path.join(workspaceDir, "memory"), (w) => w.dir === path.join(workspaceDir, "memory"),
); );
memoryWatcher?.emit(eventType, "notes.md"); memoryWatcher?.emit(eventType, "notes.md");
await vi.advanceTimersByTimeAsync(25); await vi.advanceTimersByTimeAsync(BUILT_IN_WATCH_DEBOUNCE_MS);
expect(syncSpy).toHaveBeenCalledWith({ reason: "watch" }); expect(syncSpy).toHaveBeenCalledWith({ reason: "watch" });
}, },
@@ -434,7 +436,7 @@ describe("memory watcher config", () => {
// Node docs warn that filename may be null on some platforms; conservative // Node docs warn that filename may be null on some platforms; conservative
// dirty must still be scheduled. // dirty must still be scheduled.
memoryWatcher?.emit("rename", null as unknown as string); memoryWatcher?.emit("rename", null as unknown as string);
await vi.advanceTimersByTimeAsync(50); await vi.advanceTimersByTimeAsync(BUILT_IN_WATCH_DEBOUNCE_MS);
expect(syncSpy).toHaveBeenCalledWith({ reason: "watch" }); expect(syncSpy).toHaveBeenCalledWith({ reason: "watch" });
}); });
@@ -495,7 +497,7 @@ describe("memory watcher config", () => {
); );
memoryWatcher?.emitError(new Error("watcher error: ENOSPC")); memoryWatcher?.emitError(new Error("watcher error: ENOSPC"));
await vi.advanceTimersByTimeAsync(50); await vi.advanceTimersByTimeAsync(BUILT_IN_WATCH_DEBOUNCE_MS);
expect(memoryLoggerWarn).toHaveBeenCalledWith( expect(memoryLoggerWarn).toHaveBeenCalledWith(
expect.stringContaining("memory native watcher error"), expect.stringContaining("memory native watcher error"),
@@ -511,7 +513,7 @@ describe("memory watcher config", () => {
// continues to schedule sync. // continues to schedule sync.
syncSpy.mockClear(); syncSpy.mockClear();
existingChokidar?.emit("change"); existingChokidar?.emit("change");
await vi.advanceTimersByTimeAsync(25); await vi.advanceTimersByTimeAsync(BUILT_IN_WATCH_DEBOUNCE_MS);
expect(syncSpy).toHaveBeenCalledWith({ reason: "watch" }); expect(syncSpy).toHaveBeenCalledWith({ reason: "watch" });
}); });
@@ -611,7 +613,7 @@ describe("memory watcher config", () => {
await fs.mkdir(newDir); await fs.mkdir(newDir);
const memoryWatcher = createdNativeWatchers.find((w) => w.dir === memoryDir); const memoryWatcher = createdNativeWatchers.find((w) => w.dir === memoryDir);
memoryWatcher?.emit("rename", "new-topic"); memoryWatcher?.emit("rename", "new-topic");
await vi.advanceTimersByTimeAsync(25); await vi.advanceTimersByTimeAsync(BUILT_IN_WATCH_DEBOUNCE_MS);
expect(syncSpy).toHaveBeenCalledWith({ reason: "watch" }); expect(syncSpy).toHaveBeenCalledWith({ reason: "watch" });
expect( expect(
@@ -1005,10 +1007,10 @@ describe("memory watcher config", () => {
extraWatcher?.emit("change", "notes.md"); extraWatcher?.emit("change", "notes.md");
await fs.writeFile(notesPath, "hello updated"); await fs.writeFile(notesPath, "hello updated");
await vi.advanceTimersByTimeAsync(25); await vi.advanceTimersByTimeAsync(BUILT_IN_WATCH_DEBOUNCE_MS);
expect(syncSpy).not.toHaveBeenCalled(); expect(syncSpy).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(25); await vi.advanceTimersByTimeAsync(BUILT_IN_WATCH_DEBOUNCE_MS);
expect(syncSpy).toHaveBeenCalledWith({ reason: "watch" }); expect(syncSpy).toHaveBeenCalledWith({ reason: "watch" });
// Recorded path should match the resolved absolute path under extraDir. // Recorded path should match the resolved absolute path under extraDir.
const recordedStats = (initialStats as unknown as { isDirectory: () => boolean }).isDirectory(); const recordedStats = (initialStats as unknown as { isDirectory: () => boolean }).isDirectory();
@@ -49,6 +49,7 @@ const MEMORY_EMBEDDING_PROVIDERS_KEY = Symbol.for("openclaw.memoryEmbeddingProvi
const MCPORTER_STATE_KEY = Symbol.for("openclaw.mcporterState"); const MCPORTER_STATE_KEY = Symbol.for("openclaw.mcporterState");
const QMD_EMBED_QUEUE_KEY = Symbol.for("openclaw.qmdEmbedQueueTail"); const QMD_EMBED_QUEUE_KEY = Symbol.for("openclaw.qmdEmbedQueueTail");
const QMD_UPDATE_QUEUE_KEY = Symbol.for("openclaw.qmdUpdateQueueState"); const QMD_UPDATE_QUEUE_KEY = Symbol.for("openclaw.qmdUpdateQueueState");
const BUILT_IN_WATCH_DEBOUNCE_MS = 1_500;
type WatchOptions = { type WatchOptions = {
ignored?: (watchPath: string) => boolean; ignored?: (watchPath: string) => boolean;
@@ -1193,7 +1194,7 @@ describe("QmdMemoryManager", () => {
}); });
expect(manager.status().dirty).toBe(true); expect(manager.status().dirty).toBe(true);
await vi.advanceTimersByTimeAsync(25); await vi.advanceTimersByTimeAsync(BUILT_IN_WATCH_DEBOUNCE_MS);
const updateCalls = spawnMock.mock.calls.filter((call) => call[1]?.[0] === "update"); const updateCalls = spawnMock.mock.calls.filter((call) => call[1]?.[0] === "update");
expect(updateCalls).toHaveLength(1); expect(updateCalls).toHaveLength(1);
@@ -1328,10 +1329,10 @@ describe("QmdMemoryManager", () => {
}); });
await fs.writeFile(notesPath, "hello updated"); await fs.writeFile(notesPath, "hello updated");
await vi.advanceTimersByTimeAsync(25); await vi.advanceTimersByTimeAsync(BUILT_IN_WATCH_DEBOUNCE_MS);
expect(spawnMock.mock.calls.filter((call) => call[1]?.[0] === "update")).toHaveLength(0); expect(spawnMock.mock.calls.filter((call) => call[1]?.[0] === "update")).toHaveLength(0);
await vi.advanceTimersByTimeAsync(25); await vi.advanceTimersByTimeAsync(BUILT_IN_WATCH_DEBOUNCE_MS);
expect(spawnMock.mock.calls.filter((call) => call[1]?.[0] === "update")).toHaveLength(1); expect(spawnMock.mock.calls.filter((call) => call[1]?.[0] === "update")).toHaveLength(1);
await manager.close(); await manager.close();
@@ -1,3 +1,4 @@
import { DEFAULT_TIMING } from "openclaw/plugin-sdk/channel-feedback";
import { expect, it, vi } from "vitest"; import { expect, it, vi } from "vitest";
import { import {
createChannelMessageReplyPipeline, createChannelMessageReplyPipeline,
@@ -12,6 +13,7 @@ import { notifyTelegramInboundEventOutboundSuccess } from "./inbound-event-deliv
describeTelegramDispatch("dispatchTelegramMessage pipeline-init", () => { describeTelegramDispatch("dispatchTelegramMessage pipeline-init", () => {
it("cleans delivery correlation when reply-pipeline initialization fails", async () => { it("cleans delivery correlation when reply-pipeline initialization fails", async () => {
vi.useFakeTimers();
const sessionKey = "agent:main:telegram:direct:pipeline-init-failure"; const sessionKey = "agent:main:telegram:direct:pipeline-init-failure";
const statusReactionController = createStatusReactionController(); const statusReactionController = createStatusReactionController();
const reactionApi = vi.fn(async () => undefined); const reactionApi = vi.fn(async () => undefined);
@@ -41,7 +43,8 @@ describeTelegramDispatch("dispatchTelegramMessage pipeline-init", () => {
suppressFailureFallback: true, suppressFailureFallback: true,
}); });
await vi.waitFor(() => expect(statusReactionController.restoreInitial).toHaveBeenCalled()); await vi.advanceTimersByTimeAsync(DEFAULT_TIMING.errorHoldMs);
expect(statusReactionController.restoreInitial).toHaveBeenCalled();
expect(reactionApi).not.toHaveBeenCalled(); expect(reactionApi).not.toHaveBeenCalled();
}); });
}); });
@@ -325,14 +325,14 @@ describe("createTelegramBot", () => {
globalThis.fetch = originalFetch; globalThis.fetch = originalFetch;
} }
}); });
it("applies global and per-account timeoutSeconds", () => { it("ignores removed global and per-account timeoutSeconds", () => {
loadConfig.mockReturnValue({ loadConfig.mockReturnValue({
channels: { channels: {
telegram: { dmPolicy: "open", allowFrom: ["*"], timeoutSeconds: 60 }, telegram: { dmPolicy: "open", allowFrom: ["*"], timeoutSeconds: 60 },
}, },
}); });
createTelegramBot({ token: "tok" }); createTelegramBot({ token: "tok" });
expectBotClientFields({ timeoutSeconds: 60 }); expectBotClientFields({ timeoutSeconds: undefined });
botCtorSpy.mockClear(); botCtorSpy.mockClear();
loadConfig.mockReturnValue({ loadConfig.mockReturnValue({
@@ -348,7 +348,7 @@ describe("createTelegramBot", () => {
}, },
}); });
createTelegramBot({ token: "tok", accountId: "foo" }); createTelegramBot({ token: "tok", accountId: "foo" });
expectBotClientFields({ timeoutSeconds: 61 }); expectBotClientFields({ timeoutSeconds: undefined });
}); });
it("keeps low timeoutSeconds above the outbound request guard", () => { it("keeps low timeoutSeconds above the outbound request guard", () => {
@@ -358,7 +358,7 @@ describe("createTelegramBot", () => {
}, },
}); });
createTelegramBot({ token: "tok" }); createTelegramBot({ token: "tok" });
expectBotClientFields({ timeoutSeconds: 60 }); expectBotClientFields({ timeoutSeconds: undefined });
}); });
it("keeps polling client timeout above the outbound request guard", () => { it("keeps polling client timeout above the outbound request guard", () => {
@@ -368,7 +368,7 @@ describe("createTelegramBot", () => {
}, },
}); });
createTelegramBot({ token: "tok", minimumClientTimeoutSeconds: 45 }); createTelegramBot({ token: "tok", minimumClientTimeoutSeconds: 45 });
expectBotClientFields({ timeoutSeconds: 60 }); expectBotClientFields({ timeoutSeconds: undefined });
}); });
it("passes startup probe botInfo to grammY", () => { it("passes startup probe botInfo to grammY", () => {
@@ -552,7 +552,7 @@ describe("telegramPlugin gateway startup", () => {
).resolves.toBeNull(); ).resolves.toBeNull();
}); });
it("honors higher per-account timeoutSeconds for startup probe", async () => { it("uses the built-in startup probe timeout", async () => {
installTelegramRuntime(); installTelegramRuntime();
probeTelegram.mockResolvedValue({ probeTelegram.mockResolvedValue({
ok: true, ok: true,
@@ -565,7 +565,7 @@ describe("telegramPlugin gateway startup", () => {
const { ctx, task } = startTelegramAccount("ops", { timeoutSeconds: 60 }); const { ctx, task } = startTelegramAccount("ops", { timeoutSeconds: 60 });
await expect(task).resolves.toBeUndefined(); await expect(task).resolves.toBeUndefined();
expect(probeTelegram).toHaveBeenCalledWith("123456:bad-token", 60_000, { expect(probeTelegram).toHaveBeenCalledWith("123456:bad-token", 15_000, {
abortSignal: ctx.abortSignal, abortSignal: ctx.abortSignal,
accountId: "ops", accountId: "ops",
proxyUrl: undefined, proxyUrl: undefined,
+3 -3
View File
@@ -853,13 +853,13 @@ describe("sendMessageTelegram", () => {
expect(botCtorSpy).not.toHaveBeenCalled(); expect(botCtorSpy).not.toHaveBeenCalled();
}); });
it("applies timeoutSeconds config precedence", async () => { it("ignores removed timeoutSeconds config", async () => {
const cases = [ const cases = [
{ {
name: "global telegram timeout", name: "global telegram timeout",
cfg: { channels: { telegram: { timeoutSeconds: 60 } } }, cfg: { channels: { telegram: { timeoutSeconds: 60 } } },
opts: { cfg: TELEGRAM_TEST_CFG, token: "tok" }, opts: { cfg: TELEGRAM_TEST_CFG, token: "tok" },
expectedTimeout: 60, expectedTimeout: undefined,
}, },
{ {
name: "per-account timeout override", name: "per-account timeout override",
@@ -872,7 +872,7 @@ describe("sendMessageTelegram", () => {
}, },
}, },
opts: { cfg: TELEGRAM_TEST_CFG, token: "tok", accountId: "foo" }, opts: { cfg: TELEGRAM_TEST_CFG, token: "tok", accountId: "foo" },
expectedTimeout: 61, expectedTimeout: undefined,
}, },
] as const; ] as const;
for (const testCase of cases) { for (const testCase of cases) {
@@ -20,6 +20,14 @@ export function assertSuspendedProbes(
health: Record<string, unknown>, health: Record<string, unknown>,
readiness: Record<string, unknown>, readiness: Record<string, unknown>,
): void; ): void;
export function prepareReadySuspension(
options: {
deadline: number;
requestId: string;
rpc: (method: string, params: Record<string, unknown>) => Promise<Record<string, unknown>>;
},
deps?: { delayImpl?: (ms: number) => Promise<void>; now?: () => number },
): Promise<Record<string, unknown>>;
export function runGatewaySuspensionPreRestartClient( export function runGatewaySuspensionPreRestartClient(
options: { statePath: string; token: string; url: string; timeoutMs?: number }, options: { statePath: string; token: string; url: string; timeoutMs?: number },
deps?: { fetchImpl?: typeof fetch }, deps?: { fetchImpl?: typeof fetch },
+39 -9
View File
@@ -148,6 +148,30 @@ export function assertReadySuspensionResponse(response, now = Date.now()) {
return payload; return payload;
} }
export async function prepareReadySuspension(
{ deadline, requestId, rpc },
{ delayImpl = delay, now = Date.now } = {},
) {
while (true) {
if (now() >= deadline) {
throw new DOMException("gateway suspension preparation timeout", "TimeoutError");
}
const response = await rpc("gateway.suspend.prepare", { requestId });
if (response?.status !== 200 || response.body?.ok !== true) {
return assertReadySuspensionResponse(response, now());
}
const payload = response.body.payload;
if (payload?.status !== "busy") {
return assertReadySuspensionResponse(response, now());
}
const retryAfterMs =
typeof payload.retryAfterMs === "number" && Number.isFinite(payload.retryAfterMs)
? Math.max(1, Math.floor(payload.retryAfterMs))
: 100;
await delayImpl(Math.min(retryAfterMs, Math.max(1, deadline - now())));
}
}
export function assertGatewaySuspendingError(response) { export function assertGatewaySuspendingError(response) {
assert.equal(response?.ok, false, "normal RPC must fail during suspension"); assert.equal(response?.ok, false, "normal RPC must fail during suspension");
assert.equal(response.error?.code, "UNAVAILABLE", "normal RPC must be unavailable"); assert.equal(response.error?.code, "UNAVAILABLE", "normal RPC must be unavailable");
@@ -206,9 +230,11 @@ export async function runGatewaySuspensionPreRestartClient(
url, url,
}; };
const rpc = (method, params) => adminRpc(requestContext, method, params); const rpc = (method, params) => adminRpc(requestContext, method, params);
const firstLease = assertReadySuspensionResponse( const firstLease = await prepareReadySuspension({
await rpc("gateway.suspend.prepare", { requestId: "gateway-network-live-contract" }), deadline: requestContext.deadline,
); requestId: "gateway-network-live-contract",
rpc,
});
assertSuspendedProbes( assertSuspendedProbes(
await readProbe(requestContext, "/healthz"), await readProbe(requestContext, "/healthz"),
@@ -263,9 +289,11 @@ export async function runGatewaySuspensionPreRestartClient(
); );
const requestId = "gateway-network-restart-contract"; const requestId = "gateway-network-restart-contract";
const secondLease = assertReadySuspensionResponse( const secondLease = await prepareReadySuspension({
await rpc("gateway.suspend.prepare", { requestId }), deadline: requestContext.deadline,
); requestId,
rpc,
});
await writeFile( await writeFile(
statePath, statePath,
JSON.stringify({ JSON.stringify({
@@ -309,9 +337,11 @@ export async function runGatewaySuspensionPostRestartClient(
); );
assertAdminSuccess(await rpc("health"), "Admin health after restart"); assertAdminSuccess(await rpc("health"), "Admin health after restart");
const replacement = assertReadySuspensionResponse( const replacement = await prepareReadySuspension({
await rpc("gateway.suspend.prepare", { requestId: state.requestId }), deadline: requestContext.deadline,
); requestId: state.requestId,
rpc,
});
assert( assert(
replacement.suspensionId !== state.suspensionId, replacement.suspensionId !== state.suspensionId,
"reused request id must create a fresh suspension lease after restart", "reused request id must create a fresh suspension lease after restart",
@@ -8,6 +8,7 @@ import {
assertGatewaySuspendingError, assertGatewaySuspendingError,
assertReadySuspensionResponse, assertReadySuspensionResponse,
assertSuspendedProbes, assertSuspendedProbes,
prepareReadySuspension,
runGatewayNetworkClient, runGatewayNetworkClient,
runGatewaySuspensionPostRestartClient, runGatewaySuspensionPostRestartClient,
runGatewaySuspensionPreRestartClient, runGatewaySuspensionPreRestartClient,
@@ -80,6 +81,68 @@ describe("gateway network client", () => {
).toBe(3000); ).toBe(3000);
}); });
it("retries busy suspension preparation using the server delay", async () => {
const expiresAtMs = Date.now() + 10_000;
const rpc = vi
.fn()
.mockResolvedValueOnce({
status: 200,
body: {
ok: true,
payload: { status: "busy", retryAfterMs: 250, activeCount: 1, blockers: ["agent"] },
},
})
.mockResolvedValueOnce({
status: 200,
body: {
ok: true,
payload: {
status: "ready",
suspensionId: "lease-1",
expiresAtMs,
activeCount: 0,
blockers: [],
},
},
});
const delayImpl = vi.fn(async () => undefined);
await expect(
prepareReadySuspension(
{ deadline: Date.now() + 5_000, requestId: "request-1", rpc },
{ delayImpl },
),
).resolves.toMatchObject({ status: "ready", suspensionId: "lease-1" });
expect(rpc).toHaveBeenCalledTimes(2);
expect(rpc).toHaveBeenNthCalledWith(1, "gateway.suspend.prepare", {
requestId: "request-1",
});
expect(delayImpl).toHaveBeenCalledWith(250);
});
it("stops busy suspension retries at the client deadline", async () => {
let nowMs = 1_000;
const rpc = vi.fn(async () => ({
status: 200,
body: {
ok: true,
payload: { status: "busy", retryAfterMs: 250, activeCount: 1, blockers: ["agent"] },
},
}));
const delayImpl = vi.fn(async (ms: number) => {
nowMs += ms;
});
await expect(
prepareReadySuspension(
{ deadline: 1_250, requestId: "request-busy", rpc },
{ delayImpl, now: () => nowMs },
),
).rejects.toMatchObject({ name: "TimeoutError" });
expect(rpc).toHaveBeenCalledOnce();
expect(delayImpl).toHaveBeenCalledWith(250);
});
it("bounds a stalled suspension admin request by the client deadline", async () => { it("bounds a stalled suspension admin request by the client deadline", async () => {
let requestSignal: AbortSignal | null | undefined; let requestSignal: AbortSignal | null | undefined;
const fetchImpl = vi.fn<typeof fetch>((_input, init) => { const fetchImpl = vi.fn<typeof fetch>((_input, init) => {