perf(session-catalog): bound slow node fan-out (#110106)

This commit is contained in:
Peter Steinberger
2026-07-17 19:24:27 +01:00
committed by GitHub
parent f104c51e85
commit 67a5ffab77
4 changed files with 111 additions and 22 deletions
@@ -1407,6 +1407,45 @@ describe("Claude session catalog", () => {
]);
});
it("bounds how long a hung paired-node catalog can delay the caller", async () => {
vi.useFakeTimers();
try {
const invoke = vi.fn<PluginRuntime["nodes"]["invoke"]>(
async () => await new Promise<never>(() => {}),
);
const provider = captureCatalogProvider({
nodes: {
list: vi.fn().mockResolvedValue({
nodes: [
{
nodeId: "slow-node",
displayName: "Slow node",
connected: true,
commands: [CLAUDE_SESSIONS_LIST_COMMAND],
},
],
}),
invoke,
},
} as unknown as PluginRuntime);
const pending = provider.list({ hostIds: ["node:slow-node"] });
await vi.advanceTimersByTimeAsync(8_000);
await expect(pending).resolves.toEqual([
expect.objectContaining({
hostId: "node:slow-node",
connected: true,
sessions: [],
error: expect.objectContaining({ code: "NODE_INVOKE_FAILED" }),
}),
]);
expect(invoke).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 30_000 }));
} finally {
vi.useRealTimers();
}
});
it("starts paired-node discovery while the local catalog is still reading", async () => {
const home = await createHome();
process.env.HOME = home;
+19 -11
View File
@@ -4,6 +4,7 @@ import path from "node:path";
import { resolveDefaultAgentId } from "openclaw/plugin-sdk/agent-runtime";
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime";
import { withTimeout } from "openclaw/plugin-sdk/security-runtime";
import type {
SessionCatalogHost,
SessionCatalogProvider,
@@ -66,6 +67,9 @@ const MAX_TRANSCRIPT_SCAN_BYTES = 64 * 1024 * 1024;
const MAX_TRANSCRIPT_PAGE_BYTES = 20 * 1024 * 1024;
const NODE_INVOKE_TIMEOUT_MS = 30_000;
// Catalog refresh is fail-soft: one unhealthy machine must not hold the whole sidebar.
// The node invoke keeps running so cold native discovery can warm the next poll.
const NODE_CATALOG_LIST_RESPONSE_TIMEOUT_MS = 8_000;
const CLAUDE_HISTORY_IMPORT_MAX_ITEMS = 200;
const CLAUDE_HISTORY_IMPORT_MAX_BYTES = 512 * 1024;
@@ -1061,17 +1065,21 @@ async function listClaudeSessionCatalog(params: {
});
}
try {
const raw = await params.runtime.nodes.invoke({
nodeId: node.nodeId,
command: CLAUDE_SESSIONS_LIST_COMMAND,
params: {
limit: query.limitPerHost,
...(query.search ? { searchTerm: query.search } : {}),
...(query.cursors?.[hostId] ? { cursor: query.cursors[hostId] } : {}),
},
timeoutMs: NODE_INVOKE_TIMEOUT_MS,
scopes: ["operator.write"],
});
const raw = await withTimeout(
params.runtime.nodes.invoke({
nodeId: node.nodeId,
command: CLAUDE_SESSIONS_LIST_COMMAND,
params: {
limit: query.limitPerHost,
...(query.search ? { searchTerm: query.search } : {}),
...(query.cursors?.[hostId] ? { cursor: query.cursors[hostId] } : {}),
},
timeoutMs: NODE_INVOKE_TIMEOUT_MS,
scopes: ["operator.write"],
}),
NODE_CATALOG_LIST_RESPONSE_TIMEOUT_MS,
{ message: "paired node Claude session catalog timed out" },
);
return Object.assign(common, parseCatalogPage(unwrapNodePayload(raw)));
} catch {
return Object.assign(common, {
@@ -2,6 +2,7 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime";
import type { CodexThread } from "./app-server/protocol.js";
import { withTimeout } from "./app-server/timeout.js";
import { createCodexCliNodeConversationBindingData } from "./conversation-binding-data.js";
import { CODEX_CLI_SESSION_RESUME_COMMAND } from "./node-cli-sessions.js";
import {
@@ -44,6 +45,10 @@ const CODEX_NODE_CONTINUE_COMMANDS = [
CODEX_CLI_SESSION_RESUME_COMMAND,
] as const;
// Catalog refresh is fail-soft: one unhealthy machine must not hold the whole sidebar.
// The node invoke keeps running so cold native discovery can warm the next poll.
const NODE_CATALOG_LIST_RESPONSE_TIMEOUT_MS = 8_000;
export type CatalogNode = Awaited<ReturnType<PluginRuntime["nodes"]["list"]>>["nodes"][number];
export function nodeLabel(node: CatalogNode): string {
@@ -96,17 +101,21 @@ export async function listPairedNode(params: {
};
}
try {
const raw = await params.runtime.nodes.invoke({
nodeId: params.node.nodeId,
command: CODEX_APP_SERVER_THREADS_LIST_COMMAND,
params: {
cursor: params.query.cursors?.[hostId],
limit: params.query.limitPerHost,
searchTerm: params.query.search,
},
timeoutMs: NODE_INVOKE_TIMEOUT_MS,
scopes: ["operator.write"],
});
const raw = await withTimeout(
params.runtime.nodes.invoke({
nodeId: params.node.nodeId,
command: CODEX_APP_SERVER_THREADS_LIST_COMMAND,
params: {
cursor: params.query.cursors?.[hostId],
limit: params.query.limitPerHost,
searchTerm: params.query.search,
},
timeoutMs: NODE_INVOKE_TIMEOUT_MS,
scopes: ["operator.write"],
}),
NODE_CATALOG_LIST_RESPONSE_TIMEOUT_MS,
"paired node Codex session catalog timed out",
);
const page = filterCatalogPageByTitle(
parseCatalogPage(unwrapNodeInvokePayload(raw)),
params.query.search,
@@ -17,6 +17,7 @@ import {
type CodexAppServerBindingStore,
type CodexAppServerThreadBinding,
} from "./app-server/session-binding.test-helpers.js";
import { listPairedNode } from "./session-catalog-node-continue.js";
import { catalogError } from "./session-catalog-parsing.js";
import { CODEX_TERMINAL_RESUME_COMMAND } from "./session-catalog-terminal.js";
import {
@@ -995,6 +996,38 @@ describe("Codex supervision catalog", () => {
expect(JSON.stringify(result)).not.toContain("private transcript");
});
it("bounds how long a hung paired-node catalog can delay the caller", async () => {
vi.useFakeTimers();
try {
const invoke = vi.fn<PluginRuntime["nodes"]["invoke"]>(
async () => await new Promise<never>(() => {}),
);
const pending = listPairedNode({
runtime: { nodes: { invoke } } as unknown as PluginRuntime,
node: {
nodeId: "slow-node",
displayName: "Slow node",
connected: true,
commands: [CODEX_APP_SERVER_THREADS_LIST_COMMAND],
},
query: { limitPerHost: 40 },
adoptedSessions: new Map(),
});
await vi.advanceTimersByTimeAsync(8_000);
await expect(pending).resolves.toMatchObject({
hostId: "node:slow-node",
connected: true,
sessions: [],
error: { code: "NODE_INVOKE_FAILED" },
});
expect(invoke).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 65_000 }));
} finally {
vi.useRealTimers();
}
});
it("serves one bounded transcript page from the node host command", async () => {
const listTurnPage = vi.fn(async () => ({
data: [