fix(qa): isolate plugin tools MCP smoke

This commit is contained in:
Vincent Koc
2026-07-12 07:37:47 +02:00
parent d14bb8ec44
commit 8f8b44a3ce
2 changed files with 159 additions and 30 deletions
@@ -1,6 +1,9 @@
import { mkdir, writeFile } from "node:fs/promises";
import { createRequire } from "node:module";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import type { Client } from "@modelcontextprotocol/sdk/client/index.js";
import type { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../../helpers/temp-dir.js";
import { testing } from "./gateway-mcp-real-transports.js";
@@ -66,4 +69,59 @@ describe("gateway MCP real transport producer", () => {
]);
expect(mcp.envPatch).toStrictEqual({});
});
it("isolates the source plugin-tools MCP invocation", () => {
const root = createRepoRoot();
const invocation = testing.resolvePluginToolsMcpInvocation({
configPath: "/tmp/plugin-tools/openclaw.json",
homeDir: "/tmp/plugin-tools/home",
repoRoot: root,
stateDir: "/tmp/plugin-tools/state",
});
expect(invocation).toStrictEqual({
command: process.execPath,
args: [
"--import",
createRequire(import.meta.url).resolve("tsx"),
path.join(root, "src/mcp/plugin-tools-serve.ts"),
],
cwd: root,
env: {
HOME: "/tmp/plugin-tools/home",
OPENCLAW_CONFIG_PATH: "/tmp/plugin-tools/openclaw.json",
OPENCLAW_DISABLE_BUNDLED_PLUGINS: "1",
OPENCLAW_HOME: "/tmp/plugin-tools/home",
OPENCLAW_STATE_DIR: "/tmp/plugin-tools/state",
},
});
});
it("sets explicit timeouts for each plugin-tools MCP request", async () => {
const connect = vi.fn().mockResolvedValue(undefined);
const listTools = vi.fn().mockResolvedValue({
tools: [{ name: "memory_search" }],
});
const callTool = vi.fn().mockResolvedValue({
content: [{ type: "text", text: "MCP fact: the codename is ORBIT-9." }],
isError: false,
});
const client = { callTool, connect, listTools } as unknown as Client;
const transport = { pid: 42 } as StdioClientTransport;
await expect(testing.runMcpPluginToolsClientProof({ client, transport })).resolves.toContain(
"real plugin-tools pid=42",
);
expect(connect).toHaveBeenCalledWith(transport, { timeout: 180_000 });
expect(listTools).toHaveBeenCalledWith({}, { timeout: 180_000 });
expect(callTool).toHaveBeenCalledWith(
{
name: "memory_search",
arguments: { query: "ORBIT-9 codename", maxResults: 3 },
},
undefined,
{ timeout: 180_000 },
);
});
});
@@ -2,6 +2,7 @@ import { existsSync } from "node:fs";
import fs from "node:fs/promises";
// QA Lab producer proves Gateway and MCP scenarios across real process and protocol boundaries.
import { createServer, type Server } from "node:http";
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
@@ -29,7 +30,9 @@ const FIXTURE_TOOL_NAME = "memory_search";
const FIXTURE_FACT = "MCP fact: the codename is ORBIT-9.";
const STARTUP_GATE_TIMEOUT_MS = 30_000;
const MCP_CONNECT_TIMEOUT_MS = 30_000;
const MCP_PLUGIN_TOOLS_REQUEST_TIMEOUT_MS = 180_000;
const SOURCE_PATH = "test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts";
const requireFromHere = createRequire(import.meta.url);
type ScenarioId = "gateway-smoke" | "mcp-gateway-connect-startup-retry" | "mcp-plugin-tools-call";
@@ -71,6 +74,13 @@ type McpClientHandle = {
transport: StdioClientTransport;
};
type PluginToolsMcpInvocation = {
args: string[];
command: string;
cwd: string;
env: Record<string, string>;
};
const SCENARIOS = {
"gateway-smoke": {
title: "Gateway smoke evidence",
@@ -273,6 +283,86 @@ function resolveChannelMcpInvocation(params: {
);
}
function resolvePluginToolsMcpInvocation(params: {
configPath: string;
homeDir: string;
repoRoot: string;
stateDir: string;
}): PluginToolsMcpInvocation {
return {
command: process.execPath,
args: [
"--import",
requireFromHere.resolve("tsx"),
path.join(params.repoRoot, "src/mcp/plugin-tools-serve.ts"),
],
cwd: params.repoRoot,
env: {
HOME: params.homeDir,
OPENCLAW_CONFIG_PATH: params.configPath,
OPENCLAW_DISABLE_BUNDLED_PLUGINS: "1",
OPENCLAW_HOME: params.homeDir,
OPENCLAW_STATE_DIR: params.stateDir,
},
};
}
async function runMcpPluginToolsPhase<T>(phase: string, run: () => Promise<T>) {
const startedAt = Date.now();
try {
const value = await run();
return {
durationMs: Math.max(1, Date.now() - startedAt),
value,
};
} catch (error) {
throw new Error(
`plugin-tools MCP ${phase} failed after ${Math.max(1, Date.now() - startedAt)}ms: ${formatErrorMessage(error)}`,
{ cause: error },
);
}
}
async function runMcpPluginToolsClientProof(params: {
client: Client;
transport: StdioClientTransport;
}): Promise<string> {
const connected = await runMcpPluginToolsPhase("connect", () =>
params.client.connect(params.transport, { timeout: MCP_PLUGIN_TOOLS_REQUEST_TIMEOUT_MS }),
);
const listed = await runMcpPluginToolsPhase("listTools", () =>
params.client.listTools({}, { timeout: MCP_PLUGIN_TOOLS_REQUEST_TIMEOUT_MS }),
);
if (!listed.value.tools.some((tool) => tool.name === FIXTURE_TOOL_NAME)) {
throw new Error(
`fixture plugin tool was not listed: ${listed.value.tools.map((tool) => tool.name).join(", ")}`,
);
}
const called = await runMcpPluginToolsPhase("callTool", () =>
params.client.callTool(
{
name: FIXTURE_TOOL_NAME,
arguments: { query: "ORBIT-9 codename", maxResults: 3 },
},
undefined,
{ timeout: MCP_PLUGIN_TOOLS_REQUEST_TIMEOUT_MS },
),
);
if (called.value.isError || !JSON.stringify(called.value.content).includes(FIXTURE_FACT)) {
throw new Error(
`fixture plugin tool returned unexpected payload: ${JSON.stringify(called.value)}`,
);
}
return [
`real plugin-tools pid=${params.transport.pid ?? "unknown"}`,
`connect=${connected.durationMs}ms`,
`listTools=${listed.durationMs}ms`,
`callTool=${called.durationMs}ms`,
`listed and called ${FIXTURE_TOOL_NAME}`,
"received ORBIT-9",
].join("; ");
}
function parseJsonFrame(data: RawData): Record<string, unknown> | null {
try {
const text = Array.isArray(data)
@@ -638,21 +728,14 @@ async function runMcpPluginToolsProof(options: ProducerOptions): Promise<string>
]);
const configPath = await writePluginToolsConfig(runtimeRoot, fixture.pluginDir);
const stderrChunks: Buffer[] = [];
const invocation = resolvePluginToolsMcpInvocation({
configPath,
homeDir,
repoRoot: options.repoRoot,
stateDir,
});
const transport = new StdioClientTransport({
command: process.execPath,
args: [
"--import",
"tsx",
"--eval",
`import(${JSON.stringify(pathToFileURL(path.join(options.repoRoot, "src/mcp/plugin-tools-serve.ts")).href)}).then((module) => module.servePluginToolsMcp())`,
],
cwd: options.repoRoot,
env: {
...process.env,
HOME: homeDir,
OPENCLAW_CONFIG_PATH: configPath,
OPENCLAW_STATE_DIR: stateDir,
},
...invocation,
stderr: "pipe",
});
transport.stderr?.on("data", (chunk) => stderrChunks.push(Buffer.from(chunk)));
@@ -660,21 +743,7 @@ async function runMcpPluginToolsProof(options: ProducerOptions): Promise<string>
let details = "";
let proofError: Error | undefined;
try {
await client.connect(transport);
const listed = await client.listTools();
if (!listed.tools.some((tool) => tool.name === FIXTURE_TOOL_NAME)) {
throw new Error(
`fixture plugin tool was not listed: ${listed.tools.map((tool) => tool.name).join(", ")}`,
);
}
const result = await client.callTool({
name: FIXTURE_TOOL_NAME,
arguments: { query: "ORBIT-9 codename", maxResults: 3 },
});
if (result.isError || !JSON.stringify(result.content).includes(FIXTURE_FACT)) {
throw new Error(`fixture plugin tool returned unexpected payload: ${JSON.stringify(result)}`);
}
details = `real plugin-tools pid=${transport.pid ?? "unknown"}; listed and called ${FIXTURE_TOOL_NAME}; received ORBIT-9`;
details = await runMcpPluginToolsClientProof({ client, transport });
} catch (error) {
const stderr = Buffer.concat(stderrChunks).toString("utf8");
proofError = new Error(
@@ -761,4 +830,6 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
export const testing = {
resolveChannelMcpInvocation,
resolvePluginToolsMcpInvocation,
runMcpPluginToolsClientProof,
};