mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
test(qa): add provider runtime evidence (#101051)
This commit is contained in:
Dallin Romney
@@ -1,6 +1,6 @@
|
||||
// Diagnostics Otel plugin entrypoint registers its OpenClaw integration.
|
||||
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { createDiagnosticsOtelService } from "./src/service.js";
|
||||
import { createDiagnosticsOtelService } from "./runtime-api.js";
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "diagnostics-otel",
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
// Diagnostics Otel runtime API exposes the service factory to trusted runtime callers.
|
||||
export { createDiagnosticsOtelService } from "./src/service.js";
|
||||
export type { OpenClawPluginServiceContext } from "./api.js";
|
||||
@@ -222,6 +222,7 @@ describe("qa scenario catalog", () => {
|
||||
it("loads native test execution scenarios from YAML", () => {
|
||||
const scenario = readQaScenarioById("control-ui-chat-flow-playwright");
|
||||
const uxMatrix = readQaScenarioById("ux-matrix-evidence-dashboard");
|
||||
const otelSmoke = readQaScenarioById("qa-otel-smoke");
|
||||
|
||||
expect(scenario.execution.kind).toBe("playwright");
|
||||
if (scenario.execution.kind !== "playwright") {
|
||||
@@ -238,6 +239,17 @@ describe("qa scenario catalog", () => {
|
||||
expect(uxMatrix.execution.args).toStrictEqual(["--artifact-base", "${outputDir}"]);
|
||||
expect(uxMatrix.execution.config).toBeUndefined();
|
||||
expect(uxMatrix.coverage?.primary).toContain("qa.artifact-safety");
|
||||
expect(otelSmoke.execution.kind).toBe("script");
|
||||
if (otelSmoke.execution.kind !== "script") {
|
||||
throw new Error(`expected script scenario, got ${otelSmoke.execution.kind}`);
|
||||
}
|
||||
expect(otelSmoke.execution.args).toStrictEqual([
|
||||
"--output-dir",
|
||||
"${outputDir}",
|
||||
"--logs-exporter",
|
||||
"both",
|
||||
]);
|
||||
expect(otelSmoke.coverage?.secondary).not.toContain("harness.qa-lab");
|
||||
});
|
||||
|
||||
it("loads helper-backed HTTP API scenarios as supporting taxonomy coverage", () => {
|
||||
|
||||
@@ -7,13 +7,12 @@ scenario:
|
||||
primary:
|
||||
- telemetry.otel
|
||||
secondary:
|
||||
- harness.qa-lab
|
||||
- telemetry.plugin-sdk-runtime-exports
|
||||
objective: Execute the bounded local OTLP runtime producer directly and publish its OpenTelemetry assertions through the shared QA script evidence writer.
|
||||
successCriteria:
|
||||
- QA Lab launches the runtime producer directly with a bounded local collector configuration.
|
||||
- The producer emits shared script evidence plus its OTEL assertion summary.
|
||||
- The producer captures release-critical traces, metrics, and correlated logs from a real QA runtime execution.
|
||||
- The producer captures release-critical traces, metrics, and correlated logs from the diagnostics runtime.
|
||||
- Smoke assertions reject failed OTLP requests, missing required signals, and configured leak needles.
|
||||
docsRefs:
|
||||
- docs/gateway/opentelemetry.md
|
||||
@@ -21,6 +20,7 @@ scenario:
|
||||
codeRefs:
|
||||
- test/e2e/qa-lab/runtime/script-evidence.ts
|
||||
- test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts
|
||||
- extensions/diagnostics-otel/runtime-api.ts
|
||||
execution:
|
||||
kind: script
|
||||
path: test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts
|
||||
@@ -29,10 +29,6 @@ scenario:
|
||||
args:
|
||||
- --output-dir
|
||||
- ${outputDir}
|
||||
- --provider-mode
|
||||
- mock-openai
|
||||
- --scenario
|
||||
- otel-both-log-smoke
|
||||
- --logs-exporter
|
||||
- both
|
||||
timeoutMs: 120000
|
||||
|
||||
@@ -2077,6 +2077,14 @@ const SOURCE_TEST_TARGETS = new Map([
|
||||
"test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts",
|
||||
["test/e2e/qa-lab/runtime/qa-otel-smoke.e2e.test.ts"],
|
||||
],
|
||||
[
|
||||
"test/e2e/qa-lab/runtime/heartbeat-active-hours-runtime.ts",
|
||||
["test/e2e/qa-lab/runtime/heartbeat-active-hours-runtime.test.ts"],
|
||||
],
|
||||
[
|
||||
"test/e2e/qa-lab/runtime/telegram-bot-token-runtime.ts",
|
||||
["test/e2e/qa-lab/runtime/telegram-bot-token-runtime.test.ts"],
|
||||
],
|
||||
["src/plugins/runtime-sidecar-paths-baseline.ts", RUNTIME_SIDECAR_BASELINE_OWNER_TEST_TARGETS],
|
||||
["src/plugins/runtime-sidecar-paths.ts", RUNTIME_SIDECAR_PATH_CONSUMER_TEST_TARGETS],
|
||||
["ui/config/control-ui-chunking.ts", ["ui/src/app/control-ui-chunking.test.ts"]],
|
||||
|
||||
@@ -43,6 +43,14 @@ const UNGUARDED_RUNTIME_API_PLUGIN_IDS = [
|
||||
] as const;
|
||||
|
||||
const RUNTIME_API_EXPORT_GUARDS: Record<string, readonly string[]> = {
|
||||
[bundledPluginFile({
|
||||
rootDir: ROOT_DIR,
|
||||
pluginId: "diagnostics-otel",
|
||||
relativePath: "runtime-api.ts",
|
||||
})]: [
|
||||
'export { createDiagnosticsOtelService } from "./src/service.js";',
|
||||
'export type { OpenClawPluginServiceContext } from "./api.js";',
|
||||
],
|
||||
[bundledPluginFile({ rootDir: ROOT_DIR, pluginId: "discord", relativePath: "runtime-api.ts" })]: [
|
||||
'export { discordMessageActions, handleDiscordAction, isDiscordModerationAction, readDiscordChannelCreateParams, readDiscordChannelEditParams, readDiscordChannelMoveParams, readDiscordModerationCommand, readDiscordParentIdParam, requiredGuildPermissionForModerationAction, type DiscordModerationAction, type DiscordModerationCommand } from "./runtime-api.actions.js";',
|
||||
'export { auditDiscordChannelPermissions, collectDiscordAuditChannelIds, fetchDiscordApplicationId, fetchDiscordApplicationSummary, listDiscordDirectoryGroupsLive, listDiscordDirectoryPeersLive, parseApplicationIdFromToken, probeDiscord, resolveDiscordChannelAllowlist, resolveDiscordPrivilegedIntentsFromFlags, resolveDiscordUserAllowlist, setDiscordRuntime, type DiscordApplicationSummary, type DiscordChannelResolution, type DiscordPrivilegedIntentsSummary, type DiscordPrivilegedIntentStatus, type DiscordProbe, type DiscordUserResolution } from "./runtime-api.lookup.js";',
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { runHeartbeatActiveHoursRuntime } from "./heartbeat-active-hours-runtime.js";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { force: true, recursive: true })));
|
||||
});
|
||||
|
||||
describe("heartbeat active-hours runtime evidence", () => {
|
||||
it("observes active fire, quiet-hours skip, and reload fire", async () => {
|
||||
const artifactBase = await fs.mkdtemp(path.join(os.tmpdir(), "heartbeat-active-hours-"));
|
||||
tempDirs.push(artifactBase);
|
||||
const evidence = await runHeartbeatActiveHoursRuntime({
|
||||
artifactBase,
|
||||
repoRoot: process.cwd(),
|
||||
timeoutMs: 2_000,
|
||||
});
|
||||
|
||||
expect(evidence.entries[0]?.result.status).toBe("pass");
|
||||
const summary = JSON.parse(
|
||||
await fs.readFile(path.join(artifactBase, "heartbeat-active-hours-summary.json"), "utf8"),
|
||||
) as { observations: Array<{ outcome: string }> };
|
||||
expect(summary.observations.map((entry) => entry.outcome)).toEqual([
|
||||
"active-fire",
|
||||
"quiet-hours-skip",
|
||||
"active-fire",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,168 @@
|
||||
// Heartbeat active-hours evidence runs the real bounded scheduler and reload path.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import type { OpenClawConfig } from "../../../../src/config/types.openclaw.js";
|
||||
import { formatErrorMessage } from "../../../../src/infra/errors.js";
|
||||
import { isWithinActiveHours } from "../../../../src/infra/heartbeat-active-hours.js";
|
||||
import { startHeartbeatRunner } from "../../../../src/infra/heartbeat-runner.js";
|
||||
import { createQaScriptEvidenceWriter } from "./script-evidence.js";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 5_000;
|
||||
const HEARTBEAT_INTERVAL = "100ms";
|
||||
|
||||
type HeartbeatRuntimeOptions = {
|
||||
artifactBase: string;
|
||||
repoRoot: string;
|
||||
timeoutMs: number;
|
||||
};
|
||||
|
||||
type SchedulerObservation = {
|
||||
at: string;
|
||||
outcome: "active-fire" | "quiet-hours-skip";
|
||||
};
|
||||
|
||||
function parseOptions(argv: string[], repoRoot = process.cwd()): HeartbeatRuntimeOptions {
|
||||
let artifactBase = path.join(repoRoot, ".artifacts", "qa-e2e", "heartbeat-active-hours");
|
||||
let timeoutMs = DEFAULT_TIMEOUT_MS;
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--output-dir") {
|
||||
artifactBase = path.resolve(repoRoot, argv[++index] ?? "");
|
||||
continue;
|
||||
}
|
||||
if (arg === "--timeout-ms") {
|
||||
timeoutMs = Number(argv[++index]);
|
||||
continue;
|
||||
}
|
||||
if (arg === "--") {
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
||||
throw new Error("--timeout-ms must be a positive number");
|
||||
}
|
||||
return { artifactBase, repoRoot, timeoutMs };
|
||||
}
|
||||
|
||||
function heartbeatConfig(quietHours: boolean): OpenClawConfig {
|
||||
return {
|
||||
agents: {
|
||||
defaults: {
|
||||
heartbeat: {
|
||||
activeHours: quietHours
|
||||
? { start: "00:00", end: "00:00", timezone: "UTC" }
|
||||
: { start: "00:00", end: "24:00", timezone: "UTC" },
|
||||
every: HEARTBEAT_INTERVAL,
|
||||
target: "none",
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForObservation(
|
||||
observations: SchedulerObservation[],
|
||||
outcome: SchedulerObservation["outcome"],
|
||||
afterCount: number,
|
||||
timeoutMs: number,
|
||||
) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (observations.slice(afterCount).some((entry) => entry.outcome === outcome)) {
|
||||
return;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
}
|
||||
throw new Error(`heartbeat scheduler did not observe ${outcome} within ${timeoutMs}ms`);
|
||||
}
|
||||
|
||||
function createWriter(options: HeartbeatRuntimeOptions) {
|
||||
return createQaScriptEvidenceWriter({
|
||||
artifactBase: options.artifactBase,
|
||||
logFileName: "heartbeat-active-hours.log",
|
||||
primaryModel: "heartbeat/scheduler",
|
||||
providerMode: "mock-openai",
|
||||
repoRoot: options.repoRoot,
|
||||
target: {
|
||||
id: "heartbeat-active-hours",
|
||||
title: "Heartbeat active-hours scheduler",
|
||||
sourcePath: "test/e2e/qa-lab/runtime/heartbeat-active-hours-runtime.ts",
|
||||
primaryCoverageIds: ["automation.active-hours"],
|
||||
docsRefs: ["docs/gateway/heartbeat.md"],
|
||||
codeRefs: [
|
||||
"test/e2e/qa-lab/runtime/heartbeat-active-hours-runtime.ts",
|
||||
"src/infra/heartbeat-runner.ts",
|
||||
"src/infra/heartbeat-active-hours.ts",
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function runHeartbeatActiveHoursRuntime(options: HeartbeatRuntimeOptions) {
|
||||
await fs.mkdir(options.artifactBase, { recursive: true });
|
||||
const writer = createWriter(options);
|
||||
const startedAt = Date.now();
|
||||
const observations: SchedulerObservation[] = [];
|
||||
let currentConfig = heartbeatConfig(false);
|
||||
const runner = startHeartbeatRunner({
|
||||
cfg: currentConfig,
|
||||
readCurrentConfig: () => currentConfig,
|
||||
runOnce: async ({ cfg, heartbeat }) => {
|
||||
const active = isWithinActiveHours(cfg, heartbeat);
|
||||
const outcome = active ? "active-fire" : "quiet-hours-skip";
|
||||
observations.push({ at: new Date().toISOString(), outcome });
|
||||
writer.appendLog(`heartbeat-active-hours: ${outcome}\n`);
|
||||
return active
|
||||
? { status: "ran", durationMs: 1 }
|
||||
: { status: "skipped", reason: "quiet-hours" };
|
||||
},
|
||||
stableSchedulerSeed: "qa-heartbeat-active-hours",
|
||||
});
|
||||
try {
|
||||
await waitForObservation(observations, "active-fire", 0, options.timeoutMs);
|
||||
const beforeQuiet = observations.length;
|
||||
currentConfig = heartbeatConfig(true);
|
||||
runner.updateConfig(currentConfig);
|
||||
await waitForObservation(observations, "quiet-hours-skip", beforeQuiet, options.timeoutMs);
|
||||
const beforeReload = observations.length;
|
||||
currentConfig = heartbeatConfig(false);
|
||||
runner.updateConfig(currentConfig);
|
||||
await waitForObservation(observations, "active-fire", beforeReload, options.timeoutMs);
|
||||
|
||||
const summaryPath = path.join(options.artifactBase, "heartbeat-active-hours-summary.json");
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify({ observations }, null, 2)}\n`, "utf8");
|
||||
return await writer.write({
|
||||
artifacts: [{ kind: "summary", filePath: summaryPath }],
|
||||
details: "Observed active fire, quiet-hours skip, and active-hours reload fire",
|
||||
durationMs: Math.max(1, Date.now() - startedAt),
|
||||
status: "pass",
|
||||
});
|
||||
} catch (error) {
|
||||
const details = formatErrorMessage(error);
|
||||
writer.appendLog(`heartbeat-active-hours: ${details}\n`);
|
||||
return await writer.write({
|
||||
details,
|
||||
durationMs: Math.max(1, Date.now() - startedAt),
|
||||
status: "fail",
|
||||
});
|
||||
} finally {
|
||||
runner.stop();
|
||||
}
|
||||
}
|
||||
|
||||
export const testing = { heartbeatConfig, parseOptions, waitForObservation };
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
runHeartbeatActiveHoursRuntime(parseOptions(process.argv.slice(2)))
|
||||
.then((evidence) => {
|
||||
const status = evidence.entries[0]?.result.status;
|
||||
process.stdout.write(`heartbeat-active-hours: ${status}\n`);
|
||||
process.exitCode = status === "pass" ? 0 : 1;
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
process.stderr.write(`heartbeat-active-hours: ${formatErrorMessage(error)}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -1,18 +1,25 @@
|
||||
// QA OTEL Smoke runtime supports OpenClaw repository automation.
|
||||
|
||||
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
|
||||
import { spawn } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdir, mkdtemp, open, rm, writeFile } from "node:fs/promises";
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||
import { Socket } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { gunzipSync } from "node:zlib";
|
||||
import { resolveTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion";
|
||||
import { stripLeadingPackageManagerSeparator } from "../../../../scripts/lib/arg-utils.mjs";
|
||||
import { resolveWindowsTaskkillPath } from "../../../../scripts/lib/windows-taskkill.mjs";
|
||||
import {
|
||||
createDiagnosticTraceContext,
|
||||
emitTrustedDiagnosticEvent,
|
||||
emitTrustedDiagnosticEventWithPrivateData,
|
||||
waitForDiagnosticEventsDrained,
|
||||
} from "openclaw/plugin-sdk/diagnostic-runtime";
|
||||
import {
|
||||
createDiagnosticsOtelService,
|
||||
type OpenClawPluginServiceContext,
|
||||
} from "../../../../extensions/diagnostics-otel/runtime-api.js";
|
||||
import { onTrustedInternalDiagnosticEvent } from "../../../../src/infra/diagnostic-events.js";
|
||||
import { createQaScriptEvidenceWriter } from "./script-evidence.js";
|
||||
|
||||
type CollectorMode = "local" | "docker";
|
||||
@@ -53,10 +60,6 @@ type CliOptions = {
|
||||
collectorMode: CollectorMode;
|
||||
logsExporter: OtelLogsExporter;
|
||||
outputDir: string;
|
||||
providerMode: string;
|
||||
scenarioId: string;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
help: boolean;
|
||||
};
|
||||
|
||||
@@ -108,12 +111,6 @@ type StdoutDiagnosticLogRecord = {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
const DEFAULT_SCENARIO_ID = "otel-trace-smoke";
|
||||
const LOGS_EXPORTER_SCENARIO_IDS = {
|
||||
otlp: "otel-trace-smoke",
|
||||
stdout: "otel-stdout-log-smoke",
|
||||
both: "otel-both-log-smoke",
|
||||
} satisfies Record<OtelLogsExporter, string>;
|
||||
const DEFAULT_DOCKER_COLLECTOR_IMAGE =
|
||||
process.env.OPENCLAW_QA_OTEL_COLLECTOR_IMAGE || "otel/opentelemetry-collector:0.104.0";
|
||||
const OTLP_SIGNAL_PATHS = new Map<string, OtlpSignal>([
|
||||
@@ -128,6 +125,8 @@ const REQUIRED_SPAN_NAMES = [
|
||||
"openclaw.message.delivery",
|
||||
] as const;
|
||||
const REQUIRED_METRIC_NAMES = ["openclaw.harness.duration_ms"] as const;
|
||||
const DIRECT_RUN_ID = "qa-otel-direct-run";
|
||||
const DIRECT_CALL_ID = "qa-otel-direct-call";
|
||||
const DISALLOWED_ATTRIBUTE_KEYS = new Set([
|
||||
"openclaw.runId",
|
||||
"openclaw.chatId",
|
||||
@@ -144,7 +143,12 @@ const DISALLOWED_ATTRIBUTE_KEYS = new Set([
|
||||
"openclaw.call_id",
|
||||
"openclaw.tool_call_id",
|
||||
]);
|
||||
const DISALLOWED_BODY_NEEDLES = ["OTEL-QA-SECRET", "OTEL-QA-OK"];
|
||||
const DISALLOWED_BODY_NEEDLES = [
|
||||
"OTEL-QA-SECRET",
|
||||
"OTEL-QA-OK",
|
||||
DIRECT_RUN_ID,
|
||||
DIRECT_CALL_ID,
|
||||
];
|
||||
const COLLECTOR_OUTPUT_TAIL_BYTES = 16_000;
|
||||
const POSITIVE_INTEGER_PATTERN = /^[1-9]\d*$/u;
|
||||
const MAX_OTLP_COMPRESSED_BODY_BYTES = readPositiveIntegerEnv(
|
||||
@@ -159,16 +163,10 @@ const MAX_CAPTURED_BODY_TEXT_BYTES = readPositiveIntegerEnv(
|
||||
"OPENCLAW_QA_OTEL_MAX_CAPTURED_BODY_TEXT_BYTES",
|
||||
512 * 1024,
|
||||
);
|
||||
const QA_SUITE_TIMEOUT_MS = readPositiveIntegerEnv(
|
||||
"OPENCLAW_QA_OTEL_SUITE_TIMEOUT_MS",
|
||||
10 * 60 * 1000,
|
||||
);
|
||||
const QA_SUITE_KILL_GRACE_MS = readPositiveIntegerEnv("OPENCLAW_QA_OTEL_SUITE_KILL_GRACE_MS", 5000);
|
||||
const MAX_STDOUT_DIAGNOSTIC_LINE_BYTES = readPositiveIntegerEnv(
|
||||
"OPENCLAW_QA_OTEL_MAX_STDOUT_DIAGNOSTIC_LINE_BYTES",
|
||||
512 * 1024,
|
||||
);
|
||||
const GATEWAY_STDOUT_ARTIFACT_READ_CHUNK_BYTES = 64 * 1024;
|
||||
const QA_OTEL_ENV_TO_CLEAR = [
|
||||
"OTEL_SDK_DISABLED",
|
||||
"OTEL_TRACES_EXPORTER",
|
||||
@@ -226,24 +224,21 @@ function oversizedBodyError(
|
||||
}
|
||||
|
||||
function usage(): string {
|
||||
return `Usage: pnpm qa:otel:smoke [--collector local|docker] [--logs-exporter otlp|stdout|both] [--output-dir <path>] [--provider-mode <mode>] [--scenario <id>] [--model <ref>] [--alt-model <ref>]
|
||||
return `Usage: pnpm qa:otel:smoke [--collector local|docker] [--logs-exporter otlp|stdout|both] [--output-dir <path>]
|
||||
|
||||
Runs a QA-lab scenario with diagnostics-otel enabled, then asserts the emitted
|
||||
signal shape and privacy contract. The default collector is an in-process
|
||||
OTLP/HTTP receiver. Use --collector docker to put a real OpenTelemetry
|
||||
Collector container in front of the receiver.
|
||||
Runs the diagnostics-otel runtime producer directly, then asserts the emitted
|
||||
signal shape and privacy contract. The default collector is an in-process OTLP/HTTP
|
||||
receiver. Use --collector docker to put a real OpenTelemetry Collector container
|
||||
in front of the receiver.
|
||||
`;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): CliOptions {
|
||||
const args = stripLeadingPackageManagerSeparator(argv);
|
||||
let scenarioExplicit = false;
|
||||
const args = argv[0] === "--" ? argv.slice(1) : argv;
|
||||
const options: CliOptions = {
|
||||
collectorMode: "local",
|
||||
logsExporter: "otlp",
|
||||
outputDir: path.join(".artifacts", "qa-e2e", `otel-smoke-${createOtelSmokeRunId()}`),
|
||||
providerMode: "mock-openai",
|
||||
scenarioId: DEFAULT_SCENARIO_ID,
|
||||
help: false,
|
||||
};
|
||||
const seen = new Set<string>();
|
||||
@@ -288,55 +283,16 @@ function parseArgs(argv: string[]): CliOptions {
|
||||
);
|
||||
}
|
||||
options.logsExporter = value;
|
||||
} else if (arg === "--provider-mode") {
|
||||
const value = readValue();
|
||||
recordOnce(arg);
|
||||
options.providerMode = value;
|
||||
} else if (arg === "--scenario") {
|
||||
const value = readValue();
|
||||
recordOnce(arg);
|
||||
options.scenarioId = value;
|
||||
scenarioExplicit = true;
|
||||
} else if (arg === "--model") {
|
||||
const value = readValue();
|
||||
recordOnce(arg);
|
||||
options.primaryModel = value;
|
||||
} else if (arg === "--alt-model") {
|
||||
const value = readValue();
|
||||
recordOnce(arg);
|
||||
options.alternateModel = value;
|
||||
} else {
|
||||
throw new Error(`unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
|
||||
const expectedScenarioId = LOGS_EXPORTER_SCENARIO_IDS[options.logsExporter];
|
||||
const knownLogsExporterScenarioIds = new Set(Object.values(LOGS_EXPORTER_SCENARIO_IDS));
|
||||
if (
|
||||
scenarioExplicit &&
|
||||
knownLogsExporterScenarioIds.has(options.scenarioId) &&
|
||||
options.scenarioId !== expectedScenarioId
|
||||
) {
|
||||
throw new Error(
|
||||
`--logs-exporter ${options.logsExporter} requires --scenario ${expectedScenarioId}; ` +
|
||||
`got ${options.scenarioId}`,
|
||||
);
|
||||
}
|
||||
if (!scenarioExplicit) {
|
||||
options.scenarioId = expectedScenarioId;
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function disallowedBodyNeedles(options: CliOptions): string[] {
|
||||
const scenarioId = options.scenarioId.trim();
|
||||
const needles = new Set(DISALLOWED_BODY_NEEDLES);
|
||||
if (scenarioId) {
|
||||
needles.add(`agent:qa:${scenarioId}`);
|
||||
needles.add(`Agent:qa:${scenarioId}`);
|
||||
}
|
||||
return [...needles];
|
||||
function disallowedBodyNeedles(): string[] {
|
||||
return [...DISALLOWED_BODY_NEEDLES];
|
||||
}
|
||||
|
||||
async function readRequestBody(
|
||||
@@ -1117,50 +1073,6 @@ function createStdoutDiagnosticLogCapture(maxLineBytes = MAX_STDOUT_DIAGNOSTIC_L
|
||||
};
|
||||
}
|
||||
|
||||
async function appendGatewayStdoutArtifactLogs(params: {
|
||||
capture: ReturnType<typeof createStdoutDiagnosticLogCapture>;
|
||||
outputDir: string;
|
||||
}): Promise<void> {
|
||||
const gatewayStdoutPath = path.join(
|
||||
params.outputDir,
|
||||
"artifacts",
|
||||
"gateway-runtime",
|
||||
"gateway.stdout.log",
|
||||
);
|
||||
try {
|
||||
await appendUtf8FileToStdoutDiagnosticCapture(
|
||||
gatewayStdoutPath,
|
||||
params.capture,
|
||||
GATEWAY_STDOUT_ARTIFACT_READ_CHUNK_BYTES,
|
||||
);
|
||||
params.capture.flush();
|
||||
} catch (error) {
|
||||
if (!isErrnoCode(error, "ENOENT")) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function appendUtf8FileToStdoutDiagnosticCapture(
|
||||
filePath: string,
|
||||
capture: ReturnType<typeof createStdoutDiagnosticLogCapture>,
|
||||
chunkBytes = GATEWAY_STDOUT_ARTIFACT_READ_CHUNK_BYTES,
|
||||
): Promise<void> {
|
||||
const file = await open(filePath, "r");
|
||||
try {
|
||||
const buffer = Buffer.alloc(Math.max(1, chunkBytes));
|
||||
for (;;) {
|
||||
const { bytesRead } = await file.read(buffer, 0, buffer.length);
|
||||
if (bytesRead === 0) {
|
||||
break;
|
||||
}
|
||||
capture.append(buffer.subarray(0, bytesRead));
|
||||
}
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function stopDockerContainer(name: string): Promise<void> {
|
||||
await new Promise<void>((resolve) => {
|
||||
const child = spawn("docker", ["stop", name], {
|
||||
@@ -1297,245 +1209,6 @@ service:
|
||||
};
|
||||
}
|
||||
|
||||
function openClawEntryArgs(): string[] {
|
||||
if (existsSync(path.join(process.cwd(), "scripts", "run-node.mjs"))) {
|
||||
return ["scripts/run-node.mjs"];
|
||||
}
|
||||
return ["openclaw.mjs"];
|
||||
}
|
||||
|
||||
function spawnOpenClaw(args: string[], env: NodeJS.ProcessEnv): ChildProcess {
|
||||
return spawn(process.execPath, [...openClawEntryArgs(), ...args], {
|
||||
detached: process.platform !== "win32",
|
||||
env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForChild(
|
||||
child: ChildProcess,
|
||||
timeoutMs = QA_SUITE_TIMEOUT_MS,
|
||||
killGraceMs = QA_SUITE_KILL_GRACE_MS,
|
||||
): Promise<number> {
|
||||
const resolvedTimeoutMs = resolveTimerTimeoutMs(timeoutMs, QA_SUITE_TIMEOUT_MS);
|
||||
const childExit = new Promise<number>((resolve) => {
|
||||
child.once("close", (code) => resolve(code ?? 1));
|
||||
});
|
||||
let timeoutHandle: NodeJS.Timeout | undefined;
|
||||
const timeout = new Promise<"timeout">((resolve) => {
|
||||
timeoutHandle = setTimeout(() => resolve("timeout"), resolvedTimeoutMs);
|
||||
timeoutHandle.unref();
|
||||
});
|
||||
const result = await Promise.race([childExit, timeout]).finally(() => {
|
||||
if (timeoutHandle) {
|
||||
clearTimeout(timeoutHandle);
|
||||
}
|
||||
});
|
||||
if (result !== "timeout") {
|
||||
return result;
|
||||
}
|
||||
|
||||
const cleanupPids = collectChildProcessTreePids(child);
|
||||
terminateChildTree(child, "SIGTERM", cleanupPids);
|
||||
if (!(await waitForProcessTreeExit(child, killGraceMs, cleanupPids))) {
|
||||
terminateChildTree(child, "SIGKILL", cleanupPids);
|
||||
await waitForProcessTreeExit(child, 1000, cleanupPids);
|
||||
}
|
||||
throw new Error(`openclaw qa suite timed out after ${resolvedTimeoutMs}ms`);
|
||||
}
|
||||
|
||||
function collectChildProcessTreePids(child: ChildProcess): number[] {
|
||||
if (process.platform === "win32" || typeof child.pid !== "number") {
|
||||
return [];
|
||||
}
|
||||
const ps = spawnSync("ps", ["-axo", "pid=,ppid="], { encoding: "utf8" });
|
||||
if (ps.status !== 0) {
|
||||
return [child.pid];
|
||||
}
|
||||
const childrenByParent = new Map<number, number[]>();
|
||||
for (const line of ps.stdout.split("\n")) {
|
||||
const match = line.trim().match(/^(\d+)\s+(\d+)$/u);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const pid = Number(match[1]);
|
||||
const ppid = Number(match[2]);
|
||||
const siblings = childrenByParent.get(ppid) ?? [];
|
||||
siblings.push(pid);
|
||||
childrenByParent.set(ppid, siblings);
|
||||
}
|
||||
const pids = [child.pid];
|
||||
for (const parentPid of pids) {
|
||||
for (const pid of childrenByParent.get(parentPid) ?? []) {
|
||||
pids.push(pid);
|
||||
}
|
||||
}
|
||||
return [...new Set(pids)];
|
||||
}
|
||||
|
||||
function terminateChildTree(
|
||||
child: ChildProcess,
|
||||
signal: NodeJS.Signals,
|
||||
pids = collectChildProcessTreePids(child),
|
||||
platform = process.platform,
|
||||
runTaskkill = spawnSync,
|
||||
): void {
|
||||
if (platform === "win32") {
|
||||
if (typeof child.pid === "number") {
|
||||
const result = runTaskkill(
|
||||
resolveWindowsTaskkillPath(),
|
||||
["/PID", String(child.pid), "/T", "/F"],
|
||||
{
|
||||
stdio: "ignore",
|
||||
},
|
||||
);
|
||||
if (result.status === 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
child.kill(signal);
|
||||
return;
|
||||
}
|
||||
if (pids.length > 0) {
|
||||
for (const pid of pids.toReversed()) {
|
||||
signalProcessGroupOrPid(pid, signal);
|
||||
}
|
||||
return;
|
||||
}
|
||||
child.kill(signal);
|
||||
}
|
||||
|
||||
function signalProcessGroupOrPid(pid: number, signal: NodeJS.Signals): void {
|
||||
try {
|
||||
process.kill(-pid, signal);
|
||||
} catch {
|
||||
try {
|
||||
process.kill(pid, signal);
|
||||
} catch {
|
||||
// Already gone.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isErrnoCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
"code" in error &&
|
||||
typeof error.code === "string" &&
|
||||
error.code === code
|
||||
);
|
||||
}
|
||||
|
||||
function processIdOrGroupIsAlive(pid: number): boolean {
|
||||
try {
|
||||
process.kill(-pid, 0);
|
||||
return true;
|
||||
} catch (groupError) {
|
||||
if (isErrnoCode(groupError, "EPERM")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch (pidError) {
|
||||
return isErrnoCode(pidError, "EPERM");
|
||||
}
|
||||
}
|
||||
|
||||
function processTreeIsAlive(
|
||||
child: ChildProcess,
|
||||
pids = collectChildProcessTreePids(child),
|
||||
): boolean {
|
||||
if (process.platform === "win32") {
|
||||
return child.exitCode === null && child.signalCode === null;
|
||||
}
|
||||
return pids.some((pid) => processIdOrGroupIsAlive(pid));
|
||||
}
|
||||
|
||||
async function waitForProcessTreeExit(
|
||||
child: ChildProcess,
|
||||
timeoutMs: number,
|
||||
pids = collectChildProcessTreePids(child),
|
||||
): Promise<boolean> {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
if (!processTreeIsAlive(child, pids)) {
|
||||
return true;
|
||||
}
|
||||
await delay(50);
|
||||
}
|
||||
return !processTreeIsAlive(child, pids);
|
||||
}
|
||||
|
||||
function relayParentSignalsToChild(child: ChildProcess): () => void {
|
||||
if (process.platform === "win32") {
|
||||
return () => {};
|
||||
}
|
||||
const handlers: Array<{ signal: NodeJS.Signals; handler: () => void }> = [];
|
||||
const cleanup = () => {
|
||||
for (const { signal, handler } of handlers) {
|
||||
process.off(signal, handler);
|
||||
}
|
||||
handlers.length = 0;
|
||||
};
|
||||
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
|
||||
const handler = () => {
|
||||
terminateChildTree(child, signal);
|
||||
cleanup();
|
||||
process.kill(process.pid, signal);
|
||||
};
|
||||
handlers.push({ signal, handler });
|
||||
process.once(signal, handler);
|
||||
}
|
||||
return cleanup;
|
||||
}
|
||||
|
||||
function buildQaEnv(port: number): NodeJS.ProcessEnv {
|
||||
const env = { ...process.env };
|
||||
for (const key of QA_OTEL_ENV_TO_CLEAR) {
|
||||
delete env[key];
|
||||
}
|
||||
env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = `http://127.0.0.1:${port}/v1/traces`;
|
||||
env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT = `http://127.0.0.1:${port}/v1/metrics`;
|
||||
env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = `http://127.0.0.1:${port}/v1/logs`;
|
||||
env.OTEL_SERVICE_NAME = "openclaw-qa-lab-otel-smoke";
|
||||
env.OTEL_SEMCONV_STABILITY_OPT_IN = "gen_ai_latest_experimental";
|
||||
env.OPENCLAW_QA_SUITE_PROGRESS = env.OPENCLAW_QA_SUITE_PROGRESS ?? "1";
|
||||
return env;
|
||||
}
|
||||
|
||||
function resolveQaOutputDir(outputDir: string, repoRoot = process.cwd()): string {
|
||||
const relative = path.relative(repoRoot, path.resolve(repoRoot, outputDir));
|
||||
if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
||||
throw new Error("--output-dir must stay within the repo root");
|
||||
}
|
||||
return relative ? relative.split(path.sep).join("/") : ".";
|
||||
}
|
||||
|
||||
function buildQaArgs(options: CliOptions, repoRoot = process.cwd()): string[] {
|
||||
const args = [
|
||||
"qa",
|
||||
"suite",
|
||||
"--provider-mode",
|
||||
options.providerMode,
|
||||
"--scenario",
|
||||
options.scenarioId,
|
||||
"--concurrency",
|
||||
"1",
|
||||
"--output-dir",
|
||||
resolveQaOutputDir(options.outputDir, repoRoot),
|
||||
"--fast",
|
||||
];
|
||||
if (options.primaryModel) {
|
||||
args.push("--model", options.primaryModel);
|
||||
}
|
||||
if (options.alternateModel) {
|
||||
args.push("--alt-model", options.alternateModel);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function collectAttributeKeys(spans: CapturedSpan[]): Set<string> {
|
||||
const keys = new Set<string>();
|
||||
for (const span of spans) {
|
||||
@@ -1587,6 +1260,191 @@ async function delay(ms: number): Promise<void> {
|
||||
});
|
||||
}
|
||||
|
||||
function createDirectProducerContext(params: {
|
||||
endpoint: string;
|
||||
logsExporter: OtelLogsExporter;
|
||||
outputDir: string;
|
||||
writeLog: (line: string) => void;
|
||||
}): OpenClawPluginServiceContext {
|
||||
return {
|
||||
config: {
|
||||
diagnostics: {
|
||||
enabled: true,
|
||||
otel: {
|
||||
enabled: true,
|
||||
endpoint: params.endpoint,
|
||||
protocol: "http/protobuf",
|
||||
traces: true,
|
||||
metrics: true,
|
||||
logs: true,
|
||||
logsExporter: params.logsExporter,
|
||||
},
|
||||
},
|
||||
},
|
||||
internalDiagnostics: {
|
||||
emit: emitTrustedDiagnosticEventWithPrivateData,
|
||||
onEvent: onTrustedInternalDiagnosticEvent,
|
||||
},
|
||||
logger: {
|
||||
debug: (...args) => params.writeLog(`${args.map(String).join(" ")}\n`),
|
||||
error: (...args) => params.writeLog(`${args.map(String).join(" ")}\n`),
|
||||
info: (...args) => params.writeLog(`${args.map(String).join(" ")}\n`),
|
||||
warn: (...args) => params.writeLog(`${args.map(String).join(" ")}\n`),
|
||||
},
|
||||
stateDir: params.outputDir,
|
||||
};
|
||||
}
|
||||
|
||||
async function runDirectTelemetryProducer(params: {
|
||||
endpoint: string;
|
||||
logsExporter: OtelLogsExporter;
|
||||
outputDir: string;
|
||||
writeLog: (line: string) => void;
|
||||
}) {
|
||||
const service = createDiagnosticsOtelService();
|
||||
const context = createDirectProducerContext(params);
|
||||
const previousEnv = new Map<string, string | undefined>();
|
||||
for (const key of QA_OTEL_ENV_TO_CLEAR) {
|
||||
previousEnv.set(key, process.env[key]);
|
||||
delete process.env[key];
|
||||
}
|
||||
previousEnv.set("OTEL_SERVICE_NAME", process.env.OTEL_SERVICE_NAME);
|
||||
previousEnv.set("OTEL_SEMCONV_STABILITY_OPT_IN", process.env.OTEL_SEMCONV_STABILITY_OPT_IN);
|
||||
process.env.OTEL_SERVICE_NAME = "openclaw-qa-lab-otel-smoke";
|
||||
process.env.OTEL_SEMCONV_STABILITY_OPT_IN = "gen_ai_latest_experimental";
|
||||
const traceId = "4bf92f3577b34da6a3ce929d0e0e4736";
|
||||
const harnessTrace = createDiagnosticTraceContext({
|
||||
traceId,
|
||||
spanId: "00f067aa0ba902b7",
|
||||
traceFlags: "01",
|
||||
});
|
||||
const runTrace = createDiagnosticTraceContext({
|
||||
traceId,
|
||||
spanId: "1111111111111111",
|
||||
parentSpanId: harnessTrace.spanId,
|
||||
traceFlags: "01",
|
||||
});
|
||||
const modelTrace = createDiagnosticTraceContext({
|
||||
traceId,
|
||||
spanId: "2222222222222222",
|
||||
parentSpanId: runTrace.spanId,
|
||||
traceFlags: "01",
|
||||
});
|
||||
await service.start(context);
|
||||
try {
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "harness.run.started",
|
||||
runId: DIRECT_RUN_ID,
|
||||
harnessId: "qa-otel-direct",
|
||||
pluginId: "diagnostics-otel",
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
channel: "qa",
|
||||
trace: harnessTrace,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "run.started",
|
||||
runId: DIRECT_RUN_ID,
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
channel: "qa",
|
||||
trace: runTrace,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "context.assembled",
|
||||
runId: DIRECT_RUN_ID,
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
channel: "qa",
|
||||
messageCount: 1,
|
||||
historyTextChars: 0,
|
||||
historyImageBlocks: 0,
|
||||
maxMessageTextChars: 0,
|
||||
systemPromptChars: 32,
|
||||
promptChars: 64,
|
||||
promptImages: 0,
|
||||
trace: runTrace,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "model.call.started",
|
||||
runId: DIRECT_RUN_ID,
|
||||
callId: DIRECT_CALL_ID,
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
api: "responses",
|
||||
transport: "direct",
|
||||
trace: modelTrace,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "log.record",
|
||||
level: "info",
|
||||
message: "QA OTEL direct runtime producer",
|
||||
loggerName: "qa-otel-smoke",
|
||||
trace: modelTrace,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "message.delivery.completed",
|
||||
channel: "qa",
|
||||
deliveryKind: "text",
|
||||
durationMs: 2,
|
||||
resultCount: 1,
|
||||
trace: runTrace,
|
||||
});
|
||||
emitTrustedDiagnosticEventWithPrivateData(
|
||||
{
|
||||
type: "model.call.completed",
|
||||
runId: DIRECT_RUN_ID,
|
||||
callId: DIRECT_CALL_ID,
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
api: "responses",
|
||||
transport: "direct",
|
||||
durationMs: 5,
|
||||
usage: { input: 2, output: 1, total: 3 },
|
||||
trace: modelTrace,
|
||||
},
|
||||
{
|
||||
modelContent: {
|
||||
inputMessages: ["OTEL-QA-SECRET"],
|
||||
outputMessages: ["OTEL-QA-OK"],
|
||||
},
|
||||
},
|
||||
);
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "run.completed",
|
||||
runId: DIRECT_RUN_ID,
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
channel: "qa",
|
||||
durationMs: 8,
|
||||
outcome: "completed",
|
||||
trace: runTrace,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "harness.run.completed",
|
||||
runId: DIRECT_RUN_ID,
|
||||
harnessId: "qa-otel-direct",
|
||||
pluginId: "diagnostics-otel",
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
channel: "qa",
|
||||
durationMs: 10,
|
||||
outcome: "completed",
|
||||
trace: harnessTrace,
|
||||
});
|
||||
await waitForDiagnosticEventsDrained();
|
||||
} finally {
|
||||
await service.stop?.(context);
|
||||
for (const [key, value] of previousEnv) {
|
||||
if (value === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function hasRequiredSmokeSignals(params: {
|
||||
logsExporter: OtelLogsExporter;
|
||||
receiver: ReturnType<typeof startLocalOtlpReceiver>;
|
||||
@@ -1828,18 +1686,19 @@ async function main() {
|
||||
const writer = createQaScriptEvidenceWriter({
|
||||
artifactBase: options.outputDir,
|
||||
logFileName: "qa-otel-smoke.log",
|
||||
primaryModel: options.primaryModel ?? "gpt-5.5",
|
||||
providerMode: options.providerMode as "mock-openai" | "live-frontier",
|
||||
primaryModel: "gpt-5.5",
|
||||
providerMode: "mock-openai",
|
||||
repoRoot: process.cwd(),
|
||||
target: {
|
||||
id: "qa-otel-smoke",
|
||||
title: "QA OTEL smoke evidence",
|
||||
sourcePath: "test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts",
|
||||
primaryCoverageIds: ["telemetry.otel"],
|
||||
secondaryCoverageIds: ["harness.qa-lab", "telemetry.plugin-sdk-runtime-exports"],
|
||||
secondaryCoverageIds: ["telemetry.plugin-sdk-runtime-exports"],
|
||||
docsRefs: ["docs/gateway/opentelemetry.md", "docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: [
|
||||
"test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts",
|
||||
"extensions/diagnostics-otel/runtime-api.ts",
|
||||
"extensions/diagnostics-otel/src/service.ts",
|
||||
],
|
||||
},
|
||||
@@ -1854,7 +1713,7 @@ async function main() {
|
||||
writer.appendLog(chunk);
|
||||
process.stderr.write(String(chunk));
|
||||
};
|
||||
const receiver = startLocalOtlpReceiver(disallowedBodyNeedles(options));
|
||||
const receiver = startLocalOtlpReceiver(disallowedBodyNeedles());
|
||||
const port = await receiver.listen();
|
||||
writeStdout(`qa-otel-smoke: local OTLP receiver listening on http://127.0.0.1:${port}\n`);
|
||||
|
||||
@@ -1871,30 +1730,24 @@ async function main() {
|
||||
);
|
||||
}
|
||||
|
||||
const child = spawnOpenClaw(buildQaArgs(options), buildQaEnv(exportPort));
|
||||
const cleanupSignalRelay = relayParentSignalsToChild(child);
|
||||
child.stdout?.on("data", (chunk) => {
|
||||
const originalStdoutWrite = process.stdout.write;
|
||||
process.stdout.write = ((chunk: string | Uint8Array, ...args: unknown[]) => {
|
||||
stdoutDiagnosticLogs.append(chunk);
|
||||
writeStdout(chunk);
|
||||
});
|
||||
child.stderr?.on("data", writeStderr);
|
||||
return originalStdoutWrite.call(process.stdout, chunk, ...args);
|
||||
}) as typeof process.stdout.write;
|
||||
try {
|
||||
childExitCode = await waitForChild(child);
|
||||
await runDirectTelemetryProducer({
|
||||
endpoint: `http://127.0.0.1:${exportPort}`,
|
||||
logsExporter: options.logsExporter,
|
||||
outputDir: options.outputDir,
|
||||
writeLog: writeStdout,
|
||||
});
|
||||
childExitCode = 0;
|
||||
} finally {
|
||||
cleanupSignalRelay();
|
||||
process.stdout.write = originalStdoutWrite;
|
||||
stdoutDiagnosticLogs.flush();
|
||||
}
|
||||
if (stdoutDiagnosticLogs.records.length === 0) {
|
||||
await appendGatewayStdoutArtifactLogs({
|
||||
capture: stdoutDiagnosticLogs,
|
||||
outputDir: options.outputDir,
|
||||
});
|
||||
}
|
||||
if (childExitCode === 0) {
|
||||
await waitForExpectedTelemetry(receiver, options.logsExporter, 15_000);
|
||||
} else {
|
||||
await delay(3000);
|
||||
}
|
||||
await waitForExpectedTelemetry(receiver, options.logsExporter, 15_000);
|
||||
} finally {
|
||||
try {
|
||||
await collector?.close();
|
||||
@@ -1905,7 +1758,7 @@ async function main() {
|
||||
|
||||
const assertion = assertSmoke({
|
||||
childExitCode,
|
||||
disallowedBodyNeedles: disallowedBodyNeedles(options),
|
||||
disallowedBodyNeedles: disallowedBodyNeedles(),
|
||||
logsExporter: options.logsExporter,
|
||||
spans: receiver.capturedSpans,
|
||||
metrics: receiver.capturedMetrics,
|
||||
@@ -1919,8 +1772,7 @@ async function main() {
|
||||
passed: assertion.passed,
|
||||
failures: assertion.failures,
|
||||
outputDir: options.outputDir,
|
||||
scenarioId: options.scenarioId,
|
||||
providerMode: options.providerMode,
|
||||
producer: "diagnostics-otel-direct",
|
||||
collectorMode: options.collectorMode,
|
||||
logsExporter: options.logsExporter,
|
||||
requests: receiver.capturedRequests,
|
||||
@@ -2023,11 +1875,8 @@ async function main() {
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
appendGatewayStdoutArtifactLogs,
|
||||
appendCapturedBodyText,
|
||||
assertSmoke,
|
||||
buildQaArgs,
|
||||
buildQaEnv,
|
||||
createBoundedTextAccumulator,
|
||||
createStdoutDiagnosticLogCapture,
|
||||
decodeRequestBody,
|
||||
@@ -2035,11 +1884,8 @@ export const testing = {
|
||||
parseStdoutDiagnosticLogLine,
|
||||
readPositiveIntegerEnv,
|
||||
readRequestBody,
|
||||
appendUtf8FileToStdoutDiagnosticCapture,
|
||||
startLocalOtlpReceiver,
|
||||
startDockerOtelCollector,
|
||||
terminateChildTree,
|
||||
waitForChild,
|
||||
};
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
|
||||
@@ -1,21 +1,15 @@
|
||||
// QA OTEL Smoke tests cover QA Lab telemetry evidence.
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { createConnection as createNetConnection } from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { resolveWindowsTaskkillPath } from "../../../../scripts/lib/windows-taskkill.mjs";
|
||||
import { testing } from "./qa-otel-smoke-runtime.js";
|
||||
|
||||
function expectedTaskkillPath(): string {
|
||||
return resolveWindowsTaskkillPath();
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
vi.restoreAllMocks();
|
||||
@@ -117,22 +111,10 @@ describe("qa-otel-smoke receiver bounds", () => {
|
||||
|
||||
it("accepts package-manager forwarded arguments", () => {
|
||||
expect(
|
||||
testing.parseArgs([
|
||||
"--",
|
||||
"--collector",
|
||||
"docker",
|
||||
"--provider-mode",
|
||||
"mock-openai",
|
||||
"--scenario",
|
||||
"otel-stdout-log-smoke",
|
||||
"--logs-exporter",
|
||||
"stdout",
|
||||
]),
|
||||
testing.parseArgs(["--", "--collector", "docker", "--logs-exporter", "stdout"]),
|
||||
).toMatchObject({
|
||||
collectorMode: "docker",
|
||||
logsExporter: "stdout",
|
||||
providerMode: "mock-openai",
|
||||
scenarioId: "otel-stdout-log-smoke",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -140,10 +122,6 @@ describe("qa-otel-smoke receiver bounds", () => {
|
||||
["--collector", ["--collector", "--logs-exporter"]],
|
||||
["--logs-exporter", ["--logs-exporter", "--collector"]],
|
||||
["--output-dir", ["--output-dir", "--collector"]],
|
||||
["--provider-mode", ["--provider-mode", "--collector"]],
|
||||
["--scenario", ["--scenario", "--collector"]],
|
||||
["--model", ["--model", "--collector"]],
|
||||
["--alt-model", ["--alt-model", "--collector"]],
|
||||
])("rejects missing values for %s before shifting parser state", (flag, args) => {
|
||||
expect(() => testing.parseArgs(args)).toThrow(`${flag} requires a value`);
|
||||
});
|
||||
@@ -153,10 +131,6 @@ describe("qa-otel-smoke receiver bounds", () => {
|
||||
["--collector", ["--collector", "local", "--collector", "docker"]],
|
||||
["--logs-exporter", ["--logs-exporter", "otlp", "--logs-exporter", "stdout"]],
|
||||
["--output-dir", ["--output-dir", ".artifacts/one", "--output-dir", ".artifacts/two"]],
|
||||
["--provider-mode", ["--provider-mode", "mock-openai", "--provider-mode", "live-frontier"]],
|
||||
["--scenario", ["--scenario", "custom-one", "--scenario", "custom-two"]],
|
||||
["--model", ["--model", "openai/gpt-5.5", "--model", "openai/gpt-5.4"]],
|
||||
["--alt-model", ["--alt-model", "openai/gpt-5.5", "--alt-model", "openai/gpt-5.4"]],
|
||||
] satisfies Array<[string, string[]]>;
|
||||
|
||||
for (const [flag, args] of duplicateCases) {
|
||||
@@ -164,30 +138,6 @@ describe("qa-otel-smoke receiver bounds", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("selects the matching scenario for the requested log exporter", () => {
|
||||
expect(testing.parseArgs(["--logs-exporter", "otlp"]).scenarioId).toBe("otel-trace-smoke");
|
||||
expect(testing.parseArgs(["--logs-exporter", "stdout"]).scenarioId).toBe(
|
||||
"otel-stdout-log-smoke",
|
||||
);
|
||||
expect(testing.parseArgs(["--logs-exporter", "both"]).scenarioId).toBe("otel-both-log-smoke");
|
||||
});
|
||||
|
||||
it("rejects explicit scenarios that do not match the log exporter", () => {
|
||||
expect(() =>
|
||||
testing.parseArgs(["--logs-exporter", "stdout", "--scenario", "otel-trace-smoke"]),
|
||||
).toThrow("--logs-exporter stdout requires --scenario otel-stdout-log-smoke");
|
||||
});
|
||||
|
||||
it("allows explicit custom scenarios to own their exporter config", () => {
|
||||
expect(testing.parseArgs(["--scenario", "custom-otel-smoke"]).scenarioId).toBe(
|
||||
"custom-otel-smoke",
|
||||
);
|
||||
expect(
|
||||
testing.parseArgs(["--logs-exporter", "stdout", "--scenario", "custom-stdout-smoke"])
|
||||
.scenarioId,
|
||||
).toBe("custom-stdout-smoke");
|
||||
});
|
||||
|
||||
it("uses unique default output dirs", () => {
|
||||
const firstOutputDir = testing.parseArgs([]).outputDir;
|
||||
const secondOutputDir = testing.parseArgs([]).outputDir;
|
||||
@@ -200,27 +150,6 @@ describe("qa-otel-smoke receiver bounds", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("passes a repo-relative output dir to the child QA suite", () => {
|
||||
const repoRoot = path.join(path.sep, "repo");
|
||||
const options = testing.parseArgs([
|
||||
"--output-dir",
|
||||
path.join(repoRoot, ".artifacts", "qa-e2e", "otel-smoke"),
|
||||
]);
|
||||
|
||||
expect(testing.buildQaArgs(options, repoRoot)).toContain(".artifacts/qa-e2e/otel-smoke");
|
||||
const repoRootArgs = testing.buildQaArgs(
|
||||
testing.parseArgs(["--output-dir", repoRoot]),
|
||||
repoRoot,
|
||||
);
|
||||
expect(repoRootArgs[repoRootArgs.indexOf("--output-dir") + 1]).toBe(".");
|
||||
expect(() =>
|
||||
testing.buildQaArgs(
|
||||
testing.parseArgs(["--output-dir", path.join(path.sep, "outside", "otel-smoke")]),
|
||||
repoRoot,
|
||||
),
|
||||
).toThrow("--output-dir must stay within the repo root");
|
||||
});
|
||||
|
||||
it("parses body-size limit env values as strict positive integers", () => {
|
||||
expect(testing.readPositiveIntegerEnv("OTEL_TEST_LIMIT", 64, {})).toBe(64);
|
||||
expect(
|
||||
@@ -243,38 +172,37 @@ describe("qa-otel-smoke receiver bounds", () => {
|
||||
expect(configuredBodyLimitLoad.stderr).not.toContain("ReferenceError");
|
||||
});
|
||||
|
||||
it("scrubs inherited OpenTelemetry exporter env before running the QA suite", () => {
|
||||
vi.stubEnv("OTEL_SDK_DISABLED", "true");
|
||||
vi.stubEnv("OTEL_TRACES_EXPORTER", "none");
|
||||
vi.stubEnv("OTEL_METRICS_EXPORTER", "none");
|
||||
vi.stubEnv("OTEL_LOGS_EXPORTER", "none");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.example.test:4318");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "grpc");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", "grpc");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", "grpc");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=secret");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_LOGS_HEADERS", "authorization=logs-secret");
|
||||
vi.stubEnv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=developer-laptop");
|
||||
it("ignores inherited OTEL exporter endpoints during direct producer execution", () => {
|
||||
const tempRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-env-isolation-"));
|
||||
try {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
"test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts",
|
||||
"--output-dir",
|
||||
tempRoot,
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: "http://127.0.0.1:1",
|
||||
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://127.0.0.1:2/v1/traces",
|
||||
OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://127.0.0.1:3/v1/metrics",
|
||||
OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://127.0.0.1:4/v1/logs",
|
||||
},
|
||||
timeout: 30_000,
|
||||
},
|
||||
);
|
||||
|
||||
const env = testing.buildQaEnv(4318);
|
||||
|
||||
expect(env.OTEL_SDK_DISABLED).toBeUndefined();
|
||||
expect(env.OTEL_TRACES_EXPORTER).toBeUndefined();
|
||||
expect(env.OTEL_METRICS_EXPORTER).toBeUndefined();
|
||||
expect(env.OTEL_LOGS_EXPORTER).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_ENDPOINT).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_PROTOCOL).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_METRICS_PROTOCOL).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_HEADERS).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_LOGS_HEADERS).toBeUndefined();
|
||||
expect(env.OTEL_RESOURCE_ATTRIBUTES).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT).toBe("http://127.0.0.1:4318/v1/traces");
|
||||
expect(env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT).toBe("http://127.0.0.1:4318/v1/metrics");
|
||||
expect(env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT).toBe("http://127.0.0.1:4318/v1/logs");
|
||||
expect(env.OTEL_SERVICE_NAME).toBe("openclaw-qa-lab-otel-smoke");
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
expect(result.stdout).toContain("qa-otel-smoke: passed");
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects identity OTLP bodies above the decoded byte ceiling", () => {
|
||||
@@ -607,139 +535,6 @@ describe("qa-otel-smoke receiver bounds", () => {
|
||||
expect(output.text()).not.toContain("DO_NOT_RETAIN_COLLECTOR_PREFIX");
|
||||
});
|
||||
|
||||
it("streams gateway stdout artifact records without requiring them in the tail", async () => {
|
||||
const tempRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-stdout-stream-"));
|
||||
const logPath = path.join(tempRoot, "gateway.stdout.log");
|
||||
const capture = testing.createStdoutDiagnosticLogCapture();
|
||||
const record = {
|
||||
signal: "openclaw.diagnostic.log",
|
||||
ts: "2026-06-18T00:00:00.000Z",
|
||||
"service.name": "openclaw-qa-lab-otel-smoke",
|
||||
severityText: "INFO",
|
||||
severityNumber: 9,
|
||||
body: "early log",
|
||||
attributes: {},
|
||||
};
|
||||
try {
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify(record)}\n${"x".repeat(256 * 1024)}\nGATEWAY_STDOUT_TAIL\n`,
|
||||
);
|
||||
|
||||
await testing.appendUtf8FileToStdoutDiagnosticCapture(logPath, capture);
|
||||
capture.flush();
|
||||
|
||||
expect(capture.records).toEqual([record]);
|
||||
expect(capture.lines).toHaveLength(1);
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps gateway stdout artifact fallback parsing bounded", async () => {
|
||||
const tempRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-stdout-artifact-"));
|
||||
const outputDir = path.join(tempRoot, "output");
|
||||
const artifactDir = path.join(outputDir, "artifacts", "gateway-runtime");
|
||||
const record = {
|
||||
signal: "openclaw.diagnostic.log",
|
||||
ts: "2026-06-18T00:00:00.000Z",
|
||||
"service.name": "openclaw-qa-lab-otel-smoke",
|
||||
severityText: "INFO",
|
||||
severityNumber: 9,
|
||||
body: "tail log",
|
||||
attributes: {},
|
||||
};
|
||||
try {
|
||||
mkdirSync(artifactDir, { recursive: true });
|
||||
writeFileSync(
|
||||
path.join(artifactDir, "gateway.stdout.log"),
|
||||
`${JSON.stringify(record)}\n${"x".repeat(256 * 1024)}\n`,
|
||||
);
|
||||
const capture = testing.createStdoutDiagnosticLogCapture();
|
||||
|
||||
await testing.appendGatewayStdoutArtifactLogs({ capture, outputDir });
|
||||
|
||||
expect(capture.records).toEqual([record]);
|
||||
expect(capture.lines).toHaveLength(1);
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("times out and kills a wedged QA suite child with a detached gateway", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const tempDir = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-child-"));
|
||||
const markerPath = path.join(tempDir, "marker.txt");
|
||||
try {
|
||||
const gatewayScript = [
|
||||
"import fs from 'node:fs';",
|
||||
"process.on('SIGTERM', () => {});",
|
||||
`setInterval(() => fs.appendFileSync(${JSON.stringify(markerPath)}, "x"), 20);`,
|
||||
].join("\n");
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
"--input-type=module",
|
||||
"--eval",
|
||||
[
|
||||
"import childProcess from 'node:child_process';",
|
||||
`childProcess.spawn(process.execPath, ["--input-type=module", "--eval", ${JSON.stringify(
|
||||
gatewayScript,
|
||||
)}], { detached: true, stdio: "ignore" });`,
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("\n"),
|
||||
],
|
||||
{
|
||||
detached: true,
|
||||
stdio: "ignore",
|
||||
},
|
||||
);
|
||||
|
||||
await expect(testing.waitForChild(child, 100, 100)).rejects.toThrow(
|
||||
"openclaw qa suite timed out after 100ms",
|
||||
);
|
||||
const sizeAfterReturn = existsSync(markerPath) ? statSync(markerPath).size : 0;
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 150);
|
||||
});
|
||||
const sizeAfterWait = existsSync(markerPath) ? statSync(markerPath).size : 0;
|
||||
expect(sizeAfterWait).toBe(sizeAfterReturn);
|
||||
} finally {
|
||||
rmSync(tempDir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("clamps oversized QA suite child timers before scheduling", async () => {
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
["--input-type=module", "--eval", "setTimeout(() => process.exit(0), 25);"],
|
||||
{ stdio: "ignore" },
|
||||
);
|
||||
|
||||
await expect(testing.waitForChild(child, MAX_TIMER_TIMEOUT_MS + 1, 100)).resolves.toBe(0);
|
||||
});
|
||||
|
||||
it("uses taskkill for Windows QA suite timeout cleanup", () => {
|
||||
const kill = vi.fn();
|
||||
const runTaskkill = vi.fn(() => ({ status: 0 }));
|
||||
|
||||
testing.terminateChildTree(
|
||||
{ kill, pid: 1234 } as never,
|
||||
"SIGTERM",
|
||||
[],
|
||||
"win32",
|
||||
runTaskkill as never,
|
||||
);
|
||||
|
||||
expect(runTaskkill).toHaveBeenCalledWith(expectedTaskkillPath(), ["/PID", "1234", "/T", "/F"], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
expect(kill).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("moves Docker collector telemetry off the default host port", async () => {
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stderr: EventEmitter;
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { runTelegramBotTokenRuntime, testing } from "./telegram-bot-token-runtime.js";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { force: true, recursive: true })));
|
||||
});
|
||||
|
||||
describe("telegram bot token runtime evidence", () => {
|
||||
it("resolves only dedicated leased credentials", () => {
|
||||
expect(
|
||||
testing.resolveLeasedToken({
|
||||
OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "leased-token",
|
||||
TELEGRAM_E2E_SUT_BOT_TOKEN: "secondary-leased-token",
|
||||
TELEGRAM_BOT_TOKEN: "generic-token",
|
||||
}),
|
||||
).toEqual({ key: "OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN", token: "leased-token" });
|
||||
});
|
||||
|
||||
it("writes blocked evidence without a dedicated credential", async () => {
|
||||
const artifactBase = await fs.mkdtemp(path.join(os.tmpdir(), "telegram-bot-token-"));
|
||||
tempDirs.push(artifactBase);
|
||||
const evidence = await runTelegramBotTokenRuntime(
|
||||
{ artifactBase, repoRoot: process.cwd(), startupTimeoutMs: 100 },
|
||||
{ TELEGRAM_BOT_TOKEN: "generic-token" },
|
||||
);
|
||||
|
||||
expect(evidence.entries[0]?.result.status).toBe("blocked");
|
||||
const log = await fs.readFile(path.join(artifactBase, "telegram-bot-token.log"), "utf8");
|
||||
expect(log).toContain("blocked");
|
||||
expect(log).not.toContain("generic-token");
|
||||
});
|
||||
|
||||
it("bounds monitor shutdown", async () => {
|
||||
await expect(testing.waitForMonitorShutdown(new Promise(() => {}), 10)).rejects.toThrow(
|
||||
"Telegram runtime shutdown timed out",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,220 @@
|
||||
// Telegram bot-token runtime evidence starts the real monitor through getMe.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import type { RuntimeEnv } from "openclaw/plugin-sdk/runtime-env";
|
||||
import { monitorTelegramProvider } from "../../../../extensions/telegram/runtime-api.js";
|
||||
import { formatErrorMessage } from "../../../../src/infra/errors.js";
|
||||
import { createQaScriptEvidenceWriter } from "./script-evidence.js";
|
||||
|
||||
const STARTUP_TIMEOUT_MS = 30_000;
|
||||
const TOKEN_ENV_KEYS = [
|
||||
"OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN",
|
||||
"TELEGRAM_E2E_SUT_BOT_TOKEN",
|
||||
] as const;
|
||||
|
||||
type TelegramRuntimeOptions = {
|
||||
artifactBase: string;
|
||||
repoRoot: string;
|
||||
startupTimeoutMs: number;
|
||||
};
|
||||
|
||||
async function waitForMonitorShutdown(monitorPromise: Promise<void>, timeoutMs: number) {
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(
|
||||
() => reject(new Error("Telegram runtime shutdown timed out")),
|
||||
timeoutMs,
|
||||
);
|
||||
});
|
||||
try {
|
||||
await Promise.race([monitorPromise.catch(() => undefined), timeoutPromise]);
|
||||
} finally {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseOptions(argv: string[], repoRoot = process.cwd()): TelegramRuntimeOptions {
|
||||
let artifactBase = path.join(repoRoot, ".artifacts", "qa-e2e", "telegram-bot-token");
|
||||
let startupTimeoutMs = STARTUP_TIMEOUT_MS;
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--output-dir") {
|
||||
artifactBase = path.resolve(repoRoot, argv[++index] ?? "");
|
||||
continue;
|
||||
}
|
||||
if (arg === "--timeout-ms") {
|
||||
startupTimeoutMs = Number(argv[++index]);
|
||||
continue;
|
||||
}
|
||||
if (arg === "--") {
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
if (!Number.isFinite(startupTimeoutMs) || startupTimeoutMs <= 0) {
|
||||
throw new Error("--timeout-ms must be a positive number");
|
||||
}
|
||||
return { artifactBase, repoRoot, startupTimeoutMs };
|
||||
}
|
||||
|
||||
function resolveLeasedToken(env: NodeJS.ProcessEnv = process.env) {
|
||||
for (const key of TOKEN_ENV_KEYS) {
|
||||
const token = env[key]?.trim();
|
||||
if (token) {
|
||||
return { key, token };
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function createWriter(options: TelegramRuntimeOptions) {
|
||||
return createQaScriptEvidenceWriter({
|
||||
artifactBase: options.artifactBase,
|
||||
logFileName: "telegram-bot-token.log",
|
||||
primaryModel: "telegram/bot-api",
|
||||
providerMode: "live-frontier",
|
||||
repoRoot: options.repoRoot,
|
||||
target: {
|
||||
id: "telegram-bot-token",
|
||||
title: "Telegram bot token runtime startup",
|
||||
sourcePath: "test/e2e/qa-lab/runtime/telegram-bot-token-runtime.ts",
|
||||
primaryCoverageIds: ["telegram.startup-getme"],
|
||||
docsRefs: ["docs/channels/telegram.md"],
|
||||
codeRefs: [
|
||||
"test/e2e/qa-lab/runtime/telegram-bot-token-runtime.ts",
|
||||
"extensions/telegram/src/monitor.ts",
|
||||
"extensions/telegram/src/polling-session.ts",
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForStartup(params: {
|
||||
abortController: AbortController;
|
||||
monitorPromise: Promise<void>;
|
||||
startupPromise: Promise<void>;
|
||||
timeoutMs: number;
|
||||
}) {
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(
|
||||
() => reject(new Error("Telegram runtime startup timed out")),
|
||||
params.timeoutMs,
|
||||
);
|
||||
});
|
||||
try {
|
||||
await Promise.race([
|
||||
params.startupPromise,
|
||||
params.monitorPromise.then(() => {
|
||||
throw new Error("Telegram runtime stopped before polling startup");
|
||||
}),
|
||||
timeoutPromise,
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
params.abortController.abort();
|
||||
await waitForMonitorShutdown(params.monitorPromise, params.timeoutMs);
|
||||
}
|
||||
}
|
||||
|
||||
export async function runTelegramBotTokenRuntime(
|
||||
options: TelegramRuntimeOptions,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
) {
|
||||
await fs.mkdir(options.artifactBase, { recursive: true });
|
||||
const writer = createWriter(options);
|
||||
const startedAt = Date.now();
|
||||
const credential = resolveLeasedToken(env);
|
||||
if (!credential) {
|
||||
writer.appendLog(`telegram-bot-token: blocked; none of ${TOKEN_ENV_KEYS.join(", ")} is set\n`);
|
||||
return await writer.write({
|
||||
details: "Telegram runtime proof requires a leased bot token",
|
||||
durationMs: Math.max(1, Date.now() - startedAt),
|
||||
status: "blocked",
|
||||
});
|
||||
}
|
||||
|
||||
writer.appendLog(`telegram-bot-token: using leased credential from ${credential.key}\n`);
|
||||
const abortController = new AbortController();
|
||||
let markStarted: (() => void) | undefined;
|
||||
const startupPromise = new Promise<void>((resolve) => {
|
||||
markStarted = resolve;
|
||||
});
|
||||
const runtime: RuntimeEnv = {
|
||||
log: (...args) => {
|
||||
const line = args.map(String).join(" ");
|
||||
writer.appendLog(`${line}\n`);
|
||||
if (
|
||||
line.includes("isolated polling ingress started") ||
|
||||
line.includes("polling cycle started")
|
||||
) {
|
||||
markStarted?.();
|
||||
}
|
||||
},
|
||||
error: (...args) => writer.appendLog(`${args.map(String).join(" ")}\n`),
|
||||
exit: (code) => {
|
||||
throw new Error(`Telegram runtime requested exit ${code}`);
|
||||
},
|
||||
};
|
||||
const config: OpenClawConfig = {
|
||||
channels: { telegram: { enabled: true } },
|
||||
};
|
||||
const previousStateDir = process.env.OPENCLAW_STATE_DIR;
|
||||
process.env.OPENCLAW_STATE_DIR = path.join(options.artifactBase, "state");
|
||||
try {
|
||||
const monitorPromise = monitorTelegramProvider({
|
||||
abortSignal: abortController.signal,
|
||||
config,
|
||||
isolatedIngress: { enabled: true },
|
||||
runtime,
|
||||
token: credential.token,
|
||||
});
|
||||
await waitForStartup({
|
||||
abortController,
|
||||
monitorPromise,
|
||||
startupPromise,
|
||||
timeoutMs: options.startupTimeoutMs,
|
||||
});
|
||||
writer.appendLog("telegram-bot-token: runtime started after Telegram getMe\n");
|
||||
return await writer.write({
|
||||
details: `Telegram runtime startup completed with ${credential.key}`,
|
||||
durationMs: Math.max(1, Date.now() - startedAt),
|
||||
status: "pass",
|
||||
});
|
||||
} catch (error) {
|
||||
const details = formatErrorMessage(error);
|
||||
writer.appendLog(`telegram-bot-token: ${details}\n`);
|
||||
return await writer.write({
|
||||
details,
|
||||
durationMs: Math.max(1, Date.now() - startedAt),
|
||||
status: "fail",
|
||||
});
|
||||
} finally {
|
||||
if (previousStateDir === undefined) {
|
||||
delete process.env.OPENCLAW_STATE_DIR;
|
||||
} else {
|
||||
process.env.OPENCLAW_STATE_DIR = previousStateDir;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const testing = { parseOptions, resolveLeasedToken, waitForMonitorShutdown };
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
runTelegramBotTokenRuntime(parseOptions(process.argv.slice(2)))
|
||||
.then((evidence) => {
|
||||
const status = evidence.entries[0]?.result.status;
|
||||
process.stdout.write(`telegram-bot-token: ${status}\n`);
|
||||
process.exitCode = status === "fail" ? 1 : 0;
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
process.stderr.write(`telegram-bot-token: ${formatErrorMessage(error)}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user