fix(diagnostics-otel): surface error message on run/harness error spans (#101244)

* fix(diagnostics-otel): surface error message on run/harness error spans

Errored openclaw.run / openclaw.harness.run spans only carried a
low-cardinality errorCategory (or a hardcoded "error"), so trace UIs
showed "outcome error" with no message. Thread the redacted error
message through run.completed / harness.run.completed / harness.run.error
onto an openclaw.error span attribute + span status, mirroring
recordWebhookError. The raw message stays off metric attrs to preserve
cardinality; support-bundle redaction covers the new error field by name.

* fix(diagnostics-otel): harden run failure telemetry

Co-authored-by: Alex Knight <aknight@atlassian.com>

* test(diagnostics-otel): assert wire-level redaction

* chore(changelog): defer release note to release process

---------

Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
Alex Knight
2026-07-07 04:29:39 +01:00
committed by GitHub
co-authored by Peter Steinberger
parent 118e5bd762
commit 87a17920a0
12 changed files with 449 additions and 74 deletions
+1
View File
@@ -11,6 +11,7 @@ export {
parseDiagnosticTraceparent,
type DiagnosticEventMetadata,
type DiagnosticEventPayload,
type DiagnosticEventPrivateData,
type DiagnosticTraceContext,
} from "openclaw/plugin-sdk/diagnostic-runtime";
export { emptyPluginConfigSchema, type OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
@@ -1455,6 +1455,132 @@ describe("diagnostics-otel service", () => {
await service.stop?.(ctx);
});
test("run.completed error span carries the redacted message off the metric attrs", async () => {
const service = createDiagnosticsOtelService();
const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { traces: true, metrics: true });
await service.start(ctx);
emitTrustedDiagnosticEventWithPrivateData(
{
type: "run.completed",
runId: "run-1",
provider: "openai",
model: "gpt-5.4",
outcome: "error",
errorCategory: "Error",
durationMs: 100,
},
{ errorMessage: "upstream model stream stalled then aborted" },
);
await flushDiagnosticEvents();
expect(startedSpanOptions("openclaw.run")?.attributes?.["openclaw.error"]).toBe(
"upstream model stream stalled then aborted",
);
expect(spanByName("openclaw.run").setStatus).toHaveBeenCalledWith({
code: 2,
message: "upstream model stream stalled then aborted",
});
// The raw message must never widen metric cardinality.
const runDuration = lastHistogramRecord("openclaw.run.duration_ms");
expect(runDuration?.[1]?.["openclaw.outcome"]).toBe("error");
expect(Object.hasOwn(runDuration?.[1] ?? {}, "openclaw.error")).toBe(false);
await service.stop?.(ctx);
});
test("run.completed bounds sensitive error text before export", async () => {
const service = createDiagnosticsOtelService();
const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { traces: true });
await service.start(ctx);
const secret = "sk-1234567890abcdef";
emitTrustedDiagnosticEventWithPrivateData(
{
type: "run.completed",
runId: "run-1",
outcome: "error",
errorCategory: "Error",
durationMs: 100,
},
{ errorMessage: `OPENAI_API_KEY=${secret} ${"x".repeat(8 * 1024)}` },
);
await flushDiagnosticEvents();
const status = mockCallArg(spanByName("openclaw.run").setStatus, 0) as {
message?: string;
};
expect(status.message).not.toContain(secret);
expect(status.message).toMatch(/\.\.\.\(truncated\)$/u);
expect(status.message?.length).toBeLessThanOrEqual(4 * 1024 + 20);
await service.stop?.(ctx);
});
test("harness.run.completed error span carries the redacted message", async () => {
const service = createDiagnosticsOtelService();
const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { traces: true, metrics: true });
await service.start(ctx);
emitTrustedDiagnosticEventWithPrivateData(
{
type: "harness.run.completed",
runId: "run-1",
provider: "openai",
model: "gpt-5.4",
harnessId: "openclaw",
outcome: "error",
durationMs: 90,
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
},
{ errorMessage: "model run failed during resolve phase" },
);
await flushDiagnosticEvents();
expect(startedSpanOptions("openclaw.harness.run")?.attributes?.["openclaw.error"]).toBe(
"model run failed during resolve phase",
);
expect(spanByName("openclaw.harness.run").setStatus).toHaveBeenCalledWith({
code: 2,
message: "model run failed during resolve phase",
});
const harnessDuration = lastHistogramRecord("openclaw.harness.duration_ms");
expect(Object.hasOwn(harnessDuration?.[1] ?? {}, "openclaw.error")).toBe(false);
await service.stop?.(ctx);
});
test("harness.run.error span prefers the redacted message over the category", async () => {
const service = createDiagnosticsOtelService();
const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { traces: true, metrics: true });
await service.start(ctx);
emitTrustedDiagnosticEventWithPrivateData(
{
type: "harness.run.error",
runId: "run-1",
provider: "openai",
model: "gpt-5.4",
harnessId: "openclaw",
phase: "resolve",
errorCategory: "Error",
durationMs: 90,
},
{ errorMessage: "harness cleanup threw" },
);
await flushDiagnosticEvents();
expect(startedSpanOptions("openclaw.harness.run")?.attributes?.["openclaw.error"]).toBe(
"harness cleanup threw",
);
expect(spanByName("openclaw.harness.run").setStatus).toHaveBeenCalledWith({
code: 2,
message: "harness cleanup threw",
});
await service.stop?.(ctx);
});
test("honors disabled traces when an OpenTelemetry SDK is preloaded", async () => {
process.env.OPENCLAW_OTEL_PRELOADED = "1";
const service = createDiagnosticsOtelService();
+35 -6
View File
@@ -39,6 +39,7 @@ import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
import type {
DiagnosticEventMetadata,
DiagnosticEventPayload,
DiagnosticEventPrivateData,
DiagnosticTraceContext,
OpenClawPluginService,
OpenClawPluginServiceContext,
@@ -80,6 +81,7 @@ const MAX_OTEL_CONTENT_ARRAY_ITEMS = 200;
const MAX_OTEL_LOG_BODY_CHARS = 4 * 1024;
const MAX_OTEL_LOG_ATTRIBUTE_COUNT = 64;
const MAX_OTEL_LOG_ATTRIBUTE_VALUE_CHARS = 4 * 1024;
const MAX_OTEL_ERROR_MESSAGE_CHARS = 4 * 1024;
const LOG_RECORD_EXPORT_FAILURE_REPORT_INTERVAL_MS = 60_000;
const OTEL_LOG_RAW_ATTRIBUTE_KEY_RE = /^[A-Za-z0-9_.:-]{1,64}$/u;
const OTEL_LOG_ATTRIBUTE_KEY_RE = /^[A-Za-z0-9_.:-]{1,96}$/u;
@@ -683,6 +685,14 @@ function normalizeOtelLogString(value: string, maxChars: number): string {
return clampOtelLogText(redactSensitiveText(value), maxChars);
}
function normalizeOtelErrorMessage(value: string | undefined): string | undefined {
if (!value) {
return undefined;
}
const normalized = normalizeOtelLogString(value.trim(), MAX_OTEL_ERROR_MESSAGE_CHARS);
return normalized || undefined;
}
function resolveContentCapturePolicy(value: unknown): OtelContentCapturePolicy {
if (value === true) {
return {
@@ -3066,6 +3076,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
const recordRunCompleted = (
evt: Extract<DiagnosticEventPayload, { type: "run.completed" }>,
metadata: DiagnosticEventMetadata,
privateData: DiagnosticEventPrivateData,
) => {
const attrs: Record<string, string | number> = {
"openclaw.outcome": evt.outcome,
@@ -3092,6 +3103,11 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
if (evt.errorCategory) {
spanAttrs["openclaw.errorCategory"] = lowCardinalityAttr(evt.errorCategory, "other");
}
// Redacted message goes on the span only, never the low-cardinality metric attrs.
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
if (redactedError) {
spanAttrs["openclaw.error"] = redactedError;
}
const trustedTrace = trustedTraceContext(evt, metadata);
const trackedSpan = trustedTrace?.spanId
? activeTrustedSpans.get(trustedTrace.spanId)
@@ -3104,9 +3120,12 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
});
setSpanAttrs(span, spanAttrs);
if (evt.outcome === "error") {
const message =
redactedError ??
(evt.errorCategory ? redactSensitiveText(evt.errorCategory) : undefined);
span.setStatus({
code: SpanStatusCode.ERROR,
...(evt.errorCategory ? { message: redactSensitiveText(evt.errorCategory) } : {}),
...(message ? { message } : {}),
});
}
if (trackedSpan && trustedTrace?.spanId) {
@@ -3149,6 +3168,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
const recordHarnessRunCompleted = (
evt: Extract<DiagnosticEventPayload, { type: "harness.run.completed" }>,
metadata: DiagnosticEventMetadata,
privateData: DiagnosticEventPrivateData,
) => {
harnessDurationHistogram.record(evt.durationMs, harnessRunMetricAttrs(evt));
if (!tracesEnabled) {
@@ -3170,6 +3190,11 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
spanAttrs["openclaw.harness.items.completed"] = evt.itemLifecycle.completedCount;
spanAttrs["openclaw.harness.items.active"] = evt.itemLifecycle.activeCount;
}
// Redacted message goes on the span only, never the low-cardinality metric attrs.
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
if (redactedError) {
spanAttrs["openclaw.error"] = redactedError;
}
const trustedTrace = trustedTraceContext(evt, metadata);
const trackedSpan = trustedTrace?.spanId
? activeTrustedSpans.get(trustedTrace.spanId)
@@ -3184,7 +3209,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
if (evt.outcome === "error") {
span.setStatus({
code: SpanStatusCode.ERROR,
message: "error",
message: redactedError ?? "error",
});
}
if (trackedSpan && trustedTrace?.spanId) {
@@ -3197,6 +3222,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
const recordHarnessRunError = (
evt: Extract<DiagnosticEventPayload, { type: "harness.run.error" }>,
metadata: DiagnosticEventMetadata,
privateData: DiagnosticEventPrivateData,
) => {
const errorType = lowCardinalityAttr(evt.errorCategory, "other");
const attrs = {
@@ -3208,9 +3234,12 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
if (!tracesEnabled) {
return;
}
// Redacted message goes on the span only; attrs above feed the metric.
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
const spanAttrs: Record<string, string | number | boolean> = {
...attrs,
"error.type": errorType,
...(redactedError ? { "openclaw.error": redactedError } : {}),
...(evt.cleanupFailed ? { "openclaw.harness.cleanup_failed": true } : {}),
};
const span =
@@ -3222,7 +3251,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
setSpanAttrs(span, spanAttrs);
span.setStatus({
code: SpanStatusCode.ERROR,
message: errorType,
message: redactedError ?? errorType,
});
span.end(evt.ts);
};
@@ -3887,16 +3916,16 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
recordRunStarted(evt, metadata);
return;
case "run.completed":
recordRunCompleted(evt, metadata);
recordRunCompleted(evt, metadata, privateData);
return;
case "harness.run.started":
recordHarnessRunStarted(evt, metadata);
return;
case "harness.run.completed":
recordHarnessRunCompleted(evt, metadata);
recordHarnessRunCompleted(evt, metadata, privateData);
return;
case "harness.run.error":
recordHarnessRunError(evt, metadata);
recordHarnessRunError(evt, metadata, privateData);
return;
case "context.assembled":
recordContextAssembled(evt, metadata);
@@ -0,0 +1,69 @@
// Coverage for trusted diagnostics emitted by a full embedded attempt.
import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
import {
onTrustedInternalDiagnosticEvent,
resetDiagnosticEventsForTest,
waitForDiagnosticEventsDrained,
type DiagnosticEventPrivateData,
type DiagnosticEventPayload,
} from "../../../infra/diagnostic-events.js";
import {
cleanupTempPaths,
createContextEngineAttemptRunner,
createContextEngineBootstrapAndAssemble,
preloadRunEmbeddedAttemptForTests,
resetEmbeddedAttemptHarness,
} from "./attempt.spawn-workspace.test-support.js";
describe("runEmbeddedAttempt diagnostics", () => {
const tempPaths: string[] = [];
beforeAll(async () => {
await preloadRunEmbeddedAttemptForTests();
});
beforeEach(() => {
resetEmbeddedAttemptHarness();
resetDiagnosticEventsForTest();
});
afterEach(async () => {
await cleanupTempPaths(tempPaths);
resetDiagnosticEventsForTest();
});
it("keeps run failure text on the trusted private channel", async () => {
const completed: Array<{
event: DiagnosticEventPayload;
privateData: DiagnosticEventPrivateData;
}> = [];
const unsubscribe = onTrustedInternalDiagnosticEvent((event, _metadata, privateData) => {
if (event.type === "run.completed") {
completed.push({ event, privateData });
}
});
try {
await createContextEngineAttemptRunner({
contextEngine: createContextEngineBootstrapAndAssemble(),
sessionKey: "agent:main:diagnostic-failure",
tempPaths,
sessionPrompt: async () => {
throw new Error("provider stream failed");
},
});
await waitForDiagnosticEventsDrained();
} finally {
unsubscribe();
}
expect(completed).toHaveLength(1);
expect(completed[0]?.event).toMatchObject({
type: "run.completed",
outcome: "error",
errorCategory: "Error",
});
expect(completed[0]?.event).not.toHaveProperty("error");
expect(completed[0]?.privateData.errorMessage).toBe("provider stream failed");
});
});
+21 -10
View File
@@ -33,8 +33,14 @@ import {
import { resolveContextEngineOwnerPluginId } from "../../../context-engine/registry.js";
import { buildContextEngineRuntimeSettings } from "../../../context-engine/runtime-settings.js";
import type { AssembleResult } from "../../../context-engine/types.js";
import { diagnosticErrorCategory } from "../../../infra/diagnostic-error-metadata.js";
import { emitTrustedDiagnosticEvent } from "../../../infra/diagnostic-events.js";
import {
diagnosticErrorCategory,
diagnosticErrorMessage,
} from "../../../infra/diagnostic-error-metadata.js";
import {
emitTrustedDiagnosticEvent,
emitTrustedDiagnosticEventWithPrivateData,
} from "../../../infra/diagnostic-events.js";
import { resolveDiagnosticModelContentCapturePolicy } from "../../../infra/diagnostic-llm-content.js";
import {
createChildDiagnosticTraceContext,
@@ -1188,14 +1194,19 @@ export async function runEmbeddedAttempt(
return;
}
diagnosticRunCompleted = true;
emitTrustedDiagnosticEvent({
type: "run.completed",
...diagnosticRunBase,
durationMs: Date.now() - diagnosticRunStartedAt,
outcome,
...(extra?.blockedBy ? { blockedBy: extra.blockedBy } : {}),
...(err && outcome !== "blocked" ? { errorCategory: diagnosticErrorCategory(err) } : {}),
});
const failed = err != null && outcome !== "blocked";
const errorMessage = failed ? diagnosticErrorMessage(err) : undefined;
emitTrustedDiagnosticEventWithPrivateData(
{
type: "run.completed",
...diagnosticRunBase,
durationMs: Date.now() - diagnosticRunStartedAt,
outcome,
...(extra?.blockedBy ? { blockedBy: extra.blockedBy } : {}),
...(failed ? { errorCategory: diagnosticErrorCategory(err) } : {}),
},
errorMessage ? { errorMessage } : undefined,
);
};
const corePluginToolStages = createEmbeddedRunStageTracker();
const forceDirectMessageTool =
+50 -5
View File
@@ -4,8 +4,9 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import { OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST } from "../../context-engine/host-compat.js";
import type { ContextEngine } from "../../context-engine/types.js";
import {
onInternalDiagnosticEvent,
onTrustedInternalDiagnosticEvent,
resetDiagnosticEventsForTest,
type DiagnosticEventPrivateData,
type DiagnosticEventMetadata,
type DiagnosticEventPayload,
} from "../../infra/diagnostic-events.js";
@@ -110,13 +111,21 @@ function captureDiagnosticEvents(
filter: (event: DiagnosticEventPayload) => boolean = (event) =>
event.type.startsWith("harness.run."),
): {
events: Array<{ event: DiagnosticEventPayload; metadata: DiagnosticEventMetadata }>;
events: Array<{
event: DiagnosticEventPayload;
metadata: DiagnosticEventMetadata;
privateData: DiagnosticEventPrivateData;
}>;
unsubscribe: () => void;
} {
const events: Array<{ event: DiagnosticEventPayload; metadata: DiagnosticEventMetadata }> = [];
const unsubscribe = onInternalDiagnosticEvent((event, metadata) => {
const events: Array<{
event: DiagnosticEventPayload;
metadata: DiagnosticEventMetadata;
privateData: DiagnosticEventPrivateData;
}> = [];
const unsubscribe = onTrustedInternalDiagnosticEvent((event, metadata, privateData) => {
if (filter(event)) {
events.push({ event, metadata });
events.push({ event, metadata, privateData });
}
});
return { events, unsubscribe };
@@ -320,6 +329,41 @@ describe("AgentHarness lifecycle runner", () => {
expect(attemptResult?.diagnosticTrace).toEqual(harnessTrace);
});
it("keeps plugin harness failure messages on the trusted private channel", async () => {
const params = createAttemptParams();
const harnessTrace = createDiagnosticTrace();
const result = {
...createAttemptResult(),
promptError: new Error("provider stream failed"),
} as EmbeddedRunAttemptResult;
const harness: AgentHarness = {
id: "codex",
label: "Codex",
supports: () => ({ supported: true }),
runAttempt: async () => result,
};
const diagnostics = captureDiagnosticEvents(
(event) => event.type === "run.completed" || event.type === "harness.run.completed",
);
try {
await runWithDiagnosticTraceContext(harnessTrace, () =>
runAgentHarnessLifecycleAttempt(harness, params),
);
await flushDiagnosticEvents();
} finally {
diagnostics.unsubscribe();
}
expect(diagnostics.events.map(({ event }) => event.type)).toEqual([
"run.completed",
"harness.run.completed",
]);
for (const { event, privateData } of diagnostics.events) {
expect(event).not.toHaveProperty("error");
expect(privateData.errorMessage).toBe("provider stream failed");
}
});
it("emits plugin before-agent-run hook blocks as blocked run completions", async () => {
resetDiagnosticEventsForTest();
const params = createAttemptParams();
@@ -390,6 +434,7 @@ describe("AgentHarness lifecycle runner", () => {
expect(errorEvent?.type).toBe("harness.run.error");
expect(errorEvent?.phase).toBe("send");
expect(errorEvent?.errorCategory).toBe("Error");
expect(diagnostics.events[1]?.privateData.errorMessage).toBe("codex app-server send failed");
expect(errorEvent).not.toHaveProperty("cleanupFailed");
expect(errorEvent?.harnessId).toBe("codex");
expect(typeof errorEvent?.durationMs).toBe("number");
+47 -29
View File
@@ -8,9 +8,13 @@ import {
assertContextEngineHostSupport,
type ContextEngineHostSupport,
} from "../../context-engine/host-compat.js";
import { diagnosticErrorCategory } from "../../infra/diagnostic-error-metadata.js";
import {
diagnosticErrorCategory,
diagnosticErrorMessage,
} from "../../infra/diagnostic-error-metadata.js";
import {
emitTrustedDiagnosticEvent,
emitTrustedDiagnosticEventWithPrivateData,
type DiagnosticHarnessRunErrorEvent,
type DiagnosticHarnessRunOutcome,
} from "../../infra/diagnostic-events.js";
@@ -167,17 +171,24 @@ function emitAgentHarnessRunCompleted(params: {
trace?: DiagnosticTraceContext;
}): void {
const { harness, attemptParams, result, startedAt, trace } = params;
emitTrustedDiagnosticEvent({
type: "harness.run.completed",
...agentHarnessDiagnosticBase(harness, attemptParams, trace ?? result.diagnosticTrace),
durationMs: Date.now() - startedAt,
outcome: agentHarnessRunOutcome(result),
...(result.agentHarnessResultClassification
? { resultClassification: result.agentHarnessResultClassification }
: {}),
...(typeof result.yieldDetected === "boolean" ? { yieldDetected: result.yieldDetected } : {}),
itemLifecycle: { ...result.itemLifecycle },
});
const outcome = agentHarnessRunOutcome(result);
// A classified (non-thrown) failure carries its error on result.promptError;
// forward the message so the error span shows more than a bare category.
const errorMessage = outcome === "error" ? diagnosticErrorMessage(result.promptError) : undefined;
emitTrustedDiagnosticEventWithPrivateData(
{
type: "harness.run.completed",
...agentHarnessDiagnosticBase(harness, attemptParams, trace ?? result.diagnosticTrace),
durationMs: Date.now() - startedAt,
outcome,
...(result.agentHarnessResultClassification
? { resultClassification: result.agentHarnessResultClassification }
: {}),
...(typeof result.yieldDetected === "boolean" ? { yieldDetected: result.yieldDetected } : {}),
itemLifecycle: { ...result.itemLifecycle },
},
errorMessage ? { errorMessage } : undefined,
);
}
function emitAgentHarnessRunError(params: {
@@ -189,13 +200,17 @@ function emitAgentHarnessRunError(params: {
trace?: DiagnosticTraceContext;
}): void {
const { harness, attemptParams, startedAt, phase, error, trace } = params;
emitTrustedDiagnosticEvent({
type: "harness.run.error",
...agentHarnessDiagnosticBase(harness, attemptParams, trace),
durationMs: Date.now() - startedAt,
phase,
errorCategory: diagnosticErrorCategory(error),
});
const errorMessage = diagnosticErrorMessage(error);
emitTrustedDiagnosticEventWithPrivateData(
{
type: "harness.run.error",
...agentHarnessDiagnosticBase(harness, attemptParams, trace),
durationMs: Date.now() - startedAt,
phase,
errorCategory: diagnosticErrorCategory(error),
},
errorMessage ? { errorMessage } : undefined,
);
}
/** Runs one harness attempt with diagnostics, tracing, and result classification. */
@@ -215,16 +230,19 @@ export async function runAgentHarnessLifecycleAttempt(
return;
}
agentRunCompleted = true;
emitTrustedDiagnosticEvent({
type: "run.completed",
...agentRunDiagnosticBase(params, agentRunTrace),
durationMs: Date.now() - agentRunStartedAt,
outcome: completion.outcome,
...(completion.blockedBy ? { blockedBy: completion.blockedBy } : {}),
...(completion.error && completion.outcome === "error"
? { errorCategory: diagnosticErrorCategory(completion.error) }
: {}),
});
const failed = completion.outcome === "error" && completion.error != null;
const errorMessage = failed ? diagnosticErrorMessage(completion.error) : undefined;
emitTrustedDiagnosticEventWithPrivateData(
{
type: "run.completed",
...agentRunDiagnosticBase(params, agentRunTrace),
durationMs: Date.now() - agentRunStartedAt,
outcome: completion.outcome,
...(completion.blockedBy ? { blockedBy: completion.blockedBy } : {}),
...(failed ? { errorCategory: diagnosticErrorCategory(completion.error) } : {}),
},
errorMessage ? { errorMessage } : undefined,
);
};
emitAgentHarnessRunStarted(harness, params, activeHarnessTrace);
@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";
import {
diagnosticErrorCategory,
diagnosticErrorFailureKind,
diagnosticErrorMessage,
diagnosticHttpStatusCode,
diagnosticProviderRequestIdHash,
} from "./diagnostic-error-metadata.js";
@@ -22,6 +23,24 @@ describe("diagnostic error metadata", () => {
expect(diagnosticErrorCategory(null)).toBe("null");
});
it("normalizes Error and string messages", () => {
expect(diagnosticErrorMessage(new Error(" provider failed "))).toBe("provider failed");
expect(diagnosticErrorMessage(" connection closed ")).toBe("connection closed");
expect(diagnosticErrorMessage(" ")).toBeUndefined();
});
it("does not invoke message getters", () => {
const errorLike = Object.create(null, {
message: {
get() {
throw new Error("getter must not run");
},
},
});
expect(diagnosticErrorMessage(errorLike)).toBeUndefined();
});
it("accepts only own HTTP status data properties as error codes", () => {
expect(diagnosticHttpStatusCode({ status: 429 })).toBe("429");
expect(diagnosticHttpStatusCode({ statusCode: 503 })).toBe("503");
+13
View File
@@ -157,6 +157,19 @@ export function diagnosticErrorCategory(err: unknown): string {
return typeof err;
}
/**
* Human-readable error message for diagnostics. Complements
* {@link diagnosticErrorCategory} (low-cardinality class name) with the actual
* message so error spans carry a real status message instead of a bare
* category. Reads only an own data property so diagnostics never invoke a
* user-defined getter.
*/
export function diagnosticErrorMessage(err: unknown): string | undefined {
const text = readDirectMessage(err);
const trimmed = text?.trim();
return trimmed ? trimmed : undefined;
}
/** Extracts a safe HTTP status code from own `status` or `statusCode` data properties. */
export function diagnosticHttpStatusCode(err: unknown): string | undefined {
const status = readOwnDataProperty(err, "status");
+2
View File
@@ -839,6 +839,8 @@ export type DiagnosticSkillUsagePrivateData = Readonly<{
}>;
export type DiagnosticEventPrivateData = Readonly<{
/** Raw failure text for trusted diagnostics exporters; never part of the public event payload. */
errorMessage?: string;
modelContent?: DiagnosticModelCallContent;
skillUsage?: DiagnosticSkillUsagePrivateData;
toolContent?: DiagnosticToolCallContent;
@@ -127,6 +127,8 @@ const REQUIRED_SPAN_NAMES = [
const REQUIRED_METRIC_NAMES = ["openclaw.harness.duration_ms"] as const;
const DIRECT_RUN_ID = "qa-otel-direct-run";
const DIRECT_CALL_ID = "qa-otel-direct-call";
const DIRECT_ERROR_MESSAGE = "QA OTEL provider stream failed";
const DIRECT_ERROR_SECRET = "sk-1234567890abcdef";
const DISALLOWED_ATTRIBUTE_KEYS = new Set([
"openclaw.runId",
"openclaw.chatId",
@@ -146,6 +148,7 @@ const DISALLOWED_ATTRIBUTE_KEYS = new Set([
const DISALLOWED_BODY_NEEDLES = [
"OTEL-QA-SECRET",
"OTEL-QA-OK",
DIRECT_ERROR_SECRET,
DIRECT_RUN_ID,
DIRECT_CALL_ID,
];
@@ -1410,28 +1413,38 @@ async function runDirectTelemetryProducer(params: {
},
},
);
emitTrustedDiagnosticEvent({
type: "run.completed",
runId: DIRECT_RUN_ID,
provider: "openai",
model: "gpt-5.5",
channel: "qa",
durationMs: 8,
outcome: "completed",
trace: runTrace,
});
emitTrustedDiagnosticEvent({
type: "harness.run.completed",
runId: DIRECT_RUN_ID,
harnessId: "qa-otel-direct",
pluginId: "diagnostics-otel",
provider: "openai",
model: "gpt-5.5",
channel: "qa",
durationMs: 10,
outcome: "completed",
trace: harnessTrace,
});
const failurePrivateData = {
errorMessage: `${DIRECT_ERROR_MESSAGE} OPENAI_API_KEY=${DIRECT_ERROR_SECRET}`,
};
emitTrustedDiagnosticEventWithPrivateData(
{
type: "run.completed",
runId: DIRECT_RUN_ID,
provider: "openai",
model: "gpt-5.5",
channel: "qa",
durationMs: 8,
outcome: "error",
errorCategory: "Error",
trace: runTrace,
},
failurePrivateData,
);
emitTrustedDiagnosticEventWithPrivateData(
{
type: "harness.run.completed",
runId: DIRECT_RUN_ID,
harnessId: "qa-otel-direct",
pluginId: "diagnostics-otel",
provider: "openai",
model: "gpt-5.5",
channel: "qa",
durationMs: 10,
outcome: "error",
trace: harnessTrace,
},
failurePrivateData,
);
await waitForDiagnosticEventsDrained();
} finally {
await service.stop?.(context);
@@ -1629,6 +1642,23 @@ function assertSmoke(params: {
failures.push("successful QA run exported model-call error attributes");
}
const failedRunSpans = params.spans.filter(
(span) =>
(span.name === "openclaw.run" || span.name === "openclaw.harness.run") &&
span.attributes["openclaw.error"] === `${DIRECT_ERROR_MESSAGE} OPENAI_API_KEY=***`,
);
if (failedRunSpans.length !== 2) {
const observed = params.spans
.filter((span) => span.name === "openclaw.run" || span.name === "openclaw.harness.run")
.map((span) => ({ name: span.name, error: span.attributes["openclaw.error"] }));
failures.push(
`run and harness spans did not export the redacted failure message: ${JSON.stringify(observed)}`,
);
}
if ((params.bodyText.metrics ?? []).some((body) => body.includes(DIRECT_ERROR_MESSAGE))) {
failures.push("run failure message leaked into OTLP metric attributes");
}
const serializedAttributes = JSON.stringify(params.spans.map((span) => span.attributes));
if (serializedAttributes.includes("StreamAbandoned")) {
failures.push("StreamAbandoned leaked into OTEL attributes");
@@ -91,8 +91,20 @@ describe("qa-otel-smoke receiver bounds", () => {
stdoutLogLines: [],
stdoutLogRecords: [],
spans: [
{ name: "openclaw.run", parent: false, attributes: {} },
{ name: "openclaw.harness.run", parent: true, attributes: {} },
{
name: "openclaw.run",
parent: false,
attributes: {
"openclaw.error": "QA OTEL provider stream failed OPENAI_API_KEY=***",
},
},
{
name: "openclaw.harness.run",
parent: true,
attributes: {
"openclaw.error": "QA OTEL provider stream failed OPENAI_API_KEY=***",
},
},
{ name: "openclaw.context.assembled", parent: true, attributes: {} },
{ name: "openclaw.message.delivery", parent: true, attributes: {} },
{