mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
fix(diagnostics-otel): surface error message on run/harness error spans (#101244)
* fix(diagnostics-otel): surface error message on run/harness error spans Errored openclaw.run / openclaw.harness.run spans only carried a low-cardinality errorCategory (or a hardcoded "error"), so trace UIs showed "outcome error" with no message. Thread the redacted error message through run.completed / harness.run.completed / harness.run.error onto an openclaw.error span attribute + span status, mirroring recordWebhookError. The raw message stays off metric attrs to preserve cardinality; support-bundle redaction covers the new error field by name. * fix(diagnostics-otel): harden run failure telemetry Co-authored-by: Alex Knight <aknight@atlassian.com> * test(diagnostics-otel): assert wire-level redaction * chore(changelog): defer release note to release process --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
co-authored by
Peter Steinberger
parent
118e5bd762
commit
87a17920a0
@@ -11,6 +11,7 @@ export {
|
||||
parseDiagnosticTraceparent,
|
||||
type DiagnosticEventMetadata,
|
||||
type DiagnosticEventPayload,
|
||||
type DiagnosticEventPrivateData,
|
||||
type DiagnosticTraceContext,
|
||||
} from "openclaw/plugin-sdk/diagnostic-runtime";
|
||||
export { emptyPluginConfigSchema, type OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
||||
|
||||
@@ -1455,6 +1455,132 @@ describe("diagnostics-otel service", () => {
|
||||
await service.stop?.(ctx);
|
||||
});
|
||||
|
||||
test("run.completed error span carries the redacted message off the metric attrs", async () => {
|
||||
const service = createDiagnosticsOtelService();
|
||||
const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { traces: true, metrics: true });
|
||||
await service.start(ctx);
|
||||
|
||||
emitTrustedDiagnosticEventWithPrivateData(
|
||||
{
|
||||
type: "run.completed",
|
||||
runId: "run-1",
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
outcome: "error",
|
||||
errorCategory: "Error",
|
||||
durationMs: 100,
|
||||
},
|
||||
{ errorMessage: "upstream model stream stalled then aborted" },
|
||||
);
|
||||
await flushDiagnosticEvents();
|
||||
|
||||
expect(startedSpanOptions("openclaw.run")?.attributes?.["openclaw.error"]).toBe(
|
||||
"upstream model stream stalled then aborted",
|
||||
);
|
||||
expect(spanByName("openclaw.run").setStatus).toHaveBeenCalledWith({
|
||||
code: 2,
|
||||
message: "upstream model stream stalled then aborted",
|
||||
});
|
||||
// The raw message must never widen metric cardinality.
|
||||
const runDuration = lastHistogramRecord("openclaw.run.duration_ms");
|
||||
expect(runDuration?.[1]?.["openclaw.outcome"]).toBe("error");
|
||||
expect(Object.hasOwn(runDuration?.[1] ?? {}, "openclaw.error")).toBe(false);
|
||||
|
||||
await service.stop?.(ctx);
|
||||
});
|
||||
|
||||
test("run.completed bounds sensitive error text before export", async () => {
|
||||
const service = createDiagnosticsOtelService();
|
||||
const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { traces: true });
|
||||
await service.start(ctx);
|
||||
const secret = "sk-1234567890abcdef";
|
||||
|
||||
emitTrustedDiagnosticEventWithPrivateData(
|
||||
{
|
||||
type: "run.completed",
|
||||
runId: "run-1",
|
||||
outcome: "error",
|
||||
errorCategory: "Error",
|
||||
durationMs: 100,
|
||||
},
|
||||
{ errorMessage: `OPENAI_API_KEY=${secret} ${"x".repeat(8 * 1024)}` },
|
||||
);
|
||||
await flushDiagnosticEvents();
|
||||
|
||||
const status = mockCallArg(spanByName("openclaw.run").setStatus, 0) as {
|
||||
message?: string;
|
||||
};
|
||||
expect(status.message).not.toContain(secret);
|
||||
expect(status.message).toMatch(/\.\.\.\(truncated\)$/u);
|
||||
expect(status.message?.length).toBeLessThanOrEqual(4 * 1024 + 20);
|
||||
|
||||
await service.stop?.(ctx);
|
||||
});
|
||||
|
||||
test("harness.run.completed error span carries the redacted message", async () => {
|
||||
const service = createDiagnosticsOtelService();
|
||||
const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { traces: true, metrics: true });
|
||||
await service.start(ctx);
|
||||
|
||||
emitTrustedDiagnosticEventWithPrivateData(
|
||||
{
|
||||
type: "harness.run.completed",
|
||||
runId: "run-1",
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
harnessId: "openclaw",
|
||||
outcome: "error",
|
||||
durationMs: 90,
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
},
|
||||
{ errorMessage: "model run failed during resolve phase" },
|
||||
);
|
||||
await flushDiagnosticEvents();
|
||||
|
||||
expect(startedSpanOptions("openclaw.harness.run")?.attributes?.["openclaw.error"]).toBe(
|
||||
"model run failed during resolve phase",
|
||||
);
|
||||
expect(spanByName("openclaw.harness.run").setStatus).toHaveBeenCalledWith({
|
||||
code: 2,
|
||||
message: "model run failed during resolve phase",
|
||||
});
|
||||
const harnessDuration = lastHistogramRecord("openclaw.harness.duration_ms");
|
||||
expect(Object.hasOwn(harnessDuration?.[1] ?? {}, "openclaw.error")).toBe(false);
|
||||
|
||||
await service.stop?.(ctx);
|
||||
});
|
||||
|
||||
test("harness.run.error span prefers the redacted message over the category", async () => {
|
||||
const service = createDiagnosticsOtelService();
|
||||
const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { traces: true, metrics: true });
|
||||
await service.start(ctx);
|
||||
|
||||
emitTrustedDiagnosticEventWithPrivateData(
|
||||
{
|
||||
type: "harness.run.error",
|
||||
runId: "run-1",
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
harnessId: "openclaw",
|
||||
phase: "resolve",
|
||||
errorCategory: "Error",
|
||||
durationMs: 90,
|
||||
},
|
||||
{ errorMessage: "harness cleanup threw" },
|
||||
);
|
||||
await flushDiagnosticEvents();
|
||||
|
||||
expect(startedSpanOptions("openclaw.harness.run")?.attributes?.["openclaw.error"]).toBe(
|
||||
"harness cleanup threw",
|
||||
);
|
||||
expect(spanByName("openclaw.harness.run").setStatus).toHaveBeenCalledWith({
|
||||
code: 2,
|
||||
message: "harness cleanup threw",
|
||||
});
|
||||
|
||||
await service.stop?.(ctx);
|
||||
});
|
||||
|
||||
test("honors disabled traces when an OpenTelemetry SDK is preloaded", async () => {
|
||||
process.env.OPENCLAW_OTEL_PRELOADED = "1";
|
||||
const service = createDiagnosticsOtelService();
|
||||
|
||||
@@ -39,6 +39,7 @@ import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type {
|
||||
DiagnosticEventMetadata,
|
||||
DiagnosticEventPayload,
|
||||
DiagnosticEventPrivateData,
|
||||
DiagnosticTraceContext,
|
||||
OpenClawPluginService,
|
||||
OpenClawPluginServiceContext,
|
||||
@@ -80,6 +81,7 @@ const MAX_OTEL_CONTENT_ARRAY_ITEMS = 200;
|
||||
const MAX_OTEL_LOG_BODY_CHARS = 4 * 1024;
|
||||
const MAX_OTEL_LOG_ATTRIBUTE_COUNT = 64;
|
||||
const MAX_OTEL_LOG_ATTRIBUTE_VALUE_CHARS = 4 * 1024;
|
||||
const MAX_OTEL_ERROR_MESSAGE_CHARS = 4 * 1024;
|
||||
const LOG_RECORD_EXPORT_FAILURE_REPORT_INTERVAL_MS = 60_000;
|
||||
const OTEL_LOG_RAW_ATTRIBUTE_KEY_RE = /^[A-Za-z0-9_.:-]{1,64}$/u;
|
||||
const OTEL_LOG_ATTRIBUTE_KEY_RE = /^[A-Za-z0-9_.:-]{1,96}$/u;
|
||||
@@ -683,6 +685,14 @@ function normalizeOtelLogString(value: string, maxChars: number): string {
|
||||
return clampOtelLogText(redactSensitiveText(value), maxChars);
|
||||
}
|
||||
|
||||
function normalizeOtelErrorMessage(value: string | undefined): string | undefined {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = normalizeOtelLogString(value.trim(), MAX_OTEL_ERROR_MESSAGE_CHARS);
|
||||
return normalized || undefined;
|
||||
}
|
||||
|
||||
function resolveContentCapturePolicy(value: unknown): OtelContentCapturePolicy {
|
||||
if (value === true) {
|
||||
return {
|
||||
@@ -3066,6 +3076,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
|
||||
const recordRunCompleted = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "run.completed" }>,
|
||||
metadata: DiagnosticEventMetadata,
|
||||
privateData: DiagnosticEventPrivateData,
|
||||
) => {
|
||||
const attrs: Record<string, string | number> = {
|
||||
"openclaw.outcome": evt.outcome,
|
||||
@@ -3092,6 +3103,11 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
|
||||
if (evt.errorCategory) {
|
||||
spanAttrs["openclaw.errorCategory"] = lowCardinalityAttr(evt.errorCategory, "other");
|
||||
}
|
||||
// Redacted message goes on the span only, never the low-cardinality metric attrs.
|
||||
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
||||
if (redactedError) {
|
||||
spanAttrs["openclaw.error"] = redactedError;
|
||||
}
|
||||
const trustedTrace = trustedTraceContext(evt, metadata);
|
||||
const trackedSpan = trustedTrace?.spanId
|
||||
? activeTrustedSpans.get(trustedTrace.spanId)
|
||||
@@ -3104,9 +3120,12 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
|
||||
});
|
||||
setSpanAttrs(span, spanAttrs);
|
||||
if (evt.outcome === "error") {
|
||||
const message =
|
||||
redactedError ??
|
||||
(evt.errorCategory ? redactSensitiveText(evt.errorCategory) : undefined);
|
||||
span.setStatus({
|
||||
code: SpanStatusCode.ERROR,
|
||||
...(evt.errorCategory ? { message: redactSensitiveText(evt.errorCategory) } : {}),
|
||||
...(message ? { message } : {}),
|
||||
});
|
||||
}
|
||||
if (trackedSpan && trustedTrace?.spanId) {
|
||||
@@ -3149,6 +3168,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
|
||||
const recordHarnessRunCompleted = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "harness.run.completed" }>,
|
||||
metadata: DiagnosticEventMetadata,
|
||||
privateData: DiagnosticEventPrivateData,
|
||||
) => {
|
||||
harnessDurationHistogram.record(evt.durationMs, harnessRunMetricAttrs(evt));
|
||||
if (!tracesEnabled) {
|
||||
@@ -3170,6 +3190,11 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
|
||||
spanAttrs["openclaw.harness.items.completed"] = evt.itemLifecycle.completedCount;
|
||||
spanAttrs["openclaw.harness.items.active"] = evt.itemLifecycle.activeCount;
|
||||
}
|
||||
// Redacted message goes on the span only, never the low-cardinality metric attrs.
|
||||
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
||||
if (redactedError) {
|
||||
spanAttrs["openclaw.error"] = redactedError;
|
||||
}
|
||||
const trustedTrace = trustedTraceContext(evt, metadata);
|
||||
const trackedSpan = trustedTrace?.spanId
|
||||
? activeTrustedSpans.get(trustedTrace.spanId)
|
||||
@@ -3184,7 +3209,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
|
||||
if (evt.outcome === "error") {
|
||||
span.setStatus({
|
||||
code: SpanStatusCode.ERROR,
|
||||
message: "error",
|
||||
message: redactedError ?? "error",
|
||||
});
|
||||
}
|
||||
if (trackedSpan && trustedTrace?.spanId) {
|
||||
@@ -3197,6 +3222,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
|
||||
const recordHarnessRunError = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "harness.run.error" }>,
|
||||
metadata: DiagnosticEventMetadata,
|
||||
privateData: DiagnosticEventPrivateData,
|
||||
) => {
|
||||
const errorType = lowCardinalityAttr(evt.errorCategory, "other");
|
||||
const attrs = {
|
||||
@@ -3208,9 +3234,12 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
|
||||
if (!tracesEnabled) {
|
||||
return;
|
||||
}
|
||||
// Redacted message goes on the span only; attrs above feed the metric.
|
||||
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
||||
const spanAttrs: Record<string, string | number | boolean> = {
|
||||
...attrs,
|
||||
"error.type": errorType,
|
||||
...(redactedError ? { "openclaw.error": redactedError } : {}),
|
||||
...(evt.cleanupFailed ? { "openclaw.harness.cleanup_failed": true } : {}),
|
||||
};
|
||||
const span =
|
||||
@@ -3222,7 +3251,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
|
||||
setSpanAttrs(span, spanAttrs);
|
||||
span.setStatus({
|
||||
code: SpanStatusCode.ERROR,
|
||||
message: errorType,
|
||||
message: redactedError ?? errorType,
|
||||
});
|
||||
span.end(evt.ts);
|
||||
};
|
||||
@@ -3887,16 +3916,16 @@ export function createDiagnosticsOtelService(): OpenClawPluginService {
|
||||
recordRunStarted(evt, metadata);
|
||||
return;
|
||||
case "run.completed":
|
||||
recordRunCompleted(evt, metadata);
|
||||
recordRunCompleted(evt, metadata, privateData);
|
||||
return;
|
||||
case "harness.run.started":
|
||||
recordHarnessRunStarted(evt, metadata);
|
||||
return;
|
||||
case "harness.run.completed":
|
||||
recordHarnessRunCompleted(evt, metadata);
|
||||
recordHarnessRunCompleted(evt, metadata, privateData);
|
||||
return;
|
||||
case "harness.run.error":
|
||||
recordHarnessRunError(evt, metadata);
|
||||
recordHarnessRunError(evt, metadata, privateData);
|
||||
return;
|
||||
case "context.assembled":
|
||||
recordContextAssembled(evt, metadata);
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
// Coverage for trusted diagnostics emitted by a full embedded attempt.
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
onTrustedInternalDiagnosticEvent,
|
||||
resetDiagnosticEventsForTest,
|
||||
waitForDiagnosticEventsDrained,
|
||||
type DiagnosticEventPrivateData,
|
||||
type DiagnosticEventPayload,
|
||||
} from "../../../infra/diagnostic-events.js";
|
||||
import {
|
||||
cleanupTempPaths,
|
||||
createContextEngineAttemptRunner,
|
||||
createContextEngineBootstrapAndAssemble,
|
||||
preloadRunEmbeddedAttemptForTests,
|
||||
resetEmbeddedAttemptHarness,
|
||||
} from "./attempt.spawn-workspace.test-support.js";
|
||||
|
||||
describe("runEmbeddedAttempt diagnostics", () => {
|
||||
const tempPaths: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
await preloadRunEmbeddedAttemptForTests();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
resetEmbeddedAttemptHarness();
|
||||
resetDiagnosticEventsForTest();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await cleanupTempPaths(tempPaths);
|
||||
resetDiagnosticEventsForTest();
|
||||
});
|
||||
|
||||
it("keeps run failure text on the trusted private channel", async () => {
|
||||
const completed: Array<{
|
||||
event: DiagnosticEventPayload;
|
||||
privateData: DiagnosticEventPrivateData;
|
||||
}> = [];
|
||||
const unsubscribe = onTrustedInternalDiagnosticEvent((event, _metadata, privateData) => {
|
||||
if (event.type === "run.completed") {
|
||||
completed.push({ event, privateData });
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
await createContextEngineAttemptRunner({
|
||||
contextEngine: createContextEngineBootstrapAndAssemble(),
|
||||
sessionKey: "agent:main:diagnostic-failure",
|
||||
tempPaths,
|
||||
sessionPrompt: async () => {
|
||||
throw new Error("provider stream failed");
|
||||
},
|
||||
});
|
||||
await waitForDiagnosticEventsDrained();
|
||||
} finally {
|
||||
unsubscribe();
|
||||
}
|
||||
|
||||
expect(completed).toHaveLength(1);
|
||||
expect(completed[0]?.event).toMatchObject({
|
||||
type: "run.completed",
|
||||
outcome: "error",
|
||||
errorCategory: "Error",
|
||||
});
|
||||
expect(completed[0]?.event).not.toHaveProperty("error");
|
||||
expect(completed[0]?.privateData.errorMessage).toBe("provider stream failed");
|
||||
});
|
||||
});
|
||||
@@ -33,8 +33,14 @@ import {
|
||||
import { resolveContextEngineOwnerPluginId } from "../../../context-engine/registry.js";
|
||||
import { buildContextEngineRuntimeSettings } from "../../../context-engine/runtime-settings.js";
|
||||
import type { AssembleResult } from "../../../context-engine/types.js";
|
||||
import { diagnosticErrorCategory } from "../../../infra/diagnostic-error-metadata.js";
|
||||
import { emitTrustedDiagnosticEvent } from "../../../infra/diagnostic-events.js";
|
||||
import {
|
||||
diagnosticErrorCategory,
|
||||
diagnosticErrorMessage,
|
||||
} from "../../../infra/diagnostic-error-metadata.js";
|
||||
import {
|
||||
emitTrustedDiagnosticEvent,
|
||||
emitTrustedDiagnosticEventWithPrivateData,
|
||||
} from "../../../infra/diagnostic-events.js";
|
||||
import { resolveDiagnosticModelContentCapturePolicy } from "../../../infra/diagnostic-llm-content.js";
|
||||
import {
|
||||
createChildDiagnosticTraceContext,
|
||||
@@ -1188,14 +1194,19 @@ export async function runEmbeddedAttempt(
|
||||
return;
|
||||
}
|
||||
diagnosticRunCompleted = true;
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "run.completed",
|
||||
...diagnosticRunBase,
|
||||
durationMs: Date.now() - diagnosticRunStartedAt,
|
||||
outcome,
|
||||
...(extra?.blockedBy ? { blockedBy: extra.blockedBy } : {}),
|
||||
...(err && outcome !== "blocked" ? { errorCategory: diagnosticErrorCategory(err) } : {}),
|
||||
});
|
||||
const failed = err != null && outcome !== "blocked";
|
||||
const errorMessage = failed ? diagnosticErrorMessage(err) : undefined;
|
||||
emitTrustedDiagnosticEventWithPrivateData(
|
||||
{
|
||||
type: "run.completed",
|
||||
...diagnosticRunBase,
|
||||
durationMs: Date.now() - diagnosticRunStartedAt,
|
||||
outcome,
|
||||
...(extra?.blockedBy ? { blockedBy: extra.blockedBy } : {}),
|
||||
...(failed ? { errorCategory: diagnosticErrorCategory(err) } : {}),
|
||||
},
|
||||
errorMessage ? { errorMessage } : undefined,
|
||||
);
|
||||
};
|
||||
const corePluginToolStages = createEmbeddedRunStageTracker();
|
||||
const forceDirectMessageTool =
|
||||
|
||||
@@ -4,8 +4,9 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST } from "../../context-engine/host-compat.js";
|
||||
import type { ContextEngine } from "../../context-engine/types.js";
|
||||
import {
|
||||
onInternalDiagnosticEvent,
|
||||
onTrustedInternalDiagnosticEvent,
|
||||
resetDiagnosticEventsForTest,
|
||||
type DiagnosticEventPrivateData,
|
||||
type DiagnosticEventMetadata,
|
||||
type DiagnosticEventPayload,
|
||||
} from "../../infra/diagnostic-events.js";
|
||||
@@ -110,13 +111,21 @@ function captureDiagnosticEvents(
|
||||
filter: (event: DiagnosticEventPayload) => boolean = (event) =>
|
||||
event.type.startsWith("harness.run."),
|
||||
): {
|
||||
events: Array<{ event: DiagnosticEventPayload; metadata: DiagnosticEventMetadata }>;
|
||||
events: Array<{
|
||||
event: DiagnosticEventPayload;
|
||||
metadata: DiagnosticEventMetadata;
|
||||
privateData: DiagnosticEventPrivateData;
|
||||
}>;
|
||||
unsubscribe: () => void;
|
||||
} {
|
||||
const events: Array<{ event: DiagnosticEventPayload; metadata: DiagnosticEventMetadata }> = [];
|
||||
const unsubscribe = onInternalDiagnosticEvent((event, metadata) => {
|
||||
const events: Array<{
|
||||
event: DiagnosticEventPayload;
|
||||
metadata: DiagnosticEventMetadata;
|
||||
privateData: DiagnosticEventPrivateData;
|
||||
}> = [];
|
||||
const unsubscribe = onTrustedInternalDiagnosticEvent((event, metadata, privateData) => {
|
||||
if (filter(event)) {
|
||||
events.push({ event, metadata });
|
||||
events.push({ event, metadata, privateData });
|
||||
}
|
||||
});
|
||||
return { events, unsubscribe };
|
||||
@@ -320,6 +329,41 @@ describe("AgentHarness lifecycle runner", () => {
|
||||
expect(attemptResult?.diagnosticTrace).toEqual(harnessTrace);
|
||||
});
|
||||
|
||||
it("keeps plugin harness failure messages on the trusted private channel", async () => {
|
||||
const params = createAttemptParams();
|
||||
const harnessTrace = createDiagnosticTrace();
|
||||
const result = {
|
||||
...createAttemptResult(),
|
||||
promptError: new Error("provider stream failed"),
|
||||
} as EmbeddedRunAttemptResult;
|
||||
const harness: AgentHarness = {
|
||||
id: "codex",
|
||||
label: "Codex",
|
||||
supports: () => ({ supported: true }),
|
||||
runAttempt: async () => result,
|
||||
};
|
||||
const diagnostics = captureDiagnosticEvents(
|
||||
(event) => event.type === "run.completed" || event.type === "harness.run.completed",
|
||||
);
|
||||
try {
|
||||
await runWithDiagnosticTraceContext(harnessTrace, () =>
|
||||
runAgentHarnessLifecycleAttempt(harness, params),
|
||||
);
|
||||
await flushDiagnosticEvents();
|
||||
} finally {
|
||||
diagnostics.unsubscribe();
|
||||
}
|
||||
|
||||
expect(diagnostics.events.map(({ event }) => event.type)).toEqual([
|
||||
"run.completed",
|
||||
"harness.run.completed",
|
||||
]);
|
||||
for (const { event, privateData } of diagnostics.events) {
|
||||
expect(event).not.toHaveProperty("error");
|
||||
expect(privateData.errorMessage).toBe("provider stream failed");
|
||||
}
|
||||
});
|
||||
|
||||
it("emits plugin before-agent-run hook blocks as blocked run completions", async () => {
|
||||
resetDiagnosticEventsForTest();
|
||||
const params = createAttemptParams();
|
||||
@@ -390,6 +434,7 @@ describe("AgentHarness lifecycle runner", () => {
|
||||
expect(errorEvent?.type).toBe("harness.run.error");
|
||||
expect(errorEvent?.phase).toBe("send");
|
||||
expect(errorEvent?.errorCategory).toBe("Error");
|
||||
expect(diagnostics.events[1]?.privateData.errorMessage).toBe("codex app-server send failed");
|
||||
expect(errorEvent).not.toHaveProperty("cleanupFailed");
|
||||
expect(errorEvent?.harnessId).toBe("codex");
|
||||
expect(typeof errorEvent?.durationMs).toBe("number");
|
||||
|
||||
@@ -8,9 +8,13 @@ import {
|
||||
assertContextEngineHostSupport,
|
||||
type ContextEngineHostSupport,
|
||||
} from "../../context-engine/host-compat.js";
|
||||
import { diagnosticErrorCategory } from "../../infra/diagnostic-error-metadata.js";
|
||||
import {
|
||||
diagnosticErrorCategory,
|
||||
diagnosticErrorMessage,
|
||||
} from "../../infra/diagnostic-error-metadata.js";
|
||||
import {
|
||||
emitTrustedDiagnosticEvent,
|
||||
emitTrustedDiagnosticEventWithPrivateData,
|
||||
type DiagnosticHarnessRunErrorEvent,
|
||||
type DiagnosticHarnessRunOutcome,
|
||||
} from "../../infra/diagnostic-events.js";
|
||||
@@ -167,17 +171,24 @@ function emitAgentHarnessRunCompleted(params: {
|
||||
trace?: DiagnosticTraceContext;
|
||||
}): void {
|
||||
const { harness, attemptParams, result, startedAt, trace } = params;
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "harness.run.completed",
|
||||
...agentHarnessDiagnosticBase(harness, attemptParams, trace ?? result.diagnosticTrace),
|
||||
durationMs: Date.now() - startedAt,
|
||||
outcome: agentHarnessRunOutcome(result),
|
||||
...(result.agentHarnessResultClassification
|
||||
? { resultClassification: result.agentHarnessResultClassification }
|
||||
: {}),
|
||||
...(typeof result.yieldDetected === "boolean" ? { yieldDetected: result.yieldDetected } : {}),
|
||||
itemLifecycle: { ...result.itemLifecycle },
|
||||
});
|
||||
const outcome = agentHarnessRunOutcome(result);
|
||||
// A classified (non-thrown) failure carries its error on result.promptError;
|
||||
// forward the message so the error span shows more than a bare category.
|
||||
const errorMessage = outcome === "error" ? diagnosticErrorMessage(result.promptError) : undefined;
|
||||
emitTrustedDiagnosticEventWithPrivateData(
|
||||
{
|
||||
type: "harness.run.completed",
|
||||
...agentHarnessDiagnosticBase(harness, attemptParams, trace ?? result.diagnosticTrace),
|
||||
durationMs: Date.now() - startedAt,
|
||||
outcome,
|
||||
...(result.agentHarnessResultClassification
|
||||
? { resultClassification: result.agentHarnessResultClassification }
|
||||
: {}),
|
||||
...(typeof result.yieldDetected === "boolean" ? { yieldDetected: result.yieldDetected } : {}),
|
||||
itemLifecycle: { ...result.itemLifecycle },
|
||||
},
|
||||
errorMessage ? { errorMessage } : undefined,
|
||||
);
|
||||
}
|
||||
|
||||
function emitAgentHarnessRunError(params: {
|
||||
@@ -189,13 +200,17 @@ function emitAgentHarnessRunError(params: {
|
||||
trace?: DiagnosticTraceContext;
|
||||
}): void {
|
||||
const { harness, attemptParams, startedAt, phase, error, trace } = params;
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "harness.run.error",
|
||||
...agentHarnessDiagnosticBase(harness, attemptParams, trace),
|
||||
durationMs: Date.now() - startedAt,
|
||||
phase,
|
||||
errorCategory: diagnosticErrorCategory(error),
|
||||
});
|
||||
const errorMessage = diagnosticErrorMessage(error);
|
||||
emitTrustedDiagnosticEventWithPrivateData(
|
||||
{
|
||||
type: "harness.run.error",
|
||||
...agentHarnessDiagnosticBase(harness, attemptParams, trace),
|
||||
durationMs: Date.now() - startedAt,
|
||||
phase,
|
||||
errorCategory: diagnosticErrorCategory(error),
|
||||
},
|
||||
errorMessage ? { errorMessage } : undefined,
|
||||
);
|
||||
}
|
||||
|
||||
/** Runs one harness attempt with diagnostics, tracing, and result classification. */
|
||||
@@ -215,16 +230,19 @@ export async function runAgentHarnessLifecycleAttempt(
|
||||
return;
|
||||
}
|
||||
agentRunCompleted = true;
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "run.completed",
|
||||
...agentRunDiagnosticBase(params, agentRunTrace),
|
||||
durationMs: Date.now() - agentRunStartedAt,
|
||||
outcome: completion.outcome,
|
||||
...(completion.blockedBy ? { blockedBy: completion.blockedBy } : {}),
|
||||
...(completion.error && completion.outcome === "error"
|
||||
? { errorCategory: diagnosticErrorCategory(completion.error) }
|
||||
: {}),
|
||||
});
|
||||
const failed = completion.outcome === "error" && completion.error != null;
|
||||
const errorMessage = failed ? diagnosticErrorMessage(completion.error) : undefined;
|
||||
emitTrustedDiagnosticEventWithPrivateData(
|
||||
{
|
||||
type: "run.completed",
|
||||
...agentRunDiagnosticBase(params, agentRunTrace),
|
||||
durationMs: Date.now() - agentRunStartedAt,
|
||||
outcome: completion.outcome,
|
||||
...(completion.blockedBy ? { blockedBy: completion.blockedBy } : {}),
|
||||
...(failed ? { errorCategory: diagnosticErrorCategory(completion.error) } : {}),
|
||||
},
|
||||
errorMessage ? { errorMessage } : undefined,
|
||||
);
|
||||
};
|
||||
|
||||
emitAgentHarnessRunStarted(harness, params, activeHarnessTrace);
|
||||
|
||||
@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
diagnosticErrorCategory,
|
||||
diagnosticErrorFailureKind,
|
||||
diagnosticErrorMessage,
|
||||
diagnosticHttpStatusCode,
|
||||
diagnosticProviderRequestIdHash,
|
||||
} from "./diagnostic-error-metadata.js";
|
||||
@@ -22,6 +23,24 @@ describe("diagnostic error metadata", () => {
|
||||
expect(diagnosticErrorCategory(null)).toBe("null");
|
||||
});
|
||||
|
||||
it("normalizes Error and string messages", () => {
|
||||
expect(diagnosticErrorMessage(new Error(" provider failed "))).toBe("provider failed");
|
||||
expect(diagnosticErrorMessage(" connection closed ")).toBe("connection closed");
|
||||
expect(diagnosticErrorMessage(" ")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not invoke message getters", () => {
|
||||
const errorLike = Object.create(null, {
|
||||
message: {
|
||||
get() {
|
||||
throw new Error("getter must not run");
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(diagnosticErrorMessage(errorLike)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("accepts only own HTTP status data properties as error codes", () => {
|
||||
expect(diagnosticHttpStatusCode({ status: 429 })).toBe("429");
|
||||
expect(diagnosticHttpStatusCode({ statusCode: 503 })).toBe("503");
|
||||
|
||||
@@ -157,6 +157,19 @@ export function diagnosticErrorCategory(err: unknown): string {
|
||||
return typeof err;
|
||||
}
|
||||
|
||||
/**
|
||||
* Human-readable error message for diagnostics. Complements
|
||||
* {@link diagnosticErrorCategory} (low-cardinality class name) with the actual
|
||||
* message so error spans carry a real status message instead of a bare
|
||||
* category. Reads only an own data property so diagnostics never invoke a
|
||||
* user-defined getter.
|
||||
*/
|
||||
export function diagnosticErrorMessage(err: unknown): string | undefined {
|
||||
const text = readDirectMessage(err);
|
||||
const trimmed = text?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
/** Extracts a safe HTTP status code from own `status` or `statusCode` data properties. */
|
||||
export function diagnosticHttpStatusCode(err: unknown): string | undefined {
|
||||
const status = readOwnDataProperty(err, "status");
|
||||
|
||||
@@ -839,6 +839,8 @@ export type DiagnosticSkillUsagePrivateData = Readonly<{
|
||||
}>;
|
||||
|
||||
export type DiagnosticEventPrivateData = Readonly<{
|
||||
/** Raw failure text for trusted diagnostics exporters; never part of the public event payload. */
|
||||
errorMessage?: string;
|
||||
modelContent?: DiagnosticModelCallContent;
|
||||
skillUsage?: DiagnosticSkillUsagePrivateData;
|
||||
toolContent?: DiagnosticToolCallContent;
|
||||
|
||||
@@ -127,6 +127,8 @@ const REQUIRED_SPAN_NAMES = [
|
||||
const REQUIRED_METRIC_NAMES = ["openclaw.harness.duration_ms"] as const;
|
||||
const DIRECT_RUN_ID = "qa-otel-direct-run";
|
||||
const DIRECT_CALL_ID = "qa-otel-direct-call";
|
||||
const DIRECT_ERROR_MESSAGE = "QA OTEL provider stream failed";
|
||||
const DIRECT_ERROR_SECRET = "sk-1234567890abcdef";
|
||||
const DISALLOWED_ATTRIBUTE_KEYS = new Set([
|
||||
"openclaw.runId",
|
||||
"openclaw.chatId",
|
||||
@@ -146,6 +148,7 @@ const DISALLOWED_ATTRIBUTE_KEYS = new Set([
|
||||
const DISALLOWED_BODY_NEEDLES = [
|
||||
"OTEL-QA-SECRET",
|
||||
"OTEL-QA-OK",
|
||||
DIRECT_ERROR_SECRET,
|
||||
DIRECT_RUN_ID,
|
||||
DIRECT_CALL_ID,
|
||||
];
|
||||
@@ -1410,28 +1413,38 @@ async function runDirectTelemetryProducer(params: {
|
||||
},
|
||||
},
|
||||
);
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "run.completed",
|
||||
runId: DIRECT_RUN_ID,
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
channel: "qa",
|
||||
durationMs: 8,
|
||||
outcome: "completed",
|
||||
trace: runTrace,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "harness.run.completed",
|
||||
runId: DIRECT_RUN_ID,
|
||||
harnessId: "qa-otel-direct",
|
||||
pluginId: "diagnostics-otel",
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
channel: "qa",
|
||||
durationMs: 10,
|
||||
outcome: "completed",
|
||||
trace: harnessTrace,
|
||||
});
|
||||
const failurePrivateData = {
|
||||
errorMessage: `${DIRECT_ERROR_MESSAGE} OPENAI_API_KEY=${DIRECT_ERROR_SECRET}`,
|
||||
};
|
||||
emitTrustedDiagnosticEventWithPrivateData(
|
||||
{
|
||||
type: "run.completed",
|
||||
runId: DIRECT_RUN_ID,
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
channel: "qa",
|
||||
durationMs: 8,
|
||||
outcome: "error",
|
||||
errorCategory: "Error",
|
||||
trace: runTrace,
|
||||
},
|
||||
failurePrivateData,
|
||||
);
|
||||
emitTrustedDiagnosticEventWithPrivateData(
|
||||
{
|
||||
type: "harness.run.completed",
|
||||
runId: DIRECT_RUN_ID,
|
||||
harnessId: "qa-otel-direct",
|
||||
pluginId: "diagnostics-otel",
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
channel: "qa",
|
||||
durationMs: 10,
|
||||
outcome: "error",
|
||||
trace: harnessTrace,
|
||||
},
|
||||
failurePrivateData,
|
||||
);
|
||||
await waitForDiagnosticEventsDrained();
|
||||
} finally {
|
||||
await service.stop?.(context);
|
||||
@@ -1629,6 +1642,23 @@ function assertSmoke(params: {
|
||||
failures.push("successful QA run exported model-call error attributes");
|
||||
}
|
||||
|
||||
const failedRunSpans = params.spans.filter(
|
||||
(span) =>
|
||||
(span.name === "openclaw.run" || span.name === "openclaw.harness.run") &&
|
||||
span.attributes["openclaw.error"] === `${DIRECT_ERROR_MESSAGE} OPENAI_API_KEY=***`,
|
||||
);
|
||||
if (failedRunSpans.length !== 2) {
|
||||
const observed = params.spans
|
||||
.filter((span) => span.name === "openclaw.run" || span.name === "openclaw.harness.run")
|
||||
.map((span) => ({ name: span.name, error: span.attributes["openclaw.error"] }));
|
||||
failures.push(
|
||||
`run and harness spans did not export the redacted failure message: ${JSON.stringify(observed)}`,
|
||||
);
|
||||
}
|
||||
if ((params.bodyText.metrics ?? []).some((body) => body.includes(DIRECT_ERROR_MESSAGE))) {
|
||||
failures.push("run failure message leaked into OTLP metric attributes");
|
||||
}
|
||||
|
||||
const serializedAttributes = JSON.stringify(params.spans.map((span) => span.attributes));
|
||||
if (serializedAttributes.includes("StreamAbandoned")) {
|
||||
failures.push("StreamAbandoned leaked into OTEL attributes");
|
||||
|
||||
@@ -91,8 +91,20 @@ describe("qa-otel-smoke receiver bounds", () => {
|
||||
stdoutLogLines: [],
|
||||
stdoutLogRecords: [],
|
||||
spans: [
|
||||
{ name: "openclaw.run", parent: false, attributes: {} },
|
||||
{ name: "openclaw.harness.run", parent: true, attributes: {} },
|
||||
{
|
||||
name: "openclaw.run",
|
||||
parent: false,
|
||||
attributes: {
|
||||
"openclaw.error": "QA OTEL provider stream failed OPENAI_API_KEY=***",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "openclaw.harness.run",
|
||||
parent: true,
|
||||
attributes: {
|
||||
"openclaw.error": "QA OTEL provider stream failed OPENAI_API_KEY=***",
|
||||
},
|
||||
},
|
||||
{ name: "openclaw.context.assembled", parent: true, attributes: {} },
|
||||
{ name: "openclaw.message.delivery", parent: true, attributes: {} },
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user