From 6651813bd0b44d498f4c8695a6693e66431b9e74 Mon Sep 17 00:00:00 2001 From: ooiuuii Date: Fri, 17 Jul 2026 15:25:15 +0800 Subject: [PATCH] fix(logging): redact auth-style HTTP headers (#107999) * fix(logging): redact auth-style HTTP headers * fix(logging): redact structured auth headers * test(logging): keep auth fixtures synthetic * test(logging): preserve existing gateway fixtures * fix(logging): cover serialized auth credentials * fix(logging): harden auth header redaction * fix(logging): cover serialized auth whitespace * chore(logging): refresh reviewed PR head * test(logging): expect complete API key redaction --------- Co-authored-by: Peter Steinberger --- packages/acp-core/src/error-format.test.ts | 322 ++++++++++++++ packages/acp-core/src/error-format.ts | 83 +++- packages/acp-core/src/index.ts | 1 + .../acp-core/src/structured-auth-redaction.ts | 409 ++++++++++++++++++ .../openai-transport-stream.base.test.ts | 7 +- src/logging/logger-redaction-behavior.test.ts | 24 + src/logging/redact.test.ts | 397 +++++++++++++++++ src/logging/redact.ts | 65 ++- 8 files changed, 1293 insertions(+), 15 deletions(-) create mode 100644 packages/acp-core/src/structured-auth-redaction.ts diff --git a/packages/acp-core/src/error-format.test.ts b/packages/acp-core/src/error-format.test.ts index 78f00f2d72d..8bc8e120891 100644 --- a/packages/acp-core/src/error-format.test.ts +++ b/packages/acp-core/src/error-format.test.ts @@ -33,4 +33,326 @@ describe("redactSensitiveText", () => { configureAcpErrorRedactor(undefined); } }); + + it("redacts unquoted auth-style HTTP headers in fallback errors", () => { + const keyHeader = ["api", "-", "key"].join(""); + const googleHeader = ["x", "-", "goog", "-", "api", "-", "key"].join(""); + const accessHeader = ["x", "-", "access", "-", "token"].join(""); + const digestUser = ["digest", "user", "example"].join("-"); + const digestResponse = ["digest", "response", "1234567890abcdef"].join("-"); + const digestExtension = ["digest", "extension", "1234567890abcdef"].join("-"); + const digestTail = ["digest", "tail", "1234567890abcdef"].join("-"); + const awsCredential = [ + "AK", + "IA", + "EXAMPLE", + "1234567890", + "/20260717/eu-west-1/s3/aws4_request", + ].join(""); + const awsSignature = ["aws", "signature", "1234567890abcdef"].join("-"); + const input = [ + ["Authorization", ": token ", "samplevalue1234567890abcd"].join(""), + ["Proxy-Authorization", ": Digest ", "sampleproxyvalue1234567890"].join(""), + `Authorization: Digest username="${digestUser}", 2fa="${digestExtension}", response="${digestResponse}", extension="${digestTail}", cnonce="tail-nonce"; request_id=digest-example`, + `Authorization: AWS4-HMAC-SHA256 Credential=${awsCredential}, SignedHeaders=:authority;x_custom;x.custom, Signature=${awsSignature}; status=403`, + `Proxy-Authorization: Digest username="${digestUser}", response="${digestResponse}"; request_id=proxy-example`, + [keyHeader, ": ", "samplekeyvalue1234567890"].join(""), + [googleHeader, "=", "samplegoogvalue1234567890"].join(""), + [accessHeader, ": ", "sampleaccessvalue1234567890"].join(""), + ].join("\n"); + + expect(redactSensitiveText(input)).toBe( + [ + ["Authorization", ": token ", "[REDACTED]"].join(""), + ["Proxy-Authorization", ": Digest ", "[REDACTED]"].join(""), + "Authorization: Digest [REDACTED]; request_id=digest-example", + "Authorization: AWS4-HMAC-SHA256 [REDACTED]; status=403", + "Proxy-Authorization: Digest [REDACTED]; request_id=proxy-example", + [keyHeader, ": ", "[REDACTED]"].join(""), + [googleHeader, "=", "[REDACTED]"].join(""), + [accessHeader, ": ", "[REDACTED]"].join(""), + ].join("\n"), + ); + }); + + it("redacts escaped structured authorization fields", () => { + const response = ["escaped", "digest", "response", "1234567890abcdef"].join("-"); + const input = `Authorization: Digest realm=\\"Example Realm\\", response=\\"${response}\\"; status=401`; + + expect(redactSensitiveText(input)).toBe("Authorization: Digest [REDACTED]; status=401"); + }); + + it("redacts consecutive, prefixed, and serialized auth headers", () => { + const proxyValue = ["cHJveH", "k6cGFz", "cw=="].join(""); + const customValue = ["Y3VzdG", "9tOnBh", "c3M="].join(""); + const accessValue = ["sample", "access", "value", "1234567890"].join("-"); + const googleValue = ["sample", "google", "value", "1234567890"].join("-"); + const input = [ + "Proxy-Authorization: Foo", + `Proxy-Authorization: Basic ${proxyValue}`, + `X-Authorization: Basic ${customValue}`, + JSON.stringify({ + "x-access-token": accessValue, + "x-goog-api-key": googleValue, + }), + ].join("\n"); + + expect(redactSensitiveText(input)).toBe( + [ + "Proxy-Authorization: [REDACTED]", + "Proxy-Authorization: Basic [REDACTED]", + "X-Authorization: Basic [REDACTED]", + JSON.stringify({ + "x-access-token": "[REDACTED]", + "x-goog-api-key": "[REDACTED]", + }), + ].join("\n"), + ); + }); + + it("redacts later auth params and token credentials after punctuation", () => { + const responseValue = ["later", "response", "value", "1234567890"].join("-"); + const negotiateValue = ["cHJvb2", "YxMjM0", "NTY3ODkw"].join(""); + const foldedValue = ["Zm9sZG", "VkOnNl", "Y3JldA=="].join(""); + const rawValue = ["raw", "header", "value", "1234567890"].join("-"); + const input = [ + `Authorization: Digest username="sample",,response="${responseValue}"; status=401`, + `Authorization: Digest damaged,,response="${responseValue}"; status=403`, + `Authorization: Digest username="sample", uri=/bad, response="${responseValue}"; status=407`, + `Authorization: Digest username="sample",\r\n response="${responseValue}"; status=408`, + `Authorization: Digest uri=http://service, response="${responseValue}"; status=409`, + `Authorization: Digest response='${responseValue}'; status=410`, + `Authorization: Digest realm=sample, authorization-param=${responseValue}; status=412`, + `Authorization: Digest username=sample,\\r\\n response=${responseValue}; status=413`, + `Authorization: Digest username=sample,\\r\\n\\tresponse=${responseValue}; status=414`, + `(Authorization: Negotiate ${negotiateValue})`, + `Authorization:\r\n Basic ${foldedValue}`, + `Authorization:\nBasic ${foldedValue}`, + `Authorization:\\nBasic ${foldedValue}`, + `Authorization:\\tBearer ${foldedValue}`, + `Authorization: Bearer\\t${foldedValue}`, + `Authorization: ${rawValue} `, + ].join("\n"); + + expect(redactSensitiveText(input)).toBe( + [ + "Authorization: Digest [REDACTED]; status=401", + "Authorization: Digest [REDACTED]; status=403", + "Authorization: Digest [REDACTED]; status=407", + "Authorization: Digest [REDACTED]; status=408", + "Authorization: Digest [REDACTED]; status=409", + "Authorization: Digest [REDACTED]; status=410", + "Authorization: Digest [REDACTED]; status=412", + "Authorization: Digest [REDACTED]; status=413", + "Authorization: Digest [REDACTED]; status=414", + "(Authorization: Negotiate [REDACTED])", + "Authorization:\r\n Basic [REDACTED]", + "Authorization:\nBasic [REDACTED]", + "Authorization:\\nBasic [REDACTED]", + "Authorization:\\tBearer [REDACTED]", + "Authorization: Bearer\\t[REDACTED]", + "Authorization: [REDACTED] ", + ].join("\n"), + ); + + const serializedLine = JSON.stringify( + `prefix\nAuthorization: Digest response="${responseValue}"`, + ); + expect(redactSensitiveText(serializedLine)).toBe( + JSON.stringify("prefix\nAuthorization: Digest [REDACTED]"), + ); + }); + + it("bounds recovery between repeated malformed auth headers", () => { + const response = ["final", "response", "1234567890abcdef"].join("-"); + const malformed = Array.from({ length: 128 }, () => "Authorization: Digest damaged").join(" "); + const output = redactSensitiveText( + `${malformed} Authorization: Digest response="${response}"; status=411`, + ); + + expect(output).not.toContain(response); + expect(output).toContain("Authorization: Digest [REDACTED]; status=411"); + }); + + it("redacts parameterized schemes and encoded quoted-pairs", () => { + const proof = ["hawk", "credential", "proof", "1234567890abcdef"].join("-"); + const response = ["escaped", "quoted", "response", "1234567890abcdef"].join("-"); + const input = [ + `Authorization: Hawk id="client", mac="${proof}"; status=401`, + `Authorization: Digest realm=\\"Example \\\\\\"Realm\\\\\\"\\", response=\\"${response}\\"; status=403`, + ].join("\n"); + + expect(redactSensitiveText(input)).toBe( + [ + "Authorization: Hawk [REDACTED]; status=401", + "Authorization: Digest [REDACTED]; status=403", + ].join("\n"), + ); + }); + + it("redacts full auth-param tokens and serialized header objects", () => { + const id = ["abc", "'", "def", "`", "ghi"].join(""); + const proof = ["token", "grammar", "proof", "1234567890abcdef"].join("-"); + const response = ["json", "digest", "response", "1234567890abcdef"].join("-"); + const input = [ + `Authorization: Foo id=${id}, proof=${proof}; status=401`, + `{"Authorization":"Digest username=\\"example\\", response=\\"${response}\\""}`, + ].join("\n"); + + expect(redactSensitiveText(input)).toBe( + ["Authorization: Foo [REDACTED]; status=401", `{"Authorization":"[REDACTED]"}`].join("\n"), + ); + }); + + it("redacts nested serialized headers and punctuated token schemes", () => { + const response = ["nested", "digest", "response", "1234567890abcdef"].join("-"); + const header = { Authorization: `Digest response="${response}\\\\"` }; + const serialized = JSON.stringify(JSON.stringify(header)); + const token = ["extension", "token", "1234567890abcdef"].join("-"); + const basicCredential = ["dXNl", "cjpw", "YXNz"].join(""); + const nestedBasic = JSON.stringify( + JSON.stringify({ Authorization: `Basic ${basicCredential}` }), + ); + const bearerCredential = ["/opaque", "~bearer", "1234567890abcdef"].join("-"); + const nestedBearer = JSON.stringify( + JSON.stringify({ Authorization: `Bearer ${bearerCredential}` }), + ); + const opaqueCredential = ["opaque", "credential", "1234567890abcdef"].join("-"); + const nestedOpaque = JSON.stringify( + JSON.stringify({ + Authorization: opaqueCredential, + "Proxy-Authorization": opaqueCredential, + }), + ); + + expect(redactSensitiveText(serialized)).toBe( + JSON.stringify(JSON.stringify({ Authorization: "Digest [REDACTED]" })), + ); + expect(redactSensitiveText(JSON.stringify(header))).toBe( + JSON.stringify({ Authorization: "[REDACTED]" }), + ); + expect(redactSensitiveText(`Authorization: Foo+Bar ${token}; status=401`)).toBe( + "Authorization: Foo+Bar [REDACTED]; status=401", + ); + expect(redactSensitiveText(`Authorization: Basic+Foo ${token}; status=401`)).toBe( + "Authorization: Basic+Foo [REDACTED]; status=401", + ); + expect(redactSensitiveText(nestedBasic)).toBe( + JSON.stringify(JSON.stringify({ Authorization: "Basic [REDACTED]" })), + ); + expect(redactSensitiveText(`Authorization: Bearer ${bearerCredential}`)).toBe( + "Authorization: Bearer [REDACTED]", + ); + expect(redactSensitiveText(`request failed: Bearer ${bearerCredential}`)).toBe( + "request failed: Bearer [REDACTED]", + ); + expect(redactSensitiveText(nestedBearer)).toBe( + JSON.stringify(JSON.stringify({ Authorization: "Bearer [REDACTED]" })), + ); + expect(redactSensitiveText(nestedOpaque)).toBe( + JSON.stringify( + JSON.stringify({ + Authorization: "[REDACTED]", + "Proxy-Authorization": "[REDACTED]", + }), + ), + ); + }); + + it("does not confuse real marker prefixes with internal redaction state", () => { + const envKey = ["API", "_", "KEY"].join(""); + const prefixedSecret = ["***", "live", "secret", "1234567890abcdef"].join("-"); + const response = ["marker", "digest", "response", "1234567890abcdef"].join("-"); + const input = [ + `${envKey}=${prefixedSecret}`, + `Authorization: Digest ***ext=one, response="${response}"; status=401`, + ].join("\n"); + + expect(redactSensitiveText(input)).toBe( + [`${envKey}=[REDACTED]`, "Authorization: Digest [REDACTED]; status=401"].join("\n"), + ); + }); + + it("preserves marker-shaped input while redacting structured auth", () => { + const markerZero = ";__openclaw_structured_auth_redacted_0;"; + const markerOne = ";__openclaw_structured_auth_redacted_1;"; + const response = ["collision", "response", "1234567890abcdef"].join("-"); + const input = [ + markerZero, + markerOne, + `Authorization: Digest response="${response}"; status=401`, + ].join("\n"); + + expect(redactSensitiveText(input)).toBe( + [markerZero, markerOne, "Authorization: Digest [REDACTED]; status=401"].join("\n"), + ); + }); + + it("preserves Basic padding diagnostics and masks punctuation in header values", () => { + const keyHeader = ["api", "-", "key"].join(""); + const input = [ + "Authorization: Basic dXNlcg==, status=401", + `${keyHeader}: prefix)sensitive-suffix`, + ].join("\n"); + + expect(redactSensitiveText(input)).toBe( + ["Authorization: Basic [REDACTED], status=401", `${keyHeader}: [REDACTED]`].join("\n"), + ); + }); + + it("redacts quoted diagnostics while preserving structural closers", () => { + const response = ["quoted", "header", "response", "1234567890abcdef"].join("-"); + const signature = ["structural", "aws", "signature", "1234567890abcdef"].join("-"); + const awsScopeField = ["Cred", "ential", "=scope/path"].join(""); + const input = [ + `curl -H 'Authorization: Digest username="example", response="${response}"'`, + `{Authorization: AWS4-HMAC-SHA256 ${awsScopeField}, SignedHeaders=host, Signature=${signature}}`, + ].join("\n"); + + expect(redactSensitiveText(input)).toBe( + [ + "curl -H 'Authorization: Digest [REDACTED]'", + "{Authorization: AWS4-HMAC-SHA256 [REDACTED]}", + ].join("\n"), + ); + }); + + it("keeps diagnostics adjacent to single-token auth headers", () => { + const keyHeader = ["api", "-", "key"].join(""); + const token = ["opaque", "auth", "value", "1234567890abcdef"].join("-"); + const input = [ + `Authorization: ${token}, status=401`, + `Proxy-Authorization: ${token}; request_id=example`, + `${keyHeader}: ${token}, status=500`, + ].join("\n"); + + expect(redactSensitiveText(input)).toBe( + [ + "Authorization: [REDACTED], status=401", + "Proxy-Authorization: [REDACTED]; request_id=example", + `${keyHeader}: [REDACTED], status=500`, + ].join("\n"), + ); + }); + + it("keeps diagnostics adjacent to scheme-token auth headers", () => { + const token = ["scheme", "auth", "value", "1234567890abcdef"].join("-"); + const input = [ + `Authorization: Token ${token}, status=401`, + `Proxy-Authorization: Basic ${token}; request_id=example`, + ].join("\n"); + + expect(redactSensitiveText(input)).toBe( + [ + "Authorization: Token [REDACTED], status=401", + "Proxy-Authorization: Basic [REDACTED]; request_id=example", + ].join("\n"), + ); + }); + + it("does not redact ordinary authorization prose in fallback errors", () => { + const input = "the authorization model is open"; + + expect(redactSensitiveText(input)).toBe(input); + }); }); diff --git a/packages/acp-core/src/error-format.ts b/packages/acp-core/src/error-format.ts index 908286e28d0..d0f0cfd0478 100644 --- a/packages/acp-core/src/error-format.ts +++ b/packages/acp-core/src/error-format.ts @@ -1,4 +1,18 @@ // ACP Core helper module supports error format behavior. +import { + CREDENTIAL_STYLE_HEADER_REDACT_PATTERN, + HTTP_AUTH_HEADER_BOUNDARY_PATTERN, + HTTP_AUTH_LEGACY_VALUE_WHITESPACE_PATTERN, + HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN, + HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN, + HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN, + HTTP_AUTH_SCHEME_PATTERN, + HTTP_AUTH_SERIALIZED_QUOTE_PATTERN, + redactStructuredAuthHeaders, +} from "./structured-auth-redaction.js"; + +const STRUCTURED_AUTH_MARKER_PREFIX = ";__openclaw_structured_auth_redacted_"; + const SECRET_PATTERNS: RegExp[] = [ /\b[A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|CARD[_-]?NUMBER|CARD[_-]?CVC|CARD[_-]?CVV|CVC|CVV|SECURITY[_-]?CODE|PAYMENT[_-]?CREDENTIAL|SHARED[_-]?PAYMENT[_-]?TOKEN)\b\s*[=:]\s*(["']?)([^\s"'\\]+)\1/g, /\b[A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|CARD[_-]?NUMBER|CARD[_-]?CVC|CARD[_-]?CVV|CVC|CVV|SECURITY[_-]?CODE|PAYMENT[_-]?CREDENTIAL|SHARED[_-]?PAYMENT[_-]?TOKEN)\b\s*[=:]\s*\\+(["'])([^\s"'\\]+)\\+\1/g, @@ -7,10 +21,33 @@ const SECRET_PATTERNS: RegExp[] = [ /(^|[\s,{])["']?(?:api[-_]key|access[-_]token|refresh[-_]token|authToken|auth[-_]token|clientSecret|client[-_]secret|appSecret|app[-_]secret)["']?\s*[:=]\s*(["'])([^"'\r\n]+)\2/gi, /(^|[\s,{])["']?(?:authorization|proxy-authorization|cookie|set-cookie|x-api-key|x-auth-token)["']?\s*[:=]\s*(["'])([^"'\r\n]+)\2/gi, /--(?:api[-_]?key|hook[-_]?token|token|secret|password|passwd|card[-_]?number|card[-_]?cvc|card[-_]?cvv|cvc|cvv|security[-_]?code|payment[-_]?credential|shared[-_]?payment[-_]?token)\s+(["']?)([^\s"']+)\1/gi, - /Authorization\s*[:=]\s*Bearer\s+([A-Za-z0-9._\-+=]+)/gi, - /Authorization\s*[:=]\s*Basic\s+([A-Za-z0-9+/=]+)/gi, + new RegExp( + String.raw`Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_LEGACY_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}Bearer${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}(${HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN})`, + "gi", + ), + new RegExp( + String.raw`Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_LEGACY_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}Basic${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}(${HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN})`, + "gi", + ), + new RegExp( + String.raw`${HTTP_AUTH_HEADER_BOUNDARY_PATTERN}Proxy-Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}${HTTP_AUTH_SCHEME_PATTERN}${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}(${HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN})`, + "gi", + ), + new RegExp( + String.raw`${HTTP_AUTH_HEADER_BOUNDARY_PATTERN}Proxy-Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}(?!${HTTP_AUTH_SCHEME_PATTERN}${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}${STRUCTURED_AUTH_MARKER_PREFIX})(${HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN})[ \t]*(?=${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}(?:$|[,;)}\]]|\r?\n(?![ \t])))`, + "gi", + ), + new RegExp( + String.raw`${HTTP_AUTH_HEADER_BOUNDARY_PATTERN}Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}(?!(?:Bearer|Basic)(?=${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}))${HTTP_AUTH_SCHEME_PATTERN}${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}(${HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN})`, + "gi", + ), + new RegExp( + String.raw`${HTTP_AUTH_HEADER_BOUNDARY_PATTERN}Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}(?!(?:Bearer|Basic)(?=${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}))(?!${HTTP_AUTH_SCHEME_PATTERN}${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}${STRUCTURED_AUTH_MARKER_PREFIX})(${HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN})[ \t]*(?=${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}(?:$|[,;)}\]]|\r?\n(?![ \t])))`, + "gi", + ), + new RegExp(CREDENTIAL_STYLE_HEADER_REDACT_PATTERN, "gi"), /(?:X-OpenClaw-Token|x-pomerium-jwt-assertion|X-Api-Key|X-Auth-Token)\s*[:=]\s*([^\s"',;]+)/gi, - /\bBearer\s+([A-Za-z0-9._\-+=]{18,})\b/g, + /\bBearer\s+([-A-Za-z0-9._~+/=]{18,})(?![-A-Za-z0-9._~+/=])/g, /(^|[\s,;])(?:access_token|refresh_token|auth[-_]?token|api[-_]?key|client[-_]?secret|app[-_]?secret|token|secret|password|passwd|card[-_]?number|card[-_]?cvc|card[-_]?cvv|cvc|cvv|security[-_]?code|payment[-_]?credential|shared[-_]?payment[-_]?token)=([^\s&#]+)/gi, /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]+?-----END [A-Z ]*PRIVATE KEY-----/g, /\b(sk-[A-Za-z0-9_-]{8,})\b/g, @@ -35,6 +72,40 @@ const SECRET_PATTERNS: RegExp[] = [ let configuredRedactor: ((value: string) => string) | undefined; +function createStructuredAuthMarker(value: string): string { + const usedIds = new Set(); + const maxIdDigits = String(value.length).length; + let cursor = 0; + for (;;) { + const markerStart = value.indexOf(STRUCTURED_AUTH_MARKER_PREFIX, cursor); + if (markerStart < 0) { + break; + } + const idStart = markerStart + STRUCTURED_AUTH_MARKER_PREFIX.length; + let idEnd = idStart; + while (idEnd - idStart <= maxIdDigits) { + const char = value[idEnd]; + if (char === undefined || char < "0" || char > "9") { + break; + } + idEnd += 1; + } + if (idEnd > idStart && value[idEnd] === ";" && idEnd - idStart <= maxIdDigits) { + const id = Number(value.slice(idStart, idEnd)); + if (id <= value.length) { + usedIds.add(id); + } + } + cursor = idStart; + } + + let id = 0; + while (usedIds.has(id)) { + id += 1; + } + return `${STRUCTURED_AUTH_MARKER_PREFIX}${id};`; +} + /** Installs a host-provided redactor used before ACP fallback secret-pattern redaction. */ export function configureAcpErrorRedactor(redactor: ((value: string) => string) | undefined): void { configuredRedactor = redactor; @@ -42,7 +113,9 @@ export function configureAcpErrorRedactor(redactor: ((value: string) => string) /** Redacts common provider, GitHub, HTTP, payment, bot, and private-key secrets from error text. */ export function redactSensitiveText(value: string): string { - let redacted = configuredRedactor ? configuredRedactor(value) : value; + const configured = configuredRedactor ? configuredRedactor(value) : value; + const structuredAuthMarker = createStructuredAuthMarker(configured); + let redacted = redactStructuredAuthHeaders(configured, structuredAuthMarker); for (const pattern of SECRET_PATTERNS) { redacted = redacted.replace(pattern, (match, ...args: string[]) => { if (match.includes("PRIVATE KEY-----")) { @@ -54,7 +127,7 @@ export function redactSensitiveText(value: string): string { return token ? match.replace(token, "[REDACTED]") : "[REDACTED]"; }); } - return redacted; + return redacted.replaceAll(structuredAuthMarker, "[REDACTED]"); } export { stringifyNonErrorCause } from "@openclaw/normalization-core/error-coercion"; diff --git a/packages/acp-core/src/index.ts b/packages/acp-core/src/index.ts index 6ee833caecb..e81b991bd9c 100644 --- a/packages/acp-core/src/index.ts +++ b/packages/acp-core/src/index.ts @@ -8,6 +8,7 @@ export * from "./record-shared.js"; export * from "./session-interaction-mode.js"; export * from "./session-lineage-meta.js"; export * from "./session.js"; +export * from "./structured-auth-redaction.js"; export * from "./types.js"; export * from "./runtime/error-text.js"; export * from "./runtime/errors.js"; diff --git a/packages/acp-core/src/structured-auth-redaction.ts b/packages/acp-core/src/structured-auth-redaction.ts new file mode 100644 index 00000000000..0b480977135 --- /dev/null +++ b/packages/acp-core/src/structured-auth-redaction.ts @@ -0,0 +1,409 @@ +export const HTTP_AUTH_SCHEME_PATTERN = "[A-Za-z0-9!#$%&'*+.^_`|~-]+"; +export const HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN = String.raw`(?:\[REDACTED\]|[^\s\\"',;&#?<>)}\]]+)`; +const HTTP_AUTH_SERIALIZED_TAB_PATTERN = String.raw`\\{1,64}t`; +const HTTP_AUTH_SERIALIZED_INDENT_PATTERN = String.raw`(?:[ \t]+|${HTTP_AUTH_SERIALIZED_TAB_PATTERN})`; +export const HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN = String.raw`(?:[ \t]*\r?\n${HTTP_AUTH_SERIALIZED_INDENT_PATTERN}|[ \t]*\\{1,64}r\\{1,64}n${HTTP_AUTH_SERIALIZED_INDENT_PATTERN}|[ \t]*\\{1,64}n${HTTP_AUTH_SERIALIZED_INDENT_PATTERN}|[ \t]*${HTTP_AUTH_SERIALIZED_TAB_PATTERN}[ \t]*|[ \t]*)`; +export const HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN = String.raw`(?:[ \t]*\r?\n${HTTP_AUTH_SERIALIZED_INDENT_PATTERN}|[ \t]*\\{1,64}r\\{1,64}n${HTTP_AUTH_SERIALIZED_INDENT_PATTERN}|[ \t]*\\{1,64}n${HTTP_AUTH_SERIALIZED_INDENT_PATTERN}|[ \t]*${HTTP_AUTH_SERIALIZED_TAB_PATTERN}[ \t]*|[ \t]+)`; +export const HTTP_AUTH_LEGACY_VALUE_WHITESPACE_PATTERN = String.raw`(?:[ \t\r\n]*|[ \t]*\\{1,64}r\\{1,64}n(?:[ \t]*|${HTTP_AUTH_SERIALIZED_TAB_PATTERN})|[ \t]*\\{1,64}n(?:[ \t]*|${HTTP_AUTH_SERIALIZED_TAB_PATTERN})|[ \t]*${HTTP_AUTH_SERIALIZED_TAB_PATTERN}[ \t]*)`; +export const HTTP_AUTH_HEADER_BOUNDARY_PATTERN = String.raw`(^|[^A-Za-z0-9_-]|\\{1,64}[rn])`; +// Each JSON encoding doubles delimiter slashes and adds one. The cap covers six nested +// encodings while keeping credential redaction regex work bounded on hostile diagnostics. +export const HTTP_AUTH_SERIALIZED_QUOTE_PATTERN = String.raw`(?:\\{1,64}["']|["']|)`; +export const CREDENTIAL_STYLE_HEADER_REDACT_PATTERN = String.raw`${HTTP_AUTH_HEADER_BOUNDARY_PATTERN}(?:x-goog-api-key|api-key|apikey|x-api-token|x-access-token)${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}([^\s\\"',;]+)`; +const STRUCTURED_AUTH_HEADER_RE = new RegExp( + String.raw`${HTTP_AUTH_HEADER_BOUNDARY_PATTERN}(?:Proxy-)?Authorization${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}[ \t]*[:=]${HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN}${HTTP_AUTH_SERIALIZED_QUOTE_PATTERN}(${HTTP_AUTH_SCHEME_PATTERN})${HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN}`, + "giu", +); +const AUTH_PARAM_NAME_RE = /^[A-Za-z0-9!#$%&'*+.^_`|~-]+/u; +const AUTH_PARAM_TOKEN_RE = /^[A-Za-z0-9!#$%&'*+.^_`|~-]+/u; +const AWS_SCOPE_VALUE_RE = /^[A-Za-z0-9!#$%&'*+.^_`|~:/-]+/u; + +export type StructuredAuthParamRange = { start: number; end: number }; + +function skipHorizontalWhitespace(value: string, start: number): number { + let cursor = start; + while (value[cursor] === " " || value[cursor] === "\t") { + cursor += 1; + } + return cursor; +} + +function readSerializedLineEnd(value: string, start: number): number | null { + let cursor = start; + let slashCount = 0; + while (slashCount < 64 && value[cursor] === "\\") { + slashCount += 1; + cursor += 1; + } + if (slashCount === 0) { + return null; + } + if (value[cursor] === "n") { + return cursor + 1; + } + if (value[cursor] !== "r") { + return null; + } + cursor += 1; + slashCount = 0; + while (slashCount < 64 && value[cursor] === "\\") { + slashCount += 1; + cursor += 1; + } + return slashCount > 0 && value[cursor] === "n" ? cursor + 1 : null; +} + +function readSerializedTabEnd(value: string, start: number): number | null { + let cursor = start; + let slashCount = 0; + while (slashCount < 64 && value[cursor] === "\\") { + slashCount += 1; + cursor += 1; + } + return slashCount > 0 && value[cursor] === "t" ? cursor + 1 : null; +} + +function skipAuthWhitespace(value: string, start: number): number { + let cursor = start; + for (;;) { + cursor = skipHorizontalWhitespace(value, cursor); + const tabEnd = readSerializedTabEnd(value, cursor); + if (tabEnd !== null) { + cursor = tabEnd; + continue; + } + const lineEnd = + value[cursor] === "\r" && value[cursor + 1] === "\n" + ? cursor + 2 + : value[cursor] === "\n" + ? cursor + 1 + : readSerializedLineEnd(value, cursor); + if ( + lineEnd === null || + (value[lineEnd] !== " " && + value[lineEnd] !== "\t" && + readSerializedTabEnd(value, lineEnd) === null) + ) { + return cursor; + } + cursor = lineEnd; + } +} + +function readAuthParamName(value: string, start: number): { name: string; end: number } | null { + const match = AUTH_PARAM_NAME_RE.exec(value.slice(start)); + return match ? { name: match[0].toLowerCase(), end: start + match[0].length } : null; +} + +function isAuthHeaderStart(value: string, index: number): boolean { + const previous = value[index - 1]; + let serializedLineBoundary = false; + if (previous === "n" || previous === "r") { + let slashCursor = index - 2; + let slashCount = 0; + while (slashCount < 64 && value[slashCursor] === "\\") { + slashCount += 1; + slashCursor -= 1; + } + serializedLineBoundary = slashCount > 0; + } + if (!serializedLineBoundary && previous !== undefined && /[A-Za-z0-9_-]/u.test(previous)) { + return false; + } + const proxyName = "proxy-authorization"; + const directName = "authorization"; + const candidate = value.slice(index, index + proxyName.length).toLowerCase(); + const name = + candidate === proxyName ? proxyName : candidate.startsWith(directName) ? directName : null; + if (!name) { + return false; + } + + let cursor = index + name.length; + let slashCount = 0; + while (slashCount < 64 && value[cursor] === "\\") { + slashCount += 1; + cursor += 1; + } + if (value[cursor] === '"' || value[cursor] === "'") { + cursor += 1; + } else if (slashCount > 0) { + return false; + } + cursor = skipHorizontalWhitespace(value, cursor); + return value[cursor] === ":" || value[cursor] === "="; +} + +function findNextAuthParamStart(value: string, start: number): number | null { + let cursor = start; + for (;;) { + cursor = skipAuthWhitespace(value, cursor); + if (cursor > start && isAuthHeaderStart(value, cursor)) { + return null; + } + if ( + cursor >= value.length || + value[cursor] === "\r" || + value[cursor] === "\n" || + value[cursor] === ";" + ) { + return null; + } + if (value[cursor] === ",") { + cursor += 1; + continue; + } + + const param = readAuthParamName(value, cursor); + if (param) { + const equals = skipAuthWhitespace(value, param.end); + if (value[equals] === "=" && value[equals + 1] !== "=") { + return cursor; + } + } + + while (cursor < value.length) { + const whitespaceEnd = skipAuthWhitespace(value, cursor); + if (whitespaceEnd > cursor) { + cursor = whitespaceEnd; + continue; + } + if (cursor > start && isAuthHeaderStart(value, cursor)) { + return null; + } + const char = value[cursor]; + if (char === "\r" || char === "\n" || char === ";") { + return null; + } + cursor += 1; + if (char === ",") { + break; + } + } + } +} + +function usesAuthParams(scheme: string): boolean { + return scheme === "digest" || scheme === "hawk" || scheme.startsWith("aws4-"); +} + +function findAuthFieldEnd(value: string, start: number): number { + let cursor = start; + while (cursor < value.length) { + const whitespaceEnd = skipAuthWhitespace(value, cursor); + if (whitespaceEnd > cursor) { + cursor = whitespaceEnd; + continue; + } + if (cursor > start && isAuthHeaderStart(value, cursor)) { + break; + } + const char = value[cursor]; + if ( + char === "\r" || + char === "\n" || + char === ";" || + char === "\\" || + char === '"' || + char === "'" || + char === "}" || + char === "]" + ) { + break; + } + cursor += 1; + } + return cursor; +} + +function readParamValue( + value: string, + start: number, + options: { awsScope: boolean; signedHeaders: boolean }, +): number | null { + let escapedQuoteSlashCount = 0; + while (value[start + escapedQuoteSlashCount] === "\\") { + escapedQuoteSlashCount += 1; + } + const escapedQuotes = escapedQuoteSlashCount > 0 && value[start + escapedQuoteSlashCount] === '"'; + const quote = value[start] === '"' || value[start] === "'" ? value[start] : undefined; + if (quote || escapedQuotes) { + let cursor = start + (escapedQuotes ? escapedQuoteSlashCount + 1 : 1); + while (cursor < value.length) { + if (value[cursor] === "\r" || value[cursor] === "\n") { + const whitespaceEnd = skipAuthWhitespace(value, cursor); + if (whitespaceEnd === cursor) { + break; + } + cursor = whitespaceEnd; + continue; + } + if (escapedQuotes && value[cursor] === "\\") { + let slashEnd = cursor + 1; + while (value[slashEnd] === "\\") { + slashEnd += 1; + } + if (value[slashEnd] === '"') { + const slashCount = slashEnd - cursor; + if (slashCount % (2 * (escapedQuoteSlashCount + 1)) === escapedQuoteSlashCount) { + return slashEnd + 1; + } + cursor = slashEnd + 1; + continue; + } + cursor = slashEnd; + continue; + } + if (!escapedQuotes && value[cursor] === "\\" && cursor + 1 < value.length) { + cursor += 2; + continue; + } + if (!escapedQuotes && value[cursor] === quote) { + return cursor + 1; + } + cursor += 1; + } + return cursor > start + 1 ? cursor : null; + } + + if (options.signedHeaders) { + const match = /^:?[A-Za-z0-9!#$%&'*+.^_`|~-]+(?:;:?[A-Za-z0-9!#$%&'*+.^_`|~-]+)*/u.exec( + value.slice(start), + ); + if (!match) { + return null; + } + const end = start + match[0].length; + const next = value[end]; + return next === undefined || + next === "," || + next === " " || + next === "\t" || + next === "\r" || + next === "\n" + ? end + : null; + } + const match = (options.awsScope ? AWS_SCOPE_VALUE_RE : AUTH_PARAM_TOKEN_RE).exec( + value.slice(start), + ); + return match ? start + match[0].length : null; +} + +export function findStructuredAuthParamRanges(value: string): StructuredAuthParamRange[] { + const ranges: StructuredAuthParamRange[] = []; + for (const header of value.matchAll(STRUCTURED_AUTH_HEADER_RE)) { + const scheme = (header[2] ?? "").toLowerCase(); + let cursor = (header.index ?? 0) + header[0].length; + const rangeStart = cursor; + let rangeEnd = cursor; + + const directParam = readAuthParamName(value, cursor); + const directEquals = directParam ? skipAuthWhitespace(value, directParam.end) : undefined; + if ( + !directParam || + directEquals === undefined || + value[directEquals] !== "=" || + value[directEquals + 1] === "=" + ) { + const firstNonWhitespace = skipAuthWhitespace(value, cursor); + // An opaque token followed by `, status=...` is not an auth-param list. Only known + // parameterized schemes or an explicit empty first member justify resynchronizing here. + if (value[firstNonWhitespace] !== "," && !usesAuthParams(scheme)) { + continue; + } + const firstParamStart = findNextAuthParamStart(value, cursor); + if (firstParamStart === null) { + continue; + } + cursor = firstParamStart; + } + + // Commas belong to the credential grammar. Only an explicit field boundary or line end can + // end the value; malformed list members resynchronize so later credentials cannot leak. + for (;;) { + const param = readAuthParamName(value, cursor); + if (!param) { + break; + } + cursor = skipAuthWhitespace(value, param.end); + if (value[cursor] !== "=") { + break; + } + cursor = skipAuthWhitespace(value, cursor + 1); + const valueEnd = readParamValue(value, cursor, { + awsScope: scheme.startsWith("aws4-") && param.name === "credential", + signedHeaders: param.name === "signedheaders", + }); + if (valueEnd === null) { + const nextParamStart = findNextAuthParamStart(value, cursor); + if (nextParamStart !== null) { + cursor = nextParamStart; + continue; + } + rangeEnd = Math.max(rangeEnd, findAuthFieldEnd(value, cursor)); + break; + } + rangeEnd = valueEnd; + + const separator = skipAuthWhitespace(value, valueEnd); + if (value[separator] !== ",") { + if ( + value[separator] !== undefined && + value[separator] !== "\r" && + value[separator] !== "\n" && + value[separator] !== ";" && + value[separator] !== "\\" && + value[separator] !== '"' && + value[separator] !== "'" && + value[separator] !== "}" && + value[separator] !== "]" + ) { + const nextParamStart = findNextAuthParamStart(value, separator); + if (nextParamStart !== null) { + cursor = nextParamStart; + continue; + } + rangeEnd = Math.max(rangeEnd, findAuthFieldEnd(value, separator)); + } + break; + } + // RFC list syntax permits empty members. Resynchronize after malformed members too so a + // damaged diagnostic cannot expose a later credential parameter on the same header line. + const nextParamStart = findNextAuthParamStart(value, separator + 1); + if (nextParamStart === null) { + break; + } + cursor = nextParamStart; + } + + if (rangeEnd > rangeStart) { + ranges.push({ start: rangeStart, end: rangeEnd }); + } + } + return ranges; +} + +export function redactStructuredAuthHeaders(value: string, replacement: string): string { + const ranges = findStructuredAuthParamRanges(value); + if (ranges.length === 0) { + return value; + } + const merged: StructuredAuthParamRange[] = []; + for (const range of ranges) { + const previous = merged.at(-1); + if (previous && range.start <= previous.end) { + previous.end = Math.max(previous.end, range.end); + } else { + merged.push({ ...range }); + } + } + const parts: string[] = []; + let cursor = 0; + for (const range of merged) { + parts.push(value.slice(cursor, range.start), replacement); + cursor = range.end; + } + parts.push(value.slice(cursor)); + return parts.join(""); +} diff --git a/src/agents/openai-transport-stream.base.test.ts b/src/agents/openai-transport-stream.base.test.ts index 83cabb8660f..0066d6ffc1d 100644 --- a/src/agents/openai-transport-stream.base.test.ts +++ b/src/agents/openai-transport-stream.base.test.ts @@ -820,16 +820,17 @@ describe("openai transport stream", () => { const previous = process.env.OPENCLAW_DEBUG_MODEL_PAYLOAD; process.env.OPENCLAW_DEBUG_MODEL_PAYLOAD = "full-redacted"; try { + const apiKey = "test-api-key"; const summary = testing.summarizeResponsesPayload({ model: "gpt-5.5", stream: true, input: [], tools: [{ type: "function", name: "exec" }], - apiKey: "sk-abcdefghijklmnopqrstuvwxyz", + apiKey, }); expect(summary).toContain("payload="); - expect(summary).toContain("sk-abc"); - expect(summary).not.toContain("sk-abcdefghijklmnopqrstuvwxyz"); + expect(summary).toContain('"apiKey":"***"'); + expect(summary).not.toContain(apiKey); } finally { if (previous === undefined) { delete process.env.OPENCLAW_DEBUG_MODEL_PAYLOAD; diff --git a/src/logging/logger-redaction-behavior.test.ts b/src/logging/logger-redaction-behavior.test.ts index dee213179a8..9f6d628f749 100644 --- a/src/logging/logger-redaction-behavior.test.ts +++ b/src/logging/logger-redaction-behavior.test.ts @@ -60,6 +60,30 @@ describe("file log redaction", () => { expect(content).not.toContain(secret); }); + it("redacts structured authorization fields before writing JSONL file logs", () => { + const logPath = logPathTracker.nextPath(); + const digestResponse = ["runtime", "digest", "response", "1234567890abcdef"].join("-"); + const awsScopeField = ["Cred", "ential", "=test-fixture-scope"].join(""); + const awsProofField = ["runtime", "aws", "proof", "1234567890abcdef"].join("-"); + setLoggerOverride({ level: "info", file: logPath }); + + getLogger().warn({ + message: [ + `Authorization: Digest username="example", response="${digestResponse}"`, + `Authorization: AWS4-HMAC-SHA256 ${awsScopeField}, Signature=${awsProofField}`, + ].join("\n"), + }); + + const content = fs.readFileSync(logPath, "utf8"); + expect(() => JSON.parse(content.trim())).not.toThrow(); + expect(content).toContain("Authorization: Digest"); + expect(content).toContain("Authorization: AWS4-HMAC-SHA256"); + expect(content).not.toContain(digestResponse); + expect(content).not.toContain(awsProofField); + expect(content).not.toContain("response="); + expect(content).not.toContain("Signature="); + }); + it("redacts sensitive structured fields before emitting diagnostic log records", async () => { const logPath = logPathTracker.nextPath(); setLoggerOverride({ level: "info", file: logPath }); diff --git a/src/logging/redact.test.ts b/src/logging/redact.test.ts index 84aa8315fb7..4960c9944a3 100644 --- a/src/logging/redact.test.ts +++ b/src/logging/redact.test.ts @@ -352,6 +352,376 @@ describe("redactSensitiveText", () => { expect(output).not.toContain(secret); }); + it("masks non-Bearer authorization schemes", () => { + const firstValue = ["sample", "value", "1234567890abcd"].join(""); + const secondValue = ["sample", "proxy", "value", "1234567890"].join(""); + const input = [ + ["Authorization", ": token ", firstValue].join(""), + ["Proxy-Authorization", ": Digest ", secondValue].join(""), + ].join("\n"); + const output = redactSensitiveText(input, { mode: "tools" }); + + expect(output).toContain("Authorization: token "); + expect(output).toContain("Proxy-Authorization: Digest "); + expect(output).not.toContain(firstValue); + expect(output).not.toContain(secondValue); + }); + + it("masks complete structured authorization fields", () => { + const digestUser = ["digest", "user", "example"].join("-"); + const digestResponse = ["digest", "response", "1234567890abcdef"].join("-"); + const digestExtension = ["digest", "extension", "1234567890abcdef"].join("-"); + const digestTail = ["digest", "tail", "1234567890abcdef"].join("-"); + const awsCredential = [ + "AK", + "IA", + "EXAMPLE", + "1234567890", + "/20260717/eu-west-1/s3/aws4_request", + ].join(""); + const awsSignature = ["aws", "signature", "1234567890abcdef"].join("-"); + const input = [ + `Authorization: Digest username="${digestUser}", 2fa="${digestExtension}", response="${digestResponse}", extension="${digestTail}", cnonce="tail-nonce"; request_id=digest-example`, + `Authorization: AWS4-HMAC-SHA256 Credential=${awsCredential}, SignedHeaders=:authority;x_custom;x.custom, Signature=${awsSignature}; status=403`, + `Proxy-Authorization: Digest username="${digestUser}", response="${digestResponse}"; request_id=proxy-example`, + ].join("\n"); + const output = redactSensitiveText(input, { mode: "tools" }); + + expect(output).toContain("Authorization: Digest "); + expect(output).toContain("Authorization: AWS4-HMAC-SHA256 "); + expect(output).toContain("Proxy-Authorization: Digest "); + for (const value of [ + digestUser, + digestExtension, + digestResponse, + digestTail, + awsCredential, + awsSignature, + ]) { + expect(output).not.toContain(value); + } + expect(output).not.toContain("username="); + expect(output).not.toContain("Credential="); + expect(output).not.toContain("Signature="); + expect(output).toContain("; request_id=digest-example"); + expect(output).toContain("; status=403"); + expect(output).toContain("; request_id=proxy-example"); + }); + + it("masks consecutive, prefixed, and serialized auth headers", () => { + const proxyValue = ["cHJveH", "k6cGFz", "cw=="].join(""); + const customValue = ["Y3VzdG", "9tOnBh", "c3M="].join(""); + const accessValue = ["sample", "access", "value", "1234567890"].join("-"); + const googleValue = ["sample", "google", "value", "1234567890"].join("-"); + const input = [ + "Proxy-Authorization: Foo", + `Proxy-Authorization: Basic ${proxyValue}`, + `X-Authorization: Basic ${customValue}`, + JSON.stringify({ + "x-access-token": accessValue, + "x-goog-api-key": googleValue, + }), + ].join("\n"); + const output = redactSensitiveText(input, { mode: "tools" }); + + expect(output).toContain("Proxy-Authorization: Basic ***"); + expect(output).toContain("X-Authorization: Basic ***"); + for (const credential of [proxyValue, customValue, accessValue, googleValue]) { + expect(output).not.toContain(credential); + } + }); + + it("masks later auth params and token credentials after punctuation", () => { + const responseValue = ["later", "response", "value", "1234567890"].join("-"); + const negotiateValue = ["cHJvb2", "YxMjM0", "NTY3ODkw"].join(""); + const foldedValue = ["Zm9sZG", "VkOnNl", "Y3JldA=="].join(""); + const rawValue = ["raw", "header", "value", "1234567890"].join("-"); + const input = [ + `Authorization: Digest username="sample",,response="${responseValue}"; status=401`, + `Authorization: Digest damaged,,response="${responseValue}"; status=403`, + `Authorization: Digest username="sample", uri=/bad, response="${responseValue}"; status=407`, + `Authorization: Digest username="sample",\r\n response="${responseValue}"; status=408`, + `Authorization: Digest uri=http://service, response="${responseValue}"; status=409`, + `Authorization: Digest response='${responseValue}'; status=410`, + `Authorization: Digest realm=sample, authorization-param=${responseValue}; status=412`, + `Authorization: Digest username=sample,\\r\\n response=${responseValue}; status=413`, + `Authorization: Digest username=sample,\\r\\n\\tresponse=${responseValue}; status=414`, + `(Authorization: Negotiate ${negotiateValue})`, + `Authorization:\r\n Basic ${foldedValue}`, + `Authorization:\nBasic ${foldedValue}`, + `Authorization:\\nBasic ${foldedValue}`, + `Authorization:\\tBearer ${foldedValue}`, + `Authorization: Bearer\\t${foldedValue}`, + `Authorization: ${rawValue} `, + ].join("\n"); + const output = redactSensitiveText(input, { mode: "tools" }); + + expect(output).toBe( + [ + "Authorization: Digest ***; status=401", + "Authorization: Digest ***; status=403", + "Authorization: Digest ***; status=407", + "Authorization: Digest ***; status=408", + "Authorization: Digest ***; status=409", + "Authorization: Digest ***; status=410", + "Authorization: Digest ***; status=412", + "Authorization: Digest ***; status=413", + "Authorization: Digest ***; status=414", + "(Authorization: Negotiate cHJvb2…ODkw)", + "Authorization:\r\n Basic Zm9sZG…dA==", + "Authorization:\nBasic Zm9sZG…dA==", + "Authorization:\\nBasic Zm9sZG…dA==", + "Authorization:\\tBearer Zm9sZG…dA==", + "Authorization: Bearer\\tZm9sZG…dA==", + "Authorization: raw-he…7890 ", + ].join("\n"), + ); + expect(output).not.toContain(responseValue); + expect(output).not.toContain(negotiateValue); + expect(output).not.toContain(foldedValue); + + const serializedLine = JSON.stringify( + `prefix\nAuthorization: Digest response="${responseValue}"`, + ); + expect(redactSensitiveText(serializedLine, { mode: "tools" })).toBe( + JSON.stringify("prefix\nAuthorization: Digest ***"), + ); + }); + + it("masks escaped structured authorization fields", () => { + const response = ["escaped", "digest", "response", "1234567890abcdef"].join("-"); + const input = `Authorization: Digest realm=\\"Example Realm\\", response=\\"${response}\\"; status=401`; + const output = redactSensitiveText(input, { mode: "tools" }); + + expect(output).toBe("Authorization: Digest ***; status=401"); + expect(output).not.toContain(response); + }); + + it("masks parameterized authorization schemes", () => { + const proof = ["hawk", "credential", "proof", "1234567890abcdef"].join("-"); + const output = redactSensitiveText( + `Authorization: Hawk id="client", mac="${proof}"; status=401`, + { mode: "tools" }, + ); + + expect(output).toBe("Authorization: Hawk ***; status=401"); + expect(output).not.toContain(proof); + }); + + it("masks full token grammar in auth-param values", () => { + const id = ["abc", "'", "def", "`", "ghi"].join(""); + const proof = ["token", "grammar", "proof", "1234567890abcdef"].join("-"); + const output = redactSensitiveText(`Authorization: Foo id=${id}, proof=${proof}; status=401`, { + mode: "tools", + }); + + expect(output).toBe("Authorization: Foo ***; status=401"); + expect(output).not.toContain(proof); + }); + + it("does not confuse auth-param names beginning with the display marker", () => { + const response = ["marker", "digest", "response", "1234567890abcdef"].join("-"); + const output = redactSensitiveText( + `Authorization: Digest ***ext=one, response="${response}"; status=401`, + { mode: "tools" }, + ); + + expect(output).toBe("Authorization: Digest ***; status=401"); + expect(output).not.toContain(response); + }); + + it("masks structured auth in serialized header objects", () => { + const response = ["json", "digest", "response", "1234567890abcdef"].join("-"); + const input = `{"Authorization":"Digest username=\\"example\\", response=\\"${response}\\""}`; + + expect(redactSensitiveText(input, { mode: "tools" })).toBe(`{"Authorization":"***"}`); + }); + + it("masks nested serialized auth objects", () => { + const response = ["nested", "digest", "response", "1234567890abcdef"].join("-"); + const header = { Authorization: `Digest response="${response}\\\\"` }; + const input = JSON.stringify(JSON.stringify(header)); + + expect(redactSensitiveText(input, { mode: "tools" })).toBe( + JSON.stringify(JSON.stringify({ Authorization: "Digest ***" })), + ); + expect(redactSensitiveText(JSON.stringify(header), { mode: "tools" })).toBe( + JSON.stringify({ Authorization: "***" }), + ); + }); + + it("masks token credentials for punctuated auth schemes", () => { + const token = ["extension", "token", "1234567890abcdef"].join("-"); + const basicCredential = ["dXNl", "cjpw", "YXNz"].join(""); + const nestedBasic = JSON.stringify( + JSON.stringify({ Authorization: `Basic ${basicCredential}` }), + ); + const bearerCredential = ["/opaque", "~bearer", "1234567890abcdef"].join("-"); + const nestedBearer = JSON.stringify( + JSON.stringify({ Authorization: `Bearer ${bearerCredential}` }), + ); + const opaqueCredential = ["opaque", "credential", "1234567890abcdef"].join("-"); + const nestedOpaque = JSON.stringify( + JSON.stringify({ + Authorization: opaqueCredential, + "Proxy-Authorization": opaqueCredential, + }), + ); + + expect( + redactSensitiveText(`Authorization: Foo+Bar ${token}; status=401`, { mode: "tools" }), + ).toBe("Authorization: Foo+Bar extens…cdef; status=401"); + expect( + redactSensitiveText(`Authorization: Basic+Foo ${token}; status=401`, { mode: "tools" }), + ).toBe("Authorization: Basic+Foo extens…cdef; status=401"); + expect(redactSensitiveText(nestedBasic, { mode: "tools" })).toBe( + JSON.stringify(JSON.stringify({ Authorization: "Basic ***" })), + ); + expect( + redactSensitiveText(`Authorization: Bearer ${bearerCredential}`, { mode: "tools" }), + ).toBe("Authorization: Bearer /opaqu…cdef"); + expect(redactSensitiveText(nestedBearer, { mode: "tools" })).toBe( + JSON.stringify(JSON.stringify({ Authorization: "Bearer /opaqu…cdef" })), + ); + expect(redactSensitiveText(nestedOpaque, { mode: "tools" })).toBe( + JSON.stringify( + JSON.stringify({ + Authorization: "opaque…cdef", + "Proxy-Authorization": "opaque…cdef", + }), + ), + ); + }); + + it("keeps token68 padding out of structured auth parsing", () => { + expect( + redactSensitiveText("Authorization: Basic dXNlcg==, status=401", { mode: "tools" }), + ).toBe("Authorization: Basic ***, status=401"); + }); + + it("masks structured authorization inside quoted diagnostics", () => { + const response = ["quoted", "header", "response", "1234567890abcdef"].join("-"); + const input = `curl -H 'Authorization: Digest username="example", response="${response}"'`; + + expect(redactSensitiveText(input, { mode: "tools" })).toBe( + "curl -H 'Authorization: Digest ***'", + ); + }); + + it("preserves structural closers after unquoted auth parameters", () => { + const signature = ["structural", "aws", "signature", "1234567890abcdef"].join("-"); + const awsScopeField = ["Cred", "ential", "=scope/path"].join(""); + const input = `{Authorization: AWS4-HMAC-SHA256 ${awsScopeField}, SignedHeaders=host, Signature=${signature}}`; + + expect(redactSensitiveText(input, { mode: "tools" })).toBe( + "{Authorization: AWS4-HMAC-SHA256 ***}", + ); + }); + + it("masks escaped auth fields containing encoded quoted-pairs", () => { + const response = ["escaped", "quoted", "response", "1234567890abcdef"].join("-"); + const input = `Authorization: Digest realm=\\"Example \\\\\\"Realm\\\\\\"\\", response=\\"${response}\\"; status=401`; + const output = redactSensitiveText(input, { mode: "tools" }); + + expect(output).toBe("Authorization: Digest ***; status=401"); + expect(output).not.toContain(response); + }); + + it("masks structured authorization fields across bounded-replacement chunks", () => { + const response = ["cross", "chunk", "response", "1234567890abcdef"].join("-"); + const input = `${"x".repeat(32_768)}\nAuthorization: Digest username="example", response="${response}"`; + const output = redactSensitiveText(input, { mode: "tools" }); + + expect(output).not.toContain(response); + expect(output).not.toContain("response="); + }); + + it("masks opaque authorization across bounded-replacement chunks", () => { + const headerValue = `${"A".repeat(96)}==`; + const standaloneValue = `${"B".repeat(96)}==`; + const input = `${"x".repeat(32_760)} Authorization: Bearer ${headerValue}\nrequest failed: Bearer ${standaloneValue}`; + const output = redactSensitiveText(input, { mode: "tools" }); + + expect(output).not.toContain(headerValue); + expect(output).not.toContain(standaloneValue); + expect(output).toContain("Authorization: Bearer AAAAAA…AA=="); + expect(output).toContain("request failed: Bearer BBBBBB…BB=="); + }); + + it("masks token authorization fields without consuming adjacent diagnostics", () => { + const token = ["opaque", "auth", "value", "1234567890abcdef"].join("-"); + const input = [ + `Authorization: ${token}, status=401`, + `Proxy-Authorization: ${token}; request_id=example`, + ].join("\n"); + const output = redactSensitiveText(input, { mode: "tools" }); + + expect(output).not.toContain(token); + expect(output).toContain(", status=401"); + expect(output).toContain("; request_id=example"); + }); + + it("masks scheme tokens without consuming adjacent diagnostics", () => { + const token = ["scheme", "auth", "value", "1234567890abcdef"].join("-"); + const input = [ + `Authorization: Token ${token}, status=401`, + `Proxy-Authorization: Basic ${token}; request_id=example`, + ].join("\n"); + const output = redactSensitiveText(input, { mode: "tools" }); + + expect(output).not.toContain(token); + expect(output).toContain(", status=401"); + expect(output).toContain("; request_id=example"); + }); + + it("masks unquoted credential-style headers", () => { + const firstValue = ["sample", "key", "value", "1234567890"].join(""); + const secondValue = ["sample", "goog", "value", "1234567890"].join(""); + const thirdValue = ["sample", "access", "value", "1234567890"].join(""); + const keyHeader = ["api", "-", "key"].join(""); + const googleHeader = ["x", "-", "goog", "-", "api", "-", "key"].join(""); + const accessHeader = ["x", "-", "access", "-", "token"].join(""); + const input = [ + [keyHeader, ": ", firstValue].join(""), + [googleHeader, "=", secondValue].join(""), + [accessHeader, ": ", thirdValue].join(""), + ].join("\n"); + const output = redactSensitiveText(input, { mode: "tools" }); + + expect(output).toContain(`${keyHeader}: `); + expect(output).toContain(`${googleHeader}=`); + expect(output).toContain(`${accessHeader}: `); + expect(output).not.toContain(firstValue); + expect(output).not.toContain(secondValue); + expect(output).not.toContain(thirdValue); + }); + + it("preserves diagnostics following unquoted credential-style headers", () => { + const keyHeader = ["api", "-", "key"].join(""); + const value = ["sample", "key", "value", "1234567890"].join(""); + const output = redactSensitiveText(`${keyHeader}: ${value}, request_id=example, status=500`, { + mode: "tools", + }); + + expect(output).not.toContain(value); + expect(output).toContain(", request_id=example, status=500"); + }); + + it("masks punctuation inside unquoted credential-style header values", () => { + const keyHeader = ["api", "-", "key"].join(""); + const output = redactSensitiveText(`${keyHeader}: prefix)sensitive-suffix`, { + mode: "tools", + }); + + expect(output).not.toContain("sensitive-suffix"); + }); + + it("does not redact ordinary authorization prose", () => { + const input = "the authorization model is open"; + + expect(redactSensitiveText(input, { mode: "tools" })).toBe(input); + }); + it("masks named Gateway security headers", () => { const openClawToken = "supersecretgatewaytoken1234567890"; const pomeriumJwt = "eyJheaderabcd.eyJpayloadabcd.signatureabcd123456"; @@ -1346,6 +1716,33 @@ describe("redactSensitiveLines", () => { expect(redactSensitiveLines(lines, resolved)).toEqual(lines); }); + it("redacts structured auth when form-body preprocessing is disabled", () => { + const resolved = { + ...resolveRedactOptions({ mode: "tools" }), + redactFormBodies: false, + }; + const response = ["line", "digest", "response", "1234567890abcdef"].join("-"); + + expect( + redactSensitiveLines( + [`Authorization: Digest username="example", response="${response}"; status=401`], + resolved, + ), + ).toEqual(["Authorization: Digest ***; status=401"]); + }); + + it("redacts folded structured auth across line batches", () => { + const resolved = resolveRedactOptions({ mode: "tools" }); + const response = ["folded", "line", "response", "1234567890abcdef"].join("-"); + + expect( + redactSensitiveLines( + ["Authorization: Digest", ` response="${response}"; status=401`], + resolved, + ), + ).toEqual(["Authorization: Digest", " ***; status=401"]); + }); + it("returns lines unmodified when resolved patterns is empty — does not fall back to defaults", () => { // Simulates the case where all user-configured patterns fail to compile. // The pre-resolved empty array must be honored, not silently replaced with defaults. diff --git a/src/logging/redact.ts b/src/logging/redact.ts index 0fdc48a754c..3c61dc9ef33 100644 --- a/src/logging/redact.ts +++ b/src/logging/redact.ts @@ -1,3 +1,15 @@ +import { + CREDENTIAL_STYLE_HEADER_REDACT_PATTERN, + findStructuredAuthParamRanges, + HTTP_AUTH_HEADER_BOUNDARY_PATTERN, + HTTP_AUTH_LEGACY_VALUE_WHITESPACE_PATTERN, + HTTP_AUTH_OPAQUE_CREDENTIAL_PATTERN, + HTTP_AUTH_OPTIONAL_VALUE_WHITESPACE_PATTERN, + HTTP_AUTH_REQUIRED_VALUE_WHITESPACE_PATTERN, + HTTP_AUTH_SCHEME_PATTERN, + HTTP_AUTH_SERIALIZED_QUOTE_PATTERN, + redactStructuredAuthHeaders, +} from "@openclaw/acp-core"; import { expectDefined } from "@openclaw/normalization-core"; // Redaction helpers scrub secrets and sensitive identifiers from log output. import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; @@ -130,6 +142,17 @@ const BASE64_SAFE_TOKEN_BOUNDARY = String.raw`(^|[^A-Za-z0-9])(?(); // Patterns whose left-context assertions or complete token can cross a chunk boundary must run @@ -162,11 +190,12 @@ const DEFAULT_REDACT_PATTERNS: string[] = [ // CLI flags. String.raw`--(?:api[-_]?key|hook[-_]?token|access[-_]?token|refresh[-_]?token|id[-_]?token|token|secret|password|passwd|credential|private[-_]?key|client[-_]?secret|${PAYMENT_CREDENTIAL_QUERY_KEYS})\s+(?!(?:or|and)\b(?=\s+--))(["']?)([^\s"']+)\1`, // Authorization headers. - String.raw`Authorization\s*[:=]\s*Bearer\s+([A-Za-z0-9._\-+=]+)`, - String.raw`Authorization\s*[:=]\s*Basic\s+([A-Za-z0-9+/=]+)`, - String.raw`Authorization\s*[:=]\s*Bot\s+([A-Za-z0-9._\-+=]{18,})`, + AUTHORIZATION_BEARER_REDACT_PATTERN, + AUTHORIZATION_BASIC_REDACT_PATTERN, + AUTHORIZATION_BOT_REDACT_PATTERN, + ...HTTP_AUTH_HEADER_REDACT_PATTERNS, String.raw`(?:X-OpenClaw-Token|x-pomerium-jwt-assertion|X-Api-Key|X-Auth-Token)\s*[:=]\s*([^\s"',;]+)`, - String.raw`\bBearer\s+([A-Za-z0-9._\-+=]{18,})\b`, + STANDALONE_BEARER_REDACT_PATTERN, // URL userinfo and common connection-string password slots. String.raw`\b(?:https?|wss?|ftp):\/\/[^\/\s:@]*:([^\/\s@]+)@`, String.raw`\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|rediss?|amqps?):\/\/[^:\s/@]*:([^@\s]+)@`, @@ -301,6 +330,7 @@ export type ResolvedRedactOptions = { mode: RedactSensitiveMode; patterns: RegExp[]; redactFormBodies: boolean; + redactStructuredAuthHeaders?: boolean; }; function normalizeMode(value?: string): RedactSensitiveMode { @@ -865,9 +895,16 @@ function redactMatch( function redactText( text: string, patterns: RegExp[], - options?: { fullContext?: boolean; redactFormBodies?: boolean }, + options?: { + fullContext?: boolean; + redactFormBodies?: boolean; + redactStructuredAuthHeaders?: boolean; + }, ): string { let next = text; + if (options?.redactStructuredAuthHeaders) { + next = redactStructuredAuthHeaders(next, "***"); + } if (options?.redactFormBodies) { next = redactUrlQueryPairs(next); next = redactFormBody(next); @@ -947,6 +984,11 @@ export function computeSensitiveRedactionBitmap( if (resolved.mode === "off" || !resolved.patterns.length || !text) { return bitmap; } + if (resolved.redactStructuredAuthHeaders) { + for (const range of findStructuredAuthParamRanges(text)) { + markBitmapRange(bitmap, range.start, range.end); + } + } if (resolved.redactFormBodies) { markUrlQueryPairRedactions(text, bitmap); markFormBodyRedactions(text, bitmap); @@ -990,10 +1032,12 @@ export function resolveRedactOptions(options?: RedactOptions): ResolvedRedactOpt }; } const patterns = resolvePatterns(resolved.patterns); + const includesDefaults = patterns.length > 0 && includesDefaultRedactPatterns(resolved.patterns); return { mode, patterns, - redactFormBodies: patterns.length > 0 && includesDefaultRedactPatterns(resolved.patterns), + redactFormBodies: includesDefaults, + redactStructuredAuthHeaders: includesDefaults, }; } @@ -1015,6 +1059,7 @@ export function redactSensitiveText(text: string, options?: RedactOptions): stri } return redactText(exactRedacted, resolved.patterns, { redactFormBodies: resolved.redactFormBodies, + redactStructuredAuthHeaders: resolved.redactStructuredAuthHeaders, }); } @@ -1053,6 +1098,7 @@ export function redactToolPayloadTextWithConfig( return redactText(exactRedacted, resolved.patterns, { fullContext: true, redactFormBodies: resolved.redactFormBodies, + redactStructuredAuthHeaders: resolved.redactStructuredAuthHeaders, }); } return redactSensitiveText(text, resolveToolPayloadRedaction(loggingConfig)); @@ -1075,6 +1121,7 @@ function redactSensitiveFieldValueWithOptions( } const redacted = redactText(exactRedacted, resolved.patterns, { redactFormBodies: resolved.redactFormBodies, + redactStructuredAuthHeaders: resolved.redactStructuredAuthHeaders, }); const shouldRedactAppPassword = redacted !== value || STRUCTURED_APP_PASSWORD_FIELD_RE.test(key); if (shouldRedactAppPassword) { @@ -1238,6 +1285,10 @@ export function redactSensitiveLines(lines: string[], resolved: ResolvedRedactOp const redactedLines = resolved.redactFormBodies ? lines.map((line) => redactFormBody(redactUrlQueryPairs(line))) : lines; - return redactText(redactedLines.join("\n"), resolved.patterns).split("\n"); + let redacted = redactedLines.join("\n"); + if (resolved.redactStructuredAuthHeaders) { + redacted = redactStructuredAuthHeaders(redacted, "***"); + } + return redactText(redacted, resolved.patterns).split("\n"); } /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */